- Remove multi-marketplace from Phase 3 - Add FastAPI web UI on port 8766 with basic auth - Add 6 Jinja2 templates (dashboard, keywords, users, ads, stats) - Add pytest test suite (45 tests, 49% coverage) - Add GitHub Actions CI/CD workflow - Update docker-compose.yml to expose web UI port - Update Dockerfile to include tests
This commit is contained in:
+20
-1
@@ -388,6 +388,21 @@ async def main() -> None:
|
||||
await site.start()
|
||||
logger.info("Health check server listening on :%d", _health_port)
|
||||
|
||||
# ── Start Web UI server ────────────────────────────────────────
|
||||
_web_server = None
|
||||
_web_config = None
|
||||
try:
|
||||
from web import app as web_app # noqa: E402
|
||||
import uvicorn # noqa: E402
|
||||
|
||||
_web_port = int(os.getenv("WEB_UI_PORT", "8766"))
|
||||
_web_config = uvicorn.Config(web_app, host="0.0.0.0", port=_web_port, log_level="info")
|
||||
_web_server = uvicorn.Server(_web_config)
|
||||
asyncio.ensure_future(_web_server.serve())
|
||||
logger.info("Web UI listening on :%d", _web_port)
|
||||
except Exception:
|
||||
logger.exception("Failed to start Web UI (optional)")
|
||||
|
||||
scheduler = asyncio.ensure_future(scheduler_task(pool, app.bot))
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
@@ -435,7 +450,11 @@ async def main() -> None:
|
||||
# ── Close health server ───────────────────────────────────────
|
||||
logger.info("Stopping health check server...")
|
||||
await runner.cleanup()
|
||||
|
||||
# ── Close Web UI server ──────────────────────────────────────
|
||||
if _web_server is not None:
|
||||
logger.info("Stopping Web UI server...")
|
||||
_web_server.should_exit = True
|
||||
await asyncio.sleep(1)
|
||||
# ── Close HTTP client ────────────────────────────────────────
|
||||
logger.info("Closing HTTP client...")
|
||||
from scraper import close_client as close_http_client # noqa: E402
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Ads — Willhaben Tracker{% endblock %}
|
||||
{% block content %}
|
||||
<div class="page-header">
|
||||
<h2>Recent Ads</h2>
|
||||
</div>
|
||||
|
||||
{% if not error and ads %}
|
||||
<div class="card" style="overflow-x: auto;">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Title</th>
|
||||
<th>Price</th>
|
||||
<th>Location</th>
|
||||
<th>Postcode</th>
|
||||
<th>URL</th>
|
||||
<th>Published</th>
|
||||
<th>First Seen</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for ad in ads %}
|
||||
<tr>
|
||||
<td>{{ ad.title[:70] }}{% if ad.title|length > 70 %}…{% endif %}</td>
|
||||
<td>{{ format_price(ad.price) }}</td>
|
||||
<td>{{ ad.location or '—' }}</td>
|
||||
<td>{{ ad.postcode or '—' }}</td>
|
||||
<td><a href="{{ ad.url }}" target="_blank">Link</a></td>
|
||||
<td>{{ ad.published_at.strftime('%Y-%m-%d %H:%M') if ad.published_at else '—' }}</td>
|
||||
<td>{{ ad.first_seen_at.strftime('%Y-%m-%d %H:%M') }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% elif not error %}
|
||||
<div class="card"><em>No ads found.</em></div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,131 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>{% block title %}Willhaben Tracker{% endblock %}</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body {
|
||||
font-family: system-ui, -apple-system, sans-serif;
|
||||
background: #1a1a2e;
|
||||
color: #e0e0e0;
|
||||
display: flex;
|
||||
min-height: 100vh;
|
||||
}
|
||||
/* Sidebar */
|
||||
.sidebar {
|
||||
width: 220px;
|
||||
background: #16213e;
|
||||
padding: 20px 0;
|
||||
border-right: 1px solid #0f3460;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.sidebar h1 {
|
||||
color: #e94560;
|
||||
font-size: 18px;
|
||||
padding: 0 20px 20px;
|
||||
border-bottom: 1px solid #0f3460;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
.sidebar a {
|
||||
display: block;
|
||||
color: #a0a0b0;
|
||||
text-decoration: none;
|
||||
padding: 10px 20px;
|
||||
font-size: 14px;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
.sidebar a:hover, .sidebar a.active {
|
||||
background: #0f3460;
|
||||
color: #e94560;
|
||||
}
|
||||
/* Main */
|
||||
.main {
|
||||
flex: 1;
|
||||
padding: 30px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
.page-header {
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
.page-header h2 {
|
||||
color: #e94560;
|
||||
font-size: 24px;
|
||||
}
|
||||
/* Cards */
|
||||
.card {
|
||||
background: #16213e;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
margin-bottom: 16px;
|
||||
border: 1px solid #0f3460;
|
||||
}
|
||||
/* Tables */
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
}
|
||||
th {
|
||||
text-align: left;
|
||||
padding: 12px 16px;
|
||||
background: #0f3460;
|
||||
color: #e94560;
|
||||
font-size: 12px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
td {
|
||||
padding: 10px 16px;
|
||||
border-bottom: 1px solid #0f3460;
|
||||
font-size: 14px;
|
||||
}
|
||||
tr:hover td {
|
||||
background: rgba(15, 52, 96, 0.3);
|
||||
}
|
||||
/* Badges */
|
||||
.badge {
|
||||
display: inline-block;
|
||||
padding: 3px 10px;
|
||||
border-radius: 12px;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
}
|
||||
.badge-green { background: #0f9b58; color: #fff; }
|
||||
.badge-red { background: #e94560; color: #fff; }
|
||||
.badge-yellow { background: #f0ad4e; color: #000; }
|
||||
/* Error */
|
||||
.error-banner {
|
||||
background: #e94560;
|
||||
color: #fff;
|
||||
padding: 16px 20px;
|
||||
border-radius: 8px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
/* Link */
|
||||
a { color: #e94560; text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
/* Responsive */
|
||||
@media (max-width: 768px) {
|
||||
body { flex-direction: column; }
|
||||
.sidebar { width: 100%; border-right: none; border-bottom: 1px solid #0f3460; }
|
||||
.main { padding: 16px; }
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<nav class="sidebar">
|
||||
<h1>Willhaben Tracker</h1>
|
||||
<a href="/" class="{{ 'active' if request.url.path == '/' else '' }}">Dashboard</a>
|
||||
<a href="/keywords" class="{{ 'active' if 'keywords' in request.url.path else '' }}">Keywords</a>
|
||||
<a href="/users" class="{{ 'active' if request.url.path == '/users' else '' }}">Users</a>
|
||||
<a href="/ads" class="{{ 'active' if request.url.path == '/ads' else '' }}">Ads</a>
|
||||
</nav>
|
||||
<main class="main">
|
||||
{% if error %}
|
||||
<div class="error-banner">{{ error }}</div>
|
||||
{% endif %}
|
||||
{% block content %}{% endblock %}
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,43 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Dashboard — Willhaben Tracker{% endblock %}
|
||||
{% block content %}
|
||||
<div class="page-header">
|
||||
<h2>Dashboard</h2>
|
||||
</div>
|
||||
|
||||
{% if data %}
|
||||
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 16px;">
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Total Keywords</div>
|
||||
<div style="font-size: 32px; font-weight: 700; margin-top: 8px;">{{ data.total_keywords }}</div>
|
||||
<div style="font-size: 12px; color: #0f9b58; margin-top: 4px;">{{ data.active_keywords }} active</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Total Ads</div>
|
||||
<div style="font-size: 32px; font-weight: 700; margin-top: 8px;">{{ data.total_ads }}</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Active Users</div>
|
||||
<div style="font-size: 32px; font-weight: 700; margin-top: 8px;">{{ data.total_users }}</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Notifications Sent</div>
|
||||
<div style="font-size: 32px; font-weight: 700; margin-top: 8px;">{{ data.notifications_sent }}</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Queue Pending</div>
|
||||
<div style="font-size: 32px; font-weight: 700; margin-top: 8px;">{{ data.queue_pending }}</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Queue Dead</div>
|
||||
<div style="font-size: 32px; font-weight: 700; margin-top: 8px; color: #e94560;">{{ data.queue_dead }}</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Last Scheduler Run</div>
|
||||
<div style="font-size: 16px; font-weight: 700; margin-top: 8px;">
|
||||
{{ data.last_scheduler.strftime('%Y-%m-%d %H:%M:%S') if data.last_scheduler else 'Never' }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,122 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}{{ keyword.keyword }} — Willhaben Tracker{% endblock %}
|
||||
{% block content %}
|
||||
<div class="page-header">
|
||||
<h2>{{ keyword.keyword }}</h2>
|
||||
</div>
|
||||
|
||||
{% if keyword %}
|
||||
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 16px; margin-bottom: 24px;">
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Status</div>
|
||||
<div style="margin-top: 8px;">
|
||||
{% if keyword.is_active %}
|
||||
<span class="badge badge-green">Active</span>
|
||||
{% else %}
|
||||
<span class="badge badge-red">Stopped</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Interval</div>
|
||||
<div style="font-size: 24px; font-weight: 700; margin-top: 8px;">{{ keyword.interval_minutes }}m</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Subscribers</div>
|
||||
<div style="font-size: 24px; font-weight: 700; margin-top: 8px;">{{ subscriber_count }}</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Price Range</div>
|
||||
<div style="font-size: 16px; font-weight: 700; margin-top: 8px;">
|
||||
{% if keyword.price_min %}€{{ (keyword.price_min / 100)|round(2) }}{% else %}No min{% endif %}
|
||||
—
|
||||
{% if keyword.price_max %}€{{ (keyword.price_max / 100)|round(2) }}{% else %}No max{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Postcodes</div>
|
||||
<div style="font-size: 16px; font-weight: 700; margin-top: 8px;">
|
||||
{{ ', '.join(keyword.allowed_postcodes) if keyword.allowed_postcodes else 'All' }}
|
||||
</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Last Scraped</div>
|
||||
<div style="font-size: 16px; font-weight: 700; margin-top: 8px;">
|
||||
{{ keyword.last_scraped_at.strftime('%Y-%m-%d %H:%M') if keyword.last_scraped_at else 'Never' }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h3 style="color: #e94560; margin-bottom: 12px;">Recent Ads</h3>
|
||||
{% if ads %}
|
||||
<div class="card" style="overflow-x: auto;">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Title</th>
|
||||
<th>Price</th>
|
||||
<th>Location</th>
|
||||
<th>Postcode</th>
|
||||
<th>URL</th>
|
||||
<th>Published</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for ad in ads %}
|
||||
<tr>
|
||||
<td>{{ ad.title[:60] }}{% if ad.title|length > 60 %}…{% endif %}</td>
|
||||
<td>{{ format_price(ad.price) }}</td>
|
||||
<td>{{ ad.location or '—' }}</td>
|
||||
<td>{{ ad.postcode or '—' }}</td>
|
||||
<td><a href="{{ ad.url }}" target="_blank">Link</a></td>
|
||||
<td>{{ ad.published_at.strftime('%Y-%m-%d') if ad.published_at else '—' }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="card"><em>No ads found for this keyword.</em></div>
|
||||
{% endif %}
|
||||
|
||||
<h3 style="color: #e94560; margin: 24px 0 12px;">Recent Scrape Logs</h3>
|
||||
{% if logs %}
|
||||
<div class="card" style="overflow-x: auto;">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Time</th>
|
||||
<th>Status</th>
|
||||
<th>Ads Found</th>
|
||||
<th>New Ads</th>
|
||||
<th>Error</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for log in logs %}
|
||||
<tr>
|
||||
<td>{{ log.scraped_at.strftime('%Y-%m-%d %H:%M:%S') }}</td>
|
||||
<td>
|
||||
{% if log.status == 'success' %}
|
||||
<span class="badge badge-green">Success</span>
|
||||
{% elif log.status == 'rate_limited' %}
|
||||
<span class="badge badge-yellow">Rate Limited</span>
|
||||
{% else %}
|
||||
<span class="badge badge-red">Error</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ log.ads_found }}</td>
|
||||
<td>{{ log.new_ads }}</td>
|
||||
<td style="max-width: 200px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;">
|
||||
{{ log.error_message or '—' }}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="card"><em>No scrape logs found.</em></div>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,56 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Keywords — Willhaben Tracker{% endblock %}
|
||||
{% block content %}
|
||||
<div class="page-header">
|
||||
<h2>Keywords</h2>
|
||||
</div>
|
||||
|
||||
{% if not error and keywords %}
|
||||
<div class="card" style="overflow-x: auto;">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>ID</th>
|
||||
<th>Keyword</th>
|
||||
<th>Status</th>
|
||||
<th>Interval</th>
|
||||
<th>Price Min</th>
|
||||
<th>Price Max</th>
|
||||
<th>Postcodes</th>
|
||||
<th>Subscribers</th>
|
||||
<th>Last Scraped</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for kw in keywords %}
|
||||
<tr>
|
||||
<td><a href="/keywords/{{ kw.id }}">{{ kw.id[:8] }}…</a></td>
|
||||
<td><strong>{{ kw.keyword }}</strong></td>
|
||||
<td>
|
||||
{% if kw.is_active %}
|
||||
<span class="badge badge-green">Active</span>
|
||||
{% else %}
|
||||
<span class="badge badge-red">Stopped</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ kw.interval_minutes }}m</td>
|
||||
<td>{{ (kw.price_min / 100)|round(2) if kw.price_min else '—' }}</td>
|
||||
<td>{{ (kw.price_max / 100)|round(2) if kw.price_max else '—' }}</td>
|
||||
<td>{{ ', '.join(kw.allowed_postcodes) if kw.allowed_postcodes else '—' }}</td>
|
||||
<td>{{ kw.subscriber_count }}</td>
|
||||
<td>
|
||||
{% if kw.last_scraped_at %}
|
||||
{{ kw.last_scraped_at.strftime('%Y-%m-%d %H:%M') }}
|
||||
{% else %}
|
||||
Never
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% elif not error %}
|
||||
<div class="card"><em>No keywords found.</em></div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,66 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Users — Willhaben Tracker{% endblock %}
|
||||
{% block content %}
|
||||
<div class="page-header">
|
||||
<h2>Users</h2>
|
||||
</div>
|
||||
|
||||
{% if not error and users %}
|
||||
<div class="card" style="overflow-x: auto;">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Telegram ID</th>
|
||||
<th>Username</th>
|
||||
<th>First Name</th>
|
||||
<th>Admin</th>
|
||||
<th>Status</th>
|
||||
<th>Mute Hours</th>
|
||||
<th>Digest Mode</th>
|
||||
<th>Digest Interval</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for user in users %}
|
||||
<tr>
|
||||
<td>{{ user.telegram_id }}</td>
|
||||
<td>@{{ user.username }}{% if not user.username %}—{% endif %}</td>
|
||||
<td>{{ user.first_name or '—' }}</td>
|
||||
<td>
|
||||
{% if user.is_admin %}
|
||||
<span class="badge badge-yellow">Admin</span>
|
||||
{% else %}
|
||||
<span class="badge" style="background: #555; color: #fff;">User</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if user.is_active %}
|
||||
<span class="badge badge-green">Active</span>
|
||||
{% else %}
|
||||
<span class="badge badge-red">Inactive</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if user.mute_start and user.mute_end %}
|
||||
{{ user.mute_start.strftime('%H:%M') }} — {{ user.mute_end.strftime('%H:%M') }}
|
||||
{% else %}
|
||||
—
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if user.digest_mode %}
|
||||
<span class="badge badge-green">On</span>
|
||||
{% else %}
|
||||
<span class="badge" style="background: #555; color: #fff;">Off</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ user.digest_interval }}m</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% elif not error %}
|
||||
<div class="card"><em>No users found.</em></div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,313 @@
|
||||
import os
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, Request, HTTPException
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from fastapi.security import HTTPBasic, HTTPBasicCredentials
|
||||
import asyncpg
|
||||
|
||||
from db import get_pool
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
USERNAME = os.getenv("WEB_UI_USERNAME", "admin")
|
||||
PASSWORD = os.getenv("WEB_UI_PASSWORD", "admin")
|
||||
|
||||
security = HTTPBasic()
|
||||
templates = Jinja2Templates(directory="templates")
|
||||
|
||||
|
||||
def get_current_user(credentials: HTTPBasicCredentials):
|
||||
if credentials.username == USERNAME and credentials.password == PASSWORD:
|
||||
return credentials.username
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail="Invalid credentials",
|
||||
headers={"WWW-Authenticate": "Basic"},
|
||||
)
|
||||
|
||||
|
||||
async def query(sql: str, *args) -> list:
|
||||
"""Execute a query and return rows as dicts."""
|
||||
try:
|
||||
pool = await get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
rows = await conn.fetch(sql, *args)
|
||||
return [dict(r) for r in rows]
|
||||
except Exception as e:
|
||||
logger.error("Database query error: %s", e)
|
||||
raise
|
||||
|
||||
|
||||
async def query_one(sql: str, *args) -> dict | None:
|
||||
"""Execute a query and return a single row as dict."""
|
||||
try:
|
||||
pool = await get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
row = await conn.fetchrow(sql, *args)
|
||||
return dict(row) if row else None
|
||||
except Exception as e:
|
||||
logger.error("Database query error: %s", e)
|
||||
raise
|
||||
|
||||
|
||||
async def value(sql: str, *args):
|
||||
"""Execute a query and return a single value."""
|
||||
try:
|
||||
pool = await get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
return await conn.fetchval(sql, *args)
|
||||
except Exception as e:
|
||||
logger.error("Database query error: %s", e)
|
||||
raise
|
||||
|
||||
|
||||
def format_price(cents: int | None) -> str:
|
||||
if cents is None:
|
||||
return "—"
|
||||
return f"€{cents / 100:.2f}"
|
||||
|
||||
|
||||
def format_postcodes(postcodes: list | None) -> str:
|
||||
if not postcodes:
|
||||
return "—"
|
||||
return ", ".join(str(p) for p in postcodes)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
logger.info("Web UI starting up")
|
||||
yield
|
||||
logger.info("Web UI shutting down")
|
||||
|
||||
|
||||
app = FastAPI(title="Willhaben Tracker Web UI", lifespan=lifespan)
|
||||
|
||||
|
||||
# ─── Middleware ───────────────────────────────────────────────
|
||||
|
||||
@app.middleware("http")
|
||||
async def auth_middleware(request: Request, call_next):
|
||||
credentials = None
|
||||
auth_header = request.headers.get("authorization")
|
||||
if auth_header and auth_header.startswith("Basic "):
|
||||
import base64
|
||||
try:
|
||||
decoded = base64.b64decode(auth_header[6:]).decode("utf-8")
|
||||
username, password = decoded.split(":", 1)
|
||||
credentials = HTTPBasicCredentials(username=username, password=password)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not credentials or not get_current_user(credentials):
|
||||
return JSONResponse(
|
||||
status_code=401,
|
||||
content={"detail": "Authentication required"},
|
||||
headers={"WWW-Authenticate": "Basic realm='Willhaben Tracker'"},
|
||||
)
|
||||
|
||||
response = await call_next(request)
|
||||
return response
|
||||
|
||||
|
||||
# ─── Routes ───────────────────────────────────────────────────
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
async def dashboard(request: Request):
|
||||
try:
|
||||
total_keywords = await value("SELECT COUNT(*) FROM keywords")
|
||||
active_keywords = await value("SELECT COUNT(*) FROM keywords WHERE is_active = true")
|
||||
total_ads = await value("SELECT COUNT(*) FROM ads")
|
||||
total_users = await value("SELECT COUNT(*) FROM users WHERE is_active = true")
|
||||
notifications_sent = await value("SELECT COUNT(*) FROM notifications")
|
||||
queue_pending = await value("SELECT COUNT(*) FROM notification_queue WHERE status = 'pending'")
|
||||
queue_dead = await value("SELECT COUNT(*) FROM notification_queue WHERE status = 'dead'")
|
||||
last_scheduler = await query_one(
|
||||
"SELECT scraped_at FROM scrape_logs ORDER BY scraped_at DESC LIMIT 1"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Dashboard query error: %s", e)
|
||||
return templates.TemplateResponse(
|
||||
"dashboard.html",
|
||||
{"request": request, "error": "Database unavailable", "data": None},
|
||||
)
|
||||
|
||||
data = {
|
||||
"total_keywords": total_keywords or 0,
|
||||
"active_keywords": active_keywords or 0,
|
||||
"total_ads": total_ads or 0,
|
||||
"total_users": total_users or 0,
|
||||
"notifications_sent": notifications_sent or 0,
|
||||
"queue_pending": queue_pending or 0,
|
||||
"queue_dead": queue_dead or 0,
|
||||
"last_scheduler": last_scheduler["scraped_at"] if last_scheduler else None,
|
||||
}
|
||||
return templates.TemplateResponse("dashboard.html", {"request": request, "error": None, "data": data})
|
||||
|
||||
|
||||
@app.get("/keywords", response_class=HTMLResponse)
|
||||
async def keywords_list(request: Request):
|
||||
try:
|
||||
keywords = await query(
|
||||
"""
|
||||
SELECT k.*,
|
||||
COUNT(DISTINCT ks.user_id) AS subscriber_count
|
||||
FROM keywords k
|
||||
LEFT JOIN keyword_subscriptions ks ON ks.keyword_id = k.id
|
||||
GROUP BY k.id
|
||||
ORDER BY k.created_at DESC
|
||||
"""
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Keywords query error: %s", e)
|
||||
return templates.TemplateResponse(
|
||||
"keywords.html",
|
||||
{"request": request, "error": "Database unavailable", "keywords": []},
|
||||
)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"keywords.html",
|
||||
{"request": request, "error": None, "keywords": keywords},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/keywords/{keyword_id}", response_class=HTMLResponse)
|
||||
async def keyword_detail(request: Request, keyword_id: str):
|
||||
try:
|
||||
kw = await query_one("SELECT * FROM keywords WHERE id = $1", keyword_id)
|
||||
if not kw:
|
||||
raise HTTPException(status_code=404, detail="Keyword not found")
|
||||
|
||||
subscriber_count = await value(
|
||||
"SELECT COUNT(*) FROM keyword_subscriptions WHERE keyword_id = $1", keyword_id
|
||||
)
|
||||
|
||||
# Get recent ads associated with this keyword via scrape_logs
|
||||
ads = await query(
|
||||
"""
|
||||
SELECT DISTINCT ON (a.id) a.*, sl.scraped_at as last_scrape
|
||||
FROM ads a
|
||||
JOIN scrape_logs sl ON sl.keyword_id = $1
|
||||
ORDER BY a.id, sl.scraped_at DESC
|
||||
LIMIT 20
|
||||
""",
|
||||
keyword_id,
|
||||
)
|
||||
|
||||
logs = await query(
|
||||
"""
|
||||
SELECT * FROM scrape_logs
|
||||
WHERE keyword_id = $1
|
||||
ORDER BY scraped_at DESC
|
||||
LIMIT 10
|
||||
""",
|
||||
keyword_id,
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Keyword detail error: %s", e)
|
||||
return templates.TemplateResponse(
|
||||
"keyword_detail.html",
|
||||
{"request": request, "error": "Database unavailable", "keyword": None, "ads": [], "logs": [], "subscriber_count": 0},
|
||||
)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"keyword_detail.html",
|
||||
{
|
||||
"request": request,
|
||||
"error": None,
|
||||
"keyword": kw,
|
||||
"ads": ads,
|
||||
"logs": logs,
|
||||
"subscriber_count": subscriber_count or 0,
|
||||
"format_price": format_price,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/users", response_class=HTMLResponse)
|
||||
async def users_list(request: Request):
|
||||
try:
|
||||
users = await query(
|
||||
"""
|
||||
SELECT u.*,
|
||||
us.mute_start,
|
||||
us.mute_end,
|
||||
us.digest_mode,
|
||||
us.digest_interval
|
||||
FROM users u
|
||||
LEFT JOIN user_settings us ON us.telegram_id = u.telegram_id::text
|
||||
ORDER BY u.created_at DESC
|
||||
"""
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Users query error: %s", e)
|
||||
return templates.TemplateResponse(
|
||||
"users.html",
|
||||
{"request": request, "error": "Database unavailable", "users": []},
|
||||
)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"users.html",
|
||||
{"request": request, "error": None, "users": users},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/ads", response_class=HTMLResponse)
|
||||
async def ads_list(request: Request):
|
||||
try:
|
||||
ads = await query(
|
||||
"""
|
||||
SELECT * FROM ads
|
||||
ORDER BY first_seen_at DESC
|
||||
LIMIT 50
|
||||
"""
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Ads query error: %s", e)
|
||||
return templates.TemplateResponse(
|
||||
"ads.html",
|
||||
{"request": request, "error": "Database unavailable", "ads": []},
|
||||
)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"ads.html",
|
||||
{"request": request, "error": None, "ads": ads, "format_price": format_price},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/stats")
|
||||
async def stats_json():
|
||||
try:
|
||||
total_keywords = await value("SELECT COUNT(*) FROM keywords")
|
||||
active_keywords = await value("SELECT COUNT(*) FROM keywords WHERE is_active = true")
|
||||
total_ads = await value("SELECT COUNT(*) FROM ads")
|
||||
total_users = await value("SELECT COUNT(*) FROM users WHERE is_active = true")
|
||||
notifications_sent = await value("SELECT COUNT(*) FROM notifications")
|
||||
queue_pending = await value("SELECT COUNT(*) FROM notification_queue WHERE status = 'pending'")
|
||||
queue_dead = await value("SELECT COUNT(*) FROM notification_queue WHERE status = 'dead'")
|
||||
queue_failed = await value("SELECT COUNT(*) FROM notification_queue WHERE status = 'failed'")
|
||||
digest_buffered = await value("SELECT COUNT(*) FROM digest_buffer")
|
||||
last_scheduler = await query_one(
|
||||
"SELECT scraped_at FROM scrape_logs ORDER BY scraped_at DESC LIMIT 1"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Stats query error: %s", e)
|
||||
return JSONResponse(status_code=503, content={"error": "Database unavailable"})
|
||||
|
||||
return {
|
||||
"total_keywords": total_keywords or 0,
|
||||
"active_keywords": active_keywords or 0,
|
||||
"total_ads": total_ads or 0,
|
||||
"total_users": total_users or 0,
|
||||
"notifications_sent": notifications_sent or 0,
|
||||
"queue_pending": queue_pending or 0,
|
||||
"queue_dead": queue_dead or 0,
|
||||
"queue_failed": queue_failed or 0,
|
||||
"digest_buffered": digest_buffered or 0,
|
||||
"last_scheduler_run": last_scheduler["scraped_at"].isoformat() if last_scheduler else None,
|
||||
}
|
||||
Reference in New Issue
Block a user