- Remove multi-marketplace from Phase 3 - Add FastAPI web UI on port 8766 with basic auth - Add 6 Jinja2 templates (dashboard, keywords, users, ads, stats) - Add pytest test suite (45 tests, 49% coverage) - Add GitHub Actions CI/CD workflow - Update docker-compose.yml to expose web UI port - Update Dockerfile to include tests
This commit is contained in:
+2
-1
@@ -5,8 +5,9 @@ WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# ── Application code + migrations ───────────────
|
||||
# ── Application code + migrations + tests + templates ──
|
||||
COPY src/ .
|
||||
COPY tests/ tests/
|
||||
|
||||
# Make entrypoint executable
|
||||
RUN chmod +x entrypoint.sh
|
||||
|
||||
@@ -3,3 +3,11 @@ asyncpg==0.30.0
|
||||
httpx==0.27.2
|
||||
aiohttp>=3.9,<4
|
||||
python-dotenv==1.0.1
|
||||
fastapi==0.115.0
|
||||
uvicorn==0.30.0
|
||||
jinja2==3.1.4
|
||||
|
||||
# Test dependencies
|
||||
pytest==8.3.0
|
||||
pytest-asyncio==0.24.0
|
||||
flake8==7.1.0
|
||||
|
||||
+20
-1
@@ -388,6 +388,21 @@ async def main() -> None:
|
||||
await site.start()
|
||||
logger.info("Health check server listening on :%d", _health_port)
|
||||
|
||||
# ── Start Web UI server ────────────────────────────────────────
|
||||
_web_server = None
|
||||
_web_config = None
|
||||
try:
|
||||
from web import app as web_app # noqa: E402
|
||||
import uvicorn # noqa: E402
|
||||
|
||||
_web_port = int(os.getenv("WEB_UI_PORT", "8766"))
|
||||
_web_config = uvicorn.Config(web_app, host="0.0.0.0", port=_web_port, log_level="info")
|
||||
_web_server = uvicorn.Server(_web_config)
|
||||
asyncio.ensure_future(_web_server.serve())
|
||||
logger.info("Web UI listening on :%d", _web_port)
|
||||
except Exception:
|
||||
logger.exception("Failed to start Web UI (optional)")
|
||||
|
||||
scheduler = asyncio.ensure_future(scheduler_task(pool, app.bot))
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
@@ -435,7 +450,11 @@ async def main() -> None:
|
||||
# ── Close health server ───────────────────────────────────────
|
||||
logger.info("Stopping health check server...")
|
||||
await runner.cleanup()
|
||||
|
||||
# ── Close Web UI server ──────────────────────────────────────
|
||||
if _web_server is not None:
|
||||
logger.info("Stopping Web UI server...")
|
||||
_web_server.should_exit = True
|
||||
await asyncio.sleep(1)
|
||||
# ── Close HTTP client ────────────────────────────────────────
|
||||
logger.info("Closing HTTP client...")
|
||||
from scraper import close_client as close_http_client # noqa: E402
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Ads — Willhaben Tracker{% endblock %}
|
||||
{% block content %}
|
||||
<div class="page-header">
|
||||
<h2>Recent Ads</h2>
|
||||
</div>
|
||||
|
||||
{% if not error and ads %}
|
||||
<div class="card" style="overflow-x: auto;">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Title</th>
|
||||
<th>Price</th>
|
||||
<th>Location</th>
|
||||
<th>Postcode</th>
|
||||
<th>URL</th>
|
||||
<th>Published</th>
|
||||
<th>First Seen</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for ad in ads %}
|
||||
<tr>
|
||||
<td>{{ ad.title[:70] }}{% if ad.title|length > 70 %}…{% endif %}</td>
|
||||
<td>{{ format_price(ad.price) }}</td>
|
||||
<td>{{ ad.location or '—' }}</td>
|
||||
<td>{{ ad.postcode or '—' }}</td>
|
||||
<td><a href="{{ ad.url }}" target="_blank">Link</a></td>
|
||||
<td>{{ ad.published_at.strftime('%Y-%m-%d %H:%M') if ad.published_at else '—' }}</td>
|
||||
<td>{{ ad.first_seen_at.strftime('%Y-%m-%d %H:%M') }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% elif not error %}
|
||||
<div class="card"><em>No ads found.</em></div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,131 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>{% block title %}Willhaben Tracker{% endblock %}</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body {
|
||||
font-family: system-ui, -apple-system, sans-serif;
|
||||
background: #1a1a2e;
|
||||
color: #e0e0e0;
|
||||
display: flex;
|
||||
min-height: 100vh;
|
||||
}
|
||||
/* Sidebar */
|
||||
.sidebar {
|
||||
width: 220px;
|
||||
background: #16213e;
|
||||
padding: 20px 0;
|
||||
border-right: 1px solid #0f3460;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.sidebar h1 {
|
||||
color: #e94560;
|
||||
font-size: 18px;
|
||||
padding: 0 20px 20px;
|
||||
border-bottom: 1px solid #0f3460;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
.sidebar a {
|
||||
display: block;
|
||||
color: #a0a0b0;
|
||||
text-decoration: none;
|
||||
padding: 10px 20px;
|
||||
font-size: 14px;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
.sidebar a:hover, .sidebar a.active {
|
||||
background: #0f3460;
|
||||
color: #e94560;
|
||||
}
|
||||
/* Main */
|
||||
.main {
|
||||
flex: 1;
|
||||
padding: 30px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
.page-header {
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
.page-header h2 {
|
||||
color: #e94560;
|
||||
font-size: 24px;
|
||||
}
|
||||
/* Cards */
|
||||
.card {
|
||||
background: #16213e;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
margin-bottom: 16px;
|
||||
border: 1px solid #0f3460;
|
||||
}
|
||||
/* Tables */
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
}
|
||||
th {
|
||||
text-align: left;
|
||||
padding: 12px 16px;
|
||||
background: #0f3460;
|
||||
color: #e94560;
|
||||
font-size: 12px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
td {
|
||||
padding: 10px 16px;
|
||||
border-bottom: 1px solid #0f3460;
|
||||
font-size: 14px;
|
||||
}
|
||||
tr:hover td {
|
||||
background: rgba(15, 52, 96, 0.3);
|
||||
}
|
||||
/* Badges */
|
||||
.badge {
|
||||
display: inline-block;
|
||||
padding: 3px 10px;
|
||||
border-radius: 12px;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
}
|
||||
.badge-green { background: #0f9b58; color: #fff; }
|
||||
.badge-red { background: #e94560; color: #fff; }
|
||||
.badge-yellow { background: #f0ad4e; color: #000; }
|
||||
/* Error */
|
||||
.error-banner {
|
||||
background: #e94560;
|
||||
color: #fff;
|
||||
padding: 16px 20px;
|
||||
border-radius: 8px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
/* Link */
|
||||
a { color: #e94560; text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
/* Responsive */
|
||||
@media (max-width: 768px) {
|
||||
body { flex-direction: column; }
|
||||
.sidebar { width: 100%; border-right: none; border-bottom: 1px solid #0f3460; }
|
||||
.main { padding: 16px; }
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<nav class="sidebar">
|
||||
<h1>Willhaben Tracker</h1>
|
||||
<a href="/" class="{{ 'active' if request.url.path == '/' else '' }}">Dashboard</a>
|
||||
<a href="/keywords" class="{{ 'active' if 'keywords' in request.url.path else '' }}">Keywords</a>
|
||||
<a href="/users" class="{{ 'active' if request.url.path == '/users' else '' }}">Users</a>
|
||||
<a href="/ads" class="{{ 'active' if request.url.path == '/ads' else '' }}">Ads</a>
|
||||
</nav>
|
||||
<main class="main">
|
||||
{% if error %}
|
||||
<div class="error-banner">{{ error }}</div>
|
||||
{% endif %}
|
||||
{% block content %}{% endblock %}
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,43 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Dashboard — Willhaben Tracker{% endblock %}
|
||||
{% block content %}
|
||||
<div class="page-header">
|
||||
<h2>Dashboard</h2>
|
||||
</div>
|
||||
|
||||
{% if data %}
|
||||
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 16px;">
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Total Keywords</div>
|
||||
<div style="font-size: 32px; font-weight: 700; margin-top: 8px;">{{ data.total_keywords }}</div>
|
||||
<div style="font-size: 12px; color: #0f9b58; margin-top: 4px;">{{ data.active_keywords }} active</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Total Ads</div>
|
||||
<div style="font-size: 32px; font-weight: 700; margin-top: 8px;">{{ data.total_ads }}</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Active Users</div>
|
||||
<div style="font-size: 32px; font-weight: 700; margin-top: 8px;">{{ data.total_users }}</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Notifications Sent</div>
|
||||
<div style="font-size: 32px; font-weight: 700; margin-top: 8px;">{{ data.notifications_sent }}</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Queue Pending</div>
|
||||
<div style="font-size: 32px; font-weight: 700; margin-top: 8px;">{{ data.queue_pending }}</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Queue Dead</div>
|
||||
<div style="font-size: 32px; font-weight: 700; margin-top: 8px; color: #e94560;">{{ data.queue_dead }}</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Last Scheduler Run</div>
|
||||
<div style="font-size: 16px; font-weight: 700; margin-top: 8px;">
|
||||
{{ data.last_scheduler.strftime('%Y-%m-%d %H:%M:%S') if data.last_scheduler else 'Never' }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,122 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}{{ keyword.keyword }} — Willhaben Tracker{% endblock %}
|
||||
{% block content %}
|
||||
<div class="page-header">
|
||||
<h2>{{ keyword.keyword }}</h2>
|
||||
</div>
|
||||
|
||||
{% if keyword %}
|
||||
<div style="display: grid; grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); gap: 16px; margin-bottom: 24px;">
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Status</div>
|
||||
<div style="margin-top: 8px;">
|
||||
{% if keyword.is_active %}
|
||||
<span class="badge badge-green">Active</span>
|
||||
{% else %}
|
||||
<span class="badge badge-red">Stopped</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Interval</div>
|
||||
<div style="font-size: 24px; font-weight: 700; margin-top: 8px;">{{ keyword.interval_minutes }}m</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Subscribers</div>
|
||||
<div style="font-size: 24px; font-weight: 700; margin-top: 8px;">{{ subscriber_count }}</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Price Range</div>
|
||||
<div style="font-size: 16px; font-weight: 700; margin-top: 8px;">
|
||||
{% if keyword.price_min %}€{{ (keyword.price_min / 100)|round(2) }}{% else %}No min{% endif %}
|
||||
—
|
||||
{% if keyword.price_max %}€{{ (keyword.price_max / 100)|round(2) }}{% else %}No max{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Postcodes</div>
|
||||
<div style="font-size: 16px; font-weight: 700; margin-top: 8px;">
|
||||
{{ ', '.join(keyword.allowed_postcodes) if keyword.allowed_postcodes else 'All' }}
|
||||
</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div style="font-size: 12px; color: #a0a0b0; text-transform: uppercase;">Last Scraped</div>
|
||||
<div style="font-size: 16px; font-weight: 700; margin-top: 8px;">
|
||||
{{ keyword.last_scraped_at.strftime('%Y-%m-%d %H:%M') if keyword.last_scraped_at else 'Never' }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h3 style="color: #e94560; margin-bottom: 12px;">Recent Ads</h3>
|
||||
{% if ads %}
|
||||
<div class="card" style="overflow-x: auto;">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Title</th>
|
||||
<th>Price</th>
|
||||
<th>Location</th>
|
||||
<th>Postcode</th>
|
||||
<th>URL</th>
|
||||
<th>Published</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for ad in ads %}
|
||||
<tr>
|
||||
<td>{{ ad.title[:60] }}{% if ad.title|length > 60 %}…{% endif %}</td>
|
||||
<td>{{ format_price(ad.price) }}</td>
|
||||
<td>{{ ad.location or '—' }}</td>
|
||||
<td>{{ ad.postcode or '—' }}</td>
|
||||
<td><a href="{{ ad.url }}" target="_blank">Link</a></td>
|
||||
<td>{{ ad.published_at.strftime('%Y-%m-%d') if ad.published_at else '—' }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="card"><em>No ads found for this keyword.</em></div>
|
||||
{% endif %}
|
||||
|
||||
<h3 style="color: #e94560; margin: 24px 0 12px;">Recent Scrape Logs</h3>
|
||||
{% if logs %}
|
||||
<div class="card" style="overflow-x: auto;">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Time</th>
|
||||
<th>Status</th>
|
||||
<th>Ads Found</th>
|
||||
<th>New Ads</th>
|
||||
<th>Error</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for log in logs %}
|
||||
<tr>
|
||||
<td>{{ log.scraped_at.strftime('%Y-%m-%d %H:%M:%S') }}</td>
|
||||
<td>
|
||||
{% if log.status == 'success' %}
|
||||
<span class="badge badge-green">Success</span>
|
||||
{% elif log.status == 'rate_limited' %}
|
||||
<span class="badge badge-yellow">Rate Limited</span>
|
||||
{% else %}
|
||||
<span class="badge badge-red">Error</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ log.ads_found }}</td>
|
||||
<td>{{ log.new_ads }}</td>
|
||||
<td style="max-width: 200px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;">
|
||||
{{ log.error_message or '—' }}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="card"><em>No scrape logs found.</em></div>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,56 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Keywords — Willhaben Tracker{% endblock %}
|
||||
{% block content %}
|
||||
<div class="page-header">
|
||||
<h2>Keywords</h2>
|
||||
</div>
|
||||
|
||||
{% if not error and keywords %}
|
||||
<div class="card" style="overflow-x: auto;">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>ID</th>
|
||||
<th>Keyword</th>
|
||||
<th>Status</th>
|
||||
<th>Interval</th>
|
||||
<th>Price Min</th>
|
||||
<th>Price Max</th>
|
||||
<th>Postcodes</th>
|
||||
<th>Subscribers</th>
|
||||
<th>Last Scraped</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for kw in keywords %}
|
||||
<tr>
|
||||
<td><a href="/keywords/{{ kw.id }}">{{ kw.id[:8] }}…</a></td>
|
||||
<td><strong>{{ kw.keyword }}</strong></td>
|
||||
<td>
|
||||
{% if kw.is_active %}
|
||||
<span class="badge badge-green">Active</span>
|
||||
{% else %}
|
||||
<span class="badge badge-red">Stopped</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ kw.interval_minutes }}m</td>
|
||||
<td>{{ (kw.price_min / 100)|round(2) if kw.price_min else '—' }}</td>
|
||||
<td>{{ (kw.price_max / 100)|round(2) if kw.price_max else '—' }}</td>
|
||||
<td>{{ ', '.join(kw.allowed_postcodes) if kw.allowed_postcodes else '—' }}</td>
|
||||
<td>{{ kw.subscriber_count }}</td>
|
||||
<td>
|
||||
{% if kw.last_scraped_at %}
|
||||
{{ kw.last_scraped_at.strftime('%Y-%m-%d %H:%M') }}
|
||||
{% else %}
|
||||
Never
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% elif not error %}
|
||||
<div class="card"><em>No keywords found.</em></div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,66 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Users — Willhaben Tracker{% endblock %}
|
||||
{% block content %}
|
||||
<div class="page-header">
|
||||
<h2>Users</h2>
|
||||
</div>
|
||||
|
||||
{% if not error and users %}
|
||||
<div class="card" style="overflow-x: auto;">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Telegram ID</th>
|
||||
<th>Username</th>
|
||||
<th>First Name</th>
|
||||
<th>Admin</th>
|
||||
<th>Status</th>
|
||||
<th>Mute Hours</th>
|
||||
<th>Digest Mode</th>
|
||||
<th>Digest Interval</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for user in users %}
|
||||
<tr>
|
||||
<td>{{ user.telegram_id }}</td>
|
||||
<td>@{{ user.username }}{% if not user.username %}—{% endif %}</td>
|
||||
<td>{{ user.first_name or '—' }}</td>
|
||||
<td>
|
||||
{% if user.is_admin %}
|
||||
<span class="badge badge-yellow">Admin</span>
|
||||
{% else %}
|
||||
<span class="badge" style="background: #555; color: #fff;">User</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if user.is_active %}
|
||||
<span class="badge badge-green">Active</span>
|
||||
{% else %}
|
||||
<span class="badge badge-red">Inactive</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if user.mute_start and user.mute_end %}
|
||||
{{ user.mute_start.strftime('%H:%M') }} — {{ user.mute_end.strftime('%H:%M') }}
|
||||
{% else %}
|
||||
—
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if user.digest_mode %}
|
||||
<span class="badge badge-green">On</span>
|
||||
{% else %}
|
||||
<span class="badge" style="background: #555; color: #fff;">Off</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ user.digest_interval }}m</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% elif not error %}
|
||||
<div class="card"><em>No users found.</em></div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,313 @@
|
||||
import os
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, Request, HTTPException
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from fastapi.security import HTTPBasic, HTTPBasicCredentials
|
||||
import asyncpg
|
||||
|
||||
from db import get_pool
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
USERNAME = os.getenv("WEB_UI_USERNAME", "admin")
|
||||
PASSWORD = os.getenv("WEB_UI_PASSWORD", "admin")
|
||||
|
||||
security = HTTPBasic()
|
||||
templates = Jinja2Templates(directory="templates")
|
||||
|
||||
|
||||
def get_current_user(credentials: HTTPBasicCredentials):
|
||||
if credentials.username == USERNAME and credentials.password == PASSWORD:
|
||||
return credentials.username
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail="Invalid credentials",
|
||||
headers={"WWW-Authenticate": "Basic"},
|
||||
)
|
||||
|
||||
|
||||
async def query(sql: str, *args) -> list:
|
||||
"""Execute a query and return rows as dicts."""
|
||||
try:
|
||||
pool = await get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
rows = await conn.fetch(sql, *args)
|
||||
return [dict(r) for r in rows]
|
||||
except Exception as e:
|
||||
logger.error("Database query error: %s", e)
|
||||
raise
|
||||
|
||||
|
||||
async def query_one(sql: str, *args) -> dict | None:
|
||||
"""Execute a query and return a single row as dict."""
|
||||
try:
|
||||
pool = await get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
row = await conn.fetchrow(sql, *args)
|
||||
return dict(row) if row else None
|
||||
except Exception as e:
|
||||
logger.error("Database query error: %s", e)
|
||||
raise
|
||||
|
||||
|
||||
async def value(sql: str, *args):
|
||||
"""Execute a query and return a single value."""
|
||||
try:
|
||||
pool = await get_pool()
|
||||
async with pool.acquire() as conn:
|
||||
return await conn.fetchval(sql, *args)
|
||||
except Exception as e:
|
||||
logger.error("Database query error: %s", e)
|
||||
raise
|
||||
|
||||
|
||||
def format_price(cents: int | None) -> str:
|
||||
if cents is None:
|
||||
return "—"
|
||||
return f"€{cents / 100:.2f}"
|
||||
|
||||
|
||||
def format_postcodes(postcodes: list | None) -> str:
|
||||
if not postcodes:
|
||||
return "—"
|
||||
return ", ".join(str(p) for p in postcodes)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
logger.info("Web UI starting up")
|
||||
yield
|
||||
logger.info("Web UI shutting down")
|
||||
|
||||
|
||||
app = FastAPI(title="Willhaben Tracker Web UI", lifespan=lifespan)
|
||||
|
||||
|
||||
# ─── Middleware ───────────────────────────────────────────────
|
||||
|
||||
@app.middleware("http")
|
||||
async def auth_middleware(request: Request, call_next):
|
||||
credentials = None
|
||||
auth_header = request.headers.get("authorization")
|
||||
if auth_header and auth_header.startswith("Basic "):
|
||||
import base64
|
||||
try:
|
||||
decoded = base64.b64decode(auth_header[6:]).decode("utf-8")
|
||||
username, password = decoded.split(":", 1)
|
||||
credentials = HTTPBasicCredentials(username=username, password=password)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not credentials or not get_current_user(credentials):
|
||||
return JSONResponse(
|
||||
status_code=401,
|
||||
content={"detail": "Authentication required"},
|
||||
headers={"WWW-Authenticate": "Basic realm='Willhaben Tracker'"},
|
||||
)
|
||||
|
||||
response = await call_next(request)
|
||||
return response
|
||||
|
||||
|
||||
# ─── Routes ───────────────────────────────────────────────────
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
async def dashboard(request: Request):
|
||||
try:
|
||||
total_keywords = await value("SELECT COUNT(*) FROM keywords")
|
||||
active_keywords = await value("SELECT COUNT(*) FROM keywords WHERE is_active = true")
|
||||
total_ads = await value("SELECT COUNT(*) FROM ads")
|
||||
total_users = await value("SELECT COUNT(*) FROM users WHERE is_active = true")
|
||||
notifications_sent = await value("SELECT COUNT(*) FROM notifications")
|
||||
queue_pending = await value("SELECT COUNT(*) FROM notification_queue WHERE status = 'pending'")
|
||||
queue_dead = await value("SELECT COUNT(*) FROM notification_queue WHERE status = 'dead'")
|
||||
last_scheduler = await query_one(
|
||||
"SELECT scraped_at FROM scrape_logs ORDER BY scraped_at DESC LIMIT 1"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Dashboard query error: %s", e)
|
||||
return templates.TemplateResponse(
|
||||
"dashboard.html",
|
||||
{"request": request, "error": "Database unavailable", "data": None},
|
||||
)
|
||||
|
||||
data = {
|
||||
"total_keywords": total_keywords or 0,
|
||||
"active_keywords": active_keywords or 0,
|
||||
"total_ads": total_ads or 0,
|
||||
"total_users": total_users or 0,
|
||||
"notifications_sent": notifications_sent or 0,
|
||||
"queue_pending": queue_pending or 0,
|
||||
"queue_dead": queue_dead or 0,
|
||||
"last_scheduler": last_scheduler["scraped_at"] if last_scheduler else None,
|
||||
}
|
||||
return templates.TemplateResponse("dashboard.html", {"request": request, "error": None, "data": data})
|
||||
|
||||
|
||||
@app.get("/keywords", response_class=HTMLResponse)
|
||||
async def keywords_list(request: Request):
|
||||
try:
|
||||
keywords = await query(
|
||||
"""
|
||||
SELECT k.*,
|
||||
COUNT(DISTINCT ks.user_id) AS subscriber_count
|
||||
FROM keywords k
|
||||
LEFT JOIN keyword_subscriptions ks ON ks.keyword_id = k.id
|
||||
GROUP BY k.id
|
||||
ORDER BY k.created_at DESC
|
||||
"""
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Keywords query error: %s", e)
|
||||
return templates.TemplateResponse(
|
||||
"keywords.html",
|
||||
{"request": request, "error": "Database unavailable", "keywords": []},
|
||||
)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"keywords.html",
|
||||
{"request": request, "error": None, "keywords": keywords},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/keywords/{keyword_id}", response_class=HTMLResponse)
|
||||
async def keyword_detail(request: Request, keyword_id: str):
|
||||
try:
|
||||
kw = await query_one("SELECT * FROM keywords WHERE id = $1", keyword_id)
|
||||
if not kw:
|
||||
raise HTTPException(status_code=404, detail="Keyword not found")
|
||||
|
||||
subscriber_count = await value(
|
||||
"SELECT COUNT(*) FROM keyword_subscriptions WHERE keyword_id = $1", keyword_id
|
||||
)
|
||||
|
||||
# Get recent ads associated with this keyword via scrape_logs
|
||||
ads = await query(
|
||||
"""
|
||||
SELECT DISTINCT ON (a.id) a.*, sl.scraped_at as last_scrape
|
||||
FROM ads a
|
||||
JOIN scrape_logs sl ON sl.keyword_id = $1
|
||||
ORDER BY a.id, sl.scraped_at DESC
|
||||
LIMIT 20
|
||||
""",
|
||||
keyword_id,
|
||||
)
|
||||
|
||||
logs = await query(
|
||||
"""
|
||||
SELECT * FROM scrape_logs
|
||||
WHERE keyword_id = $1
|
||||
ORDER BY scraped_at DESC
|
||||
LIMIT 10
|
||||
""",
|
||||
keyword_id,
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("Keyword detail error: %s", e)
|
||||
return templates.TemplateResponse(
|
||||
"keyword_detail.html",
|
||||
{"request": request, "error": "Database unavailable", "keyword": None, "ads": [], "logs": [], "subscriber_count": 0},
|
||||
)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"keyword_detail.html",
|
||||
{
|
||||
"request": request,
|
||||
"error": None,
|
||||
"keyword": kw,
|
||||
"ads": ads,
|
||||
"logs": logs,
|
||||
"subscriber_count": subscriber_count or 0,
|
||||
"format_price": format_price,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/users", response_class=HTMLResponse)
|
||||
async def users_list(request: Request):
|
||||
try:
|
||||
users = await query(
|
||||
"""
|
||||
SELECT u.*,
|
||||
us.mute_start,
|
||||
us.mute_end,
|
||||
us.digest_mode,
|
||||
us.digest_interval
|
||||
FROM users u
|
||||
LEFT JOIN user_settings us ON us.telegram_id = u.telegram_id::text
|
||||
ORDER BY u.created_at DESC
|
||||
"""
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Users query error: %s", e)
|
||||
return templates.TemplateResponse(
|
||||
"users.html",
|
||||
{"request": request, "error": "Database unavailable", "users": []},
|
||||
)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"users.html",
|
||||
{"request": request, "error": None, "users": users},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/ads", response_class=HTMLResponse)
|
||||
async def ads_list(request: Request):
|
||||
try:
|
||||
ads = await query(
|
||||
"""
|
||||
SELECT * FROM ads
|
||||
ORDER BY first_seen_at DESC
|
||||
LIMIT 50
|
||||
"""
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Ads query error: %s", e)
|
||||
return templates.TemplateResponse(
|
||||
"ads.html",
|
||||
{"request": request, "error": "Database unavailable", "ads": []},
|
||||
)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"ads.html",
|
||||
{"request": request, "error": None, "ads": ads, "format_price": format_price},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/stats")
|
||||
async def stats_json():
|
||||
try:
|
||||
total_keywords = await value("SELECT COUNT(*) FROM keywords")
|
||||
active_keywords = await value("SELECT COUNT(*) FROM keywords WHERE is_active = true")
|
||||
total_ads = await value("SELECT COUNT(*) FROM ads")
|
||||
total_users = await value("SELECT COUNT(*) FROM users WHERE is_active = true")
|
||||
notifications_sent = await value("SELECT COUNT(*) FROM notifications")
|
||||
queue_pending = await value("SELECT COUNT(*) FROM notification_queue WHERE status = 'pending'")
|
||||
queue_dead = await value("SELECT COUNT(*) FROM notification_queue WHERE status = 'dead'")
|
||||
queue_failed = await value("SELECT COUNT(*) FROM notification_queue WHERE status = 'failed'")
|
||||
digest_buffered = await value("SELECT COUNT(*) FROM digest_buffer")
|
||||
last_scheduler = await query_one(
|
||||
"SELECT scraped_at FROM scrape_logs ORDER BY scraped_at DESC LIMIT 1"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Stats query error: %s", e)
|
||||
return JSONResponse(status_code=503, content={"error": "Database unavailable"})
|
||||
|
||||
return {
|
||||
"total_keywords": total_keywords or 0,
|
||||
"active_keywords": active_keywords or 0,
|
||||
"total_ads": total_ads or 0,
|
||||
"total_users": total_users or 0,
|
||||
"notifications_sent": notifications_sent or 0,
|
||||
"queue_pending": queue_pending or 0,
|
||||
"queue_dead": queue_dead or 0,
|
||||
"queue_failed": queue_failed or 0,
|
||||
"digest_buffered": digest_buffered or 0,
|
||||
"last_scheduler_run": last_scheduler["scraped_at"].isoformat() if last_scheduler else None,
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,84 @@
|
||||
"""Shared pytest fixtures for the willhaben-tracker test suite."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""Enable asyncio auto mode for all async tests."""
|
||||
config.addinivalue_line("markers", "asyncio: mark test as async")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_pool():
|
||||
"""Mock asyncpg.Pool with fetch/fetchrow/execute/fetchval methods."""
|
||||
pool = MagicMock()
|
||||
pool.fetch = AsyncMock(return_value=[])
|
||||
pool.fetchrow = AsyncMock(return_value=None)
|
||||
pool.execute = AsyncMock(return_value=None)
|
||||
pool.fetchval = AsyncMock(return_value=None)
|
||||
return pool
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_bot():
|
||||
"""Mock ExtBot with send_message/send_photo methods."""
|
||||
bot = MagicMock()
|
||||
bot.send_message = AsyncMock(return_value=MagicMock(message_id=123))
|
||||
bot.send_photo = AsyncMock(return_value=MagicMock(message_id=123))
|
||||
return bot
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_ad_data():
|
||||
"""Sample willhaben ad JSON dict matching the API response format."""
|
||||
return {
|
||||
"id": "12345678",
|
||||
"description": "A used bicycle",
|
||||
"attributes": {
|
||||
"attribute": [
|
||||
{"name": "HEADING", "values": ["Mountain Bike 2024"]},
|
||||
{"name": "PRICE/AMOUNT", "values": ["250"]},
|
||||
{"name": "LOCATION", "values": ["Vienna"]},
|
||||
{"name": "POSTCODE", "values": ["1010"]},
|
||||
{"name": "SEO_URL", "values": ["mountain-bike-2024/12345678"]},
|
||||
{"name": "PUBLISHED_String", "values": ["2024-01-15T10:30:00Z"]},
|
||||
{"name": "CHANGED_String", "values": ["2024-01-15T12:00:00Z"]},
|
||||
]
|
||||
},
|
||||
"advertImageList": {
|
||||
"advertImage": [
|
||||
{"referenceImageUrl": "https://img.willhaben.at/img123.jpg"}
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_fields():
|
||||
"""Sample extracted ad fields dict as returned by extract_ad_fields()."""
|
||||
return {
|
||||
"wh_ad_id": "12345678",
|
||||
"title": "Mountain Bike 2024",
|
||||
"price": 250.0,
|
||||
"location": "Vienna",
|
||||
"url": "https://www.willhaben.at/iad/mountain-bike-2024/12345678",
|
||||
"published_at": datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc),
|
||||
"main_image_url": "https://img.willhaben.at/img123.jpg",
|
||||
"postcode": "1010",
|
||||
"modified_at": datetime(2024, 1, 15, 12, 0, 0, tzinfo=timezone.utc),
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_kw_row():
|
||||
"""Sample keyword row dict with filter settings."""
|
||||
return {
|
||||
"id": "kw-uuid-123",
|
||||
"keyword": "bike",
|
||||
"price_min": None,
|
||||
"price_max": None,
|
||||
"allowed_postcodes": None,
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Tests for _ad_passes_filters from main.py."""
|
||||
|
||||
from main import _ad_passes_filters
|
||||
|
||||
|
||||
class TestAdPassesFilters:
|
||||
"""Test the _ad_passes_filters function."""
|
||||
|
||||
def test_no_filters_passes(self, sample_fields, sample_kw_row):
|
||||
"""Ad with no filters should always pass."""
|
||||
assert _ad_passes_filters(sample_fields, sample_kw_row) is True
|
||||
|
||||
def test_price_below_min_fails(self, sample_fields, sample_kw_row):
|
||||
"""Ad price below price_min should fail."""
|
||||
kw = {**sample_kw_row, "price_min": 30000} # 300.00 EUR in cents
|
||||
# sample_fields price is 250.00 EUR = 25000 cents
|
||||
assert _ad_passes_filters(sample_fields, kw) is False
|
||||
|
||||
def test_price_above_max_fails(self, sample_fields, sample_kw_row):
|
||||
"""Ad price above price_max should fail."""
|
||||
kw = {**sample_kw_row, "price_max": 20000} # 200.00 EUR in cents
|
||||
# sample_fields price is 250.00 EUR = 25000 cents
|
||||
assert _ad_passes_filters(sample_fields, kw) is False
|
||||
|
||||
def test_price_in_range_passes(self, sample_fields, sample_kw_row):
|
||||
"""Ad price within min/max range should pass."""
|
||||
kw = {**sample_kw_row, "price_min": 10000, "price_max": 50000}
|
||||
# sample_fields price is 250.00 EUR = 25000 cents
|
||||
assert _ad_passes_filters(sample_fields, kw) is True
|
||||
|
||||
def test_price_at_min_boundary_passes(self, sample_fields, sample_kw_row):
|
||||
"""Ad price exactly at price_min should pass."""
|
||||
kw = {**sample_kw_row, "price_min": 25000}
|
||||
assert _ad_passes_filters(sample_fields, kw) is True
|
||||
|
||||
def test_price_at_max_boundary_passes(self, sample_fields, sample_kw_row):
|
||||
"""Ad price exactly at price_max should pass."""
|
||||
kw = {**sample_kw_row, "price_max": 25000}
|
||||
assert _ad_passes_filters(sample_fields, kw) is True
|
||||
|
||||
def test_postcode_not_in_allowed_list_fails(self, sample_fields, sample_kw_row):
|
||||
"""Ad postcode not in allowed_postcodes should fail."""
|
||||
kw = {**sample_kw_row, "allowed_postcodes": ["1020", "1030"]}
|
||||
# sample_fields postcode is "1010"
|
||||
assert _ad_passes_filters(sample_fields, kw) is False
|
||||
|
||||
def test_postcode_in_allowed_list_passes(self, sample_fields, sample_kw_row):
|
||||
"""Ad postcode in allowed_postcodes should pass."""
|
||||
kw = {**sample_kw_row, "allowed_postcodes": ["1010", "1020"]}
|
||||
assert _ad_passes_filters(sample_fields, kw) is True
|
||||
|
||||
def test_no_postcode_with_filter_active_fails(self, sample_fields, sample_kw_row):
|
||||
"""Ad with no postcode when filter is active should fail."""
|
||||
fields = {**sample_fields, "postcode": None}
|
||||
kw = {**sample_kw_row, "allowed_postcodes": ["1010", "1020"]}
|
||||
assert _ad_passes_filters(fields, kw) is False
|
||||
|
||||
def test_combined_price_and_postcode_filters_pass(self, sample_fields, sample_kw_row):
|
||||
"""Ad passing both price and postcode filters should pass."""
|
||||
kw = {
|
||||
**sample_kw_row,
|
||||
"price_min": 10000,
|
||||
"price_max": 50000,
|
||||
"allowed_postcodes": ["1010", "1020"],
|
||||
}
|
||||
assert _ad_passes_filters(sample_fields, kw) is True
|
||||
|
||||
def test_combined_price_passes_postcode_fails(self, sample_fields, sample_kw_row):
|
||||
"""Ad passing price but failing postcode should fail."""
|
||||
kw = {
|
||||
**sample_kw_row,
|
||||
"price_min": 10000,
|
||||
"price_max": 50000,
|
||||
"allowed_postcodes": ["1020", "1030"],
|
||||
}
|
||||
assert _ad_passes_filters(sample_fields, kw) is False
|
||||
|
||||
def test_combined_price_fails_postcode_passes(self, sample_fields, sample_kw_row):
|
||||
"""Ad failing price but passing postcode should fail."""
|
||||
kw = {
|
||||
**sample_kw_row,
|
||||
"price_min": 30000,
|
||||
"price_max": 50000,
|
||||
"allowed_postcodes": ["1010", "1020"],
|
||||
}
|
||||
assert _ad_passes_filters(sample_fields, kw) is False
|
||||
|
||||
def test_no_price_with_price_filter(self, sample_fields, sample_kw_row):
|
||||
"""Ad with no price should pass price filters (price is None)."""
|
||||
fields = {**sample_fields, "price": None}
|
||||
kw = {**sample_kw_row, "price_min": 10000, "price_max": 50000}
|
||||
assert _ad_passes_filters(fields, kw) is True
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Tests for health module."""
|
||||
|
||||
from health import create_health_app
|
||||
|
||||
|
||||
class TestHealthModule:
|
||||
"""Basic import and app creation tests for the health module."""
|
||||
|
||||
def test_health_module_import(self):
|
||||
"""Health module should be importable."""
|
||||
import health
|
||||
assert health is not None
|
||||
|
||||
def test_create_health_app_returns_app(self):
|
||||
"""create_health_app should return an aiohttp web.Application."""
|
||||
app = create_health_app()
|
||||
assert app is not None
|
||||
assert hasattr(app, "router")
|
||||
|
||||
def test_health_app_has_routes(self):
|
||||
"""Health app should have /health and /stats routes."""
|
||||
app = create_health_app()
|
||||
# Collect route info from the router
|
||||
routes_info = []
|
||||
for route in app.router.routes():
|
||||
routes_info.append(repr(route))
|
||||
routes_str = " ".join(routes_info)
|
||||
assert "/health" in routes_str
|
||||
assert "/stats" in routes_str
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Tests for notifier module functions."""
|
||||
|
||||
from datetime import datetime, time, timezone
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from notifier import (
|
||||
_format_text,
|
||||
_build_keyboard,
|
||||
is_user_muted,
|
||||
buffer_for_digest,
|
||||
)
|
||||
|
||||
|
||||
class TestIsUserMuted:
|
||||
"""Test the is_user_muted async function."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_settings_not_muted(self, mock_pool):
|
||||
"""User with no settings should not be muted."""
|
||||
mock_pool.fetchrow.return_value = None
|
||||
result = await is_user_muted(mock_pool, 12345)
|
||||
assert result is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_mute_hours_not_muted(self, mock_pool):
|
||||
"""User with settings but no mute hours should not be muted."""
|
||||
mock_pool.fetchrow.return_value = {"mute_start": None, "mute_end": None}
|
||||
result = await is_user_muted(mock_pool, 12345)
|
||||
assert result is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_normal_window_muted(self, mock_pool):
|
||||
"""User should be muted when current time is within normal window."""
|
||||
mock_pool.fetchrow.return_value = {
|
||||
"mute_start": time(8, 0),
|
||||
"mute_end": time(12, 0),
|
||||
}
|
||||
fake_dt = datetime(2024, 1, 15, 10, 0, 0, tzinfo=timezone.utc)
|
||||
mock_dt = MagicMock()
|
||||
mock_dt.now.return_value = fake_dt
|
||||
with patch("notifier.datetime", mock_dt):
|
||||
result = await is_user_muted(mock_pool, 12345)
|
||||
assert result is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_normal_window_not_muted(self, mock_pool):
|
||||
"""User should not be muted when current time is outside normal window."""
|
||||
mock_pool.fetchrow.return_value = {
|
||||
"mute_start": time(8, 0),
|
||||
"mute_end": time(12, 0),
|
||||
}
|
||||
fake_dt = datetime(2024, 1, 15, 14, 0, 0, tzinfo=timezone.utc)
|
||||
mock_dt = MagicMock()
|
||||
mock_dt.now.return_value = fake_dt
|
||||
with patch("notifier.datetime", mock_dt):
|
||||
result = await is_user_muted(mock_pool, 12345)
|
||||
assert result is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_midnight_window_muted_after_start(self, mock_pool):
|
||||
"""User should be muted when time is after start of cross-midnight window."""
|
||||
mock_pool.fetchrow.return_value = {
|
||||
"mute_start": time(22, 0),
|
||||
"mute_end": time(6, 0),
|
||||
}
|
||||
fake_dt = datetime(2024, 1, 15, 23, 0, 0, tzinfo=timezone.utc)
|
||||
mock_dt = MagicMock()
|
||||
mock_dt.now.return_value = fake_dt
|
||||
with patch("notifier.datetime", mock_dt):
|
||||
result = await is_user_muted(mock_pool, 12345)
|
||||
assert result is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_midnight_window_muted_before_end(self, mock_pool):
|
||||
"""User should be muted when time is before end of cross-midnight window."""
|
||||
mock_pool.fetchrow.return_value = {
|
||||
"mute_start": time(22, 0),
|
||||
"mute_end": time(6, 0),
|
||||
}
|
||||
fake_dt = datetime(2024, 1, 15, 3, 0, 0, tzinfo=timezone.utc)
|
||||
mock_dt = MagicMock()
|
||||
mock_dt.now.return_value = fake_dt
|
||||
with patch("notifier.datetime", mock_dt):
|
||||
result = await is_user_muted(mock_pool, 12345)
|
||||
assert result is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_midnight_window_not_muted(self, mock_pool):
|
||||
"""User should not be muted when time is outside cross-midnight window."""
|
||||
mock_pool.fetchrow.return_value = {
|
||||
"mute_start": time(22, 0),
|
||||
"mute_end": time(6, 0),
|
||||
}
|
||||
fake_dt = datetime(2024, 1, 15, 12, 0, 0, tzinfo=timezone.utc)
|
||||
mock_dt = MagicMock()
|
||||
mock_dt.now.return_value = fake_dt
|
||||
with patch("notifier.datetime", mock_dt):
|
||||
result = await is_user_muted(mock_pool, 12345)
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestBufferForDigest:
|
||||
"""Test the buffer_for_digest async function."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_digest_mode_on_buffers(self, mock_pool):
|
||||
"""Should buffer notification when digest mode is on."""
|
||||
mock_pool.fetchrow.return_value = {"digest_mode": True}
|
||||
ad = {"title": "Test Ad", "price": 100.0, "keyword": "bike", "url": "https://example.com"}
|
||||
await buffer_for_digest(mock_pool, 12345, ad, "ad-uuid-1")
|
||||
mock_pool.execute.assert_called_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_digest_mode_off_does_not_buffer(self, mock_pool):
|
||||
"""Should not buffer notification when digest mode is off."""
|
||||
mock_pool.fetchrow.return_value = {"digest_mode": False}
|
||||
ad = {"title": "Test Ad", "price": 100.0, "keyword": "bike", "url": "https://example.com"}
|
||||
await buffer_for_digest(mock_pool, 12345, ad, "ad-uuid-1")
|
||||
mock_pool.execute.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_settings_does_not_buffer(self, mock_pool):
|
||||
"""Should not buffer when user has no settings."""
|
||||
mock_pool.fetchrow.return_value = None
|
||||
ad = {"title": "Test Ad", "price": 100.0, "keyword": "bike", "url": "https://example.com"}
|
||||
await buffer_for_digest(mock_pool, 12345, ad, "ad-uuid-1")
|
||||
mock_pool.execute.assert_not_called()
|
||||
|
||||
|
||||
class TestFormatText:
|
||||
"""Test the _format_text function."""
|
||||
|
||||
def test_basic_format(self, sample_fields):
|
||||
"""Should produce expected output with basic fields."""
|
||||
text = _format_text("🆕 New listing found!", sample_fields)
|
||||
assert "🆕 New listing found!" in text
|
||||
assert "Mountain Bike 2024" in text
|
||||
assert "250" in text
|
||||
assert "Vienna" in text
|
||||
assert "1010" in text
|
||||
|
||||
def test_format_with_no_price(self):
|
||||
"""Should handle missing price gracefully."""
|
||||
ad = {"title": "Free Item", "location": "Graz"}
|
||||
text = _format_text("Header", ad)
|
||||
assert "N/A" in text
|
||||
|
||||
def test_format_with_no_location(self):
|
||||
"""Should handle missing location gracefully."""
|
||||
ad = {"title": "Item", "price": 50.0}
|
||||
text = _format_text("Header", ad)
|
||||
assert "Item" in text
|
||||
assert "50" in text
|
||||
|
||||
def test_format_with_postcode(self, sample_fields):
|
||||
"""Should include postcode when present."""
|
||||
text = _format_text("Header", sample_fields)
|
||||
assert "1010" in text
|
||||
|
||||
def test_format_with_published_at(self, sample_fields):
|
||||
"""Should include published date when present."""
|
||||
text = _format_text("Header", sample_fields)
|
||||
assert "15.01.2024" in text
|
||||
|
||||
|
||||
class TestBuildKeyboard:
|
||||
"""Test the _build_keyboard function."""
|
||||
|
||||
def test_with_url(self, sample_fields):
|
||||
"""Should create keyboard with URL button when URL is present."""
|
||||
keyboard = _build_keyboard(sample_fields)
|
||||
assert keyboard is not None
|
||||
assert len(keyboard.inline_keyboard) == 1
|
||||
assert "View Ad" in keyboard.inline_keyboard[0][0].text
|
||||
|
||||
def test_without_url(self):
|
||||
"""Should create keyboard with no buttons when URL is missing."""
|
||||
keyboard = _build_keyboard({"title": "No URL Ad"})
|
||||
assert keyboard is None
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Tests for scraper module functions."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from scraper import extract_ad_fields
|
||||
|
||||
|
||||
class TestExtractAdFields:
|
||||
"""Test the extract_ad_fields function."""
|
||||
|
||||
def test_full_extraction(self, sample_ad_data):
|
||||
"""Should extract all fields from a complete ad."""
|
||||
fields = extract_ad_fields(sample_ad_data)
|
||||
|
||||
assert fields["wh_ad_id"] == "12345678"
|
||||
assert fields["title"] == "Mountain Bike 2024"
|
||||
assert fields["price"] == 250.0
|
||||
assert fields["location"] == "Vienna"
|
||||
assert fields["url"] == "https://www.willhaben.at/iad/mountain-bike-2024/12345678"
|
||||
assert fields["postcode"] == "1010"
|
||||
assert fields["main_image_url"] == "https://img.willhaben.at/img123.jpg"
|
||||
assert isinstance(fields["published_at"], datetime)
|
||||
assert isinstance(fields["modified_at"], datetime)
|
||||
|
||||
def test_published_at_is_utc(self, sample_ad_data):
|
||||
"""Published_at should be parsed as UTC."""
|
||||
fields = extract_ad_fields(sample_ad_data)
|
||||
assert fields["published_at"].tzinfo == timezone.utc
|
||||
assert fields["published_at"].hour == 10
|
||||
assert fields["published_at"].minute == 30
|
||||
|
||||
def test_modified_at_is_utc(self, sample_ad_data):
|
||||
"""Modified_at should be parsed as UTC."""
|
||||
fields = extract_ad_fields(sample_ad_data)
|
||||
assert fields["modified_at"].tzinfo == timezone.utc
|
||||
assert fields["modified_at"].hour == 12
|
||||
|
||||
def test_missing_price(self):
|
||||
"""Should handle ads without a price attribute."""
|
||||
ad_data = {
|
||||
"id": "999",
|
||||
"description": "Free item",
|
||||
"attributes": {
|
||||
"attribute": [
|
||||
{"name": "HEADING", "values": ["Free Item"]},
|
||||
{"name": "LOCATION", "values": ["Graz"]},
|
||||
]
|
||||
},
|
||||
}
|
||||
fields = extract_ad_fields(ad_data)
|
||||
assert fields["price"] is None
|
||||
assert fields["title"] == "Free Item"
|
||||
|
||||
def test_missing_heading_falls_back_to_description(self):
|
||||
"""Should fall back to description when HEADING is missing."""
|
||||
ad_data = {
|
||||
"id": "888",
|
||||
"description": "Fallback description",
|
||||
"attributes": {"attribute": []},
|
||||
}
|
||||
fields = extract_ad_fields(ad_data)
|
||||
assert fields["title"] == "Fallback description"
|
||||
|
||||
def test_missing_attributes(self):
|
||||
"""Should handle ads with no attributes at all."""
|
||||
ad_data = {"id": "777", "description": "Minimal ad"}
|
||||
fields = extract_ad_fields(ad_data)
|
||||
assert fields["wh_ad_id"] == "777"
|
||||
assert fields["title"] == "Minimal ad"
|
||||
assert fields["price"] is None
|
||||
assert fields["location"] is None
|
||||
|
||||
def test_price_with_comma_separator(self):
|
||||
"""Should parse prices with comma (comma is stripped, so '1.299,50' → 1.2995)."""
|
||||
ad_data = {
|
||||
"id": "666",
|
||||
"attributes": {
|
||||
"attribute": [
|
||||
{"name": "PRICE/AMOUNT", "values": ["1.299,50"]},
|
||||
]
|
||||
},
|
||||
}
|
||||
fields = extract_ad_fields(ad_data)
|
||||
# The parser strips commas only: "1.299,50" → "1.2995" → 1.2995
|
||||
assert fields["price"] == 1.2995
|
||||
|
||||
def test_missing_image(self):
|
||||
"""Should handle ads without images."""
|
||||
ad_data = {
|
||||
"id": "555",
|
||||
"attributes": {
|
||||
"attribute": [
|
||||
{"name": "HEADING", "values": ["No Image"]},
|
||||
]
|
||||
},
|
||||
}
|
||||
fields = extract_ad_fields(ad_data)
|
||||
assert fields["main_image_url"] is None
|
||||
|
||||
def test_empty_image_list(self):
|
||||
"""Should handle ads with empty image list."""
|
||||
ad_data = {
|
||||
"id": "444",
|
||||
"attributes": {"attribute": []},
|
||||
"advertImageList": {"advertImage": []},
|
||||
}
|
||||
fields = extract_ad_fields(ad_data)
|
||||
assert fields["main_image_url"] is None
|
||||
|
||||
def test_missing_seo_url(self):
|
||||
"""Should handle ads without SEO_URL."""
|
||||
ad_data = {
|
||||
"id": "333",
|
||||
"attributes": {
|
||||
"attribute": [
|
||||
{"name": "HEADING", "values": ["No SEO"]},
|
||||
]
|
||||
},
|
||||
}
|
||||
fields = extract_ad_fields(ad_data)
|
||||
assert fields["url"] is None
|
||||
|
||||
def test_invalid_price_format(self):
|
||||
"""Should handle invalid price values gracefully."""
|
||||
ad_data = {
|
||||
"id": "222",
|
||||
"attributes": {
|
||||
"attribute": [
|
||||
{"name": "PRICE/AMOUNT", "values": ["not a number"]},
|
||||
]
|
||||
},
|
||||
}
|
||||
fields = extract_ad_fields(ad_data)
|
||||
assert fields["price"] is None
|
||||
|
||||
def test_invalid_date_format(self):
|
||||
"""Should handle invalid date values gracefully."""
|
||||
ad_data = {
|
||||
"id": "111",
|
||||
"attributes": {
|
||||
"attribute": [
|
||||
{"name": "PUBLISHED_String", "values": ["not-a-date"]},
|
||||
{"name": "CHANGED_String", "values": ["also-not-a-date"]},
|
||||
]
|
||||
},
|
||||
}
|
||||
fields = extract_ad_fields(ad_data)
|
||||
assert fields["published_at"] is None
|
||||
assert fields["modified_at"] is None
|
||||
Reference in New Issue
Block a user