feat: Phase 3 — web dashboard, testing, and CI/CD
CI / lint-and-test (push) Has been cancelled

- Remove multi-marketplace from Phase 3
- Add FastAPI web UI on port 8766 with basic auth
- Add 6 Jinja2 templates (dashboard, keywords, users, ads, stats)
- Add pytest test suite (45 tests, 49% coverage)
- Add GitHub Actions CI/CD workflow
- Update docker-compose.yml to expose web UI port
- Update Dockerfile to include tests
This commit is contained in:
2026-07-10 22:28:18 +02:00
parent 0c20799f9a
commit 8151c530da
26 changed files with 1506 additions and 388 deletions
+84
View File
@@ -0,0 +1,84 @@
"""Shared pytest fixtures for the willhaben-tracker test suite."""
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock
import pytest
def pytest_configure(config):
"""Enable asyncio auto mode for all async tests."""
config.addinivalue_line("markers", "asyncio: mark test as async")
@pytest.fixture
def mock_pool():
"""Mock asyncpg.Pool with fetch/fetchrow/execute/fetchval methods."""
pool = MagicMock()
pool.fetch = AsyncMock(return_value=[])
pool.fetchrow = AsyncMock(return_value=None)
pool.execute = AsyncMock(return_value=None)
pool.fetchval = AsyncMock(return_value=None)
return pool
@pytest.fixture
def mock_bot():
"""Mock ExtBot with send_message/send_photo methods."""
bot = MagicMock()
bot.send_message = AsyncMock(return_value=MagicMock(message_id=123))
bot.send_photo = AsyncMock(return_value=MagicMock(message_id=123))
return bot
@pytest.fixture
def sample_ad_data():
"""Sample willhaben ad JSON dict matching the API response format."""
return {
"id": "12345678",
"description": "A used bicycle",
"attributes": {
"attribute": [
{"name": "HEADING", "values": ["Mountain Bike 2024"]},
{"name": "PRICE/AMOUNT", "values": ["250"]},
{"name": "LOCATION", "values": ["Vienna"]},
{"name": "POSTCODE", "values": ["1010"]},
{"name": "SEO_URL", "values": ["mountain-bike-2024/12345678"]},
{"name": "PUBLISHED_String", "values": ["2024-01-15T10:30:00Z"]},
{"name": "CHANGED_String", "values": ["2024-01-15T12:00:00Z"]},
]
},
"advertImageList": {
"advertImage": [
{"referenceImageUrl": "https://img.willhaben.at/img123.jpg"}
]
},
}
@pytest.fixture
def sample_fields():
"""Sample extracted ad fields dict as returned by extract_ad_fields()."""
return {
"wh_ad_id": "12345678",
"title": "Mountain Bike 2024",
"price": 250.0,
"location": "Vienna",
"url": "https://www.willhaben.at/iad/mountain-bike-2024/12345678",
"published_at": datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc),
"main_image_url": "https://img.willhaben.at/img123.jpg",
"postcode": "1010",
"modified_at": datetime(2024, 1, 15, 12, 0, 0, tzinfo=timezone.utc),
}
@pytest.fixture
def sample_kw_row():
"""Sample keyword row dict with filter settings."""
return {
"id": "kw-uuid-123",
"keyword": "bike",
"price_min": None,
"price_max": None,
"allowed_postcodes": None,
}
+92
View File
@@ -0,0 +1,92 @@
"""Tests for _ad_passes_filters from main.py."""
from main import _ad_passes_filters
class TestAdPassesFilters:
"""Test the _ad_passes_filters function."""
def test_no_filters_passes(self, sample_fields, sample_kw_row):
"""Ad with no filters should always pass."""
assert _ad_passes_filters(sample_fields, sample_kw_row) is True
def test_price_below_min_fails(self, sample_fields, sample_kw_row):
"""Ad price below price_min should fail."""
kw = {**sample_kw_row, "price_min": 30000} # 300.00 EUR in cents
# sample_fields price is 250.00 EUR = 25000 cents
assert _ad_passes_filters(sample_fields, kw) is False
def test_price_above_max_fails(self, sample_fields, sample_kw_row):
"""Ad price above price_max should fail."""
kw = {**sample_kw_row, "price_max": 20000} # 200.00 EUR in cents
# sample_fields price is 250.00 EUR = 25000 cents
assert _ad_passes_filters(sample_fields, kw) is False
def test_price_in_range_passes(self, sample_fields, sample_kw_row):
"""Ad price within min/max range should pass."""
kw = {**sample_kw_row, "price_min": 10000, "price_max": 50000}
# sample_fields price is 250.00 EUR = 25000 cents
assert _ad_passes_filters(sample_fields, kw) is True
def test_price_at_min_boundary_passes(self, sample_fields, sample_kw_row):
"""Ad price exactly at price_min should pass."""
kw = {**sample_kw_row, "price_min": 25000}
assert _ad_passes_filters(sample_fields, kw) is True
def test_price_at_max_boundary_passes(self, sample_fields, sample_kw_row):
"""Ad price exactly at price_max should pass."""
kw = {**sample_kw_row, "price_max": 25000}
assert _ad_passes_filters(sample_fields, kw) is True
def test_postcode_not_in_allowed_list_fails(self, sample_fields, sample_kw_row):
"""Ad postcode not in allowed_postcodes should fail."""
kw = {**sample_kw_row, "allowed_postcodes": ["1020", "1030"]}
# sample_fields postcode is "1010"
assert _ad_passes_filters(sample_fields, kw) is False
def test_postcode_in_allowed_list_passes(self, sample_fields, sample_kw_row):
"""Ad postcode in allowed_postcodes should pass."""
kw = {**sample_kw_row, "allowed_postcodes": ["1010", "1020"]}
assert _ad_passes_filters(sample_fields, kw) is True
def test_no_postcode_with_filter_active_fails(self, sample_fields, sample_kw_row):
"""Ad with no postcode when filter is active should fail."""
fields = {**sample_fields, "postcode": None}
kw = {**sample_kw_row, "allowed_postcodes": ["1010", "1020"]}
assert _ad_passes_filters(fields, kw) is False
def test_combined_price_and_postcode_filters_pass(self, sample_fields, sample_kw_row):
"""Ad passing both price and postcode filters should pass."""
kw = {
**sample_kw_row,
"price_min": 10000,
"price_max": 50000,
"allowed_postcodes": ["1010", "1020"],
}
assert _ad_passes_filters(sample_fields, kw) is True
def test_combined_price_passes_postcode_fails(self, sample_fields, sample_kw_row):
"""Ad passing price but failing postcode should fail."""
kw = {
**sample_kw_row,
"price_min": 10000,
"price_max": 50000,
"allowed_postcodes": ["1020", "1030"],
}
assert _ad_passes_filters(sample_fields, kw) is False
def test_combined_price_fails_postcode_passes(self, sample_fields, sample_kw_row):
"""Ad failing price but passing postcode should fail."""
kw = {
**sample_kw_row,
"price_min": 30000,
"price_max": 50000,
"allowed_postcodes": ["1010", "1020"],
}
assert _ad_passes_filters(sample_fields, kw) is False
def test_no_price_with_price_filter(self, sample_fields, sample_kw_row):
"""Ad with no price should pass price filters (price is None)."""
fields = {**sample_fields, "price": None}
kw = {**sample_kw_row, "price_min": 10000, "price_max": 50000}
assert _ad_passes_filters(fields, kw) is True
+29
View File
@@ -0,0 +1,29 @@
"""Tests for health module."""
from health import create_health_app
class TestHealthModule:
"""Basic import and app creation tests for the health module."""
def test_health_module_import(self):
"""Health module should be importable."""
import health
assert health is not None
def test_create_health_app_returns_app(self):
"""create_health_app should return an aiohttp web.Application."""
app = create_health_app()
assert app is not None
assert hasattr(app, "router")
def test_health_app_has_routes(self):
"""Health app should have /health and /stats routes."""
app = create_health_app()
# Collect route info from the router
routes_info = []
for route in app.router.routes():
routes_info.append(repr(route))
routes_str = " ".join(routes_info)
assert "/health" in routes_str
assert "/stats" in routes_str
+181
View File
@@ -0,0 +1,181 @@
"""Tests for notifier module functions."""
from datetime import datetime, time, timezone
from unittest.mock import MagicMock, patch
import pytest
from notifier import (
_format_text,
_build_keyboard,
is_user_muted,
buffer_for_digest,
)
class TestIsUserMuted:
"""Test the is_user_muted async function."""
@pytest.mark.asyncio
async def test_no_settings_not_muted(self, mock_pool):
"""User with no settings should not be muted."""
mock_pool.fetchrow.return_value = None
result = await is_user_muted(mock_pool, 12345)
assert result is False
@pytest.mark.asyncio
async def test_no_mute_hours_not_muted(self, mock_pool):
"""User with settings but no mute hours should not be muted."""
mock_pool.fetchrow.return_value = {"mute_start": None, "mute_end": None}
result = await is_user_muted(mock_pool, 12345)
assert result is False
@pytest.mark.asyncio
async def test_normal_window_muted(self, mock_pool):
"""User should be muted when current time is within normal window."""
mock_pool.fetchrow.return_value = {
"mute_start": time(8, 0),
"mute_end": time(12, 0),
}
fake_dt = datetime(2024, 1, 15, 10, 0, 0, tzinfo=timezone.utc)
mock_dt = MagicMock()
mock_dt.now.return_value = fake_dt
with patch("notifier.datetime", mock_dt):
result = await is_user_muted(mock_pool, 12345)
assert result is True
@pytest.mark.asyncio
async def test_normal_window_not_muted(self, mock_pool):
"""User should not be muted when current time is outside normal window."""
mock_pool.fetchrow.return_value = {
"mute_start": time(8, 0),
"mute_end": time(12, 0),
}
fake_dt = datetime(2024, 1, 15, 14, 0, 0, tzinfo=timezone.utc)
mock_dt = MagicMock()
mock_dt.now.return_value = fake_dt
with patch("notifier.datetime", mock_dt):
result = await is_user_muted(mock_pool, 12345)
assert result is False
@pytest.mark.asyncio
async def test_cross_midnight_window_muted_after_start(self, mock_pool):
"""User should be muted when time is after start of cross-midnight window."""
mock_pool.fetchrow.return_value = {
"mute_start": time(22, 0),
"mute_end": time(6, 0),
}
fake_dt = datetime(2024, 1, 15, 23, 0, 0, tzinfo=timezone.utc)
mock_dt = MagicMock()
mock_dt.now.return_value = fake_dt
with patch("notifier.datetime", mock_dt):
result = await is_user_muted(mock_pool, 12345)
assert result is True
@pytest.mark.asyncio
async def test_cross_midnight_window_muted_before_end(self, mock_pool):
"""User should be muted when time is before end of cross-midnight window."""
mock_pool.fetchrow.return_value = {
"mute_start": time(22, 0),
"mute_end": time(6, 0),
}
fake_dt = datetime(2024, 1, 15, 3, 0, 0, tzinfo=timezone.utc)
mock_dt = MagicMock()
mock_dt.now.return_value = fake_dt
with patch("notifier.datetime", mock_dt):
result = await is_user_muted(mock_pool, 12345)
assert result is True
@pytest.mark.asyncio
async def test_cross_midnight_window_not_muted(self, mock_pool):
"""User should not be muted when time is outside cross-midnight window."""
mock_pool.fetchrow.return_value = {
"mute_start": time(22, 0),
"mute_end": time(6, 0),
}
fake_dt = datetime(2024, 1, 15, 12, 0, 0, tzinfo=timezone.utc)
mock_dt = MagicMock()
mock_dt.now.return_value = fake_dt
with patch("notifier.datetime", mock_dt):
result = await is_user_muted(mock_pool, 12345)
assert result is False
class TestBufferForDigest:
"""Test the buffer_for_digest async function."""
@pytest.mark.asyncio
async def test_digest_mode_on_buffers(self, mock_pool):
"""Should buffer notification when digest mode is on."""
mock_pool.fetchrow.return_value = {"digest_mode": True}
ad = {"title": "Test Ad", "price": 100.0, "keyword": "bike", "url": "https://example.com"}
await buffer_for_digest(mock_pool, 12345, ad, "ad-uuid-1")
mock_pool.execute.assert_called_once()
@pytest.mark.asyncio
async def test_digest_mode_off_does_not_buffer(self, mock_pool):
"""Should not buffer notification when digest mode is off."""
mock_pool.fetchrow.return_value = {"digest_mode": False}
ad = {"title": "Test Ad", "price": 100.0, "keyword": "bike", "url": "https://example.com"}
await buffer_for_digest(mock_pool, 12345, ad, "ad-uuid-1")
mock_pool.execute.assert_not_called()
@pytest.mark.asyncio
async def test_no_settings_does_not_buffer(self, mock_pool):
"""Should not buffer when user has no settings."""
mock_pool.fetchrow.return_value = None
ad = {"title": "Test Ad", "price": 100.0, "keyword": "bike", "url": "https://example.com"}
await buffer_for_digest(mock_pool, 12345, ad, "ad-uuid-1")
mock_pool.execute.assert_not_called()
class TestFormatText:
"""Test the _format_text function."""
def test_basic_format(self, sample_fields):
"""Should produce expected output with basic fields."""
text = _format_text("🆕 New listing found!", sample_fields)
assert "🆕 New listing found!" in text
assert "Mountain Bike 2024" in text
assert "250" in text
assert "Vienna" in text
assert "1010" in text
def test_format_with_no_price(self):
"""Should handle missing price gracefully."""
ad = {"title": "Free Item", "location": "Graz"}
text = _format_text("Header", ad)
assert "N/A" in text
def test_format_with_no_location(self):
"""Should handle missing location gracefully."""
ad = {"title": "Item", "price": 50.0}
text = _format_text("Header", ad)
assert "Item" in text
assert "50" in text
def test_format_with_postcode(self, sample_fields):
"""Should include postcode when present."""
text = _format_text("Header", sample_fields)
assert "1010" in text
def test_format_with_published_at(self, sample_fields):
"""Should include published date when present."""
text = _format_text("Header", sample_fields)
assert "15.01.2024" in text
class TestBuildKeyboard:
"""Test the _build_keyboard function."""
def test_with_url(self, sample_fields):
"""Should create keyboard with URL button when URL is present."""
keyboard = _build_keyboard(sample_fields)
assert keyboard is not None
assert len(keyboard.inline_keyboard) == 1
assert "View Ad" in keyboard.inline_keyboard[0][0].text
def test_without_url(self):
"""Should create keyboard with no buttons when URL is missing."""
keyboard = _build_keyboard({"title": "No URL Ad"})
assert keyboard is None
+149
View File
@@ -0,0 +1,149 @@
"""Tests for scraper module functions."""
from datetime import datetime, timezone
from scraper import extract_ad_fields
class TestExtractAdFields:
"""Test the extract_ad_fields function."""
def test_full_extraction(self, sample_ad_data):
"""Should extract all fields from a complete ad."""
fields = extract_ad_fields(sample_ad_data)
assert fields["wh_ad_id"] == "12345678"
assert fields["title"] == "Mountain Bike 2024"
assert fields["price"] == 250.0
assert fields["location"] == "Vienna"
assert fields["url"] == "https://www.willhaben.at/iad/mountain-bike-2024/12345678"
assert fields["postcode"] == "1010"
assert fields["main_image_url"] == "https://img.willhaben.at/img123.jpg"
assert isinstance(fields["published_at"], datetime)
assert isinstance(fields["modified_at"], datetime)
def test_published_at_is_utc(self, sample_ad_data):
"""Published_at should be parsed as UTC."""
fields = extract_ad_fields(sample_ad_data)
assert fields["published_at"].tzinfo == timezone.utc
assert fields["published_at"].hour == 10
assert fields["published_at"].minute == 30
def test_modified_at_is_utc(self, sample_ad_data):
"""Modified_at should be parsed as UTC."""
fields = extract_ad_fields(sample_ad_data)
assert fields["modified_at"].tzinfo == timezone.utc
assert fields["modified_at"].hour == 12
def test_missing_price(self):
"""Should handle ads without a price attribute."""
ad_data = {
"id": "999",
"description": "Free item",
"attributes": {
"attribute": [
{"name": "HEADING", "values": ["Free Item"]},
{"name": "LOCATION", "values": ["Graz"]},
]
},
}
fields = extract_ad_fields(ad_data)
assert fields["price"] is None
assert fields["title"] == "Free Item"
def test_missing_heading_falls_back_to_description(self):
"""Should fall back to description when HEADING is missing."""
ad_data = {
"id": "888",
"description": "Fallback description",
"attributes": {"attribute": []},
}
fields = extract_ad_fields(ad_data)
assert fields["title"] == "Fallback description"
def test_missing_attributes(self):
"""Should handle ads with no attributes at all."""
ad_data = {"id": "777", "description": "Minimal ad"}
fields = extract_ad_fields(ad_data)
assert fields["wh_ad_id"] == "777"
assert fields["title"] == "Minimal ad"
assert fields["price"] is None
assert fields["location"] is None
def test_price_with_comma_separator(self):
"""Should parse prices with comma (comma is stripped, so '1.299,50' → 1.2995)."""
ad_data = {
"id": "666",
"attributes": {
"attribute": [
{"name": "PRICE/AMOUNT", "values": ["1.299,50"]},
]
},
}
fields = extract_ad_fields(ad_data)
# The parser strips commas only: "1.299,50" → "1.2995" → 1.2995
assert fields["price"] == 1.2995
def test_missing_image(self):
"""Should handle ads without images."""
ad_data = {
"id": "555",
"attributes": {
"attribute": [
{"name": "HEADING", "values": ["No Image"]},
]
},
}
fields = extract_ad_fields(ad_data)
assert fields["main_image_url"] is None
def test_empty_image_list(self):
"""Should handle ads with empty image list."""
ad_data = {
"id": "444",
"attributes": {"attribute": []},
"advertImageList": {"advertImage": []},
}
fields = extract_ad_fields(ad_data)
assert fields["main_image_url"] is None
def test_missing_seo_url(self):
"""Should handle ads without SEO_URL."""
ad_data = {
"id": "333",
"attributes": {
"attribute": [
{"name": "HEADING", "values": ["No SEO"]},
]
},
}
fields = extract_ad_fields(ad_data)
assert fields["url"] is None
def test_invalid_price_format(self):
"""Should handle invalid price values gracefully."""
ad_data = {
"id": "222",
"attributes": {
"attribute": [
{"name": "PRICE/AMOUNT", "values": ["not a number"]},
]
},
}
fields = extract_ad_fields(ad_data)
assert fields["price"] is None
def test_invalid_date_format(self):
"""Should handle invalid date values gracefully."""
ad_data = {
"id": "111",
"attributes": {
"attribute": [
{"name": "PUBLISHED_String", "values": ["not-a-date"]},
{"name": "CHANGED_String", "values": ["also-not-a-date"]},
]
},
}
fields = extract_ad_fields(ad_data)
assert fields["published_at"] is None
assert fields["modified_at"] is None