feat: Phase 1 — reliability & completeness improvements

- httpx singleton: replace per-call AsyncClient with module-level singleton
  using keepalive pool (configurable via env vars). Close client on shutdown.

- Scraper pagination: multi-page fetching (default 2 pages/60 ads, configurable
  via SCRAPE_MAX_PAGES). Cursor-based early-stop to skip stale pages. Tracks
  ads_cursor per keyword for cross-cycle deduplication.

- Notification retry queue: persistent notification_queue table with exponential
  backoff (max 5 attempts → dead). Failed Telegram notifications are enqueued
  instead of silently dropped. Queue processed at start of each scheduler cycle
  (50 items max limit). /stats endpoint reports pending/dead counts.
This commit is contained in:
2026-07-05 22:22:28 +02:00
parent c9dd9ba076
commit 3e7e5d0b32
10 changed files with 347 additions and 56 deletions
@@ -0,0 +1,5 @@
-- Add ads_cursor column to keywords for pagination cursor tracking
ALTER TABLE keywords ADD COLUMN IF NOT EXISTS ads_cursor timestamptz;
COMMENT ON COLUMN keywords.ads_cursor IS
'Timestamp of oldest ad processed in last cycle, for pagination cursor';
@@ -0,0 +1,26 @@
-- Notification retry queue table
CREATE TYPE notification_type AS ENUM ('new', 'drop');
CREATE TYPE notification_status AS ENUM ('pending', 'sent', 'failed', 'dead');
CREATE TABLE IF NOT EXISTS notification_queue (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
ad_id uuid NOT NULL REFERENCES ads(id) ON DELETE SET NULL,
telegram_id text NOT NULL,
message_text text NOT NULL,
type notification_type NOT NULL,
attempts int NOT NULL DEFAULT 0,
max_attempts int NOT NULL DEFAULT 5,
last_error text,
status notification_status NOT NULL DEFAULT 'pending',
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
-- Index for efficient queue polling
CREATE INDEX IF NOT EXISTS idx_notif_queue_poll
ON notification_queue(status, attempts, updated_at)
WHERE status IN ('pending', 'failed');
COMMENT ON TABLE notification_queue IS
'Persistent retry queue for failed Telegram notifications';