feat: Phase 1 — reliability & completeness improvements
- httpx singleton: replace per-call AsyncClient with module-level singleton using keepalive pool (configurable via env vars). Close client on shutdown. - Scraper pagination: multi-page fetching (default 2 pages/60 ads, configurable via SCRAPE_MAX_PAGES). Cursor-based early-stop to skip stale pages. Tracks ads_cursor per keyword for cross-cycle deduplication. - Notification retry queue: persistent notification_queue table with exponential backoff (max 5 attempts → dead). Failed Telegram notifications are enqueued instead of silently dropped. Queue processed at start of each scheduler cycle (50 items max limit). /stats endpoint reports pending/dead counts.
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
-- Add ads_cursor column to keywords for pagination cursor tracking
|
||||
|
||||
ALTER TABLE keywords ADD COLUMN IF NOT EXISTS ads_cursor timestamptz;
|
||||
COMMENT ON COLUMN keywords.ads_cursor IS
|
||||
'Timestamp of oldest ad processed in last cycle, for pagination cursor';
|
||||
@@ -0,0 +1,26 @@
|
||||
-- Notification retry queue table
|
||||
|
||||
CREATE TYPE notification_type AS ENUM ('new', 'drop');
|
||||
CREATE TYPE notification_status AS ENUM ('pending', 'sent', 'failed', 'dead');
|
||||
|
||||
CREATE TABLE IF NOT EXISTS notification_queue (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
ad_id uuid NOT NULL REFERENCES ads(id) ON DELETE SET NULL,
|
||||
telegram_id text NOT NULL,
|
||||
message_text text NOT NULL,
|
||||
type notification_type NOT NULL,
|
||||
attempts int NOT NULL DEFAULT 0,
|
||||
max_attempts int NOT NULL DEFAULT 5,
|
||||
last_error text,
|
||||
status notification_status NOT NULL DEFAULT 'pending',
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Index for efficient queue polling
|
||||
CREATE INDEX IF NOT EXISTS idx_notif_queue_poll
|
||||
ON notification_queue(status, attempts, updated_at)
|
||||
WHERE status IN ('pending', 'failed');
|
||||
|
||||
COMMENT ON TABLE notification_queue IS
|
||||
'Persistent retry queue for failed Telegram notifications';
|
||||
Reference in New Issue
Block a user