fix: notification retry queue - resolve hour-long delays
CI / lint-and-test (push) Has been cancelled
CI / lint-and-test (push) Has been cancelled
- Pending notifications now send immediately (no backoff) - Backoff uses created_at instead of updated_at (fixed timeline) - Increase batch size from 50 to 200 for faster queue drain - Add queue depth logging and skip debug logging
This commit is contained in:
+18
-5
@@ -59,20 +59,33 @@ async def process_notification_queue(pool: asyncpg.Pool, bot: ExtBot) -> int:
|
|||||||
"""Process pending notifications from the retry queue."""
|
"""Process pending notifications from the retry queue."""
|
||||||
rows = await pool.fetch("""
|
rows = await pool.fetch("""
|
||||||
SELECT id, ad_id, telegram_id, message_text, type, attempts,
|
SELECT id, ad_id, telegram_id, message_text, type, attempts,
|
||||||
max_attempts, last_error, updated_at
|
max_attempts, last_error, updated_at, created_at
|
||||||
FROM notification_queue
|
FROM notification_queue
|
||||||
WHERE status IN ('pending', 'failed')
|
WHERE status IN ('pending', 'failed')
|
||||||
ORDER BY attempts ASC, updated_at ASC
|
ORDER BY attempts ASC, created_at ASC
|
||||||
LIMIT 50
|
LIMIT 200
|
||||||
""")
|
""")
|
||||||
|
|
||||||
|
# Log queue depth
|
||||||
|
queue_stats = await pool.fetchrow("SELECT count(*) FROM notification_queue WHERE status IN ('pending', 'failed')")
|
||||||
|
total_queued = queue_stats["count"]
|
||||||
|
logger.info("Notification queue: %d total, processing up to %d", total_queued, len(rows))
|
||||||
|
|
||||||
processed = 0
|
processed = 0
|
||||||
|
|
||||||
for row in rows:
|
for row in rows:
|
||||||
backoff_min = min(2 ** row["attempts"], 60)
|
# Pending (never tried) — send immediately
|
||||||
retry_after = row["updated_at"] + timedelta(minutes=backoff_min)
|
if row["status"] == "pending":
|
||||||
|
backoff_min = 0
|
||||||
|
else:
|
||||||
|
# Failed — exponential backoff from created_at (fixed timeline)
|
||||||
|
backoff_min = min(2 ** row["attempts"], 60)
|
||||||
|
|
||||||
|
retry_after = row["created_at"] + timedelta(minutes=backoff_min)
|
||||||
|
|
||||||
if datetime.now(tz=timezone.utc) < retry_after:
|
if datetime.now(tz=timezone.utc) < retry_after:
|
||||||
|
logger.debug("Skipping notification %s — retry after %s (now %s)",
|
||||||
|
row["id"], retry_after, datetime.now(tz=timezone.utc))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|||||||
Reference in New Issue
Block a user