From 98c71d4232447331b028888555a140d5a7bb38b4 Mon Sep 17 00:00:00 2001 From: Jose Lago Date: Sat, 4 Jul 2026 10:33:27 +0200 Subject: [PATCH 1/3] Migrate from local Supabase stack to external Supabase instance - Reduce docker-compose from 6 services to 1 (worker only) - Connect via supavisor pooled port (6543) on dedicated Supabase - Add willhaben_tracker schema with search_path in asyncpg pool init - Create consolidated migration SQL for Supabase SQL Editor - Update .env.example with new connection pattern --- .env.example | 9 +- docker-compose.yml | 95 +--------------- supabase/migrations/supabase-migration.sql | 121 +++++++++++++++++++++ worker/src/db.py | 1 + 4 files changed, 128 insertions(+), 98 deletions(-) create mode 100644 supabase/migrations/supabase-migration.sql diff --git a/.env.example b/.env.example index 4a0d619..6ccf086 100644 --- a/.env.example +++ b/.env.example @@ -1,14 +1,13 @@ # Telegram Bot Token (from @BotFather) TELEGRAM_BOT_TOKEN=your-bot-token-here -# PostgreSQL Credentials +# Supabase PostgreSQL Connection (pooled via supavisor on port 6543) +POSTGRES_HOST=192.168.178.3 +POSTGRES_PORT=6543 POSTGRES_USER=postgres -POSTGRES_PASSWORD=change-me-strong-password +POSTGRES_PASSWORD=your-supabase-db-password POSTGRES_DB=postgres -# PostgREST JWT Secret (random string for signing tokens) -JWT_SECRET=your-super-secret-jwt-key-change-in-production - # Worker Configuration DEFAULT_INTERVAL_MINUTES=60 ADMIN_TELEGRAM_IDS=123456789 # Comma-separated Telegram user IDs with admin access diff --git a/docker-compose.yml b/docker-compose.yml index 1902e36..dafdb2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,103 +1,12 @@ -version: "3.9" - services: - db: - image: postgres:15-alpine - restart: unless-stopped - environment: - POSTGRES_USER: ${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: ${POSTGRES_DB:-postgres} - POSTGRES_HOST_AUTH_METHOD: trust - volumes: - - ./data/db:/var/lib/postgresql/data - - ./supabase/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro - - ./supabase/migrations/00-run-init.sh:/docker-entrypoint-initdb.d/00-run-init.sh:ro - - ./supabase/migrations/01-schema.sql:/docker-entrypoint-initdb.d/01-schema.sql:ro - - ./supabase/migrations/post-boot.sql:/docker-entrypoint-initdb.d/post-boot.sql:ro - command: > - postgres - -c hba_file=/etc/postgresql/pg_hba.conf - -c wal_level=logical - -c max_wal_senders=0 - -c max_replication_slots=0 - -c idle_in_transaction_session_timeout=1min - ports: - - "55632:5432" - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-postgres}"] - interval: 5s - timeout: 5s - retries: 10 - - rest: - image: postgrest/postgrest:v12.2.0 - restart: unless-stopped - environment: - PGRST_DB_URI: postgres://authenticator@db:5432/${POSTGRES_DB:-postgres} - PGRST_DB_SCHEMAS: public - PGRST_DB_ANON_ROLE: authenticator - PGRST_JWT_SECRET: ${JWT_SECRET:-your-super-secret-jwt-key-change-in-production} - PGRST_DB_EXTRA_SEARCH_PATH: public - depends_on: - db: - condition: service_healthy - - kong: - image: kong:2.8.1 - restart: unless-stopped - environment: - KONG_DATABASE: "off" - KONG_DECLARATIVE_CONFIG: /etc/kong/kong.yml - KONG_PLUGINS: request-transformer,cors - KONG_PROXY_LISTEN: "0.0.0.0:8000" - KONG_NGINX_WORKER_PROCESSES: 1 - volumes: - - ./supabase/kong.yml:/etc/kong/kong.yml:ro - ports: - - "55621:8000" - depends_on: - rest: - condition: service_started - - studio: - image: supabase/studio - restart: unless-stopped - environment: - STUDIO_PG_META_URL: http://meta:8080 - DEFAULT_ORGANIZATION_NAME: Local - DEFAULT_PROJECT_NAME: willhaben-tracker - ports: - - "55630:3000" - depends_on: - meta: - condition: service_started - - meta: - image: supabase/postgres-meta:v0.84.2 - restart: unless-stopped - environment: - PG_META_PORT: 8080 - PG_META_DB_HOST: db - PG_META_DB_PORT: 5432 - PG_META_DB_NAME: ${POSTGRES_DB:-postgres} - PG_META_DB_USER: ${POSTGRES_USER:-postgres} - PG_META_DB_PASSWORD: ${POSTGRES_PASSWORD} - depends_on: - db: - condition: service_healthy - worker: build: ./worker restart: unless-stopped environment: TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN} - POSTGRES_HOST: db - POSTGRES_PORT: "5432" + POSTGRES_HOST: ${POSTGRES_HOST:-192.168.178.3} + POSTGRES_PORT: "6543" POSTGRES_USER: ${POSTGRES_USER:-postgres} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_DB: ${POSTGRES_DB:-postgres} DEFAULT_INTERVAL_MINUTES: ${DEFAULT_INTERVAL_MINUTES:-60} - depends_on: - db: - condition: service_healthy diff --git a/supabase/migrations/supabase-migration.sql b/supabase/migrations/supabase-migration.sql new file mode 100644 index 0000000..3447414 --- /dev/null +++ b/supabase/migrations/supabase-migration.sql @@ -0,0 +1,121 @@ +CREATE SCHEMA IF NOT EXISTS willhaben_tracker; +SET search_path TO willhaben_tracker; + +-- ----------------------------------------------------------- +-- users (whitelisted Telegram users) +-- ----------------------------------------------------------- +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + telegram_id bigint UNIQUE NOT NULL, + username text, + first_name text, + is_admin boolean NOT NULL DEFAULT false, + is_active boolean NOT NULL DEFAULT true, + created_at timestamptz NOT NULL DEFAULT now() +); + +-- ----------------------------------------------------------- +-- keywords (global search keywords — deduplicated across users) +-- ----------------------------------------------------------- +CREATE TABLE IF NOT EXISTS keywords ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + keyword text NOT NULL, + interval_minutes int NOT NULL DEFAULT 60, + is_active boolean NOT NULL DEFAULT true, + initial_loaded boolean NOT NULL DEFAULT false, + last_scraped_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_keywords_unique_lower + ON keywords(LOWER(keyword)); + +-- ----------------------------------------------------------- +-- keyword_subscriptions (many-to-many: user <-> keyword) +-- ----------------------------------------------------------- +CREATE TABLE IF NOT EXISTS keyword_subscriptions ( + keyword_id uuid REFERENCES keywords(id) ON DELETE CASCADE NOT NULL, + user_id uuid REFERENCES users(id) ON DELETE CASCADE NOT NULL, + PRIMARY KEY (keyword_id, user_id), + created_at timestamptz NOT NULL DEFAULT now() +); + +-- ----------------------------------------------------------- +-- ads (raw ad snapshots, globally deduplicated by wh_ad_id) +-- ----------------------------------------------------------- +CREATE TABLE IF NOT EXISTS ads ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + wh_ad_id text UNIQUE NOT NULL, + raw_json jsonb NOT NULL, + title text NOT NULL, + price numeric, + location text, + url text, + published_at timestamptz, + first_seen_at timestamptz NOT NULL DEFAULT now(), + main_image_url text, + postcode text, + modified_at timestamptz +); + +-- ----------------------------------------------------------- +-- price_history (track price changes per ad) +-- ----------------------------------------------------------- +CREATE TABLE IF NOT EXISTS price_history ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + ad_id uuid REFERENCES ads(id) ON DELETE CASCADE NOT NULL, + old_price numeric NOT NULL, + new_price numeric NOT NULL, + changed_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (ad_id, old_price, new_price) +); + +-- ----------------------------------------------------------- +-- notifications (audit log of sent Telegram messages) +-- ----------------------------------------------------------- +CREATE TABLE IF NOT EXISTS notifications ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + user_id uuid REFERENCES users(id) ON DELETE CASCADE NOT NULL, + ad_id uuid REFERENCES ads(id) ON DELETE SET NULL, + message_id int, + sent_at timestamptz NOT NULL DEFAULT now() +); + +-- ----------------------------------------------------------- +-- scrape_logs (worker health / debugging) +-- ----------------------------------------------------------- +CREATE TABLE IF NOT EXISTS scrape_logs ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + keyword_id uuid REFERENCES keywords(id) ON DELETE CASCADE NOT NULL, + status text NOT NULL CHECK (status IN ('success', 'error', 'rate_limited')), + ads_found int NOT NULL DEFAULT 0, + new_ads int NOT NULL DEFAULT 0, + error_message text, + scraped_at timestamptz NOT NULL DEFAULT now() +); + +-- ============================================================ +-- Indexes +-- ============================================================ + +CREATE INDEX IF NOT EXISTS idx_keywords_active_scraped + ON keywords(is_active, last_scraped_at) WHERE is_active = true; + +CREATE INDEX IF NOT EXISTS idx_keyword_subscriptions_user_id + ON keyword_subscriptions(user_id); + +CREATE INDEX IF NOT EXISTS idx_price_history_ad_id + ON price_history(ad_id); + +CREATE INDEX IF NOT EXISTS idx_notifications_user_sent + ON notifications(user_id, sent_at DESC); + +CREATE INDEX IF NOT EXISTS idx_scrape_logs_keyword_at + ON scrape_logs(keyword_id, scraped_at DESC); + +-- ----------------------------------------------------------- +-- Seed: initial admin user +-- ----------------------------------------------------------- +INSERT INTO users (telegram_id, username, first_name, is_admin, is_active) +VALUES (298181113, NULL, 'Admin', true, true) +ON CONFLICT (telegram_id) DO NOTHING; diff --git a/worker/src/db.py b/worker/src/db.py index b30af52..b73ed63 100644 --- a/worker/src/db.py +++ b/worker/src/db.py @@ -18,6 +18,7 @@ async def get_pool() -> asyncpg.Pool: database=os.getenv("POSTGRES_DB", "postgres"), min_size=2, max_size=10, + init="SET search_path TO willhaben_tracker" ) logger.info("Database pool initialized") return _pool From 753174b9468efe9fce630a6ae2cd809a942bb1bd Mon Sep 17 00:00:00 2001 From: Jose Lago Date: Sat, 4 Jul 2026 11:21:01 +0200 Subject: [PATCH 2/3] switch to direct DB connection on supabase_default network, bypass supavisor --- .env.example | 6 +++--- docker-compose.yml | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.env.example b/.env.example index 6ccf086..cb05fd4 100644 --- a/.env.example +++ b/.env.example @@ -1,9 +1,9 @@ # Telegram Bot Token (from @BotFather) TELEGRAM_BOT_TOKEN=your-bot-token-here -# Supabase PostgreSQL Connection (pooled via supavisor on port 6543) -POSTGRES_HOST=192.168.178.3 -POSTGRES_PORT=6543 +# Direct Supabase Postgres connection (join supabase_default network, connect to db:5432) +POSTGRES_HOST=db +POSTGRES_PORT=5432 POSTGRES_USER=postgres POSTGRES_PASSWORD=your-supabase-db-password POSTGRES_DB=postgres diff --git a/docker-compose.yml b/docker-compose.yml index dafdb2d..fd06548 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,11 +2,11 @@ services: worker: build: ./worker restart: unless-stopped - environment: - TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN} - POSTGRES_HOST: ${POSTGRES_HOST:-192.168.178.3} - POSTGRES_PORT: "6543" - POSTGRES_USER: ${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: ${POSTGRES_DB:-postgres} - DEFAULT_INTERVAL_MINUTES: ${DEFAULT_INTERVAL_MINUTES:-60} + env_file: + - .env + networks: + - supabase_default + +networks: + supabase_default: + external: true From fa551556e69adc3866228cc24b9756fa39c6a7ab Mon Sep 17 00:00:00 2001 From: Jose Lago Date: Sat, 4 Jul 2026 11:23:15 +0200 Subject: [PATCH 3/3] fix: use async callable for asyncpg pool init instead of string --- worker/src/db.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/worker/src/db.py b/worker/src/db.py index b73ed63..a6fffe4 100644 --- a/worker/src/db.py +++ b/worker/src/db.py @@ -7,6 +7,10 @@ logger = logging.getLogger(__name__) _pool: asyncpg.Pool | None = None +async def _init_connection(conn: asyncpg.Connection) -> None: + await conn.execute("SET search_path TO willhaben_tracker") + + async def get_pool() -> asyncpg.Pool: global _pool if _pool is None: @@ -18,7 +22,7 @@ async def get_pool() -> asyncpg.Pool: database=os.getenv("POSTGRES_DB", "postgres"), min_size=2, max_size=10, - init="SET search_path TO willhaben_tracker" + init=_init_connection ) logger.info("Database pool initialized") return _pool