diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..4a0d619 --- /dev/null +++ b/.env.example @@ -0,0 +1,14 @@ +# Telegram Bot Token (from @BotFather) +TELEGRAM_BOT_TOKEN=your-bot-token-here + +# PostgreSQL Credentials +POSTGRES_USER=postgres +POSTGRES_PASSWORD=change-me-strong-password +POSTGRES_DB=postgres + +# PostgREST JWT Secret (random string for signing tokens) +JWT_SECRET=your-super-secret-jwt-key-change-in-production + +# Worker Configuration +DEFAULT_INTERVAL_MINUTES=60 +ADMIN_TELEGRAM_IDS=123456789 # Comma-separated Telegram user IDs with admin access diff --git a/README.md b/README.md index ee5acf8..40e1af2 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,53 @@ Telegram bot + scraper for willhaben.at classified ads. Self-hosted on Unraid vi ## Stack -- **Postgres 15** with Supabase (PostgREST, Kong) +- **Postgres 15** with logical WAL, init scripts run alphabetically on first boot +- **PostgREST** — auto-generated REST API over Postgres `public` schema +- **Kong** — reverse proxy routing `/rest/v1/` to PostgREST +- **Supabase Studio** — database browser and management UI - **Python worker** — Telegram long polling + scrape scheduler -- **Docker Compose** deployed via Portainer at `/mnt/user/appdata/willhaben-tracker` + +## Services + +| Service | Image | Port | Description | +|---------|-------|------|-------------| +| db | postgres:15-alpine | `55632` | PostgreSQL with init migrations | +| rest | postgrest/postgrest:v12.2.0 | internal | REST API over Postgres | +| kong | kong:2.8.1 | `55621` | API gateway / reverse proxy | +| studio | supabase/studio | `55630` | Supabase dashboard UI | +| meta | supabase/postgres-meta:v0.84.2 | internal | Database introspection for Studio | +| worker | custom (./worker) | none | Bot + scraper process | ## Quick Start ```bash cp .env.example .env # Edit TELEGRAM_BOT_TOKEN and POSTGRES_PASSWORD in .env -docker compose up -d +docker compose up -d --build +``` + +On first boot, Postgres init scripts run automatically in order: + +1. `00-run-init.sh` — creates roles (authenticator, dashboard_user) +2. `01-init.sql` — creates tables and indexes +3. `post-boot.sql` — applies grants on created tables + +## Deployment + +### Unraid + Portainer + +1. Set the Docker Compose project path to `/mnt/user/appdata/willhaben-tracker` +2. Ensure `.env` is present with valid credentials +3. Deploy via Portainer: **Stacks → Add stack**, paste `docker-compose.yml` contents and attach `.env` +4. Postgres data persists at `/mnt/user/appdata/willhaben-tracker/data/db` + +### Manual (Linux) + +```bash +cd /path/to/willhaben-tracker +cp .env.example .env +# Edit .env with your credentials +docker compose up -d --build ``` ## Telegram Commands @@ -24,11 +61,3 @@ docker compose up -d - `/pause ` / `/resume ` — Toggle query - `/delete ` — Remove query - `/stats` — Tracking statistics - -## Ports - -| Service | Port | -|---------|------| -| Kong API | `55621` | -| Studio | `55630` | -| Postgres | `55632` | diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..0009cd2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,104 @@ +version: "3.9" + +services: + db: + image: postgres:15-alpine + restart: unless-stopped + environment: + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB:-postgres} + volumes: + - /mnt/user/appdata/willhaben-tracker/data/db:/var/lib/postgresql/data + - ./supabase/postgresql.conf:/etc/postgresql.conf:ro + - ./supabase/migrations/00-run-init.sh:/docker-entrypoint-initdb.d/00-run-init.sh:ro + - ./supabase/migrations/01-init.sql:/docker-entrypoint-initdb.d/01-init.sql:ro + - ./supabase/migrations/post-boot.sql:/docker-entrypoint-initdb.d/post-boot.sql:ro + command: > + postgres + -c config_file=/etc/postgresql.conf + -c wal_level=logical + -c max_wal_senders=0 + -c max_replication_slots=0 + -c idle_in_transaction_session_timeout=1min + -c jsonb_output_as_text=true + ports: + - "55632:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-postgres}"] + interval: 5s + timeout: 5s + retries: 10 + + rest: + image: postgrest/postgrest:v12.2.0 + restart: unless-stopped + environment: + PGRST_DB_URI: postgres://authenticator:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB:-postgres} + PGRST_DB_SCHEMAS: public + PGRST_DB_ANON_ROLE: authenticator + PGRST_JWT_SECRET: ${JWT_SECRET:-your-super-secret-jwt-key-change-in-production} + PGRST_DB_EXTRA_SEARCH_PATH: public + depends_on: + db: + condition: service_healthy + + kong: + image: kong:2.8.1 + restart: unless-stopped + environment: + KONG_DATABASE: "off" + KONG_DECLARATIVE_CONFIG: /etc/kong/kong.yml + KONG_PLUGINS: request-transformer,cors + KONG_PROXY_LISTEN: "0.0.0.0:8000" + KONG_NGINX_WORKER_PROCESSES: 1 + volumes: + - ./supabase/kong.yml:/etc/kong/kong.yml:ro + ports: + - "55621:8000" + depends_on: + rest: + condition: service_started + + studio: + image: supabase/studio + restart: unless-stopped + environment: + STUDIO_PG_META_URL: http://meta:8080 + DEFAULT_ORGANIZATION_NAME: Local + DEFAULT_PROJECT_NAME: willhaben-tracker + ports: + - "55630:3000" + depends_on: + meta: + condition: service_started + + meta: + image: supabase/postgres-meta:v0.84.2 + restart: unless-stopped + environment: + PG_META_PORT: 8080 + PG_META_DB_HOST: db + PG_META_DB_PORT: 5432 + PG_META_DB_NAME: ${POSTGRES_DB:-postgres} + PG_META_DB_USER: ${POSTGRES_USER:-postgres} + PG_META_DB_PASSWORD: ${POSTGRES_PASSWORD} + depends_on: + db: + condition: service_healthy + + worker: + build: ./worker + restart: unless-stopped + environment: + TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN} + POSTGRES_HOST: db + POSTGRES_PORT: "5432" + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB:-postgres} + DEFAULT_INTERVAL_MINUTES: ${DEFAULT_INTERVAL_MINUTES:-60} + ADMIN_TELEGRAM_IDS: ${ADMIN_TELEGRAM_IDS:-} + depends_on: + db: + condition: service_healthy diff --git a/supabase/postgresql.conf b/supabase/postgresql.conf new file mode 100644 index 0000000..cdedcdd --- /dev/null +++ b/supabase/postgresql.conf @@ -0,0 +1,20 @@ +# willhaben-tracker — PostgreSQL configuration +listen_addresses = '*' +port = 5432 +max_connections = 100 + +# Replication (needed for Supabase realtime) +wal_level = logical +max_wal_senders = 0 +max_replication_slots = 0 + +# Performance +shared_buffers = 128MB +work_mem = 4MB +maintenance_work_mem = 64MB + +# Timeouts +idle_in_transaction_session_timeout = 60000 + +# JSONB output as text for PostgREST compatibility +jsonb_output_as_text = true