HTTP Proxy
-
- {{ 'Enabled' if data.proxy_enabled else 'Disabled' }}
+
+ {% if data.proxy_enabled %}Active{% elif data.proxy_available %}Bypassed{% else %}Unavailable{% endif %}
- {{ 'Outbound scraping uses proxy' if data.proxy_enabled else 'Direct connection' }}
+ {% if data.proxy_enabled %}
+ Scraper uses proxy
+ {% elif data.proxy_available %}
+ Proxy present, scraper direct
+ {% else %}
+ No proxy in .env
+ {% endif %}
+
@@ -91,7 +102,7 @@
{% if data.proxy_enabled %}
Scheduler runs every ~{{ '%.1f'|format(data.turbo_effective_sleep_s) }}s
{% else %}
- Enable proxy first to activate turbo
+ Activate proxy first to enable turbo
{% endif %}
{% if data.seconds_until_next is not none %}
diff --git a/worker/src/web.py b/worker/src/web.py
index fc054e3..2f5823a 100644
--- a/worker/src/web.py
+++ b/worker/src/web.py
@@ -1,18 +1,16 @@
-import os
import logging
import time
from contextlib import asynccontextmanager
from datetime import datetime, timezone
import httpx
-from fastapi import FastAPI, Request
+from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
from fastapi.templating import Jinja2Templates
-import asyncpg
from db import get_pool
from scraper import get_network_status, refresh_network_status
-from settings import get_turbo_mode, set_turbo_mode
+from settings import get_proxy_enabled, get_turbo_mode, proxy_available, set_proxy_enabled, set_turbo_mode
logger = logging.getLogger(__name__)
@@ -81,10 +79,6 @@ def format_postcodes(postcodes: list | None) -> str:
return ", ".join(str(p) for p in postcodes)
-def _proxy_enabled() -> bool:
- return bool((os.getenv("HTTPS_PROXY") or os.getenv("https_proxy") or "").strip())
-
-
def _flag_from_country_code(country_code: str | None) -> str:
if not country_code or len(country_code) != 2:
return ""
@@ -162,6 +156,9 @@ async def dashboard(request: Request):
{"request": request, "error": "Database unavailable", "data": None},
)
+ proxy_configured = proxy_available()
+ proxy_setting_enabled = await get_proxy_enabled(pool)
+
try:
network = await refresh_network_status()
except Exception:
@@ -169,7 +166,7 @@ async def dashboard(request: Request):
network = get_network_status()
system_ip = network.get("system_public_ip")
last_used_ip = network.get("last_used_public_ip")
- proxy_enabled = bool(network.get("proxy_enabled"))
+ proxy_enabled = proxy_configured and proxy_setting_enabled
system_country = await _country_code_for_ip(system_ip if isinstance(system_ip, str) else None)
last_used_country = await _country_code_for_ip(last_used_ip if isinstance(last_used_ip, str) else None)
turbo_mode = await get_turbo_mode(pool)
@@ -199,7 +196,9 @@ async def dashboard(request: Request):
"queue_pending": queue_pending or 0,
"queue_dead": queue_dead or 0,
"last_scheduler": last_scheduler["scraped_at"] if last_scheduler else None,
+ "proxy_available": proxy_configured,
"proxy_enabled": proxy_enabled,
+ "proxy_setting_enabled": proxy_setting_enabled,
"system_public_ip": system_ip,
"last_used_public_ip": last_used_ip,
"system_country_code": system_country,
@@ -222,6 +221,17 @@ async def toggle_turbo(request: Request):
return RedirectResponse(url="/", status_code=303)
+@app.post("/admin/proxy")
+async def toggle_proxy(request: Request):
+ if not proxy_available():
+ return RedirectResponse(url="/", status_code=303)
+
+ pool = await get_pool()
+ current = await get_proxy_enabled(pool)
+ await set_proxy_enabled(pool, not current)
+ return RedirectResponse(url="/", status_code=303)
+
+
@app.get("/keywords", response_class=HTMLResponse)
async def keywords_list(request: Request):
try:
@@ -384,4 +394,4 @@ async def stats_json():
"queue_failed": queue_failed or 0,
"digest_buffered": digest_buffered or 0,
"last_scheduler_run": last_scheduler["scraped_at"].isoformat() if last_scheduler else None,
- }
\ No newline at end of file
+ }
diff --git a/worker/tests/test_proxy_settings.py b/worker/tests/test_proxy_settings.py
new file mode 100644
index 0000000..b9b1659
--- /dev/null
+++ b/worker/tests/test_proxy_settings.py
@@ -0,0 +1,81 @@
+"""Tests for the app-level proxy setting."""
+
+import pytest
+
+from settings import get_proxy_enabled, set_proxy_enabled
+
+
+def _clear_proxy_env(monkeypatch):
+ monkeypatch.delenv("HTTPS_PROXY", raising=False)
+ monkeypatch.delenv("https_proxy", raising=False)
+
+
+@pytest.mark.asyncio
+async def test_get_proxy_enabled_defaults_true_when_https_proxy_is_usable(
+ mock_pool, monkeypatch
+):
+ """First-run default should enable only with a usable proxy."""
+ _clear_proxy_env(monkeypatch)
+ monkeypatch.setenv("HTTPS_PROXY", "proxy.example:8080:user:pass")
+ mock_pool.fetchval.return_value = None
+
+ assert await get_proxy_enabled(mock_pool) is True
+
+ mock_pool.fetchval.assert_awaited_once()
+ assert "proxy_enabled" in mock_pool.fetchval.await_args.args
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("proxy_value", [None, "", "proxy.example:8080:user"])
+async def test_get_proxy_enabled_defaults_false_without_usable_https_proxy(
+ mock_pool, monkeypatch, proxy_value
+):
+ """Missing, blank, and malformed proxy env values default disabled."""
+ _clear_proxy_env(monkeypatch)
+ if proxy_value is not None:
+ monkeypatch.setenv("HTTPS_PROXY", proxy_value)
+ mock_pool.fetchval.return_value = None
+
+ assert await get_proxy_enabled(mock_pool) is False
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("stored_value", "expected"),
+ [
+ ("true", True),
+ ("1", True),
+ ("yes", True),
+ ("on", True),
+ ("false", False),
+ ("0", False),
+ ("no", False),
+ ("off", False),
+ ],
+)
+async def test_get_proxy_enabled_uses_persisted_setting(
+ mock_pool, monkeypatch, stored_value, expected
+):
+ """Once saved, the DB setting should control the UI/runtime toggle."""
+ _clear_proxy_env(monkeypatch)
+ monkeypatch.setenv("HTTPS_PROXY", "proxy.example:8080:user:pass")
+ mock_pool.fetchval.return_value = stored_value
+
+ assert await get_proxy_enabled(mock_pool) is expected
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("enabled", "stored_value"),
+ [(True, "true"), (False, "false")],
+)
+async def test_set_proxy_enabled_persists_proxy_toggle(
+ mock_pool, enabled, stored_value
+):
+ """The proxy toggle should be stored under the app setting key."""
+ await set_proxy_enabled(mock_pool, enabled)
+
+ mock_pool.execute.assert_awaited()
+ sql, *args = mock_pool.execute.await_args.args
+ assert "proxy_enabled" in sql or "proxy_enabled" in args
+ assert stored_value in args
diff --git a/worker/tests/test_scraper_proxy.py b/worker/tests/test_scraper_proxy.py
new file mode 100644
index 0000000..692576f
--- /dev/null
+++ b/worker/tests/test_scraper_proxy.py
@@ -0,0 +1,129 @@
+"""Proxy parsing and runtime-status tests for scraper."""
+
+import inspect
+
+import pytest
+
+import scraper
+
+
+def _clear_proxy_env(monkeypatch):
+ monkeypatch.delenv("HTTPS_PROXY", raising=False)
+ monkeypatch.delenv("https_proxy", raising=False)
+
+
+def _proxy_url_parser():
+ parser = getattr(scraper, "_get_proxy_url", None)
+ if parser is None:
+ pytest.skip("scraper does not expose a proxy URL parser")
+ return parser
+
+
+async def _get_network_status(mock_pool, monkeypatch):
+ monkeypatch.setattr(
+ scraper,
+ "_proxy_enabled_effective",
+ None,
+ raising=False,
+ )
+ status_fn = getattr(scraper, "get_network_status")
+ signature = inspect.signature(status_fn)
+ if "pool" in signature.parameters:
+ result = status_fn(mock_pool)
+ if inspect.isawaitable(result):
+ return await result
+ return result
+
+ effective_proxy_fn = getattr(scraper, "_get_effective_proxy_url", None)
+ if effective_proxy_fn is not None:
+ async def fake_get_pool():
+ return mock_pool
+
+ monkeypatch.setattr(scraper, "get_pool", fake_get_pool, raising=False)
+ result = effective_proxy_fn()
+ if inspect.isawaitable(result):
+ await result
+
+ result = status_fn()
+ if inspect.isawaitable(result):
+ return await result
+ return result
+
+
+def _block_public_ip_fetch(monkeypatch):
+ async def fail_fetch(*_args, **_kwargs):
+ raise AssertionError("network status tests must not fetch public IPs")
+
+ monkeypatch.setattr(scraper, "_fetch_public_ip", fail_fetch, raising=False)
+
+
+def test_proxy_url_parser_accepts_full_proxy_url(monkeypatch):
+ _clear_proxy_env(monkeypatch)
+ monkeypatch.setenv("HTTPS_PROXY", "http://user:pass@proxy.example:8080")
+
+ assert _proxy_url_parser()() == "http://user:pass@proxy.example:8080"
+
+
+def test_proxy_url_parser_builds_url_from_colon_format(monkeypatch):
+ _clear_proxy_env(monkeypatch)
+ monkeypatch.setenv("HTTPS_PROXY", "proxy.example:8080:user@example:p a/s")
+
+ assert (
+ _proxy_url_parser()()
+ == "http://user%40example:p%20a%2Fs@proxy.example:8080"
+ )
+
+
+@pytest.mark.parametrize("proxy_value", ["", "proxy.example:8080:user"])
+def test_proxy_url_parser_rejects_missing_or_malformed_proxy(
+ monkeypatch, proxy_value
+):
+ _clear_proxy_env(monkeypatch)
+ monkeypatch.setenv("HTTPS_PROXY", proxy_value)
+
+ assert _proxy_url_parser()() is None
+
+
+@pytest.mark.asyncio
+async def test_network_status_reports_proxy_available_when_env_is_usable(
+ mock_pool, monkeypatch
+):
+ _clear_proxy_env(monkeypatch)
+ _block_public_ip_fetch(monkeypatch)
+ monkeypatch.setenv("HTTPS_PROXY", "proxy.example:8080:user:pass")
+ mock_pool.fetchval.return_value = None
+
+ status = await _get_network_status(mock_pool, monkeypatch)
+
+ assert status["proxy_available"] is True
+ assert status["proxy_enabled"] is True
+
+
+@pytest.mark.asyncio
+async def test_network_status_respects_disabled_proxy_setting(
+ mock_pool, monkeypatch
+):
+ _clear_proxy_env(monkeypatch)
+ _block_public_ip_fetch(monkeypatch)
+ monkeypatch.setenv("HTTPS_PROXY", "proxy.example:8080:user:pass")
+ mock_pool.fetchval.return_value = "false"
+
+ status = await _get_network_status(mock_pool, monkeypatch)
+
+ assert status["proxy_available"] is True
+ assert status["proxy_enabled"] is False
+
+
+@pytest.mark.asyncio
+async def test_network_status_reports_unavailable_for_malformed_proxy(
+ mock_pool, monkeypatch
+):
+ _clear_proxy_env(monkeypatch)
+ _block_public_ip_fetch(monkeypatch)
+ monkeypatch.setenv("HTTPS_PROXY", "proxy.example:8080:user")
+ mock_pool.fetchval.return_value = None
+
+ status = await _get_network_status(mock_pool, monkeypatch)
+
+ assert status["proxy_available"] is False
+ assert status["proxy_enabled"] is False