Private
Public Access
S4: the throttle keyed on the first X-Forwarded-For entry, which a client or a proxy that appends can choose, and disabled itself entirely when Redis was down. EVECTION_TRUSTED_PROXIES (IPs/CIDRs of your reverse proxy) makes it strict: X-Forwarded-For is only believed when the TCP peer is a trusted proxy, and the rightmost entry that isn't itself a trusted proxy — the address the proxy actually saw — is used instead of the spoofable first one. Unset keeps the previous behavior so existing installs don't suddenly share one bucket behind a proxy, and `manage.py check --deploy` warns (evection.W001) until it's set. During a Redis outage the throttle now falls back to a bounded in-process limiter instead of switching off. C5: /health/ready/ reuses its queue/feed aggregates for 15s (EVECTION_HEALTH_CACHE_SECONDS) so a fast-polling monitor can't create steady load; the database, Redis and poller probes stay live. Docs cover both settings, restricting the endpoint at the proxy, and the tombstone/purge behavior from the previous changes. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aacb41d1d3
commit
ac5e951749
12 files changed
+386
-53
No files matched your search
@@ -23,3 +23,9 @@ EVECTION_MIN_FETCH_INTERVAL_MINUTES=15
|
|||||||
# Bump this on deploys that change cached static assets to invalidate old
|
# Bump this on deploys that change cached static assets to invalidate old
|
||||||
# service worker caches.
|
# service worker caches.
|
||||||
EVECTION_PWA_CACHE_VERSION=v1
|
EVECTION_PWA_CACHE_VERSION=v1
|
||||||
|
|
||||||
|
# IP address(es) or CIDR(s) of the reverse proxy in front of gunicorn,
|
||||||
|
# comma-separated, e.g. 192.168.1.10 or 10.0.0.0/24. When set, the login
|
||||||
|
# throttle trusts X-Forwarded-For only from these addresses. Strongly
|
||||||
|
# recommended; `manage.py check --deploy` warns until it's set.
|
||||||
|
# EVECTION_TRUSTED_PROXIES=
|
||||||
@@ -4,3 +4,6 @@ from django.apps import AppConfig
|
|||||||
class AccountsConfig(AppConfig):
|
class AccountsConfig(AppConfig):
|
||||||
default_auto_field = "django.db.models.BigAutoField"
|
default_auto_field = "django.db.models.BigAutoField"
|
||||||
name = "apps.accounts"
|
name = "apps.accounts"
|
||||||
|
|
||||||
|
def ready(self):
|
||||||
|
from . import checks # noqa: F401 (registers the deploy check)
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
from django.conf import settings
|
||||||
|
from django.core.checks import Tags, Warning, register
|
||||||
|
|
||||||
|
|
||||||
|
@register(Tags.security, deploy=True)
|
||||||
|
def trusted_proxies_configured(app_configs, **kwargs):
|
||||||
|
if settings.DEBUG or settings.EVECTION_TRUSTED_PROXIES:
|
||||||
|
return []
|
||||||
|
return [
|
||||||
|
Warning(
|
||||||
|
"EVECTION_TRUSTED_PROXIES is not set.",
|
||||||
|
hint=(
|
||||||
|
"The login throttle then trusts the first X-Forwarded-For entry from "
|
||||||
|
"anyone, which only holds while gunicorn is reachable by nothing but a "
|
||||||
|
"proxy that overwrites that header. Set EVECTION_TRUSTED_PROXIES to your "
|
||||||
|
"reverse proxy's IP/CIDR in .env (see docs/install.md)."
|
||||||
|
),
|
||||||
|
id="evection.W001",
|
||||||
|
)
|
||||||
|
]
|
||||||
@@ -74,19 +74,144 @@ def test_too_many_failed_attempts_against_one_username_blocks_that_username(clie
|
|||||||
assert response.wsgi_request.user == bob
|
assert response.wsgi_request.user == bob
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.fixture
|
||||||
def test_login_still_works_when_the_throttle_cache_is_down(client, monkeypatch):
|
def cache_down(monkeypatch):
|
||||||
from unittest.mock import Mock
|
from unittest.mock import Mock
|
||||||
|
|
||||||
from django.contrib.auth import get_user_model
|
|
||||||
|
|
||||||
get_user_model().objects.create_user("alice", password="pw-alice-123")
|
|
||||||
broken = Mock(side_effect=ConnectionError("redis down"))
|
broken = Mock(side_effect=ConnectionError("redis down"))
|
||||||
monkeypatch.setattr("apps.accounts.views.cache.get", broken)
|
for name in ("get", "incr", "set"):
|
||||||
monkeypatch.setattr("apps.accounts.views.cache.incr", broken)
|
monkeypatch.setattr(f"apps.accounts.throttle.cache.{name}", broken)
|
||||||
monkeypatch.setattr("apps.accounts.views.cache.set", broken)
|
|
||||||
|
|
||||||
bad = client.post(reverse("accounts:login"), {"username": "alice", "password": "wrong"})
|
|
||||||
assert bad.status_code == 200
|
@pytest.fixture(autouse=True)
|
||||||
|
def _reset_local_limiter():
|
||||||
|
from .views import login_throttle
|
||||||
|
|
||||||
|
login_throttle._local.clear()
|
||||||
|
yield
|
||||||
|
login_throttle._local.clear()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_login_still_works_when_the_throttle_cache_is_down(client, alice, cache_down):
|
||||||
|
assert _bad_login(client).status_code == 200
|
||||||
good = client.post(reverse("accounts:login"), {"username": "alice", "password": "pw-alice-123"})
|
good = client.post(reverse("accounts:login"), {"username": "alice", "password": "pw-alice-123"})
|
||||||
assert good.status_code == 302
|
assert good.status_code == 302
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_a_redis_outage_degrades_to_a_local_limit_instead_of_no_limit(client, alice, cache_down):
|
||||||
|
"""R2-S4: previously every attempt was allowed while the cache was
|
||||||
|
down, so an outage switched brute-force protection off entirely."""
|
||||||
|
for _ in range(LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME):
|
||||||
|
_bad_login(client)
|
||||||
|
|
||||||
|
blocked = client.post(
|
||||||
|
reverse("accounts:login"), {"username": "alice", "password": "pw-alice-123"}
|
||||||
|
)
|
||||||
|
assert blocked.status_code == 302
|
||||||
|
assert blocked.url == reverse("accounts:login"), "even the right password is refused"
|
||||||
|
|
||||||
|
|
||||||
|
class TestLocalLimiter:
|
||||||
|
def test_counts_per_key_and_expires(self, monkeypatch):
|
||||||
|
import time as time_module
|
||||||
|
|
||||||
|
from .throttle import LoginThrottle
|
||||||
|
|
||||||
|
throttle = LoginThrottle(window_seconds=60)
|
||||||
|
for _ in range(3):
|
||||||
|
throttle._local_record("k")
|
||||||
|
assert throttle._local_count("k") == 3
|
||||||
|
assert throttle._local_count("other") == 0
|
||||||
|
|
||||||
|
real = time_module.monotonic()
|
||||||
|
monkeypatch.setattr("apps.accounts.throttle.time.monotonic", lambda: real + 61)
|
||||||
|
assert throttle._local_count("k") == 0
|
||||||
|
|
||||||
|
def test_memory_is_bounded(self, monkeypatch):
|
||||||
|
from . import throttle as throttle_module
|
||||||
|
|
||||||
|
monkeypatch.setattr(throttle_module, "LOCAL_MAX_KEYS", 5)
|
||||||
|
throttle = throttle_module.LoginThrottle(window_seconds=60)
|
||||||
|
for i in range(50):
|
||||||
|
throttle._local_record(f"k{i}")
|
||||||
|
assert len(throttle._local) <= 5
|
||||||
|
|
||||||
|
|
||||||
|
class TestClientAddress:
|
||||||
|
def _request(self, remote, forwarded=None):
|
||||||
|
from django.test import RequestFactory
|
||||||
|
|
||||||
|
extra = {"REMOTE_ADDR": remote}
|
||||||
|
if forwarded is not None:
|
||||||
|
extra["HTTP_X_FORWARDED_FOR"] = forwarded
|
||||||
|
return RequestFactory().get("/", **extra)
|
||||||
|
|
||||||
|
def test_unset_keeps_the_legacy_first_hop_behavior(self, settings):
|
||||||
|
from .throttle import client_address
|
||||||
|
|
||||||
|
settings.EVECTION_TRUSTED_PROXIES = []
|
||||||
|
assert client_address(self._request("10.0.0.5", "1.2.3.4, 10.0.0.5")) == "1.2.3.4"
|
||||||
|
assert client_address(self._request("10.0.0.5")) == "10.0.0.5"
|
||||||
|
|
||||||
|
def test_trusted_proxy_uses_the_rightmost_untrusted_hop_not_the_spoofable_first(self, settings):
|
||||||
|
import ipaddress
|
||||||
|
|
||||||
|
from .throttle import client_address
|
||||||
|
|
||||||
|
settings.EVECTION_TRUSTED_PROXIES = [ipaddress.ip_network("10.0.0.5/32")]
|
||||||
|
# The client sent "X-Forwarded-For: 6.6.6.6"; the proxy appended the
|
||||||
|
# address it actually saw (203.0.113.9).
|
||||||
|
request = self._request("10.0.0.5", "6.6.6.6, 203.0.113.9")
|
||||||
|
assert client_address(request) == "203.0.113.9"
|
||||||
|
|
||||||
|
def test_forwarded_for_from_an_untrusted_peer_is_ignored(self, settings):
|
||||||
|
import ipaddress
|
||||||
|
|
||||||
|
from .throttle import client_address
|
||||||
|
|
||||||
|
settings.EVECTION_TRUSTED_PROXIES = [ipaddress.ip_network("10.0.0.5/32")]
|
||||||
|
# Someone reaching gunicorn directly and claiming to be 1.2.3.4.
|
||||||
|
assert client_address(self._request("198.51.100.7", "1.2.3.4")) == "198.51.100.7"
|
||||||
|
|
||||||
|
def test_chained_trusted_proxies_are_skipped(self, settings):
|
||||||
|
import ipaddress
|
||||||
|
|
||||||
|
from .throttle import client_address
|
||||||
|
|
||||||
|
settings.EVECTION_TRUSTED_PROXIES = [ipaddress.ip_network("10.0.0.0/24")]
|
||||||
|
request = self._request("10.0.0.5", "203.0.113.9, 10.0.0.7")
|
||||||
|
assert client_address(request) == "203.0.113.9"
|
||||||
|
|
||||||
|
def test_trusted_peer_with_no_forwarded_header_falls_back_to_the_peer(self, settings):
|
||||||
|
import ipaddress
|
||||||
|
|
||||||
|
from .throttle import client_address
|
||||||
|
|
||||||
|
settings.EVECTION_TRUSTED_PROXIES = [ipaddress.ip_network("10.0.0.5/32")]
|
||||||
|
assert client_address(self._request("10.0.0.5")) == "10.0.0.5"
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeployCheck:
|
||||||
|
def test_warns_when_trusted_proxies_is_unset_in_production(self, settings):
|
||||||
|
from .checks import trusted_proxies_configured
|
||||||
|
|
||||||
|
settings.DEBUG = False
|
||||||
|
settings.EVECTION_TRUSTED_PROXIES = []
|
||||||
|
warnings = trusted_proxies_configured(None)
|
||||||
|
|
||||||
|
assert [w.id for w in warnings] == ["evection.W001"]
|
||||||
|
|
||||||
|
def test_quiet_when_configured_or_in_debug(self, settings):
|
||||||
|
import ipaddress
|
||||||
|
|
||||||
|
from .checks import trusted_proxies_configured
|
||||||
|
|
||||||
|
settings.DEBUG = False
|
||||||
|
settings.EVECTION_TRUSTED_PROXIES = [ipaddress.ip_network("10.0.0.5/32")]
|
||||||
|
assert trusted_proxies_configured(None) == []
|
||||||
|
|
||||||
|
settings.DEBUG = True
|
||||||
|
settings.EVECTION_TRUSTED_PROXIES = []
|
||||||
|
assert trusted_proxies_configured(None) == []
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
"""Login attempt throttling: counters live in the shared cache (Redis) so
|
||||||
|
every gunicorn worker sees them, with an in-process fallback when the
|
||||||
|
cache is unreachable (R2-S4) — so a Redis outage degrades throttling to
|
||||||
|
per-worker limits instead of switching it off for anyone who happens to
|
||||||
|
choose that moment."""
|
||||||
|
|
||||||
|
import ipaddress
|
||||||
|
import logging
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
from django.conf import settings
|
||||||
|
from django.core.cache import cache
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
LOCAL_MAX_KEYS = 10_000
|
||||||
|
|
||||||
|
|
||||||
|
class LoginThrottle:
|
||||||
|
def __init__(self, window_seconds: int):
|
||||||
|
self.window = window_seconds
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._local: dict[str, tuple[int, float]] = {}
|
||||||
|
|
||||||
|
def is_blocked(self, key: str, limit: int) -> bool:
|
||||||
|
try:
|
||||||
|
return cache.get(key, 0) >= limit
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Login throttle cache unavailable; using local limiter")
|
||||||
|
return self._local_count(key) >= limit
|
||||||
|
|
||||||
|
def record(self, key: str) -> None:
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
cache.incr(key)
|
||||||
|
except ValueError:
|
||||||
|
cache.set(key, 1, self.window)
|
||||||
|
except Exception:
|
||||||
|
logger.warning("Login throttle cache unavailable; using local limiter")
|
||||||
|
self._local_record(key)
|
||||||
|
|
||||||
|
def _local_count(self, key: str) -> int:
|
||||||
|
with self._lock:
|
||||||
|
entry = self._local.get(key)
|
||||||
|
if entry is None or entry[1] <= time.monotonic():
|
||||||
|
return 0
|
||||||
|
return entry[0]
|
||||||
|
|
||||||
|
def _local_record(self, key: str) -> None:
|
||||||
|
now = time.monotonic()
|
||||||
|
with self._lock:
|
||||||
|
if len(self._local) >= LOCAL_MAX_KEYS:
|
||||||
|
self._local = {k: v for k, v in self._local.items() if v[1] > now}
|
||||||
|
if len(self._local) >= LOCAL_MAX_KEYS:
|
||||||
|
self._local.clear() # under a flood, bounded memory beats perfect memory
|
||||||
|
count, expires = self._local.get(key, (0, 0.0))
|
||||||
|
if expires <= now:
|
||||||
|
count, expires = 0, now + self.window
|
||||||
|
self._local[key] = (count + 1, expires)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_ip(value: str):
|
||||||
|
try:
|
||||||
|
return ipaddress.ip_address(value.strip())
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _is_trusted_proxy(value: str, trusted) -> bool:
|
||||||
|
ip = _parse_ip(value)
|
||||||
|
return ip is not None and any(ip in network for network in trusted)
|
||||||
|
|
||||||
|
|
||||||
|
def client_address(request) -> str:
|
||||||
|
"""The address to throttle a login attempt by.
|
||||||
|
|
||||||
|
EVECTION_TRUSTED_PROXIES (comma-separated IPs/CIDRs of your reverse
|
||||||
|
proxy) makes this strict: X-Forwarded-For is only believed when the
|
||||||
|
TCP peer is one of those proxies, and then the *rightmost* entry that
|
||||||
|
isn't itself a trusted proxy is used — the address your proxy actually
|
||||||
|
saw — rather than the first entry, which is whatever the client chose
|
||||||
|
to send. A client reaching gunicorn directly, or a proxy outside the
|
||||||
|
list, gets its own peer address.
|
||||||
|
|
||||||
|
Unset (the default), it keeps the previous behavior — trust the first
|
||||||
|
X-Forwarded-For entry — which is only safe while gunicorn is reachable
|
||||||
|
by nothing but a proxy that overwrites that header (docs/install.md);
|
||||||
|
`manage.py check --deploy` warns until it's set.
|
||||||
|
"""
|
||||||
|
remote = request.META.get("REMOTE_ADDR", "unknown")
|
||||||
|
forwarded = [
|
||||||
|
part.strip()
|
||||||
|
for part in request.META.get("HTTP_X_FORWARDED_FOR", "").split(",")
|
||||||
|
if part.strip()
|
||||||
|
]
|
||||||
|
trusted = settings.EVECTION_TRUSTED_PROXIES
|
||||||
|
|
||||||
|
if not trusted:
|
||||||
|
return forwarded[0] if forwarded else remote
|
||||||
|
if not _is_trusted_proxy(remote, trusted):
|
||||||
|
return remote
|
||||||
|
for address in reversed(forwarded):
|
||||||
|
if not _is_trusted_proxy(address, trusted):
|
||||||
|
return address
|
||||||
|
return forwarded[0] if forwarded else remote
|
||||||
+7
-38
@@ -4,13 +4,13 @@ from django.contrib import messages
|
|||||||
from django.contrib.auth import login
|
from django.contrib.auth import login
|
||||||
from django.contrib.auth.decorators import user_passes_test
|
from django.contrib.auth.decorators import user_passes_test
|
||||||
from django.contrib.auth.views import LoginView as DjangoLoginView
|
from django.contrib.auth.views import LoginView as DjangoLoginView
|
||||||
from django.core.cache import cache
|
|
||||||
from django.db import IntegrityError, transaction
|
from django.db import IntegrityError, transaction
|
||||||
from django.shortcuts import redirect, render
|
from django.shortcuts import redirect, render
|
||||||
|
|
||||||
from .forms import CreateUserForm, SetupForm
|
from .forms import CreateUserForm, SetupForm
|
||||||
from .middleware import SetupWizardMiddleware
|
from .middleware import SetupWizardMiddleware
|
||||||
from .models import SetupLock, User
|
from .models import SetupLock, User
|
||||||
|
from .throttle import LoginThrottle, client_address
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -27,44 +27,13 @@ LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_ADDRESS = 15
|
|||||||
LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME = 8
|
LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME = 8
|
||||||
|
|
||||||
|
|
||||||
def _client_address(request) -> str:
|
login_throttle = LoginThrottle(LOGIN_RATE_LIMIT_WINDOW_SECONDS)
|
||||||
# gunicorn is always run behind a reverse proxy in every deployment
|
|
||||||
# (see docs/install.md) and its bind address is never meant to be
|
|
||||||
# reachable from outside that proxy — the same trust boundary
|
|
||||||
# config.settings.prod already relies on for X-Forwarded-Proto. A
|
|
||||||
# direct request (e.g. local dev) has no X-Forwarded-For and falls
|
|
||||||
# back to REMOTE_ADDR.
|
|
||||||
forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR", "")
|
|
||||||
if forwarded_for:
|
|
||||||
return forwarded_for.split(",")[0].strip()
|
|
||||||
return request.META.get("REMOTE_ADDR", "unknown")
|
|
||||||
|
|
||||||
|
|
||||||
def _attempts_key(kind: str, value: str) -> str:
|
def _attempts_key(kind: str, value: str) -> str:
|
||||||
return f"login-attempts:{kind}:{value}"
|
return f"login-attempts:{kind}:{value}"
|
||||||
|
|
||||||
|
|
||||||
def _too_many_attempts(key: str, limit: int) -> bool:
|
|
||||||
# Fails open if the cache (Redis) is unreachable: logging in must keep
|
|
||||||
# working through a cache outage, at the cost of no throttling until
|
|
||||||
# it's back — an outage shouldn't lock every user out of the app.
|
|
||||||
try:
|
|
||||||
return cache.get(key, 0) >= limit
|
|
||||||
except Exception:
|
|
||||||
logger.warning("Login throttle cache unavailable; not rate limiting", exc_info=True)
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def _record_failed_attempt(key: str) -> None:
|
|
||||||
try:
|
|
||||||
try:
|
|
||||||
cache.incr(key)
|
|
||||||
except ValueError:
|
|
||||||
cache.set(key, 1, LOGIN_RATE_LIMIT_WINDOW_SECONDS)
|
|
||||||
except Exception:
|
|
||||||
logger.warning("Login throttle cache unavailable; attempt not recorded", exc_info=True)
|
|
||||||
|
|
||||||
|
|
||||||
class LoginView(DjangoLoginView):
|
class LoginView(DjangoLoginView):
|
||||||
"""Wraps Django's own LoginView with the throttle above — everything
|
"""Wraps Django's own LoginView with the throttle above — everything
|
||||||
else (form, template, redirect behavior) is unchanged."""
|
else (form, template, redirect behavior) is unchanged."""
|
||||||
@@ -72,11 +41,11 @@ class LoginView(DjangoLoginView):
|
|||||||
template_name = "accounts/login.html"
|
template_name = "accounts/login.html"
|
||||||
|
|
||||||
def post(self, request, *args, **kwargs):
|
def post(self, request, *args, **kwargs):
|
||||||
address_key = _attempts_key("addr", _client_address(request))
|
address_key = _attempts_key("addr", client_address(request))
|
||||||
username_key = _attempts_key("user", request.POST.get("username", "")[:150])
|
username_key = _attempts_key("user", request.POST.get("username", "")[:150])
|
||||||
if _too_many_attempts(
|
if login_throttle.is_blocked(
|
||||||
address_key, LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_ADDRESS
|
address_key, LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_ADDRESS
|
||||||
) or _too_many_attempts(username_key, LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME):
|
) or login_throttle.is_blocked(username_key, LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME):
|
||||||
messages.error(
|
messages.error(
|
||||||
request, "Too many login attempts. Please wait a few minutes and try again."
|
request, "Too many login attempts. Please wait a few minutes and try again."
|
||||||
)
|
)
|
||||||
@@ -84,8 +53,8 @@ class LoginView(DjangoLoginView):
|
|||||||
return super().post(request, *args, **kwargs)
|
return super().post(request, *args, **kwargs)
|
||||||
|
|
||||||
def form_invalid(self, form):
|
def form_invalid(self, form):
|
||||||
_record_failed_attempt(_attempts_key("addr", _client_address(self.request)))
|
login_throttle.record(_attempts_key("addr", client_address(self.request)))
|
||||||
_record_failed_attempt(_attempts_key("user", self.request.POST.get("username", "")[:150]))
|
login_throttle.record(_attempts_key("user", self.request.POST.get("username", "")[:150]))
|
||||||
return super().form_invalid(form)
|
return super().form_invalid(form)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ from datetime import timedelta
|
|||||||
|
|
||||||
import redis
|
import redis
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
|
from django.core.cache import cache
|
||||||
from django.db.models import Max, Min
|
from django.db.models import Max, Min
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
@@ -42,6 +43,33 @@ def _success_window() -> timedelta:
|
|||||||
return timedelta(minutes=max(3 * base, MIN_SUCCESS_WINDOW_MINUTES))
|
return timedelta(minutes=max(3 * base, MIN_SUCCESS_WINDOW_MINUTES))
|
||||||
|
|
||||||
|
|
||||||
|
PIPELINE_CACHE_KEY = "health-ready-pipeline"
|
||||||
|
|
||||||
|
|
||||||
|
def cached_fetch_pipeline_status() -> tuple[dict[str, bool], dict]:
|
||||||
|
"""fetch_pipeline_status, reused for EVECTION_HEALTH_CACHE_SECONDS
|
||||||
|
(R2-C5): the queue/feed aggregates behind it cost real queries, and a
|
||||||
|
monitor that polls /health/ready/ every second shouldn't turn that
|
||||||
|
into recurring database and Redis load. The cheap liveness probes
|
||||||
|
(database, Redis, poll heartbeat) are not cached — those stay live.
|
||||||
|
A cache failure just means computing it fresh."""
|
||||||
|
ttl = settings.EVECTION_HEALTH_CACHE_SECONDS
|
||||||
|
if ttl > 0:
|
||||||
|
try:
|
||||||
|
cached = cache.get(PIPELINE_CACHE_KEY)
|
||||||
|
except Exception:
|
||||||
|
cached = None
|
||||||
|
if cached is not None:
|
||||||
|
return cached
|
||||||
|
result = fetch_pipeline_status()
|
||||||
|
if ttl > 0:
|
||||||
|
try:
|
||||||
|
cache.set(PIPELINE_CACHE_KEY, result, timeout=ttl)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def fetch_pipeline_status(now=None) -> tuple[dict[str, bool], dict]:
|
def fetch_pipeline_status(now=None) -> tuple[dict[str, bool], dict]:
|
||||||
"""Checks that fetching is actually *succeeding*, not just scheduled
|
"""Checks that fetching is actually *succeeding*, not just scheduled
|
||||||
(the poll heartbeat only proves beat and a worker are alive): returns
|
(the poll heartbeat only proves beat and a worker are alive): returns
|
||||||
|
|||||||
@@ -265,3 +265,45 @@ def test_health_ready_flags_most_feeds_failing_but_not_a_single_broken_one(clien
|
|||||||
assert body["checks"]["feed_failures"] is False
|
assert body["checks"]["feed_failures"] is False
|
||||||
assert body["failing_feeds"] == 3
|
assert body["failing_feeds"] == 3
|
||||||
assert healthy.pk and broken
|
assert healthy.pk and broken
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_readiness_reuses_the_pipeline_aggregates_for_a_short_time(client, broker_queue, settings):
|
||||||
|
"""R2-C5: the expensive part of the readiness check is cached briefly,
|
||||||
|
while the cheap liveness probes stay live."""
|
||||||
|
from django.core.cache import cache
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY
|
||||||
|
|
||||||
|
settings.EVECTION_HEALTH_CACHE_SECONDS = 30
|
||||||
|
cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now())
|
||||||
|
Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
||||||
|
|
||||||
|
first = client.get(reverse("core:health_ready")).json()
|
||||||
|
Feed.objects.create(feed_url="https://b.example.com/feed.xml")
|
||||||
|
second = client.get(reverse("core:health_ready")).json()
|
||||||
|
|
||||||
|
assert first["active_feeds"] == 1
|
||||||
|
assert second["active_feeds"] == 1, "served from the short-lived cache"
|
||||||
|
|
||||||
|
cache.delete("health-ready-pipeline")
|
||||||
|
third = client.get(reverse("core:health_ready")).json()
|
||||||
|
assert third["active_feeds"] == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_readiness_liveness_probes_are_never_cached(client, broker_queue, settings):
|
||||||
|
from django.core.cache import cache
|
||||||
|
|
||||||
|
settings.EVECTION_HEALTH_CACHE_SECONDS = 30
|
||||||
|
first = client.get(reverse("core:health_ready"))
|
||||||
|
assert first.json()["checks"]["poll_heartbeat"] is False
|
||||||
|
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY
|
||||||
|
|
||||||
|
cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now())
|
||||||
|
second = client.get(reverse("core:health_ready"))
|
||||||
|
assert second.json()["checks"]["poll_heartbeat"] is True
|
||||||
+2
-2
@@ -12,7 +12,7 @@ from apps.articles.models import Article
|
|||||||
from apps.feeds.models import Feed, Subscription
|
from apps.feeds.models import Feed, Subscription
|
||||||
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, enqueue_fetch, reset_backoff
|
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, enqueue_fetch, reset_backoff
|
||||||
|
|
||||||
from .health import fetch_pipeline_status
|
from .health import cached_fetch_pipeline_status
|
||||||
|
|
||||||
is_admin = user_passes_test(lambda u: u.is_authenticated and u.is_staff)
|
is_admin = user_passes_test(lambda u: u.is_authenticated and u.is_staff)
|
||||||
|
|
||||||
@@ -70,7 +70,7 @@ def health_ready(request):
|
|||||||
checks["poll_heartbeat"] = age <= POLL_HEARTBEAT_MAX_AGE_SECONDS
|
checks["poll_heartbeat"] = age <= POLL_HEARTBEAT_MAX_AGE_SECONDS
|
||||||
|
|
||||||
if checks["database"]:
|
if checks["database"]:
|
||||||
pipeline_checks, pipeline_detail = fetch_pipeline_status()
|
pipeline_checks, pipeline_detail = cached_fetch_pipeline_status()
|
||||||
checks.update(pipeline_checks)
|
checks.update(pipeline_checks)
|
||||||
detail.update(pipeline_detail)
|
detail.update(pipeline_detail)
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ Base Django settings for Evection, shared by all environments.
|
|||||||
Environment-specific overrides live in dev.py / prod.py / test.py.
|
Environment-specific overrides live in dev.py / prod.py / test.py.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import ipaddress
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import environ
|
import environ
|
||||||
@@ -122,6 +123,18 @@ EVECTION_SIGNUP_ENABLED = env.bool("EVECTION_SIGNUP_ENABLED", default=False)
|
|||||||
EVECTION_MIN_FETCH_INTERVAL_MINUTES = env.int("EVECTION_MIN_FETCH_INTERVAL_MINUTES", default=15)
|
EVECTION_MIN_FETCH_INTERVAL_MINUTES = env.int("EVECTION_MIN_FETCH_INTERVAL_MINUTES", default=15)
|
||||||
EVECTION_PWA_CACHE_VERSION = env.str("EVECTION_PWA_CACHE_VERSION", default="v1")
|
EVECTION_PWA_CACHE_VERSION = env.str("EVECTION_PWA_CACHE_VERSION", default="v1")
|
||||||
|
|
||||||
|
# How long /health/ready/ reuses its queue/feed aggregates (0 = never).
|
||||||
|
EVECTION_HEALTH_CACHE_SECONDS = env.int("EVECTION_HEALTH_CACHE_SECONDS", default=15)
|
||||||
|
|
||||||
|
# IPs/CIDRs of the reverse proxy in front of gunicorn. When set, the login
|
||||||
|
# throttle only believes X-Forwarded-For from these peers (see
|
||||||
|
# apps.accounts.throttle.client_address). Empty keeps the legacy behavior.
|
||||||
|
EVECTION_TRUSTED_PROXIES = [
|
||||||
|
ipaddress.ip_network(entry.strip(), strict=False)
|
||||||
|
for entry in env.list("EVECTION_TRUSTED_PROXIES", default=[])
|
||||||
|
if entry.strip()
|
||||||
|
]
|
||||||
|
|
||||||
# Redis-backed rather than Django's default per-process LocMemCache: login
|
# Redis-backed rather than Django's default per-process LocMemCache: login
|
||||||
# rate limiting (apps.accounts.views) must share counters across every
|
# rate limiting (apps.accounts.views) must share counters across every
|
||||||
# gunicorn worker, not just whichever one happened to handle a given
|
# gunicorn worker, not just whichever one happened to handle a given
|
||||||
|
|||||||
@@ -12,3 +12,6 @@ PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
|
|||||||
# location is per-process but shared across tests in that process; see
|
# location is per-process but shared across tests in that process; see
|
||||||
# apps.accounts.test_rate_limit's cache.clear() fixture).
|
# apps.accounts.test_rate_limit's cache.clear() fixture).
|
||||||
CACHES = {"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}}
|
CACHES = {"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}}
|
||||||
|
|
||||||
|
# Tests change state between calls to /health/ready/; caching it would hide that.
|
||||||
|
EVECTION_HEALTH_CACHE_SECONDS = 0
|
||||||
+21
-3
@@ -150,6 +150,15 @@ At minimum, set the following in `.env`:
|
|||||||
- `REDIS_URL` — `redis://localhost:6379/0` is fine for a single-host install
|
- `REDIS_URL` — `redis://localhost:6379/0` is fine for a single-host install
|
||||||
- `EVECTION_GUNICORN_BIND` — the `address:port` gunicorn listens on; see
|
- `EVECTION_GUNICORN_BIND` — the `address:port` gunicorn listens on; see
|
||||||
"Reverse proxy" below for how to pick this
|
"Reverse proxy" below for how to pick this
|
||||||
|
- `EVECTION_TRUSTED_PROXIES` — the IP address (or CIDR, comma-separated for more
|
||||||
|
than one) of your reverse proxy, as gunicorn sees it connecting. The login
|
||||||
|
throttle only believes `X-Forwarded-For` from these addresses, and takes the
|
||||||
|
address your proxy actually saw rather than the first entry (which a client can
|
||||||
|
forge). Leave it unset and it falls back to trusting that header from anyone,
|
||||||
|
which is only safe while nothing but your proxy can reach gunicorn;
|
||||||
|
`manage.py check --deploy` warns until it's set. With Nginx Proxy Manager on
|
||||||
|
another host, use that host's address (check `ss -tn` on this machine while
|
||||||
|
loading the site if you're not sure what gunicorn sees).
|
||||||
|
|
||||||
How often every feed is polled is a site-wide setting (Django admin → Site
|
How often every feed is polled is a site-wide setting (Django admin → Site
|
||||||
settings), not a `.env` value — `EVECTION_MIN_FETCH_INTERVAL_MINUTES` only sets
|
settings), not a `.env` value — `EVECTION_MIN_FETCH_INTERVAL_MINUTES` only sets
|
||||||
@@ -339,7 +348,12 @@ with a per-check breakdown when any of these fail:
|
|||||||
failing at once points at your server, DNS, or outbound network).
|
failing at once points at your server, DNS, or outbound network).
|
||||||
|
|
||||||
The payload also reports `queue_depth`, `active_feeds`, `failing_feeds`, and
|
The payload also reports `queue_depth`, `active_feeds`, `failing_feeds`, and
|
||||||
the age of the last successful fetch, so you can graph them.
|
the age of the last successful fetch, so you can graph them. The queue and feed
|
||||||
|
figures are reused for 15 seconds (`EVECTION_HEALTH_CACHE_SECONDS`, 0 disables
|
||||||
|
it) so a fast-polling monitor can't turn them into steady database load; the
|
||||||
|
database, Redis and poller checks are always live. The endpoint is
|
||||||
|
unauthenticated, so if you'd rather not expose those numbers publicly, restrict
|
||||||
|
`/health/ready/` to your monitor's address at the proxy.
|
||||||
|
|
||||||
**Retention.** By default every fetched article is kept forever. In the Django
|
**Retention.** By default every fetched article is kept forever. In the Django
|
||||||
admin, "Site settings" has a *Retention days* field: set it to delete articles
|
admin, "Site settings" has a *Retention days* field: set it to delete articles
|
||||||
@@ -347,8 +361,12 @@ fetched longer ago than that, nightly (03:30 UTC, via Celery beat). Articles
|
|||||||
anyone has starred or saved for later are always kept; read state and tags for
|
anyone has starred or saved for later are always kept; read state and tags for
|
||||||
deleted articles go with them. A small record (feed and entry id) is kept for
|
deleted articles go with them. A small record (feed and entry id) is kept for
|
||||||
each deleted article so a feed that still lists it can't bring it back as new
|
each deleted article so a feed that still lists it can't bring it back as new
|
||||||
and unread, and entries whose own publish date is already older than the window
|
and unread — those records are themselves removed after a year, so an entry a
|
||||||
aren't fetched in the first place. The dashboard shows the article count and the
|
feed is *still* publishing a year after it was purged (with no publish date, or
|
||||||
|
a recent one) can come back as new. Entries whose own publish date is already
|
||||||
|
older than the window aren't fetched in the first place. A large first purge
|
||||||
|
works in time-boxed runs (about four minutes each) that queue their own
|
||||||
|
continuation, so it never monopolises a worker. The dashboard shows the article count and the
|
||||||
age of the oldest one, so you can see what retention is doing.
|
age of the oldest one, so you can see what retention is doing.
|
||||||
|
|
||||||
**SQLite.** SQLite installs run in WAL mode with a 20-second busy timeout (set
|
**SQLite.** SQLite installs run in WAL mode with a 20-second busy timeout (set
|
||||||
|
|||||||
Reference in new issue
Block a user