Private
Public Access
S4: the throttle keyed on the first X-Forwarded-For entry, which a client or a proxy that appends can choose, and disabled itself entirely when Redis was down. EVECTION_TRUSTED_PROXIES (IPs/CIDRs of your reverse proxy) makes it strict: X-Forwarded-For is only believed when the TCP peer is a trusted proxy, and the rightmost entry that isn't itself a trusted proxy — the address the proxy actually saw — is used instead of the spoofable first one. Unset keeps the previous behavior so existing installs don't suddenly share one bucket behind a proxy, and `manage.py check --deploy` warns (evection.W001) until it's set. During a Redis outage the throttle now falls back to a bounded in-process limiter instead of switching off. C5: /health/ready/ reuses its queue/feed aggregates for 15s (EVECTION_HEALTH_CACHE_SECONDS) so a fast-polling monitor can't create steady load; the database, Redis and poller probes stay live. Docs cover both settings, restricting the endpoint at the proxy, and the tombstone/purge behavior from the previous changes. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
aacb41d1d3
commit
ac5e951749
12 files changed
+386
-53
No files matched your search
@@ -2,6 +2,7 @@ from datetime import timedelta
|
||||
|
||||
import redis
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django.db.models import Max, Min
|
||||
from django.utils import timezone
|
||||
|
||||
@@ -42,6 +43,33 @@ def _success_window() -> timedelta:
|
||||
return timedelta(minutes=max(3 * base, MIN_SUCCESS_WINDOW_MINUTES))
|
||||
|
||||
|
||||
PIPELINE_CACHE_KEY = "health-ready-pipeline"
|
||||
|
||||
|
||||
def cached_fetch_pipeline_status() -> tuple[dict[str, bool], dict]:
|
||||
"""fetch_pipeline_status, reused for EVECTION_HEALTH_CACHE_SECONDS
|
||||
(R2-C5): the queue/feed aggregates behind it cost real queries, and a
|
||||
monitor that polls /health/ready/ every second shouldn't turn that
|
||||
into recurring database and Redis load. The cheap liveness probes
|
||||
(database, Redis, poll heartbeat) are not cached — those stay live.
|
||||
A cache failure just means computing it fresh."""
|
||||
ttl = settings.EVECTION_HEALTH_CACHE_SECONDS
|
||||
if ttl > 0:
|
||||
try:
|
||||
cached = cache.get(PIPELINE_CACHE_KEY)
|
||||
except Exception:
|
||||
cached = None
|
||||
if cached is not None:
|
||||
return cached
|
||||
result = fetch_pipeline_status()
|
||||
if ttl > 0:
|
||||
try:
|
||||
cache.set(PIPELINE_CACHE_KEY, result, timeout=ttl)
|
||||
except Exception:
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def fetch_pipeline_status(now=None) -> tuple[dict[str, bool], dict]:
|
||||
"""Checks that fetching is actually *succeeding*, not just scheduled
|
||||
(the poll heartbeat only proves beat and a worker are alive): returns
|
||||
|
||||
@@ -265,3 +265,45 @@ def test_health_ready_flags_most_feeds_failing_but_not_a_single_broken_one(clien
|
||||
assert body["checks"]["feed_failures"] is False
|
||||
assert body["failing_feeds"] == 3
|
||||
assert healthy.pk and broken
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_readiness_reuses_the_pipeline_aggregates_for_a_short_time(client, broker_queue, settings):
|
||||
"""R2-C5: the expensive part of the readiness check is cached briefly,
|
||||
while the cheap liveness probes stay live."""
|
||||
from django.core.cache import cache
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY
|
||||
|
||||
settings.EVECTION_HEALTH_CACHE_SECONDS = 30
|
||||
cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now())
|
||||
Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
||||
|
||||
first = client.get(reverse("core:health_ready")).json()
|
||||
Feed.objects.create(feed_url="https://b.example.com/feed.xml")
|
||||
second = client.get(reverse("core:health_ready")).json()
|
||||
|
||||
assert first["active_feeds"] == 1
|
||||
assert second["active_feeds"] == 1, "served from the short-lived cache"
|
||||
|
||||
cache.delete("health-ready-pipeline")
|
||||
third = client.get(reverse("core:health_ready")).json()
|
||||
assert third["active_feeds"] == 2
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_readiness_liveness_probes_are_never_cached(client, broker_queue, settings):
|
||||
from django.core.cache import cache
|
||||
|
||||
settings.EVECTION_HEALTH_CACHE_SECONDS = 30
|
||||
first = client.get(reverse("core:health_ready"))
|
||||
assert first.json()["checks"]["poll_heartbeat"] is False
|
||||
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY
|
||||
|
||||
cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now())
|
||||
second = client.get(reverse("core:health_ready"))
|
||||
assert second.json()["checks"]["poll_heartbeat"] is True
|
||||
+2
-2
@@ -12,7 +12,7 @@ from apps.articles.models import Article
|
||||
from apps.feeds.models import Feed, Subscription
|
||||
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, enqueue_fetch, reset_backoff
|
||||
|
||||
from .health import fetch_pipeline_status
|
||||
from .health import cached_fetch_pipeline_status
|
||||
|
||||
is_admin = user_passes_test(lambda u: u.is_authenticated and u.is_staff)
|
||||
|
||||
@@ -70,7 +70,7 @@ def health_ready(request):
|
||||
checks["poll_heartbeat"] = age <= POLL_HEARTBEAT_MAX_AGE_SECONDS
|
||||
|
||||
if checks["database"]:
|
||||
pipeline_checks, pipeline_detail = fetch_pipeline_status()
|
||||
pipeline_checks, pipeline_detail = cached_fetch_pipeline_status()
|
||||
checks.update(pipeline_checks)
|
||||
detail.update(pipeline_detail)
|
||||
|
||||
|
||||
Reference in new issue
Block a user