Private
Public Access
S4: the throttle keyed on the first X-Forwarded-For entry, which a client or a proxy that appends can choose, and disabled itself entirely when Redis was down. EVECTION_TRUSTED_PROXIES (IPs/CIDRs of your reverse proxy) makes it strict: X-Forwarded-For is only believed when the TCP peer is a trusted proxy, and the rightmost entry that isn't itself a trusted proxy — the address the proxy actually saw — is used instead of the spoofable first one. Unset keeps the previous behavior so existing installs don't suddenly share one bucket behind a proxy, and `manage.py check --deploy` warns (evection.W001) until it's set. During a Redis outage the throttle now falls back to a bounded in-process limiter instead of switching off. C5: /health/ready/ reuses its queue/feed aggregates for 15s (EVECTION_HEALTH_CACHE_SECONDS) so a fast-polling monitor can't create steady load; the database, Redis and poller probes stay live. Docs cover both settings, restricting the endpoint at the proxy, and the tombstone/purge behavior from the previous changes. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
310 lines
10 KiB
Python
310 lines
10 KiB
Python
from unittest.mock import patch
|
|
|
|
import pytest
|
|
from django.urls import reverse
|
|
|
|
from apps.feeds.models import Feed, Subscription
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_health_check_returns_ok_without_login(client):
|
|
response = client.get(reverse("core:health"))
|
|
assert response.status_code == 200
|
|
assert response.json() == {"status": "ok"}
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_health_check_works_before_first_run_setup(client):
|
|
# No User exists yet, so SetupWizardMiddleware would normally redirect
|
|
# every request to /setup/ — scripts/upgrade.sh needs this reachable
|
|
# immediately after a fresh deploy, before anyone's created an account.
|
|
response = client.get(reverse("core:health"))
|
|
assert response.status_code == 200
|
|
|
|
|
|
@pytest.fixture
|
|
def broker_queue(monkeypatch):
|
|
"""Stands in for the Redis broker's queue length (tests have no Redis):
|
|
set `broker_queue["depth"]` to a number, or None for "unreachable"."""
|
|
state = {"depth": 0}
|
|
monkeypatch.setattr("apps.core.health.queue_depth", lambda: state["depth"])
|
|
return state
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_health_ready_is_degraded_until_a_poll_tick_has_happened(client, broker_queue):
|
|
response = client.get(reverse("core:health_ready"))
|
|
assert response.status_code == 503
|
|
body = response.json()
|
|
assert body["status"] == "degraded"
|
|
assert body["checks"]["database"] is True
|
|
assert body["checks"]["redis"] is True
|
|
assert body["checks"]["poll_heartbeat"] is False
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_health_ready_is_ok_after_poll_due_feeds_runs(client, broker_queue):
|
|
from apps.feeds.tasks import poll_due_feeds
|
|
|
|
poll_due_feeds()
|
|
|
|
response = client.get(reverse("core:health_ready"))
|
|
assert response.status_code == 200
|
|
checks = response.json()["checks"]
|
|
assert checks == {
|
|
"database": True,
|
|
"redis": True,
|
|
"poll_heartbeat": True,
|
|
"queue_depth": True,
|
|
"recent_fetch_success": True,
|
|
"feed_failures": True,
|
|
}
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_health_ready_flags_a_stale_poll_heartbeat(client, broker_queue):
|
|
from datetime import timedelta
|
|
|
|
from django.core.cache import cache
|
|
from django.utils import timezone
|
|
|
|
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY
|
|
|
|
cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now() - timedelta(minutes=30))
|
|
|
|
response = client.get(reverse("core:health_ready"))
|
|
assert response.status_code == 503
|
|
assert response.json()["checks"]["poll_heartbeat"] is False
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_dashboard_requires_staff(client, django_user_model):
|
|
django_user_model.objects.create_user("regular", password="pw-regular-123")
|
|
client.login(username="regular", password="pw-regular-123")
|
|
|
|
response = client.get(reverse("core:dashboard"))
|
|
assert response.status_code == 302
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_dashboard_lists_erroring_feeds(client, django_user_model):
|
|
admin = django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True)
|
|
client.login(username="admin", password="pw-admin-123")
|
|
|
|
ok_feed = Feed.objects.create(feed_url="https://ok.example.com/feed", title="OK Feed")
|
|
broken_feed = Feed.objects.create(
|
|
feed_url="https://broken.example.com/feed",
|
|
title="Broken Feed",
|
|
last_fetch_status=Feed.STATUS_ERROR,
|
|
fetch_error_count=3,
|
|
last_error_message="Connection timed out",
|
|
)
|
|
Subscription.objects.create(user=admin, feed=ok_feed)
|
|
Subscription.objects.create(user=admin, feed=broken_feed)
|
|
|
|
response = client.get(reverse("core:dashboard"))
|
|
assert response.status_code == 200
|
|
assert b"Broken Feed" in response.content
|
|
assert b"Connection timed out" in response.content
|
|
# "OK Feed" legitimately appears elsewhere on the page (the sidebar
|
|
# lists every subscription), so check the dashboard's own error table
|
|
# scope via context rather than raw page content.
|
|
assert list(response.context["erroring_feeds"]) == [broken_feed]
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_retry_feed_resets_backoff_then_claims_and_queues_one_fetch(client, django_user_model):
|
|
django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True)
|
|
client.login(username="admin", password="pw-admin-123")
|
|
|
|
feed = Feed.objects.create(
|
|
feed_url="https://broken.example.com/feed",
|
|
fetch_error_count=5,
|
|
last_fetch_status=Feed.STATUS_ERROR,
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_fetch:
|
|
response = client.post(reverse("core:retry_feed", args=[feed.id]))
|
|
|
|
assert response.status_code == 302
|
|
mock_fetch.assert_called_once_with(feed.id)
|
|
feed.refresh_from_db()
|
|
assert feed.fetch_error_count == 0
|
|
assert feed.claimed_at is not None, "the fetch is claimed so the poller can't also take it"
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_retry_feed_does_not_queue_a_second_fetch_while_one_is_in_flight(client, django_user_model):
|
|
from django.utils import timezone
|
|
|
|
django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True)
|
|
client.login(username="admin", password="pw-admin-123")
|
|
feed = Feed.objects.create(feed_url="https://busy.example.com/feed", claimed_at=timezone.now())
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_fetch:
|
|
client.post(reverse("core:retry_feed", args=[feed.id]))
|
|
|
|
mock_fetch.assert_not_called()
|
|
|
|
|
|
def _ready(client):
|
|
from django.core.cache import cache
|
|
from django.utils import timezone
|
|
|
|
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY
|
|
|
|
# Record the heartbeat directly: running poll_due_feeds here would
|
|
# execute fetches eagerly (CELERY_TASK_ALWAYS_EAGER) and rewrite the
|
|
# feed states each test sets up.
|
|
cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now())
|
|
response = client.get(reverse("core:health_ready"))
|
|
return response.status_code, response.json()
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_health_ready_flags_a_backed_up_queue(client, broker_queue):
|
|
from .health import MAX_QUEUE_DEPTH
|
|
|
|
broker_queue["depth"] = MAX_QUEUE_DEPTH + 1
|
|
|
|
status, body = _ready(client)
|
|
|
|
assert status == 503
|
|
assert body["checks"]["queue_depth"] is False
|
|
assert body["queue_depth"] == MAX_QUEUE_DEPTH + 1
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_health_ready_flags_an_unreachable_broker_queue(client, broker_queue):
|
|
broker_queue["depth"] = None
|
|
|
|
status, body = _ready(client)
|
|
|
|
assert status == 503
|
|
assert body["checks"]["queue_depth"] is False
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_health_ready_flags_no_recent_successful_fetch(client, broker_queue):
|
|
from datetime import timedelta
|
|
|
|
from django.utils import timezone
|
|
|
|
feed = Feed.objects.create(
|
|
feed_url="https://a.example.com/feed.xml",
|
|
last_fetch_status=Feed.STATUS_OK,
|
|
last_fetched_at=timezone.now() - timedelta(days=2),
|
|
)
|
|
Feed.objects.filter(pk=feed.pk).update(created_at=timezone.now() - timedelta(days=30))
|
|
|
|
status, body = _ready(client)
|
|
|
|
assert status == 503
|
|
assert body["checks"]["recent_fetch_success"] is False
|
|
assert body["last_successful_fetch_age_seconds"] > 86400
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_health_ready_is_ok_with_a_recent_successful_fetch(client, broker_queue):
|
|
from django.utils import timezone
|
|
|
|
Feed.objects.create(
|
|
feed_url="https://a.example.com/feed.xml",
|
|
last_fetch_status=Feed.STATUS_NOT_MODIFIED,
|
|
last_fetched_at=timezone.now(),
|
|
)
|
|
|
|
status, body = _ready(client)
|
|
|
|
assert status == 200
|
|
assert body["active_feeds"] == 1
|
|
assert body["failing_feeds"] == 0
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_health_ready_gives_a_brand_new_feed_time_before_it_counts_as_stale(client, broker_queue):
|
|
Feed.objects.create(feed_url="https://new.example.com/feed.xml")
|
|
|
|
status, body = _ready(client)
|
|
|
|
assert status == 200
|
|
assert body["checks"]["recent_fetch_success"] is True
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_health_ready_flags_most_feeds_failing_but_not_a_single_broken_one(client, broker_queue):
|
|
from django.utils import timezone
|
|
|
|
now = timezone.now()
|
|
healthy = Feed.objects.create(
|
|
feed_url="https://ok.example.com/feed.xml",
|
|
last_fetch_status=Feed.STATUS_OK,
|
|
last_fetched_at=now,
|
|
)
|
|
broken = [
|
|
Feed.objects.create(
|
|
feed_url=f"https://bad{i}.example.com/feed.xml",
|
|
last_fetch_status=Feed.STATUS_ERROR,
|
|
fetch_error_count=3,
|
|
last_fetched_at=now,
|
|
)
|
|
for i in range(2)
|
|
]
|
|
|
|
status, body = _ready(client)
|
|
assert status == 200, "2 of 3 failing is below the minimum feed count for the ratio"
|
|
|
|
Feed.objects.create(
|
|
feed_url="https://bad9.example.com/feed.xml",
|
|
last_fetch_status=Feed.STATUS_ERROR,
|
|
fetch_error_count=1,
|
|
last_fetched_at=now,
|
|
)
|
|
status, body = _ready(client)
|
|
assert status == 503
|
|
assert body["checks"]["feed_failures"] is False
|
|
assert body["failing_feeds"] == 3
|
|
assert healthy.pk and broken
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_readiness_reuses_the_pipeline_aggregates_for_a_short_time(client, broker_queue, settings):
|
|
"""R2-C5: the expensive part of the readiness check is cached briefly,
|
|
while the cheap liveness probes stay live."""
|
|
from django.core.cache import cache
|
|
from django.utils import timezone
|
|
|
|
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY
|
|
|
|
settings.EVECTION_HEALTH_CACHE_SECONDS = 30
|
|
cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now())
|
|
Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
|
|
|
first = client.get(reverse("core:health_ready")).json()
|
|
Feed.objects.create(feed_url="https://b.example.com/feed.xml")
|
|
second = client.get(reverse("core:health_ready")).json()
|
|
|
|
assert first["active_feeds"] == 1
|
|
assert second["active_feeds"] == 1, "served from the short-lived cache"
|
|
|
|
cache.delete("health-ready-pipeline")
|
|
third = client.get(reverse("core:health_ready")).json()
|
|
assert third["active_feeds"] == 2
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_readiness_liveness_probes_are_never_cached(client, broker_queue, settings):
|
|
from django.core.cache import cache
|
|
|
|
settings.EVECTION_HEALTH_CACHE_SECONDS = 30
|
|
first = client.get(reverse("core:health_ready"))
|
|
assert first.json()["checks"]["poll_heartbeat"] is False
|
|
|
|
from django.utils import timezone
|
|
|
|
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY
|
|
|
|
cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now())
|
|
second = client.get(reverse("core:health_ready"))
|
|
assert second.json()["checks"]["poll_heartbeat"] is True
|