Batch feed ingestion, add a readiness check, fail open without Redis (C2, C8)

Ingestion did a get_or_create per entry. It now prefetches existing
articles for the poll in one query, bulk-inserts new ones and
bulk-updates changed ones (falling back to row-by-row only if a
concurrent fetch wins a race), and dispatches rule evaluation in
bounded chunks so a huge first fetch is several small tasks.

/health/ stays a DB-only liveness check (upgrade.sh gates on it right
after restart). New /health/ready/ also checks Redis and that
poll_due_feeds ticked within five minutes — proving beat and a worker
are both alive — and returns 503 with per-check detail otherwise.

The sidebar unread-count cache and login throttle now degrade instead
of erroring when Redis is unreachable: every authenticated page and
every login would otherwise 500 during a cache outage. Found while
verifying in a sandbox with no Redis.

Also includes the dashboard's oldest-article metric (core/views.py).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5.5 committed 2026-11-01 08:51:20 -07:00
1 parent 8d2900424a
commit e5f1277014
14 files changed
+342 -36

No files matched your search

+1
View File
@@ -45,4 +45,5 @@ def _setup_exempt_paths():
reverse("pwa:service_worker"), reverse("pwa:service_worker"),
reverse("pwa:offline"), reverse("pwa:offline"),
reverse("core:health"), reverse("core:health"),
reverse("core:health_ready"),
} }
+18
View File
@@ -72,3 +72,21 @@ def test_too_many_failed_attempts_against_one_username_blocks_that_username(clie
response = client.post(reverse("accounts:login"), {"username": "bob", "password": "pw-bob-123"}) response = client.post(reverse("accounts:login"), {"username": "bob", "password": "pw-bob-123"})
assert response.status_code == 302 assert response.status_code == 302
assert response.wsgi_request.user == bob assert response.wsgi_request.user == bob
@pytest.mark.django_db
def test_login_still_works_when_the_throttle_cache_is_down(client, monkeypatch):
from unittest.mock import Mock
from django.contrib.auth import get_user_model
get_user_model().objects.create_user("alice", password="pw-alice-123")
broken = Mock(side_effect=ConnectionError("redis down"))
monkeypatch.setattr("apps.accounts.views.cache.get", broken)
monkeypatch.setattr("apps.accounts.views.cache.incr", broken)
monkeypatch.setattr("apps.accounts.views.cache.set", broken)
bad = client.post(reverse("accounts:login"), {"username": "alice", "password": "wrong"})
assert bad.status_code == 200
good = client.post(reverse("accounts:login"), {"username": "alice", "password": "pw-alice-123"})
assert good.status_code == 302
+14
View File
@@ -1,3 +1,5 @@
import logging
from django.contrib import messages from django.contrib import messages
from django.contrib.auth import login from django.contrib.auth import login
from django.contrib.auth.decorators import user_passes_test from django.contrib.auth.decorators import user_passes_test
@@ -10,6 +12,8 @@ from .forms import CreateUserForm, SetupForm
from .middleware import SetupWizardMiddleware from .middleware import SetupWizardMiddleware
from .models import SetupLock, User from .models import SetupLock, User
logger = logging.getLogger(__name__)
is_admin = user_passes_test(lambda u: u.is_authenticated and u.is_staff) is_admin = user_passes_test(lambda u: u.is_authenticated and u.is_staff)
# Two independent throttles, either of which can block an attempt: by # Two independent throttles, either of which can block an attempt: by
@@ -41,14 +45,24 @@ def _attempts_key(kind: str, value: str) -> str:
def _too_many_attempts(key: str, limit: int) -> bool: def _too_many_attempts(key: str, limit: int) -> bool:
# Fails open if the cache (Redis) is unreachable: logging in must keep
# working through a cache outage, at the cost of no throttling until
# it's back — an outage shouldn't lock every user out of the app.
try:
return cache.get(key, 0) >= limit return cache.get(key, 0) >= limit
except Exception:
logger.warning("Login throttle cache unavailable; not rate limiting", exc_info=True)
return False
def _record_failed_attempt(key: str) -> None: def _record_failed_attempt(key: str) -> None:
try:
try: try:
cache.incr(key) cache.incr(key)
except ValueError: except ValueError:
cache.set(key, 1, LOGIN_RATE_LIMIT_WINDOW_SECONDS) cache.set(key, 1, LOGIN_RATE_LIMIT_WINDOW_SECONDS)
except Exception:
logger.warning("Login throttle cache unavailable; attempt not recorded", exc_info=True)
class LoginView(DjangoLoginView): class LoginView(DjangoLoginView):
+14
View File
@@ -1,3 +1,5 @@
import logging
from django.core.cache import cache from django.core.cache import cache
from django.db.models import Count from django.db.models import Count
@@ -5,6 +7,8 @@ from apps.core.cache_utils import SIDEBAR_UNREAD_CACHE_TTL_SECONDS, sidebar_unre
from .models import SAVED_TAG_NAME, STARRED_TAG_NAME, Article, ArticleTag, UserArticleState from .models import SAVED_TAG_NAME, STARRED_TAG_NAME, Article, ArticleTag, UserArticleState
logger = logging.getLogger(__name__)
def with_user_state(articles, user): def with_user_state(articles, user):
"""Annotates an iterable of Article instances with is_read/is_starred/ """Annotates an iterable of Article instances with is_read/is_starred/
@@ -56,7 +60,14 @@ def unread_counts_by_feed(user):
the current user's own read state, so this is rarely visibly stale. the current user's own read state, so this is rarely visibly stale.
""" """
cache_key = sidebar_unread_cache_key(user.pk) cache_key = sidebar_unread_cache_key(user.pk)
# The cache is an optimization here, never a dependency: every
# authenticated page render goes through this, so a Redis outage must
# degrade to recomputing the count, not to a 500 on every page.
try:
counts = cache.get(cache_key) counts = cache.get(cache_key)
except Exception:
logger.warning("Sidebar cache unavailable; computing unread counts directly")
counts = None
if counts is not None: if counts is not None:
return counts return counts
@@ -68,5 +79,8 @@ def unread_counts_by_feed(user):
.annotate(unread=Count("id")) .annotate(unread=Count("id"))
) )
counts = {row["feed_id"]: row["unread"] for row in rows} counts = {row["feed_id"]: row["unread"] for row in rows}
try:
cache.set(cache_key, counts, SIDEBAR_UNREAD_CACHE_TTL_SECONDS) cache.set(cache_key, counts, SIDEBAR_UNREAD_CACHE_TTL_SECONDS)
except Exception:
logger.warning("Sidebar cache unavailable; skipping cache write")
return counts return counts
+16
View File
@@ -58,3 +58,19 @@ def test_unread_counts_by_feed_cache_is_per_user(alice, feed):
assert unread_counts_by_feed(alice) == {feed.pk: 1} assert unread_counts_by_feed(alice) == {feed.pk: 1}
assert unread_counts_by_feed(bob) == {feed.pk: 1} assert unread_counts_by_feed(bob) == {feed.pk: 1}
@pytest.mark.django_db
def test_unread_counts_still_work_when_the_cache_is_down(alice, feed, monkeypatch):
"""A Redis outage must degrade the sidebar to recomputing counts, not
500 every authenticated page."""
from unittest.mock import Mock
Subscription.objects.create(user=alice, feed=feed)
Article.objects.create(feed=feed, guid="a-1")
broken = Mock(side_effect=ConnectionError("redis down"))
monkeypatch.setattr("apps.articles.queries.cache.get", broken)
monkeypatch.setattr("apps.articles.queries.cache.set", broken)
assert unread_counts_by_feed(alice) == {feed.pk: 1}
invalidate_sidebar_unread_cache(alice.pk)
+7
View File
@@ -1,5 +1,9 @@
import logging
from django.core.cache import cache from django.core.cache import cache
logger = logging.getLogger(__name__)
# Short enough that a missed invalidation point self-heals within one # Short enough that a missed invalidation point self-heals within one
# page load's worth of staleness; long enough to actually absorb repeat # page load's worth of staleness; long enough to actually absorb repeat
# authenticated-page renders (apps.core.context_processors.sidebar runs # authenticated-page renders (apps.core.context_processors.sidebar runs
@@ -19,4 +23,7 @@ def invalidate_sidebar_unread_cache(user_id: int) -> None:
out the TTL above. Rule-driven state changes during background feed out the TTL above. Rule-driven state changes during background feed
ingestion aren't invalidated this way — those self-heal within the ingestion aren't invalidated this way — those self-heal within the
TTL, same as a missed call site here would.""" TTL, same as a missed call site here would."""
try:
cache.delete(sidebar_unread_cache_key(user_id)) cache.delete(sidebar_unread_cache_key(user_id))
except Exception:
logger.warning("Sidebar cache unavailable; nothing to invalidate")
+42
View File
@@ -22,6 +22,48 @@ def test_health_check_works_before_first_run_setup(client):
assert response.status_code == 200 assert response.status_code == 200
@pytest.mark.django_db
def test_health_ready_is_degraded_until_a_poll_tick_has_happened(client):
response = client.get(reverse("core:health_ready"))
assert response.status_code == 503
body = response.json()
assert body["status"] == "degraded"
assert body["checks"]["database"] is True
assert body["checks"]["redis"] is True
assert body["checks"]["poll_heartbeat"] is False
@pytest.mark.django_db
def test_health_ready_is_ok_after_poll_due_feeds_runs(client):
from apps.feeds.tasks import poll_due_feeds
poll_due_feeds()
response = client.get(reverse("core:health_ready"))
assert response.status_code == 200
assert response.json()["checks"] == {
"database": True,
"redis": True,
"poll_heartbeat": True,
}
@pytest.mark.django_db
def test_health_ready_flags_a_stale_poll_heartbeat(client):
from datetime import timedelta
from django.core.cache import cache
from django.utils import timezone
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY
cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now() - timedelta(minutes=30))
response = client.get(reverse("core:health_ready"))
assert response.status_code == 503
assert response.json()["checks"]["poll_heartbeat"] is False
@pytest.mark.django_db @pytest.mark.django_db
def test_dashboard_requires_staff(client, django_user_model): def test_dashboard_requires_staff(client, django_user_model):
django_user_model.objects.create_user("regular", password="pw-regular-123") django_user_model.objects.create_user("regular", password="pw-regular-123")
+1
View File
@@ -6,6 +6,7 @@ app_name = "core"
urlpatterns = [ urlpatterns = [
path("health/", views.health, name="health"), path("health/", views.health, name="health"),
path("health/ready/", views.health_ready, name="health_ready"),
path("dashboard/", views.dashboard, name="dashboard"), path("dashboard/", views.dashboard, name="dashboard"),
path("dashboard/feeds/<int:feed_id>/retry/", views.retry_feed, name="retry_feed"), path("dashboard/feeds/<int:feed_id>/retry/", views.retry_feed, name="retry_feed"),
] ]
+50 -3
View File
@@ -1,20 +1,27 @@
from django.contrib import messages from django.contrib import messages
from django.contrib.auth.decorators import user_passes_test from django.contrib.auth.decorators import user_passes_test
from django.core.cache import cache
from django.db import connection from django.db import connection
from django.db.models import Count, Q from django.db.models import Count, Min, Q
from django.http import JsonResponse from django.http import JsonResponse
from django.shortcuts import get_object_or_404, redirect, render from django.shortcuts import get_object_or_404, redirect, render
from django.utils import timezone
from django.views.decorators.http import require_POST from django.views.decorators.http import require_POST
from apps.articles.models import Article from apps.articles.models import Article
from apps.feeds.models import Feed, Subscription from apps.feeds.models import Feed, Subscription
from apps.feeds.tasks import fetch_feed, reset_feed_backoff from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, fetch_feed, reset_feed_backoff
is_admin = user_passes_test(lambda u: u.is_authenticated and u.is_staff) is_admin = user_passes_test(lambda u: u.is_authenticated and u.is_staff)
# Beat ticks poll_due_feeds every minute; this many minutes without a tick
# means beat or the worker (or the broker between them) isn't working.
POLL_HEARTBEAT_MAX_AGE_SECONDS = 5 * 60
def health(request): def health(request):
"""Unauthenticated liveness/readiness check — reachable even before """Unauthenticated liveness check — reachable even before
first-run setup (see SetupWizardMiddleware). Used by scripts/upgrade.sh first-run setup (see SetupWizardMiddleware). Used by scripts/upgrade.sh
to confirm a freshly restarted deploy actually came up, and is a to confirm a freshly restarted deploy actually came up, and is a
reasonable target for any external uptime monitor too.""" reasonable target for any external uptime monitor too."""
@@ -26,6 +33,45 @@ def health(request):
return JsonResponse({"status": "ok"}) return JsonResponse({"status": "ok"})
def health_ready(request):
"""Readiness: liveness plus everything background fetching depends on
(C8) — the database, the Redis cache/broker, and a recent
poll_due_feeds tick, which proves beat is scheduling AND a worker is
consuming. /health/ above stays DB-only on purpose: scripts/upgrade.sh
gates a deploy on it right after restart, before beat has had a
chance to tick. Point an uptime monitor at this one instead."""
checks = {"database": False, "redis": False, "poll_heartbeat": False}
detail = {}
try:
with connection.cursor() as cursor:
cursor.execute("SELECT 1")
checks["database"] = True
except Exception:
pass
try:
cache.set("health-ready-probe", "1", timeout=10)
checks["redis"] = cache.get("health-ready-probe") == "1"
except Exception:
pass
try:
last_tick = cache.get(POLL_HEARTBEAT_CACHE_KEY)
except Exception:
last_tick = None
if last_tick is not None:
age = (timezone.now() - last_tick).total_seconds()
detail["poll_heartbeat_age_seconds"] = int(age)
checks["poll_heartbeat"] = age <= POLL_HEARTBEAT_MAX_AGE_SECONDS
ok = all(checks.values())
return JsonResponse(
{"status": "ok" if ok else "degraded", "checks": checks, **detail},
status=200 if ok else 503,
)
@is_admin @is_admin
def dashboard(request): def dashboard(request):
feeds = Feed.objects.annotate(subscriber_count=Count("subscriptions")) feeds = Feed.objects.annotate(subscriber_count=Count("subscriptions"))
@@ -39,6 +85,7 @@ def dashboard(request):
"error_feed_count": erroring_feeds.count(), "error_feed_count": erroring_feeds.count(),
"subscription_count": Subscription.objects.count(), "subscription_count": Subscription.objects.count(),
"article_count": Article.objects.count(), "article_count": Article.objects.count(),
"oldest_article_at": Article.objects.aggregate(oldest=Min("fetched_at"))["oldest"],
} }
return render( return render(
+69 -31
View File
@@ -18,6 +18,7 @@ import feedparser
import nh3 import nh3
import requests import requests
from django.conf import settings from django.conf import settings
from django.db import IntegrityError, transaction
from django.utils import timezone from django.utils import timezone
from apps.articles.models import Article from apps.articles.models import Article
@@ -131,6 +132,26 @@ def _entry_content_html(entry) -> str:
return getattr(entry, "summary", "") or "" return getattr(entry, "summary", "") or ""
UPDATABLE_ARTICLE_FIELDS = (
"title",
"link",
"author",
"summary",
"content_html",
"published_at",
"content_hash",
)
def _ensure_pks(feed: Feed, articles: list[Article]) -> list[int]:
"""bulk_create only populates primary keys on backends that can
RETURN them (Postgres, SQLite 3.35+); look the rest up by guid."""
if all(a.pk for a in articles):
return [a.pk for a in articles]
guids = [a.guid for a in articles]
return list(Article.objects.filter(feed=feed, guid__in=guids).values_list("pk", flat=True))
def effective_interval_minutes(feed: Feed, configured_interval_minutes: int) -> int: def effective_interval_minutes(feed: Feed, configured_interval_minutes: int) -> int:
"""The actual polling interval to use for `feed`, applying exponential """The actual polling interval to use for `feed`, applying exponential
backoff on top of the site-wide `configured_interval_minutes` (see backoff on top of the site-wide `configured_interval_minutes` (see
@@ -217,52 +238,69 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
) )
entries = entries[:MAX_ENTRIES_PER_POLL] entries = entries[:MAX_ENTRIES_PER_POLL]
# One prefetch + bulk writes for the whole poll (C2), instead of a
# get_or_create (SELECT, then INSERT) per entry. Entries are keyed by
# guid so a feed that repeats one within a single response is only
# processed once.
parsed_entries: dict[str, dict] = {}
for entry in entries: for entry in entries:
guid = _entry_guid(entry) guid = _entry_guid(entry)
if guid in parsed_entries:
continue
title = (getattr(entry, "title", "") or "")[:1000] title = (getattr(entry, "title", "") or "")[:1000]
link = (getattr(entry, "link", "") or "")[:2000] link = (getattr(entry, "link", "") or "")[:2000]
author = (getattr(entry, "author", "") or "")[:500] author = (getattr(entry, "author", "") or "")[:500]
summary = (getattr(entry, "summary", "") or "")[:5000] summary = (getattr(entry, "summary", "") or "")[:5000]
content_html = sanitize_html(_entry_content_html(entry)) content_html = sanitize_html(_entry_content_html(entry))
published_at = _entry_published_at(entry) published_at = _entry_published_at(entry)
content_hash = _content_hash(title, link, author, summary, content_html, published_at) parsed_entries[guid] = {
article, created = Article.objects.get_or_create(
feed=feed,
guid=guid,
defaults={
"link": link,
"title": title, "title": title,
"link": link,
"author": author, "author": author,
"content_html": content_html,
"summary": summary, "summary": summary,
"content_hash": content_hash, "content_html": content_html,
"published_at": published_at, "published_at": published_at,
}, "content_hash": _content_hash(title, link, author, summary, content_html, published_at),
) }
if created: existing = {a.guid: a for a in Article.objects.filter(feed=feed, guid__in=parsed_entries)}
new_ids.append(article.pk)
elif article.content_hash != content_hash: to_create = [
article.title = title Article(feed=feed, guid=guid, **fields)
article.link = link for guid, fields in parsed_entries.items()
article.author = author if guid not in existing
article.summary = summary
article.content_html = content_html
article.published_at = published_at
article.content_hash = content_hash
article.save(
update_fields=[
"title",
"link",
"author",
"summary",
"content_html",
"published_at",
"content_hash",
] ]
to_update = []
for guid, article in existing.items():
fields = parsed_entries[guid]
if article.content_hash != fields["content_hash"]:
for name, value in fields.items():
setattr(article, name, value)
to_update.append(article)
if to_update:
Article.objects.bulk_update(to_update, list(UPDATABLE_ARTICLE_FIELDS), batch_size=100)
updated_ids.extend(a.pk for a in to_update)
if to_create:
try:
with transaction.atomic():
Article.objects.bulk_create(to_create)
except IntegrityError:
# Another worker inserted some of these guids between the
# prefetch above and now (two fetches of the same feed can
# overlap) — fall back to row-by-row so nothing is lost or
# duplicated, at the cost of the old per-entry round trips
# for this rare case only.
for article in to_create:
fields = {name: getattr(article, name) for name in UPDATABLE_ARTICLE_FIELDS}
obj, created = Article.objects.get_or_create(
feed=feed, guid=article.guid, defaults=fields
) )
updated_ids.append(article.pk) if created:
new_ids.append(obj.pk)
else:
new_ids.extend(_ensure_pks(feed, to_create))
return FetchResult( return FetchResult(
status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids
+15 -1
View File
@@ -4,6 +4,7 @@ from io import BytesIO
from celery import shared_task from celery import shared_task
from django.contrib.auth import get_user_model from django.contrib.auth import get_user_model
from django.core.cache import cache
from django.db.models import Q from django.db.models import Q
from django.utils import timezone from django.utils import timezone
@@ -27,6 +28,12 @@ CLAIM_STALE_MINUTES = 10
# across several beat ticks instead of enqueuing everything — and every # across several beat ticks instead of enqueuing everything — and every
# bounded Feed query below stays index-only work, not a full scan (C1). # bounded Feed query below stays index-only work, not a full scan (C1).
POLL_BATCH_SIZE = 500 POLL_BATCH_SIZE = 500
RULE_DISPATCH_CHUNK_SIZE = 50
# Written on every poll_due_feeds tick; apps.core.views.health_ready
# reads it to tell whether beat is scheduling and a worker is consuming.
POLL_HEARTBEAT_CACHE_KEY = "poll-due-feeds-heartbeat"
POLL_HEARTBEAT_TTL_SECONDS = 3600
@shared_task @shared_task
@@ -39,6 +46,7 @@ def poll_due_feeds():
here, not via one Celery Beat entry per feed. here, not via one Celery Beat entry per feed.
""" """
now = timezone.now() now = timezone.now()
cache.set(POLL_HEARTBEAT_CACHE_KEY, now, timeout=POLL_HEARTBEAT_TTL_SECONDS)
claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES) claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
due_feed_ids = list( due_feed_ids = list(
Feed.objects.filter(is_active=True) Feed.objects.filter(is_active=True)
@@ -125,7 +133,13 @@ def fetch_feed(feed_id: int):
len(result.new_article_ids), len(result.new_article_ids),
len(result.updated_article_ids), len(result.updated_article_ids),
) )
apply_filter_rules_for_feed.delay(feed_id, changed_article_ids) # Chunked so one huge poll (a first fetch of a long archive feed)
# becomes several bounded rule-evaluation tasks, each well inside
# the worker's time limits, instead of one unbounded one (C2).
for start in range(0, len(changed_article_ids), RULE_DISPATCH_CHUNK_SIZE):
apply_filter_rules_for_feed.delay(
feed_id, changed_article_ids[start : start + RULE_DISPATCH_CHUNK_SIZE]
)
return { return {
"status": result.status, "status": result.status,
+74
View File
@@ -264,3 +264,77 @@ def test_fetch_only_processes_the_first_max_entries_per_poll(feed):
assert len(result.new_article_ids) == MAX_ENTRIES_PER_POLL assert len(result.new_article_ids) == MAX_ENTRIES_PER_POLL
assert Article.objects.filter(feed=feed).count() == MAX_ENTRIES_PER_POLL assert Article.objects.filter(feed=feed).count() == MAX_ENTRIES_PER_POLL
assert not Article.objects.filter(feed=feed, guid=f"guid-{MAX_ENTRIES_PER_POLL}").exists() assert not Article.objects.filter(feed=feed, guid=f"guid-{MAX_ENTRIES_PER_POLL}").exists()
def _feed_xml(count, title_suffix=""):
items = "".join(
f"<item><title>Item {i}{title_suffix}</title><guid>guid-{i}</guid></item>"
for i in range(count)
)
return (
f'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>{items}</channel></rss>'
)
@pytest.mark.django_db
@responses.activate
def test_ingest_uses_a_constant_number_of_article_queries(feed):
"""C2: one prefetch + bulk writes per poll, not a get_or_create per
entry — query count must not grow with the number of entries."""
from django.db import connection
from django.test.utils import CaptureQueriesContext
responses.add(responses.GET, FEED_URL, body=_feed_xml(40), status=200)
with CaptureQueriesContext(connection) as ctx:
result = fetch_and_ingest(feed)
assert len(result.new_article_ids) == 40
assert len(ctx.captured_queries) < 10
assert Article.objects.filter(feed=feed).count() == 40
@pytest.mark.django_db
@responses.activate
def test_ingest_dedups_a_guid_repeated_within_one_response(feed):
xml = (
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
"<item><title>A</title><guid>same</guid></item>"
"<item><title>B</title><guid>same</guid></item>"
"</channel></rss>"
)
responses.add(responses.GET, FEED_URL, body=xml, status=200)
result = fetch_and_ingest(feed)
assert len(result.new_article_ids) == 1
assert Article.objects.filter(feed=feed).count() == 1
@pytest.mark.django_db
@responses.activate
def test_ingest_reports_updates_separately_from_new_articles(feed):
responses.add(responses.GET, FEED_URL, body=_feed_xml(3), status=200)
fetch_and_ingest(feed)
responses.replace(responses.GET, FEED_URL, body=_feed_xml(4, " edited"), status=200)
result = fetch_and_ingest(feed)
assert len(result.new_article_ids) == 1
assert len(result.updated_article_ids) == 3
assert Article.objects.get(feed=feed, guid="guid-0").title == "Item 0 edited"
@pytest.mark.django_db
@responses.activate
def test_ingest_falls_back_to_row_by_row_when_a_concurrent_insert_wins(feed):
from unittest.mock import patch
from django.db import IntegrityError
responses.add(responses.GET, FEED_URL, body=_feed_xml(3), status=200)
with patch.object(Article.objects, "bulk_create", side_effect=IntegrityError("race")):
result = fetch_and_ingest(feed)
assert len(result.new_article_ids) == 3
assert Article.objects.filter(feed=feed).count() == 3
+20
View File
@@ -208,3 +208,23 @@ def test_fetch_feed_schedules_the_next_fetch_using_the_site_wide_interval():
assert feed.next_fetch_at is not None assert feed.next_fetch_at is not None
expected = before + timedelta(minutes=120) expected = before + timedelta(minutes=120)
assert abs((feed.next_fetch_at - expected).total_seconds()) < 5 assert abs((feed.next_fetch_at - expected).total_seconds()) < 5
@pytest.mark.django_db
@responses.activate
def test_fetch_feed_dispatches_rule_evaluation_in_bounded_chunks(monkeypatch):
monkeypatch.setattr("apps.feeds.tasks.RULE_DISPATCH_CHUNK_SIZE", 2)
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
items = "".join(f"<item><title>T{i}</title><guid>g{i}</guid></item>" for i in range(5))
responses.add(
responses.GET,
feed.feed_url,
body=f'<?xml version="1.0"?><rss version="2.0"><channel>{items}</channel></rss>',
status=200,
)
with patch("apps.feeds.tasks.apply_filter_rules_for_feed.delay") as mock_delay:
fetch_feed(feed.id)
chunk_sizes = [len(call.args[1]) for call in mock_delay.call_args_list]
assert chunk_sizes == [2, 2, 1]
+1 -1
View File
@@ -36,7 +36,7 @@ SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
# over loopback (see ALLOWED_HOSTS above) it has no X-Forwarded-Proto # over loopback (see ALLOWED_HOSTS above) it has no X-Forwarded-Proto
# header to prove it's "secure", and there's no TLS listener on gunicorn's # header to prove it's "secure", and there's no TLS listener on gunicorn's
# own port to redirect to anyway. # own port to redirect to anyway.
SECURE_REDIRECT_EXEMPT = [r"^health/$"] SECURE_REDIRECT_EXEMPT = [r"^health/$", r"^health/ready/$"]
# WhiteNoise serves /static/ directly from gunicorn instead of relying on # WhiteNoise serves /static/ directly from gunicorn instead of relying on
# the reverse proxy to alias the staticfiles directory on disk — the proxy # the reverse proxy to alias the staticfiles directory on disk — the proxy