Private
Public Access
Batch feed ingestion, add a readiness check, fail open without Redis (C2, C8)
Ingestion did a get_or_create per entry. It now prefetches existing articles for the poll in one query, bulk-inserts new ones and bulk-updates changed ones (falling back to row-by-row only if a concurrent fetch wins a race), and dispatches rule evaluation in bounded chunks so a huge first fetch is several small tasks. /health/ stays a DB-only liveness check (upgrade.sh gates on it right after restart). New /health/ready/ also checks Redis and that poll_due_feeds ticked within five minutes — proving beat and a worker are both alive — and returns 503 with per-check detail otherwise. The sidebar unread-count cache and login throttle now degrade instead of erroring when Redis is unreachable: every authenticated page and every login would otherwise 500 during a cache outage. Found while verifying in a sandbox with no Redis. Also includes the dashboard's oldest-article metric (core/views.py). Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8d2900424a
commit
e5f1277014
14 files changed
+354
-48
No files matched your search
@@ -264,3 +264,77 @@ def test_fetch_only_processes_the_first_max_entries_per_poll(feed):
|
||||
assert len(result.new_article_ids) == MAX_ENTRIES_PER_POLL
|
||||
assert Article.objects.filter(feed=feed).count() == MAX_ENTRIES_PER_POLL
|
||||
assert not Article.objects.filter(feed=feed, guid=f"guid-{MAX_ENTRIES_PER_POLL}").exists()
|
||||
|
||||
|
||||
def _feed_xml(count, title_suffix=""):
|
||||
items = "".join(
|
||||
f"<item><title>Item {i}{title_suffix}</title><guid>guid-{i}</guid></item>"
|
||||
for i in range(count)
|
||||
)
|
||||
return (
|
||||
f'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>{items}</channel></rss>'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_ingest_uses_a_constant_number_of_article_queries(feed):
|
||||
"""C2: one prefetch + bulk writes per poll, not a get_or_create per
|
||||
entry — query count must not grow with the number of entries."""
|
||||
from django.db import connection
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
|
||||
responses.add(responses.GET, FEED_URL, body=_feed_xml(40), status=200)
|
||||
with CaptureQueriesContext(connection) as ctx:
|
||||
result = fetch_and_ingest(feed)
|
||||
|
||||
assert len(result.new_article_ids) == 40
|
||||
assert len(ctx.captured_queries) < 10
|
||||
assert Article.objects.filter(feed=feed).count() == 40
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_ingest_dedups_a_guid_repeated_within_one_response(feed):
|
||||
xml = (
|
||||
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
|
||||
"<item><title>A</title><guid>same</guid></item>"
|
||||
"<item><title>B</title><guid>same</guid></item>"
|
||||
"</channel></rss>"
|
||||
)
|
||||
responses.add(responses.GET, FEED_URL, body=xml, status=200)
|
||||
|
||||
result = fetch_and_ingest(feed)
|
||||
|
||||
assert len(result.new_article_ids) == 1
|
||||
assert Article.objects.filter(feed=feed).count() == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_ingest_reports_updates_separately_from_new_articles(feed):
|
||||
responses.add(responses.GET, FEED_URL, body=_feed_xml(3), status=200)
|
||||
fetch_and_ingest(feed)
|
||||
|
||||
responses.replace(responses.GET, FEED_URL, body=_feed_xml(4, " edited"), status=200)
|
||||
result = fetch_and_ingest(feed)
|
||||
|
||||
assert len(result.new_article_ids) == 1
|
||||
assert len(result.updated_article_ids) == 3
|
||||
assert Article.objects.get(feed=feed, guid="guid-0").title == "Item 0 edited"
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_ingest_falls_back_to_row_by_row_when_a_concurrent_insert_wins(feed):
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.db import IntegrityError
|
||||
|
||||
responses.add(responses.GET, FEED_URL, body=_feed_xml(3), status=200)
|
||||
|
||||
with patch.object(Article.objects, "bulk_create", side_effect=IntegrityError("race")):
|
||||
result = fetch_and_ingest(feed)
|
||||
|
||||
assert len(result.new_article_ids) == 3
|
||||
assert Article.objects.filter(feed=feed).count() == 3
|
||||
Reference in new issue
Block a user