Batch feed ingestion, add a readiness check, fail open without Redis (C2, C8)

Ingestion did a get_or_create per entry. It now prefetches existing
articles for the poll in one query, bulk-inserts new ones and
bulk-updates changed ones (falling back to row-by-row only if a
concurrent fetch wins a race), and dispatches rule evaluation in
bounded chunks so a huge first fetch is several small tasks.

/health/ stays a DB-only liveness check (upgrade.sh gates on it right
after restart). New /health/ready/ also checks Redis and that
poll_due_feeds ticked within five minutes — proving beat and a worker
are both alive — and returns 503 with per-check detail otherwise.

The sidebar unread-count cache and login throttle now degrade instead
of erroring when Redis is unreachable: every authenticated page and
every login would otherwise 500 during a cache outage. Found while
verifying in a sandbox with no Redis.

Also includes the dashboard's oldest-article metric (core/views.py).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5.5 committed 2026-11-01 08:51:20 -07:00
1 parent 8d2900424a
commit e5f1277014
14 files changed
+354 -48

No files matched your search

+74 -36
View File
@@ -18,6 +18,7 @@ import feedparser
import nh3
import requests
from django.conf import settings
from django.db import IntegrityError, transaction
from django.utils import timezone
from apps.articles.models import Article
@@ -131,6 +132,26 @@ def _entry_content_html(entry) -> str:
return getattr(entry, "summary", "") or ""
UPDATABLE_ARTICLE_FIELDS = (
"title",
"link",
"author",
"summary",
"content_html",
"published_at",
"content_hash",
)
def _ensure_pks(feed: Feed, articles: list[Article]) -> list[int]:
"""bulk_create only populates primary keys on backends that can
RETURN them (Postgres, SQLite 3.35+); look the rest up by guid."""
if all(a.pk for a in articles):
return [a.pk for a in articles]
guids = [a.guid for a in articles]
return list(Article.objects.filter(feed=feed, guid__in=guids).values_list("pk", flat=True))
def effective_interval_minutes(feed: Feed, configured_interval_minutes: int) -> int:
"""The actual polling interval to use for `feed`, applying exponential
backoff on top of the site-wide `configured_interval_minutes` (see
@@ -217,52 +238,69 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
)
entries = entries[:MAX_ENTRIES_PER_POLL]
# One prefetch + bulk writes for the whole poll (C2), instead of a
# get_or_create (SELECT, then INSERT) per entry. Entries are keyed by
# guid so a feed that repeats one within a single response is only
# processed once.
parsed_entries: dict[str, dict] = {}
for entry in entries:
guid = _entry_guid(entry)
if guid in parsed_entries:
continue
title = (getattr(entry, "title", "") or "")[:1000]
link = (getattr(entry, "link", "") or "")[:2000]
author = (getattr(entry, "author", "") or "")[:500]
summary = (getattr(entry, "summary", "") or "")[:5000]
content_html = sanitize_html(_entry_content_html(entry))
published_at = _entry_published_at(entry)
content_hash = _content_hash(title, link, author, summary, content_html, published_at)
parsed_entries[guid] = {
"title": title,
"link": link,
"author": author,
"summary": summary,
"content_html": content_html,
"published_at": published_at,
"content_hash": _content_hash(title, link, author, summary, content_html, published_at),
}
article, created = Article.objects.get_or_create(
feed=feed,
guid=guid,
defaults={
"link": link,
"title": title,
"author": author,
"content_html": content_html,
"summary": summary,
"content_hash": content_hash,
"published_at": published_at,
},
)
existing = {a.guid: a for a in Article.objects.filter(feed=feed, guid__in=parsed_entries)}
if created:
new_ids.append(article.pk)
elif article.content_hash != content_hash:
article.title = title
article.link = link
article.author = author
article.summary = summary
article.content_html = content_html
article.published_at = published_at
article.content_hash = content_hash
article.save(
update_fields=[
"title",
"link",
"author",
"summary",
"content_html",
"published_at",
"content_hash",
]
)
updated_ids.append(article.pk)
to_create = [
Article(feed=feed, guid=guid, **fields)
for guid, fields in parsed_entries.items()
if guid not in existing
]
to_update = []
for guid, article in existing.items():
fields = parsed_entries[guid]
if article.content_hash != fields["content_hash"]:
for name, value in fields.items():
setattr(article, name, value)
to_update.append(article)
if to_update:
Article.objects.bulk_update(to_update, list(UPDATABLE_ARTICLE_FIELDS), batch_size=100)
updated_ids.extend(a.pk for a in to_update)
if to_create:
try:
with transaction.atomic():
Article.objects.bulk_create(to_create)
except IntegrityError:
# Another worker inserted some of these guids between the
# prefetch above and now (two fetches of the same feed can
# overlap) — fall back to row-by-row so nothing is lost or
# duplicated, at the cost of the old per-entry round trips
# for this rare case only.
for article in to_create:
fields = {name: getattr(article, name) for name in UPDATABLE_ARTICLE_FIELDS}
obj, created = Article.objects.get_or_create(
feed=feed, guid=article.guid, defaults=fields
)
if created:
new_ids.append(obj.pk)
else:
new_ids.extend(_ensure_pks(feed, to_create))
return FetchResult(
status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids