Files
Evection/apps/feeds/services/fetcher.py
T
ksmithandClaude Sonnet 5.5 e5f1277014 Batch feed ingestion, add a readiness check, fail open without Redis (C2, C8)
Ingestion did a get_or_create per entry. It now prefetches existing
articles for the poll in one query, bulk-inserts new ones and
bulk-updates changed ones (falling back to row-by-row only if a
concurrent fetch wins a race), and dispatches rule evaluation in
bounded chunks so a huge first fetch is several small tasks.

/health/ stays a DB-only liveness check (upgrade.sh gates on it right
after restart). New /health/ready/ also checks Redis and that
poll_due_feeds ticked within five minutes — proving beat and a worker
are both alive — and returns 503 with per-check detail otherwise.

The sidebar unread-count cache and login throttle now degrade instead
of erroring when Redis is unreachable: every authenticated page and
every login would otherwise 500 during a cache outage. Found while
verifying in a sandbox with no Redis.

Also includes the dashboard's oldest-article metric (core/views.py).

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-01 08:51:20 -07:00

318 lines
11 KiB
Python

"""HTTP fetching + feedparser glue for polling a single Feed.
Kept separate from tasks.py so the actual fetch/parse/dedup logic can be
unit-tested with plain function calls (no Celery/broker involved) and
exercised from a shell for manual verification.
"""
from __future__ import annotations
import hashlib
import logging
from calendar import timegm
from dataclasses import dataclass
from datetime import datetime
from datetime import timezone as dt_timezone
import feedparser
import nh3
import requests
from django.conf import settings
from django.db import IntegrityError, transaction
from django.utils import timezone
from apps.articles.models import Article
from apps.feeds.models import Feed
from .ssrf_guard import SSRFBlocked, safe_get
logger = logging.getLogger(__name__)
FETCH_TIMEOUT_SECONDS = 15
MAX_BACKOFF_MINUTES = 24 * 60
USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)"
# ssrf_guard.MAX_RESPONSE_BYTES already bounds the raw download; these
# bound what a single poll does *with* that response — a feed within the
# byte cap can still carry an unreasonable number of entries, or one
# entry with an unreasonably large body, and both cost real CPU/DB time
# (nh3 sanitization, an Article row/UPDATE) per entry processed.
MAX_ENTRIES_PER_POLL = 200
MAX_CONTENT_HTML_INPUT_CHARS = 200_000 # bounds sanitizer cost, not just storage
MAX_CONTENT_HTML_CHARS = 50_000 # bounds what's actually stored/rendered
ALLOWED_TAGS = {
"a",
"p",
"br",
"hr",
"strong",
"em",
"b",
"i",
"u",
"s",
"blockquote",
"code",
"pre",
"ul",
"ol",
"li",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"img",
"figure",
"figcaption",
"table",
"thead",
"tbody",
"tr",
"th",
"td",
"span",
"div",
}
ALLOWED_ATTRIBUTES = {
"a": {"href", "title", "rel"},
"img": {"src", "alt", "title", "width", "height"},
}
@dataclass
class FetchResult:
status: str # Feed.STATUS_OK / STATUS_NOT_MODIFIED / STATUS_ERROR
new_article_ids: list[int]
updated_article_ids: list[int]
error_message: str = ""
def sanitize_html(raw_html: str) -> str:
if not raw_html:
return ""
bounded = raw_html[:MAX_CONTENT_HTML_INPUT_CHARS]
cleaned = nh3.clean(bounded, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None)
return cleaned[:MAX_CONTENT_HTML_CHARS]
def _content_hash(
title: str, link: str, author: str, summary: str, content_html: str, published_at
) -> str:
"""Covers every mutable Article field an entry carries (T6) — a feed
editing its link/author/summary/published date, not just title or
body, must still be detected as a change worth re-saving."""
published_key = published_at.isoformat() if published_at else ""
payload = "\n".join([title, link, author, summary, content_html, published_key])
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
def _entry_guid(entry) -> str:
if getattr(entry, "id", None):
return entry.id
if getattr(entry, "link", None):
return entry.link
title = getattr(entry, "title", "")
published = getattr(entry, "published", "")
return hashlib.sha256(f"{title}|{published}".encode("utf-8")).hexdigest()
def _entry_published_at(entry):
parsed = getattr(entry, "published_parsed", None) or getattr(entry, "updated_parsed", None)
if not parsed:
return None
return datetime.fromtimestamp(timegm(parsed), tz=dt_timezone.utc)
def _entry_content_html(entry) -> str:
if getattr(entry, "content", None):
return entry.content[0].value
return getattr(entry, "summary", "") or ""
UPDATABLE_ARTICLE_FIELDS = (
"title",
"link",
"author",
"summary",
"content_html",
"published_at",
"content_hash",
)
def _ensure_pks(feed: Feed, articles: list[Article]) -> list[int]:
"""bulk_create only populates primary keys on backends that can
RETURN them (Postgres, SQLite 3.35+); look the rest up by guid."""
if all(a.pk for a in articles):
return [a.pk for a in articles]
guids = [a.guid for a in articles]
return list(Article.objects.filter(feed=feed, guid__in=guids).values_list("pk", flat=True))
def effective_interval_minutes(feed: Feed, configured_interval_minutes: int) -> int:
"""The actual polling interval to use for `feed`, applying exponential
backoff on top of the site-wide `configured_interval_minutes` (see
apps.core.models.SiteSettings) for a feed that's currently erroring.
Takes the configured value as a parameter rather than querying
SiteSettings itself, since this runs once per feed (apps.feeds.tasks
calls it from a list comprehension over every active feed) and the
site-wide value doesn't vary by feed.
"""
floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15)
base = max(configured_interval_minutes, floor)
if feed.fetch_error_count <= 0:
return base
backoff = base * (2**feed.fetch_error_count)
return min(backoff, MAX_BACKOFF_MINUTES)
def fetch_and_ingest(feed: Feed) -> FetchResult:
"""Fetch one Feed over HTTP, parse it, and dedup/insert its entries.
Pure function over a Feed instance — callers are responsible for
saving it afterwards. Safe to call directly from a shell or a test,
with no Celery broker required.
"""
headers = {"User-Agent": USER_AGENT}
if feed.etag:
headers["If-None-Match"] = feed.etag
if feed.last_modified:
headers["If-Modified-Since"] = feed.last_modified
try:
response = safe_get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS)
except SSRFBlocked as exc:
feed.last_fetched_at = timezone.now()
return record_fetch_error(feed, f"Refused to fetch: {exc}")
except requests.RequestException as exc:
feed.last_fetched_at = timezone.now()
return record_fetch_error(feed, f"Request failed: {exc}")
feed.last_fetched_at = timezone.now()
if response.status_code == 304:
feed.last_fetch_status = Feed.STATUS_NOT_MODIFIED
feed.fetch_error_count = 0
return FetchResult(
status=Feed.STATUS_NOT_MODIFIED, new_article_ids=[], updated_article_ids=[]
)
if response.status_code >= 400:
return record_fetch_error(feed, f"HTTP {response.status_code}")
parsed = feedparser.parse(response.content)
if parsed.bozo and not parsed.entries:
return record_fetch_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}")
if parsed.bozo:
logger.warning(
"Feed %s parsed with warnings: %s", feed.feed_url, parsed.get("bozo_exception")
)
feed.etag = response.headers.get("ETag", "")
feed.last_modified = response.headers.get("Last-Modified", "")
feed.last_fetch_status = Feed.STATUS_OK
feed.fetch_error_count = 0
channel_title = parsed.feed.get("title")
if channel_title and not feed.title:
feed.title = channel_title[:500]
channel_link = parsed.feed.get("link")
if channel_link and not feed.site_url:
feed.site_url = channel_link[:1000]
new_ids: list[int] = []
updated_ids: list[int] = []
entries = parsed.entries
if len(entries) > MAX_ENTRIES_PER_POLL:
logger.warning(
"Feed %s: %d entries in one poll, only processing the first %d",
feed.feed_url,
len(entries),
MAX_ENTRIES_PER_POLL,
)
entries = entries[:MAX_ENTRIES_PER_POLL]
# One prefetch + bulk writes for the whole poll (C2), instead of a
# get_or_create (SELECT, then INSERT) per entry. Entries are keyed by
# guid so a feed that repeats one within a single response is only
# processed once.
parsed_entries: dict[str, dict] = {}
for entry in entries:
guid = _entry_guid(entry)
if guid in parsed_entries:
continue
title = (getattr(entry, "title", "") or "")[:1000]
link = (getattr(entry, "link", "") or "")[:2000]
author = (getattr(entry, "author", "") or "")[:500]
summary = (getattr(entry, "summary", "") or "")[:5000]
content_html = sanitize_html(_entry_content_html(entry))
published_at = _entry_published_at(entry)
parsed_entries[guid] = {
"title": title,
"link": link,
"author": author,
"summary": summary,
"content_html": content_html,
"published_at": published_at,
"content_hash": _content_hash(title, link, author, summary, content_html, published_at),
}
existing = {a.guid: a for a in Article.objects.filter(feed=feed, guid__in=parsed_entries)}
to_create = [
Article(feed=feed, guid=guid, **fields)
for guid, fields in parsed_entries.items()
if guid not in existing
]
to_update = []
for guid, article in existing.items():
fields = parsed_entries[guid]
if article.content_hash != fields["content_hash"]:
for name, value in fields.items():
setattr(article, name, value)
to_update.append(article)
if to_update:
Article.objects.bulk_update(to_update, list(UPDATABLE_ARTICLE_FIELDS), batch_size=100)
updated_ids.extend(a.pk for a in to_update)
if to_create:
try:
with transaction.atomic():
Article.objects.bulk_create(to_create)
except IntegrityError:
# Another worker inserted some of these guids between the
# prefetch above and now (two fetches of the same feed can
# overlap) — fall back to row-by-row so nothing is lost or
# duplicated, at the cost of the old per-entry round trips
# for this rare case only.
for article in to_create:
fields = {name: getattr(article, name) for name in UPDATABLE_ARTICLE_FIELDS}
obj, created = Article.objects.get_or_create(
feed=feed, guid=article.guid, defaults=fields
)
if created:
new_ids.append(obj.pk)
else:
new_ids.extend(_ensure_pks(feed, to_create))
return FetchResult(
status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids
)
def record_fetch_error(feed: Feed, message: str) -> FetchResult:
feed.last_fetch_status = Feed.STATUS_ERROR
feed.last_error_message = message[:2000]
feed.fetch_error_count += 1
logger.warning("Feed %s fetch error: %s", feed.feed_url, message)
return FetchResult(
status=Feed.STATUS_ERROR, new_article_ids=[], updated_article_ids=[], error_message=message
)