Private
Public Access
The nightly purge deleted articles, but a feed that still listed an entry would re-insert it on its next poll as a brand-new, unread article. For feeds with long archives, old items could reappear after every retention run. The purge now leaves a (feed, guid) tombstone per deleted article, and ingestion skips tombstoned guids. With retention on, entries whose own publish date is already outside the window are not ingested at all, so a long archive isn't fetched only to be purged and tombstoned the same night. Undated entries rely on the tombstone. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
337 lines
11 KiB
Python
337 lines
11 KiB
Python
"""HTTP fetching + feedparser glue for polling a single Feed.
|
|
|
|
Kept separate from tasks.py so the actual fetch/parse/dedup logic can be
|
|
unit-tested with plain function calls (no Celery/broker involved) and
|
|
exercised from a shell for manual verification.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
from calendar import timegm
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timedelta
|
|
from datetime import timezone as dt_timezone
|
|
|
|
import feedparser
|
|
import nh3
|
|
import requests
|
|
from django.conf import settings
|
|
from django.db import IntegrityError, transaction
|
|
from django.utils import timezone
|
|
|
|
from apps.articles.models import Article, PurgedArticle
|
|
from apps.core.models import SiteSettings
|
|
from apps.feeds.models import Feed
|
|
|
|
from .ssrf_guard import SSRFBlocked, safe_get
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
FETCH_TIMEOUT_SECONDS = 15
|
|
MAX_BACKOFF_MINUTES = 24 * 60
|
|
USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)"
|
|
|
|
# ssrf_guard.MAX_RESPONSE_BYTES already bounds the raw download; these
|
|
# bound what a single poll does *with* that response — a feed within the
|
|
# byte cap can still carry an unreasonable number of entries, or one
|
|
# entry with an unreasonably large body, and both cost real CPU/DB time
|
|
# (nh3 sanitization, an Article row/UPDATE) per entry processed.
|
|
MAX_ENTRIES_PER_POLL = 200
|
|
MAX_CONTENT_HTML_INPUT_CHARS = 200_000 # bounds sanitizer cost, not just storage
|
|
MAX_CONTENT_HTML_CHARS = 50_000 # bounds what's actually stored/rendered
|
|
|
|
ALLOWED_TAGS = {
|
|
"a",
|
|
"p",
|
|
"br",
|
|
"hr",
|
|
"strong",
|
|
"em",
|
|
"b",
|
|
"i",
|
|
"u",
|
|
"s",
|
|
"blockquote",
|
|
"code",
|
|
"pre",
|
|
"ul",
|
|
"ol",
|
|
"li",
|
|
"h1",
|
|
"h2",
|
|
"h3",
|
|
"h4",
|
|
"h5",
|
|
"h6",
|
|
"img",
|
|
"figure",
|
|
"figcaption",
|
|
"table",
|
|
"thead",
|
|
"tbody",
|
|
"tr",
|
|
"th",
|
|
"td",
|
|
"span",
|
|
"div",
|
|
}
|
|
ALLOWED_ATTRIBUTES = {
|
|
"a": {"href", "title", "rel"},
|
|
"img": {"src", "alt", "title", "width", "height"},
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class FetchResult:
|
|
status: str # Feed.STATUS_OK / STATUS_NOT_MODIFIED / STATUS_ERROR
|
|
new_article_ids: list[int]
|
|
updated_article_ids: list[int]
|
|
error_message: str = ""
|
|
|
|
|
|
def sanitize_html(raw_html: str) -> str:
|
|
if not raw_html:
|
|
return ""
|
|
bounded = raw_html[:MAX_CONTENT_HTML_INPUT_CHARS]
|
|
cleaned = nh3.clean(bounded, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None)
|
|
return cleaned[:MAX_CONTENT_HTML_CHARS]
|
|
|
|
|
|
def _content_hash(
|
|
title: str, link: str, author: str, summary: str, content_html: str, published_at
|
|
) -> str:
|
|
"""Covers every mutable Article field an entry carries (T6) — a feed
|
|
editing its link/author/summary/published date, not just title or
|
|
body, must still be detected as a change worth re-saving."""
|
|
published_key = published_at.isoformat() if published_at else ""
|
|
payload = "\n".join([title, link, author, summary, content_html, published_key])
|
|
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _entry_guid(entry) -> str:
|
|
if getattr(entry, "id", None):
|
|
return entry.id
|
|
if getattr(entry, "link", None):
|
|
return entry.link
|
|
title = getattr(entry, "title", "")
|
|
published = getattr(entry, "published", "")
|
|
return hashlib.sha256(f"{title}|{published}".encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _entry_published_at(entry):
|
|
parsed = getattr(entry, "published_parsed", None) or getattr(entry, "updated_parsed", None)
|
|
if not parsed:
|
|
return None
|
|
return datetime.fromtimestamp(timegm(parsed), tz=dt_timezone.utc)
|
|
|
|
|
|
def _entry_content_html(entry) -> str:
|
|
if getattr(entry, "content", None):
|
|
return entry.content[0].value
|
|
return getattr(entry, "summary", "") or ""
|
|
|
|
|
|
UPDATABLE_ARTICLE_FIELDS = (
|
|
"title",
|
|
"link",
|
|
"author",
|
|
"summary",
|
|
"content_html",
|
|
"published_at",
|
|
"content_hash",
|
|
)
|
|
|
|
|
|
def _ensure_pks(feed: Feed, articles: list[Article]) -> list[int]:
|
|
"""bulk_create only populates primary keys on backends that can
|
|
RETURN them (Postgres, SQLite 3.35+); look the rest up by guid."""
|
|
if all(a.pk for a in articles):
|
|
return [a.pk for a in articles]
|
|
guids = [a.guid for a in articles]
|
|
return list(Article.objects.filter(feed=feed, guid__in=guids).values_list("pk", flat=True))
|
|
|
|
|
|
def effective_interval_minutes(feed: Feed, configured_interval_minutes: int) -> int:
|
|
"""The actual polling interval to use for `feed`, applying exponential
|
|
backoff on top of the site-wide `configured_interval_minutes` (see
|
|
apps.core.models.SiteSettings) for a feed that's currently erroring.
|
|
Takes the configured value as a parameter rather than querying
|
|
SiteSettings itself, since this runs once per feed (apps.feeds.tasks
|
|
calls it from a list comprehension over every active feed) and the
|
|
site-wide value doesn't vary by feed.
|
|
"""
|
|
floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15)
|
|
base = max(configured_interval_minutes, floor)
|
|
if feed.fetch_error_count <= 0:
|
|
return base
|
|
backoff = base * (2**feed.fetch_error_count)
|
|
return min(backoff, MAX_BACKOFF_MINUTES)
|
|
|
|
|
|
def fetch_and_ingest(feed: Feed) -> FetchResult:
|
|
"""Fetch one Feed over HTTP, parse it, and dedup/insert its entries.
|
|
|
|
Pure function over a Feed instance — callers are responsible for
|
|
saving it afterwards. Safe to call directly from a shell or a test,
|
|
with no Celery broker required.
|
|
"""
|
|
headers = {"User-Agent": USER_AGENT}
|
|
if feed.etag:
|
|
headers["If-None-Match"] = feed.etag
|
|
if feed.last_modified:
|
|
headers["If-Modified-Since"] = feed.last_modified
|
|
|
|
try:
|
|
response = safe_get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS)
|
|
except SSRFBlocked as exc:
|
|
feed.last_fetched_at = timezone.now()
|
|
return record_fetch_error(feed, f"Refused to fetch: {exc}")
|
|
except requests.RequestException as exc:
|
|
feed.last_fetched_at = timezone.now()
|
|
return record_fetch_error(feed, f"Request failed: {exc}")
|
|
|
|
feed.last_fetched_at = timezone.now()
|
|
|
|
if response.status_code == 304:
|
|
feed.last_fetch_status = Feed.STATUS_NOT_MODIFIED
|
|
feed.fetch_error_count = 0
|
|
return FetchResult(
|
|
status=Feed.STATUS_NOT_MODIFIED, new_article_ids=[], updated_article_ids=[]
|
|
)
|
|
|
|
if response.status_code >= 400:
|
|
return record_fetch_error(feed, f"HTTP {response.status_code}")
|
|
|
|
parsed = feedparser.parse(response.content)
|
|
|
|
if parsed.bozo and not parsed.entries:
|
|
return record_fetch_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}")
|
|
|
|
if parsed.bozo:
|
|
logger.warning(
|
|
"Feed %s parsed with warnings: %s", feed.feed_url, parsed.get("bozo_exception")
|
|
)
|
|
|
|
feed.etag = response.headers.get("ETag", "")
|
|
feed.last_modified = response.headers.get("Last-Modified", "")
|
|
feed.last_fetch_status = Feed.STATUS_OK
|
|
feed.fetch_error_count = 0
|
|
|
|
channel_title = parsed.feed.get("title")
|
|
if channel_title and not feed.title:
|
|
feed.title = channel_title[:500]
|
|
channel_link = parsed.feed.get("link")
|
|
if channel_link and not feed.site_url:
|
|
feed.site_url = channel_link[:1000]
|
|
|
|
new_ids: list[int] = []
|
|
updated_ids: list[int] = []
|
|
|
|
entries = parsed.entries
|
|
if len(entries) > MAX_ENTRIES_PER_POLL:
|
|
logger.warning(
|
|
"Feed %s: %d entries in one poll, only processing the first %d",
|
|
feed.feed_url,
|
|
len(entries),
|
|
MAX_ENTRIES_PER_POLL,
|
|
)
|
|
entries = entries[:MAX_ENTRIES_PER_POLL]
|
|
|
|
# One prefetch + bulk writes for the whole poll (C2), instead of a
|
|
# get_or_create (SELECT, then INSERT) per entry. Entries are keyed by
|
|
# guid so a feed that repeats one within a single response is only
|
|
# processed once.
|
|
parsed_entries: dict[str, dict] = {}
|
|
for entry in entries:
|
|
guid = _entry_guid(entry)
|
|
if guid in parsed_entries:
|
|
continue
|
|
title = (getattr(entry, "title", "") or "")[:1000]
|
|
link = (getattr(entry, "link", "") or "")[:2000]
|
|
author = (getattr(entry, "author", "") or "")[:500]
|
|
summary = (getattr(entry, "summary", "") or "")[:5000]
|
|
content_html = sanitize_html(_entry_content_html(entry))
|
|
published_at = _entry_published_at(entry)
|
|
parsed_entries[guid] = {
|
|
"title": title,
|
|
"link": link,
|
|
"author": author,
|
|
"summary": summary,
|
|
"content_html": content_html,
|
|
"published_at": published_at,
|
|
"content_hash": _content_hash(title, link, author, summary, content_html, published_at),
|
|
}
|
|
|
|
# Entries the retention purge already deleted (tombstoned) and, when
|
|
# retention is on, entries so old by their own publish date that the
|
|
# next purge would delete them again, are not (re-)ingested — see
|
|
# apps.articles.tasks.purge_old_articles.
|
|
retention_days = SiteSettings.load().retention_days
|
|
if retention_days > 0:
|
|
too_old_before = timezone.now() - timedelta(days=retention_days)
|
|
for guid in [
|
|
g
|
|
for g, f in parsed_entries.items()
|
|
if f["published_at"] is not None and f["published_at"] < too_old_before
|
|
]:
|
|
del parsed_entries[guid]
|
|
for guid in PurgedArticle.objects.filter(feed=feed, guid__in=parsed_entries).values_list(
|
|
"guid", flat=True
|
|
):
|
|
del parsed_entries[guid]
|
|
|
|
existing = {a.guid: a for a in Article.objects.filter(feed=feed, guid__in=parsed_entries)}
|
|
|
|
to_create = [
|
|
Article(feed=feed, guid=guid, **fields)
|
|
for guid, fields in parsed_entries.items()
|
|
if guid not in existing
|
|
]
|
|
to_update = []
|
|
for guid, article in existing.items():
|
|
fields = parsed_entries[guid]
|
|
if article.content_hash != fields["content_hash"]:
|
|
for name, value in fields.items():
|
|
setattr(article, name, value)
|
|
to_update.append(article)
|
|
|
|
if to_update:
|
|
Article.objects.bulk_update(to_update, list(UPDATABLE_ARTICLE_FIELDS), batch_size=100)
|
|
updated_ids.extend(a.pk for a in to_update)
|
|
|
|
if to_create:
|
|
try:
|
|
with transaction.atomic():
|
|
Article.objects.bulk_create(to_create)
|
|
except IntegrityError:
|
|
# Another worker inserted some of these guids between the
|
|
# prefetch above and now (two fetches of the same feed can
|
|
# overlap) — fall back to row-by-row so nothing is lost or
|
|
# duplicated, at the cost of the old per-entry round trips
|
|
# for this rare case only.
|
|
for article in to_create:
|
|
fields = {name: getattr(article, name) for name in UPDATABLE_ARTICLE_FIELDS}
|
|
obj, created = Article.objects.get_or_create(
|
|
feed=feed, guid=article.guid, defaults=fields
|
|
)
|
|
if created:
|
|
new_ids.append(obj.pk)
|
|
else:
|
|
new_ids.extend(_ensure_pks(feed, to_create))
|
|
|
|
return FetchResult(
|
|
status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids
|
|
)
|
|
|
|
|
|
def record_fetch_error(feed: Feed, message: str) -> FetchResult:
|
|
feed.last_fetch_status = Feed.STATUS_ERROR
|
|
feed.last_error_message = message[:2000]
|
|
feed.fetch_error_count += 1
|
|
logger.warning("Feed %s fetch error: %s", feed.feed_url, message)
|
|
return FetchResult(
|
|
status=Feed.STATUS_ERROR, new_article_ids=[], updated_article_ids=[], error_message=message
|
|
)
|