"""HTTP fetching + feedparser glue for polling a single Feed. Kept separate from tasks.py so the actual fetch/parse/dedup logic can be unit-tested with plain function calls (no Celery/broker involved) and exercised from a shell for manual verification. """ from __future__ import annotations import hashlib import logging from calendar import timegm from dataclasses import dataclass from datetime import datetime, timedelta from datetime import timezone as dt_timezone import feedparser import nh3 import requests from django.conf import settings from django.db import IntegrityError, transaction from django.utils import timezone from apps.articles.models import Article, PurgedArticle from apps.core.models import SiteSettings from apps.feeds.models import Feed from .ssrf_guard import SSRFBlocked, safe_get logger = logging.getLogger(__name__) FETCH_TIMEOUT_SECONDS = 15 MAX_BACKOFF_MINUTES = 24 * 60 USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)" # ssrf_guard.MAX_RESPONSE_BYTES already bounds the raw download; these # bound what a single poll does *with* that response — a feed within the # byte cap can still carry an unreasonable number of entries, or one # entry with an unreasonably large body, and both cost real CPU/DB time # (nh3 sanitization, an Article row/UPDATE) per entry processed. MAX_ENTRIES_PER_POLL = 200 MAX_CONTENT_HTML_INPUT_CHARS = 200_000 # bounds sanitizer cost, not just storage MAX_CONTENT_HTML_CHARS = 50_000 # bounds what's actually stored/rendered ALLOWED_TAGS = { "a", "p", "br", "hr", "strong", "em", "b", "i", "u", "s", "blockquote", "code", "pre", "ul", "ol", "li", "h1", "h2", "h3", "h4", "h5", "h6", "img", "figure", "figcaption", "table", "thead", "tbody", "tr", "th", "td", "span", "div", } ALLOWED_ATTRIBUTES = { "a": {"href", "title", "rel"}, "img": {"src", "alt", "title", "width", "height"}, } @dataclass class FetchResult: status: str # Feed.STATUS_OK / STATUS_NOT_MODIFIED / STATUS_ERROR new_article_ids: list[int] updated_article_ids: list[int] error_message: str = "" def sanitize_html(raw_html: str) -> str: if not raw_html: return "" bounded = raw_html[:MAX_CONTENT_HTML_INPUT_CHARS] cleaned = nh3.clean(bounded, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None) return cleaned[:MAX_CONTENT_HTML_CHARS] def _content_hash( title: str, link: str, author: str, summary: str, content_html: str, published_at ) -> str: """Covers every mutable Article field an entry carries (T6) — a feed editing its link/author/summary/published date, not just title or body, must still be detected as a change worth re-saving.""" published_key = published_at.isoformat() if published_at else "" payload = "\n".join([title, link, author, summary, content_html, published_key]) return hashlib.sha256(payload.encode("utf-8")).hexdigest() def _entry_guid(entry) -> str: if getattr(entry, "id", None): return entry.id if getattr(entry, "link", None): return entry.link title = getattr(entry, "title", "") published = getattr(entry, "published", "") return hashlib.sha256(f"{title}|{published}".encode("utf-8")).hexdigest() def _entry_published_at(entry): parsed = getattr(entry, "published_parsed", None) or getattr(entry, "updated_parsed", None) if not parsed: return None return datetime.fromtimestamp(timegm(parsed), tz=dt_timezone.utc) def _entry_content_html(entry) -> str: if getattr(entry, "content", None): return entry.content[0].value return getattr(entry, "summary", "") or "" UPDATABLE_ARTICLE_FIELDS = ( "title", "link", "author", "summary", "content_html", "published_at", "content_hash", ) def _ensure_pks(feed: Feed, articles: list[Article]) -> list[int]: """bulk_create only populates primary keys on backends that can RETURN them (Postgres, SQLite 3.35+); look the rest up by guid.""" if all(a.pk for a in articles): return [a.pk for a in articles] guids = [a.guid for a in articles] return list(Article.objects.filter(feed=feed, guid__in=guids).values_list("pk", flat=True)) def effective_interval_minutes(feed: Feed, configured_interval_minutes: int) -> int: """The actual polling interval to use for `feed`, applying exponential backoff on top of the site-wide `configured_interval_minutes` (see apps.core.models.SiteSettings) for a feed that's currently erroring. Takes the configured value as a parameter rather than querying SiteSettings itself, since this runs once per feed (apps.feeds.tasks calls it from a list comprehension over every active feed) and the site-wide value doesn't vary by feed. """ floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15) base = max(configured_interval_minutes, floor) if feed.fetch_error_count <= 0: return base backoff = base * (2**feed.fetch_error_count) return min(backoff, MAX_BACKOFF_MINUTES) def fetch_and_ingest(feed: Feed) -> FetchResult: """Fetch one Feed over HTTP, parse it, and dedup/insert its entries. Pure function over a Feed instance — callers are responsible for saving it afterwards. Safe to call directly from a shell or a test, with no Celery broker required. """ headers = {"User-Agent": USER_AGENT} if feed.etag: headers["If-None-Match"] = feed.etag if feed.last_modified: headers["If-Modified-Since"] = feed.last_modified try: response = safe_get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS) except SSRFBlocked as exc: feed.last_fetched_at = timezone.now() return record_fetch_error(feed, f"Refused to fetch: {exc}") except requests.RequestException as exc: feed.last_fetched_at = timezone.now() return record_fetch_error(feed, f"Request failed: {exc}") feed.last_fetched_at = timezone.now() if response.status_code == 304: feed.last_fetch_status = Feed.STATUS_NOT_MODIFIED feed.fetch_error_count = 0 return FetchResult( status=Feed.STATUS_NOT_MODIFIED, new_article_ids=[], updated_article_ids=[] ) if response.status_code >= 400: return record_fetch_error(feed, f"HTTP {response.status_code}") parsed = feedparser.parse(response.content) if parsed.bozo and not parsed.entries: return record_fetch_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}") if parsed.bozo: logger.warning( "Feed %s parsed with warnings: %s", feed.feed_url, parsed.get("bozo_exception") ) feed.etag = response.headers.get("ETag", "") feed.last_modified = response.headers.get("Last-Modified", "") feed.last_fetch_status = Feed.STATUS_OK feed.fetch_error_count = 0 channel_title = parsed.feed.get("title") if channel_title and not feed.title: feed.title = channel_title[:500] channel_link = parsed.feed.get("link") if channel_link and not feed.site_url: feed.site_url = channel_link[:1000] new_ids: list[int] = [] updated_ids: list[int] = [] entries = parsed.entries if len(entries) > MAX_ENTRIES_PER_POLL: logger.warning( "Feed %s: %d entries in one poll, only processing the first %d", feed.feed_url, len(entries), MAX_ENTRIES_PER_POLL, ) entries = entries[:MAX_ENTRIES_PER_POLL] # One prefetch + bulk writes for the whole poll (C2), instead of a # get_or_create (SELECT, then INSERT) per entry. Entries are keyed by # guid so a feed that repeats one within a single response is only # processed once. parsed_entries: dict[str, dict] = {} for entry in entries: guid = _entry_guid(entry) if guid in parsed_entries: continue title = (getattr(entry, "title", "") or "")[:1000] link = (getattr(entry, "link", "") or "")[:2000] author = (getattr(entry, "author", "") or "")[:500] summary = (getattr(entry, "summary", "") or "")[:5000] content_html = sanitize_html(_entry_content_html(entry)) published_at = _entry_published_at(entry) parsed_entries[guid] = { "title": title, "link": link, "author": author, "summary": summary, "content_html": content_html, "published_at": published_at, "content_hash": _content_hash(title, link, author, summary, content_html, published_at), } # Entries the retention purge already deleted (tombstoned) and, when # retention is on, entries so old by their own publish date that the # next purge would delete them again, are not (re-)ingested — see # apps.articles.tasks.purge_old_articles. retention_days = SiteSettings.load().retention_days if retention_days > 0: too_old_before = timezone.now() - timedelta(days=retention_days) for guid in [ g for g, f in parsed_entries.items() if f["published_at"] is not None and f["published_at"] < too_old_before ]: del parsed_entries[guid] for guid in PurgedArticle.objects.filter(feed=feed, guid__in=parsed_entries).values_list( "guid", flat=True ): del parsed_entries[guid] existing = {a.guid: a for a in Article.objects.filter(feed=feed, guid__in=parsed_entries)} to_create = [ Article(feed=feed, guid=guid, **fields) for guid, fields in parsed_entries.items() if guid not in existing ] to_update = [] for guid, article in existing.items(): fields = parsed_entries[guid] if article.content_hash != fields["content_hash"]: for name, value in fields.items(): setattr(article, name, value) to_update.append(article) if to_update: Article.objects.bulk_update(to_update, list(UPDATABLE_ARTICLE_FIELDS), batch_size=100) updated_ids.extend(a.pk for a in to_update) if to_create: try: with transaction.atomic(): Article.objects.bulk_create(to_create) except IntegrityError: # Another worker inserted some of these guids between the # prefetch above and now (two fetches of the same feed can # overlap) — fall back to row-by-row so nothing is lost or # duplicated, at the cost of the old per-entry round trips # for this rare case only. for article in to_create: fields = {name: getattr(article, name) for name in UPDATABLE_ARTICLE_FIELDS} obj, created = Article.objects.get_or_create( feed=feed, guid=article.guid, defaults=fields ) if created: new_ids.append(obj.pk) else: new_ids.extend(_ensure_pks(feed, to_create)) return FetchResult( status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids ) def record_fetch_error(feed: Feed, message: str) -> FetchResult: feed.last_fetch_status = Feed.STATUS_ERROR feed.last_error_message = message[:2000] feed.fetch_error_count += 1 logger.warning("Feed %s fetch error: %s", feed.feed_url, message) return FetchResult( status=Feed.STATUS_ERROR, new_article_ids=[], updated_article_ids=[], error_message=message )