"""HTTP fetching + feedparser glue for polling a single Feed. Kept separate from tasks.py so the actual fetch/parse/dedup logic can be unit-tested with plain function calls (no Celery/broker involved) and exercised from a shell for manual verification. """ from __future__ import annotations import hashlib import logging from calendar import timegm from dataclasses import dataclass from datetime import datetime from datetime import timezone as dt_timezone import feedparser import nh3 import requests from django.conf import settings from django.db import IntegrityError, transaction from django.utils import timezone from apps.articles.models import Article from apps.feeds.models import Feed from .ssrf_guard import SSRFBlocked, safe_get logger = logging.getLogger(__name__) FETCH_TIMEOUT_SECONDS = 15 MAX_BACKOFF_MINUTES = 24 * 60 USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)" # ssrf_guard.MAX_RESPONSE_BYTES already bounds the raw download; these # bound what a single poll does *with* that response — a feed within the # byte cap can still carry an unreasonable number of entries, or one # entry with an unreasonably large body, and both cost real CPU/DB time # (nh3 sanitization, an Article row/UPDATE) per entry processed. MAX_ENTRIES_PER_POLL = 200 MAX_CONTENT_HTML_INPUT_CHARS = 200_000 # bounds sanitizer cost, not just storage MAX_CONTENT_HTML_CHARS = 50_000 # bounds what's actually stored/rendered ALLOWED_TAGS = { "a", "p", "br", "hr", "strong", "em", "b", "i", "u", "s", "blockquote", "code", "pre", "ul", "ol", "li", "h1", "h2", "h3", "h4", "h5", "h6", "img", "figure", "figcaption", "table", "thead", "tbody", "tr", "th", "td", "span", "div", } ALLOWED_ATTRIBUTES = { "a": {"href", "title", "rel"}, "img": {"src", "alt", "title", "width", "height"}, } @dataclass class FetchResult: status: str # Feed.STATUS_OK / STATUS_NOT_MODIFIED / STATUS_ERROR new_article_ids: list[int] updated_article_ids: list[int] error_message: str = "" def sanitize_html(raw_html: str) -> str: if not raw_html: return "" bounded = raw_html[:MAX_CONTENT_HTML_INPUT_CHARS] cleaned = nh3.clean(bounded, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None) return cleaned[:MAX_CONTENT_HTML_CHARS] def _content_hash( title: str, link: str, author: str, summary: str, content_html: str, published_at ) -> str: """Covers every mutable Article field an entry carries (T6) — a feed editing its link/author/summary/published date, not just title or body, must still be detected as a change worth re-saving.""" published_key = published_at.isoformat() if published_at else "" payload = "\n".join([title, link, author, summary, content_html, published_key]) return hashlib.sha256(payload.encode("utf-8")).hexdigest() def _entry_guid(entry) -> str: if getattr(entry, "id", None): return entry.id if getattr(entry, "link", None): return entry.link title = getattr(entry, "title", "") published = getattr(entry, "published", "") return hashlib.sha256(f"{title}|{published}".encode("utf-8")).hexdigest() def _entry_published_at(entry): parsed = getattr(entry, "published_parsed", None) or getattr(entry, "updated_parsed", None) if not parsed: return None return datetime.fromtimestamp(timegm(parsed), tz=dt_timezone.utc) def _entry_content_html(entry) -> str: if getattr(entry, "content", None): return entry.content[0].value return getattr(entry, "summary", "") or "" UPDATABLE_ARTICLE_FIELDS = ( "title", "link", "author", "summary", "content_html", "published_at", "content_hash", ) def _ensure_pks(feed: Feed, articles: list[Article]) -> list[int]: """bulk_create only populates primary keys on backends that can RETURN them (Postgres, SQLite 3.35+); look the rest up by guid.""" if all(a.pk for a in articles): return [a.pk for a in articles] guids = [a.guid for a in articles] return list(Article.objects.filter(feed=feed, guid__in=guids).values_list("pk", flat=True)) def effective_interval_minutes(feed: Feed, configured_interval_minutes: int) -> int: """The actual polling interval to use for `feed`, applying exponential backoff on top of the site-wide `configured_interval_minutes` (see apps.core.models.SiteSettings) for a feed that's currently erroring. Takes the configured value as a parameter rather than querying SiteSettings itself, since this runs once per feed (apps.feeds.tasks calls it from a list comprehension over every active feed) and the site-wide value doesn't vary by feed. """ floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15) base = max(configured_interval_minutes, floor) if feed.fetch_error_count <= 0: return base backoff = base * (2**feed.fetch_error_count) return min(backoff, MAX_BACKOFF_MINUTES) def fetch_and_ingest(feed: Feed) -> FetchResult: """Fetch one Feed over HTTP, parse it, and dedup/insert its entries. Pure function over a Feed instance — callers are responsible for saving it afterwards. Safe to call directly from a shell or a test, with no Celery broker required. """ headers = {"User-Agent": USER_AGENT} if feed.etag: headers["If-None-Match"] = feed.etag if feed.last_modified: headers["If-Modified-Since"] = feed.last_modified try: response = safe_get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS) except SSRFBlocked as exc: feed.last_fetched_at = timezone.now() return record_fetch_error(feed, f"Refused to fetch: {exc}") except requests.RequestException as exc: feed.last_fetched_at = timezone.now() return record_fetch_error(feed, f"Request failed: {exc}") feed.last_fetched_at = timezone.now() if response.status_code == 304: feed.last_fetch_status = Feed.STATUS_NOT_MODIFIED feed.fetch_error_count = 0 return FetchResult( status=Feed.STATUS_NOT_MODIFIED, new_article_ids=[], updated_article_ids=[] ) if response.status_code >= 400: return record_fetch_error(feed, f"HTTP {response.status_code}") parsed = feedparser.parse(response.content) if parsed.bozo and not parsed.entries: return record_fetch_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}") if parsed.bozo: logger.warning( "Feed %s parsed with warnings: %s", feed.feed_url, parsed.get("bozo_exception") ) feed.etag = response.headers.get("ETag", "") feed.last_modified = response.headers.get("Last-Modified", "") feed.last_fetch_status = Feed.STATUS_OK feed.fetch_error_count = 0 channel_title = parsed.feed.get("title") if channel_title and not feed.title: feed.title = channel_title[:500] channel_link = parsed.feed.get("link") if channel_link and not feed.site_url: feed.site_url = channel_link[:1000] new_ids: list[int] = [] updated_ids: list[int] = [] entries = parsed.entries if len(entries) > MAX_ENTRIES_PER_POLL: logger.warning( "Feed %s: %d entries in one poll, only processing the first %d", feed.feed_url, len(entries), MAX_ENTRIES_PER_POLL, ) entries = entries[:MAX_ENTRIES_PER_POLL] # One prefetch + bulk writes for the whole poll (C2), instead of a # get_or_create (SELECT, then INSERT) per entry. Entries are keyed by # guid so a feed that repeats one within a single response is only # processed once. parsed_entries: dict[str, dict] = {} for entry in entries: guid = _entry_guid(entry) if guid in parsed_entries: continue title = (getattr(entry, "title", "") or "")[:1000] link = (getattr(entry, "link", "") or "")[:2000] author = (getattr(entry, "author", "") or "")[:500] summary = (getattr(entry, "summary", "") or "")[:5000] content_html = sanitize_html(_entry_content_html(entry)) published_at = _entry_published_at(entry) parsed_entries[guid] = { "title": title, "link": link, "author": author, "summary": summary, "content_html": content_html, "published_at": published_at, "content_hash": _content_hash(title, link, author, summary, content_html, published_at), } existing = {a.guid: a for a in Article.objects.filter(feed=feed, guid__in=parsed_entries)} to_create = [ Article(feed=feed, guid=guid, **fields) for guid, fields in parsed_entries.items() if guid not in existing ] to_update = [] for guid, article in existing.items(): fields = parsed_entries[guid] if article.content_hash != fields["content_hash"]: for name, value in fields.items(): setattr(article, name, value) to_update.append(article) if to_update: Article.objects.bulk_update(to_update, list(UPDATABLE_ARTICLE_FIELDS), batch_size=100) updated_ids.extend(a.pk for a in to_update) if to_create: try: with transaction.atomic(): Article.objects.bulk_create(to_create) except IntegrityError: # Another worker inserted some of these guids between the # prefetch above and now (two fetches of the same feed can # overlap) — fall back to row-by-row so nothing is lost or # duplicated, at the cost of the old per-entry round trips # for this rare case only. for article in to_create: fields = {name: getattr(article, name) for name in UPDATABLE_ARTICLE_FIELDS} obj, created = Article.objects.get_or_create( feed=feed, guid=article.guid, defaults=fields ) if created: new_ids.append(obj.pk) else: new_ids.extend(_ensure_pks(feed, to_create)) return FetchResult( status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids ) def record_fetch_error(feed: Feed, message: str) -> FetchResult: feed.last_fetch_status = Feed.STATUS_ERROR feed.last_error_message = message[:2000] feed.fetch_error_count += 1 logger.warning("Feed %s fetch error: %s", feed.feed_url, message) return FetchResult( status=Feed.STATUS_ERROR, new_article_ids=[], updated_article_ids=[], error_message=message )