"""HTTP fetching + feedparser glue for polling a single Feed. Kept separate from tasks.py so the actual fetch/parse/dedup logic can be unit-tested with plain function calls (no Celery/broker involved) and exercised from a shell for manual verification. """ from __future__ import annotations import hashlib import logging from calendar import timegm from dataclasses import dataclass from datetime import datetime from datetime import timezone as dt_timezone import feedparser import nh3 import requests from django.conf import settings from django.utils import timezone from apps.articles.models import Article from apps.feeds.models import Feed from .ssrf_guard import SSRFBlocked, safe_get logger = logging.getLogger(__name__) FETCH_TIMEOUT_SECONDS = 15 MAX_BACKOFF_MINUTES = 24 * 60 USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)" ALLOWED_TAGS = { "a", "p", "br", "hr", "strong", "em", "b", "i", "u", "s", "blockquote", "code", "pre", "ul", "ol", "li", "h1", "h2", "h3", "h4", "h5", "h6", "img", "figure", "figcaption", "table", "thead", "tbody", "tr", "th", "td", "span", "div", } ALLOWED_ATTRIBUTES = { "a": {"href", "title", "rel"}, "img": {"src", "alt", "title", "width", "height"}, } @dataclass class FetchResult: status: str # Feed.STATUS_OK / STATUS_NOT_MODIFIED / STATUS_ERROR new_article_ids: list[int] updated_article_ids: list[int] error_message: str = "" def sanitize_html(raw_html: str) -> str: if not raw_html: return "" return nh3.clean(raw_html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None) def _content_hash(title: str, content_html: str) -> str: return hashlib.sha256(f"{title}\n{content_html}".encode("utf-8")).hexdigest() def _entry_guid(entry) -> str: if getattr(entry, "id", None): return entry.id if getattr(entry, "link", None): return entry.link title = getattr(entry, "title", "") published = getattr(entry, "published", "") return hashlib.sha256(f"{title}|{published}".encode("utf-8")).hexdigest() def _entry_published_at(entry): parsed = getattr(entry, "published_parsed", None) or getattr(entry, "updated_parsed", None) if not parsed: return None return datetime.fromtimestamp(timegm(parsed), tz=dt_timezone.utc) def _entry_content_html(entry) -> str: if getattr(entry, "content", None): return entry.content[0].value return getattr(entry, "summary", "") or "" def effective_interval_minutes(feed: Feed) -> int: """The actual polling interval to use, applying exponential backoff on top of the admin-configured ``fetch_interval_minutes`` for feeds that are currently erroring. ``fetch_interval_minutes`` itself is never mutated by the fetcher — it's the admin's chosen baseline. """ floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15) base = max(feed.fetch_interval_minutes, floor) if feed.fetch_error_count <= 0: return base backoff = base * (2**feed.fetch_error_count) return min(backoff, MAX_BACKOFF_MINUTES) def fetch_and_ingest(feed: Feed) -> FetchResult: """Fetch one Feed over HTTP, parse it, and dedup/insert its entries. Pure function over a Feed instance — callers are responsible for saving it afterwards. Safe to call directly from a shell or a test, with no Celery broker required. """ headers = {"User-Agent": USER_AGENT} if feed.etag: headers["If-None-Match"] = feed.etag if feed.last_modified: headers["If-Modified-Since"] = feed.last_modified try: response = safe_get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS) except SSRFBlocked as exc: feed.last_fetched_at = timezone.now() return _record_error(feed, f"Refused to fetch: {exc}") except requests.RequestException as exc: feed.last_fetched_at = timezone.now() return _record_error(feed, f"Request failed: {exc}") feed.last_fetched_at = timezone.now() if response.status_code == 304: feed.last_fetch_status = Feed.STATUS_NOT_MODIFIED feed.fetch_error_count = 0 return FetchResult( status=Feed.STATUS_NOT_MODIFIED, new_article_ids=[], updated_article_ids=[] ) if response.status_code >= 400: return _record_error(feed, f"HTTP {response.status_code}") parsed = feedparser.parse(response.content) if parsed.bozo and not parsed.entries: return _record_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}") if parsed.bozo: logger.warning( "Feed %s parsed with warnings: %s", feed.feed_url, parsed.get("bozo_exception") ) feed.etag = response.headers.get("ETag", "") feed.last_modified = response.headers.get("Last-Modified", "") feed.last_fetch_status = Feed.STATUS_OK feed.fetch_error_count = 0 channel_title = parsed.feed.get("title") if channel_title and not feed.title: feed.title = channel_title[:500] channel_link = parsed.feed.get("link") if channel_link and not feed.site_url: feed.site_url = channel_link[:1000] new_ids: list[int] = [] updated_ids: list[int] = [] for entry in parsed.entries: guid = _entry_guid(entry) title = (getattr(entry, "title", "") or "")[:1000] content_html = sanitize_html(_entry_content_html(entry)) content_hash = _content_hash(title, content_html) article, created = Article.objects.get_or_create( feed=feed, guid=guid, defaults={ "link": (getattr(entry, "link", "") or "")[:2000], "title": title, "author": (getattr(entry, "author", "") or "")[:500], "content_html": content_html, "summary": (getattr(entry, "summary", "") or "")[:5000], "content_hash": content_hash, "published_at": _entry_published_at(entry), }, ) if created: new_ids.append(article.pk) elif article.content_hash != content_hash: article.title = title article.content_html = content_html article.content_hash = content_hash article.save(update_fields=["title", "content_html", "content_hash"]) updated_ids.append(article.pk) return FetchResult( status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids ) def _record_error(feed: Feed, message: str) -> FetchResult: feed.last_fetch_status = Feed.STATUS_ERROR feed.last_error_message = message[:2000] feed.fetch_error_count += 1 logger.warning("Feed %s fetch error: %s", feed.feed_url, message) return FetchResult( status=Feed.STATUS_ERROR, new_article_ids=[], updated_article_ids=[], error_message=message )