Files
Evection/apps/feeds/services/fetcher.py
T
ksmithandClaude Sonnet 5.5 41f99e9405 Allow only http(s) URLs from feeds and OPML, at storage and render (R2-S1, #33)
A feed's channel link, an entry's link and an OPML htmlUrl were stored
as given and rendered in href attributes (bulk writes skip model
validation), so a malicious feed could supply a javascript: or data:
link. CSP is a second layer, not the control.

apps.core.urlutils accepts only absolute http(s) URLs. Fetching stores
channel/entry links through it — relative links now resolve against the
feed's own URL instead of being stored as dead relative paths — OPML
import skips non-http(s) feed URLs and blanks unsafe htmlUrl, and the
add-feed form rejects non-http(s) (URLField also allows ftp). A
safe_http_url template filter guards the two places a feed-supplied URL
becomes a link (the article's "Original" link and the dashboard), so
values already in the database from before this fix are neutralized too.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-01 11:57:23 -07:00

338 lines
11 KiB
Python

"""HTTP fetching + feedparser glue for polling a single Feed.
Kept separate from tasks.py so the actual fetch/parse/dedup logic can be
unit-tested with plain function calls (no Celery/broker involved) and
exercised from a shell for manual verification.
"""
from __future__ import annotations
import hashlib
import logging
from calendar import timegm
from dataclasses import dataclass
from datetime import datetime, timedelta
from datetime import timezone as dt_timezone
import feedparser
import nh3
import requests
from django.conf import settings
from django.db import IntegrityError, transaction
from django.utils import timezone
from apps.articles.models import Article, PurgedArticle
from apps.core.models import SiteSettings
from apps.core.urlutils import resolve_http_url
from apps.feeds.models import Feed
from .ssrf_guard import SSRFBlocked, safe_get
logger = logging.getLogger(__name__)
FETCH_TIMEOUT_SECONDS = 15
MAX_BACKOFF_MINUTES = 24 * 60
USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)"
# ssrf_guard.MAX_RESPONSE_BYTES already bounds the raw download; these
# bound what a single poll does *with* that response — a feed within the
# byte cap can still carry an unreasonable number of entries, or one
# entry with an unreasonably large body, and both cost real CPU/DB time
# (nh3 sanitization, an Article row/UPDATE) per entry processed.
MAX_ENTRIES_PER_POLL = 200
MAX_CONTENT_HTML_INPUT_CHARS = 200_000 # bounds sanitizer cost, not just storage
MAX_CONTENT_HTML_CHARS = 50_000 # bounds what's actually stored/rendered
ALLOWED_TAGS = {
"a",
"p",
"br",
"hr",
"strong",
"em",
"b",
"i",
"u",
"s",
"blockquote",
"code",
"pre",
"ul",
"ol",
"li",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"img",
"figure",
"figcaption",
"table",
"thead",
"tbody",
"tr",
"th",
"td",
"span",
"div",
}
ALLOWED_ATTRIBUTES = {
"a": {"href", "title", "rel"},
"img": {"src", "alt", "title", "width", "height"},
}
@dataclass
class FetchResult:
status: str # Feed.STATUS_OK / STATUS_NOT_MODIFIED / STATUS_ERROR
new_article_ids: list[int]
updated_article_ids: list[int]
error_message: str = ""
def sanitize_html(raw_html: str) -> str:
if not raw_html:
return ""
bounded = raw_html[:MAX_CONTENT_HTML_INPUT_CHARS]
cleaned = nh3.clean(bounded, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None)
return cleaned[:MAX_CONTENT_HTML_CHARS]
def _content_hash(
title: str, link: str, author: str, summary: str, content_html: str, published_at
) -> str:
"""Covers every mutable Article field an entry carries (T6) — a feed
editing its link/author/summary/published date, not just title or
body, must still be detected as a change worth re-saving."""
published_key = published_at.isoformat() if published_at else ""
payload = "\n".join([title, link, author, summary, content_html, published_key])
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
def _entry_guid(entry) -> str:
if getattr(entry, "id", None):
return entry.id
if getattr(entry, "link", None):
return entry.link
title = getattr(entry, "title", "")
published = getattr(entry, "published", "")
return hashlib.sha256(f"{title}|{published}".encode("utf-8")).hexdigest()
def _entry_published_at(entry):
parsed = getattr(entry, "published_parsed", None) or getattr(entry, "updated_parsed", None)
if not parsed:
return None
return datetime.fromtimestamp(timegm(parsed), tz=dt_timezone.utc)
def _entry_content_html(entry) -> str:
if getattr(entry, "content", None):
return entry.content[0].value
return getattr(entry, "summary", "") or ""
UPDATABLE_ARTICLE_FIELDS = (
"title",
"link",
"author",
"summary",
"content_html",
"published_at",
"content_hash",
)
def _ensure_pks(feed: Feed, articles: list[Article]) -> list[int]:
"""bulk_create only populates primary keys on backends that can
RETURN them (Postgres, SQLite 3.35+); look the rest up by guid."""
if all(a.pk for a in articles):
return [a.pk for a in articles]
guids = [a.guid for a in articles]
return list(Article.objects.filter(feed=feed, guid__in=guids).values_list("pk", flat=True))
def effective_interval_minutes(feed: Feed, configured_interval_minutes: int) -> int:
"""The actual polling interval to use for `feed`, applying exponential
backoff on top of the site-wide `configured_interval_minutes` (see
apps.core.models.SiteSettings) for a feed that's currently erroring.
Takes the configured value as a parameter rather than querying
SiteSettings itself, since this runs once per feed (apps.feeds.tasks
calls it from a list comprehension over every active feed) and the
site-wide value doesn't vary by feed.
"""
floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15)
base = max(configured_interval_minutes, floor)
if feed.fetch_error_count <= 0:
return base
backoff = base * (2**feed.fetch_error_count)
return min(backoff, MAX_BACKOFF_MINUTES)
def fetch_and_ingest(feed: Feed) -> FetchResult:
"""Fetch one Feed over HTTP, parse it, and dedup/insert its entries.
Pure function over a Feed instance — callers are responsible for
saving it afterwards. Safe to call directly from a shell or a test,
with no Celery broker required.
"""
headers = {"User-Agent": USER_AGENT}
if feed.etag:
headers["If-None-Match"] = feed.etag
if feed.last_modified:
headers["If-Modified-Since"] = feed.last_modified
try:
response = safe_get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS)
except SSRFBlocked as exc:
feed.last_fetched_at = timezone.now()
return record_fetch_error(feed, f"Refused to fetch: {exc}")
except requests.RequestException as exc:
feed.last_fetched_at = timezone.now()
return record_fetch_error(feed, f"Request failed: {exc}")
feed.last_fetched_at = timezone.now()
if response.status_code == 304:
feed.last_fetch_status = Feed.STATUS_NOT_MODIFIED
feed.fetch_error_count = 0
return FetchResult(
status=Feed.STATUS_NOT_MODIFIED, new_article_ids=[], updated_article_ids=[]
)
if response.status_code >= 400:
return record_fetch_error(feed, f"HTTP {response.status_code}")
parsed = feedparser.parse(response.content)
if parsed.bozo and not parsed.entries:
return record_fetch_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}")
if parsed.bozo:
logger.warning(
"Feed %s parsed with warnings: %s", feed.feed_url, parsed.get("bozo_exception")
)
feed.etag = response.headers.get("ETag", "")
feed.last_modified = response.headers.get("Last-Modified", "")
feed.last_fetch_status = Feed.STATUS_OK
feed.fetch_error_count = 0
channel_title = parsed.feed.get("title")
if channel_title and not feed.title:
feed.title = channel_title[:500]
channel_link = resolve_http_url(feed.feed_url, parsed.feed.get("link"))
if channel_link and not feed.site_url:
feed.site_url = channel_link[:1000]
new_ids: list[int] = []
updated_ids: list[int] = []
entries = parsed.entries
if len(entries) > MAX_ENTRIES_PER_POLL:
logger.warning(
"Feed %s: %d entries in one poll, only processing the first %d",
feed.feed_url,
len(entries),
MAX_ENTRIES_PER_POLL,
)
entries = entries[:MAX_ENTRIES_PER_POLL]
# One prefetch + bulk writes for the whole poll (C2), instead of a
# get_or_create (SELECT, then INSERT) per entry. Entries are keyed by
# guid so a feed that repeats one within a single response is only
# processed once.
parsed_entries: dict[str, dict] = {}
for entry in entries:
guid = _entry_guid(entry)
if guid in parsed_entries:
continue
title = (getattr(entry, "title", "") or "")[:1000]
link = resolve_http_url(feed.feed_url, getattr(entry, "link", ""))[:2000]
author = (getattr(entry, "author", "") or "")[:500]
summary = (getattr(entry, "summary", "") or "")[:5000]
content_html = sanitize_html(_entry_content_html(entry))
published_at = _entry_published_at(entry)
parsed_entries[guid] = {
"title": title,
"link": link,
"author": author,
"summary": summary,
"content_html": content_html,
"published_at": published_at,
"content_hash": _content_hash(title, link, author, summary, content_html, published_at),
}
# Entries the retention purge already deleted (tombstoned) and, when
# retention is on, entries so old by their own publish date that the
# next purge would delete them again, are not (re-)ingested — see
# apps.articles.tasks.purge_old_articles.
retention_days = SiteSettings.load().retention_days
if retention_days > 0:
too_old_before = timezone.now() - timedelta(days=retention_days)
for guid in [
g
for g, f in parsed_entries.items()
if f["published_at"] is not None and f["published_at"] < too_old_before
]:
del parsed_entries[guid]
for guid in PurgedArticle.objects.filter(feed=feed, guid__in=parsed_entries).values_list(
"guid", flat=True
):
del parsed_entries[guid]
existing = {a.guid: a for a in Article.objects.filter(feed=feed, guid__in=parsed_entries)}
to_create = [
Article(feed=feed, guid=guid, **fields)
for guid, fields in parsed_entries.items()
if guid not in existing
]
to_update = []
for guid, article in existing.items():
fields = parsed_entries[guid]
if article.content_hash != fields["content_hash"]:
for name, value in fields.items():
setattr(article, name, value)
to_update.append(article)
if to_update:
Article.objects.bulk_update(to_update, list(UPDATABLE_ARTICLE_FIELDS), batch_size=100)
updated_ids.extend(a.pk for a in to_update)
if to_create:
try:
with transaction.atomic():
Article.objects.bulk_create(to_create)
except IntegrityError:
# Another worker inserted some of these guids between the
# prefetch above and now (two fetches of the same feed can
# overlap) — fall back to row-by-row so nothing is lost or
# duplicated, at the cost of the old per-entry round trips
# for this rare case only.
for article in to_create:
fields = {name: getattr(article, name) for name in UPDATABLE_ARTICLE_FIELDS}
obj, created = Article.objects.get_or_create(
feed=feed, guid=article.guid, defaults=fields
)
if created:
new_ids.append(obj.pk)
else:
new_ids.extend(_ensure_pks(feed, to_create))
return FetchResult(
status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids
)
def record_fetch_error(feed: Feed, message: str) -> FetchResult:
feed.last_fetch_status = Feed.STATUS_ERROR
feed.last_error_message = message[:2000]
feed.fetch_error_count += 1
logger.warning("Feed %s fetch error: %s", feed.feed_url, message)
return FetchResult(
status=Feed.STATUS_ERROR, new_article_ids=[], updated_article_ids=[], error_message=message
)