Files
Evection/apps/feeds/services/fetcher.py
T
ksmithandClaude Sonnet 5 9439f9cc59 Add SSRF protection to feed fetching: validate scheme/address, bound redirects and response size (S1)
fetch_and_ingest fetched a feed_url supplied straight from the "add
feed" form or an uploaded OPML file, with requests following redirects
on its own and no check that the target (or a redirect's target) was
actually a public, fetchable feed rather than an internal service,
localhost, or a cloud metadata endpoint — reachable from wherever this
app runs, on any authenticated user's behalf.

apps.feeds.services.ssrf_guard.safe_get() replaces the direct
requests.get() call: it resolves and validates the hostname (rejecting
anything that isn't a globally routable address) before the initial
request and before following each redirect hop manually, caps the
number of hops, and bounds the response body size by streaming it
instead of buffering it unbounded. OPML-imported feeds go through the
same check the first time poll_due_feeds fetches them, since this is
the one chokepoint every feed URL passes through regardless of which
path created the Feed row.

Known residual gap, documented in ssrf_guard.py: this validates then
connects rather than pinning the TCP connection to the exact address it
validated, so a narrow DNS-rebinding race isn't closed. Fixing that
needs a custom transport and felt like more machinery than this
self-hosted app's threat model currently justifies.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-31 12:25:20 -06:00

231 lines
6.9 KiB
Python

"""HTTP fetching + feedparser glue for polling a single Feed.
Kept separate from tasks.py so the actual fetch/parse/dedup logic can be
unit-tested with plain function calls (no Celery/broker involved) and
exercised from a shell for manual verification.
"""
from __future__ import annotations
import hashlib
import logging
from calendar import timegm
from dataclasses import dataclass
from datetime import datetime
from datetime import timezone as dt_timezone
import feedparser
import nh3
import requests
from django.conf import settings
from django.utils import timezone
from apps.articles.models import Article
from apps.feeds.models import Feed
from .ssrf_guard import SSRFBlocked, safe_get
logger = logging.getLogger(__name__)
FETCH_TIMEOUT_SECONDS = 15
MAX_BACKOFF_MINUTES = 24 * 60
USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)"
ALLOWED_TAGS = {
"a",
"p",
"br",
"hr",
"strong",
"em",
"b",
"i",
"u",
"s",
"blockquote",
"code",
"pre",
"ul",
"ol",
"li",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"img",
"figure",
"figcaption",
"table",
"thead",
"tbody",
"tr",
"th",
"td",
"span",
"div",
}
ALLOWED_ATTRIBUTES = {
"a": {"href", "title", "rel"},
"img": {"src", "alt", "title", "width", "height"},
}
@dataclass
class FetchResult:
status: str # Feed.STATUS_OK / STATUS_NOT_MODIFIED / STATUS_ERROR
new_article_ids: list[int]
updated_article_ids: list[int]
error_message: str = ""
def sanitize_html(raw_html: str) -> str:
if not raw_html:
return ""
return nh3.clean(raw_html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None)
def _content_hash(title: str, content_html: str) -> str:
return hashlib.sha256(f"{title}\n{content_html}".encode("utf-8")).hexdigest()
def _entry_guid(entry) -> str:
if getattr(entry, "id", None):
return entry.id
if getattr(entry, "link", None):
return entry.link
title = getattr(entry, "title", "")
published = getattr(entry, "published", "")
return hashlib.sha256(f"{title}|{published}".encode("utf-8")).hexdigest()
def _entry_published_at(entry):
parsed = getattr(entry, "published_parsed", None) or getattr(entry, "updated_parsed", None)
if not parsed:
return None
return datetime.fromtimestamp(timegm(parsed), tz=dt_timezone.utc)
def _entry_content_html(entry) -> str:
if getattr(entry, "content", None):
return entry.content[0].value
return getattr(entry, "summary", "") or ""
def effective_interval_minutes(feed: Feed) -> int:
"""The actual polling interval to use, applying exponential backoff on
top of the admin-configured ``fetch_interval_minutes`` for feeds that
are currently erroring. ``fetch_interval_minutes`` itself is never
mutated by the fetcher — it's the admin's chosen baseline.
"""
floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15)
base = max(feed.fetch_interval_minutes, floor)
if feed.fetch_error_count <= 0:
return base
backoff = base * (2**feed.fetch_error_count)
return min(backoff, MAX_BACKOFF_MINUTES)
def fetch_and_ingest(feed: Feed) -> FetchResult:
"""Fetch one Feed over HTTP, parse it, and dedup/insert its entries.
Pure function over a Feed instance — callers are responsible for
saving it afterwards. Safe to call directly from a shell or a test,
with no Celery broker required.
"""
headers = {"User-Agent": USER_AGENT}
if feed.etag:
headers["If-None-Match"] = feed.etag
if feed.last_modified:
headers["If-Modified-Since"] = feed.last_modified
try:
response = safe_get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS)
except SSRFBlocked as exc:
feed.last_fetched_at = timezone.now()
return _record_error(feed, f"Refused to fetch: {exc}")
except requests.RequestException as exc:
feed.last_fetched_at = timezone.now()
return _record_error(feed, f"Request failed: {exc}")
feed.last_fetched_at = timezone.now()
if response.status_code == 304:
feed.last_fetch_status = Feed.STATUS_NOT_MODIFIED
feed.fetch_error_count = 0
return FetchResult(
status=Feed.STATUS_NOT_MODIFIED, new_article_ids=[], updated_article_ids=[]
)
if response.status_code >= 400:
return _record_error(feed, f"HTTP {response.status_code}")
parsed = feedparser.parse(response.content)
if parsed.bozo and not parsed.entries:
return _record_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}")
if parsed.bozo:
logger.warning(
"Feed %s parsed with warnings: %s", feed.feed_url, parsed.get("bozo_exception")
)
feed.etag = response.headers.get("ETag", "")
feed.last_modified = response.headers.get("Last-Modified", "")
feed.last_fetch_status = Feed.STATUS_OK
feed.fetch_error_count = 0
channel_title = parsed.feed.get("title")
if channel_title and not feed.title:
feed.title = channel_title[:500]
channel_link = parsed.feed.get("link")
if channel_link and not feed.site_url:
feed.site_url = channel_link[:1000]
new_ids: list[int] = []
updated_ids: list[int] = []
for entry in parsed.entries:
guid = _entry_guid(entry)
title = (getattr(entry, "title", "") or "")[:1000]
content_html = sanitize_html(_entry_content_html(entry))
content_hash = _content_hash(title, content_html)
article, created = Article.objects.get_or_create(
feed=feed,
guid=guid,
defaults={
"link": (getattr(entry, "link", "") or "")[:2000],
"title": title,
"author": (getattr(entry, "author", "") or "")[:500],
"content_html": content_html,
"summary": (getattr(entry, "summary", "") or "")[:5000],
"content_hash": content_hash,
"published_at": _entry_published_at(entry),
},
)
if created:
new_ids.append(article.pk)
elif article.content_hash != content_hash:
article.title = title
article.content_html = content_html
article.content_hash = content_hash
article.save(update_fields=["title", "content_html", "content_hash"])
updated_ids.append(article.pk)
return FetchResult(
status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids
)
def _record_error(feed: Feed, message: str) -> FetchResult:
feed.last_fetch_status = Feed.STATUS_ERROR
feed.last_error_message = message[:2000]
feed.fetch_error_count += 1
logger.warning("Feed %s fetch error: %s", feed.feed_url, message)
return FetchResult(
status=Feed.STATUS_ERROR, new_article_ids=[], updated_article_ids=[], error_message=message
)