Private
Public Access
fetch_and_ingest fetched a feed_url supplied straight from the "add feed" form or an uploaded OPML file, with requests following redirects on its own and no check that the target (or a redirect's target) was actually a public, fetchable feed rather than an internal service, localhost, or a cloud metadata endpoint — reachable from wherever this app runs, on any authenticated user's behalf. apps.feeds.services.ssrf_guard.safe_get() replaces the direct requests.get() call: it resolves and validates the hostname (rejecting anything that isn't a globally routable address) before the initial request and before following each redirect hop manually, caps the number of hops, and bounds the response body size by streaming it instead of buffering it unbounded. OPML-imported feeds go through the same check the first time poll_due_feeds fetches them, since this is the one chokepoint every feed URL passes through regardless of which path created the Feed row. Known residual gap, documented in ssrf_guard.py: this validates then connects rather than pinning the TCP connection to the exact address it validated, so a narrow DNS-rebinding race isn't closed. Fixing that needs a custom transport and felt like more machinery than this self-hosted app's threat model currently justifies. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
231 lines
6.9 KiB
Python
231 lines
6.9 KiB
Python
"""HTTP fetching + feedparser glue for polling a single Feed.
|
|
|
|
Kept separate from tasks.py so the actual fetch/parse/dedup logic can be
|
|
unit-tested with plain function calls (no Celery/broker involved) and
|
|
exercised from a shell for manual verification.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
from calendar import timegm
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
from datetime import timezone as dt_timezone
|
|
|
|
import feedparser
|
|
import nh3
|
|
import requests
|
|
from django.conf import settings
|
|
from django.utils import timezone
|
|
|
|
from apps.articles.models import Article
|
|
from apps.feeds.models import Feed
|
|
|
|
from .ssrf_guard import SSRFBlocked, safe_get
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
FETCH_TIMEOUT_SECONDS = 15
|
|
MAX_BACKOFF_MINUTES = 24 * 60
|
|
USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)"
|
|
|
|
ALLOWED_TAGS = {
|
|
"a",
|
|
"p",
|
|
"br",
|
|
"hr",
|
|
"strong",
|
|
"em",
|
|
"b",
|
|
"i",
|
|
"u",
|
|
"s",
|
|
"blockquote",
|
|
"code",
|
|
"pre",
|
|
"ul",
|
|
"ol",
|
|
"li",
|
|
"h1",
|
|
"h2",
|
|
"h3",
|
|
"h4",
|
|
"h5",
|
|
"h6",
|
|
"img",
|
|
"figure",
|
|
"figcaption",
|
|
"table",
|
|
"thead",
|
|
"tbody",
|
|
"tr",
|
|
"th",
|
|
"td",
|
|
"span",
|
|
"div",
|
|
}
|
|
ALLOWED_ATTRIBUTES = {
|
|
"a": {"href", "title", "rel"},
|
|
"img": {"src", "alt", "title", "width", "height"},
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class FetchResult:
|
|
status: str # Feed.STATUS_OK / STATUS_NOT_MODIFIED / STATUS_ERROR
|
|
new_article_ids: list[int]
|
|
updated_article_ids: list[int]
|
|
error_message: str = ""
|
|
|
|
|
|
def sanitize_html(raw_html: str) -> str:
|
|
if not raw_html:
|
|
return ""
|
|
return nh3.clean(raw_html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None)
|
|
|
|
|
|
def _content_hash(title: str, content_html: str) -> str:
|
|
return hashlib.sha256(f"{title}\n{content_html}".encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _entry_guid(entry) -> str:
|
|
if getattr(entry, "id", None):
|
|
return entry.id
|
|
if getattr(entry, "link", None):
|
|
return entry.link
|
|
title = getattr(entry, "title", "")
|
|
published = getattr(entry, "published", "")
|
|
return hashlib.sha256(f"{title}|{published}".encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _entry_published_at(entry):
|
|
parsed = getattr(entry, "published_parsed", None) or getattr(entry, "updated_parsed", None)
|
|
if not parsed:
|
|
return None
|
|
return datetime.fromtimestamp(timegm(parsed), tz=dt_timezone.utc)
|
|
|
|
|
|
def _entry_content_html(entry) -> str:
|
|
if getattr(entry, "content", None):
|
|
return entry.content[0].value
|
|
return getattr(entry, "summary", "") or ""
|
|
|
|
|
|
def effective_interval_minutes(feed: Feed) -> int:
|
|
"""The actual polling interval to use, applying exponential backoff on
|
|
top of the admin-configured ``fetch_interval_minutes`` for feeds that
|
|
are currently erroring. ``fetch_interval_minutes`` itself is never
|
|
mutated by the fetcher — it's the admin's chosen baseline.
|
|
"""
|
|
floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15)
|
|
base = max(feed.fetch_interval_minutes, floor)
|
|
if feed.fetch_error_count <= 0:
|
|
return base
|
|
backoff = base * (2**feed.fetch_error_count)
|
|
return min(backoff, MAX_BACKOFF_MINUTES)
|
|
|
|
|
|
def fetch_and_ingest(feed: Feed) -> FetchResult:
|
|
"""Fetch one Feed over HTTP, parse it, and dedup/insert its entries.
|
|
|
|
Pure function over a Feed instance — callers are responsible for
|
|
saving it afterwards. Safe to call directly from a shell or a test,
|
|
with no Celery broker required.
|
|
"""
|
|
headers = {"User-Agent": USER_AGENT}
|
|
if feed.etag:
|
|
headers["If-None-Match"] = feed.etag
|
|
if feed.last_modified:
|
|
headers["If-Modified-Since"] = feed.last_modified
|
|
|
|
try:
|
|
response = safe_get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS)
|
|
except SSRFBlocked as exc:
|
|
feed.last_fetched_at = timezone.now()
|
|
return _record_error(feed, f"Refused to fetch: {exc}")
|
|
except requests.RequestException as exc:
|
|
feed.last_fetched_at = timezone.now()
|
|
return _record_error(feed, f"Request failed: {exc}")
|
|
|
|
feed.last_fetched_at = timezone.now()
|
|
|
|
if response.status_code == 304:
|
|
feed.last_fetch_status = Feed.STATUS_NOT_MODIFIED
|
|
feed.fetch_error_count = 0
|
|
return FetchResult(
|
|
status=Feed.STATUS_NOT_MODIFIED, new_article_ids=[], updated_article_ids=[]
|
|
)
|
|
|
|
if response.status_code >= 400:
|
|
return _record_error(feed, f"HTTP {response.status_code}")
|
|
|
|
parsed = feedparser.parse(response.content)
|
|
|
|
if parsed.bozo and not parsed.entries:
|
|
return _record_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}")
|
|
|
|
if parsed.bozo:
|
|
logger.warning(
|
|
"Feed %s parsed with warnings: %s", feed.feed_url, parsed.get("bozo_exception")
|
|
)
|
|
|
|
feed.etag = response.headers.get("ETag", "")
|
|
feed.last_modified = response.headers.get("Last-Modified", "")
|
|
feed.last_fetch_status = Feed.STATUS_OK
|
|
feed.fetch_error_count = 0
|
|
|
|
channel_title = parsed.feed.get("title")
|
|
if channel_title and not feed.title:
|
|
feed.title = channel_title[:500]
|
|
channel_link = parsed.feed.get("link")
|
|
if channel_link and not feed.site_url:
|
|
feed.site_url = channel_link[:1000]
|
|
|
|
new_ids: list[int] = []
|
|
updated_ids: list[int] = []
|
|
|
|
for entry in parsed.entries:
|
|
guid = _entry_guid(entry)
|
|
title = (getattr(entry, "title", "") or "")[:1000]
|
|
content_html = sanitize_html(_entry_content_html(entry))
|
|
content_hash = _content_hash(title, content_html)
|
|
|
|
article, created = Article.objects.get_or_create(
|
|
feed=feed,
|
|
guid=guid,
|
|
defaults={
|
|
"link": (getattr(entry, "link", "") or "")[:2000],
|
|
"title": title,
|
|
"author": (getattr(entry, "author", "") or "")[:500],
|
|
"content_html": content_html,
|
|
"summary": (getattr(entry, "summary", "") or "")[:5000],
|
|
"content_hash": content_hash,
|
|
"published_at": _entry_published_at(entry),
|
|
},
|
|
)
|
|
|
|
if created:
|
|
new_ids.append(article.pk)
|
|
elif article.content_hash != content_hash:
|
|
article.title = title
|
|
article.content_html = content_html
|
|
article.content_hash = content_hash
|
|
article.save(update_fields=["title", "content_html", "content_hash"])
|
|
updated_ids.append(article.pk)
|
|
|
|
return FetchResult(
|
|
status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids
|
|
)
|
|
|
|
|
|
def _record_error(feed: Feed, message: str) -> FetchResult:
|
|
feed.last_fetch_status = Feed.STATUS_ERROR
|
|
feed.last_error_message = message[:2000]
|
|
feed.fetch_error_count += 1
|
|
logger.warning("Feed %s fetch error: %s", feed.feed_url, message)
|
|
return FetchResult(
|
|
status=Feed.STATUS_ERROR, new_article_ids=[], updated_article_ids=[], error_message=message
|
|
)
|