Private
Public Access
Replaces Feed.fetch_interval_minutes with a single admin-editable SiteSettings.fetch_interval_minutes (apps.core), following the same singleton-row pattern as accounts.SetupLock. Per-feed error backoff is unchanged — only the configured baseline interval is now shared across every feed, loaded once per poll_due_feeds run rather than queried per feed. EVECTION_MIN_FETCH_INTERVAL_MINUTES keeps working as the floor underneath it. SiteSettings is registered in Django admin as a singleton: "Add" is only offered when no row exists yet, and delete is always refused. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
255 lines
8.0 KiB
Python
255 lines
8.0 KiB
Python
"""HTTP fetching + feedparser glue for polling a single Feed.
|
|
|
|
Kept separate from tasks.py so the actual fetch/parse/dedup logic can be
|
|
unit-tested with plain function calls (no Celery/broker involved) and
|
|
exercised from a shell for manual verification.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
from calendar import timegm
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
from datetime import timezone as dt_timezone
|
|
|
|
import feedparser
|
|
import nh3
|
|
import requests
|
|
from django.conf import settings
|
|
from django.utils import timezone
|
|
|
|
from apps.articles.models import Article
|
|
from apps.feeds.models import Feed
|
|
|
|
from .ssrf_guard import SSRFBlocked, safe_get
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
FETCH_TIMEOUT_SECONDS = 15
|
|
MAX_BACKOFF_MINUTES = 24 * 60
|
|
USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)"
|
|
|
|
# ssrf_guard.MAX_RESPONSE_BYTES already bounds the raw download; these
|
|
# bound what a single poll does *with* that response — a feed within the
|
|
# byte cap can still carry an unreasonable number of entries, or one
|
|
# entry with an unreasonably large body, and both cost real CPU/DB time
|
|
# (nh3 sanitization, an Article row/UPDATE) per entry processed.
|
|
MAX_ENTRIES_PER_POLL = 200
|
|
MAX_CONTENT_HTML_INPUT_CHARS = 200_000 # bounds sanitizer cost, not just storage
|
|
MAX_CONTENT_HTML_CHARS = 50_000 # bounds what's actually stored/rendered
|
|
|
|
ALLOWED_TAGS = {
|
|
"a",
|
|
"p",
|
|
"br",
|
|
"hr",
|
|
"strong",
|
|
"em",
|
|
"b",
|
|
"i",
|
|
"u",
|
|
"s",
|
|
"blockquote",
|
|
"code",
|
|
"pre",
|
|
"ul",
|
|
"ol",
|
|
"li",
|
|
"h1",
|
|
"h2",
|
|
"h3",
|
|
"h4",
|
|
"h5",
|
|
"h6",
|
|
"img",
|
|
"figure",
|
|
"figcaption",
|
|
"table",
|
|
"thead",
|
|
"tbody",
|
|
"tr",
|
|
"th",
|
|
"td",
|
|
"span",
|
|
"div",
|
|
}
|
|
ALLOWED_ATTRIBUTES = {
|
|
"a": {"href", "title", "rel"},
|
|
"img": {"src", "alt", "title", "width", "height"},
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class FetchResult:
|
|
status: str # Feed.STATUS_OK / STATUS_NOT_MODIFIED / STATUS_ERROR
|
|
new_article_ids: list[int]
|
|
updated_article_ids: list[int]
|
|
error_message: str = ""
|
|
|
|
|
|
def sanitize_html(raw_html: str) -> str:
|
|
if not raw_html:
|
|
return ""
|
|
bounded = raw_html[:MAX_CONTENT_HTML_INPUT_CHARS]
|
|
cleaned = nh3.clean(bounded, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None)
|
|
return cleaned[:MAX_CONTENT_HTML_CHARS]
|
|
|
|
|
|
def _content_hash(title: str, content_html: str) -> str:
|
|
return hashlib.sha256(f"{title}\n{content_html}".encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _entry_guid(entry) -> str:
|
|
if getattr(entry, "id", None):
|
|
return entry.id
|
|
if getattr(entry, "link", None):
|
|
return entry.link
|
|
title = getattr(entry, "title", "")
|
|
published = getattr(entry, "published", "")
|
|
return hashlib.sha256(f"{title}|{published}".encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _entry_published_at(entry):
|
|
parsed = getattr(entry, "published_parsed", None) or getattr(entry, "updated_parsed", None)
|
|
if not parsed:
|
|
return None
|
|
return datetime.fromtimestamp(timegm(parsed), tz=dt_timezone.utc)
|
|
|
|
|
|
def _entry_content_html(entry) -> str:
|
|
if getattr(entry, "content", None):
|
|
return entry.content[0].value
|
|
return getattr(entry, "summary", "") or ""
|
|
|
|
|
|
def effective_interval_minutes(feed: Feed, configured_interval_minutes: int) -> int:
|
|
"""The actual polling interval to use for `feed`, applying exponential
|
|
backoff on top of the site-wide `configured_interval_minutes` (see
|
|
apps.core.models.SiteSettings) for a feed that's currently erroring.
|
|
Takes the configured value as a parameter rather than querying
|
|
SiteSettings itself, since this runs once per feed (apps.feeds.tasks
|
|
calls it from a list comprehension over every active feed) and the
|
|
site-wide value doesn't vary by feed.
|
|
"""
|
|
floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15)
|
|
base = max(configured_interval_minutes, floor)
|
|
if feed.fetch_error_count <= 0:
|
|
return base
|
|
backoff = base * (2**feed.fetch_error_count)
|
|
return min(backoff, MAX_BACKOFF_MINUTES)
|
|
|
|
|
|
def fetch_and_ingest(feed: Feed) -> FetchResult:
|
|
"""Fetch one Feed over HTTP, parse it, and dedup/insert its entries.
|
|
|
|
Pure function over a Feed instance — callers are responsible for
|
|
saving it afterwards. Safe to call directly from a shell or a test,
|
|
with no Celery broker required.
|
|
"""
|
|
headers = {"User-Agent": USER_AGENT}
|
|
if feed.etag:
|
|
headers["If-None-Match"] = feed.etag
|
|
if feed.last_modified:
|
|
headers["If-Modified-Since"] = feed.last_modified
|
|
|
|
try:
|
|
response = safe_get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS)
|
|
except SSRFBlocked as exc:
|
|
feed.last_fetched_at = timezone.now()
|
|
return record_fetch_error(feed, f"Refused to fetch: {exc}")
|
|
except requests.RequestException as exc:
|
|
feed.last_fetched_at = timezone.now()
|
|
return record_fetch_error(feed, f"Request failed: {exc}")
|
|
|
|
feed.last_fetched_at = timezone.now()
|
|
|
|
if response.status_code == 304:
|
|
feed.last_fetch_status = Feed.STATUS_NOT_MODIFIED
|
|
feed.fetch_error_count = 0
|
|
return FetchResult(
|
|
status=Feed.STATUS_NOT_MODIFIED, new_article_ids=[], updated_article_ids=[]
|
|
)
|
|
|
|
if response.status_code >= 400:
|
|
return record_fetch_error(feed, f"HTTP {response.status_code}")
|
|
|
|
parsed = feedparser.parse(response.content)
|
|
|
|
if parsed.bozo and not parsed.entries:
|
|
return record_fetch_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}")
|
|
|
|
if parsed.bozo:
|
|
logger.warning(
|
|
"Feed %s parsed with warnings: %s", feed.feed_url, parsed.get("bozo_exception")
|
|
)
|
|
|
|
feed.etag = response.headers.get("ETag", "")
|
|
feed.last_modified = response.headers.get("Last-Modified", "")
|
|
feed.last_fetch_status = Feed.STATUS_OK
|
|
feed.fetch_error_count = 0
|
|
|
|
channel_title = parsed.feed.get("title")
|
|
if channel_title and not feed.title:
|
|
feed.title = channel_title[:500]
|
|
channel_link = parsed.feed.get("link")
|
|
if channel_link and not feed.site_url:
|
|
feed.site_url = channel_link[:1000]
|
|
|
|
new_ids: list[int] = []
|
|
updated_ids: list[int] = []
|
|
|
|
entries = parsed.entries
|
|
if len(entries) > MAX_ENTRIES_PER_POLL:
|
|
logger.warning(
|
|
"Feed %s: %d entries in one poll, only processing the first %d",
|
|
feed.feed_url,
|
|
len(entries),
|
|
MAX_ENTRIES_PER_POLL,
|
|
)
|
|
entries = entries[:MAX_ENTRIES_PER_POLL]
|
|
|
|
for entry in entries:
|
|
guid = _entry_guid(entry)
|
|
title = (getattr(entry, "title", "") or "")[:1000]
|
|
content_html = sanitize_html(_entry_content_html(entry))
|
|
content_hash = _content_hash(title, content_html)
|
|
|
|
article, created = Article.objects.get_or_create(
|
|
feed=feed,
|
|
guid=guid,
|
|
defaults={
|
|
"link": (getattr(entry, "link", "") or "")[:2000],
|
|
"title": title,
|
|
"author": (getattr(entry, "author", "") or "")[:500],
|
|
"content_html": content_html,
|
|
"summary": (getattr(entry, "summary", "") or "")[:5000],
|
|
"content_hash": content_hash,
|
|
"published_at": _entry_published_at(entry),
|
|
},
|
|
)
|
|
|
|
if created:
|
|
new_ids.append(article.pk)
|
|
elif article.content_hash != content_hash:
|
|
article.title = title
|
|
article.content_html = content_html
|
|
article.content_hash = content_hash
|
|
article.save(update_fields=["title", "content_html", "content_hash"])
|
|
updated_ids.append(article.pk)
|
|
|
|
return FetchResult(
|
|
status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids
|
|
)
|
|
|
|
|
|
def record_fetch_error(feed: Feed, message: str) -> FetchResult:
|
|
feed.last_fetch_status = Feed.STATUS_ERROR
|
|
feed.last_error_message = message[:2000]
|
|
feed.fetch_error_count += 1
|
|
logger.warning("Feed %s fetch error: %s", feed.feed_url, message)
|
|
return FetchResult(
|
|
status=Feed.STATUS_ERROR, new_article_ids=[], updated_article_ids=[], error_message=message
|
|
)
|