Add core data models and the feed fetch pipeline

Implements Folder/Feed/Subscription (feeds app) and Article/
UserArticleState/Tag/ArticleTag (articles app), preserving the
shared-Feed/shared-Article dedup design: a feed is fetched once
regardless of subscriber count, and per-user read/star/save/hide state
is created lazily rather than eagerly per subscriber.

Adds the fetch pipeline: feedparser-based parsing with conditional GET
(ETag/Last-Modified), nh3 HTML sanitization, guid-based dedup with
content-hash change detection that never resets a user's read/star
state, and exponential backoff on fetch errors respecting a configured
minimum interval floor. Wired into Celery via poll_due_feeds/fetch_feed
tasks, plus a minimal "add feed by URL" view that fetches synchronously
so new subscribers see results immediately.

Verified live against a real feed (hnrss.org) for ingestion, 304
dedup, and error backoff, in addition to the unit/integration test
suite (28 tests, all passing).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5 committed 2026-10-30 16:03:20 -06:00
1 parent c635cd9330
commit d200ab80f7
19 files changed
+1355 -8

No files matched your search

View File
Whitespace-only changes.
+225
View File
@@ -0,0 +1,225 @@
"""HTTP fetching + feedparser glue for polling a single Feed.
Kept separate from tasks.py so the actual fetch/parse/dedup logic can be
unit-tested with plain function calls (no Celery/broker involved) and
exercised from a shell for manual verification.
"""
from __future__ import annotations
import hashlib
import logging
from calendar import timegm
from dataclasses import dataclass
from datetime import datetime
from datetime import timezone as dt_timezone
import feedparser
import nh3
import requests
from django.conf import settings
from django.utils import timezone
from apps.articles.models import Article
from apps.feeds.models import Feed
logger = logging.getLogger(__name__)
FETCH_TIMEOUT_SECONDS = 15
MAX_BACKOFF_MINUTES = 24 * 60
USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)"
ALLOWED_TAGS = {
"a",
"p",
"br",
"hr",
"strong",
"em",
"b",
"i",
"u",
"s",
"blockquote",
"code",
"pre",
"ul",
"ol",
"li",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"img",
"figure",
"figcaption",
"table",
"thead",
"tbody",
"tr",
"th",
"td",
"span",
"div",
}
ALLOWED_ATTRIBUTES = {
"a": {"href", "title", "rel"},
"img": {"src", "alt", "title", "width", "height"},
}
@dataclass
class FetchResult:
status: str # Feed.STATUS_OK / STATUS_NOT_MODIFIED / STATUS_ERROR
new_article_ids: list[int]
updated_article_ids: list[int]
error_message: str = ""
def sanitize_html(raw_html: str) -> str:
if not raw_html:
return ""
return nh3.clean(raw_html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None)
def _content_hash(title: str, content_html: str) -> str:
return hashlib.sha256(f"{title}\n{content_html}".encode("utf-8")).hexdigest()
def _entry_guid(entry) -> str:
if getattr(entry, "id", None):
return entry.id
if getattr(entry, "link", None):
return entry.link
title = getattr(entry, "title", "")
published = getattr(entry, "published", "")
return hashlib.sha256(f"{title}|{published}".encode("utf-8")).hexdigest()
def _entry_published_at(entry):
parsed = getattr(entry, "published_parsed", None) or getattr(entry, "updated_parsed", None)
if not parsed:
return None
return datetime.fromtimestamp(timegm(parsed), tz=dt_timezone.utc)
def _entry_content_html(entry) -> str:
if getattr(entry, "content", None):
return entry.content[0].value
return getattr(entry, "summary", "") or ""
def effective_interval_minutes(feed: Feed) -> int:
"""The actual polling interval to use, applying exponential backoff on
top of the admin-configured ``fetch_interval_minutes`` for feeds that
are currently erroring. ``fetch_interval_minutes`` itself is never
mutated by the fetcher — it's the admin's chosen baseline.
"""
floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15)
base = max(feed.fetch_interval_minutes, floor)
if feed.fetch_error_count <= 0:
return base
backoff = base * (2**feed.fetch_error_count)
return min(backoff, MAX_BACKOFF_MINUTES)
def fetch_and_ingest(feed: Feed) -> FetchResult:
"""Fetch one Feed over HTTP, parse it, and dedup/insert its entries.
Pure function over a Feed instance — callers are responsible for
saving it afterwards. Safe to call directly from a shell or a test,
with no Celery broker required.
"""
headers = {"User-Agent": USER_AGENT}
if feed.etag:
headers["If-None-Match"] = feed.etag
if feed.last_modified:
headers["If-Modified-Since"] = feed.last_modified
try:
response = requests.get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS)
except requests.RequestException as exc:
feed.last_fetched_at = timezone.now()
return _record_error(feed, f"Request failed: {exc}")
feed.last_fetched_at = timezone.now()
if response.status_code == 304:
feed.last_fetch_status = Feed.STATUS_NOT_MODIFIED
feed.fetch_error_count = 0
return FetchResult(
status=Feed.STATUS_NOT_MODIFIED, new_article_ids=[], updated_article_ids=[]
)
if response.status_code >= 400:
return _record_error(feed, f"HTTP {response.status_code}")
parsed = feedparser.parse(response.content)
if parsed.bozo and not parsed.entries:
return _record_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}")
if parsed.bozo:
logger.warning(
"Feed %s parsed with warnings: %s", feed.feed_url, parsed.get("bozo_exception")
)
feed.etag = response.headers.get("ETag", "")
feed.last_modified = response.headers.get("Last-Modified", "")
feed.last_fetch_status = Feed.STATUS_OK
feed.fetch_error_count = 0
channel_title = parsed.feed.get("title")
if channel_title and not feed.title:
feed.title = channel_title[:500]
channel_link = parsed.feed.get("link")
if channel_link and not feed.site_url:
feed.site_url = channel_link[:1000]
new_ids: list[int] = []
updated_ids: list[int] = []
for entry in parsed.entries:
guid = _entry_guid(entry)
title = (getattr(entry, "title", "") or "")[:1000]
content_html = sanitize_html(_entry_content_html(entry))
content_hash = _content_hash(title, content_html)
article, created = Article.objects.get_or_create(
feed=feed,
guid=guid,
defaults={
"link": (getattr(entry, "link", "") or "")[:2000],
"title": title,
"author": (getattr(entry, "author", "") or "")[:500],
"content_html": content_html,
"summary": (getattr(entry, "summary", "") or "")[:5000],
"content_hash": content_hash,
"published_at": _entry_published_at(entry),
},
)
if created:
new_ids.append(article.pk)
elif article.content_hash != content_hash:
article.title = title
article.content_html = content_html
article.content_hash = content_hash
article.save(update_fields=["title", "content_html", "content_hash"])
updated_ids.append(article.pk)
return FetchResult(
status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids
)
def _record_error(feed: Feed, message: str) -> FetchResult:
feed.last_fetch_status = Feed.STATUS_ERROR
feed.last_error_message = message[:2000]
feed.fetch_error_count += 1
logger.warning("Feed %s fetch error: %s", feed.feed_url, message)
return FetchResult(
status=Feed.STATUS_ERROR, new_article_ids=[], updated_article_ids=[], error_message=message
)