Private
Public Access
Add core data models and the feed fetch pipeline
Implements Folder/Feed/Subscription (feeds app) and Article/ UserArticleState/Tag/ArticleTag (articles app), preserving the shared-Feed/shared-Article dedup design: a feed is fetched once regardless of subscriber count, and per-user read/star/save/hide state is created lazily rather than eagerly per subscriber. Adds the fetch pipeline: feedparser-based parsing with conditional GET (ETag/Last-Modified), nh3 HTML sanitization, guid-based dedup with content-hash change detection that never resets a user's read/star state, and exponential backoff on fetch errors respecting a configured minimum interval floor. Wired into Celery via poll_due_feeds/fetch_feed tasks, plus a minimal "add feed by URL" view that fetches synchronously so new subscribers see results immediately. Verified live against a real feed (hnrss.org) for ingestion, 304 dedup, and error backoff, in addition to the unit/integration test suite (28 tests, all passing). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c635cd9330
commit
d200ab80f7
19 files changed
+1355
-8
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,225 @@
|
||||
"""HTTP fetching + feedparser glue for polling a single Feed.
|
||||
|
||||
Kept separate from tasks.py so the actual fetch/parse/dedup logic can be
|
||||
unit-tested with plain function calls (no Celery/broker involved) and
|
||||
exercised from a shell for manual verification.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from calendar import timegm
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from datetime import timezone as dt_timezone
|
||||
|
||||
import feedparser
|
||||
import nh3
|
||||
import requests
|
||||
from django.conf import settings
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.articles.models import Article
|
||||
from apps.feeds.models import Feed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FETCH_TIMEOUT_SECONDS = 15
|
||||
MAX_BACKOFF_MINUTES = 24 * 60
|
||||
USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)"
|
||||
|
||||
ALLOWED_TAGS = {
|
||||
"a",
|
||||
"p",
|
||||
"br",
|
||||
"hr",
|
||||
"strong",
|
||||
"em",
|
||||
"b",
|
||||
"i",
|
||||
"u",
|
||||
"s",
|
||||
"blockquote",
|
||||
"code",
|
||||
"pre",
|
||||
"ul",
|
||||
"ol",
|
||||
"li",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"img",
|
||||
"figure",
|
||||
"figcaption",
|
||||
"table",
|
||||
"thead",
|
||||
"tbody",
|
||||
"tr",
|
||||
"th",
|
||||
"td",
|
||||
"span",
|
||||
"div",
|
||||
}
|
||||
ALLOWED_ATTRIBUTES = {
|
||||
"a": {"href", "title", "rel"},
|
||||
"img": {"src", "alt", "title", "width", "height"},
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class FetchResult:
|
||||
status: str # Feed.STATUS_OK / STATUS_NOT_MODIFIED / STATUS_ERROR
|
||||
new_article_ids: list[int]
|
||||
updated_article_ids: list[int]
|
||||
error_message: str = ""
|
||||
|
||||
|
||||
def sanitize_html(raw_html: str) -> str:
|
||||
if not raw_html:
|
||||
return ""
|
||||
return nh3.clean(raw_html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None)
|
||||
|
||||
|
||||
def _content_hash(title: str, content_html: str) -> str:
|
||||
return hashlib.sha256(f"{title}\n{content_html}".encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _entry_guid(entry) -> str:
|
||||
if getattr(entry, "id", None):
|
||||
return entry.id
|
||||
if getattr(entry, "link", None):
|
||||
return entry.link
|
||||
title = getattr(entry, "title", "")
|
||||
published = getattr(entry, "published", "")
|
||||
return hashlib.sha256(f"{title}|{published}".encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _entry_published_at(entry):
|
||||
parsed = getattr(entry, "published_parsed", None) or getattr(entry, "updated_parsed", None)
|
||||
if not parsed:
|
||||
return None
|
||||
return datetime.fromtimestamp(timegm(parsed), tz=dt_timezone.utc)
|
||||
|
||||
|
||||
def _entry_content_html(entry) -> str:
|
||||
if getattr(entry, "content", None):
|
||||
return entry.content[0].value
|
||||
return getattr(entry, "summary", "") or ""
|
||||
|
||||
|
||||
def effective_interval_minutes(feed: Feed) -> int:
|
||||
"""The actual polling interval to use, applying exponential backoff on
|
||||
top of the admin-configured ``fetch_interval_minutes`` for feeds that
|
||||
are currently erroring. ``fetch_interval_minutes`` itself is never
|
||||
mutated by the fetcher — it's the admin's chosen baseline.
|
||||
"""
|
||||
floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15)
|
||||
base = max(feed.fetch_interval_minutes, floor)
|
||||
if feed.fetch_error_count <= 0:
|
||||
return base
|
||||
backoff = base * (2**feed.fetch_error_count)
|
||||
return min(backoff, MAX_BACKOFF_MINUTES)
|
||||
|
||||
|
||||
def fetch_and_ingest(feed: Feed) -> FetchResult:
|
||||
"""Fetch one Feed over HTTP, parse it, and dedup/insert its entries.
|
||||
|
||||
Pure function over a Feed instance — callers are responsible for
|
||||
saving it afterwards. Safe to call directly from a shell or a test,
|
||||
with no Celery broker required.
|
||||
"""
|
||||
headers = {"User-Agent": USER_AGENT}
|
||||
if feed.etag:
|
||||
headers["If-None-Match"] = feed.etag
|
||||
if feed.last_modified:
|
||||
headers["If-Modified-Since"] = feed.last_modified
|
||||
|
||||
try:
|
||||
response = requests.get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS)
|
||||
except requests.RequestException as exc:
|
||||
feed.last_fetched_at = timezone.now()
|
||||
return _record_error(feed, f"Request failed: {exc}")
|
||||
|
||||
feed.last_fetched_at = timezone.now()
|
||||
|
||||
if response.status_code == 304:
|
||||
feed.last_fetch_status = Feed.STATUS_NOT_MODIFIED
|
||||
feed.fetch_error_count = 0
|
||||
return FetchResult(
|
||||
status=Feed.STATUS_NOT_MODIFIED, new_article_ids=[], updated_article_ids=[]
|
||||
)
|
||||
|
||||
if response.status_code >= 400:
|
||||
return _record_error(feed, f"HTTP {response.status_code}")
|
||||
|
||||
parsed = feedparser.parse(response.content)
|
||||
|
||||
if parsed.bozo and not parsed.entries:
|
||||
return _record_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}")
|
||||
|
||||
if parsed.bozo:
|
||||
logger.warning(
|
||||
"Feed %s parsed with warnings: %s", feed.feed_url, parsed.get("bozo_exception")
|
||||
)
|
||||
|
||||
feed.etag = response.headers.get("ETag", "")
|
||||
feed.last_modified = response.headers.get("Last-Modified", "")
|
||||
feed.last_fetch_status = Feed.STATUS_OK
|
||||
feed.fetch_error_count = 0
|
||||
|
||||
channel_title = parsed.feed.get("title")
|
||||
if channel_title and not feed.title:
|
||||
feed.title = channel_title[:500]
|
||||
channel_link = parsed.feed.get("link")
|
||||
if channel_link and not feed.site_url:
|
||||
feed.site_url = channel_link[:1000]
|
||||
|
||||
new_ids: list[int] = []
|
||||
updated_ids: list[int] = []
|
||||
|
||||
for entry in parsed.entries:
|
||||
guid = _entry_guid(entry)
|
||||
title = (getattr(entry, "title", "") or "")[:1000]
|
||||
content_html = sanitize_html(_entry_content_html(entry))
|
||||
content_hash = _content_hash(title, content_html)
|
||||
|
||||
article, created = Article.objects.get_or_create(
|
||||
feed=feed,
|
||||
guid=guid,
|
||||
defaults={
|
||||
"link": (getattr(entry, "link", "") or "")[:2000],
|
||||
"title": title,
|
||||
"author": (getattr(entry, "author", "") or "")[:500],
|
||||
"content_html": content_html,
|
||||
"summary": (getattr(entry, "summary", "") or "")[:5000],
|
||||
"content_hash": content_hash,
|
||||
"published_at": _entry_published_at(entry),
|
||||
},
|
||||
)
|
||||
|
||||
if created:
|
||||
new_ids.append(article.pk)
|
||||
elif article.content_hash != content_hash:
|
||||
article.title = title
|
||||
article.content_html = content_html
|
||||
article.content_hash = content_hash
|
||||
article.save(update_fields=["title", "content_html", "content_hash"])
|
||||
updated_ids.append(article.pk)
|
||||
|
||||
return FetchResult(
|
||||
status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids
|
||||
)
|
||||
|
||||
|
||||
def _record_error(feed: Feed, message: str) -> FetchResult:
|
||||
feed.last_fetch_status = Feed.STATUS_ERROR
|
||||
feed.last_error_message = message[:2000]
|
||||
feed.fetch_error_count += 1
|
||||
logger.warning("Feed %s fetch error: %s", feed.feed_url, message)
|
||||
return FetchResult(
|
||||
status=Feed.STATUS_ERROR, new_article_ids=[], updated_article_ids=[], error_message=message
|
||||
)
|
||||
Reference in new issue
Block a user