Add core data models and the feed fetch pipeline

Implements Folder/Feed/Subscription (feeds app) and Article/
UserArticleState/Tag/ArticleTag (articles app), preserving the
shared-Feed/shared-Article dedup design: a feed is fetched once
regardless of subscriber count, and per-user read/star/save/hide state
is created lazily rather than eagerly per subscriber.

Adds the fetch pipeline: feedparser-based parsing with conditional GET
(ETag/Last-Modified), nh3 HTML sanitization, guid-based dedup with
content-hash change detection that never resets a user's read/star
state, and exponential backoff on fetch errors respecting a configured
minimum interval floor. Wired into Celery via poll_due_feeds/fetch_feed
tasks, plus a minimal "add feed by URL" view that fetches synchronously
so new subscribers see results immediately.

Verified live against a real feed (hnrss.org) for ingestion, 304
dedup, and error backoff, in addition to the unit/integration test
suite (28 tests, all passing).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5 committed 2026-10-30 16:03:20 -06:00
1 parent c635cd9330
commit d200ab80f7
19 files changed
+1355 -8

No files matched your search

+196
View File
@@ -0,0 +1,196 @@
import pytest
import requests
import responses
from django.utils import timezone
from apps.articles.models import Article, UserArticleState
from apps.feeds.models import Feed
from apps.feeds.services.fetcher import effective_interval_minutes, fetch_and_ingest
FEED_URL = "https://example.com/feed.xml"
GOOD_FEED_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>Example Blog</title>
<link>https://example.com</link>
<item>
<title>First post</title>
<link>https://example.com/1</link>
<guid>https://example.com/1</guid>
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
<description>&lt;p&gt;Hello &lt;script&gt;alert(1)&lt;/script&gt;world&lt;/p&gt;</description>
</item>
</channel></rss>
"""
NO_GUID_FEED_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>No Guid Blog</title>
<item>
<title>Only a link</title>
<link>https://example.com/only-link</link>
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
<description>Body</description>
</item>
</channel></rss>
"""
MALFORMED_BUT_SALVAGEABLE_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>Slightly Broken</title>
<item>
<title>Unescaped & ampersand but still has entries</title>
<guid>guid-1</guid>
</item>
</channel></rss>
"""
TOTAL_GARBAGE = "this is not xml at all {{{"
@pytest.fixture
def feed(db):
return Feed.objects.create(feed_url=FEED_URL)
@pytest.mark.django_db
@responses.activate
def test_fetch_ingests_new_articles_and_sanitizes_html(feed):
responses.add(
responses.GET,
FEED_URL,
body=GOOD_FEED_XML,
status=200,
headers={"ETag": '"abc123"', "Last-Modified": "Mon, 01 Jan 2024 00:00:00 GMT"},
)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_OK
assert len(result.new_article_ids) == 1
assert feed.title == "Example Blog"
assert feed.etag == '"abc123"'
assert feed.last_modified == "Mon, 01 Jan 2024 00:00:00 GMT"
article = Article.objects.get(feed=feed, guid="https://example.com/1")
assert "<script>" not in article.content_html
assert "world" in article.content_html
@pytest.mark.django_db
@responses.activate
def test_second_poll_with_304_does_not_duplicate_or_error(feed):
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
fetch_and_ingest(feed)
feed.save()
assert Article.objects.filter(feed=feed).count() == 1
responses.add(responses.GET, FEED_URL, status=304)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_NOT_MODIFIED
assert Article.objects.filter(feed=feed).count() == 1
assert feed.fetch_error_count == 0
@pytest.mark.django_db
@responses.activate
def test_entry_without_guid_falls_back_to_link(feed):
responses.add(responses.GET, FEED_URL, body=NO_GUID_FEED_XML, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert len(result.new_article_ids) == 1
article = Article.objects.get(pk=result.new_article_ids[0])
assert article.guid == "https://example.com/only-link"
@pytest.mark.django_db
@responses.activate
def test_bozo_feed_with_entries_still_ingests(feed):
responses.add(responses.GET, FEED_URL, body=MALFORMED_BUT_SALVAGEABLE_XML, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_OK
assert len(result.new_article_ids) == 1
@pytest.mark.django_db
@responses.activate
def test_unparseable_feed_records_error_and_backs_off(feed):
responses.add(responses.GET, FEED_URL, body=TOTAL_GARBAGE, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_ERROR
assert feed.fetch_error_count == 1
assert effective_interval_minutes(feed) > feed.fetch_interval_minutes
@pytest.mark.django_db
@responses.activate
def test_http_500_records_error(feed):
responses.add(responses.GET, FEED_URL, status=500)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_ERROR
assert feed.last_fetch_status == Feed.STATUS_ERROR
assert feed.fetch_error_count == 1
@pytest.mark.django_db
@responses.activate
def test_connection_error_still_updates_last_fetched_at(feed):
responses.add(
responses.GET,
FEED_URL,
body=requests.exceptions.ConnectionError("boom"),
)
before = timezone.now()
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_ERROR
assert feed.last_fetched_at is not None
assert feed.last_fetched_at >= before
@pytest.mark.django_db
@responses.activate
def test_refetching_updated_content_does_not_reset_user_state():
from django.contrib.auth import get_user_model
User = get_user_model()
user = User.objects.create_user("alice", password="pw-alice-123")
feed = Feed.objects.create(feed_url=FEED_URL)
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
fetch_and_ingest(feed)
feed.save()
article = Article.objects.get(feed=feed)
state = UserArticleState.objects.create(
user=user, article=article, is_read=True, is_starred=True
)
updated_xml = GOOD_FEED_XML.replace("First post", "First post (edited)")
responses.add(responses.GET, FEED_URL, body=updated_xml, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.updated_article_ids == [article.pk]
article.refresh_from_db()
assert article.title == "First post (edited)"
state.refresh_from_db()
assert state.is_read is True
assert state.is_starred is True