Add core data models and the feed fetch pipeline

Implements Folder/Feed/Subscription (feeds app) and Article/
UserArticleState/Tag/ArticleTag (articles app), preserving the
shared-Feed/shared-Article dedup design: a feed is fetched once
regardless of subscriber count, and per-user read/star/save/hide state
is created lazily rather than eagerly per subscriber.

Adds the fetch pipeline: feedparser-based parsing with conditional GET
(ETag/Last-Modified), nh3 HTML sanitization, guid-based dedup with
content-hash change detection that never resets a user's read/star
state, and exponential backoff on fetch errors respecting a configured
minimum interval floor. Wired into Celery via poll_due_feeds/fetch_feed
tasks, plus a minimal "add feed by URL" view that fetches synchronously
so new subscribers see results immediately.

Verified live against a real feed (hnrss.org) for ingestion, 304
dedup, and error backoff, in addition to the unit/integration test
suite (28 tests, all passing).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5 committed 2026-10-30 16:03:20 -06:00
1 parent c635cd9330
commit d200ab80f7
19 files changed
+1355 -8

No files matched your search

+77
View File
@@ -0,0 +1,77 @@
from datetime import timedelta
from unittest.mock import patch
import pytest
import responses
from django.utils import timezone
from apps.articles.models import Article
from apps.feeds.models import Feed
from apps.feeds.tasks import fetch_feed, poll_due_feeds
@pytest.mark.django_db
def test_poll_due_feeds_dispatches_only_due_feeds():
never_fetched = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
recently_fetched = Feed.objects.create(
feed_url="https://b.example.com/feed.xml",
last_fetched_at=timezone.now(),
fetch_interval_minutes=60,
)
overdue = Feed.objects.create(
feed_url="https://c.example.com/feed.xml",
last_fetched_at=timezone.now() - timedelta(hours=2),
fetch_interval_minutes=60,
)
inactive_but_overdue = Feed.objects.create(
feed_url="https://d.example.com/feed.xml",
last_fetched_at=timezone.now() - timedelta(hours=2),
fetch_interval_minutes=60,
is_active=False,
)
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
dispatched_count = poll_due_feeds()
dispatched_ids = {call.args[0] for call in mock_delay.call_args_list}
assert dispatched_ids == {never_fetched.id, overdue.id}
assert dispatched_count == 2
assert recently_fetched.id not in dispatched_ids
assert inactive_but_overdue.id not in dispatched_ids
GOOD_FEED_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>Example Blog</title>
<item>
<title>First post</title>
<guid>guid-1</guid>
</item>
</channel></rss>
"""
@pytest.mark.django_db
@responses.activate
def test_fetch_feed_task_ingests_and_saves_feed_state():
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
result = fetch_feed(feed.id)
assert result["status"] == Feed.STATUS_OK
assert result["new"] == 1
feed.refresh_from_db()
assert feed.last_fetch_status == Feed.STATUS_OK
assert Article.objects.filter(feed=feed, guid="guid-1").exists()
@pytest.mark.django_db
def test_fetch_feed_task_skips_inactive_feed():
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", is_active=False)
assert fetch_feed(feed.id) is None
@pytest.mark.django_db
def test_fetch_feed_task_skips_missing_feed():
assert fetch_feed(999999) is None