Private
Public Access
Implements Folder/Feed/Subscription (feeds app) and Article/ UserArticleState/Tag/ArticleTag (articles app), preserving the shared-Feed/shared-Article dedup design: a feed is fetched once regardless of subscriber count, and per-user read/star/save/hide state is created lazily rather than eagerly per subscriber. Adds the fetch pipeline: feedparser-based parsing with conditional GET (ETag/Last-Modified), nh3 HTML sanitization, guid-based dedup with content-hash change detection that never resets a user's read/star state, and exponential backoff on fetch errors respecting a configured minimum interval floor. Wired into Celery via poll_due_feeds/fetch_feed tasks, plus a minimal "add feed by URL" view that fetches synchronously so new subscribers see results immediately. Verified live against a real feed (hnrss.org) for ingestion, 304 dedup, and error backoff, in addition to the unit/integration test suite (28 tests, all passing). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
78 lines
2.4 KiB
Python
78 lines
2.4 KiB
Python
from datetime import timedelta
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
import responses
|
|
from django.utils import timezone
|
|
|
|
from apps.articles.models import Article
|
|
from apps.feeds.models import Feed
|
|
from apps.feeds.tasks import fetch_feed, poll_due_feeds
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_poll_due_feeds_dispatches_only_due_feeds():
|
|
never_fetched = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
|
recently_fetched = Feed.objects.create(
|
|
feed_url="https://b.example.com/feed.xml",
|
|
last_fetched_at=timezone.now(),
|
|
fetch_interval_minutes=60,
|
|
)
|
|
overdue = Feed.objects.create(
|
|
feed_url="https://c.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
fetch_interval_minutes=60,
|
|
)
|
|
inactive_but_overdue = Feed.objects.create(
|
|
feed_url="https://d.example.com/feed.xml",
|
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
|
fetch_interval_minutes=60,
|
|
is_active=False,
|
|
)
|
|
|
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
|
dispatched_count = poll_due_feeds()
|
|
|
|
dispatched_ids = {call.args[0] for call in mock_delay.call_args_list}
|
|
assert dispatched_ids == {never_fetched.id, overdue.id}
|
|
assert dispatched_count == 2
|
|
assert recently_fetched.id not in dispatched_ids
|
|
assert inactive_but_overdue.id not in dispatched_ids
|
|
|
|
|
|
GOOD_FEED_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>Example Blog</title>
|
|
<item>
|
|
<title>First post</title>
|
|
<guid>guid-1</guid>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_feed_task_ingests_and_saves_feed_state():
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
|
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
|
|
|
result = fetch_feed(feed.id)
|
|
|
|
assert result["status"] == Feed.STATUS_OK
|
|
assert result["new"] == 1
|
|
feed.refresh_from_db()
|
|
assert feed.last_fetch_status == Feed.STATUS_OK
|
|
assert Article.objects.filter(feed=feed, guid="guid-1").exists()
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_fetch_feed_task_skips_inactive_feed():
|
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", is_active=False)
|
|
assert fetch_feed(feed.id) is None
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_fetch_feed_task_skips_missing_feed():
|
|
assert fetch_feed(999999) is None
|