Private
Public Access
Implements Folder/Feed/Subscription (feeds app) and Article/ UserArticleState/Tag/ArticleTag (articles app), preserving the shared-Feed/shared-Article dedup design: a feed is fetched once regardless of subscriber count, and per-user read/star/save/hide state is created lazily rather than eagerly per subscriber. Adds the fetch pipeline: feedparser-based parsing with conditional GET (ETag/Last-Modified), nh3 HTML sanitization, guid-based dedup with content-hash change detection that never resets a user's read/star state, and exponential backoff on fetch errors respecting a configured minimum interval floor. Wired into Celery via poll_due_feeds/fetch_feed tasks, plus a minimal "add feed by URL" view that fetches synchronously so new subscribers see results immediately. Verified live against a real feed (hnrss.org) for ingestion, 304 dedup, and error backoff, in addition to the unit/integration test suite (28 tests, all passing). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
197 lines
5.5 KiB
Python
197 lines
5.5 KiB
Python
import pytest
|
|
import requests
|
|
import responses
|
|
from django.utils import timezone
|
|
|
|
from apps.articles.models import Article, UserArticleState
|
|
from apps.feeds.models import Feed
|
|
from apps.feeds.services.fetcher import effective_interval_minutes, fetch_and_ingest
|
|
|
|
FEED_URL = "https://example.com/feed.xml"
|
|
|
|
GOOD_FEED_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>Example Blog</title>
|
|
<link>https://example.com</link>
|
|
<item>
|
|
<title>First post</title>
|
|
<link>https://example.com/1</link>
|
|
<guid>https://example.com/1</guid>
|
|
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
|
<description><p>Hello <script>alert(1)</script>world</p></description>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
NO_GUID_FEED_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>No Guid Blog</title>
|
|
<item>
|
|
<title>Only a link</title>
|
|
<link>https://example.com/only-link</link>
|
|
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
|
<description>Body</description>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
MALFORMED_BUT_SALVAGEABLE_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>Slightly Broken</title>
|
|
<item>
|
|
<title>Unescaped & ampersand but still has entries</title>
|
|
<guid>guid-1</guid>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
TOTAL_GARBAGE = "this is not xml at all {{{"
|
|
|
|
|
|
@pytest.fixture
|
|
def feed(db):
|
|
return Feed.objects.create(feed_url=FEED_URL)
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_ingests_new_articles_and_sanitizes_html(feed):
|
|
responses.add(
|
|
responses.GET,
|
|
FEED_URL,
|
|
body=GOOD_FEED_XML,
|
|
status=200,
|
|
headers={"ETag": '"abc123"', "Last-Modified": "Mon, 01 Jan 2024 00:00:00 GMT"},
|
|
)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_OK
|
|
assert len(result.new_article_ids) == 1
|
|
assert feed.title == "Example Blog"
|
|
assert feed.etag == '"abc123"'
|
|
assert feed.last_modified == "Mon, 01 Jan 2024 00:00:00 GMT"
|
|
|
|
article = Article.objects.get(feed=feed, guid="https://example.com/1")
|
|
assert "<script>" not in article.content_html
|
|
assert "world" in article.content_html
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_second_poll_with_304_does_not_duplicate_or_error(feed):
|
|
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
|
|
fetch_and_ingest(feed)
|
|
feed.save()
|
|
assert Article.objects.filter(feed=feed).count() == 1
|
|
|
|
responses.add(responses.GET, FEED_URL, status=304)
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_NOT_MODIFIED
|
|
assert Article.objects.filter(feed=feed).count() == 1
|
|
assert feed.fetch_error_count == 0
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_entry_without_guid_falls_back_to_link(feed):
|
|
responses.add(responses.GET, FEED_URL, body=NO_GUID_FEED_XML, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert len(result.new_article_ids) == 1
|
|
article = Article.objects.get(pk=result.new_article_ids[0])
|
|
assert article.guid == "https://example.com/only-link"
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_bozo_feed_with_entries_still_ingests(feed):
|
|
responses.add(responses.GET, FEED_URL, body=MALFORMED_BUT_SALVAGEABLE_XML, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_OK
|
|
assert len(result.new_article_ids) == 1
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_unparseable_feed_records_error_and_backs_off(feed):
|
|
responses.add(responses.GET, FEED_URL, body=TOTAL_GARBAGE, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_ERROR
|
|
assert feed.fetch_error_count == 1
|
|
assert effective_interval_minutes(feed) > feed.fetch_interval_minutes
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_http_500_records_error(feed):
|
|
responses.add(responses.GET, FEED_URL, status=500)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_ERROR
|
|
assert feed.last_fetch_status == Feed.STATUS_ERROR
|
|
assert feed.fetch_error_count == 1
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_connection_error_still_updates_last_fetched_at(feed):
|
|
responses.add(
|
|
responses.GET,
|
|
FEED_URL,
|
|
body=requests.exceptions.ConnectionError("boom"),
|
|
)
|
|
|
|
before = timezone.now()
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_ERROR
|
|
assert feed.last_fetched_at is not None
|
|
assert feed.last_fetched_at >= before
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_refetching_updated_content_does_not_reset_user_state():
|
|
from django.contrib.auth import get_user_model
|
|
|
|
User = get_user_model()
|
|
user = User.objects.create_user("alice", password="pw-alice-123")
|
|
feed = Feed.objects.create(feed_url=FEED_URL)
|
|
|
|
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
|
|
fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
article = Article.objects.get(feed=feed)
|
|
state = UserArticleState.objects.create(
|
|
user=user, article=article, is_read=True, is_starred=True
|
|
)
|
|
|
|
updated_xml = GOOD_FEED_XML.replace("First post", "First post (edited)")
|
|
responses.add(responses.GET, FEED_URL, body=updated_xml, status=200)
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.updated_article_ids == [article.pk]
|
|
article.refresh_from_db()
|
|
assert article.title == "First post (edited)"
|
|
|
|
state.refresh_from_db()
|
|
assert state.is_read is True
|
|
assert state.is_starred is True
|