Files
Evection/apps/feeds/test_fetcher.py
T
ksmithandClaude Sonnet 5.5 f8cd1eb33e Stop retention from bringing deleted articles back (#29)
The nightly purge deleted articles, but a feed that still listed an
entry would re-insert it on its next poll as a brand-new, unread
article. For feeds with long archives, old items could reappear after
every retention run.

The purge now leaves a (feed, guid) tombstone per deleted article, and
ingestion skips tombstoned guids. With retention on, entries whose own
publish date is already outside the window are not ingested at all, so
a long archive isn't fetched only to be purged and tombstoned the same
night. Undated entries rely on the tombstone.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-01 09:30:00 -07:00

402 lines
13 KiB
Python

import datetime as dt
import pytest
import requests
import responses
from django.utils import timezone
from apps.articles.models import Article, UserArticleState
from apps.feeds.models import Feed
from apps.feeds.services.fetcher import (
MAX_CONTENT_HTML_CHARS,
MAX_ENTRIES_PER_POLL,
effective_interval_minutes,
fetch_and_ingest,
sanitize_html,
)
FEED_URL = "https://example.com/feed.xml"
GOOD_FEED_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>Example Blog</title>
<link>https://example.com</link>
<item>
<title>First post</title>
<link>https://example.com/1</link>
<guid>https://example.com/1</guid>
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
<description>&lt;p&gt;Hello &lt;script&gt;alert(1)&lt;/script&gt;world&lt;/p&gt;</description>
</item>
</channel></rss>
"""
NO_GUID_FEED_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>No Guid Blog</title>
<item>
<title>Only a link</title>
<link>https://example.com/only-link</link>
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
<description>Body</description>
</item>
</channel></rss>
"""
MALFORMED_BUT_SALVAGEABLE_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>Slightly Broken</title>
<item>
<title>Unescaped & ampersand but still has entries</title>
<guid>guid-1</guid>
</item>
</channel></rss>
"""
TOTAL_GARBAGE = "this is not xml at all {{{"
@pytest.fixture
def feed(db):
return Feed.objects.create(feed_url=FEED_URL)
@pytest.mark.django_db
@responses.activate
def test_fetch_ingests_new_articles_and_sanitizes_html(feed):
responses.add(
responses.GET,
FEED_URL,
body=GOOD_FEED_XML,
status=200,
headers={"ETag": '"abc123"', "Last-Modified": "Mon, 01 Jan 2024 00:00:00 GMT"},
)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_OK
assert len(result.new_article_ids) == 1
assert feed.title == "Example Blog"
assert feed.etag == '"abc123"'
assert feed.last_modified == "Mon, 01 Jan 2024 00:00:00 GMT"
article = Article.objects.get(feed=feed, guid="https://example.com/1")
assert "<script>" not in article.content_html
assert "world" in article.content_html
@pytest.mark.django_db
@responses.activate
def test_second_poll_with_304_does_not_duplicate_or_error(feed):
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
fetch_and_ingest(feed)
feed.save()
assert Article.objects.filter(feed=feed).count() == 1
responses.add(responses.GET, FEED_URL, status=304)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_NOT_MODIFIED
assert Article.objects.filter(feed=feed).count() == 1
assert feed.fetch_error_count == 0
@pytest.mark.django_db
@responses.activate
def test_entry_without_guid_falls_back_to_link(feed):
responses.add(responses.GET, FEED_URL, body=NO_GUID_FEED_XML, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert len(result.new_article_ids) == 1
article = Article.objects.get(pk=result.new_article_ids[0])
assert article.guid == "https://example.com/only-link"
@pytest.mark.django_db
@responses.activate
def test_bozo_feed_with_entries_still_ingests(feed):
responses.add(responses.GET, FEED_URL, body=MALFORMED_BUT_SALVAGEABLE_XML, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_OK
assert len(result.new_article_ids) == 1
@pytest.mark.django_db
@responses.activate
def test_unparseable_feed_records_error_and_backs_off(feed):
responses.add(responses.GET, FEED_URL, body=TOTAL_GARBAGE, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_ERROR
assert feed.fetch_error_count == 1
assert effective_interval_minutes(feed, 60) > 60
@pytest.mark.django_db
@responses.activate
def test_http_500_records_error(feed):
responses.add(responses.GET, FEED_URL, status=500)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_ERROR
assert feed.last_fetch_status == Feed.STATUS_ERROR
assert feed.fetch_error_count == 1
@pytest.mark.django_db
@responses.activate
def test_connection_error_still_updates_last_fetched_at(feed):
responses.add(
responses.GET,
FEED_URL,
body=requests.exceptions.ConnectionError("boom"),
)
before = timezone.now()
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_ERROR
assert feed.last_fetched_at is not None
assert feed.last_fetched_at >= before
@pytest.mark.django_db
@responses.activate
def test_refetching_updated_content_does_not_reset_user_state():
from django.contrib.auth import get_user_model
User = get_user_model()
user = User.objects.create_user("alice", password="pw-alice-123")
feed = Feed.objects.create(feed_url=FEED_URL)
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
fetch_and_ingest(feed)
feed.save()
from apps.articles.models import STARRED_TAG_NAME, ArticleTag, get_system_tag
article = Article.objects.get(feed=feed)
state = UserArticleState.objects.create(user=user, article=article, is_read=True)
starred_tag = get_system_tag(user, STARRED_TAG_NAME)
ArticleTag.objects.create(user=user, article=article, tag=starred_tag)
updated_xml = GOOD_FEED_XML.replace("First post", "First post (edited)")
responses.add(responses.GET, FEED_URL, body=updated_xml, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.updated_article_ids == [article.pk]
article.refresh_from_db()
assert article.title == "First post (edited)"
state.refresh_from_db()
assert state.is_read is True
assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists()
@pytest.mark.django_db
@responses.activate
def test_refetching_picks_up_link_author_summary_and_published_at_changes():
"""Previously only title/content_html/content_hash were updated on
refetch (T6) — a feed correcting its link, author, summary, or
publication date without touching the title went unnoticed forever."""
feed = Feed.objects.create(feed_url=FEED_URL)
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
fetch_and_ingest(feed)
feed.save()
article = Article.objects.get(feed=feed)
assert article.link == "https://example.com/1"
assert article.author == ""
edited_xml = GOOD_FEED_XML.replace(
"<link>https://example.com/1</link>",
"<link>https://example.com/1-moved</link>",
).replace(
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>",
"<pubDate>Tue, 02 Jan 2024 00:00:00 GMT</pubDate>\n <author>jane@example.com</author>",
)
responses.add(responses.GET, FEED_URL, body=edited_xml, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.updated_article_ids == [article.pk]
article.refresh_from_db()
assert article.link == "https://example.com/1-moved"
assert article.author == "jane@example.com"
assert article.published_at == dt.datetime(2024, 1, 2, tzinfo=dt.timezone.utc)
def test_sanitize_html_caps_stored_output_length():
huge = "<p>" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "</p>"
cleaned = sanitize_html(huge)
assert len(cleaned) <= MAX_CONTENT_HTML_CHARS
@pytest.mark.django_db
@responses.activate
def test_fetch_only_processes_the_first_max_entries_per_poll(feed):
items = "".join(
f"<item><title>Item {i}</title><guid>guid-{i}</guid></item>"
for i in range(MAX_ENTRIES_PER_POLL + 5)
)
xml = f"""<?xml version="1.0"?>
<rss version="2.0"><channel><title>Big feed</title>{items}</channel></rss>
"""
responses.add(responses.GET, FEED_URL, body=xml, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert len(result.new_article_ids) == MAX_ENTRIES_PER_POLL
assert Article.objects.filter(feed=feed).count() == MAX_ENTRIES_PER_POLL
assert not Article.objects.filter(feed=feed, guid=f"guid-{MAX_ENTRIES_PER_POLL}").exists()
def _feed_xml(count, title_suffix=""):
items = "".join(
f"<item><title>Item {i}{title_suffix}</title><guid>guid-{i}</guid></item>"
for i in range(count)
)
return (
f'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>{items}</channel></rss>'
)
@pytest.mark.django_db
@responses.activate
def test_ingest_uses_a_constant_number_of_article_queries(feed):
"""C2: one prefetch + bulk writes per poll, not a get_or_create per
entry — query count must not grow with the number of entries."""
from django.db import connection
from django.test.utils import CaptureQueriesContext
responses.add(responses.GET, FEED_URL, body=_feed_xml(40), status=200)
with CaptureQueriesContext(connection) as ctx:
result = fetch_and_ingest(feed)
assert len(result.new_article_ids) == 40
assert len(ctx.captured_queries) < 10
assert Article.objects.filter(feed=feed).count() == 40
@pytest.mark.django_db
@responses.activate
def test_ingest_dedups_a_guid_repeated_within_one_response(feed):
xml = (
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
"<item><title>A</title><guid>same</guid></item>"
"<item><title>B</title><guid>same</guid></item>"
"</channel></rss>"
)
responses.add(responses.GET, FEED_URL, body=xml, status=200)
result = fetch_and_ingest(feed)
assert len(result.new_article_ids) == 1
assert Article.objects.filter(feed=feed).count() == 1
@pytest.mark.django_db
@responses.activate
def test_ingest_reports_updates_separately_from_new_articles(feed):
responses.add(responses.GET, FEED_URL, body=_feed_xml(3), status=200)
fetch_and_ingest(feed)
responses.replace(responses.GET, FEED_URL, body=_feed_xml(4, " edited"), status=200)
result = fetch_and_ingest(feed)
assert len(result.new_article_ids) == 1
assert len(result.updated_article_ids) == 3
assert Article.objects.get(feed=feed, guid="guid-0").title == "Item 0 edited"
@pytest.mark.django_db
@responses.activate
def test_ingest_falls_back_to_row_by_row_when_a_concurrent_insert_wins(feed):
from unittest.mock import patch
from django.db import IntegrityError
responses.add(responses.GET, FEED_URL, body=_feed_xml(3), status=200)
with patch.object(Article.objects, "bulk_create", side_effect=IntegrityError("race")):
result = fetch_and_ingest(feed)
assert len(result.new_article_ids) == 3
assert Article.objects.filter(feed=feed).count() == 3
@pytest.mark.django_db
@responses.activate
def test_purged_article_is_not_resurrected_by_a_feed_that_still_lists_it(feed):
"""#29: a feed still listing an entry the retention purge deleted must
not bring it back as a brand-new, unread article."""
from apps.articles.tasks import purge_old_articles
from apps.core.models import SiteSettings
SiteSettings.objects.create(retention_days=30)
responses.add(responses.GET, FEED_URL, body=_feed_xml(3), status=200)
fetch_and_ingest(feed)
Article.objects.filter(feed=feed, guid="guid-0").update(
fetched_at=timezone.now() - dt.timedelta(days=90)
)
assert purge_old_articles() == 1
assert not Article.objects.filter(feed=feed, guid="guid-0").exists()
result = fetch_and_ingest(feed)
assert result.new_article_ids == []
assert not Article.objects.filter(feed=feed, guid="guid-0").exists()
assert Article.objects.filter(feed=feed).count() == 2
@pytest.mark.django_db
@responses.activate
def test_entries_older_than_the_retention_window_are_not_ingested(feed):
"""With retention on, a feed's long archive isn't ingested only to be
purged (and tombstoned) the same night."""
from apps.core.models import SiteSettings
SiteSettings.objects.create(retention_days=30)
xml = (
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
"<item><title>Ancient</title><guid>old</guid>"
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate></item>"
"<item><title>Undated</title><guid>undated</guid></item>"
"</channel></rss>"
)
responses.add(responses.GET, FEED_URL, body=xml, status=200)
result = fetch_and_ingest(feed)
assert len(result.new_article_ids) == 1
assert Article.objects.filter(feed=feed).get().guid == "undated"
@pytest.mark.django_db
@responses.activate
def test_old_entries_are_ingested_normally_when_retention_is_off(feed):
xml = (
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
"<item><title>Ancient</title><guid>old</guid>"
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate></item>"
"</channel></rss>"
)
responses.add(responses.GET, FEED_URL, body=xml, status=200)
assert len(fetch_and_ingest(feed).new_article_ids) == 1