Private
Public Access
The nightly purge deleted articles, but a feed that still listed an entry would re-insert it on its next poll as a brand-new, unread article. For feeds with long archives, old items could reappear after every retention run. The purge now leaves a (feed, guid) tombstone per deleted article, and ingestion skips tombstoned guids. With retention on, entries whose own publish date is already outside the window are not ingested at all, so a long archive isn't fetched only to be purged and tombstoned the same night. Undated entries rely on the tombstone. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
402 lines
13 KiB
Python
402 lines
13 KiB
Python
import datetime as dt
|
|
|
|
import pytest
|
|
import requests
|
|
import responses
|
|
from django.utils import timezone
|
|
|
|
from apps.articles.models import Article, UserArticleState
|
|
from apps.feeds.models import Feed
|
|
from apps.feeds.services.fetcher import (
|
|
MAX_CONTENT_HTML_CHARS,
|
|
MAX_ENTRIES_PER_POLL,
|
|
effective_interval_minutes,
|
|
fetch_and_ingest,
|
|
sanitize_html,
|
|
)
|
|
|
|
FEED_URL = "https://example.com/feed.xml"
|
|
|
|
GOOD_FEED_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>Example Blog</title>
|
|
<link>https://example.com</link>
|
|
<item>
|
|
<title>First post</title>
|
|
<link>https://example.com/1</link>
|
|
<guid>https://example.com/1</guid>
|
|
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
|
<description><p>Hello <script>alert(1)</script>world</p></description>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
NO_GUID_FEED_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>No Guid Blog</title>
|
|
<item>
|
|
<title>Only a link</title>
|
|
<link>https://example.com/only-link</link>
|
|
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
|
<description>Body</description>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
MALFORMED_BUT_SALVAGEABLE_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>Slightly Broken</title>
|
|
<item>
|
|
<title>Unescaped & ampersand but still has entries</title>
|
|
<guid>guid-1</guid>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
TOTAL_GARBAGE = "this is not xml at all {{{"
|
|
|
|
|
|
@pytest.fixture
|
|
def feed(db):
|
|
return Feed.objects.create(feed_url=FEED_URL)
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_ingests_new_articles_and_sanitizes_html(feed):
|
|
responses.add(
|
|
responses.GET,
|
|
FEED_URL,
|
|
body=GOOD_FEED_XML,
|
|
status=200,
|
|
headers={"ETag": '"abc123"', "Last-Modified": "Mon, 01 Jan 2024 00:00:00 GMT"},
|
|
)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_OK
|
|
assert len(result.new_article_ids) == 1
|
|
assert feed.title == "Example Blog"
|
|
assert feed.etag == '"abc123"'
|
|
assert feed.last_modified == "Mon, 01 Jan 2024 00:00:00 GMT"
|
|
|
|
article = Article.objects.get(feed=feed, guid="https://example.com/1")
|
|
assert "<script>" not in article.content_html
|
|
assert "world" in article.content_html
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_second_poll_with_304_does_not_duplicate_or_error(feed):
|
|
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
|
|
fetch_and_ingest(feed)
|
|
feed.save()
|
|
assert Article.objects.filter(feed=feed).count() == 1
|
|
|
|
responses.add(responses.GET, FEED_URL, status=304)
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_NOT_MODIFIED
|
|
assert Article.objects.filter(feed=feed).count() == 1
|
|
assert feed.fetch_error_count == 0
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_entry_without_guid_falls_back_to_link(feed):
|
|
responses.add(responses.GET, FEED_URL, body=NO_GUID_FEED_XML, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert len(result.new_article_ids) == 1
|
|
article = Article.objects.get(pk=result.new_article_ids[0])
|
|
assert article.guid == "https://example.com/only-link"
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_bozo_feed_with_entries_still_ingests(feed):
|
|
responses.add(responses.GET, FEED_URL, body=MALFORMED_BUT_SALVAGEABLE_XML, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_OK
|
|
assert len(result.new_article_ids) == 1
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_unparseable_feed_records_error_and_backs_off(feed):
|
|
responses.add(responses.GET, FEED_URL, body=TOTAL_GARBAGE, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_ERROR
|
|
assert feed.fetch_error_count == 1
|
|
assert effective_interval_minutes(feed, 60) > 60
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_http_500_records_error(feed):
|
|
responses.add(responses.GET, FEED_URL, status=500)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_ERROR
|
|
assert feed.last_fetch_status == Feed.STATUS_ERROR
|
|
assert feed.fetch_error_count == 1
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_connection_error_still_updates_last_fetched_at(feed):
|
|
responses.add(
|
|
responses.GET,
|
|
FEED_URL,
|
|
body=requests.exceptions.ConnectionError("boom"),
|
|
)
|
|
|
|
before = timezone.now()
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_ERROR
|
|
assert feed.last_fetched_at is not None
|
|
assert feed.last_fetched_at >= before
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_refetching_updated_content_does_not_reset_user_state():
|
|
from django.contrib.auth import get_user_model
|
|
|
|
User = get_user_model()
|
|
user = User.objects.create_user("alice", password="pw-alice-123")
|
|
feed = Feed.objects.create(feed_url=FEED_URL)
|
|
|
|
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
|
|
fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
from apps.articles.models import STARRED_TAG_NAME, ArticleTag, get_system_tag
|
|
|
|
article = Article.objects.get(feed=feed)
|
|
state = UserArticleState.objects.create(user=user, article=article, is_read=True)
|
|
starred_tag = get_system_tag(user, STARRED_TAG_NAME)
|
|
ArticleTag.objects.create(user=user, article=article, tag=starred_tag)
|
|
|
|
updated_xml = GOOD_FEED_XML.replace("First post", "First post (edited)")
|
|
responses.add(responses.GET, FEED_URL, body=updated_xml, status=200)
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.updated_article_ids == [article.pk]
|
|
article.refresh_from_db()
|
|
assert article.title == "First post (edited)"
|
|
|
|
state.refresh_from_db()
|
|
assert state.is_read is True
|
|
assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists()
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_refetching_picks_up_link_author_summary_and_published_at_changes():
|
|
"""Previously only title/content_html/content_hash were updated on
|
|
refetch (T6) — a feed correcting its link, author, summary, or
|
|
publication date without touching the title went unnoticed forever."""
|
|
feed = Feed.objects.create(feed_url=FEED_URL)
|
|
|
|
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
|
|
fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
article = Article.objects.get(feed=feed)
|
|
assert article.link == "https://example.com/1"
|
|
assert article.author == ""
|
|
|
|
edited_xml = GOOD_FEED_XML.replace(
|
|
"<link>https://example.com/1</link>",
|
|
"<link>https://example.com/1-moved</link>",
|
|
).replace(
|
|
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>",
|
|
"<pubDate>Tue, 02 Jan 2024 00:00:00 GMT</pubDate>\n <author>jane@example.com</author>",
|
|
)
|
|
responses.add(responses.GET, FEED_URL, body=edited_xml, status=200)
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.updated_article_ids == [article.pk]
|
|
article.refresh_from_db()
|
|
assert article.link == "https://example.com/1-moved"
|
|
assert article.author == "jane@example.com"
|
|
assert article.published_at == dt.datetime(2024, 1, 2, tzinfo=dt.timezone.utc)
|
|
|
|
|
|
def test_sanitize_html_caps_stored_output_length():
|
|
huge = "<p>" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "</p>"
|
|
cleaned = sanitize_html(huge)
|
|
assert len(cleaned) <= MAX_CONTENT_HTML_CHARS
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_only_processes_the_first_max_entries_per_poll(feed):
|
|
items = "".join(
|
|
f"<item><title>Item {i}</title><guid>guid-{i}</guid></item>"
|
|
for i in range(MAX_ENTRIES_PER_POLL + 5)
|
|
)
|
|
xml = f"""<?xml version="1.0"?>
|
|
<rss version="2.0"><channel><title>Big feed</title>{items}</channel></rss>
|
|
"""
|
|
responses.add(responses.GET, FEED_URL, body=xml, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert len(result.new_article_ids) == MAX_ENTRIES_PER_POLL
|
|
assert Article.objects.filter(feed=feed).count() == MAX_ENTRIES_PER_POLL
|
|
assert not Article.objects.filter(feed=feed, guid=f"guid-{MAX_ENTRIES_PER_POLL}").exists()
|
|
|
|
|
|
def _feed_xml(count, title_suffix=""):
|
|
items = "".join(
|
|
f"<item><title>Item {i}{title_suffix}</title><guid>guid-{i}</guid></item>"
|
|
for i in range(count)
|
|
)
|
|
return (
|
|
f'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>{items}</channel></rss>'
|
|
)
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_ingest_uses_a_constant_number_of_article_queries(feed):
|
|
"""C2: one prefetch + bulk writes per poll, not a get_or_create per
|
|
entry — query count must not grow with the number of entries."""
|
|
from django.db import connection
|
|
from django.test.utils import CaptureQueriesContext
|
|
|
|
responses.add(responses.GET, FEED_URL, body=_feed_xml(40), status=200)
|
|
with CaptureQueriesContext(connection) as ctx:
|
|
result = fetch_and_ingest(feed)
|
|
|
|
assert len(result.new_article_ids) == 40
|
|
assert len(ctx.captured_queries) < 10
|
|
assert Article.objects.filter(feed=feed).count() == 40
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_ingest_dedups_a_guid_repeated_within_one_response(feed):
|
|
xml = (
|
|
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
|
|
"<item><title>A</title><guid>same</guid></item>"
|
|
"<item><title>B</title><guid>same</guid></item>"
|
|
"</channel></rss>"
|
|
)
|
|
responses.add(responses.GET, FEED_URL, body=xml, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
|
|
assert len(result.new_article_ids) == 1
|
|
assert Article.objects.filter(feed=feed).count() == 1
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_ingest_reports_updates_separately_from_new_articles(feed):
|
|
responses.add(responses.GET, FEED_URL, body=_feed_xml(3), status=200)
|
|
fetch_and_ingest(feed)
|
|
|
|
responses.replace(responses.GET, FEED_URL, body=_feed_xml(4, " edited"), status=200)
|
|
result = fetch_and_ingest(feed)
|
|
|
|
assert len(result.new_article_ids) == 1
|
|
assert len(result.updated_article_ids) == 3
|
|
assert Article.objects.get(feed=feed, guid="guid-0").title == "Item 0 edited"
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_ingest_falls_back_to_row_by_row_when_a_concurrent_insert_wins(feed):
|
|
from unittest.mock import patch
|
|
|
|
from django.db import IntegrityError
|
|
|
|
responses.add(responses.GET, FEED_URL, body=_feed_xml(3), status=200)
|
|
|
|
with patch.object(Article.objects, "bulk_create", side_effect=IntegrityError("race")):
|
|
result = fetch_and_ingest(feed)
|
|
|
|
assert len(result.new_article_ids) == 3
|
|
assert Article.objects.filter(feed=feed).count() == 3
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_purged_article_is_not_resurrected_by_a_feed_that_still_lists_it(feed):
|
|
"""#29: a feed still listing an entry the retention purge deleted must
|
|
not bring it back as a brand-new, unread article."""
|
|
from apps.articles.tasks import purge_old_articles
|
|
from apps.core.models import SiteSettings
|
|
|
|
SiteSettings.objects.create(retention_days=30)
|
|
responses.add(responses.GET, FEED_URL, body=_feed_xml(3), status=200)
|
|
fetch_and_ingest(feed)
|
|
Article.objects.filter(feed=feed, guid="guid-0").update(
|
|
fetched_at=timezone.now() - dt.timedelta(days=90)
|
|
)
|
|
assert purge_old_articles() == 1
|
|
assert not Article.objects.filter(feed=feed, guid="guid-0").exists()
|
|
|
|
result = fetch_and_ingest(feed)
|
|
|
|
assert result.new_article_ids == []
|
|
assert not Article.objects.filter(feed=feed, guid="guid-0").exists()
|
|
assert Article.objects.filter(feed=feed).count() == 2
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_entries_older_than_the_retention_window_are_not_ingested(feed):
|
|
"""With retention on, a feed's long archive isn't ingested only to be
|
|
purged (and tombstoned) the same night."""
|
|
from apps.core.models import SiteSettings
|
|
|
|
SiteSettings.objects.create(retention_days=30)
|
|
xml = (
|
|
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
|
|
"<item><title>Ancient</title><guid>old</guid>"
|
|
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate></item>"
|
|
"<item><title>Undated</title><guid>undated</guid></item>"
|
|
"</channel></rss>"
|
|
)
|
|
responses.add(responses.GET, FEED_URL, body=xml, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
|
|
assert len(result.new_article_ids) == 1
|
|
assert Article.objects.filter(feed=feed).get().guid == "undated"
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_old_entries_are_ingested_normally_when_retention_is_off(feed):
|
|
xml = (
|
|
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
|
|
"<item><title>Ancient</title><guid>old</guid>"
|
|
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate></item>"
|
|
"</channel></rss>"
|
|
)
|
|
responses.add(responses.GET, FEED_URL, body=xml, status=200)
|
|
|
|
assert len(fetch_and_ingest(feed).new_article_ids) == 1
|