Files
Evection/apps/feeds/test_fetcher.py
T
ksmithandClaude Sonnet 5 17e7f63e61 Update link/author/summary/published_at on refetch, not just title (T6)
A changed guid's article only ever had its title/content_html/
content_hash refreshed on a subsequent poll — legitimate feed edits to
an entry's link, author, summary, or publication date were silently
ignored forever, which can affect navigation, ordering, and rules that
key off those fields.

content_hash is now computed over every mutable field feedparser gives
us for an entry (title, link, author, summary, content_html,
published_at), and all of them are written together when the hash
changes. Per-user read/star state is untouched either way, same as
before.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-31 14:44:15 -06:00

267 lines
8.1 KiB
Python

import datetime as dt
import pytest
import requests
import responses
from django.utils import timezone
from apps.articles.models import Article, UserArticleState
from apps.feeds.models import Feed
from apps.feeds.services.fetcher import (
MAX_CONTENT_HTML_CHARS,
MAX_ENTRIES_PER_POLL,
effective_interval_minutes,
fetch_and_ingest,
sanitize_html,
)
FEED_URL = "https://example.com/feed.xml"
GOOD_FEED_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>Example Blog</title>
<link>https://example.com</link>
<item>
<title>First post</title>
<link>https://example.com/1</link>
<guid>https://example.com/1</guid>
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
<description>&lt;p&gt;Hello &lt;script&gt;alert(1)&lt;/script&gt;world&lt;/p&gt;</description>
</item>
</channel></rss>
"""
NO_GUID_FEED_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>No Guid Blog</title>
<item>
<title>Only a link</title>
<link>https://example.com/only-link</link>
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
<description>Body</description>
</item>
</channel></rss>
"""
MALFORMED_BUT_SALVAGEABLE_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>Slightly Broken</title>
<item>
<title>Unescaped & ampersand but still has entries</title>
<guid>guid-1</guid>
</item>
</channel></rss>
"""
TOTAL_GARBAGE = "this is not xml at all {{{"
@pytest.fixture
def feed(db):
return Feed.objects.create(feed_url=FEED_URL)
@pytest.mark.django_db
@responses.activate
def test_fetch_ingests_new_articles_and_sanitizes_html(feed):
responses.add(
responses.GET,
FEED_URL,
body=GOOD_FEED_XML,
status=200,
headers={"ETag": '"abc123"', "Last-Modified": "Mon, 01 Jan 2024 00:00:00 GMT"},
)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_OK
assert len(result.new_article_ids) == 1
assert feed.title == "Example Blog"
assert feed.etag == '"abc123"'
assert feed.last_modified == "Mon, 01 Jan 2024 00:00:00 GMT"
article = Article.objects.get(feed=feed, guid="https://example.com/1")
assert "<script>" not in article.content_html
assert "world" in article.content_html
@pytest.mark.django_db
@responses.activate
def test_second_poll_with_304_does_not_duplicate_or_error(feed):
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
fetch_and_ingest(feed)
feed.save()
assert Article.objects.filter(feed=feed).count() == 1
responses.add(responses.GET, FEED_URL, status=304)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_NOT_MODIFIED
assert Article.objects.filter(feed=feed).count() == 1
assert feed.fetch_error_count == 0
@pytest.mark.django_db
@responses.activate
def test_entry_without_guid_falls_back_to_link(feed):
responses.add(responses.GET, FEED_URL, body=NO_GUID_FEED_XML, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert len(result.new_article_ids) == 1
article = Article.objects.get(pk=result.new_article_ids[0])
assert article.guid == "https://example.com/only-link"
@pytest.mark.django_db
@responses.activate
def test_bozo_feed_with_entries_still_ingests(feed):
responses.add(responses.GET, FEED_URL, body=MALFORMED_BUT_SALVAGEABLE_XML, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_OK
assert len(result.new_article_ids) == 1
@pytest.mark.django_db
@responses.activate
def test_unparseable_feed_records_error_and_backs_off(feed):
responses.add(responses.GET, FEED_URL, body=TOTAL_GARBAGE, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_ERROR
assert feed.fetch_error_count == 1
assert effective_interval_minutes(feed, 60) > 60
@pytest.mark.django_db
@responses.activate
def test_http_500_records_error(feed):
responses.add(responses.GET, FEED_URL, status=500)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_ERROR
assert feed.last_fetch_status == Feed.STATUS_ERROR
assert feed.fetch_error_count == 1
@pytest.mark.django_db
@responses.activate
def test_connection_error_still_updates_last_fetched_at(feed):
responses.add(
responses.GET,
FEED_URL,
body=requests.exceptions.ConnectionError("boom"),
)
before = timezone.now()
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_ERROR
assert feed.last_fetched_at is not None
assert feed.last_fetched_at >= before
@pytest.mark.django_db
@responses.activate
def test_refetching_updated_content_does_not_reset_user_state():
from django.contrib.auth import get_user_model
User = get_user_model()
user = User.objects.create_user("alice", password="pw-alice-123")
feed = Feed.objects.create(feed_url=FEED_URL)
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
fetch_and_ingest(feed)
feed.save()
from apps.articles.models import STARRED_TAG_NAME, ArticleTag, get_system_tag
article = Article.objects.get(feed=feed)
state = UserArticleState.objects.create(user=user, article=article, is_read=True)
starred_tag = get_system_tag(user, STARRED_TAG_NAME)
ArticleTag.objects.create(user=user, article=article, tag=starred_tag)
updated_xml = GOOD_FEED_XML.replace("First post", "First post (edited)")
responses.add(responses.GET, FEED_URL, body=updated_xml, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.updated_article_ids == [article.pk]
article.refresh_from_db()
assert article.title == "First post (edited)"
state.refresh_from_db()
assert state.is_read is True
assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists()
@pytest.mark.django_db
@responses.activate
def test_refetching_picks_up_link_author_summary_and_published_at_changes():
"""Previously only title/content_html/content_hash were updated on
refetch (T6) — a feed correcting its link, author, summary, or
publication date without touching the title went unnoticed forever."""
feed = Feed.objects.create(feed_url=FEED_URL)
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
fetch_and_ingest(feed)
feed.save()
article = Article.objects.get(feed=feed)
assert article.link == "https://example.com/1"
assert article.author == ""
edited_xml = GOOD_FEED_XML.replace(
"<link>https://example.com/1</link>",
"<link>https://example.com/1-moved</link>",
).replace(
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>",
"<pubDate>Tue, 02 Jan 2024 00:00:00 GMT</pubDate>\n <author>jane@example.com</author>",
)
responses.add(responses.GET, FEED_URL, body=edited_xml, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.updated_article_ids == [article.pk]
article.refresh_from_db()
assert article.link == "https://example.com/1-moved"
assert article.author == "jane@example.com"
assert article.published_at == dt.datetime(2024, 1, 2, tzinfo=dt.timezone.utc)
def test_sanitize_html_caps_stored_output_length():
huge = "<p>" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "</p>"
cleaned = sanitize_html(huge)
assert len(cleaned) <= MAX_CONTENT_HTML_CHARS
@pytest.mark.django_db
@responses.activate
def test_fetch_only_processes_the_first_max_entries_per_poll(feed):
items = "".join(
f"<item><title>Item {i}</title><guid>guid-{i}</guid></item>"
for i in range(MAX_ENTRIES_PER_POLL + 5)
)
xml = f"""<?xml version="1.0"?>
<rss version="2.0"><channel><title>Big feed</title>{items}</channel></rss>
"""
responses.add(responses.GET, FEED_URL, body=xml, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert len(result.new_article_ids) == MAX_ENTRIES_PER_POLL
assert Article.objects.filter(feed=feed).count() == MAX_ENTRIES_PER_POLL
assert not Article.objects.filter(feed=feed, guid=f"guid-{MAX_ENTRIES_PER_POLL}").exists()