Private
Public Access
A changed guid's article only ever had its title/content_html/ content_hash refreshed on a subsequent poll — legitimate feed edits to an entry's link, author, summary, or publication date were silently ignored forever, which can affect navigation, ordering, and rules that key off those fields. content_hash is now computed over every mutable field feedparser gives us for an entry (title, link, author, summary, content_html, published_at), and all of them are written together when the hash changes. Per-user read/star state is untouched either way, same as before. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
267 lines
8.1 KiB
Python
267 lines
8.1 KiB
Python
import datetime as dt
|
|
|
|
import pytest
|
|
import requests
|
|
import responses
|
|
from django.utils import timezone
|
|
|
|
from apps.articles.models import Article, UserArticleState
|
|
from apps.feeds.models import Feed
|
|
from apps.feeds.services.fetcher import (
|
|
MAX_CONTENT_HTML_CHARS,
|
|
MAX_ENTRIES_PER_POLL,
|
|
effective_interval_minutes,
|
|
fetch_and_ingest,
|
|
sanitize_html,
|
|
)
|
|
|
|
FEED_URL = "https://example.com/feed.xml"
|
|
|
|
GOOD_FEED_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>Example Blog</title>
|
|
<link>https://example.com</link>
|
|
<item>
|
|
<title>First post</title>
|
|
<link>https://example.com/1</link>
|
|
<guid>https://example.com/1</guid>
|
|
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
|
<description><p>Hello <script>alert(1)</script>world</p></description>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
NO_GUID_FEED_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>No Guid Blog</title>
|
|
<item>
|
|
<title>Only a link</title>
|
|
<link>https://example.com/only-link</link>
|
|
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
|
<description>Body</description>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
MALFORMED_BUT_SALVAGEABLE_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>Slightly Broken</title>
|
|
<item>
|
|
<title>Unescaped & ampersand but still has entries</title>
|
|
<guid>guid-1</guid>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
TOTAL_GARBAGE = "this is not xml at all {{{"
|
|
|
|
|
|
@pytest.fixture
|
|
def feed(db):
|
|
return Feed.objects.create(feed_url=FEED_URL)
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_ingests_new_articles_and_sanitizes_html(feed):
|
|
responses.add(
|
|
responses.GET,
|
|
FEED_URL,
|
|
body=GOOD_FEED_XML,
|
|
status=200,
|
|
headers={"ETag": '"abc123"', "Last-Modified": "Mon, 01 Jan 2024 00:00:00 GMT"},
|
|
)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_OK
|
|
assert len(result.new_article_ids) == 1
|
|
assert feed.title == "Example Blog"
|
|
assert feed.etag == '"abc123"'
|
|
assert feed.last_modified == "Mon, 01 Jan 2024 00:00:00 GMT"
|
|
|
|
article = Article.objects.get(feed=feed, guid="https://example.com/1")
|
|
assert "<script>" not in article.content_html
|
|
assert "world" in article.content_html
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_second_poll_with_304_does_not_duplicate_or_error(feed):
|
|
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
|
|
fetch_and_ingest(feed)
|
|
feed.save()
|
|
assert Article.objects.filter(feed=feed).count() == 1
|
|
|
|
responses.add(responses.GET, FEED_URL, status=304)
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_NOT_MODIFIED
|
|
assert Article.objects.filter(feed=feed).count() == 1
|
|
assert feed.fetch_error_count == 0
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_entry_without_guid_falls_back_to_link(feed):
|
|
responses.add(responses.GET, FEED_URL, body=NO_GUID_FEED_XML, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert len(result.new_article_ids) == 1
|
|
article = Article.objects.get(pk=result.new_article_ids[0])
|
|
assert article.guid == "https://example.com/only-link"
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_bozo_feed_with_entries_still_ingests(feed):
|
|
responses.add(responses.GET, FEED_URL, body=MALFORMED_BUT_SALVAGEABLE_XML, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_OK
|
|
assert len(result.new_article_ids) == 1
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_unparseable_feed_records_error_and_backs_off(feed):
|
|
responses.add(responses.GET, FEED_URL, body=TOTAL_GARBAGE, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_ERROR
|
|
assert feed.fetch_error_count == 1
|
|
assert effective_interval_minutes(feed, 60) > 60
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_http_500_records_error(feed):
|
|
responses.add(responses.GET, FEED_URL, status=500)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_ERROR
|
|
assert feed.last_fetch_status == Feed.STATUS_ERROR
|
|
assert feed.fetch_error_count == 1
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_connection_error_still_updates_last_fetched_at(feed):
|
|
responses.add(
|
|
responses.GET,
|
|
FEED_URL,
|
|
body=requests.exceptions.ConnectionError("boom"),
|
|
)
|
|
|
|
before = timezone.now()
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_ERROR
|
|
assert feed.last_fetched_at is not None
|
|
assert feed.last_fetched_at >= before
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_refetching_updated_content_does_not_reset_user_state():
|
|
from django.contrib.auth import get_user_model
|
|
|
|
User = get_user_model()
|
|
user = User.objects.create_user("alice", password="pw-alice-123")
|
|
feed = Feed.objects.create(feed_url=FEED_URL)
|
|
|
|
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
|
|
fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
from apps.articles.models import STARRED_TAG_NAME, ArticleTag, get_system_tag
|
|
|
|
article = Article.objects.get(feed=feed)
|
|
state = UserArticleState.objects.create(user=user, article=article, is_read=True)
|
|
starred_tag = get_system_tag(user, STARRED_TAG_NAME)
|
|
ArticleTag.objects.create(user=user, article=article, tag=starred_tag)
|
|
|
|
updated_xml = GOOD_FEED_XML.replace("First post", "First post (edited)")
|
|
responses.add(responses.GET, FEED_URL, body=updated_xml, status=200)
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.updated_article_ids == [article.pk]
|
|
article.refresh_from_db()
|
|
assert article.title == "First post (edited)"
|
|
|
|
state.refresh_from_db()
|
|
assert state.is_read is True
|
|
assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists()
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_refetching_picks_up_link_author_summary_and_published_at_changes():
|
|
"""Previously only title/content_html/content_hash were updated on
|
|
refetch (T6) — a feed correcting its link, author, summary, or
|
|
publication date without touching the title went unnoticed forever."""
|
|
feed = Feed.objects.create(feed_url=FEED_URL)
|
|
|
|
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
|
|
fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
article = Article.objects.get(feed=feed)
|
|
assert article.link == "https://example.com/1"
|
|
assert article.author == ""
|
|
|
|
edited_xml = GOOD_FEED_XML.replace(
|
|
"<link>https://example.com/1</link>",
|
|
"<link>https://example.com/1-moved</link>",
|
|
).replace(
|
|
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>",
|
|
"<pubDate>Tue, 02 Jan 2024 00:00:00 GMT</pubDate>\n <author>jane@example.com</author>",
|
|
)
|
|
responses.add(responses.GET, FEED_URL, body=edited_xml, status=200)
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.updated_article_ids == [article.pk]
|
|
article.refresh_from_db()
|
|
assert article.link == "https://example.com/1-moved"
|
|
assert article.author == "jane@example.com"
|
|
assert article.published_at == dt.datetime(2024, 1, 2, tzinfo=dt.timezone.utc)
|
|
|
|
|
|
def test_sanitize_html_caps_stored_output_length():
|
|
huge = "<p>" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "</p>"
|
|
cleaned = sanitize_html(huge)
|
|
assert len(cleaned) <= MAX_CONTENT_HTML_CHARS
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_only_processes_the_first_max_entries_per_poll(feed):
|
|
items = "".join(
|
|
f"<item><title>Item {i}</title><guid>guid-{i}</guid></item>"
|
|
for i in range(MAX_ENTRIES_PER_POLL + 5)
|
|
)
|
|
xml = f"""<?xml version="1.0"?>
|
|
<rss version="2.0"><channel><title>Big feed</title>{items}</channel></rss>
|
|
"""
|
|
responses.add(responses.GET, FEED_URL, body=xml, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert len(result.new_article_ids) == MAX_ENTRIES_PER_POLL
|
|
assert Article.objects.filter(feed=feed).count() == MAX_ENTRIES_PER_POLL
|
|
assert not Article.objects.filter(feed=feed, guid=f"guid-{MAX_ENTRIES_PER_POLL}").exists()
|