Private
Public Access
Replaces Feed.fetch_interval_minutes with a single admin-editable SiteSettings.fetch_interval_minutes (apps.core), following the same singleton-row pattern as accounts.SetupLock. Per-feed error backoff is unchanged — only the configured baseline interval is now shared across every feed, loaded once per poll_due_feeds run rather than queried per feed. EVECTION_MIN_FETCH_INTERVAL_MINUTES keeps working as the floor underneath it. SiteSettings is registered in Django admin as a singleton: "Add" is only offered when no row exists yet, and delete is always refused. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
231 lines
6.7 KiB
Python
231 lines
6.7 KiB
Python
import pytest
|
|
import requests
|
|
import responses
|
|
from django.utils import timezone
|
|
|
|
from apps.articles.models import Article, UserArticleState
|
|
from apps.feeds.models import Feed
|
|
from apps.feeds.services.fetcher import (
|
|
MAX_CONTENT_HTML_CHARS,
|
|
MAX_ENTRIES_PER_POLL,
|
|
effective_interval_minutes,
|
|
fetch_and_ingest,
|
|
sanitize_html,
|
|
)
|
|
|
|
FEED_URL = "https://example.com/feed.xml"
|
|
|
|
GOOD_FEED_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>Example Blog</title>
|
|
<link>https://example.com</link>
|
|
<item>
|
|
<title>First post</title>
|
|
<link>https://example.com/1</link>
|
|
<guid>https://example.com/1</guid>
|
|
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
|
<description><p>Hello <script>alert(1)</script>world</p></description>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
NO_GUID_FEED_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>No Guid Blog</title>
|
|
<item>
|
|
<title>Only a link</title>
|
|
<link>https://example.com/only-link</link>
|
|
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
|
<description>Body</description>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
MALFORMED_BUT_SALVAGEABLE_XML = """<?xml version="1.0"?>
|
|
<rss version="2.0"><channel>
|
|
<title>Slightly Broken</title>
|
|
<item>
|
|
<title>Unescaped & ampersand but still has entries</title>
|
|
<guid>guid-1</guid>
|
|
</item>
|
|
</channel></rss>
|
|
"""
|
|
|
|
TOTAL_GARBAGE = "this is not xml at all {{{"
|
|
|
|
|
|
@pytest.fixture
|
|
def feed(db):
|
|
return Feed.objects.create(feed_url=FEED_URL)
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_ingests_new_articles_and_sanitizes_html(feed):
|
|
responses.add(
|
|
responses.GET,
|
|
FEED_URL,
|
|
body=GOOD_FEED_XML,
|
|
status=200,
|
|
headers={"ETag": '"abc123"', "Last-Modified": "Mon, 01 Jan 2024 00:00:00 GMT"},
|
|
)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_OK
|
|
assert len(result.new_article_ids) == 1
|
|
assert feed.title == "Example Blog"
|
|
assert feed.etag == '"abc123"'
|
|
assert feed.last_modified == "Mon, 01 Jan 2024 00:00:00 GMT"
|
|
|
|
article = Article.objects.get(feed=feed, guid="https://example.com/1")
|
|
assert "<script>" not in article.content_html
|
|
assert "world" in article.content_html
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_second_poll_with_304_does_not_duplicate_or_error(feed):
|
|
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
|
|
fetch_and_ingest(feed)
|
|
feed.save()
|
|
assert Article.objects.filter(feed=feed).count() == 1
|
|
|
|
responses.add(responses.GET, FEED_URL, status=304)
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_NOT_MODIFIED
|
|
assert Article.objects.filter(feed=feed).count() == 1
|
|
assert feed.fetch_error_count == 0
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_entry_without_guid_falls_back_to_link(feed):
|
|
responses.add(responses.GET, FEED_URL, body=NO_GUID_FEED_XML, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert len(result.new_article_ids) == 1
|
|
article = Article.objects.get(pk=result.new_article_ids[0])
|
|
assert article.guid == "https://example.com/only-link"
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_bozo_feed_with_entries_still_ingests(feed):
|
|
responses.add(responses.GET, FEED_URL, body=MALFORMED_BUT_SALVAGEABLE_XML, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_OK
|
|
assert len(result.new_article_ids) == 1
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_unparseable_feed_records_error_and_backs_off(feed):
|
|
responses.add(responses.GET, FEED_URL, body=TOTAL_GARBAGE, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_ERROR
|
|
assert feed.fetch_error_count == 1
|
|
assert effective_interval_minutes(feed, 60) > 60
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_http_500_records_error(feed):
|
|
responses.add(responses.GET, FEED_URL, status=500)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_ERROR
|
|
assert feed.last_fetch_status == Feed.STATUS_ERROR
|
|
assert feed.fetch_error_count == 1
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_connection_error_still_updates_last_fetched_at(feed):
|
|
responses.add(
|
|
responses.GET,
|
|
FEED_URL,
|
|
body=requests.exceptions.ConnectionError("boom"),
|
|
)
|
|
|
|
before = timezone.now()
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.status == Feed.STATUS_ERROR
|
|
assert feed.last_fetched_at is not None
|
|
assert feed.last_fetched_at >= before
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_refetching_updated_content_does_not_reset_user_state():
|
|
from django.contrib.auth import get_user_model
|
|
|
|
User = get_user_model()
|
|
user = User.objects.create_user("alice", password="pw-alice-123")
|
|
feed = Feed.objects.create(feed_url=FEED_URL)
|
|
|
|
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
|
|
fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
from apps.articles.models import STARRED_TAG_NAME, ArticleTag, get_system_tag
|
|
|
|
article = Article.objects.get(feed=feed)
|
|
state = UserArticleState.objects.create(user=user, article=article, is_read=True)
|
|
starred_tag = get_system_tag(user, STARRED_TAG_NAME)
|
|
ArticleTag.objects.create(user=user, article=article, tag=starred_tag)
|
|
|
|
updated_xml = GOOD_FEED_XML.replace("First post", "First post (edited)")
|
|
responses.add(responses.GET, FEED_URL, body=updated_xml, status=200)
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert result.updated_article_ids == [article.pk]
|
|
article.refresh_from_db()
|
|
assert article.title == "First post (edited)"
|
|
|
|
state.refresh_from_db()
|
|
assert state.is_read is True
|
|
assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists()
|
|
|
|
|
|
def test_sanitize_html_caps_stored_output_length():
|
|
huge = "<p>" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "</p>"
|
|
cleaned = sanitize_html(huge)
|
|
assert len(cleaned) <= MAX_CONTENT_HTML_CHARS
|
|
|
|
|
|
@pytest.mark.django_db
|
|
@responses.activate
|
|
def test_fetch_only_processes_the_first_max_entries_per_poll(feed):
|
|
items = "".join(
|
|
f"<item><title>Item {i}</title><guid>guid-{i}</guid></item>"
|
|
for i in range(MAX_ENTRIES_PER_POLL + 5)
|
|
)
|
|
xml = f"""<?xml version="1.0"?>
|
|
<rss version="2.0"><channel><title>Big feed</title>{items}</channel></rss>
|
|
"""
|
|
responses.add(responses.GET, FEED_URL, body=xml, status=200)
|
|
|
|
result = fetch_and_ingest(feed)
|
|
feed.save()
|
|
|
|
assert len(result.new_article_ids) == MAX_ENTRIES_PER_POLL
|
|
assert Article.objects.filter(feed=feed).count() == MAX_ENTRIES_PER_POLL
|
|
assert not Article.objects.filter(feed=feed, guid=f"guid-{MAX_ENTRIES_PER_POLL}").exists()
|