Cap entries per poll and stored content size (T4)

ssrf_guard.MAX_RESPONSE_BYTES (S1) already bounds the raw download, but
a feed within that byte cap can still carry an unreasonable number of
entries, or one entry with an unreasonably large body — both cost real
CPU/DB time (nh3 sanitization, an Article row/UPDATE) per entry.
fetch_and_ingest now processes at most MAX_ENTRIES_PER_POLL (200)
entries per poll (logging a warning when truncated) and sanitize_html
bounds both its input (sanitizer cost) and output (what's actually
stored/rendered) length.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5 committed 2026-10-31 12:46:40 -06:00
1 parent 64f25789f5
commit 9241a54958
1 file changed
+33 -1
+33 -1
View File
@@ -5,7 +5,13 @@ from django.utils import timezone
from apps.articles.models import Article, UserArticleState
from apps.feeds.models import Feed
from apps.feeds.services.fetcher import effective_interval_minutes, fetch_and_ingest
from apps.feeds.services.fetcher import (
MAX_CONTENT_HTML_CHARS,
MAX_ENTRIES_PER_POLL,
effective_interval_minutes,
fetch_and_ingest,
sanitize_html,
)
FEED_URL = "https://example.com/feed.xml"
@@ -196,3 +202,29 @@ def test_refetching_updated_content_does_not_reset_user_state():
state.refresh_from_db()
assert state.is_read is True
assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists()
def test_sanitize_html_caps_stored_output_length():
huge = "<p>" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "</p>"
cleaned = sanitize_html(huge)
assert len(cleaned) <= MAX_CONTENT_HTML_CHARS
@pytest.mark.django_db
@responses.activate
def test_fetch_only_processes_the_first_max_entries_per_poll(feed):
items = "".join(
f"<item><title>Item {i}</title><guid>guid-{i}</guid></item>"
for i in range(MAX_ENTRIES_PER_POLL + 5)
)
xml = f"""<?xml version="1.0"?>
<rss version="2.0"><channel><title>Big feed</title>{items}</channel></rss>
"""
responses.add(responses.GET, FEED_URL, body=xml, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert len(result.new_article_ids) == MAX_ENTRIES_PER_POLL
assert Article.objects.filter(feed=feed).count() == MAX_ENTRIES_PER_POLL
assert not Article.objects.filter(feed=feed, guid=f"guid-{MAX_ENTRIES_PER_POLL}").exists()