From 9241a54958312338f8ed84aba90ce500336a50b5 Mon Sep 17 00:00:00 2001 From: Keith Smith Date: Sat, 31 Oct 2026 12:46:40 -0600 Subject: [PATCH] Cap entries per poll and stored content size (T4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ssrf_guard.MAX_RESPONSE_BYTES (S1) already bounds the raw download, but a feed within that byte cap can still carry an unreasonable number of entries, or one entry with an unreasonably large body — both cost real CPU/DB time (nh3 sanitization, an Article row/UPDATE) per entry. fetch_and_ingest now processes at most MAX_ENTRIES_PER_POLL (200) entries per poll (logging a warning when truncated) and sanitize_html bounds both its input (sanitizer cost) and output (what's actually stored/rendered) length. Co-Authored-By: Claude Sonnet 5 --- apps/feeds/test_fetcher.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/apps/feeds/test_fetcher.py b/apps/feeds/test_fetcher.py index 5b1bd03..1d3e23a 100644 --- a/apps/feeds/test_fetcher.py +++ b/apps/feeds/test_fetcher.py @@ -5,7 +5,13 @@ from django.utils import timezone from apps.articles.models import Article, UserArticleState from apps.feeds.models import Feed -from apps.feeds.services.fetcher import effective_interval_minutes, fetch_and_ingest +from apps.feeds.services.fetcher import ( + MAX_CONTENT_HTML_CHARS, + MAX_ENTRIES_PER_POLL, + effective_interval_minutes, + fetch_and_ingest, + sanitize_html, +) FEED_URL = "https://example.com/feed.xml" @@ -196,3 +202,29 @@ def test_refetching_updated_content_does_not_reset_user_state(): state.refresh_from_db() assert state.is_read is True assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists() + + +def test_sanitize_html_caps_stored_output_length(): + huge = "

" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "

" + cleaned = sanitize_html(huge) + assert len(cleaned) <= MAX_CONTENT_HTML_CHARS + + +@pytest.mark.django_db +@responses.activate +def test_fetch_only_processes_the_first_max_entries_per_poll(feed): + items = "".join( + f"Item {i}guid-{i}" + for i in range(MAX_ENTRIES_PER_POLL + 5) + ) + xml = f""" +Big feed{items} +""" + responses.add(responses.GET, FEED_URL, body=xml, status=200) + + result = fetch_and_ingest(feed) + feed.save() + + assert len(result.new_article_ids) == MAX_ENTRIES_PER_POLL + assert Article.objects.filter(feed=feed).count() == MAX_ENTRIES_PER_POLL + assert not Article.objects.filter(feed=feed, guid=f"guid-{MAX_ENTRIES_PER_POLL}").exists()