diff --git a/apps/feeds/services/fetcher.py b/apps/feeds/services/fetcher.py index 1080843..2c44166 100644 --- a/apps/feeds/services/fetcher.py +++ b/apps/feeds/services/fetcher.py @@ -97,8 +97,15 @@ def sanitize_html(raw_html: str) -> str: return cleaned[:MAX_CONTENT_HTML_CHARS] -def _content_hash(title: str, content_html: str) -> str: - return hashlib.sha256(f"{title}\n{content_html}".encode("utf-8")).hexdigest() +def _content_hash( + title: str, link: str, author: str, summary: str, content_html: str, published_at +) -> str: + """Covers every mutable Article field an entry carries (T6) — a feed + editing its link/author/summary/published date, not just title or + body, must still be detected as a change worth re-saving.""" + published_key = published_at.isoformat() if published_at else "" + payload = "\n".join([title, link, author, summary, content_html, published_key]) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() def _entry_guid(entry) -> str: @@ -213,20 +220,24 @@ def fetch_and_ingest(feed: Feed) -> FetchResult: for entry in entries: guid = _entry_guid(entry) title = (getattr(entry, "title", "") or "")[:1000] + link = (getattr(entry, "link", "") or "")[:2000] + author = (getattr(entry, "author", "") or "")[:500] + summary = (getattr(entry, "summary", "") or "")[:5000] content_html = sanitize_html(_entry_content_html(entry)) - content_hash = _content_hash(title, content_html) + published_at = _entry_published_at(entry) + content_hash = _content_hash(title, link, author, summary, content_html, published_at) article, created = Article.objects.get_or_create( feed=feed, guid=guid, defaults={ - "link": (getattr(entry, "link", "") or "")[:2000], + "link": link, "title": title, - "author": (getattr(entry, "author", "") or "")[:500], + "author": author, "content_html": content_html, - "summary": (getattr(entry, "summary", "") or "")[:5000], + "summary": summary, "content_hash": content_hash, - "published_at": _entry_published_at(entry), + "published_at": published_at, }, ) @@ -234,9 +245,23 @@ def fetch_and_ingest(feed: Feed) -> FetchResult: new_ids.append(article.pk) elif article.content_hash != content_hash: article.title = title + article.link = link + article.author = author + article.summary = summary article.content_html = content_html + article.published_at = published_at article.content_hash = content_hash - article.save(update_fields=["title", "content_html", "content_hash"]) + article.save( + update_fields=[ + "title", + "link", + "author", + "summary", + "content_html", + "published_at", + "content_hash", + ] + ) updated_ids.append(article.pk) return FetchResult( diff --git a/apps/feeds/test_fetcher.py b/apps/feeds/test_fetcher.py index d653cbc..bfae4b1 100644 --- a/apps/feeds/test_fetcher.py +++ b/apps/feeds/test_fetcher.py @@ -1,3 +1,5 @@ +import datetime as dt + import pytest import requests import responses @@ -204,6 +206,40 @@ def test_refetching_updated_content_does_not_reset_user_state(): assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists() +@pytest.mark.django_db +@responses.activate +def test_refetching_picks_up_link_author_summary_and_published_at_changes(): + """Previously only title/content_html/content_hash were updated on + refetch (T6) — a feed correcting its link, author, summary, or + publication date without touching the title went unnoticed forever.""" + feed = Feed.objects.create(feed_url=FEED_URL) + + responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200) + fetch_and_ingest(feed) + feed.save() + + article = Article.objects.get(feed=feed) + assert article.link == "https://example.com/1" + assert article.author == "" + + edited_xml = GOOD_FEED_XML.replace( + "https://example.com/1", + "https://example.com/1-moved", + ).replace( + "Mon, 01 Jan 2024 00:00:00 GMT", + "Tue, 02 Jan 2024 00:00:00 GMT\n jane@example.com", + ) + responses.add(responses.GET, FEED_URL, body=edited_xml, status=200) + result = fetch_and_ingest(feed) + feed.save() + + assert result.updated_article_ids == [article.pk] + article.refresh_from_db() + assert article.link == "https://example.com/1-moved" + assert article.author == "jane@example.com" + assert article.published_at == dt.datetime(2024, 1, 2, tzinfo=dt.timezone.utc) + + def test_sanitize_html_caps_stored_output_length(): huge = "

" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "

" cleaned = sanitize_html(huge)