Private
Public Access
Update link/author/summary/published_at on refetch, not just title (T6)
A changed guid's article only ever had its title/content_html/ content_hash refreshed on a subsequent poll — legitimate feed edits to an entry's link, author, summary, or publication date were silently ignored forever, which can affect navigation, ordering, and rules that key off those fields. content_hash is now computed over every mutable field feedparser gives us for an entry (title, link, author, summary, content_html, published_at), and all of them are written together when the hash changes. Per-user read/star state is untouched either way, same as before. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0bfe9c8076
commit
17e7f63e61
2 files changed
+69
-8
No files matched your search
@@ -97,8 +97,15 @@ def sanitize_html(raw_html: str) -> str:
|
||||
return cleaned[:MAX_CONTENT_HTML_CHARS]
|
||||
|
||||
|
||||
def _content_hash(title: str, content_html: str) -> str:
|
||||
return hashlib.sha256(f"{title}\n{content_html}".encode("utf-8")).hexdigest()
|
||||
def _content_hash(
|
||||
title: str, link: str, author: str, summary: str, content_html: str, published_at
|
||||
) -> str:
|
||||
"""Covers every mutable Article field an entry carries (T6) — a feed
|
||||
editing its link/author/summary/published date, not just title or
|
||||
body, must still be detected as a change worth re-saving."""
|
||||
published_key = published_at.isoformat() if published_at else ""
|
||||
payload = "\n".join([title, link, author, summary, content_html, published_key])
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _entry_guid(entry) -> str:
|
||||
@@ -213,20 +220,24 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
|
||||
for entry in entries:
|
||||
guid = _entry_guid(entry)
|
||||
title = (getattr(entry, "title", "") or "")[:1000]
|
||||
link = (getattr(entry, "link", "") or "")[:2000]
|
||||
author = (getattr(entry, "author", "") or "")[:500]
|
||||
summary = (getattr(entry, "summary", "") or "")[:5000]
|
||||
content_html = sanitize_html(_entry_content_html(entry))
|
||||
content_hash = _content_hash(title, content_html)
|
||||
published_at = _entry_published_at(entry)
|
||||
content_hash = _content_hash(title, link, author, summary, content_html, published_at)
|
||||
|
||||
article, created = Article.objects.get_or_create(
|
||||
feed=feed,
|
||||
guid=guid,
|
||||
defaults={
|
||||
"link": (getattr(entry, "link", "") or "")[:2000],
|
||||
"link": link,
|
||||
"title": title,
|
||||
"author": (getattr(entry, "author", "") or "")[:500],
|
||||
"author": author,
|
||||
"content_html": content_html,
|
||||
"summary": (getattr(entry, "summary", "") or "")[:5000],
|
||||
"summary": summary,
|
||||
"content_hash": content_hash,
|
||||
"published_at": _entry_published_at(entry),
|
||||
"published_at": published_at,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -234,9 +245,23 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
|
||||
new_ids.append(article.pk)
|
||||
elif article.content_hash != content_hash:
|
||||
article.title = title
|
||||
article.link = link
|
||||
article.author = author
|
||||
article.summary = summary
|
||||
article.content_html = content_html
|
||||
article.published_at = published_at
|
||||
article.content_hash = content_hash
|
||||
article.save(update_fields=["title", "content_html", "content_hash"])
|
||||
article.save(
|
||||
update_fields=[
|
||||
"title",
|
||||
"link",
|
||||
"author",
|
||||
"summary",
|
||||
"content_html",
|
||||
"published_at",
|
||||
"content_hash",
|
||||
]
|
||||
)
|
||||
updated_ids.append(article.pk)
|
||||
|
||||
return FetchResult(
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import datetime as dt
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import responses
|
||||
@@ -204,6 +206,40 @@ def test_refetching_updated_content_does_not_reset_user_state():
|
||||
assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_refetching_picks_up_link_author_summary_and_published_at_changes():
|
||||
"""Previously only title/content_html/content_hash were updated on
|
||||
refetch (T6) — a feed correcting its link, author, summary, or
|
||||
publication date without touching the title went unnoticed forever."""
|
||||
feed = Feed.objects.create(feed_url=FEED_URL)
|
||||
|
||||
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
|
||||
fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
article = Article.objects.get(feed=feed)
|
||||
assert article.link == "https://example.com/1"
|
||||
assert article.author == ""
|
||||
|
||||
edited_xml = GOOD_FEED_XML.replace(
|
||||
"<link>https://example.com/1</link>",
|
||||
"<link>https://example.com/1-moved</link>",
|
||||
).replace(
|
||||
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>",
|
||||
"<pubDate>Tue, 02 Jan 2024 00:00:00 GMT</pubDate>\n <author>jane@example.com</author>",
|
||||
)
|
||||
responses.add(responses.GET, FEED_URL, body=edited_xml, status=200)
|
||||
result = fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
assert result.updated_article_ids == [article.pk]
|
||||
article.refresh_from_db()
|
||||
assert article.link == "https://example.com/1-moved"
|
||||
assert article.author == "jane@example.com"
|
||||
assert article.published_at == dt.datetime(2024, 1, 2, tzinfo=dt.timezone.utc)
|
||||
|
||||
|
||||
def test_sanitize_html_caps_stored_output_length():
|
||||
huge = "<p>" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "</p>"
|
||||
cleaned = sanitize_html(huge)
|
||||
|
||||
Reference in new issue
Block a user