Compare commits

...
Author SHA1 Message Date
ksmithandClaude Sonnet 5 17e7f63e61 Update link/author/summary/published_at on refetch, not just title (T6)
A changed guid's article only ever had its title/content_html/
content_hash refreshed on a subsequent poll — legitimate feed edits to
an entry's link, author, summary, or publication date were silently
ignored forever, which can affect navigation, ordering, and rules that
key off those fields.

content_hash is now computed over every mutable field feedparser gives
us for an entry (title, link, author, summary, content_html,
published_at), and all of them are written together when the hash
changes. Per-user read/star state is untouched either way, same as
before.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-31 14:44:15 -06:00
ksmithandClaude Sonnet 5 0bfe9c8076 Enqueue the initial feed fetch instead of running it inline (T3)
add_feed ran fetch_and_ingest synchronously during the POST request, so
a slow or large feed tied up a Gunicorn worker for the duration of the
fetch (up to the 15s HTTP timeout, plus parse/ingest time) and a user
could repeat it with distinct URLs to reduce request capacity.

The subscription is now created synchronously as before, but the first
fetch is dispatched via fetch_feed.delay() — the same task the periodic
poll uses, so the save/error bookkeeping is unchanged. The user sees an
immediate redirect with a message that the fetch is in progress rather
than waiting on it in the request/response cycle.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-31 14:44:09 -06:00
4 changed files with 88 additions and 30 deletions

No files matched your search

+33 -8
View File
@@ -97,8 +97,15 @@ def sanitize_html(raw_html: str) -> str:
return cleaned[:MAX_CONTENT_HTML_CHARS] return cleaned[:MAX_CONTENT_HTML_CHARS]
def _content_hash(title: str, content_html: str) -> str: def _content_hash(
return hashlib.sha256(f"{title}\n{content_html}".encode("utf-8")).hexdigest() title: str, link: str, author: str, summary: str, content_html: str, published_at
) -> str:
"""Covers every mutable Article field an entry carries (T6) — a feed
editing its link/author/summary/published date, not just title or
body, must still be detected as a change worth re-saving."""
published_key = published_at.isoformat() if published_at else ""
payload = "\n".join([title, link, author, summary, content_html, published_key])
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
def _entry_guid(entry) -> str: def _entry_guid(entry) -> str:
@@ -213,20 +220,24 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
for entry in entries: for entry in entries:
guid = _entry_guid(entry) guid = _entry_guid(entry)
title = (getattr(entry, "title", "") or "")[:1000] title = (getattr(entry, "title", "") or "")[:1000]
link = (getattr(entry, "link", "") or "")[:2000]
author = (getattr(entry, "author", "") or "")[:500]
summary = (getattr(entry, "summary", "") or "")[:5000]
content_html = sanitize_html(_entry_content_html(entry)) content_html = sanitize_html(_entry_content_html(entry))
content_hash = _content_hash(title, content_html) published_at = _entry_published_at(entry)
content_hash = _content_hash(title, link, author, summary, content_html, published_at)
article, created = Article.objects.get_or_create( article, created = Article.objects.get_or_create(
feed=feed, feed=feed,
guid=guid, guid=guid,
defaults={ defaults={
"link": (getattr(entry, "link", "") or "")[:2000], "link": link,
"title": title, "title": title,
"author": (getattr(entry, "author", "") or "")[:500], "author": author,
"content_html": content_html, "content_html": content_html,
"summary": (getattr(entry, "summary", "") or "")[:5000], "summary": summary,
"content_hash": content_hash, "content_hash": content_hash,
"published_at": _entry_published_at(entry), "published_at": published_at,
}, },
) )
@@ -234,9 +245,23 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
new_ids.append(article.pk) new_ids.append(article.pk)
elif article.content_hash != content_hash: elif article.content_hash != content_hash:
article.title = title article.title = title
article.link = link
article.author = author
article.summary = summary
article.content_html = content_html article.content_html = content_html
article.published_at = published_at
article.content_hash = content_hash article.content_hash = content_hash
article.save(update_fields=["title", "content_html", "content_hash"]) article.save(
update_fields=[
"title",
"link",
"author",
"summary",
"content_html",
"published_at",
"content_hash",
]
)
updated_ids.append(article.pk) updated_ids.append(article.pk)
return FetchResult( return FetchResult(
+36
View File
@@ -1,3 +1,5 @@
import datetime as dt
import pytest import pytest
import requests import requests
import responses import responses
@@ -204,6 +206,40 @@ def test_refetching_updated_content_does_not_reset_user_state():
assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists() assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists()
@pytest.mark.django_db
@responses.activate
def test_refetching_picks_up_link_author_summary_and_published_at_changes():
"""Previously only title/content_html/content_hash were updated on
refetch (T6) — a feed correcting its link, author, summary, or
publication date without touching the title went unnoticed forever."""
feed = Feed.objects.create(feed_url=FEED_URL)
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
fetch_and_ingest(feed)
feed.save()
article = Article.objects.get(feed=feed)
assert article.link == "https://example.com/1"
assert article.author == ""
edited_xml = GOOD_FEED_XML.replace(
"<link>https://example.com/1</link>",
"<link>https://example.com/1-moved</link>",
).replace(
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>",
"<pubDate>Tue, 02 Jan 2024 00:00:00 GMT</pubDate>\n <author>jane@example.com</author>",
)
responses.add(responses.GET, FEED_URL, body=edited_xml, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.updated_article_ids == [article.pk]
article.refresh_from_db()
assert article.link == "https://example.com/1-moved"
assert article.author == "jane@example.com"
assert article.published_at == dt.datetime(2024, 1, 2, tzinfo=dt.timezone.utc)
def test_sanitize_html_caps_stored_output_length(): def test_sanitize_html_caps_stored_output_length():
huge = "<p>" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "</p>" huge = "<p>" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "</p>"
cleaned = sanitize_html(huge) cleaned = sanitize_html(huge)
+8 -3
View File
@@ -54,7 +54,11 @@ GOOD_FEED_XML = """<?xml version="1.0"?>
@pytest.mark.django_db @pytest.mark.django_db
@responses.activate @responses.activate
def test_add_feed_view_subscribes_and_fetches_immediately(client, two_users): def test_add_feed_view_subscribes_and_enqueues_the_initial_fetch(client, two_users):
"""The subscription is created synchronously; the first fetch is
dispatched via fetch_feed.delay() (T3) rather than run inline — in
tests CELERY_TASK_ALWAYS_EAGER still runs it immediately, so the
article it ingests is visible by the time the response returns."""
alice, _ = two_users alice, _ = two_users
client.login(username="alice", password="pw-alice-123") client.login(username="alice", password="pw-alice-123")
responses.add(responses.GET, "https://example.com/feed.xml", body=GOOD_FEED_XML, status=200) responses.add(responses.GET, "https://example.com/feed.xml", body=GOOD_FEED_XML, status=200)
@@ -62,9 +66,10 @@ def test_add_feed_view_subscribes_and_fetches_immediately(client, two_users):
response = client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/feed.xml"}) response = client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/feed.xml"})
assert response.status_code == 302 assert response.status_code == 302
assert Subscription.objects.filter( subscription = Subscription.objects.get(
user=alice, feed__feed_url="https://example.com/feed.xml" user=alice, feed__feed_url="https://example.com/feed.xml"
).exists() )
assert subscription.feed.articles.filter(guid="guid-1").exists()
@pytest.mark.django_db @pytest.mark.django_db
+11 -19
View File
@@ -12,8 +12,7 @@ from apps.articles.models import SubscriptionTag, Tag
from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm
from .models import Feed, Subscription from .models import Feed, Subscription
from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml
from .services.fetcher import fetch_and_ingest from .tasks import fetch_feed, import_opml_task
from .tasks import import_opml_task
OPML_ASYNC_THRESHOLD = 50 OPML_ASYNC_THRESHOLD = 50
@@ -39,23 +38,16 @@ def add_feed(request):
user=request.user, subscription=subscription, tag=tag user=request.user, subscription=subscription, tag=tag
) )
# Fetch synchronously so the user sees results immediately; the # Enqueue the initial fetch instead of running it inline (T3) — a
# periodic Celery poll takes over from here for subsequent fetches. # slow or large feed would otherwise tie up a Gunicorn worker for
result = fetch_and_ingest(feed) # the duration of the request; fetch_feed handles the save/error
feed.save() # bookkeeping the same way the periodic poll's dispatch does.
fetch_feed.delay(feed.pk)
if result.status == Feed.STATUS_ERROR: messages.success(
messages.warning( request,
request, f"Subscribed to {feed_url}. Fetching its articles now — "
f"Subscribed to {feed_url}, but the initial fetch failed: {result.error_message}", "refresh in a few seconds if you don't see them yet.",
) )
else:
article_count = len(result.new_article_ids)
messages.success(
request,
f"Subscribed to '{subscription.display_title}' — "
f"{article_count} article(s) fetched.",
)
return redirect("reader:stream") return redirect("reader:stream")
return render(request, "feeds/add_feed.html", {"form": form}) return render(request, "feeds/add_feed.html", {"form": form})