Compare commits

..
Author SHA1 Message Date
ksmithandClaude Sonnet 5 17e7f63e61 Update link/author/summary/published_at on refetch, not just title (T6)
A changed guid's article only ever had its title/content_html/
content_hash refreshed on a subsequent poll — legitimate feed edits to
an entry's link, author, summary, or publication date were silently
ignored forever, which can affect navigation, ordering, and rules that
key off those fields.

content_hash is now computed over every mutable field feedparser gives
us for an entry (title, link, author, summary, content_html,
published_at), and all of them are written together when the hash
changes. Per-user read/star state is untouched either way, same as
before.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-31 14:44:15 -06:00
ksmithandClaude Sonnet 5 0bfe9c8076 Enqueue the initial feed fetch instead of running it inline (T3)
add_feed ran fetch_and_ingest synchronously during the POST request, so
a slow or large feed tied up a Gunicorn worker for the duration of the
fetch (up to the 15s HTTP timeout, plus parse/ingest time) and a user
could repeat it with distinct URLs to reduce request capacity.

The subscription is now created synchronously as before, but the first
fetch is dispatched via fetch_feed.delay() — the same task the periodic
poll uses, so the save/error bookkeeping is unchanged. The user sees an
immediate redirect with a message that the fetch is in progress rather
than waiting on it in the request/response cycle.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-31 14:44:09 -06:00
ksmithandClaude Sonnet 5 7bb981b545 Rebuild stream date-group headers in the viewer's local timezone
The server still groups articles into "Received today/yesterday/<date>"
headers using UTC, so an article published near UTC midnight could land
in a mismatched day group. Each article row now ships its raw UTC
instant (data-published-utc); static/js/date-groups.js discards the
server-rendered headers on load and rebuilds them using the browser's
own local calendar day, same approach as the article-time localization
this follows up on.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-10-31 14:36:27 -06:00
8 changed files with 178 additions and 28 deletions

No files matched your search

+33 -8
View File
@@ -97,8 +97,15 @@ def sanitize_html(raw_html: str) -> str:
return cleaned[:MAX_CONTENT_HTML_CHARS] return cleaned[:MAX_CONTENT_HTML_CHARS]
def _content_hash(title: str, content_html: str) -> str: def _content_hash(
return hashlib.sha256(f"{title}\n{content_html}".encode("utf-8")).hexdigest() title: str, link: str, author: str, summary: str, content_html: str, published_at
) -> str:
"""Covers every mutable Article field an entry carries (T6) — a feed
editing its link/author/summary/published date, not just title or
body, must still be detected as a change worth re-saving."""
published_key = published_at.isoformat() if published_at else ""
payload = "\n".join([title, link, author, summary, content_html, published_key])
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
def _entry_guid(entry) -> str: def _entry_guid(entry) -> str:
@@ -213,20 +220,24 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
for entry in entries: for entry in entries:
guid = _entry_guid(entry) guid = _entry_guid(entry)
title = (getattr(entry, "title", "") or "")[:1000] title = (getattr(entry, "title", "") or "")[:1000]
link = (getattr(entry, "link", "") or "")[:2000]
author = (getattr(entry, "author", "") or "")[:500]
summary = (getattr(entry, "summary", "") or "")[:5000]
content_html = sanitize_html(_entry_content_html(entry)) content_html = sanitize_html(_entry_content_html(entry))
content_hash = _content_hash(title, content_html) published_at = _entry_published_at(entry)
content_hash = _content_hash(title, link, author, summary, content_html, published_at)
article, created = Article.objects.get_or_create( article, created = Article.objects.get_or_create(
feed=feed, feed=feed,
guid=guid, guid=guid,
defaults={ defaults={
"link": (getattr(entry, "link", "") or "")[:2000], "link": link,
"title": title, "title": title,
"author": (getattr(entry, "author", "") or "")[:500], "author": author,
"content_html": content_html, "content_html": content_html,
"summary": (getattr(entry, "summary", "") or "")[:5000], "summary": summary,
"content_hash": content_hash, "content_hash": content_hash,
"published_at": _entry_published_at(entry), "published_at": published_at,
}, },
) )
@@ -234,9 +245,23 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
new_ids.append(article.pk) new_ids.append(article.pk)
elif article.content_hash != content_hash: elif article.content_hash != content_hash:
article.title = title article.title = title
article.link = link
article.author = author
article.summary = summary
article.content_html = content_html article.content_html = content_html
article.published_at = published_at
article.content_hash = content_hash article.content_hash = content_hash
article.save(update_fields=["title", "content_html", "content_hash"]) article.save(
update_fields=[
"title",
"link",
"author",
"summary",
"content_html",
"published_at",
"content_hash",
]
)
updated_ids.append(article.pk) updated_ids.append(article.pk)
return FetchResult( return FetchResult(
+36
View File
@@ -1,3 +1,5 @@
import datetime as dt
import pytest import pytest
import requests import requests
import responses import responses
@@ -204,6 +206,40 @@ def test_refetching_updated_content_does_not_reset_user_state():
assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists() assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists()
@pytest.mark.django_db
@responses.activate
def test_refetching_picks_up_link_author_summary_and_published_at_changes():
"""Previously only title/content_html/content_hash were updated on
refetch (T6) — a feed correcting its link, author, summary, or
publication date without touching the title went unnoticed forever."""
feed = Feed.objects.create(feed_url=FEED_URL)
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
fetch_and_ingest(feed)
feed.save()
article = Article.objects.get(feed=feed)
assert article.link == "https://example.com/1"
assert article.author == ""
edited_xml = GOOD_FEED_XML.replace(
"<link>https://example.com/1</link>",
"<link>https://example.com/1-moved</link>",
).replace(
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>",
"<pubDate>Tue, 02 Jan 2024 00:00:00 GMT</pubDate>\n <author>jane@example.com</author>",
)
responses.add(responses.GET, FEED_URL, body=edited_xml, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.updated_article_ids == [article.pk]
article.refresh_from_db()
assert article.link == "https://example.com/1-moved"
assert article.author == "jane@example.com"
assert article.published_at == dt.datetime(2024, 1, 2, tzinfo=dt.timezone.utc)
def test_sanitize_html_caps_stored_output_length(): def test_sanitize_html_caps_stored_output_length():
huge = "<p>" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "</p>" huge = "<p>" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "</p>"
cleaned = sanitize_html(huge) cleaned = sanitize_html(huge)
+8 -3
View File
@@ -54,7 +54,11 @@ GOOD_FEED_XML = """<?xml version="1.0"?>
@pytest.mark.django_db @pytest.mark.django_db
@responses.activate @responses.activate
def test_add_feed_view_subscribes_and_fetches_immediately(client, two_users): def test_add_feed_view_subscribes_and_enqueues_the_initial_fetch(client, two_users):
"""The subscription is created synchronously; the first fetch is
dispatched via fetch_feed.delay() (T3) rather than run inline — in
tests CELERY_TASK_ALWAYS_EAGER still runs it immediately, so the
article it ingests is visible by the time the response returns."""
alice, _ = two_users alice, _ = two_users
client.login(username="alice", password="pw-alice-123") client.login(username="alice", password="pw-alice-123")
responses.add(responses.GET, "https://example.com/feed.xml", body=GOOD_FEED_XML, status=200) responses.add(responses.GET, "https://example.com/feed.xml", body=GOOD_FEED_XML, status=200)
@@ -62,9 +66,10 @@ def test_add_feed_view_subscribes_and_fetches_immediately(client, two_users):
response = client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/feed.xml"}) response = client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/feed.xml"})
assert response.status_code == 302 assert response.status_code == 302
assert Subscription.objects.filter( subscription = Subscription.objects.get(
user=alice, feed__feed_url="https://example.com/feed.xml" user=alice, feed__feed_url="https://example.com/feed.xml"
).exists() )
assert subscription.feed.articles.filter(guid="guid-1").exists()
@pytest.mark.django_db @pytest.mark.django_db
+8 -16
View File
@@ -12,8 +12,7 @@ from apps.articles.models import SubscriptionTag, Tag
from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm
from .models import Feed, Subscription from .models import Feed, Subscription
from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml
from .services.fetcher import fetch_and_ingest from .tasks import fetch_feed, import_opml_task
from .tasks import import_opml_task
OPML_ASYNC_THRESHOLD = 50 OPML_ASYNC_THRESHOLD = 50
@@ -39,22 +38,15 @@ def add_feed(request):
user=request.user, subscription=subscription, tag=tag user=request.user, subscription=subscription, tag=tag
) )
# Fetch synchronously so the user sees results immediately; the # Enqueue the initial fetch instead of running it inline (T3) — a
# periodic Celery poll takes over from here for subsequent fetches. # slow or large feed would otherwise tie up a Gunicorn worker for
result = fetch_and_ingest(feed) # the duration of the request; fetch_feed handles the save/error
feed.save() # bookkeeping the same way the periodic poll's dispatch does.
fetch_feed.delay(feed.pk)
if result.status == Feed.STATUS_ERROR:
messages.warning(
request,
f"Subscribed to {feed_url}, but the initial fetch failed: {result.error_message}",
)
else:
article_count = len(result.new_article_ids)
messages.success( messages.success(
request, request,
f"Subscribed to '{subscription.display_title}' — " f"Subscribed to {feed_url}. Fetching its articles now — "
f"{article_count} article(s) fetched.", "refresh in a few seconds if you don't see them yet.",
) )
return redirect("reader:stream") return redirect("reader:stream")
+29
View File
@@ -1,3 +1,5 @@
import re
import pytest import pytest
from django.contrib.auth import get_user_model from django.contrib.auth import get_user_model
from django.urls import reverse from django.urls import reverse
@@ -188,3 +190,30 @@ def test_sidebar_shows_a_tags_total_unread_count(client, alice):
# The only "7" on the page — the individual feeds show 3 and 4. # The only "7" on the page — the individual feeds show 3 and 4.
content = response.content.decode() content = response.content.decode()
assert '<span class="unread-count">7</span>' in content assert '<span class="unread-count">7</span>' in content
@pytest.mark.django_db
def test_article_rows_carry_their_utc_instant_for_client_side_date_grouping(client, alice):
"""The server still groups date headers in UTC, but ships each row's
raw UTC instant so date-groups.js can rebuild the headers using the
viewer's local calendar day instead."""
from django.utils import timezone
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
Subscription.objects.create(user=alice, feed=feed)
dated = Article.objects.create(
feed=feed, guid="dated", title="Dated", published_at=timezone.now()
)
undated = Article.objects.create(feed=feed, guid="undated", title="Undated")
client.login(username="alice", password="pw-alice-123")
response = client.get(reverse("reader:stream"), {"filter": "all"})
content = response.content.decode()
expected_iso = dated.published_at.isoformat()
assert f'id="article-{dated.pk}"' in content
assert f'data-published-utc="{expected_iso}"' in content
undated_row = re.search(rf'<li id="article-{undated.pk}".*?>', content, flags=re.DOTALL)
assert undated_row is not None
assert 'data-published-utc=""' in undated_row.group(0)
+61
View File
@@ -0,0 +1,61 @@
(function () {
// The server groups articles into "Received today/yesterday/<date>"
// headers using UTC (config.settings.base TIME_ZONE), so an article
// published near UTC midnight can land in the wrong day group from the
// viewer's own perspective. Each article row carries its UTC instant in
// data-published-utc; this throws away the server's date-header <li>s
// and rebuilds them using the browser's local calendar day instead.
var MONTHS = [
"January", "February", "March", "April", "May", "June",
"July", "August", "September", "October", "November", "December",
];
function localDateOf(iso) {
if (!iso) return null;
var date = new Date(iso);
if (isNaN(date.getTime())) return null;
date.setHours(0, 0, 0, 0);
return date;
}
function labelFor(day, today) {
if (day === null) return "Undated";
var formatted = day.getDate() + " " + MONTHS[day.getMonth()] + " " + day.getFullYear();
var diffDays = Math.round((today - day) / 86400000);
if (diffDays === 0) return "Received today — " + formatted;
if (diffDays === 1) return "Received yesterday — " + formatted;
return "Received " + formatted;
}
function regroupDateHeaders(root) {
if (!root) return;
var list = root.matches && root.matches("ul.article-list") ? root : root.querySelector("ul.article-list");
if (!list || !list.querySelector("li.article-row[data-published-utc], li.article-row")) return;
list.querySelectorAll("li.date-header").forEach(function (el) {
el.parentNode.removeChild(el);
});
var today = new Date();
today.setHours(0, 0, 0, 0);
var currentKey;
list.querySelectorAll("li.article-row").forEach(function (row) {
var day = localDateOf(row.getAttribute("data-published-utc"));
var key = day === null ? "undated" : day.getTime();
if (key !== currentKey) {
var header = document.createElement("li");
header.className = "date-header";
header.textContent = labelFor(day, today);
list.insertBefore(header, row);
currentKey = key;
}
});
}
regroupDateHeaders(document);
document.body.addEventListener("htmx:afterSwap", function (event) {
regroupDateHeaders(event.target);
});
})();
+1
View File
@@ -54,6 +54,7 @@
<script src="{% static 'js/stream.js' %}" defer></script> <script src="{% static 'js/stream.js' %}" defer></script>
<script src="{% static 'js/localize-times.js' %}" defer></script> <script src="{% static 'js/localize-times.js' %}" defer></script>
<script src="{% static 'js/date-groups.js' %}" defer></script>
{% if user.is_authenticated %} {% if user.is_authenticated %}
<script src="{% static 'js/theme.js' %}" defer></script> <script src="{% static 'js/theme.js' %}" defer></script>
<script src="{% static 'js/sidebar-state.js' %}" defer></script> <script src="{% static 'js/sidebar-state.js' %}" defer></script>
+2 -1
View File
@@ -1,4 +1,5 @@
<li id="article-{{ article.pk }}" class="article-row{% if article.is_read %} is-read{% endif %}"> <li id="article-{{ article.pk }}" class="article-row{% if article.is_read %} is-read{% endif %}"
data-published-utc="{% if article.published_at %}{{ article.published_at|date:"c" }}{% endif %}">
<div class="article-row-main"> <div class="article-row-main">
{% include "reader/_article_actions.html" %} {% include "reader/_article_actions.html" %}
{% if article.feed.favicon_url %} {% if article.feed.favicon_url %}