Use keyset pagination for the stream (C4)

The stream did a COUNT(*) and OFFSET slice on every request, ordered by
a non-unique (published_at, fetched_at) — deep pages got slower and
articles sharing a timestamp could repeat or vanish across pages.

It now orders by (Coalesce(published_at, fetched_at), id) and pages with
before/after cursors, fetching one extra row to know whether another
page exists instead of counting. Pagination links also keep the feed
scope, which the old links silently dropped.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5.5 committed 2026-11-01 08:51:20 -07:00
1 parent 70b0d4f277
commit 8d2900424a
3 files changed
+156 -16

No files matched your search

+95
View File
@@ -343,3 +343,98 @@ def test_sidebar_unread_count_updates_immediately_after_mark_all_read(client, al
response = client.get(reverse("reader:stream"))
assert "unread-count" not in response.content.decode()
def _stream_article_ids(client, query):
response = client.get(reverse("reader:stream"), query)
content = response.content.decode()
ids = [int(m) for m in re.findall(r'<li id="article-(\d+)"', content)]
return ids, content
def _link_query(content, label):
match = re.search(rf'<a href="\?([^"]+)">{label}', content)
if not match:
return None
from urllib.parse import parse_qsl
return dict(parse_qsl(match.group(1).replace("&amp;", "&")))
@pytest.mark.django_db
def test_stream_keyset_pagination_pages_through_tied_timestamps_without_gaps(client, alice):
"""Keyset pagination (C4) on (sort_at, id): articles sharing an exact
timestamp must neither repeat nor go missing across page boundaries,
which offset paging over a non-unique order can't guarantee."""
from django.utils import timezone
from apps.reader.views import PAGE_SIZE
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
Subscription.objects.create(user=alice, feed=feed)
same_moment = timezone.now()
total = PAGE_SIZE * 2 + 5
created = [
Article.objects.create(feed=feed, guid=f"g{i}", title=f"T{i}", published_at=same_moment)
for i in range(total)
]
client.login(username="alice", password="pw-alice-123")
seen = []
query = {"filter": "all"}
pages = 0
while query:
ids, content = _stream_article_ids(client, query)
assert len(ids) <= PAGE_SIZE
seen.extend(ids)
query = _link_query(content, "Older")
pages += 1
assert pages < 10
assert pages == 3
assert len(seen) == total
assert sorted(seen) == sorted(a.pk for a in created)
@pytest.mark.django_db
def test_stream_newer_link_returns_to_the_previous_page(client, alice):
from django.utils import timezone
from apps.reader.views import PAGE_SIZE
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
Subscription.objects.create(user=alice, feed=feed)
now = timezone.now()
for i in range(PAGE_SIZE + 5):
Article.objects.create(
feed=feed,
guid=f"g{i}",
title=f"T{i}",
published_at=now - timezone.timedelta(minutes=i),
)
client.login(username="alice", password="pw-alice-123")
first_ids, first_content = _stream_article_ids(client, {"filter": "all"})
assert _link_query(first_content, "Newer") is None
second_ids, second_content = _stream_article_ids(client, _link_query(first_content, "Older"))
assert len(second_ids) == 5
assert _link_query(second_content, "Older") is None
back_ids, _ = _stream_article_ids(client, _link_query(second_content, "Newer"))
assert back_ids == first_ids
@pytest.mark.django_db
def test_stream_pagination_links_keep_the_feed_scope(client, alice):
from apps.reader.views import PAGE_SIZE
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
Subscription.objects.create(user=alice, feed=feed)
for i in range(PAGE_SIZE + 1):
Article.objects.create(feed=feed, guid=f"g{i}", title=f"T{i}")
client.login(username="alice", password="pw-alice-123")
_, content = _stream_article_ids(client, {"filter": "all", "feed": feed.pk})
assert _link_query(content, "Older")["feed"] == str(feed.pk)