From 8d2900424ae4b6da06453d435ac68a64b3162de1 Mon Sep 17 00:00:00 2001 From: Keith Smith Date: Sun, 1 Nov 2026 08:51:20 -0700 Subject: [PATCH] Use keyset pagination for the stream (C4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stream did a COUNT(*) and OFFSET slice on every request, ordered by a non-unique (published_at, fetched_at) — deep pages got slower and articles sharing a timestamp could repeat or vanish across pages. It now orders by (Coalesce(published_at, fetched_at), id) and pages with before/after cursors, fetching one extra row to know whether another page exists instead of counting. Pagination links also keep the feed scope, which the old links silently dropped. Co-Authored-By: Claude Sonnet 5.5 --- apps/reader/tests.py | 95 +++++++++++++++++++++++++++++ apps/reader/views.py | 69 +++++++++++++++++---- templates/reader/_article_list.html | 8 +-- 3 files changed, 156 insertions(+), 16 deletions(-) diff --git a/apps/reader/tests.py b/apps/reader/tests.py index 96a0e66..26fe5e2 100644 --- a/apps/reader/tests.py +++ b/apps/reader/tests.py @@ -343,3 +343,98 @@ def test_sidebar_unread_count_updates_immediately_after_mark_all_read(client, al response = client.get(reverse("reader:stream")) assert "unread-count" not in response.content.decode() + + +def _stream_article_ids(client, query): + response = client.get(reverse("reader:stream"), query) + content = response.content.decode() + ids = [int(m) for m in re.findall(r'
  • {label}', content) + if not match: + return None + from urllib.parse import parse_qsl + + return dict(parse_qsl(match.group(1).replace("&", "&"))) + + +@pytest.mark.django_db +def test_stream_keyset_pagination_pages_through_tied_timestamps_without_gaps(client, alice): + """Keyset pagination (C4) on (sort_at, id): articles sharing an exact + timestamp must neither repeat nor go missing across page boundaries, + which offset paging over a non-unique order can't guarantee.""" + from django.utils import timezone + + from apps.reader.views import PAGE_SIZE + + feed = Feed.objects.create(feed_url="https://example.com/feed.xml") + Subscription.objects.create(user=alice, feed=feed) + same_moment = timezone.now() + total = PAGE_SIZE * 2 + 5 + created = [ + Article.objects.create(feed=feed, guid=f"g{i}", title=f"T{i}", published_at=same_moment) + for i in range(total) + ] + + client.login(username="alice", password="pw-alice-123") + seen = [] + query = {"filter": "all"} + pages = 0 + while query: + ids, content = _stream_article_ids(client, query) + assert len(ids) <= PAGE_SIZE + seen.extend(ids) + query = _link_query(content, "Older") + pages += 1 + assert pages < 10 + + assert pages == 3 + assert len(seen) == total + assert sorted(seen) == sorted(a.pk for a in created) + + +@pytest.mark.django_db +def test_stream_newer_link_returns_to_the_previous_page(client, alice): + from django.utils import timezone + + from apps.reader.views import PAGE_SIZE + + feed = Feed.objects.create(feed_url="https://example.com/feed.xml") + Subscription.objects.create(user=alice, feed=feed) + now = timezone.now() + for i in range(PAGE_SIZE + 5): + Article.objects.create( + feed=feed, + guid=f"g{i}", + title=f"T{i}", + published_at=now - timezone.timedelta(minutes=i), + ) + + client.login(username="alice", password="pw-alice-123") + first_ids, first_content = _stream_article_ids(client, {"filter": "all"}) + assert _link_query(first_content, "Newer") is None + + second_ids, second_content = _stream_article_ids(client, _link_query(first_content, "Older")) + assert len(second_ids) == 5 + assert _link_query(second_content, "Older") is None + + back_ids, _ = _stream_article_ids(client, _link_query(second_content, "Newer")) + assert back_ids == first_ids + + +@pytest.mark.django_db +def test_stream_pagination_links_keep_the_feed_scope(client, alice): + from apps.reader.views import PAGE_SIZE + + feed = Feed.objects.create(feed_url="https://example.com/feed.xml") + Subscription.objects.create(user=alice, feed=feed) + for i in range(PAGE_SIZE + 1): + Article.objects.create(feed=feed, guid=f"g{i}", title=f"T{i}") + + client.login(username="alice", password="pw-alice-123") + _, content = _stream_article_ids(client, {"filter": "all", "feed": feed.pk}) + + assert _link_query(content, "Older")["feed"] == str(feed.pk) diff --git a/apps/reader/views.py b/apps/reader/views.py index 2b82ad1..ae5a22d 100644 --- a/apps/reader/views.py +++ b/apps/reader/views.py @@ -1,8 +1,10 @@ +from datetime import datetime from itertools import groupby from urllib.parse import urlencode from django.contrib.auth.decorators import login_required from django.db.models import Max, Q +from django.db.models.functions import Coalesce from django.http import Http404 from django.shortcuts import get_object_or_404, redirect, render from django.urls import reverse @@ -157,26 +159,68 @@ def _filtered_queryset(request, active_filter): return _apply_scope(request, queryset) +def _encode_cursor(article) -> str: + return f"{article.sort_at.isoformat()},{article.pk}" + + +def _decode_cursor(raw): + """(sort_at, id) from a cursor produced by _encode_cursor, or None for + a missing/garbled one (treated as "no cursor" — the first page).""" + if not raw: + return None + try: + timestamp, pk = raw.rsplit(",", 1) + return datetime.fromisoformat(timestamp), int(pk) + except ValueError: + return None + + @login_required def stream(request): active_filter = _resolve_active_filter(request) queryset, active_tag, active_feed = _filtered_queryset(request, active_filter) - try: - page = max(int(request.GET.get("page", 1)), 1) - except ValueError: - page = 1 - start, end = (page - 1) * PAGE_SIZE, page * PAGE_SIZE - - total_count = queryset.count() - articles = with_user_state(queryset[start:end], request.user) - scope_params = {"filter": active_filter} if active_tag: scope_params["tag"] = active_tag.pk if active_feed: scope_params["feed"] = active_feed.pk + # Keyset pagination (C4) over (sort_at, id) — sort_at falls back to + # fetched_at for entries a feed gave no published date, and id breaks + # ties so pages neither shift nor repeat articles that share a + # timestamp. No COUNT(*): fetching one row past the page size is + # enough to know whether another page exists in that direction. + ordered = queryset.annotate(sort_at=Coalesce("published_at", "fetched_at")) + after = _decode_cursor(request.GET.get("after")) + before = _decode_cursor(request.GET.get("before")) + + if after: + sort_at, pk = after + rows = list( + ordered.filter(Q(sort_at__gt=sort_at) | Q(sort_at=sort_at, id__gt=pk)).order_by( + "sort_at", "id" + )[: PAGE_SIZE + 1] + ) + has_newer_page = len(rows) > PAGE_SIZE + has_older_page = True + rows = rows[:PAGE_SIZE][::-1] + else: + if before: + sort_at, pk = before + ordered = ordered.filter(Q(sort_at__lt=sort_at) | Q(sort_at=sort_at, id__lt=pk)) + rows = list(ordered.order_by("-sort_at", "-id")[: PAGE_SIZE + 1]) + has_older_page = len(rows) > PAGE_SIZE + has_newer_page = bool(before) + rows = rows[:PAGE_SIZE] + + articles = with_user_state(rows, request.user) + + newer_qs = older_qs = "" + if articles: + newer_qs = urlencode({**scope_params, "after": _encode_cursor(articles[0])}) + older_qs = urlencode({**scope_params, "before": _encode_cursor(articles[-1])}) + context = { "date_groups": _group_by_date(articles), "active_filter": active_filter, @@ -184,9 +228,10 @@ def stream(request): "active_tag": active_tag, "active_feed": active_feed, "user_tags": Tag.objects.filter(user=request.user), - "page": page, - "has_next_page": end < total_count, - "has_previous_page": page > 1, + "has_newer_page": has_newer_page and bool(articles), + "has_older_page": has_older_page and bool(articles), + "newer_qs": newer_qs, + "older_qs": older_qs, # Lets the "new articles" banner ask, on this exact same scope, # whether anything with a higher id has shown up since this page # was rendered — id (not published_at) because it's monotonic diff --git a/templates/reader/_article_list.html b/templates/reader/_article_list.html index be9f896..bd4853d 100644 --- a/templates/reader/_article_list.html +++ b/templates/reader/_article_list.html @@ -10,10 +10,10 @@