Private
Public Access
Use keyset pagination for the stream (C4)
The stream did a COUNT(*) and OFFSET slice on every request, ordered by a non-unique (published_at, fetched_at) — deep pages got slower and articles sharing a timestamp could repeat or vanish across pages. It now orders by (Coalesce(published_at, fetched_at), id) and pages with before/after cursors, fetching one extra row to know whether another page exists instead of counting. Pagination links also keep the feed scope, which the old links silently dropped. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
70b0d4f277
commit
8d2900424a
3 files changed
+156
-16
No files matched your search
@@ -343,3 +343,98 @@ def test_sidebar_unread_count_updates_immediately_after_mark_all_read(client, al
|
||||
|
||||
response = client.get(reverse("reader:stream"))
|
||||
assert "unread-count" not in response.content.decode()
|
||||
|
||||
|
||||
def _stream_article_ids(client, query):
|
||||
response = client.get(reverse("reader:stream"), query)
|
||||
content = response.content.decode()
|
||||
ids = [int(m) for m in re.findall(r'<li id="article-(\d+)"', content)]
|
||||
return ids, content
|
||||
|
||||
|
||||
def _link_query(content, label):
|
||||
match = re.search(rf'<a href="\?([^"]+)">{label}', content)
|
||||
if not match:
|
||||
return None
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
return dict(parse_qsl(match.group(1).replace("&", "&")))
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_stream_keyset_pagination_pages_through_tied_timestamps_without_gaps(client, alice):
|
||||
"""Keyset pagination (C4) on (sort_at, id): articles sharing an exact
|
||||
timestamp must neither repeat nor go missing across page boundaries,
|
||||
which offset paging over a non-unique order can't guarantee."""
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.reader.views import PAGE_SIZE
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
same_moment = timezone.now()
|
||||
total = PAGE_SIZE * 2 + 5
|
||||
created = [
|
||||
Article.objects.create(feed=feed, guid=f"g{i}", title=f"T{i}", published_at=same_moment)
|
||||
for i in range(total)
|
||||
]
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
seen = []
|
||||
query = {"filter": "all"}
|
||||
pages = 0
|
||||
while query:
|
||||
ids, content = _stream_article_ids(client, query)
|
||||
assert len(ids) <= PAGE_SIZE
|
||||
seen.extend(ids)
|
||||
query = _link_query(content, "Older")
|
||||
pages += 1
|
||||
assert pages < 10
|
||||
|
||||
assert pages == 3
|
||||
assert len(seen) == total
|
||||
assert sorted(seen) == sorted(a.pk for a in created)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_stream_newer_link_returns_to_the_previous_page(client, alice):
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.reader.views import PAGE_SIZE
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
now = timezone.now()
|
||||
for i in range(PAGE_SIZE + 5):
|
||||
Article.objects.create(
|
||||
feed=feed,
|
||||
guid=f"g{i}",
|
||||
title=f"T{i}",
|
||||
published_at=now - timezone.timedelta(minutes=i),
|
||||
)
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
first_ids, first_content = _stream_article_ids(client, {"filter": "all"})
|
||||
assert _link_query(first_content, "Newer") is None
|
||||
|
||||
second_ids, second_content = _stream_article_ids(client, _link_query(first_content, "Older"))
|
||||
assert len(second_ids) == 5
|
||||
assert _link_query(second_content, "Older") is None
|
||||
|
||||
back_ids, _ = _stream_article_ids(client, _link_query(second_content, "Newer"))
|
||||
assert back_ids == first_ids
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_stream_pagination_links_keep_the_feed_scope(client, alice):
|
||||
from apps.reader.views import PAGE_SIZE
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
for i in range(PAGE_SIZE + 1):
|
||||
Article.objects.create(feed=feed, guid=f"g{i}", title=f"T{i}")
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
_, content = _stream_article_ids(client, {"filter": "all", "feed": feed.pk})
|
||||
|
||||
assert _link_query(content, "Older")["feed"] == str(feed.pk)
|
||||
+57
-12
@@ -1,8 +1,10 @@
|
||||
from datetime import datetime
|
||||
from itertools import groupby
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.db.models import Max, Q
|
||||
from django.db.models.functions import Coalesce
|
||||
from django.http import Http404
|
||||
from django.shortcuts import get_object_or_404, redirect, render
|
||||
from django.urls import reverse
|
||||
@@ -157,26 +159,68 @@ def _filtered_queryset(request, active_filter):
|
||||
return _apply_scope(request, queryset)
|
||||
|
||||
|
||||
def _encode_cursor(article) -> str:
|
||||
return f"{article.sort_at.isoformat()},{article.pk}"
|
||||
|
||||
|
||||
def _decode_cursor(raw):
|
||||
"""(sort_at, id) from a cursor produced by _encode_cursor, or None for
|
||||
a missing/garbled one (treated as "no cursor" — the first page)."""
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
timestamp, pk = raw.rsplit(",", 1)
|
||||
return datetime.fromisoformat(timestamp), int(pk)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
@login_required
|
||||
def stream(request):
|
||||
active_filter = _resolve_active_filter(request)
|
||||
queryset, active_tag, active_feed = _filtered_queryset(request, active_filter)
|
||||
|
||||
try:
|
||||
page = max(int(request.GET.get("page", 1)), 1)
|
||||
except ValueError:
|
||||
page = 1
|
||||
start, end = (page - 1) * PAGE_SIZE, page * PAGE_SIZE
|
||||
|
||||
total_count = queryset.count()
|
||||
articles = with_user_state(queryset[start:end], request.user)
|
||||
|
||||
scope_params = {"filter": active_filter}
|
||||
if active_tag:
|
||||
scope_params["tag"] = active_tag.pk
|
||||
if active_feed:
|
||||
scope_params["feed"] = active_feed.pk
|
||||
|
||||
# Keyset pagination (C4) over (sort_at, id) — sort_at falls back to
|
||||
# fetched_at for entries a feed gave no published date, and id breaks
|
||||
# ties so pages neither shift nor repeat articles that share a
|
||||
# timestamp. No COUNT(*): fetching one row past the page size is
|
||||
# enough to know whether another page exists in that direction.
|
||||
ordered = queryset.annotate(sort_at=Coalesce("published_at", "fetched_at"))
|
||||
after = _decode_cursor(request.GET.get("after"))
|
||||
before = _decode_cursor(request.GET.get("before"))
|
||||
|
||||
if after:
|
||||
sort_at, pk = after
|
||||
rows = list(
|
||||
ordered.filter(Q(sort_at__gt=sort_at) | Q(sort_at=sort_at, id__gt=pk)).order_by(
|
||||
"sort_at", "id"
|
||||
)[: PAGE_SIZE + 1]
|
||||
)
|
||||
has_newer_page = len(rows) > PAGE_SIZE
|
||||
has_older_page = True
|
||||
rows = rows[:PAGE_SIZE][::-1]
|
||||
else:
|
||||
if before:
|
||||
sort_at, pk = before
|
||||
ordered = ordered.filter(Q(sort_at__lt=sort_at) | Q(sort_at=sort_at, id__lt=pk))
|
||||
rows = list(ordered.order_by("-sort_at", "-id")[: PAGE_SIZE + 1])
|
||||
has_older_page = len(rows) > PAGE_SIZE
|
||||
has_newer_page = bool(before)
|
||||
rows = rows[:PAGE_SIZE]
|
||||
|
||||
articles = with_user_state(rows, request.user)
|
||||
|
||||
newer_qs = older_qs = ""
|
||||
if articles:
|
||||
newer_qs = urlencode({**scope_params, "after": _encode_cursor(articles[0])})
|
||||
older_qs = urlencode({**scope_params, "before": _encode_cursor(articles[-1])})
|
||||
|
||||
context = {
|
||||
"date_groups": _group_by_date(articles),
|
||||
"active_filter": active_filter,
|
||||
@@ -184,9 +228,10 @@ def stream(request):
|
||||
"active_tag": active_tag,
|
||||
"active_feed": active_feed,
|
||||
"user_tags": Tag.objects.filter(user=request.user),
|
||||
"page": page,
|
||||
"has_next_page": end < total_count,
|
||||
"has_previous_page": page > 1,
|
||||
"has_newer_page": has_newer_page and bool(articles),
|
||||
"has_older_page": has_older_page and bool(articles),
|
||||
"newer_qs": newer_qs,
|
||||
"older_qs": older_qs,
|
||||
# Lets the "new articles" banner ask, on this exact same scope,
|
||||
# whether anything with a higher id has shown up since this page
|
||||
# was rendered — id (not published_at) because it's monotonic
|
||||
|
||||
Reference in new issue
Block a user