Index Postgres search and harden SQLite search/concurrency (C5, C6)

Postgres recomputed a tsvector over every article's full text on every
search. A migration now adds a stored, GIN-indexed search_vector column
kept current by a trigger (so no ingest/update path can forget it), and
the backend reads that column. Untested here: no Postgres in the dev
sandbox, so the SQL is reviewed but not executed — worth one run of the
(skipped-on-SQLite) Postgres test module against a real instance.

SQLite search built one bound parameter per subscribed feed, so enough
feeds would exceed the variable limit, and the OperationalError was
swallowed into "no results". It now scopes through a subquery on
Subscription and logs failures. SQLite connections also get WAL mode,
a 20s busy timeout, and IMMEDIATE transactions so gunicorn, Celery, and
the FTS triggers stop tripping over each other.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5.5 committed 2026-11-01 08:51:20 -07:00
1 parent cb0ce5a336
commit 70b0d4f277
4 files changed
+104 -21

No files matched your search

+13 -12
View File
@@ -1,27 +1,28 @@
from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector
from apps.articles.models import Article
from .base import SearchBackend
# Matches the stored search_vector column, GIN index, and maintaining
# trigger created by apps/search/migrations/0002_postgres_search_vector.py
# — the query reads that column directly instead of recomputing a
# tsvector over every article's full text on every search (C5).
_VECTOR = "articles_article.search_vector"
_QUERY = "plainto_tsquery('english', %s)"
class PostgresSearchBackend(SearchBackend):
"""Computes the search vector on the fly at query time rather than
storing/indexing it on Article — keeps the schema identical to the
SQLite backend. Revisit with a stored+GIN-indexed column only if an
instance's archive grows large enough to need it."""
def search(self, query: str, user, limit: int = 50):
if not query.strip():
return Article.objects.none()
vector = SearchVector("title", weight="A") + SearchVector("content_html", weight="B")
search_query = SearchQuery(query)
return (
Article.objects.filter(feed__subscriptions__user=user)
.select_related("feed")
.annotate(search=vector, rank=SearchRank(vector, search_query))
.filter(search=search_query)
.extra(
select={"rank": f"ts_rank({_VECTOR}, {_QUERY})"},
select_params=[query],
where=[f"{_VECTOR} @@ {_QUERY}"],
params=[query],
)
.order_by("-rank", "-published_at")[:limit]
)
+15 -9
View File
@@ -1,12 +1,14 @@
import logging
import sqlite3
from django.db import connection
from apps.articles.models import Article
from apps.feeds.models import Subscription
from .base import SearchBackend
logger = logging.getLogger(__name__)
def _fts_phrase_query(raw_query: str) -> str:
"""Wraps the whole user input as a single FTS5 phrase so arbitrary
@@ -21,24 +23,28 @@ class SQLiteFTSBackend(SearchBackend):
if not query.strip():
return []
feed_ids = list(Subscription.objects.filter(user=user).values_list("feed_id", flat=True))
if not feed_ids:
return []
placeholders = ",".join(["%s"] * len(feed_ids))
sql = f"""
# Scoped through a subquery on Subscription rather than an IN
# list built from every subscribed feed id (C6): a user with
# thousands of feeds would otherwise exceed SQLite's bound-
# parameter limit and the whole search would fail.
sql = """
SELECT a.id
FROM articles_article a
JOIN articles_fts f ON f.rowid = a.id
WHERE articles_fts MATCH %s AND a.feed_id IN ({placeholders})
WHERE articles_fts MATCH %s
AND a.feed_id IN (SELECT feed_id FROM feeds_subscription WHERE user_id = %s)
ORDER BY rank
LIMIT %s
"""
try:
with connection.cursor() as cursor:
cursor.execute(sql, [_fts_phrase_query(query), *feed_ids, limit])
cursor.execute(sql, [_fts_phrase_query(query), user.pk, limit])
ordered_ids = [row[0] for row in cursor.fetchall()]
except sqlite3.OperationalError:
# A genuinely bad FTS query is rare (the input is quoted as a
# phrase), so this is far more likely a real operational
# failure — log it instead of silently showing "no results".
logger.exception("SQLite FTS search failed for user %s", user.pk)
return []
if not ordered_ids: