Private
Public Access
Index Postgres search and harden SQLite search/concurrency (C5, C6)
Postgres recomputed a tsvector over every article's full text on every search. A migration now adds a stored, GIN-indexed search_vector column kept current by a trigger (so no ingest/update path can forget it), and the backend reads that column. Untested here: no Postgres in the dev sandbox, so the SQL is reviewed but not executed — worth one run of the (skipped-on-SQLite) Postgres test module against a real instance. SQLite search built one bound parameter per subscribed feed, so enough feeds would exceed the variable limit, and the OperationalError was swallowed into "no results". It now scopes through a subquery on Subscription and logs failures. SQLite connections also get WAL mode, a 20s busy timeout, and IMMEDIATE transactions so gunicorn, Celery, and the FTS triggers stop tripping over each other. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
cb0ce5a336
commit
70b0d4f277
4 files changed
+104
-21
No files matched your search
@@ -1,27 +1,28 @@
|
|||||||
from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector
|
|
||||||
|
|
||||||
from apps.articles.models import Article
|
from apps.articles.models import Article
|
||||||
|
|
||||||
from .base import SearchBackend
|
from .base import SearchBackend
|
||||||
|
|
||||||
|
# Matches the stored search_vector column, GIN index, and maintaining
|
||||||
|
# trigger created by apps/search/migrations/0002_postgres_search_vector.py
|
||||||
|
# — the query reads that column directly instead of recomputing a
|
||||||
|
# tsvector over every article's full text on every search (C5).
|
||||||
|
_VECTOR = "articles_article.search_vector"
|
||||||
|
_QUERY = "plainto_tsquery('english', %s)"
|
||||||
|
|
||||||
|
|
||||||
class PostgresSearchBackend(SearchBackend):
|
class PostgresSearchBackend(SearchBackend):
|
||||||
"""Computes the search vector on the fly at query time rather than
|
|
||||||
storing/indexing it on Article — keeps the schema identical to the
|
|
||||||
SQLite backend. Revisit with a stored+GIN-indexed column only if an
|
|
||||||
instance's archive grows large enough to need it."""
|
|
||||||
|
|
||||||
def search(self, query: str, user, limit: int = 50):
|
def search(self, query: str, user, limit: int = 50):
|
||||||
if not query.strip():
|
if not query.strip():
|
||||||
return Article.objects.none()
|
return Article.objects.none()
|
||||||
|
|
||||||
vector = SearchVector("title", weight="A") + SearchVector("content_html", weight="B")
|
|
||||||
search_query = SearchQuery(query)
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
Article.objects.filter(feed__subscriptions__user=user)
|
Article.objects.filter(feed__subscriptions__user=user)
|
||||||
.select_related("feed")
|
.select_related("feed")
|
||||||
.annotate(search=vector, rank=SearchRank(vector, search_query))
|
.extra(
|
||||||
.filter(search=search_query)
|
select={"rank": f"ts_rank({_VECTOR}, {_QUERY})"},
|
||||||
|
select_params=[query],
|
||||||
|
where=[f"{_VECTOR} @@ {_QUERY}"],
|
||||||
|
params=[query],
|
||||||
|
)
|
||||||
.order_by("-rank", "-published_at")[:limit]
|
.order_by("-rank", "-published_at")[:limit]
|
||||||
)
|
)
|
||||||
@@ -1,12 +1,14 @@
|
|||||||
|
import logging
|
||||||
import sqlite3
|
import sqlite3
|
||||||
|
|
||||||
from django.db import connection
|
from django.db import connection
|
||||||
|
|
||||||
from apps.articles.models import Article
|
from apps.articles.models import Article
|
||||||
from apps.feeds.models import Subscription
|
|
||||||
|
|
||||||
from .base import SearchBackend
|
from .base import SearchBackend
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def _fts_phrase_query(raw_query: str) -> str:
|
def _fts_phrase_query(raw_query: str) -> str:
|
||||||
"""Wraps the whole user input as a single FTS5 phrase so arbitrary
|
"""Wraps the whole user input as a single FTS5 phrase so arbitrary
|
||||||
@@ -21,24 +23,28 @@ class SQLiteFTSBackend(SearchBackend):
|
|||||||
if not query.strip():
|
if not query.strip():
|
||||||
return []
|
return []
|
||||||
|
|
||||||
feed_ids = list(Subscription.objects.filter(user=user).values_list("feed_id", flat=True))
|
# Scoped through a subquery on Subscription rather than an IN
|
||||||
if not feed_ids:
|
# list built from every subscribed feed id (C6): a user with
|
||||||
return []
|
# thousands of feeds would otherwise exceed SQLite's bound-
|
||||||
|
# parameter limit and the whole search would fail.
|
||||||
placeholders = ",".join(["%s"] * len(feed_ids))
|
sql = """
|
||||||
sql = f"""
|
|
||||||
SELECT a.id
|
SELECT a.id
|
||||||
FROM articles_article a
|
FROM articles_article a
|
||||||
JOIN articles_fts f ON f.rowid = a.id
|
JOIN articles_fts f ON f.rowid = a.id
|
||||||
WHERE articles_fts MATCH %s AND a.feed_id IN ({placeholders})
|
WHERE articles_fts MATCH %s
|
||||||
|
AND a.feed_id IN (SELECT feed_id FROM feeds_subscription WHERE user_id = %s)
|
||||||
ORDER BY rank
|
ORDER BY rank
|
||||||
LIMIT %s
|
LIMIT %s
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
with connection.cursor() as cursor:
|
with connection.cursor() as cursor:
|
||||||
cursor.execute(sql, [_fts_phrase_query(query), *feed_ids, limit])
|
cursor.execute(sql, [_fts_phrase_query(query), user.pk, limit])
|
||||||
ordered_ids = [row[0] for row in cursor.fetchall()]
|
ordered_ids = [row[0] for row in cursor.fetchall()]
|
||||||
except sqlite3.OperationalError:
|
except sqlite3.OperationalError:
|
||||||
|
# A genuinely bad FTS query is rare (the input is quoted as a
|
||||||
|
# phrase), so this is far more likely a real operational
|
||||||
|
# failure — log it instead of silently showing "no results".
|
||||||
|
logger.exception("SQLite FTS search failed for user %s", user.pk)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
if not ordered_ids:
|
if not ordered_ids:
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
"""Stored, GIN-indexed tsvector for Postgres search (C5), kept current by a
|
||||||
|
trigger so no ingestion/update code path can forget to maintain it.
|
||||||
|
|
||||||
|
No-ops entirely on SQLite — its FTS5 table (0001_sqlite_fts) already plays
|
||||||
|
this role. The column is deliberately not a Django model field: a
|
||||||
|
SearchVectorField would break SQLite, and nothing but the search backend
|
||||||
|
ever reads it (apps/search/backends/postgres.py).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
VECTOR_EXPR = (
|
||||||
|
"setweight(to_tsvector('english', coalesce({row}title, '')), 'A') || "
|
||||||
|
"setweight(to_tsvector('english', coalesce({row}content_html, '')), 'B')"
|
||||||
|
)
|
||||||
|
|
||||||
|
CREATE = f"""
|
||||||
|
ALTER TABLE articles_article ADD COLUMN IF NOT EXISTS search_vector tsvector;
|
||||||
|
|
||||||
|
UPDATE articles_article SET search_vector = {VECTOR_EXPR.format(row="")};
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS articles_article_search_vector_gin
|
||||||
|
ON articles_article USING GIN (search_vector);
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION articles_article_search_vector_update() RETURNS trigger AS $$
|
||||||
|
BEGIN
|
||||||
|
NEW.search_vector := {VECTOR_EXPR.format(row="NEW.")};
|
||||||
|
RETURN NEW;
|
||||||
|
END
|
||||||
|
$$ LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
DROP TRIGGER IF EXISTS articles_article_search_vector_trg ON articles_article;
|
||||||
|
CREATE TRIGGER articles_article_search_vector_trg
|
||||||
|
BEFORE INSERT OR UPDATE OF title, content_html ON articles_article
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION articles_article_search_vector_update();
|
||||||
|
"""
|
||||||
|
|
||||||
|
DROP = """
|
||||||
|
DROP TRIGGER IF EXISTS articles_article_search_vector_trg ON articles_article;
|
||||||
|
DROP FUNCTION IF EXISTS articles_article_search_vector_update();
|
||||||
|
DROP INDEX IF EXISTS articles_article_search_vector_gin;
|
||||||
|
ALTER TABLE articles_article DROP COLUMN IF EXISTS search_vector;
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def create_vector(apps, schema_editor):
|
||||||
|
if schema_editor.connection.vendor != "postgresql":
|
||||||
|
return
|
||||||
|
schema_editor.execute(CREATE)
|
||||||
|
|
||||||
|
|
||||||
|
def drop_vector(apps, schema_editor):
|
||||||
|
if schema_editor.connection.vendor != "postgresql":
|
||||||
|
return
|
||||||
|
schema_editor.execute(DROP)
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
dependencies = [("search", "0001_sqlite_fts"), ("articles", "0001_initial")]
|
||||||
|
|
||||||
|
operations = [migrations.RunPython(create_vector, drop_vector)]
|
||||||
@@ -77,6 +77,21 @@ ASGI_APPLICATION = "config.asgi.application"
|
|||||||
# postgres://user:pass@host:5432/dbname
|
# postgres://user:pass@host:5432/dbname
|
||||||
DATABASES = {"default": env.db_url("DATABASE_URL", default=f"sqlite:///{BASE_DIR / 'db.sqlite3'}")}
|
DATABASES = {"default": env.db_url("DATABASE_URL", default=f"sqlite:///{BASE_DIR / 'db.sqlite3'}")}
|
||||||
|
|
||||||
|
# gunicorn, Celery workers, and the FTS triggers all write to the same
|
||||||
|
# SQLite file (C6): without WAL, readers and a writer block each other,
|
||||||
|
# and without a busy timeout a contended write fails immediately with
|
||||||
|
# "database is locked" instead of waiting its turn. IMMEDIATE takes the
|
||||||
|
# write lock at transaction start so a read-then-write transaction can't
|
||||||
|
# deadlock upgrading it mid-way.
|
||||||
|
if DATABASES["default"]["ENGINE"] == "django.db.backends.sqlite3":
|
||||||
|
DATABASES["default"].setdefault("OPTIONS", {}).update(
|
||||||
|
{
|
||||||
|
"timeout": 20,
|
||||||
|
"transaction_mode": "IMMEDIATE",
|
||||||
|
"init_command": "PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
AUTH_USER_MODEL = "accounts.User"
|
AUTH_USER_MODEL = "accounts.User"
|
||||||
|
|
||||||
AUTH_PASSWORD_VALIDATORS = [
|
AUTH_PASSWORD_VALIDATORS = [
|
||||||
|
|||||||
Reference in new issue
Block a user