diff --git a/apps/search/backends/postgres.py b/apps/search/backends/postgres.py index 4c50a22..11ffd48 100644 --- a/apps/search/backends/postgres.py +++ b/apps/search/backends/postgres.py @@ -1,27 +1,28 @@ -from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector - from apps.articles.models import Article from .base import SearchBackend +# Matches the stored search_vector column, GIN index, and maintaining +# trigger created by apps/search/migrations/0002_postgres_search_vector.py +# — the query reads that column directly instead of recomputing a +# tsvector over every article's full text on every search (C5). +_VECTOR = "articles_article.search_vector" +_QUERY = "plainto_tsquery('english', %s)" + class PostgresSearchBackend(SearchBackend): - """Computes the search vector on the fly at query time rather than - storing/indexing it on Article — keeps the schema identical to the - SQLite backend. Revisit with a stored+GIN-indexed column only if an - instance's archive grows large enough to need it.""" - def search(self, query: str, user, limit: int = 50): if not query.strip(): return Article.objects.none() - vector = SearchVector("title", weight="A") + SearchVector("content_html", weight="B") - search_query = SearchQuery(query) - return ( Article.objects.filter(feed__subscriptions__user=user) .select_related("feed") - .annotate(search=vector, rank=SearchRank(vector, search_query)) - .filter(search=search_query) + .extra( + select={"rank": f"ts_rank({_VECTOR}, {_QUERY})"}, + select_params=[query], + where=[f"{_VECTOR} @@ {_QUERY}"], + params=[query], + ) .order_by("-rank", "-published_at")[:limit] ) diff --git a/apps/search/backends/sqlite_fts.py b/apps/search/backends/sqlite_fts.py index 8ccdb4b..842c35f 100644 --- a/apps/search/backends/sqlite_fts.py +++ b/apps/search/backends/sqlite_fts.py @@ -1,12 +1,14 @@ +import logging import sqlite3 from django.db import connection from apps.articles.models import Article -from apps.feeds.models import Subscription from .base import SearchBackend +logger = logging.getLogger(__name__) + def _fts_phrase_query(raw_query: str) -> str: """Wraps the whole user input as a single FTS5 phrase so arbitrary @@ -21,24 +23,28 @@ class SQLiteFTSBackend(SearchBackend): if not query.strip(): return [] - feed_ids = list(Subscription.objects.filter(user=user).values_list("feed_id", flat=True)) - if not feed_ids: - return [] - - placeholders = ",".join(["%s"] * len(feed_ids)) - sql = f""" + # Scoped through a subquery on Subscription rather than an IN + # list built from every subscribed feed id (C6): a user with + # thousands of feeds would otherwise exceed SQLite's bound- + # parameter limit and the whole search would fail. + sql = """ SELECT a.id FROM articles_article a JOIN articles_fts f ON f.rowid = a.id - WHERE articles_fts MATCH %s AND a.feed_id IN ({placeholders}) + WHERE articles_fts MATCH %s + AND a.feed_id IN (SELECT feed_id FROM feeds_subscription WHERE user_id = %s) ORDER BY rank LIMIT %s """ try: with connection.cursor() as cursor: - cursor.execute(sql, [_fts_phrase_query(query), *feed_ids, limit]) + cursor.execute(sql, [_fts_phrase_query(query), user.pk, limit]) ordered_ids = [row[0] for row in cursor.fetchall()] except sqlite3.OperationalError: + # A genuinely bad FTS query is rare (the input is quoted as a + # phrase), so this is far more likely a real operational + # failure — log it instead of silently showing "no results". + logger.exception("SQLite FTS search failed for user %s", user.pk) return [] if not ordered_ids: diff --git a/apps/search/migrations/0002_postgres_search_vector.py b/apps/search/migrations/0002_postgres_search_vector.py new file mode 100644 index 0000000..ec842ab --- /dev/null +++ b/apps/search/migrations/0002_postgres_search_vector.py @@ -0,0 +1,61 @@ +"""Stored, GIN-indexed tsvector for Postgres search (C5), kept current by a +trigger so no ingestion/update code path can forget to maintain it. + +No-ops entirely on SQLite — its FTS5 table (0001_sqlite_fts) already plays +this role. The column is deliberately not a Django model field: a +SearchVectorField would break SQLite, and nothing but the search backend +ever reads it (apps/search/backends/postgres.py). +""" + +from django.db import migrations + +VECTOR_EXPR = ( + "setweight(to_tsvector('english', coalesce({row}title, '')), 'A') || " + "setweight(to_tsvector('english', coalesce({row}content_html, '')), 'B')" +) + +CREATE = f""" +ALTER TABLE articles_article ADD COLUMN IF NOT EXISTS search_vector tsvector; + +UPDATE articles_article SET search_vector = {VECTOR_EXPR.format(row="")}; + +CREATE INDEX IF NOT EXISTS articles_article_search_vector_gin + ON articles_article USING GIN (search_vector); + +CREATE OR REPLACE FUNCTION articles_article_search_vector_update() RETURNS trigger AS $$ +BEGIN + NEW.search_vector := {VECTOR_EXPR.format(row="NEW.")}; + RETURN NEW; +END +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS articles_article_search_vector_trg ON articles_article; +CREATE TRIGGER articles_article_search_vector_trg + BEFORE INSERT OR UPDATE OF title, content_html ON articles_article + FOR EACH ROW EXECUTE FUNCTION articles_article_search_vector_update(); +""" + +DROP = """ +DROP TRIGGER IF EXISTS articles_article_search_vector_trg ON articles_article; +DROP FUNCTION IF EXISTS articles_article_search_vector_update(); +DROP INDEX IF EXISTS articles_article_search_vector_gin; +ALTER TABLE articles_article DROP COLUMN IF EXISTS search_vector; +""" + + +def create_vector(apps, schema_editor): + if schema_editor.connection.vendor != "postgresql": + return + schema_editor.execute(CREATE) + + +def drop_vector(apps, schema_editor): + if schema_editor.connection.vendor != "postgresql": + return + schema_editor.execute(DROP) + + +class Migration(migrations.Migration): + dependencies = [("search", "0001_sqlite_fts"), ("articles", "0001_initial")] + + operations = [migrations.RunPython(create_vector, drop_vector)] diff --git a/config/settings/base.py b/config/settings/base.py index b2d9cbe..7fe2f8b 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -77,6 +77,21 @@ ASGI_APPLICATION = "config.asgi.application" # postgres://user:pass@host:5432/dbname DATABASES = {"default": env.db_url("DATABASE_URL", default=f"sqlite:///{BASE_DIR / 'db.sqlite3'}")} +# gunicorn, Celery workers, and the FTS triggers all write to the same +# SQLite file (C6): without WAL, readers and a writer block each other, +# and without a busy timeout a contended write fails immediately with +# "database is locked" instead of waiting its turn. IMMEDIATE takes the +# write lock at transaction start so a read-then-write transaction can't +# deadlock upgrading it mid-way. +if DATABASES["default"]["ENGINE"] == "django.db.backends.sqlite3": + DATABASES["default"].setdefault("OPTIONS", {}).update( + { + "timeout": 20, + "transaction_mode": "IMMEDIATE", + "init_command": "PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;", + } + ) + AUTH_USER_MODEL = "accounts.User" AUTH_PASSWORD_VALIDATORS = [