Add full-text search with per-backend SQLite/Postgres implementations

Implements the backend-agnostic search interface from the plan,
selected at runtime by connection.vendor:

- SQLite: an FTS5 virtual table (external-content, mirroring
  Article.title/content_html) created via a vendor-conditional
  migration, kept in sync by insert/update/delete triggers. Queries
  wrap the whole user input as a single FTS5 phrase so punctuation in
  a search box can't break MATCH syntax. `reindex_search` management
  command rebuilds it after bulk imports.
- Postgres: computes the search vector on the fly per the plan's
  decision (title weighted above content, no stored/indexed column)
  rather than diverging the schema from SQLite.

Both backends scope results to the requesting user's own subscriptions
and share the stream/detail article-row template and read/star/save
actions via a new apps.articles.queries.with_user_state() helper,
extracted from apps.reader.views to avoid duplicating that annotation
logic for search results.

Verified live: the SQLite path end-to-end in-browser against the real
Hacker News subscription (search UI, phrase matching, no-match case),
and the Postgres path against a throwaway Postgres container (search,
feed-scoping, and title-vs-content ranking), torn down afterward — not
part of this project's own infrastructure.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5 committed 2026-10-30 16:37:22 -06:00
1 parent 5ed0499845
commit 8653dbeaf9
19 files changed
+569 -20

No files matched your search

+28
View File
@@ -0,0 +1,28 @@
from .models import UserArticleState
def with_user_state(articles, user):
"""Annotates an iterable of Article instances with is_read/is_starred/
is_saved_for_later booleans for the given user, in two queries total
regardless of how many articles there are (no N+1).
Accepts either a queryset or a plain list (search backends return
plain lists already ordered by relevance), materializing it either
way.
"""
articles = list(articles)
if not articles:
return articles
states = {
state.article_id: state
for state in UserArticleState.objects.filter(
user=user, article_id__in=[a.pk for a in articles]
)
}
for article in articles:
state = states.get(article.pk)
article.is_read = state.is_read if state else False
article.is_starred = state.is_starred if state else False
article.is_saved_for_later = state.is_saved_for_later if state else False
return articles
+2 -19
View File
@@ -1,11 +1,11 @@
from django.contrib.auth.decorators import login_required
from django.db.models import Prefetch
from django.http import Http404
from django.shortcuts import get_object_or_404, render
from django.utils import timezone
from django.views.decorators.http import require_POST
from apps.articles.models import Article, Tag, UserArticleState
from apps.articles.queries import with_user_state
FILTER_CHOICES = [
("unread", "Unread"),
@@ -27,23 +27,6 @@ def _base_queryset(user):
)
def _with_user_state_annotations(queryset, user):
queryset = queryset.prefetch_related(
Prefetch(
"user_states",
queryset=UserArticleState.objects.filter(user=user),
to_attr="my_states",
)
)
articles = list(queryset)
for article in articles:
state = article.my_states[0] if article.my_states else None
article.is_read = state.is_read if state else False
article.is_starred = state.is_starred if state else False
article.is_saved_for_later = state.is_saved_for_later if state else False
return articles
def _get_or_create_state(user, article):
state, _ = UserArticleState.objects.get_or_create(user=user, article=article)
return state
@@ -81,7 +64,7 @@ def stream(request):
start, end = (page - 1) * PAGE_SIZE, page * PAGE_SIZE
total_count = queryset.count()
articles = _with_user_state_annotations(queryset[start:end], request.user)
articles = with_user_state(queryset[start:end], request.user)
context = {
"articles": articles,
+16
View File
@@ -0,0 +1,16 @@
from django.db import connection
from .base import SearchBackend
__all__ = ["SearchBackend", "get_backend"]
def get_backend() -> SearchBackend:
if connection.vendor == "postgresql":
from .postgres import PostgresSearchBackend
return PostgresSearchBackend()
from .sqlite_fts import SQLiteFTSBackend
return SQLiteFTSBackend()
+9
View File
@@ -0,0 +1,9 @@
class SearchBackend:
"""Common interface so views never branch on DB vendor directly —
see apps/search/backends/__init__.py::get_backend()."""
def search(self, query: str, user, limit: int = 50):
"""Returns an iterable of Article instances the user can see
(scoped to their own subscriptions), matching ``query``, most
relevant first."""
raise NotImplementedError
+27
View File
@@ -0,0 +1,27 @@
from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector
from apps.articles.models import Article
from .base import SearchBackend
class PostgresSearchBackend(SearchBackend):
"""Computes the search vector on the fly at query time rather than
storing/indexing it on Article — keeps the schema identical to the
SQLite backend. Revisit with a stored+GIN-indexed column only if an
instance's archive grows large enough to need it."""
def search(self, query: str, user, limit: int = 50):
if not query.strip():
return Article.objects.none()
vector = SearchVector("title", weight="A") + SearchVector("content_html", weight="B")
search_query = SearchQuery(query)
return (
Article.objects.filter(feed__subscriptions__user=user)
.select_related("feed")
.annotate(search=vector, rank=SearchRank(vector, search_query))
.filter(search=search_query)
.order_by("-rank", "-published_at")[:limit]
)
+48
View File
@@ -0,0 +1,48 @@
import sqlite3
from django.db import connection
from apps.articles.models import Article
from apps.feeds.models import Subscription
from .base import SearchBackend
def _fts_phrase_query(raw_query: str) -> str:
"""Wraps the whole user input as a single FTS5 phrase so arbitrary
punctuation (hyphens, quotes, parentheses...) in a search box doesn't
blow up as invalid FTS5 query syntax."""
escaped = raw_query.replace('"', '""')
return f'"{escaped}"'
class SQLiteFTSBackend(SearchBackend):
def search(self, query: str, user, limit: int = 50):
if not query.strip():
return []
feed_ids = list(Subscription.objects.filter(user=user).values_list("feed_id", flat=True))
if not feed_ids:
return []
placeholders = ",".join(["%s"] * len(feed_ids))
sql = f"""
SELECT a.id
FROM articles_article a
JOIN articles_fts f ON f.rowid = a.id
WHERE articles_fts MATCH %s AND a.feed_id IN ({placeholders})
ORDER BY rank
LIMIT %s
"""
try:
with connection.cursor() as cursor:
cursor.execute(sql, [_fts_phrase_query(query), *feed_ids, limit])
ordered_ids = [row[0] for row in cursor.fetchall()]
except sqlite3.OperationalError:
return []
if not ordered_ids:
return []
articles_by_id = Article.objects.select_related("feed").in_bulk(ordered_ids)
return [articles_by_id[pk] for pk in ordered_ids if pk in articles_by_id]
View File
Whitespace-only changes.
Whitespace-only changes.
@@ -0,0 +1,20 @@
from django.core.management.base import BaseCommand
from django.db import connection
class Command(BaseCommand):
help = (
"Rebuild the SQLite FTS5 search index from scratch. No-op on Postgres, "
"which computes its search vector at query time instead of maintaining "
"a separate index."
)
def handle(self, *args, **options):
if connection.vendor != "sqlite":
self.stdout.write("Not using SQLite — nothing to rebuild.")
return
with connection.cursor() as cursor:
cursor.execute("INSERT INTO articles_fts(articles_fts) VALUES ('rebuild')")
self.stdout.write(self.style.SUCCESS("Rebuilt the SQLite FTS5 search index."))
+78
View File
@@ -0,0 +1,78 @@
"""Creates (and on reverse, drops) a SQLite FTS5 virtual table mirroring
apps_articles.Article(title, content_html), plus triggers that keep it in
sync on insert/update/delete.
No-ops entirely on Postgres — the Postgres backend computes its search
vector on the fly instead of maintaining a separate index table, see
apps/search/backends/postgres.py.
"""
from django.db import migrations
CREATE_FTS_TABLE = """
CREATE VIRTUAL TABLE IF NOT EXISTS articles_fts USING fts5(
title, content_html, content='articles_article', content_rowid='id'
);
"""
POPULATE_FTS_TABLE = """
INSERT INTO articles_fts(rowid, title, content_html)
SELECT id, title, content_html FROM articles_article;
"""
CREATE_INSERT_TRIGGER = """
CREATE TRIGGER IF NOT EXISTS articles_fts_ai AFTER INSERT ON articles_article BEGIN
INSERT INTO articles_fts(rowid, title, content_html)
VALUES (new.id, new.title, new.content_html);
END;
"""
CREATE_DELETE_TRIGGER = """
CREATE TRIGGER IF NOT EXISTS articles_fts_ad AFTER DELETE ON articles_article BEGIN
INSERT INTO articles_fts(articles_fts, rowid, title, content_html)
VALUES('delete', old.id, old.title, old.content_html);
END;
"""
CREATE_UPDATE_TRIGGER = """
CREATE TRIGGER IF NOT EXISTS articles_fts_au AFTER UPDATE ON articles_article BEGIN
INSERT INTO articles_fts(articles_fts, rowid, title, content_html)
VALUES('delete', old.id, old.title, old.content_html);
INSERT INTO articles_fts(rowid, title, content_html)
VALUES (new.id, new.title, new.content_html);
END;
"""
DROP_EVERYTHING = """
DROP TRIGGER IF EXISTS articles_fts_au;
DROP TRIGGER IF EXISTS articles_fts_ad;
DROP TRIGGER IF EXISTS articles_fts_ai;
DROP TABLE IF EXISTS articles_fts;
"""
def create_fts(apps, schema_editor):
if schema_editor.connection.vendor != "sqlite":
return
with schema_editor.connection.cursor() as cursor:
cursor.execute(CREATE_FTS_TABLE)
cursor.execute(POPULATE_FTS_TABLE)
cursor.execute(CREATE_INSERT_TRIGGER)
cursor.execute(CREATE_DELETE_TRIGGER)
cursor.execute(CREATE_UPDATE_TRIGGER)
def drop_fts(apps, schema_editor):
if schema_editor.connection.vendor != "sqlite":
return
with schema_editor.connection.cursor() as cursor:
for statement in DROP_EVERYTHING.strip().split(";"):
statement = statement.strip()
if statement:
cursor.execute(statement)
class Migration(migrations.Migration):
dependencies = [("articles", "0001_initial")]
operations = [migrations.RunPython(create_fts, drop_fts)]
+99
View File
@@ -0,0 +1,99 @@
import pytest
from django.contrib.auth import get_user_model
from django.db import connection
from apps.articles.models import Article
from apps.feeds.models import Feed, Subscription
from .backends.postgres import PostgresSearchBackend
User = get_user_model()
pytestmark = pytest.mark.skipif(
connection.vendor != "postgresql",
reason="Postgres-only backend; run with DATABASE_URL pointed at a Postgres instance.",
)
@pytest.fixture
def alice(db):
return User.objects.create_user("alice", password="pw-alice-123")
@pytest.fixture
def bob(db):
return User.objects.create_user("bob", password="pw-bob-123")
@pytest.fixture
def feed(db):
return Feed.objects.create(feed_url="https://example.com/feed.xml", title="Example Feed")
@pytest.mark.django_db
def test_search_finds_article_by_title(alice, feed):
Subscription.objects.create(user=alice, feed=feed)
article = Article.objects.create(feed=feed, guid="g1", title="Django 5 release notes")
Article.objects.create(feed=feed, guid="g2", title="Completely unrelated")
results = list(PostgresSearchBackend().search("Django", alice))
assert [a.pk for a in results] == [article.pk]
@pytest.mark.django_db
def test_search_finds_article_by_content(alice, feed):
Subscription.objects.create(user=alice, feed=feed)
article = Article.objects.create(
feed=feed,
guid="g1",
title="Weekly update",
content_html="<p>We shipped WireGuard support.</p>",
)
results = list(PostgresSearchBackend().search("WireGuard", alice))
assert [a.pk for a in results] == [article.pk]
@pytest.mark.django_db
def test_search_only_returns_subscribed_feeds(alice, bob, feed):
other_feed = Feed.objects.create(feed_url="https://other.example.com/feed.xml")
Subscription.objects.create(user=alice, feed=feed)
Subscription.objects.create(user=bob, feed=other_feed)
Article.objects.create(feed=feed, guid="g1", title="Django in feed A")
article_b = Article.objects.create(feed=other_feed, guid="g2", title="Django in feed B")
alice_results = list(PostgresSearchBackend().search("Django", alice))
bob_results = list(PostgresSearchBackend().search("Django", bob))
assert len(alice_results) == 1
assert [a.pk for a in bob_results] == [article_b.pk]
@pytest.mark.django_db
def test_title_matches_rank_above_content_only_matches(alice, feed):
Subscription.objects.create(user=alice, feed=feed)
title_match = Article.objects.create(feed=feed, guid="g1", title="Django release")
content_match = Article.objects.create(
feed=feed, guid="g2", title="Something else", content_html="<p>mentions Django</p>"
)
results = list(PostgresSearchBackend().search("Django", alice))
assert [a.pk for a in results] == [title_match.pk, content_match.pk]
@pytest.mark.django_db
def test_no_matches_returns_empty(alice, feed):
Subscription.objects.create(user=alice, feed=feed)
Article.objects.create(feed=feed, guid="g1", title="Something else entirely")
assert list(PostgresSearchBackend().search("nonexistentword", alice)) == []
@pytest.mark.django_db
def test_empty_query_returns_empty(alice, feed):
Subscription.objects.create(user=alice, feed=feed)
assert list(PostgresSearchBackend().search("", alice)) == []
+127
View File
@@ -0,0 +1,127 @@
import pytest
from django.contrib.auth import get_user_model
from django.db import connection
from apps.articles.models import Article
from apps.feeds.models import Feed, Subscription
from .backends.sqlite_fts import SQLiteFTSBackend
User = get_user_model()
pytestmark = pytest.mark.skipif(
connection.vendor != "sqlite",
reason="SQLite-only backend; the articles_fts virtual table only exists on SQLite.",
)
@pytest.fixture
def alice(db):
return User.objects.create_user("alice", password="pw-alice-123")
@pytest.fixture
def bob(db):
return User.objects.create_user("bob", password="pw-bob-123")
@pytest.fixture
def feed(db):
return Feed.objects.create(feed_url="https://example.com/feed.xml", title="Example Feed")
@pytest.mark.django_db
def test_search_finds_article_by_title(alice, feed):
Subscription.objects.create(user=alice, feed=feed)
article = Article.objects.create(feed=feed, guid="g1", title="Django 5 release notes")
Article.objects.create(feed=feed, guid="g2", title="Completely unrelated")
results = SQLiteFTSBackend().search("Django", alice)
assert [a.pk for a in results] == [article.pk]
@pytest.mark.django_db
def test_search_finds_article_by_content(alice, feed):
Subscription.objects.create(user=alice, feed=feed)
article = Article.objects.create(
feed=feed,
guid="g1",
title="Weekly update",
content_html="<p>We shipped WireGuard support.</p>",
)
results = SQLiteFTSBackend().search("WireGuard", alice)
assert [a.pk for a in results] == [article.pk]
@pytest.mark.django_db
def test_search_only_returns_subscribed_feeds(alice, bob, feed):
other_feed = Feed.objects.create(feed_url="https://other.example.com/feed.xml")
Subscription.objects.create(user=alice, feed=feed)
Subscription.objects.create(user=bob, feed=other_feed)
Article.objects.create(feed=feed, guid="g1", title="Django in feed A")
article_b = Article.objects.create(feed=other_feed, guid="g2", title="Django in feed B")
alice_results = SQLiteFTSBackend().search("Django", alice)
bob_results = SQLiteFTSBackend().search("Django", bob)
assert len(alice_results) == 1
assert [a.pk for a in bob_results] == [article_b.pk]
@pytest.mark.django_db
def test_search_with_no_matches_returns_empty(alice, feed):
Subscription.objects.create(user=alice, feed=feed)
Article.objects.create(feed=feed, guid="g1", title="Something else entirely")
assert SQLiteFTSBackend().search("nonexistentword", alice) == []
@pytest.mark.django_db
def test_search_handles_special_characters_without_crashing(alice, feed):
Subscription.objects.create(user=alice, feed=feed)
Article.objects.create(feed=feed, guid="g1", title="C++ vs Rust: a comparison (2024)")
# Characters that are meaningful in raw FTS5 query syntax shouldn't
# raise — the backend wraps the whole query as a phrase.
results = SQLiteFTSBackend().search("C++ vs Rust: a comparison (2024)", alice)
assert len(results) == 1
@pytest.mark.django_db
def test_search_index_updates_on_article_edit(alice, feed):
Subscription.objects.create(user=alice, feed=feed)
article = Article.objects.create(feed=feed, guid="g1", title="Old title")
assert SQLiteFTSBackend().search("New", alice) == []
article.title = "New title"
article.save()
results = SQLiteFTSBackend().search("New", alice)
assert [a.pk for a in results] == [article.pk]
@pytest.mark.django_db
def test_search_index_updates_on_article_delete(alice, feed):
Subscription.objects.create(user=alice, feed=feed)
article = Article.objects.create(feed=feed, guid="g1", title="Removable article")
assert len(SQLiteFTSBackend().search("Removable", alice)) == 1
article.delete()
assert SQLiteFTSBackend().search("Removable", alice) == []
@pytest.mark.django_db
def test_user_with_no_subscriptions_gets_empty_results(alice):
assert SQLiteFTSBackend().search("anything", alice) == []
@pytest.mark.django_db
def test_empty_query_returns_empty(alice, feed):
Subscription.objects.create(user=alice, feed=feed)
assert SQLiteFTSBackend().search("", alice) == []
assert SQLiteFTSBackend().search(" ", alice) == []
+57
View File
@@ -0,0 +1,57 @@
import pytest
from django.contrib.auth import get_user_model
from django.urls import reverse
from apps.articles.models import Article, UserArticleState
from apps.feeds.models import Feed, Subscription
User = get_user_model()
@pytest.fixture
def alice(db):
return User.objects.create_user("alice", password="pw-alice-123")
@pytest.fixture
def feed(db):
return Feed.objects.create(feed_url="https://example.com/feed.xml", title="Example Feed")
@pytest.mark.django_db
def test_search_requires_login(client):
response = client.get(reverse("search:search"), {"q": "django"})
assert response.status_code == 302
@pytest.mark.django_db
def test_search_view_renders_matching_articles(client, alice, feed):
Subscription.objects.create(user=alice, feed=feed)
article = Article.objects.create(feed=feed, guid="g1", title="Django 5 release notes")
client.login(username="alice", password="pw-alice-123")
response = client.get(reverse("search:search"), {"q": "Django"})
assert response.status_code == 200
assert article.title in response.content.decode()
@pytest.mark.django_db
def test_search_view_with_no_query_shows_no_results_block(client, alice):
client.login(username="alice", password="pw-alice-123")
response = client.get(reverse("search:search"))
assert response.status_code == 200
assert b"article-list" not in response.content
@pytest.mark.django_db
def test_search_results_carry_user_state(client, alice, feed):
Subscription.objects.create(user=alice, feed=feed)
article = Article.objects.create(feed=feed, guid="g1", title="Django 5 release notes")
UserArticleState.objects.create(user=alice, article=article, is_starred=True)
client.login(username="alice", password="pw-alice-123")
response = client.get(reverse("search:search"), {"q": "Django"})
content = response.content.decode()
assert 'class="icon-btn active"' in content
+9
View File
@@ -0,0 +1,9 @@
from django.urls import path
from . import views
app_name = "search"
urlpatterns = [
path("search/", views.search, name="search"),
]
+16 -1
View File
@@ -1 +1,16 @@
# Create your views here.
from django.contrib.auth.decorators import login_required
from django.shortcuts import render
from apps.articles.queries import with_user_state
from .backends import get_backend
@login_required
def search(request):
query = request.GET.get("q", "").strip()
results = []
if query:
backend = get_backend()
results = with_user_state(backend.search(query, request.user), request.user)
return render(request, "search/results.html", {"query": query, "results": results})
+1
View File
@@ -7,5 +7,6 @@ urlpatterns = [
path("", include("apps.feeds.urls")),
path("", include("apps.articles.urls")),
path("", include("apps.rules.urls")),
path("", include("apps.search.urls")),
path("", include("apps.reader.urls")),
]
+10
View File
@@ -239,6 +239,16 @@ input, button {
gap: 0.35rem;
}
.search-form {
display: flex;
gap: 0.5rem;
margin: 1rem 0;
}
.search-form input {
flex: 1;
}
.article-content {
line-height: 1.6;
margin-top: 1rem;
+1
View File
@@ -7,6 +7,7 @@
<a href="{% url 'feeds:add_feed' %}">Add feed</a>
<a href="{% url 'articles:tag_list' %}">Tags</a>
<a href="{% url 'rules:rule_list' %}">Rules</a>
<a href="{% url 'search:search' %}">Search</a>
<a href="{% url 'accounts:logout' %}">Log out</a>
</nav>
</header>
+21
View File
@@ -0,0 +1,21 @@
{% extends "base.html" %}
{% block title %}Search — Evection{% endblock %}
{% block content %}
<p><a href="{% url 'reader:stream' %}">&larr; Back to stream</a></p>
<h1>Search</h1>
<form method="get" class="search-form">
<input type="search" name="q" value="{{ query }}" placeholder="Search your subscriptions" autofocus>
<button type="submit">Search</button>
</form>
{% if query %}
<ul class="article-list">
{% for article in results %}
{% include "reader/_article_row.html" %}
{% empty %}
<li class="empty-state">No results for "{{ query }}".</li>
{% endfor %}
</ul>
{% endif %}
{% endblock %}