Private
Public Access
Add full-text search with per-backend SQLite/Postgres implementations
Implements the backend-agnostic search interface from the plan, selected at runtime by connection.vendor: - SQLite: an FTS5 virtual table (external-content, mirroring Article.title/content_html) created via a vendor-conditional migration, kept in sync by insert/update/delete triggers. Queries wrap the whole user input as a single FTS5 phrase so punctuation in a search box can't break MATCH syntax. `reindex_search` management command rebuilds it after bulk imports. - Postgres: computes the search vector on the fly per the plan's decision (title weighted above content, no stored/indexed column) rather than diverging the schema from SQLite. Both backends scope results to the requesting user's own subscriptions and share the stream/detail article-row template and read/star/save actions via a new apps.articles.queries.with_user_state() helper, extracted from apps.reader.views to avoid duplicating that annotation logic for search results. Verified live: the SQLite path end-to-end in-browser against the real Hacker News subscription (search UI, phrase matching, no-match case), and the Postgres path against a throwaway Postgres container (search, feed-scoping, and title-vs-content ranking), torn down afterward — not part of this project's own infrastructure. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
5ed0499845
commit
8653dbeaf9
19 files changed
+569
-20
No files matched your search
@@ -0,0 +1,16 @@
|
||||
from django.db import connection
|
||||
|
||||
from .base import SearchBackend
|
||||
|
||||
__all__ = ["SearchBackend", "get_backend"]
|
||||
|
||||
|
||||
def get_backend() -> SearchBackend:
|
||||
if connection.vendor == "postgresql":
|
||||
from .postgres import PostgresSearchBackend
|
||||
|
||||
return PostgresSearchBackend()
|
||||
|
||||
from .sqlite_fts import SQLiteFTSBackend
|
||||
|
||||
return SQLiteFTSBackend()
|
||||
@@ -0,0 +1,9 @@
|
||||
class SearchBackend:
|
||||
"""Common interface so views never branch on DB vendor directly —
|
||||
see apps/search/backends/__init__.py::get_backend()."""
|
||||
|
||||
def search(self, query: str, user, limit: int = 50):
|
||||
"""Returns an iterable of Article instances the user can see
|
||||
(scoped to their own subscriptions), matching ``query``, most
|
||||
relevant first."""
|
||||
raise NotImplementedError
|
||||
@@ -0,0 +1,27 @@
|
||||
from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector
|
||||
|
||||
from apps.articles.models import Article
|
||||
|
||||
from .base import SearchBackend
|
||||
|
||||
|
||||
class PostgresSearchBackend(SearchBackend):
|
||||
"""Computes the search vector on the fly at query time rather than
|
||||
storing/indexing it on Article — keeps the schema identical to the
|
||||
SQLite backend. Revisit with a stored+GIN-indexed column only if an
|
||||
instance's archive grows large enough to need it."""
|
||||
|
||||
def search(self, query: str, user, limit: int = 50):
|
||||
if not query.strip():
|
||||
return Article.objects.none()
|
||||
|
||||
vector = SearchVector("title", weight="A") + SearchVector("content_html", weight="B")
|
||||
search_query = SearchQuery(query)
|
||||
|
||||
return (
|
||||
Article.objects.filter(feed__subscriptions__user=user)
|
||||
.select_related("feed")
|
||||
.annotate(search=vector, rank=SearchRank(vector, search_query))
|
||||
.filter(search=search_query)
|
||||
.order_by("-rank", "-published_at")[:limit]
|
||||
)
|
||||
@@ -0,0 +1,48 @@
|
||||
import sqlite3
|
||||
|
||||
from django.db import connection
|
||||
|
||||
from apps.articles.models import Article
|
||||
from apps.feeds.models import Subscription
|
||||
|
||||
from .base import SearchBackend
|
||||
|
||||
|
||||
def _fts_phrase_query(raw_query: str) -> str:
|
||||
"""Wraps the whole user input as a single FTS5 phrase so arbitrary
|
||||
punctuation (hyphens, quotes, parentheses...) in a search box doesn't
|
||||
blow up as invalid FTS5 query syntax."""
|
||||
escaped = raw_query.replace('"', '""')
|
||||
return f'"{escaped}"'
|
||||
|
||||
|
||||
class SQLiteFTSBackend(SearchBackend):
|
||||
def search(self, query: str, user, limit: int = 50):
|
||||
if not query.strip():
|
||||
return []
|
||||
|
||||
feed_ids = list(Subscription.objects.filter(user=user).values_list("feed_id", flat=True))
|
||||
if not feed_ids:
|
||||
return []
|
||||
|
||||
placeholders = ",".join(["%s"] * len(feed_ids))
|
||||
sql = f"""
|
||||
SELECT a.id
|
||||
FROM articles_article a
|
||||
JOIN articles_fts f ON f.rowid = a.id
|
||||
WHERE articles_fts MATCH %s AND a.feed_id IN ({placeholders})
|
||||
ORDER BY rank
|
||||
LIMIT %s
|
||||
"""
|
||||
try:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(sql, [_fts_phrase_query(query), *feed_ids, limit])
|
||||
ordered_ids = [row[0] for row in cursor.fetchall()]
|
||||
except sqlite3.OperationalError:
|
||||
return []
|
||||
|
||||
if not ordered_ids:
|
||||
return []
|
||||
|
||||
articles_by_id = Article.objects.select_related("feed").in_bulk(ordered_ids)
|
||||
return [articles_by_id[pk] for pk in ordered_ids if pk in articles_by_id]
|
||||
Whitespace-only changes.
Whitespace-only changes.
@@ -0,0 +1,20 @@
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db import connection
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = (
|
||||
"Rebuild the SQLite FTS5 search index from scratch. No-op on Postgres, "
|
||||
"which computes its search vector at query time instead of maintaining "
|
||||
"a separate index."
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if connection.vendor != "sqlite":
|
||||
self.stdout.write("Not using SQLite — nothing to rebuild.")
|
||||
return
|
||||
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("INSERT INTO articles_fts(articles_fts) VALUES ('rebuild')")
|
||||
|
||||
self.stdout.write(self.style.SUCCESS("Rebuilt the SQLite FTS5 search index."))
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Creates (and on reverse, drops) a SQLite FTS5 virtual table mirroring
|
||||
apps_articles.Article(title, content_html), plus triggers that keep it in
|
||||
sync on insert/update/delete.
|
||||
|
||||
No-ops entirely on Postgres — the Postgres backend computes its search
|
||||
vector on the fly instead of maintaining a separate index table, see
|
||||
apps/search/backends/postgres.py.
|
||||
"""
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
CREATE_FTS_TABLE = """
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS articles_fts USING fts5(
|
||||
title, content_html, content='articles_article', content_rowid='id'
|
||||
);
|
||||
"""
|
||||
|
||||
POPULATE_FTS_TABLE = """
|
||||
INSERT INTO articles_fts(rowid, title, content_html)
|
||||
SELECT id, title, content_html FROM articles_article;
|
||||
"""
|
||||
|
||||
CREATE_INSERT_TRIGGER = """
|
||||
CREATE TRIGGER IF NOT EXISTS articles_fts_ai AFTER INSERT ON articles_article BEGIN
|
||||
INSERT INTO articles_fts(rowid, title, content_html)
|
||||
VALUES (new.id, new.title, new.content_html);
|
||||
END;
|
||||
"""
|
||||
|
||||
CREATE_DELETE_TRIGGER = """
|
||||
CREATE TRIGGER IF NOT EXISTS articles_fts_ad AFTER DELETE ON articles_article BEGIN
|
||||
INSERT INTO articles_fts(articles_fts, rowid, title, content_html)
|
||||
VALUES('delete', old.id, old.title, old.content_html);
|
||||
END;
|
||||
"""
|
||||
|
||||
CREATE_UPDATE_TRIGGER = """
|
||||
CREATE TRIGGER IF NOT EXISTS articles_fts_au AFTER UPDATE ON articles_article BEGIN
|
||||
INSERT INTO articles_fts(articles_fts, rowid, title, content_html)
|
||||
VALUES('delete', old.id, old.title, old.content_html);
|
||||
INSERT INTO articles_fts(rowid, title, content_html)
|
||||
VALUES (new.id, new.title, new.content_html);
|
||||
END;
|
||||
"""
|
||||
|
||||
DROP_EVERYTHING = """
|
||||
DROP TRIGGER IF EXISTS articles_fts_au;
|
||||
DROP TRIGGER IF EXISTS articles_fts_ad;
|
||||
DROP TRIGGER IF EXISTS articles_fts_ai;
|
||||
DROP TABLE IF EXISTS articles_fts;
|
||||
"""
|
||||
|
||||
|
||||
def create_fts(apps, schema_editor):
|
||||
if schema_editor.connection.vendor != "sqlite":
|
||||
return
|
||||
with schema_editor.connection.cursor() as cursor:
|
||||
cursor.execute(CREATE_FTS_TABLE)
|
||||
cursor.execute(POPULATE_FTS_TABLE)
|
||||
cursor.execute(CREATE_INSERT_TRIGGER)
|
||||
cursor.execute(CREATE_DELETE_TRIGGER)
|
||||
cursor.execute(CREATE_UPDATE_TRIGGER)
|
||||
|
||||
|
||||
def drop_fts(apps, schema_editor):
|
||||
if schema_editor.connection.vendor != "sqlite":
|
||||
return
|
||||
with schema_editor.connection.cursor() as cursor:
|
||||
for statement in DROP_EVERYTHING.strip().split(";"):
|
||||
statement = statement.strip()
|
||||
if statement:
|
||||
cursor.execute(statement)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [("articles", "0001_initial")]
|
||||
|
||||
operations = [migrations.RunPython(create_fts, drop_fts)]
|
||||
@@ -0,0 +1,99 @@
|
||||
import pytest
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.db import connection
|
||||
|
||||
from apps.articles.models import Article
|
||||
from apps.feeds.models import Feed, Subscription
|
||||
|
||||
from .backends.postgres import PostgresSearchBackend
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
connection.vendor != "postgresql",
|
||||
reason="Postgres-only backend; run with DATABASE_URL pointed at a Postgres instance.",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def alice(db):
|
||||
return User.objects.create_user("alice", password="pw-alice-123")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bob(db):
|
||||
return User.objects.create_user("bob", password="pw-bob-123")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def feed(db):
|
||||
return Feed.objects.create(feed_url="https://example.com/feed.xml", title="Example Feed")
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_search_finds_article_by_title(alice, feed):
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
article = Article.objects.create(feed=feed, guid="g1", title="Django 5 release notes")
|
||||
Article.objects.create(feed=feed, guid="g2", title="Completely unrelated")
|
||||
|
||||
results = list(PostgresSearchBackend().search("Django", alice))
|
||||
|
||||
assert [a.pk for a in results] == [article.pk]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_search_finds_article_by_content(alice, feed):
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
article = Article.objects.create(
|
||||
feed=feed,
|
||||
guid="g1",
|
||||
title="Weekly update",
|
||||
content_html="<p>We shipped WireGuard support.</p>",
|
||||
)
|
||||
|
||||
results = list(PostgresSearchBackend().search("WireGuard", alice))
|
||||
|
||||
assert [a.pk for a in results] == [article.pk]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_search_only_returns_subscribed_feeds(alice, bob, feed):
|
||||
other_feed = Feed.objects.create(feed_url="https://other.example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
Subscription.objects.create(user=bob, feed=other_feed)
|
||||
|
||||
Article.objects.create(feed=feed, guid="g1", title="Django in feed A")
|
||||
article_b = Article.objects.create(feed=other_feed, guid="g2", title="Django in feed B")
|
||||
|
||||
alice_results = list(PostgresSearchBackend().search("Django", alice))
|
||||
bob_results = list(PostgresSearchBackend().search("Django", bob))
|
||||
|
||||
assert len(alice_results) == 1
|
||||
assert [a.pk for a in bob_results] == [article_b.pk]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_title_matches_rank_above_content_only_matches(alice, feed):
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
title_match = Article.objects.create(feed=feed, guid="g1", title="Django release")
|
||||
content_match = Article.objects.create(
|
||||
feed=feed, guid="g2", title="Something else", content_html="<p>mentions Django</p>"
|
||||
)
|
||||
|
||||
results = list(PostgresSearchBackend().search("Django", alice))
|
||||
|
||||
assert [a.pk for a in results] == [title_match.pk, content_match.pk]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_no_matches_returns_empty(alice, feed):
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
Article.objects.create(feed=feed, guid="g1", title="Something else entirely")
|
||||
|
||||
assert list(PostgresSearchBackend().search("nonexistentword", alice)) == []
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_empty_query_returns_empty(alice, feed):
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
assert list(PostgresSearchBackend().search("", alice)) == []
|
||||
@@ -0,0 +1,127 @@
|
||||
import pytest
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.db import connection
|
||||
|
||||
from apps.articles.models import Article
|
||||
from apps.feeds.models import Feed, Subscription
|
||||
|
||||
from .backends.sqlite_fts import SQLiteFTSBackend
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
connection.vendor != "sqlite",
|
||||
reason="SQLite-only backend; the articles_fts virtual table only exists on SQLite.",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def alice(db):
|
||||
return User.objects.create_user("alice", password="pw-alice-123")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bob(db):
|
||||
return User.objects.create_user("bob", password="pw-bob-123")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def feed(db):
|
||||
return Feed.objects.create(feed_url="https://example.com/feed.xml", title="Example Feed")
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_search_finds_article_by_title(alice, feed):
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
article = Article.objects.create(feed=feed, guid="g1", title="Django 5 release notes")
|
||||
Article.objects.create(feed=feed, guid="g2", title="Completely unrelated")
|
||||
|
||||
results = SQLiteFTSBackend().search("Django", alice)
|
||||
|
||||
assert [a.pk for a in results] == [article.pk]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_search_finds_article_by_content(alice, feed):
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
article = Article.objects.create(
|
||||
feed=feed,
|
||||
guid="g1",
|
||||
title="Weekly update",
|
||||
content_html="<p>We shipped WireGuard support.</p>",
|
||||
)
|
||||
|
||||
results = SQLiteFTSBackend().search("WireGuard", alice)
|
||||
|
||||
assert [a.pk for a in results] == [article.pk]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_search_only_returns_subscribed_feeds(alice, bob, feed):
|
||||
other_feed = Feed.objects.create(feed_url="https://other.example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
Subscription.objects.create(user=bob, feed=other_feed)
|
||||
|
||||
Article.objects.create(feed=feed, guid="g1", title="Django in feed A")
|
||||
article_b = Article.objects.create(feed=other_feed, guid="g2", title="Django in feed B")
|
||||
|
||||
alice_results = SQLiteFTSBackend().search("Django", alice)
|
||||
bob_results = SQLiteFTSBackend().search("Django", bob)
|
||||
|
||||
assert len(alice_results) == 1
|
||||
assert [a.pk for a in bob_results] == [article_b.pk]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_search_with_no_matches_returns_empty(alice, feed):
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
Article.objects.create(feed=feed, guid="g1", title="Something else entirely")
|
||||
|
||||
assert SQLiteFTSBackend().search("nonexistentword", alice) == []
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_search_handles_special_characters_without_crashing(alice, feed):
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
Article.objects.create(feed=feed, guid="g1", title="C++ vs Rust: a comparison (2024)")
|
||||
|
||||
# Characters that are meaningful in raw FTS5 query syntax shouldn't
|
||||
# raise — the backend wraps the whole query as a phrase.
|
||||
results = SQLiteFTSBackend().search("C++ vs Rust: a comparison (2024)", alice)
|
||||
assert len(results) == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_search_index_updates_on_article_edit(alice, feed):
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
article = Article.objects.create(feed=feed, guid="g1", title="Old title")
|
||||
assert SQLiteFTSBackend().search("New", alice) == []
|
||||
|
||||
article.title = "New title"
|
||||
article.save()
|
||||
|
||||
results = SQLiteFTSBackend().search("New", alice)
|
||||
assert [a.pk for a in results] == [article.pk]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_search_index_updates_on_article_delete(alice, feed):
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
article = Article.objects.create(feed=feed, guid="g1", title="Removable article")
|
||||
assert len(SQLiteFTSBackend().search("Removable", alice)) == 1
|
||||
|
||||
article.delete()
|
||||
|
||||
assert SQLiteFTSBackend().search("Removable", alice) == []
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_user_with_no_subscriptions_gets_empty_results(alice):
|
||||
assert SQLiteFTSBackend().search("anything", alice) == []
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_empty_query_returns_empty(alice, feed):
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
assert SQLiteFTSBackend().search("", alice) == []
|
||||
assert SQLiteFTSBackend().search(" ", alice) == []
|
||||
@@ -0,0 +1,57 @@
|
||||
import pytest
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.urls import reverse
|
||||
|
||||
from apps.articles.models import Article, UserArticleState
|
||||
from apps.feeds.models import Feed, Subscription
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def alice(db):
|
||||
return User.objects.create_user("alice", password="pw-alice-123")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def feed(db):
|
||||
return Feed.objects.create(feed_url="https://example.com/feed.xml", title="Example Feed")
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_search_requires_login(client):
|
||||
response = client.get(reverse("search:search"), {"q": "django"})
|
||||
assert response.status_code == 302
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_search_view_renders_matching_articles(client, alice, feed):
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
article = Article.objects.create(feed=feed, guid="g1", title="Django 5 release notes")
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
|
||||
response = client.get(reverse("search:search"), {"q": "Django"})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert article.title in response.content.decode()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_search_view_with_no_query_shows_no_results_block(client, alice):
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
response = client.get(reverse("search:search"))
|
||||
assert response.status_code == 200
|
||||
assert b"article-list" not in response.content
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_search_results_carry_user_state(client, alice, feed):
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
article = Article.objects.create(feed=feed, guid="g1", title="Django 5 release notes")
|
||||
UserArticleState.objects.create(user=alice, article=article, is_starred=True)
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
|
||||
response = client.get(reverse("search:search"), {"q": "Django"})
|
||||
content = response.content.decode()
|
||||
|
||||
assert 'class="icon-btn active"' in content
|
||||
@@ -0,0 +1,9 @@
|
||||
from django.urls import path
|
||||
|
||||
from . import views
|
||||
|
||||
app_name = "search"
|
||||
|
||||
urlpatterns = [
|
||||
path("search/", views.search, name="search"),
|
||||
]
|
||||
+16
-1
@@ -1 +1,16 @@
|
||||
# Create your views here.
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.shortcuts import render
|
||||
|
||||
from apps.articles.queries import with_user_state
|
||||
|
||||
from .backends import get_backend
|
||||
|
||||
|
||||
@login_required
|
||||
def search(request):
|
||||
query = request.GET.get("q", "").strip()
|
||||
results = []
|
||||
if query:
|
||||
backend = get_backend()
|
||||
results = with_user_state(backend.search(query, request.user), request.user)
|
||||
return render(request, "search/results.html", {"query": query, "results": results})
|
||||
Reference in new issue
Block a user