diff --git a/apps/articles/queries.py b/apps/articles/queries.py new file mode 100644 index 0000000..eb54edc --- /dev/null +++ b/apps/articles/queries.py @@ -0,0 +1,28 @@ +from .models import UserArticleState + + +def with_user_state(articles, user): + """Annotates an iterable of Article instances with is_read/is_starred/ + is_saved_for_later booleans for the given user, in two queries total + regardless of how many articles there are (no N+1). + + Accepts either a queryset or a plain list (search backends return + plain lists already ordered by relevance), materializing it either + way. + """ + articles = list(articles) + if not articles: + return articles + + states = { + state.article_id: state + for state in UserArticleState.objects.filter( + user=user, article_id__in=[a.pk for a in articles] + ) + } + for article in articles: + state = states.get(article.pk) + article.is_read = state.is_read if state else False + article.is_starred = state.is_starred if state else False + article.is_saved_for_later = state.is_saved_for_later if state else False + return articles diff --git a/apps/reader/views.py b/apps/reader/views.py index 58f4f7c..9e1b958 100644 --- a/apps/reader/views.py +++ b/apps/reader/views.py @@ -1,11 +1,11 @@ from django.contrib.auth.decorators import login_required -from django.db.models import Prefetch from django.http import Http404 from django.shortcuts import get_object_or_404, render from django.utils import timezone from django.views.decorators.http import require_POST from apps.articles.models import Article, Tag, UserArticleState +from apps.articles.queries import with_user_state FILTER_CHOICES = [ ("unread", "Unread"), @@ -27,23 +27,6 @@ def _base_queryset(user): ) -def _with_user_state_annotations(queryset, user): - queryset = queryset.prefetch_related( - Prefetch( - "user_states", - queryset=UserArticleState.objects.filter(user=user), - to_attr="my_states", - ) - ) - articles = list(queryset) - for article in articles: - state = article.my_states[0] if article.my_states else None - article.is_read = state.is_read if state else False - article.is_starred = state.is_starred if state else False - article.is_saved_for_later = state.is_saved_for_later if state else False - return articles - - def _get_or_create_state(user, article): state, _ = UserArticleState.objects.get_or_create(user=user, article=article) return state @@ -81,7 +64,7 @@ def stream(request): start, end = (page - 1) * PAGE_SIZE, page * PAGE_SIZE total_count = queryset.count() - articles = _with_user_state_annotations(queryset[start:end], request.user) + articles = with_user_state(queryset[start:end], request.user) context = { "articles": articles, diff --git a/apps/search/backends/__init__.py b/apps/search/backends/__init__.py new file mode 100644 index 0000000..c6c610c --- /dev/null +++ b/apps/search/backends/__init__.py @@ -0,0 +1,16 @@ +from django.db import connection + +from .base import SearchBackend + +__all__ = ["SearchBackend", "get_backend"] + + +def get_backend() -> SearchBackend: + if connection.vendor == "postgresql": + from .postgres import PostgresSearchBackend + + return PostgresSearchBackend() + + from .sqlite_fts import SQLiteFTSBackend + + return SQLiteFTSBackend() diff --git a/apps/search/backends/base.py b/apps/search/backends/base.py new file mode 100644 index 0000000..1dc09e0 --- /dev/null +++ b/apps/search/backends/base.py @@ -0,0 +1,9 @@ +class SearchBackend: + """Common interface so views never branch on DB vendor directly — + see apps/search/backends/__init__.py::get_backend().""" + + def search(self, query: str, user, limit: int = 50): + """Returns an iterable of Article instances the user can see + (scoped to their own subscriptions), matching ``query``, most + relevant first.""" + raise NotImplementedError diff --git a/apps/search/backends/postgres.py b/apps/search/backends/postgres.py new file mode 100644 index 0000000..4c50a22 --- /dev/null +++ b/apps/search/backends/postgres.py @@ -0,0 +1,27 @@ +from django.contrib.postgres.search import SearchQuery, SearchRank, SearchVector + +from apps.articles.models import Article + +from .base import SearchBackend + + +class PostgresSearchBackend(SearchBackend): + """Computes the search vector on the fly at query time rather than + storing/indexing it on Article — keeps the schema identical to the + SQLite backend. Revisit with a stored+GIN-indexed column only if an + instance's archive grows large enough to need it.""" + + def search(self, query: str, user, limit: int = 50): + if not query.strip(): + return Article.objects.none() + + vector = SearchVector("title", weight="A") + SearchVector("content_html", weight="B") + search_query = SearchQuery(query) + + return ( + Article.objects.filter(feed__subscriptions__user=user) + .select_related("feed") + .annotate(search=vector, rank=SearchRank(vector, search_query)) + .filter(search=search_query) + .order_by("-rank", "-published_at")[:limit] + ) diff --git a/apps/search/backends/sqlite_fts.py b/apps/search/backends/sqlite_fts.py new file mode 100644 index 0000000..8ccdb4b --- /dev/null +++ b/apps/search/backends/sqlite_fts.py @@ -0,0 +1,48 @@ +import sqlite3 + +from django.db import connection + +from apps.articles.models import Article +from apps.feeds.models import Subscription + +from .base import SearchBackend + + +def _fts_phrase_query(raw_query: str) -> str: + """Wraps the whole user input as a single FTS5 phrase so arbitrary + punctuation (hyphens, quotes, parentheses...) in a search box doesn't + blow up as invalid FTS5 query syntax.""" + escaped = raw_query.replace('"', '""') + return f'"{escaped}"' + + +class SQLiteFTSBackend(SearchBackend): + def search(self, query: str, user, limit: int = 50): + if not query.strip(): + return [] + + feed_ids = list(Subscription.objects.filter(user=user).values_list("feed_id", flat=True)) + if not feed_ids: + return [] + + placeholders = ",".join(["%s"] * len(feed_ids)) + sql = f""" + SELECT a.id + FROM articles_article a + JOIN articles_fts f ON f.rowid = a.id + WHERE articles_fts MATCH %s AND a.feed_id IN ({placeholders}) + ORDER BY rank + LIMIT %s + """ + try: + with connection.cursor() as cursor: + cursor.execute(sql, [_fts_phrase_query(query), *feed_ids, limit]) + ordered_ids = [row[0] for row in cursor.fetchall()] + except sqlite3.OperationalError: + return [] + + if not ordered_ids: + return [] + + articles_by_id = Article.objects.select_related("feed").in_bulk(ordered_ids) + return [articles_by_id[pk] for pk in ordered_ids if pk in articles_by_id] diff --git a/apps/search/management/__init__.py b/apps/search/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/search/management/commands/__init__.py b/apps/search/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/search/management/commands/reindex_search.py b/apps/search/management/commands/reindex_search.py new file mode 100644 index 0000000..9ad74d9 --- /dev/null +++ b/apps/search/management/commands/reindex_search.py @@ -0,0 +1,20 @@ +from django.core.management.base import BaseCommand +from django.db import connection + + +class Command(BaseCommand): + help = ( + "Rebuild the SQLite FTS5 search index from scratch. No-op on Postgres, " + "which computes its search vector at query time instead of maintaining " + "a separate index." + ) + + def handle(self, *args, **options): + if connection.vendor != "sqlite": + self.stdout.write("Not using SQLite — nothing to rebuild.") + return + + with connection.cursor() as cursor: + cursor.execute("INSERT INTO articles_fts(articles_fts) VALUES ('rebuild')") + + self.stdout.write(self.style.SUCCESS("Rebuilt the SQLite FTS5 search index.")) diff --git a/apps/search/migrations/0001_sqlite_fts.py b/apps/search/migrations/0001_sqlite_fts.py new file mode 100644 index 0000000..ec238a7 --- /dev/null +++ b/apps/search/migrations/0001_sqlite_fts.py @@ -0,0 +1,78 @@ +"""Creates (and on reverse, drops) a SQLite FTS5 virtual table mirroring +apps_articles.Article(title, content_html), plus triggers that keep it in +sync on insert/update/delete. + +No-ops entirely on Postgres — the Postgres backend computes its search +vector on the fly instead of maintaining a separate index table, see +apps/search/backends/postgres.py. +""" + +from django.db import migrations + +CREATE_FTS_TABLE = """ +CREATE VIRTUAL TABLE IF NOT EXISTS articles_fts USING fts5( + title, content_html, content='articles_article', content_rowid='id' +); +""" + +POPULATE_FTS_TABLE = """ +INSERT INTO articles_fts(rowid, title, content_html) +SELECT id, title, content_html FROM articles_article; +""" + +CREATE_INSERT_TRIGGER = """ +CREATE TRIGGER IF NOT EXISTS articles_fts_ai AFTER INSERT ON articles_article BEGIN + INSERT INTO articles_fts(rowid, title, content_html) + VALUES (new.id, new.title, new.content_html); +END; +""" + +CREATE_DELETE_TRIGGER = """ +CREATE TRIGGER IF NOT EXISTS articles_fts_ad AFTER DELETE ON articles_article BEGIN + INSERT INTO articles_fts(articles_fts, rowid, title, content_html) + VALUES('delete', old.id, old.title, old.content_html); +END; +""" + +CREATE_UPDATE_TRIGGER = """ +CREATE TRIGGER IF NOT EXISTS articles_fts_au AFTER UPDATE ON articles_article BEGIN + INSERT INTO articles_fts(articles_fts, rowid, title, content_html) + VALUES('delete', old.id, old.title, old.content_html); + INSERT INTO articles_fts(rowid, title, content_html) + VALUES (new.id, new.title, new.content_html); +END; +""" + +DROP_EVERYTHING = """ +DROP TRIGGER IF EXISTS articles_fts_au; +DROP TRIGGER IF EXISTS articles_fts_ad; +DROP TRIGGER IF EXISTS articles_fts_ai; +DROP TABLE IF EXISTS articles_fts; +""" + + +def create_fts(apps, schema_editor): + if schema_editor.connection.vendor != "sqlite": + return + with schema_editor.connection.cursor() as cursor: + cursor.execute(CREATE_FTS_TABLE) + cursor.execute(POPULATE_FTS_TABLE) + cursor.execute(CREATE_INSERT_TRIGGER) + cursor.execute(CREATE_DELETE_TRIGGER) + cursor.execute(CREATE_UPDATE_TRIGGER) + + +def drop_fts(apps, schema_editor): + if schema_editor.connection.vendor != "sqlite": + return + with schema_editor.connection.cursor() as cursor: + for statement in DROP_EVERYTHING.strip().split(";"): + statement = statement.strip() + if statement: + cursor.execute(statement) + + +class Migration(migrations.Migration): + dependencies = [("articles", "0001_initial")] + + operations = [migrations.RunPython(create_fts, drop_fts)] diff --git a/apps/search/test_postgres_backend.py b/apps/search/test_postgres_backend.py new file mode 100644 index 0000000..631e833 --- /dev/null +++ b/apps/search/test_postgres_backend.py @@ -0,0 +1,99 @@ +import pytest +from django.contrib.auth import get_user_model +from django.db import connection + +from apps.articles.models import Article +from apps.feeds.models import Feed, Subscription + +from .backends.postgres import PostgresSearchBackend + +User = get_user_model() + +pytestmark = pytest.mark.skipif( + connection.vendor != "postgresql", + reason="Postgres-only backend; run with DATABASE_URL pointed at a Postgres instance.", +) + + +@pytest.fixture +def alice(db): + return User.objects.create_user("alice", password="pw-alice-123") + + +@pytest.fixture +def bob(db): + return User.objects.create_user("bob", password="pw-bob-123") + + +@pytest.fixture +def feed(db): + return Feed.objects.create(feed_url="https://example.com/feed.xml", title="Example Feed") + + +@pytest.mark.django_db +def test_search_finds_article_by_title(alice, feed): + Subscription.objects.create(user=alice, feed=feed) + article = Article.objects.create(feed=feed, guid="g1", title="Django 5 release notes") + Article.objects.create(feed=feed, guid="g2", title="Completely unrelated") + + results = list(PostgresSearchBackend().search("Django", alice)) + + assert [a.pk for a in results] == [article.pk] + + +@pytest.mark.django_db +def test_search_finds_article_by_content(alice, feed): + Subscription.objects.create(user=alice, feed=feed) + article = Article.objects.create( + feed=feed, + guid="g1", + title="Weekly update", + content_html="

We shipped WireGuard support.

", + ) + + results = list(PostgresSearchBackend().search("WireGuard", alice)) + + assert [a.pk for a in results] == [article.pk] + + +@pytest.mark.django_db +def test_search_only_returns_subscribed_feeds(alice, bob, feed): + other_feed = Feed.objects.create(feed_url="https://other.example.com/feed.xml") + Subscription.objects.create(user=alice, feed=feed) + Subscription.objects.create(user=bob, feed=other_feed) + + Article.objects.create(feed=feed, guid="g1", title="Django in feed A") + article_b = Article.objects.create(feed=other_feed, guid="g2", title="Django in feed B") + + alice_results = list(PostgresSearchBackend().search("Django", alice)) + bob_results = list(PostgresSearchBackend().search("Django", bob)) + + assert len(alice_results) == 1 + assert [a.pk for a in bob_results] == [article_b.pk] + + +@pytest.mark.django_db +def test_title_matches_rank_above_content_only_matches(alice, feed): + Subscription.objects.create(user=alice, feed=feed) + title_match = Article.objects.create(feed=feed, guid="g1", title="Django release") + content_match = Article.objects.create( + feed=feed, guid="g2", title="Something else", content_html="

mentions Django

" + ) + + results = list(PostgresSearchBackend().search("Django", alice)) + + assert [a.pk for a in results] == [title_match.pk, content_match.pk] + + +@pytest.mark.django_db +def test_no_matches_returns_empty(alice, feed): + Subscription.objects.create(user=alice, feed=feed) + Article.objects.create(feed=feed, guid="g1", title="Something else entirely") + + assert list(PostgresSearchBackend().search("nonexistentword", alice)) == [] + + +@pytest.mark.django_db +def test_empty_query_returns_empty(alice, feed): + Subscription.objects.create(user=alice, feed=feed) + assert list(PostgresSearchBackend().search("", alice)) == [] diff --git a/apps/search/test_sqlite_backend.py b/apps/search/test_sqlite_backend.py new file mode 100644 index 0000000..3f34273 --- /dev/null +++ b/apps/search/test_sqlite_backend.py @@ -0,0 +1,127 @@ +import pytest +from django.contrib.auth import get_user_model +from django.db import connection + +from apps.articles.models import Article +from apps.feeds.models import Feed, Subscription + +from .backends.sqlite_fts import SQLiteFTSBackend + +User = get_user_model() + +pytestmark = pytest.mark.skipif( + connection.vendor != "sqlite", + reason="SQLite-only backend; the articles_fts virtual table only exists on SQLite.", +) + + +@pytest.fixture +def alice(db): + return User.objects.create_user("alice", password="pw-alice-123") + + +@pytest.fixture +def bob(db): + return User.objects.create_user("bob", password="pw-bob-123") + + +@pytest.fixture +def feed(db): + return Feed.objects.create(feed_url="https://example.com/feed.xml", title="Example Feed") + + +@pytest.mark.django_db +def test_search_finds_article_by_title(alice, feed): + Subscription.objects.create(user=alice, feed=feed) + article = Article.objects.create(feed=feed, guid="g1", title="Django 5 release notes") + Article.objects.create(feed=feed, guid="g2", title="Completely unrelated") + + results = SQLiteFTSBackend().search("Django", alice) + + assert [a.pk for a in results] == [article.pk] + + +@pytest.mark.django_db +def test_search_finds_article_by_content(alice, feed): + Subscription.objects.create(user=alice, feed=feed) + article = Article.objects.create( + feed=feed, + guid="g1", + title="Weekly update", + content_html="

We shipped WireGuard support.

", + ) + + results = SQLiteFTSBackend().search("WireGuard", alice) + + assert [a.pk for a in results] == [article.pk] + + +@pytest.mark.django_db +def test_search_only_returns_subscribed_feeds(alice, bob, feed): + other_feed = Feed.objects.create(feed_url="https://other.example.com/feed.xml") + Subscription.objects.create(user=alice, feed=feed) + Subscription.objects.create(user=bob, feed=other_feed) + + Article.objects.create(feed=feed, guid="g1", title="Django in feed A") + article_b = Article.objects.create(feed=other_feed, guid="g2", title="Django in feed B") + + alice_results = SQLiteFTSBackend().search("Django", alice) + bob_results = SQLiteFTSBackend().search("Django", bob) + + assert len(alice_results) == 1 + assert [a.pk for a in bob_results] == [article_b.pk] + + +@pytest.mark.django_db +def test_search_with_no_matches_returns_empty(alice, feed): + Subscription.objects.create(user=alice, feed=feed) + Article.objects.create(feed=feed, guid="g1", title="Something else entirely") + + assert SQLiteFTSBackend().search("nonexistentword", alice) == [] + + +@pytest.mark.django_db +def test_search_handles_special_characters_without_crashing(alice, feed): + Subscription.objects.create(user=alice, feed=feed) + Article.objects.create(feed=feed, guid="g1", title="C++ vs Rust: a comparison (2024)") + + # Characters that are meaningful in raw FTS5 query syntax shouldn't + # raise — the backend wraps the whole query as a phrase. + results = SQLiteFTSBackend().search("C++ vs Rust: a comparison (2024)", alice) + assert len(results) == 1 + + +@pytest.mark.django_db +def test_search_index_updates_on_article_edit(alice, feed): + Subscription.objects.create(user=alice, feed=feed) + article = Article.objects.create(feed=feed, guid="g1", title="Old title") + assert SQLiteFTSBackend().search("New", alice) == [] + + article.title = "New title" + article.save() + + results = SQLiteFTSBackend().search("New", alice) + assert [a.pk for a in results] == [article.pk] + + +@pytest.mark.django_db +def test_search_index_updates_on_article_delete(alice, feed): + Subscription.objects.create(user=alice, feed=feed) + article = Article.objects.create(feed=feed, guid="g1", title="Removable article") + assert len(SQLiteFTSBackend().search("Removable", alice)) == 1 + + article.delete() + + assert SQLiteFTSBackend().search("Removable", alice) == [] + + +@pytest.mark.django_db +def test_user_with_no_subscriptions_gets_empty_results(alice): + assert SQLiteFTSBackend().search("anything", alice) == [] + + +@pytest.mark.django_db +def test_empty_query_returns_empty(alice, feed): + Subscription.objects.create(user=alice, feed=feed) + assert SQLiteFTSBackend().search("", alice) == [] + assert SQLiteFTSBackend().search(" ", alice) == [] diff --git a/apps/search/test_views.py b/apps/search/test_views.py new file mode 100644 index 0000000..c450f79 --- /dev/null +++ b/apps/search/test_views.py @@ -0,0 +1,57 @@ +import pytest +from django.contrib.auth import get_user_model +from django.urls import reverse + +from apps.articles.models import Article, UserArticleState +from apps.feeds.models import Feed, Subscription + +User = get_user_model() + + +@pytest.fixture +def alice(db): + return User.objects.create_user("alice", password="pw-alice-123") + + +@pytest.fixture +def feed(db): + return Feed.objects.create(feed_url="https://example.com/feed.xml", title="Example Feed") + + +@pytest.mark.django_db +def test_search_requires_login(client): + response = client.get(reverse("search:search"), {"q": "django"}) + assert response.status_code == 302 + + +@pytest.mark.django_db +def test_search_view_renders_matching_articles(client, alice, feed): + Subscription.objects.create(user=alice, feed=feed) + article = Article.objects.create(feed=feed, guid="g1", title="Django 5 release notes") + client.login(username="alice", password="pw-alice-123") + + response = client.get(reverse("search:search"), {"q": "Django"}) + + assert response.status_code == 200 + assert article.title in response.content.decode() + + +@pytest.mark.django_db +def test_search_view_with_no_query_shows_no_results_block(client, alice): + client.login(username="alice", password="pw-alice-123") + response = client.get(reverse("search:search")) + assert response.status_code == 200 + assert b"article-list" not in response.content + + +@pytest.mark.django_db +def test_search_results_carry_user_state(client, alice, feed): + Subscription.objects.create(user=alice, feed=feed) + article = Article.objects.create(feed=feed, guid="g1", title="Django 5 release notes") + UserArticleState.objects.create(user=alice, article=article, is_starred=True) + client.login(username="alice", password="pw-alice-123") + + response = client.get(reverse("search:search"), {"q": "Django"}) + content = response.content.decode() + + assert 'class="icon-btn active"' in content diff --git a/apps/search/urls.py b/apps/search/urls.py new file mode 100644 index 0000000..9d6ffe8 --- /dev/null +++ b/apps/search/urls.py @@ -0,0 +1,9 @@ +from django.urls import path + +from . import views + +app_name = "search" + +urlpatterns = [ + path("search/", views.search, name="search"), +] diff --git a/apps/search/views.py b/apps/search/views.py index 60f00ef..fb50d23 100644 --- a/apps/search/views.py +++ b/apps/search/views.py @@ -1 +1,16 @@ -# Create your views here. +from django.contrib.auth.decorators import login_required +from django.shortcuts import render + +from apps.articles.queries import with_user_state + +from .backends import get_backend + + +@login_required +def search(request): + query = request.GET.get("q", "").strip() + results = [] + if query: + backend = get_backend() + results = with_user_state(backend.search(query, request.user), request.user) + return render(request, "search/results.html", {"query": query, "results": results}) diff --git a/config/urls.py b/config/urls.py index c202628..5639ef4 100644 --- a/config/urls.py +++ b/config/urls.py @@ -7,5 +7,6 @@ urlpatterns = [ path("", include("apps.feeds.urls")), path("", include("apps.articles.urls")), path("", include("apps.rules.urls")), + path("", include("apps.search.urls")), path("", include("apps.reader.urls")), ] diff --git a/static/css/base.css b/static/css/base.css index a831b0f..cd52bf7 100644 --- a/static/css/base.css +++ b/static/css/base.css @@ -239,6 +239,16 @@ input, button { gap: 0.35rem; } +.search-form { + display: flex; + gap: 0.5rem; + margin: 1rem 0; +} + +.search-form input { + flex: 1; +} + .article-content { line-height: 1.6; margin-top: 1rem; diff --git a/templates/reader/stream.html b/templates/reader/stream.html index d472f96..a67aba4 100644 --- a/templates/reader/stream.html +++ b/templates/reader/stream.html @@ -7,6 +7,7 @@ Add feed Tags Rules + Search Log out diff --git a/templates/search/results.html b/templates/search/results.html new file mode 100644 index 0000000..6e55f9b --- /dev/null +++ b/templates/search/results.html @@ -0,0 +1,21 @@ +{% extends "base.html" %} +{% block title %}Search — Evection{% endblock %} +{% block content %} +

← Back to stream

+

Search

+ +
+ + +
+ + {% if query %} + + {% endif %} +{% endblock %}