Private
Public Access
Compare commits
16
Commits
v2026.11.1.4
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2a3e9f632e | ||
|
|
fed5bcc49e | ||
|
|
ac5e951749 | ||
|
|
aacb41d1d3 | ||
|
|
91a4c360a4 | ||
|
|
d6e310b127 | ||
|
|
72bffbe97f | ||
|
|
2e1c0e31a8 | ||
|
|
10ba491e10 | ||
|
|
698940763f | ||
|
|
41f99e9405 | ||
|
|
85914f9107 | ||
|
|
0975a7634b | ||
|
|
fd494a9c08 | ||
|
|
f8cd1eb33e | ||
|
|
73dbee9400 |
No files matched your search
@@ -23,3 +23,9 @@ EVECTION_MIN_FETCH_INTERVAL_MINUTES=15
|
||||
# Bump this on deploys that change cached static assets to invalidate old
|
||||
# service worker caches.
|
||||
EVECTION_PWA_CACHE_VERSION=v1
|
||||
|
||||
# IP address(es) or CIDR(s) of the reverse proxy in front of gunicorn,
|
||||
# comma-separated, e.g. 192.168.1.10 or 10.0.0.0/24. When set, the login
|
||||
# throttle trusts X-Forwarded-For only from these addresses. Strongly
|
||||
# recommended; `manage.py check --deploy` warns until it's set.
|
||||
# EVECTION_TRUSTED_PROXIES=
|
||||
@@ -54,6 +54,18 @@ ruff check .
|
||||
black --check .
|
||||
```
|
||||
|
||||
### Testing against PostgreSQL
|
||||
|
||||
The suite runs on SQLite by default; the Postgres-only tests (search backend)
|
||||
are skipped there. To run everything against a real Postgres:
|
||||
|
||||
```bash
|
||||
docker run -d --name evection-pg-test -e POSTGRES_PASSWORD=testpw -e POSTGRES_DB=evection \
|
||||
-p 127.0.0.1:55432:5432 postgres:16
|
||||
DATABASE_URL=postgres://postgres:testpw@127.0.0.1:55432/evection pytest
|
||||
docker rm -f evection-pg-test
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
[GNU AGPL-3.0-or-later](LICENSE) — if you run a modified version of Evection as
|
||||
|
||||
@@ -4,3 +4,6 @@ from django.apps import AppConfig
|
||||
class AccountsConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = "apps.accounts"
|
||||
|
||||
def ready(self):
|
||||
from . import checks # noqa: F401 (registers the deploy check)
|
||||
@@ -0,0 +1,20 @@
|
||||
from django.conf import settings
|
||||
from django.core.checks import Tags, Warning, register
|
||||
|
||||
|
||||
@register(Tags.security, deploy=True)
|
||||
def trusted_proxies_configured(app_configs, **kwargs):
|
||||
if settings.DEBUG or settings.EVECTION_TRUSTED_PROXIES:
|
||||
return []
|
||||
return [
|
||||
Warning(
|
||||
"EVECTION_TRUSTED_PROXIES is not set.",
|
||||
hint=(
|
||||
"The login throttle then trusts the first X-Forwarded-For entry from "
|
||||
"anyone, which only holds while gunicorn is reachable by nothing but a "
|
||||
"proxy that overwrites that header. Set EVECTION_TRUSTED_PROXIES to your "
|
||||
"reverse proxy's IP/CIDR in .env (see docs/install.md)."
|
||||
),
|
||||
id="evection.W001",
|
||||
)
|
||||
]
|
||||
@@ -74,19 +74,144 @@ def test_too_many_failed_attempts_against_one_username_blocks_that_username(clie
|
||||
assert response.wsgi_request.user == bob
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_login_still_works_when_the_throttle_cache_is_down(client, monkeypatch):
|
||||
@pytest.fixture
|
||||
def cache_down(monkeypatch):
|
||||
from unittest.mock import Mock
|
||||
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
get_user_model().objects.create_user("alice", password="pw-alice-123")
|
||||
broken = Mock(side_effect=ConnectionError("redis down"))
|
||||
monkeypatch.setattr("apps.accounts.views.cache.get", broken)
|
||||
monkeypatch.setattr("apps.accounts.views.cache.incr", broken)
|
||||
monkeypatch.setattr("apps.accounts.views.cache.set", broken)
|
||||
for name in ("get", "incr", "set"):
|
||||
monkeypatch.setattr(f"apps.accounts.throttle.cache.{name}", broken)
|
||||
|
||||
bad = client.post(reverse("accounts:login"), {"username": "alice", "password": "wrong"})
|
||||
assert bad.status_code == 200
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_local_limiter():
|
||||
from .views import login_throttle
|
||||
|
||||
login_throttle._local.clear()
|
||||
yield
|
||||
login_throttle._local.clear()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_login_still_works_when_the_throttle_cache_is_down(client, alice, cache_down):
|
||||
assert _bad_login(client).status_code == 200
|
||||
good = client.post(reverse("accounts:login"), {"username": "alice", "password": "pw-alice-123"})
|
||||
assert good.status_code == 302
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_a_redis_outage_degrades_to_a_local_limit_instead_of_no_limit(client, alice, cache_down):
|
||||
"""R2-S4: previously every attempt was allowed while the cache was
|
||||
down, so an outage switched brute-force protection off entirely."""
|
||||
for _ in range(LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME):
|
||||
_bad_login(client)
|
||||
|
||||
blocked = client.post(
|
||||
reverse("accounts:login"), {"username": "alice", "password": "pw-alice-123"}
|
||||
)
|
||||
assert blocked.status_code == 302
|
||||
assert blocked.url == reverse("accounts:login"), "even the right password is refused"
|
||||
|
||||
|
||||
class TestLocalLimiter:
|
||||
def test_counts_per_key_and_expires(self, monkeypatch):
|
||||
import time as time_module
|
||||
|
||||
from .throttle import LoginThrottle
|
||||
|
||||
throttle = LoginThrottle(window_seconds=60)
|
||||
for _ in range(3):
|
||||
throttle._local_record("k")
|
||||
assert throttle._local_count("k") == 3
|
||||
assert throttle._local_count("other") == 0
|
||||
|
||||
real = time_module.monotonic()
|
||||
monkeypatch.setattr("apps.accounts.throttle.time.monotonic", lambda: real + 61)
|
||||
assert throttle._local_count("k") == 0
|
||||
|
||||
def test_memory_is_bounded(self, monkeypatch):
|
||||
from . import throttle as throttle_module
|
||||
|
||||
monkeypatch.setattr(throttle_module, "LOCAL_MAX_KEYS", 5)
|
||||
throttle = throttle_module.LoginThrottle(window_seconds=60)
|
||||
for i in range(50):
|
||||
throttle._local_record(f"k{i}")
|
||||
assert len(throttle._local) <= 5
|
||||
|
||||
|
||||
class TestClientAddress:
|
||||
def _request(self, remote, forwarded=None):
|
||||
from django.test import RequestFactory
|
||||
|
||||
extra = {"REMOTE_ADDR": remote}
|
||||
if forwarded is not None:
|
||||
extra["HTTP_X_FORWARDED_FOR"] = forwarded
|
||||
return RequestFactory().get("/", **extra)
|
||||
|
||||
def test_unset_keeps_the_legacy_first_hop_behavior(self, settings):
|
||||
from .throttle import client_address
|
||||
|
||||
settings.EVECTION_TRUSTED_PROXIES = []
|
||||
assert client_address(self._request("10.0.0.5", "1.2.3.4, 10.0.0.5")) == "1.2.3.4"
|
||||
assert client_address(self._request("10.0.0.5")) == "10.0.0.5"
|
||||
|
||||
def test_trusted_proxy_uses_the_rightmost_untrusted_hop_not_the_spoofable_first(self, settings):
|
||||
import ipaddress
|
||||
|
||||
from .throttle import client_address
|
||||
|
||||
settings.EVECTION_TRUSTED_PROXIES = [ipaddress.ip_network("10.0.0.5/32")]
|
||||
# The client sent "X-Forwarded-For: 6.6.6.6"; the proxy appended the
|
||||
# address it actually saw (203.0.113.9).
|
||||
request = self._request("10.0.0.5", "6.6.6.6, 203.0.113.9")
|
||||
assert client_address(request) == "203.0.113.9"
|
||||
|
||||
def test_forwarded_for_from_an_untrusted_peer_is_ignored(self, settings):
|
||||
import ipaddress
|
||||
|
||||
from .throttle import client_address
|
||||
|
||||
settings.EVECTION_TRUSTED_PROXIES = [ipaddress.ip_network("10.0.0.5/32")]
|
||||
# Someone reaching gunicorn directly and claiming to be 1.2.3.4.
|
||||
assert client_address(self._request("198.51.100.7", "1.2.3.4")) == "198.51.100.7"
|
||||
|
||||
def test_chained_trusted_proxies_are_skipped(self, settings):
|
||||
import ipaddress
|
||||
|
||||
from .throttle import client_address
|
||||
|
||||
settings.EVECTION_TRUSTED_PROXIES = [ipaddress.ip_network("10.0.0.0/24")]
|
||||
request = self._request("10.0.0.5", "203.0.113.9, 10.0.0.7")
|
||||
assert client_address(request) == "203.0.113.9"
|
||||
|
||||
def test_trusted_peer_with_no_forwarded_header_falls_back_to_the_peer(self, settings):
|
||||
import ipaddress
|
||||
|
||||
from .throttle import client_address
|
||||
|
||||
settings.EVECTION_TRUSTED_PROXIES = [ipaddress.ip_network("10.0.0.5/32")]
|
||||
assert client_address(self._request("10.0.0.5")) == "10.0.0.5"
|
||||
|
||||
|
||||
class TestDeployCheck:
|
||||
def test_warns_when_trusted_proxies_is_unset_in_production(self, settings):
|
||||
from .checks import trusted_proxies_configured
|
||||
|
||||
settings.DEBUG = False
|
||||
settings.EVECTION_TRUSTED_PROXIES = []
|
||||
warnings = trusted_proxies_configured(None)
|
||||
|
||||
assert [w.id for w in warnings] == ["evection.W001"]
|
||||
|
||||
def test_quiet_when_configured_or_in_debug(self, settings):
|
||||
import ipaddress
|
||||
|
||||
from .checks import trusted_proxies_configured
|
||||
|
||||
settings.DEBUG = False
|
||||
settings.EVECTION_TRUSTED_PROXIES = [ipaddress.ip_network("10.0.0.5/32")]
|
||||
assert trusted_proxies_configured(None) == []
|
||||
|
||||
settings.DEBUG = True
|
||||
settings.EVECTION_TRUSTED_PROXIES = []
|
||||
assert trusted_proxies_configured(None) == []
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Login attempt throttling: counters live in the shared cache (Redis) so
|
||||
every gunicorn worker sees them, with an in-process fallback when the
|
||||
cache is unreachable (R2-S4) — so a Redis outage degrades throttling to
|
||||
per-worker limits instead of switching it off for anyone who happens to
|
||||
choose that moment."""
|
||||
|
||||
import ipaddress
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
LOCAL_MAX_KEYS = 10_000
|
||||
|
||||
|
||||
class LoginThrottle:
|
||||
def __init__(self, window_seconds: int):
|
||||
self.window = window_seconds
|
||||
self._lock = threading.Lock()
|
||||
self._local: dict[str, tuple[int, float]] = {}
|
||||
|
||||
def is_blocked(self, key: str, limit: int) -> bool:
|
||||
try:
|
||||
return cache.get(key, 0) >= limit
|
||||
except Exception:
|
||||
logger.warning("Login throttle cache unavailable; using local limiter")
|
||||
return self._local_count(key) >= limit
|
||||
|
||||
def record(self, key: str) -> None:
|
||||
try:
|
||||
try:
|
||||
cache.incr(key)
|
||||
except ValueError:
|
||||
cache.set(key, 1, self.window)
|
||||
except Exception:
|
||||
logger.warning("Login throttle cache unavailable; using local limiter")
|
||||
self._local_record(key)
|
||||
|
||||
def _local_count(self, key: str) -> int:
|
||||
with self._lock:
|
||||
entry = self._local.get(key)
|
||||
if entry is None or entry[1] <= time.monotonic():
|
||||
return 0
|
||||
return entry[0]
|
||||
|
||||
def _local_record(self, key: str) -> None:
|
||||
now = time.monotonic()
|
||||
with self._lock:
|
||||
if len(self._local) >= LOCAL_MAX_KEYS:
|
||||
self._local = {k: v for k, v in self._local.items() if v[1] > now}
|
||||
if len(self._local) >= LOCAL_MAX_KEYS:
|
||||
self._local.clear() # under a flood, bounded memory beats perfect memory
|
||||
count, expires = self._local.get(key, (0, 0.0))
|
||||
if expires <= now:
|
||||
count, expires = 0, now + self.window
|
||||
self._local[key] = (count + 1, expires)
|
||||
|
||||
|
||||
def _parse_ip(value: str):
|
||||
try:
|
||||
return ipaddress.ip_address(value.strip())
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _is_trusted_proxy(value: str, trusted) -> bool:
|
||||
ip = _parse_ip(value)
|
||||
return ip is not None and any(ip in network for network in trusted)
|
||||
|
||||
|
||||
def client_address(request) -> str:
|
||||
"""The address to throttle a login attempt by.
|
||||
|
||||
EVECTION_TRUSTED_PROXIES (comma-separated IPs/CIDRs of your reverse
|
||||
proxy) makes this strict: X-Forwarded-For is only believed when the
|
||||
TCP peer is one of those proxies, and then the *rightmost* entry that
|
||||
isn't itself a trusted proxy is used — the address your proxy actually
|
||||
saw — rather than the first entry, which is whatever the client chose
|
||||
to send. A client reaching gunicorn directly, or a proxy outside the
|
||||
list, gets its own peer address.
|
||||
|
||||
Unset (the default), it keeps the previous behavior — trust the first
|
||||
X-Forwarded-For entry — which is only safe while gunicorn is reachable
|
||||
by nothing but a proxy that overwrites that header (docs/install.md);
|
||||
`manage.py check --deploy` warns until it's set.
|
||||
"""
|
||||
remote = request.META.get("REMOTE_ADDR", "unknown")
|
||||
forwarded = [
|
||||
part.strip()
|
||||
for part in request.META.get("HTTP_X_FORWARDED_FOR", "").split(",")
|
||||
if part.strip()
|
||||
]
|
||||
trusted = settings.EVECTION_TRUSTED_PROXIES
|
||||
|
||||
if not trusted:
|
||||
return forwarded[0] if forwarded else remote
|
||||
if not _is_trusted_proxy(remote, trusted):
|
||||
return remote
|
||||
for address in reversed(forwarded):
|
||||
if not _is_trusted_proxy(address, trusted):
|
||||
return address
|
||||
return forwarded[0] if forwarded else remote
|
||||
+7
-38
@@ -4,13 +4,13 @@ from django.contrib import messages
|
||||
from django.contrib.auth import login
|
||||
from django.contrib.auth.decorators import user_passes_test
|
||||
from django.contrib.auth.views import LoginView as DjangoLoginView
|
||||
from django.core.cache import cache
|
||||
from django.db import IntegrityError, transaction
|
||||
from django.shortcuts import redirect, render
|
||||
|
||||
from .forms import CreateUserForm, SetupForm
|
||||
from .middleware import SetupWizardMiddleware
|
||||
from .models import SetupLock, User
|
||||
from .throttle import LoginThrottle, client_address
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -27,44 +27,13 @@ LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_ADDRESS = 15
|
||||
LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME = 8
|
||||
|
||||
|
||||
def _client_address(request) -> str:
|
||||
# gunicorn is always run behind a reverse proxy in every deployment
|
||||
# (see docs/install.md) and its bind address is never meant to be
|
||||
# reachable from outside that proxy — the same trust boundary
|
||||
# config.settings.prod already relies on for X-Forwarded-Proto. A
|
||||
# direct request (e.g. local dev) has no X-Forwarded-For and falls
|
||||
# back to REMOTE_ADDR.
|
||||
forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR", "")
|
||||
if forwarded_for:
|
||||
return forwarded_for.split(",")[0].strip()
|
||||
return request.META.get("REMOTE_ADDR", "unknown")
|
||||
login_throttle = LoginThrottle(LOGIN_RATE_LIMIT_WINDOW_SECONDS)
|
||||
|
||||
|
||||
def _attempts_key(kind: str, value: str) -> str:
|
||||
return f"login-attempts:{kind}:{value}"
|
||||
|
||||
|
||||
def _too_many_attempts(key: str, limit: int) -> bool:
|
||||
# Fails open if the cache (Redis) is unreachable: logging in must keep
|
||||
# working through a cache outage, at the cost of no throttling until
|
||||
# it's back — an outage shouldn't lock every user out of the app.
|
||||
try:
|
||||
return cache.get(key, 0) >= limit
|
||||
except Exception:
|
||||
logger.warning("Login throttle cache unavailable; not rate limiting", exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
def _record_failed_attempt(key: str) -> None:
|
||||
try:
|
||||
try:
|
||||
cache.incr(key)
|
||||
except ValueError:
|
||||
cache.set(key, 1, LOGIN_RATE_LIMIT_WINDOW_SECONDS)
|
||||
except Exception:
|
||||
logger.warning("Login throttle cache unavailable; attempt not recorded", exc_info=True)
|
||||
|
||||
|
||||
class LoginView(DjangoLoginView):
|
||||
"""Wraps Django's own LoginView with the throttle above — everything
|
||||
else (form, template, redirect behavior) is unchanged."""
|
||||
@@ -72,11 +41,11 @@ class LoginView(DjangoLoginView):
|
||||
template_name = "accounts/login.html"
|
||||
|
||||
def post(self, request, *args, **kwargs):
|
||||
address_key = _attempts_key("addr", _client_address(request))
|
||||
address_key = _attempts_key("addr", client_address(request))
|
||||
username_key = _attempts_key("user", request.POST.get("username", "")[:150])
|
||||
if _too_many_attempts(
|
||||
if login_throttle.is_blocked(
|
||||
address_key, LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_ADDRESS
|
||||
) or _too_many_attempts(username_key, LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME):
|
||||
) or login_throttle.is_blocked(username_key, LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME):
|
||||
messages.error(
|
||||
request, "Too many login attempts. Please wait a few minutes and try again."
|
||||
)
|
||||
@@ -84,8 +53,8 @@ class LoginView(DjangoLoginView):
|
||||
return super().post(request, *args, **kwargs)
|
||||
|
||||
def form_invalid(self, form):
|
||||
_record_failed_attempt(_attempts_key("addr", _client_address(self.request)))
|
||||
_record_failed_attempt(_attempts_key("user", self.request.POST.get("username", "")[:150]))
|
||||
login_throttle.record(_attempts_key("addr", client_address(self.request)))
|
||||
login_throttle.record(_attempts_key("user", self.request.POST.get("username", "")[:150]))
|
||||
return super().form_invalid(form)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
# Generated by Django 5.2.18 on 2026-11-01 16:28
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('articles', '0004_starred_saved_as_system_tags'),
|
||||
('feeds', '0006_backfill_feed_next_fetch_at'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='PurgedArticle',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('guid', models.CharField(max_length=1000)),
|
||||
('purged_at', models.DateTimeField(auto_now_add=True)),
|
||||
('feed', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='purged_articles', to='feeds.feed')),
|
||||
],
|
||||
options={
|
||||
'unique_together': {('feed', 'guid')},
|
||||
},
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,22 @@
|
||||
# Generated by Django 5.2.18 on 2026-11-01 19:00
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('articles', '0005_purgedarticle'),
|
||||
('feeds', '0006_backfill_feed_next_fetch_at'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddIndex(
|
||||
model_name='article',
|
||||
index=models.Index(fields=['fetched_at'], name='articles_ar_fetched_eb5501_idx'),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name='purgedarticle',
|
||||
index=models.Index(fields=['purged_at'], name='articles_pu_purged__27d417_idx'),
|
||||
),
|
||||
]
|
||||
+23
-1
@@ -25,13 +25,35 @@ class Article(models.Model):
|
||||
|
||||
class Meta:
|
||||
unique_together = [("feed", "guid")]
|
||||
indexes = [models.Index(fields=["feed", "published_at"])]
|
||||
indexes = [
|
||||
models.Index(fields=["feed", "published_at"]),
|
||||
# The retention purge scans for the oldest articles by this.
|
||||
models.Index(fields=["fetched_at"]),
|
||||
]
|
||||
ordering = ["-published_at", "-fetched_at"]
|
||||
|
||||
def __str__(self):
|
||||
return self.title or self.guid
|
||||
|
||||
|
||||
class PurgedArticle(models.Model):
|
||||
"""Tombstone left by the retention purge (apps.articles.tasks.
|
||||
purge_old_articles) for every article it deletes: a feed that still
|
||||
lists an entry after we purged it would otherwise be re-inserted as a
|
||||
brand-new, unread article on its next poll. Ingestion skips any guid
|
||||
that has a tombstone here. Only the (feed, guid) pair is kept — a
|
||||
small fraction of the article it replaces.
|
||||
"""
|
||||
|
||||
feed = models.ForeignKey(Feed, on_delete=models.CASCADE, related_name="purged_articles")
|
||||
guid = models.CharField(max_length=1000)
|
||||
purged_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
unique_together = [("feed", "guid")]
|
||||
indexes = [models.Index(fields=["purged_at"])]
|
||||
|
||||
|
||||
class UserArticleState(models.Model):
|
||||
"""Per-user read/hide state for an Article.
|
||||
|
||||
|
||||
+80
-10
@@ -1,14 +1,20 @@
|
||||
import logging
|
||||
import time
|
||||
from datetime import timedelta
|
||||
|
||||
from celery import shared_task
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.core.cache_utils import invalidate_feed_subscribers_sidebar_cache
|
||||
from apps.core.models import SiteSettings
|
||||
from apps.feeds.models import Subscription
|
||||
from apps.rules.engine import evaluate_rules_for_article
|
||||
from apps.rules.models import FilterRule
|
||||
|
||||
from .models import Article, ArticleTag, SubscriptionTag
|
||||
from .models import Article, ArticleTag, PurgedArticle, SubscriptionTag
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@shared_task
|
||||
@@ -60,22 +66,73 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
|
||||
evaluate_rules_for_article(subscription.user, article, rules, subscription_tag_ids)
|
||||
evaluated += 1
|
||||
|
||||
if evaluated:
|
||||
# Rules can mark articles read or hide them, which changes counts.
|
||||
invalidate_feed_subscribers_sidebar_cache(feed_id)
|
||||
return evaluated
|
||||
|
||||
|
||||
PURGE_CHUNK_SIZE = 500
|
||||
|
||||
# One run works for at most this long, then queues itself again (R2-T4) so
|
||||
# a huge first purge never holds a worker for hours or starves feed
|
||||
# polling; the hard limits below are the backstop for a single stuck chunk.
|
||||
PURGE_RUN_BUDGET_SECONDS = 240
|
||||
PURGE_CONTINUATION_DELAY_SECONDS = 30
|
||||
PURGE_SOFT_TIME_LIMIT = 540
|
||||
PURGE_TIME_LIMIT = 600
|
||||
|
||||
@shared_task
|
||||
# How long a purged article's tombstone is kept (R2-T2). It only needs to
|
||||
# outlive the time a feed keeps *listing* the entry; after this, an entry a
|
||||
# feed still publishes (and that has no publish date, or a recent one) can
|
||||
# be ingested as new again — the price of the tombstone table not growing
|
||||
# forever. Documented in docs/install.md.
|
||||
TOMBSTONE_TTL_DAYS = 365
|
||||
|
||||
|
||||
def _expire_tombstones(out_of_time) -> int:
|
||||
cutoff = timezone.now() - timedelta(days=TOMBSTONE_TTL_DAYS)
|
||||
removed = 0
|
||||
while not out_of_time():
|
||||
chunk = list(
|
||||
PurgedArticle.objects.filter(purged_at__lt=cutoff).values_list("pk", flat=True)[
|
||||
:PURGE_CHUNK_SIZE
|
||||
]
|
||||
)
|
||||
if not chunk:
|
||||
break
|
||||
PurgedArticle.objects.filter(pk__in=chunk).delete()
|
||||
removed += len(chunk)
|
||||
return removed
|
||||
|
||||
|
||||
@shared_task(time_limit=PURGE_TIME_LIMIT, soft_time_limit=PURGE_SOFT_TIME_LIMIT)
|
||||
def purge_old_articles():
|
||||
"""Deletes articles older than SiteSettings.retention_days (C7), in
|
||||
chunks so one run never holds a huge delete in a single transaction.
|
||||
Anything a user starred or saved for later (a system ArticleTag) is
|
||||
kept — those are explicit "keep this" signals. UserArticleState and
|
||||
ArticleTag rows go with their article via ON DELETE CASCADE, and the
|
||||
SQLite FTS triggers drop the search entries. 0 days means keep
|
||||
everything forever (the default).
|
||||
chunks so one run never holds a huge delete in a single transaction,
|
||||
and within a time budget — if it runs out it queues a continuation
|
||||
(R2-T4). Anything a user starred or saved for later (a system
|
||||
ArticleTag) is kept — those are explicit "keep this" signals.
|
||||
UserArticleState and ArticleTag rows go with their article via ON
|
||||
DELETE CASCADE, and the SQLite FTS triggers drop the search entries.
|
||||
0 days means keep everything forever (the default).
|
||||
|
||||
Each deleted article leaves a PurgedArticle tombstone first, so a feed
|
||||
that still lists the entry can't bring it back as a new, unread
|
||||
article on its next poll (see apps.feeds.services.fetcher). Tombstones
|
||||
older than TOMBSTONE_TTL_DAYS are removed here too, whether or not
|
||||
retention is currently on (R2-T2). Returns the number of articles
|
||||
deleted by this run.
|
||||
"""
|
||||
started = time.monotonic()
|
||||
|
||||
def out_of_time() -> bool:
|
||||
return time.monotonic() - started > PURGE_RUN_BUDGET_SECONDS
|
||||
|
||||
expired = _expire_tombstones(out_of_time)
|
||||
if expired:
|
||||
logger.info("Removed %d expired purge tombstone(s)", expired)
|
||||
|
||||
retention_days = SiteSettings.load().retention_days
|
||||
if retention_days <= 0:
|
||||
return 0
|
||||
@@ -87,9 +144,22 @@ def purge_old_articles():
|
||||
chunk = list(
|
||||
Article.objects.filter(fetched_at__lt=cutoff)
|
||||
.exclude(pk__in=keep)
|
||||
.values_list("pk", flat=True)[:PURGE_CHUNK_SIZE]
|
||||
.values_list("pk", "feed_id", "guid")[:PURGE_CHUNK_SIZE]
|
||||
)
|
||||
if not chunk:
|
||||
return deleted_total
|
||||
Article.objects.filter(pk__in=chunk).delete()
|
||||
with transaction.atomic():
|
||||
PurgedArticle.objects.bulk_create(
|
||||
[PurgedArticle(feed_id=feed_id, guid=guid) for _pk, feed_id, guid in chunk],
|
||||
ignore_conflicts=True,
|
||||
)
|
||||
Article.objects.filter(pk__in=[pk for pk, _feed_id, _guid in chunk]).delete()
|
||||
deleted_total += len(chunk)
|
||||
if out_of_time():
|
||||
logger.info(
|
||||
"Retention purge hit its %ds budget after %d article(s); continuing shortly",
|
||||
PURGE_RUN_BUDGET_SECONDS,
|
||||
deleted_total,
|
||||
)
|
||||
purge_old_articles.apply_async(countdown=PURGE_CONTINUATION_DELAY_SECONDS)
|
||||
return deleted_total
|
||||
@@ -72,3 +72,81 @@ def test_purge_works_through_more_than_one_chunk(feed, monkeypatch):
|
||||
|
||||
assert purge_old_articles() == 5
|
||||
assert Article.objects.count() == 0
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_purge_leaves_a_tombstone_per_deleted_article(feed):
|
||||
from .models import PurgedArticle
|
||||
|
||||
SiteSettings.objects.create(retention_days=30)
|
||||
_article(feed, "gone-1", age_days=60)
|
||||
_article(feed, "gone-2", age_days=60)
|
||||
_article(feed, "kept", age_days=1)
|
||||
|
||||
purge_old_articles()
|
||||
|
||||
assert set(PurgedArticle.objects.filter(feed=feed).values_list("guid", flat=True)) == {
|
||||
"gone-1",
|
||||
"gone-2",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_expired_tombstones_are_removed_even_when_retention_is_off(feed):
|
||||
from .models import PurgedArticle
|
||||
from .tasks import TOMBSTONE_TTL_DAYS
|
||||
|
||||
old = PurgedArticle.objects.create(feed=feed, guid="old")
|
||||
fresh = PurgedArticle.objects.create(feed=feed, guid="fresh")
|
||||
PurgedArticle.objects.filter(pk=old.pk).update(
|
||||
purged_at=timezone.now() - timedelta(days=TOMBSTONE_TTL_DAYS + 1)
|
||||
)
|
||||
assert SiteSettings.load().retention_days == 0
|
||||
|
||||
purge_old_articles()
|
||||
|
||||
assert not PurgedArticle.objects.filter(pk=old.pk).exists()
|
||||
assert PurgedArticle.objects.filter(pk=fresh.pk).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_a_run_that_exhausts_its_budget_queues_a_continuation(feed, monkeypatch):
|
||||
"""R2-T4: a huge purge doesn't hold a worker for its whole length —
|
||||
after the budget it does what it can and queues another run."""
|
||||
from unittest.mock import patch
|
||||
|
||||
monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2)
|
||||
monkeypatch.setattr("apps.articles.tasks.PURGE_RUN_BUDGET_SECONDS", -1) # already over
|
||||
SiteSettings.objects.create(retention_days=30)
|
||||
for i in range(5):
|
||||
_article(feed, f"old-{i}", age_days=60)
|
||||
|
||||
with patch("apps.articles.tasks.purge_old_articles.apply_async") as mock_continue:
|
||||
deleted = purge_old_articles()
|
||||
|
||||
assert deleted == 2, "one chunk, then it yields"
|
||||
assert Article.objects.count() == 3
|
||||
mock_continue.assert_called_once()
|
||||
assert mock_continue.call_args.kwargs["countdown"] > 0
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_continuation_runs_finish_the_job(feed, monkeypatch):
|
||||
from unittest.mock import patch
|
||||
|
||||
monkeypatch.setattr("apps.articles.tasks.PURGE_CHUNK_SIZE", 2)
|
||||
SiteSettings.objects.create(retention_days=30)
|
||||
for i in range(5):
|
||||
_article(feed, f"old-{i}", age_days=60)
|
||||
|
||||
total = 0
|
||||
for run in range(5):
|
||||
# Budget runs out after every chunk, as it would on a huge archive.
|
||||
monkeypatch.setattr("apps.articles.tasks.PURGE_RUN_BUDGET_SECONDS", -1)
|
||||
with patch("apps.articles.tasks.purge_old_articles.apply_async") as mock_continue:
|
||||
total += purge_old_articles()
|
||||
if not mock_continue.called:
|
||||
break
|
||||
|
||||
assert total == 5
|
||||
assert Article.objects.count() == 0
|
||||
@@ -16,6 +16,20 @@ def sidebar_unread_cache_key(user_id: int) -> str:
|
||||
return f"sidebar-unread:{user_id}"
|
||||
|
||||
|
||||
def invalidate_feed_subscribers_sidebar_cache(feed_id: int) -> None:
|
||||
"""Drop the cached unread counts of everyone subscribed to a feed —
|
||||
called when ingestion adds articles or filter rules change their
|
||||
read/hidden state, so the sidebar's next refresh shows them at once
|
||||
instead of after the TTL."""
|
||||
from apps.feeds.models import Subscription
|
||||
|
||||
user_ids = Subscription.objects.filter(feed_id=feed_id).values_list("user_id", flat=True)
|
||||
try:
|
||||
cache.delete_many([sidebar_unread_cache_key(user_id) for user_id in user_ids])
|
||||
except Exception:
|
||||
logger.warning("Sidebar cache unavailable; nothing to invalidate")
|
||||
|
||||
|
||||
def invalidate_sidebar_unread_cache(user_id: int) -> None:
|
||||
"""Call this wherever a request changes the current user's own
|
||||
read/unread state (toggle_read, mark_all_read, viewing an article) so
|
||||
|
||||
@@ -1,8 +1,34 @@
|
||||
from apps.articles.models import STARRED_TAG_NAME, ArticleTag, SubscriptionTag, Tag
|
||||
from django.db.models import Max
|
||||
|
||||
from apps.articles.models import STARRED_TAG_NAME, Article, ArticleTag, SubscriptionTag, Tag
|
||||
from apps.articles.queries import unread_counts_by_feed
|
||||
from apps.feeds.models import Subscription
|
||||
|
||||
|
||||
def starred_count(user):
|
||||
return ArticleTag.objects.filter(
|
||||
user=user, tag__user=user, tag__name=STARRED_TAG_NAME, tag__is_system=True
|
||||
).count()
|
||||
|
||||
|
||||
def sidebar_counts(user):
|
||||
"""The sidebar's numbers on their own, for the live-update poll
|
||||
(static/js/live-updates.js) to patch into the rendered sidebar:
|
||||
unread per feed, unread per tag (summed over the feeds it covers,
|
||||
as in sidebar() below), and the starred total."""
|
||||
unread_counts = unread_counts_by_feed(user)
|
||||
tag_totals = {}
|
||||
for tag_id, feed_id in SubscriptionTag.objects.filter(
|
||||
user=user, tag__is_system=False
|
||||
).values_list("tag_id", "subscription__feed_id"):
|
||||
tag_totals[str(tag_id)] = tag_totals.get(str(tag_id), 0) + unread_counts.get(feed_id, 0)
|
||||
return {
|
||||
"feeds": {str(feed_id): count for feed_id, count in unread_counts.items()},
|
||||
"tags": tag_totals,
|
||||
"starred": starred_count(user),
|
||||
}
|
||||
|
||||
|
||||
def sidebar(request):
|
||||
"""Tag/feed tree and starred count for the persistent sidebar rendered
|
||||
by base.html on every authenticated page. Kept out of individual views
|
||||
@@ -59,7 +85,10 @@ def sidebar(request):
|
||||
for tag in tags
|
||||
],
|
||||
"sidebar_untagged": untagged,
|
||||
"sidebar_starred_count": ArticleTag.objects.filter(
|
||||
user=user, tag__user=user, tag__name=STARRED_TAG_NAME, tag__is_system=True
|
||||
).count(),
|
||||
"sidebar_starred_count": starred_count(user),
|
||||
# The page's "newest article seen" baseline for the new-articles
|
||||
# banner / tab dot; rendered into the page, not fetched, so there's
|
||||
# no gap between rendering and the first poll (and a page served
|
||||
# from the PWA cache still compares against what it actually shows).
|
||||
"latest_article_id": Article.objects.aggregate(Max("id"))["id__max"] or 0,
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
from datetime import timedelta
|
||||
|
||||
import redis
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django.db.models import Max, Min
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.feeds.models import Feed
|
||||
|
||||
from .models import SiteSettings
|
||||
|
||||
# More than this many tasks waiting means workers aren't keeping up (or
|
||||
# aren't running) — fetches are piling up faster than they complete.
|
||||
MAX_QUEUE_DEPTH = 1000
|
||||
|
||||
# Past this share of active feeds failing, it's far more likely something
|
||||
# on our side (DNS, outbound network, a bad deploy) than every publisher
|
||||
# breaking at once. Needs a handful of feeds first so one broken feed out
|
||||
# of two doesn't read as an outage.
|
||||
MAX_FAILING_RATIO = 0.5
|
||||
MIN_FEEDS_FOR_RATIO = 4
|
||||
|
||||
MIN_SUCCESS_WINDOW_MINUTES = 30
|
||||
|
||||
|
||||
def queue_depth() -> int | None:
|
||||
"""Tasks waiting in Celery's default queue, or None if the broker
|
||||
can't be reached. Celery's Redis transport keeps each queue as a Redis
|
||||
list named after the queue."""
|
||||
try:
|
||||
client = redis.Redis.from_url(
|
||||
settings.CELERY_BROKER_URL, socket_connect_timeout=2, socket_timeout=2
|
||||
)
|
||||
return int(client.llen("celery"))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _success_window() -> timedelta:
|
||||
floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15)
|
||||
base = max(SiteSettings.load().fetch_interval_minutes, floor)
|
||||
return timedelta(minutes=max(3 * base, MIN_SUCCESS_WINDOW_MINUTES))
|
||||
|
||||
|
||||
PIPELINE_CACHE_KEY = "health-ready-pipeline"
|
||||
|
||||
|
||||
def cached_fetch_pipeline_status() -> tuple[dict[str, bool], dict]:
|
||||
"""fetch_pipeline_status, reused for EVECTION_HEALTH_CACHE_SECONDS
|
||||
(R2-C5): the queue/feed aggregates behind it cost real queries, and a
|
||||
monitor that polls /health/ready/ every second shouldn't turn that
|
||||
into recurring database and Redis load. The cheap liveness probes
|
||||
(database, Redis, poll heartbeat) are not cached — those stay live.
|
||||
A cache failure just means computing it fresh."""
|
||||
ttl = settings.EVECTION_HEALTH_CACHE_SECONDS
|
||||
if ttl > 0:
|
||||
try:
|
||||
cached = cache.get(PIPELINE_CACHE_KEY)
|
||||
except Exception:
|
||||
cached = None
|
||||
if cached is not None:
|
||||
return cached
|
||||
result = fetch_pipeline_status()
|
||||
if ttl > 0:
|
||||
try:
|
||||
cache.set(PIPELINE_CACHE_KEY, result, timeout=ttl)
|
||||
except Exception:
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def fetch_pipeline_status(now=None) -> tuple[dict[str, bool], dict]:
|
||||
"""Checks that fetching is actually *succeeding*, not just scheduled
|
||||
(the poll heartbeat only proves beat and a worker are alive): returns
|
||||
({check: passed}, detail) covering queue depth, whether any feed has
|
||||
been fetched successfully recently, and the share of feeds failing.
|
||||
"""
|
||||
now = now or timezone.now()
|
||||
detail: dict = {}
|
||||
|
||||
depth = queue_depth()
|
||||
detail["queue_depth"] = depth
|
||||
checks = {"queue_depth": depth is not None and depth <= MAX_QUEUE_DEPTH}
|
||||
|
||||
active = Feed.objects.filter(is_active=True)
|
||||
active_count = active.count()
|
||||
failing = active.filter(fetch_error_count__gt=0).count()
|
||||
detail["active_feeds"] = active_count
|
||||
detail["failing_feeds"] = failing
|
||||
|
||||
window = _success_window()
|
||||
last_success = active.filter(
|
||||
last_fetch_status__in=[Feed.STATUS_OK, Feed.STATUS_NOT_MODIFIED]
|
||||
).aggregate(latest=Max("last_fetched_at"))["latest"]
|
||||
if last_success is not None:
|
||||
detail["last_successful_fetch_age_seconds"] = int((now - last_success).total_seconds())
|
||||
recent_success = now - last_success <= window
|
||||
elif active_count == 0:
|
||||
recent_success = True
|
||||
else:
|
||||
# Nothing has ever fetched successfully — only a problem once
|
||||
# the oldest feed has had a full window to be fetched.
|
||||
oldest = active.aggregate(oldest=Min("created_at"))["oldest"]
|
||||
recent_success = now - oldest <= window
|
||||
checks["recent_fetch_success"] = recent_success
|
||||
|
||||
checks["feed_failures"] = not (
|
||||
active_count >= MIN_FEEDS_FOR_RATIO and failing / active_count > MAX_FAILING_RATIO
|
||||
)
|
||||
return checks, detail
|
||||
Whitespace-only changes.
@@ -0,0 +1,12 @@
|
||||
from django import template
|
||||
|
||||
from apps.core.urlutils import http_url_or_empty
|
||||
|
||||
register = template.Library()
|
||||
|
||||
|
||||
@register.filter
|
||||
def safe_http_url(value):
|
||||
""" "" unless value is an absolute http(s) URL — use before putting any
|
||||
feed-supplied URL in an href (see apps.core.urlutils)."""
|
||||
return http_url_or_empty(value)
|
||||
@@ -0,0 +1,44 @@
|
||||
import pytest
|
||||
from django.template import Context, Template
|
||||
|
||||
from .urlutils import http_url_or_empty, resolve_http_url
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value",
|
||||
[
|
||||
"javascript:alert(1)",
|
||||
"JaVaScRiPt:alert(1)",
|
||||
"java\tscript:alert(1)",
|
||||
" javascript:alert(1)",
|
||||
"data:text/html,<script>alert(1)</script>",
|
||||
"vbscript:msgbox(1)",
|
||||
"file:///etc/passwd",
|
||||
"ftp://example.com/feed.xml",
|
||||
"//example.com/no-scheme",
|
||||
"/relative/path",
|
||||
"http://",
|
||||
"",
|
||||
None,
|
||||
],
|
||||
)
|
||||
def test_unsafe_or_non_absolute_urls_are_rejected(value):
|
||||
assert http_url_or_empty(value) == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["http://example.com/a", "https://example.com/a?b=1#c"])
|
||||
def test_http_and_https_urls_pass_through(value):
|
||||
assert http_url_or_empty(value) == value
|
||||
|
||||
|
||||
def test_relative_links_resolve_against_the_feed_url_and_are_still_checked():
|
||||
base = "https://blog.example.com/feed.xml"
|
||||
assert resolve_http_url(base, "/2026/post") == "https://blog.example.com/2026/post"
|
||||
assert resolve_http_url(base, "javascript:alert(1)") == ""
|
||||
assert resolve_http_url(base, "https://other.example.com/x") == "https://other.example.com/x"
|
||||
|
||||
|
||||
def test_template_filter_blanks_unsafe_urls():
|
||||
template = Template("{% load safe_urls %}[{{ url|safe_http_url }}]")
|
||||
assert template.render(Context({"url": "javascript:alert(1)"})) == "[]"
|
||||
assert template.render(Context({"url": "https://example.com/"})) == "[https://example.com/]"
|
||||
+198
-10
@@ -22,8 +22,17 @@ def test_health_check_works_before_first_run_setup(client):
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def broker_queue(monkeypatch):
|
||||
"""Stands in for the Redis broker's queue length (tests have no Redis):
|
||||
set `broker_queue["depth"]` to a number, or None for "unreachable"."""
|
||||
state = {"depth": 0}
|
||||
monkeypatch.setattr("apps.core.health.queue_depth", lambda: state["depth"])
|
||||
return state
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_health_ready_is_degraded_until_a_poll_tick_has_happened(client):
|
||||
def test_health_ready_is_degraded_until_a_poll_tick_has_happened(client, broker_queue):
|
||||
response = client.get(reverse("core:health_ready"))
|
||||
assert response.status_code == 503
|
||||
body = response.json()
|
||||
@@ -34,22 +43,26 @@ def test_health_ready_is_degraded_until_a_poll_tick_has_happened(client):
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_health_ready_is_ok_after_poll_due_feeds_runs(client):
|
||||
def test_health_ready_is_ok_after_poll_due_feeds_runs(client, broker_queue):
|
||||
from apps.feeds.tasks import poll_due_feeds
|
||||
|
||||
poll_due_feeds()
|
||||
|
||||
response = client.get(reverse("core:health_ready"))
|
||||
assert response.status_code == 200
|
||||
assert response.json()["checks"] == {
|
||||
checks = response.json()["checks"]
|
||||
assert checks == {
|
||||
"database": True,
|
||||
"redis": True,
|
||||
"poll_heartbeat": True,
|
||||
"queue_depth": True,
|
||||
"recent_fetch_success": True,
|
||||
"feed_failures": True,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_health_ready_flags_a_stale_poll_heartbeat(client):
|
||||
def test_health_ready_flags_a_stale_poll_heartbeat(client, broker_queue):
|
||||
from datetime import timedelta
|
||||
|
||||
from django.core.cache import cache
|
||||
@@ -100,7 +113,7 @@ def test_dashboard_lists_erroring_feeds(client, django_user_model):
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_retry_feed_resets_backoff_and_requeues(client, django_user_model):
|
||||
def test_retry_feed_resets_backoff_then_claims_and_queues_one_fetch(client, django_user_model):
|
||||
django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True)
|
||||
client.login(username="admin", password="pw-admin-123")
|
||||
|
||||
@@ -110,12 +123,187 @@ def test_retry_feed_resets_backoff_and_requeues(client, django_user_model):
|
||||
last_fetch_status=Feed.STATUS_ERROR,
|
||||
)
|
||||
|
||||
with (
|
||||
patch("apps.core.views.reset_feed_backoff.delay") as mock_reset,
|
||||
patch("apps.core.views.fetch_feed.delay") as mock_fetch,
|
||||
):
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_fetch:
|
||||
response = client.post(reverse("core:retry_feed", args=[feed.id]))
|
||||
|
||||
assert response.status_code == 302
|
||||
mock_reset.assert_called_once_with(feed.id)
|
||||
mock_fetch.assert_called_once_with(feed.id)
|
||||
feed.refresh_from_db()
|
||||
assert feed.fetch_error_count == 0
|
||||
assert feed.claimed_at is not None, "the fetch is claimed so the poller can't also take it"
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_retry_feed_does_not_queue_a_second_fetch_while_one_is_in_flight(client, django_user_model):
|
||||
from django.utils import timezone
|
||||
|
||||
django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True)
|
||||
client.login(username="admin", password="pw-admin-123")
|
||||
feed = Feed.objects.create(feed_url="https://busy.example.com/feed", claimed_at=timezone.now())
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_fetch:
|
||||
client.post(reverse("core:retry_feed", args=[feed.id]))
|
||||
|
||||
mock_fetch.assert_not_called()
|
||||
|
||||
|
||||
def _ready(client):
|
||||
from django.core.cache import cache
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY
|
||||
|
||||
# Record the heartbeat directly: running poll_due_feeds here would
|
||||
# execute fetches eagerly (CELERY_TASK_ALWAYS_EAGER) and rewrite the
|
||||
# feed states each test sets up.
|
||||
cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now())
|
||||
response = client.get(reverse("core:health_ready"))
|
||||
return response.status_code, response.json()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_health_ready_flags_a_backed_up_queue(client, broker_queue):
|
||||
from .health import MAX_QUEUE_DEPTH
|
||||
|
||||
broker_queue["depth"] = MAX_QUEUE_DEPTH + 1
|
||||
|
||||
status, body = _ready(client)
|
||||
|
||||
assert status == 503
|
||||
assert body["checks"]["queue_depth"] is False
|
||||
assert body["queue_depth"] == MAX_QUEUE_DEPTH + 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_health_ready_flags_an_unreachable_broker_queue(client, broker_queue):
|
||||
broker_queue["depth"] = None
|
||||
|
||||
status, body = _ready(client)
|
||||
|
||||
assert status == 503
|
||||
assert body["checks"]["queue_depth"] is False
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_health_ready_flags_no_recent_successful_fetch(client, broker_queue):
|
||||
from datetime import timedelta
|
||||
|
||||
from django.utils import timezone
|
||||
|
||||
feed = Feed.objects.create(
|
||||
feed_url="https://a.example.com/feed.xml",
|
||||
last_fetch_status=Feed.STATUS_OK,
|
||||
last_fetched_at=timezone.now() - timedelta(days=2),
|
||||
)
|
||||
Feed.objects.filter(pk=feed.pk).update(created_at=timezone.now() - timedelta(days=30))
|
||||
|
||||
status, body = _ready(client)
|
||||
|
||||
assert status == 503
|
||||
assert body["checks"]["recent_fetch_success"] is False
|
||||
assert body["last_successful_fetch_age_seconds"] > 86400
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_health_ready_is_ok_with_a_recent_successful_fetch(client, broker_queue):
|
||||
from django.utils import timezone
|
||||
|
||||
Feed.objects.create(
|
||||
feed_url="https://a.example.com/feed.xml",
|
||||
last_fetch_status=Feed.STATUS_NOT_MODIFIED,
|
||||
last_fetched_at=timezone.now(),
|
||||
)
|
||||
|
||||
status, body = _ready(client)
|
||||
|
||||
assert status == 200
|
||||
assert body["active_feeds"] == 1
|
||||
assert body["failing_feeds"] == 0
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_health_ready_gives_a_brand_new_feed_time_before_it_counts_as_stale(client, broker_queue):
|
||||
Feed.objects.create(feed_url="https://new.example.com/feed.xml")
|
||||
|
||||
status, body = _ready(client)
|
||||
|
||||
assert status == 200
|
||||
assert body["checks"]["recent_fetch_success"] is True
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_health_ready_flags_most_feeds_failing_but_not_a_single_broken_one(client, broker_queue):
|
||||
from django.utils import timezone
|
||||
|
||||
now = timezone.now()
|
||||
healthy = Feed.objects.create(
|
||||
feed_url="https://ok.example.com/feed.xml",
|
||||
last_fetch_status=Feed.STATUS_OK,
|
||||
last_fetched_at=now,
|
||||
)
|
||||
broken = [
|
||||
Feed.objects.create(
|
||||
feed_url=f"https://bad{i}.example.com/feed.xml",
|
||||
last_fetch_status=Feed.STATUS_ERROR,
|
||||
fetch_error_count=3,
|
||||
last_fetched_at=now,
|
||||
)
|
||||
for i in range(2)
|
||||
]
|
||||
|
||||
status, body = _ready(client)
|
||||
assert status == 200, "2 of 3 failing is below the minimum feed count for the ratio"
|
||||
|
||||
Feed.objects.create(
|
||||
feed_url="https://bad9.example.com/feed.xml",
|
||||
last_fetch_status=Feed.STATUS_ERROR,
|
||||
fetch_error_count=1,
|
||||
last_fetched_at=now,
|
||||
)
|
||||
status, body = _ready(client)
|
||||
assert status == 503
|
||||
assert body["checks"]["feed_failures"] is False
|
||||
assert body["failing_feeds"] == 3
|
||||
assert healthy.pk and broken
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_readiness_reuses_the_pipeline_aggregates_for_a_short_time(client, broker_queue, settings):
|
||||
"""R2-C5: the expensive part of the readiness check is cached briefly,
|
||||
while the cheap liveness probes stay live."""
|
||||
from django.core.cache import cache
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY
|
||||
|
||||
settings.EVECTION_HEALTH_CACHE_SECONDS = 30
|
||||
cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now())
|
||||
Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
||||
|
||||
first = client.get(reverse("core:health_ready")).json()
|
||||
Feed.objects.create(feed_url="https://b.example.com/feed.xml")
|
||||
second = client.get(reverse("core:health_ready")).json()
|
||||
|
||||
assert first["active_feeds"] == 1
|
||||
assert second["active_feeds"] == 1, "served from the short-lived cache"
|
||||
|
||||
cache.delete("health-ready-pipeline")
|
||||
third = client.get(reverse("core:health_ready")).json()
|
||||
assert third["active_feeds"] == 2
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_readiness_liveness_probes_are_never_cached(client, broker_queue, settings):
|
||||
from django.core.cache import cache
|
||||
|
||||
settings.EVECTION_HEALTH_CACHE_SECONDS = 30
|
||||
first = client.get(reverse("core:health_ready"))
|
||||
assert first.json()["checks"]["poll_heartbeat"] is False
|
||||
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY
|
||||
|
||||
cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now())
|
||||
second = client.get(reverse("core:health_ready"))
|
||||
assert second.json()["checks"]["poll_heartbeat"] is True
|
||||
@@ -0,0 +1,30 @@
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
ALLOWED_SCHEMES = {"http", "https"}
|
||||
|
||||
|
||||
def http_url_or_empty(value: str | None) -> str:
|
||||
"""The URL if it's an absolute http(s) URL, else "". Feed content is
|
||||
untrusted: a link of "javascript:..." or "data:..." is harmless inside
|
||||
an escaped attribute value but becomes script the moment someone
|
||||
clicks it, so every URL we store from a feed or OPML file, and every
|
||||
one we render as a link, goes through this. (urlsplit already ignores
|
||||
the tab/newline tricks browsers also ignore inside a scheme.)"""
|
||||
if not value:
|
||||
return ""
|
||||
value = value.strip()
|
||||
try:
|
||||
parts = urlsplit(value)
|
||||
except ValueError:
|
||||
return ""
|
||||
if parts.scheme.lower() not in ALLOWED_SCHEMES or not parts.netloc:
|
||||
return ""
|
||||
return value
|
||||
|
||||
|
||||
def resolve_http_url(base: str, value: str | None) -> str:
|
||||
"""Like http_url_or_empty, but first resolves a relative link (feeds
|
||||
often give "/2026/post") against the feed's own URL."""
|
||||
if not value:
|
||||
return ""
|
||||
return http_url_or_empty(urljoin(base, value.strip()))
|
||||
+19
-5
@@ -10,7 +10,9 @@ from django.views.decorators.http import require_POST
|
||||
|
||||
from apps.articles.models import Article
|
||||
from apps.feeds.models import Feed, Subscription
|
||||
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, fetch_feed, reset_feed_backoff
|
||||
from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, enqueue_fetch, reset_backoff
|
||||
|
||||
from .health import cached_fetch_pipeline_status
|
||||
|
||||
is_admin = user_passes_test(lambda u: u.is_authenticated and u.is_staff)
|
||||
|
||||
@@ -37,7 +39,9 @@ def health_ready(request):
|
||||
"""Readiness: liveness plus everything background fetching depends on
|
||||
(C8) — the database, the Redis cache/broker, and a recent
|
||||
poll_due_feeds tick, which proves beat is scheduling AND a worker is
|
||||
consuming. /health/ above stays DB-only on purpose: scripts/upgrade.sh
|
||||
consuming. Beyond that tick, apps.core.health checks fetching is
|
||||
actually succeeding: queue depth, a recent successful fetch, and the
|
||||
share of feeds failing. /health/ above stays DB-only on purpose: scripts/upgrade.sh
|
||||
gates a deploy on it right after restart, before beat has had a
|
||||
chance to tick. Point an uptime monitor at this one instead."""
|
||||
checks = {"database": False, "redis": False, "poll_heartbeat": False}
|
||||
@@ -65,6 +69,11 @@ def health_ready(request):
|
||||
detail["poll_heartbeat_age_seconds"] = int(age)
|
||||
checks["poll_heartbeat"] = age <= POLL_HEARTBEAT_MAX_AGE_SECONDS
|
||||
|
||||
if checks["database"]:
|
||||
pipeline_checks, pipeline_detail = cached_fetch_pipeline_status()
|
||||
checks.update(pipeline_checks)
|
||||
detail.update(pipeline_detail)
|
||||
|
||||
ok = all(checks.values())
|
||||
return JsonResponse(
|
||||
{"status": "ok" if ok else "degraded", "checks": checks, **detail},
|
||||
@@ -99,7 +108,12 @@ def dashboard(request):
|
||||
@require_POST
|
||||
def retry_feed(request, feed_id):
|
||||
feed = get_object_or_404(Feed, pk=feed_id)
|
||||
reset_feed_backoff.delay(feed.id)
|
||||
fetch_feed.delay(feed.id)
|
||||
messages.success(request, f"Queued an immediate retry for '{feed.title or feed.feed_url}'.")
|
||||
# Reset first, synchronously: as separate queued tasks the reset could
|
||||
# land after the fetch and undo the schedule it just set.
|
||||
reset_backoff(feed.id)
|
||||
name = feed.title or feed.feed_url
|
||||
if enqueue_fetch(feed.id):
|
||||
messages.success(request, f"Queued an immediate retry for '{name}'.")
|
||||
else:
|
||||
messages.info(request, f"'{name}' is already being fetched right now.")
|
||||
return redirect("core:dashboard")
|
||||
@@ -4,3 +4,6 @@ from django.apps import AppConfig
|
||||
class FeedsConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = "apps.feeds"
|
||||
|
||||
def ready(self):
|
||||
from . import signals # noqa: F401
|
||||
@@ -1,10 +1,22 @@
|
||||
from django import forms
|
||||
|
||||
from apps.core.urlutils import http_url_or_empty
|
||||
|
||||
from .models import Subscription
|
||||
from .opml import MAX_UPLOAD_BYTES
|
||||
|
||||
|
||||
class AddFeedForm(forms.Form):
|
||||
feed_url = forms.URLField(label="Feed URL", max_length=1000, assume_scheme="https")
|
||||
|
||||
def clean_feed_url(self):
|
||||
# URLField also accepts ftp:// and ftps://; feeds are only ever
|
||||
# fetched over http(s).
|
||||
url = http_url_or_empty(self.cleaned_data["feed_url"])
|
||||
if not url:
|
||||
raise forms.ValidationError("Enter a feed URL starting with http:// or https://.")
|
||||
return url
|
||||
|
||||
tag_names = forms.CharField(
|
||||
label="Tags",
|
||||
max_length=500,
|
||||
@@ -26,3 +38,14 @@ class EditSubscriptionForm(forms.ModelForm):
|
||||
|
||||
class OPMLImportForm(forms.Form):
|
||||
opml_file = forms.FileField(label="OPML file")
|
||||
|
||||
def clean_opml_file(self):
|
||||
uploaded = self.cleaned_data["opml_file"]
|
||||
# Checked on the upload's reported size before anything reads it
|
||||
# into memory (R2-S5); the reverse proxy's client_max_body_size is
|
||||
# a second, coarser limit.
|
||||
if uploaded.size > MAX_UPLOAD_BYTES:
|
||||
raise forms.ValidationError(
|
||||
f"That file is too large (limit {MAX_UPLOAD_BYTES // (1024 * 1024)} MB)."
|
||||
)
|
||||
return uploaded
|
||||
+67
-20
@@ -17,6 +17,7 @@ from io import BytesIO
|
||||
import defusedxml.ElementTree as DefusedET
|
||||
|
||||
from apps.articles.models import SubscriptionTag, Tag
|
||||
from apps.core.urlutils import http_url_or_empty
|
||||
|
||||
from .models import Feed, Subscription
|
||||
|
||||
@@ -25,6 +26,14 @@ from .models import Feed, Subscription
|
||||
# that directly instead of making people unzip it themselves first.
|
||||
MAX_ZIPPED_OPML_SIZE = 20 * 1024 * 1024
|
||||
|
||||
# An upload is untrusted: bound everything an attacker controls (R2-S5).
|
||||
# A real export of thousands of feeds is well under a megabyte.
|
||||
MAX_UPLOAD_BYTES = 5 * 1024 * 1024
|
||||
MAX_ZIP_ENTRIES = 50
|
||||
MAX_OPML_DEPTH = 24
|
||||
MAX_OPML_OUTLINES = 20_000
|
||||
MAX_OPML_FEEDS = 5_000
|
||||
|
||||
|
||||
def extract_opml_bytes(uploaded_bytes: bytes) -> bytes:
|
||||
"""If the upload is a zip (e.g. FreshRSS's export), returns the bytes
|
||||
@@ -34,6 +43,8 @@ def extract_opml_bytes(uploaded_bytes: bytes) -> bytes:
|
||||
return uploaded_bytes
|
||||
|
||||
with zipfile.ZipFile(BytesIO(uploaded_bytes)) as archive:
|
||||
if len(archive.infolist()) > MAX_ZIP_ENTRIES:
|
||||
raise ValueError("The uploaded zip contains too many files.")
|
||||
candidates = [
|
||||
info
|
||||
for info in archive.infolist()
|
||||
@@ -44,7 +55,13 @@ def extract_opml_bytes(uploaded_bytes: bytes) -> bytes:
|
||||
member = min(candidates, key=lambda info: info.file_size)
|
||||
if member.file_size > MAX_ZIPPED_OPML_SIZE:
|
||||
raise ValueError("The .opml file inside the zip is implausibly large.")
|
||||
return archive.read(member)
|
||||
# file_size is just a header field an attacker controls, so bound
|
||||
# what actually decompresses too, not only what it claims.
|
||||
with archive.open(member) as handle:
|
||||
data = handle.read(MAX_ZIPPED_OPML_SIZE + 1)
|
||||
if len(data) > MAX_ZIPPED_OPML_SIZE:
|
||||
raise ValueError("The .opml file inside the zip is implausibly large.")
|
||||
return data
|
||||
|
||||
|
||||
def parse_opml(file_obj) -> list[dict]:
|
||||
@@ -57,38 +74,68 @@ def parse_opml(file_obj) -> list[dict]:
|
||||
on the feed's own outline, the convention export_opml below writes and
|
||||
many other readers use for a flat multi-category representation.
|
||||
"""
|
||||
tree = DefusedET.parse(file_obj)
|
||||
root = tree.getroot()
|
||||
body = root.find("body")
|
||||
entries: list[dict] = []
|
||||
if body is None:
|
||||
return entries
|
||||
# Streamed with iterparse instead of building the whole tree first, so
|
||||
# a hostile file (absurd nesting, millions of outlines) is rejected as
|
||||
# soon as it crosses a limit rather than after it has used the memory.
|
||||
in_body = False
|
||||
depth = 0
|
||||
outline_count = 0
|
||||
open_feed_outlines = 0 # >0 while inside a feed outline: its children are ignored
|
||||
pushed_name: list[bool] = [] # per open category outline: did it add a tag name?
|
||||
ancestor_names: list[str] = []
|
||||
|
||||
def walk(container, ancestor_names):
|
||||
for outline in container.findall("outline"):
|
||||
xml_url = outline.get("xmlUrl")
|
||||
if xml_url:
|
||||
for event, elem in DefusedET.iterparse(file_obj, events=("start", "end")):
|
||||
if event == "start":
|
||||
depth += 1
|
||||
if depth > MAX_OPML_DEPTH:
|
||||
raise ValueError("The OPML file is nested too deeply.")
|
||||
if elem.tag == "body":
|
||||
in_body = True
|
||||
elif elem.tag == "outline" and in_body:
|
||||
outline_count += 1
|
||||
if outline_count > MAX_OPML_OUTLINES:
|
||||
raise ValueError("The OPML file contains too many entries.")
|
||||
if open_feed_outlines:
|
||||
open_feed_outlines += 1
|
||||
continue
|
||||
raw_xml_url = elem.get("xmlUrl")
|
||||
if raw_xml_url:
|
||||
open_feed_outlines = 1
|
||||
xml_url = http_url_or_empty(raw_xml_url)
|
||||
if not xml_url:
|
||||
continue # a feed URL that isn't http(s) can never be fetched
|
||||
tag_names = list(ancestor_names)
|
||||
category_attr = outline.get("category")
|
||||
if category_attr:
|
||||
for name in category_attr.split(","):
|
||||
for name in (elem.get("category") or "").split(","):
|
||||
name = name.strip()
|
||||
if name and name not in tag_names:
|
||||
tag_names.append(name)
|
||||
entries.append(
|
||||
{
|
||||
"feed_url": xml_url,
|
||||
"title": outline.get("title") or outline.get("text") or "",
|
||||
"site_url": outline.get("htmlUrl") or "",
|
||||
"title": elem.get("title") or elem.get("text") or "",
|
||||
"site_url": http_url_or_empty(elem.get("htmlUrl")),
|
||||
"tag_names": tag_names,
|
||||
}
|
||||
)
|
||||
if len(entries) > MAX_OPML_FEEDS:
|
||||
raise ValueError("The OPML file contains too many feeds.")
|
||||
else:
|
||||
nested_name = outline.get("title") or outline.get("text")
|
||||
next_ancestors = ancestor_names + [nested_name] if nested_name else ancestor_names
|
||||
walk(outline, next_ancestors)
|
||||
|
||||
walk(body, [])
|
||||
nested_name = elem.get("title") or elem.get("text")
|
||||
pushed_name.append(bool(nested_name))
|
||||
if nested_name:
|
||||
ancestor_names.append(nested_name)
|
||||
else:
|
||||
depth -= 1
|
||||
if elem.tag == "body":
|
||||
in_body = False
|
||||
elif elem.tag == "outline" and in_body:
|
||||
if open_feed_outlines:
|
||||
open_feed_outlines -= 1
|
||||
elif pushed_name:
|
||||
if pushed_name.pop():
|
||||
ancestor_names.pop()
|
||||
elem.clear()
|
||||
return entries
|
||||
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ import hashlib
|
||||
import logging
|
||||
from calendar import timegm
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import timezone as dt_timezone
|
||||
|
||||
import feedparser
|
||||
@@ -21,7 +21,9 @@ from django.conf import settings
|
||||
from django.db import IntegrityError, transaction
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.articles.models import Article
|
||||
from apps.articles.models import Article, PurgedArticle
|
||||
from apps.core.models import SiteSettings
|
||||
from apps.core.urlutils import resolve_http_url
|
||||
from apps.feeds.models import Feed
|
||||
|
||||
from .ssrf_guard import SSRFBlocked, safe_get
|
||||
@@ -221,7 +223,7 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
|
||||
channel_title = parsed.feed.get("title")
|
||||
if channel_title and not feed.title:
|
||||
feed.title = channel_title[:500]
|
||||
channel_link = parsed.feed.get("link")
|
||||
channel_link = resolve_http_url(feed.feed_url, parsed.feed.get("link"))
|
||||
if channel_link and not feed.site_url:
|
||||
feed.site_url = channel_link[:1000]
|
||||
|
||||
@@ -248,7 +250,7 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
|
||||
if guid in parsed_entries:
|
||||
continue
|
||||
title = (getattr(entry, "title", "") or "")[:1000]
|
||||
link = (getattr(entry, "link", "") or "")[:2000]
|
||||
link = resolve_http_url(feed.feed_url, getattr(entry, "link", ""))[:2000]
|
||||
author = (getattr(entry, "author", "") or "")[:500]
|
||||
summary = (getattr(entry, "summary", "") or "")[:5000]
|
||||
content_html = sanitize_html(_entry_content_html(entry))
|
||||
@@ -263,6 +265,24 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
|
||||
"content_hash": _content_hash(title, link, author, summary, content_html, published_at),
|
||||
}
|
||||
|
||||
# Entries the retention purge already deleted (tombstoned) and, when
|
||||
# retention is on, entries so old by their own publish date that the
|
||||
# next purge would delete them again, are not (re-)ingested — see
|
||||
# apps.articles.tasks.purge_old_articles.
|
||||
retention_days = SiteSettings.load().retention_days
|
||||
if retention_days > 0:
|
||||
too_old_before = timezone.now() - timedelta(days=retention_days)
|
||||
for guid in [
|
||||
g
|
||||
for g, f in parsed_entries.items()
|
||||
if f["published_at"] is not None and f["published_at"] < too_old_before
|
||||
]:
|
||||
del parsed_entries[guid]
|
||||
for guid in PurgedArticle.objects.filter(feed=feed, guid__in=parsed_entries).values_list(
|
||||
"guid", flat=True
|
||||
):
|
||||
del parsed_entries[guid]
|
||||
|
||||
existing = {a.guid: a for a in Article.objects.filter(feed=feed, guid__in=parsed_entries)}
|
||||
|
||||
to_create = [
|
||||
|
||||
@@ -14,15 +14,15 @@ globally routable address (ip.is_global) — covering loopback, RFC1918
|
||||
private ranges, link-local (including the 169.254.169.254 cloud
|
||||
metadata address), and so on, for both IPv4 and IPv6.
|
||||
|
||||
What this does *not* close: the TCP connection isn't pinned to the
|
||||
specific IP that was validated, so a DNS answer that changes between
|
||||
the validation lookup and the actual connect (classic DNS rebinding)
|
||||
could still slip through. Closing that fully needs a custom transport
|
||||
that connects to the exact validated IP while still verifying TLS
|
||||
against the original hostname — more machinery than this self-hosted
|
||||
app's threat model currently justifies; this stops the much more
|
||||
common case of a feed URL (or a redirect) simply pointing straight at
|
||||
an internal address.
|
||||
The connection is pinned too (R2-S2): the address is resolved, checked and
|
||||
connected to inside one function (_connect_validated, used by the adapter
|
||||
below as the connection's socket factory), so a DNS answer that changes
|
||||
between a separate validation lookup and the connect (DNS rebinding) can't
|
||||
steer the connection to an internal address — whatever it resolves to *at
|
||||
connect time* is what gets validated. TLS still verifies against the
|
||||
original hostname, because only the socket is replaced, not the host the
|
||||
TLS layer is told about. The up-front validate_fetch_url check stays as
|
||||
defense in depth and gives a clear error before any connection attempt.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -33,7 +33,10 @@ from dataclasses import dataclass
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from requests.structures import CaseInsensitiveDict
|
||||
from urllib3.connection import HTTPConnection, HTTPSConnection
|
||||
from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool
|
||||
|
||||
ALLOWED_SCHEMES = {"http", "https"}
|
||||
MAX_REDIRECTS = 5
|
||||
@@ -55,6 +58,10 @@ class BoundedResponse:
|
||||
content: bytes
|
||||
|
||||
|
||||
def _is_allowed_ip(ip) -> bool:
|
||||
return ip.is_global
|
||||
|
||||
|
||||
def validate_fetch_url(url: str) -> None:
|
||||
"""Raises SSRFBlocked unless `url` has an http(s) scheme, a hostname,
|
||||
and every address that hostname resolves to is globally routable."""
|
||||
@@ -71,10 +78,82 @@ def validate_fetch_url(url: str) -> None:
|
||||
|
||||
for *_, sockaddr in addrinfo:
|
||||
ip = ipaddress.ip_address(sockaddr[0])
|
||||
if not ip.is_global:
|
||||
if not _is_allowed_ip(ip):
|
||||
raise SSRFBlocked(f"{parsed.hostname!r} resolves to a non-public address: {ip}")
|
||||
|
||||
|
||||
def _connect_validated(host, port, timeout, source_address, socket_options):
|
||||
"""Resolves `host`, requires every address to be globally routable, and
|
||||
connects to one of those exact addresses — the same lookup result is
|
||||
both validated and used, so there is nothing for a rebinding DNS
|
||||
answer to change in between."""
|
||||
try:
|
||||
addrinfo = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
||||
except OSError as exc:
|
||||
raise SSRFBlocked(f"Could not resolve {host!r}: {exc}") from exc
|
||||
|
||||
candidates = []
|
||||
for family, socktype, proto, _canonname, sockaddr in addrinfo:
|
||||
ip = ipaddress.ip_address(sockaddr[0])
|
||||
if not _is_allowed_ip(ip):
|
||||
raise SSRFBlocked(f"{host!r} resolves to a non-public address: {ip}")
|
||||
candidates.append((family, socktype, proto, sockaddr))
|
||||
if not candidates:
|
||||
raise SSRFBlocked(f"{host!r} did not resolve to any address")
|
||||
|
||||
last_error: OSError | None = None
|
||||
for family, socktype, proto, sockaddr in candidates:
|
||||
sock = None
|
||||
try:
|
||||
sock = socket.socket(family, socktype, proto)
|
||||
for level, option, value in socket_options or ():
|
||||
sock.setsockopt(level, option, value)
|
||||
if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
|
||||
sock.settimeout(timeout)
|
||||
if source_address:
|
||||
sock.bind(source_address)
|
||||
sock.connect(sockaddr)
|
||||
return sock
|
||||
except OSError as exc:
|
||||
last_error = exc
|
||||
if sock is not None:
|
||||
sock.close()
|
||||
raise last_error or OSError(f"Could not connect to {host!r}")
|
||||
|
||||
|
||||
class _PinnedHTTPConnection(HTTPConnection):
|
||||
def _new_conn(self):
|
||||
return _connect_validated(
|
||||
self._dns_host, self.port, self.timeout, self.source_address, self.socket_options
|
||||
)
|
||||
|
||||
|
||||
class _PinnedHTTPSConnection(HTTPSConnection):
|
||||
def _new_conn(self):
|
||||
return _connect_validated(
|
||||
self._dns_host, self.port, self.timeout, self.source_address, self.socket_options
|
||||
)
|
||||
|
||||
|
||||
class _PinnedHTTPConnectionPool(HTTPConnectionPool):
|
||||
ConnectionCls = _PinnedHTTPConnection
|
||||
|
||||
|
||||
class _PinnedHTTPSConnectionPool(HTTPSConnectionPool):
|
||||
ConnectionCls = _PinnedHTTPSConnection
|
||||
|
||||
|
||||
class PinnedAdapter(HTTPAdapter):
|
||||
"""requests adapter whose connections go through _connect_validated."""
|
||||
|
||||
def init_poolmanager(self, *args, **kwargs):
|
||||
super().init_poolmanager(*args, **kwargs)
|
||||
self.poolmanager.pool_classes_by_scheme = {
|
||||
"http": _PinnedHTTPConnectionPool,
|
||||
"https": _PinnedHTTPSConnectionPool,
|
||||
}
|
||||
|
||||
|
||||
def safe_get(url: str, *, headers: dict, timeout: float) -> BoundedResponse:
|
||||
"""GETs `url` the way fetch_and_ingest needs it: every redirect hop is
|
||||
validated and followed manually (never left to requests/urllib3, so
|
||||
@@ -84,6 +163,9 @@ def safe_get(url: str, *, headers: dict, timeout: float) -> BoundedResponse:
|
||||
"""
|
||||
current_url = url
|
||||
with requests.Session() as session:
|
||||
adapter = PinnedAdapter()
|
||||
session.mount("http://", adapter)
|
||||
session.mount("https://", adapter)
|
||||
for _ in range(MAX_REDIRECTS + 1):
|
||||
validate_fetch_url(current_url)
|
||||
response = session.get(
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
from django.db.models.signals import post_delete, post_save
|
||||
from django.dispatch import receiver
|
||||
|
||||
from apps.core.cache_utils import invalidate_sidebar_unread_cache
|
||||
|
||||
from .models import Subscription
|
||||
|
||||
|
||||
@receiver([post_save, post_delete], sender=Subscription)
|
||||
def drop_sidebar_counts_when_subscriptions_change(sender, instance, **kwargs):
|
||||
"""The cached per-feed unread counts only cover feeds the user was
|
||||
subscribed to when they were computed, so a new subscription would
|
||||
show no count until the cache expired."""
|
||||
invalidate_sidebar_unread_cache(instance.user_id)
|
||||
+107
-31
@@ -5,10 +5,12 @@ from io import BytesIO
|
||||
from celery import shared_task
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.core.cache import cache
|
||||
from django.db import InterfaceError, OperationalError
|
||||
from django.db.models import Q
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.articles.tasks import apply_filter_rules_for_feed
|
||||
from apps.core.cache_utils import invalidate_feed_subscribers_sidebar_cache
|
||||
from apps.core.models import SiteSettings
|
||||
|
||||
from .models import Feed
|
||||
@@ -47,30 +49,57 @@ def poll_due_feeds():
|
||||
"""
|
||||
now = timezone.now()
|
||||
cache.set(POLL_HEARTBEAT_CACHE_KEY, now, timeout=POLL_HEARTBEAT_TTL_SECONDS)
|
||||
claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
|
||||
due_feed_ids = list(
|
||||
Feed.objects.filter(is_active=True)
|
||||
.filter(Q(next_fetch_at__isnull=True) | Q(next_fetch_at__lte=now))
|
||||
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before))
|
||||
.filter(
|
||||
Q(claimed_at__isnull=True)
|
||||
| Q(claimed_at__lt=now - timedelta(minutes=CLAIM_STALE_MINUTES))
|
||||
)
|
||||
.order_by("next_fetch_at")
|
||||
.values_list("id", flat=True)[:POLL_BATCH_SIZE]
|
||||
)
|
||||
claimed_ids = []
|
||||
dispatched = 0
|
||||
for feed_id in due_feed_ids:
|
||||
# The UPDATE's WHERE clause re-checks staleness at the database
|
||||
# level: it only claims rows nobody else has already claimed more
|
||||
# recently than CLAIM_STALE_MINUTES ago, so two workers racing this
|
||||
# same loop can't both get rowcount=1 for the same feed.
|
||||
claimed = (
|
||||
try:
|
||||
if enqueue_fetch(feed_id):
|
||||
dispatched += 1
|
||||
except Exception:
|
||||
# Broker trouble: enqueue_fetch already released this feed's
|
||||
# claim, so it's simply picked up again next tick.
|
||||
logger.exception("Could not queue a fetch for feed %s", feed_id)
|
||||
return dispatched
|
||||
|
||||
|
||||
def claim_feed(feed_id: int) -> bool:
|
||||
"""Atomically marks a feed as having a fetch in flight (an UPDATE whose
|
||||
WHERE clause re-checks staleness, so two callers racing for the same
|
||||
feed can't both get rowcount=1). Every path that dispatches a fetch —
|
||||
the periodic poller, a newly added feed, an admin retry — goes through
|
||||
this, so they can never run the same feed concurrently."""
|
||||
now = timezone.now()
|
||||
stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
|
||||
return bool(
|
||||
Feed.objects.filter(id=feed_id, is_active=True)
|
||||
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before))
|
||||
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=stale_before))
|
||||
.update(claimed_at=now)
|
||||
)
|
||||
if claimed:
|
||||
claimed_ids.append(feed_id)
|
||||
for feed_id in claimed_ids:
|
||||
|
||||
|
||||
def enqueue_fetch(feed_id: int) -> bool:
|
||||
"""Claim a feed and queue its fetch. False means another fetch of it is
|
||||
already in flight (nothing was queued — that one will do the work).
|
||||
If queueing itself fails the claim is released and the error re-raised,
|
||||
so a broker outage doesn't leave the feed claimed until it goes stale.
|
||||
"""
|
||||
if not claim_feed(feed_id):
|
||||
return False
|
||||
try:
|
||||
fetch_feed.delay(feed_id)
|
||||
return len(claimed_ids)
|
||||
except Exception:
|
||||
Feed.objects.filter(pk=feed_id).update(claimed_at=None)
|
||||
raise
|
||||
return True
|
||||
|
||||
|
||||
def _schedule_next_fetch(feed: Feed) -> None:
|
||||
@@ -92,9 +121,32 @@ def _schedule_next_fetch(feed: Feed) -> None:
|
||||
FETCH_FEED_SOFT_TIME_LIMIT = 60
|
||||
FETCH_FEED_TIME_LIMIT = 90
|
||||
|
||||
# Failures that say nothing about the feed and usually clear up in
|
||||
# seconds — a locked SQLite file, a dropped database connection — are
|
||||
# retried promptly (R2-T5) instead of waiting out the next poll interval,
|
||||
# which under error backoff can be hours. Bounded: three attempts, 30s /
|
||||
# 60s / 120s apart, then the normal "record the error and back off" path.
|
||||
# Anything else (a parser bug, the soft time limit) is not retried: that
|
||||
# would just repeat the same failure. The claim is held throughout so the
|
||||
# poller can't also fetch the feed; it's far shorter than CLAIM_STALE_MINUTES.
|
||||
TRANSIENT_FETCH_ERRORS = (OperationalError, InterfaceError)
|
||||
FETCH_MAX_RETRIES = 3
|
||||
FETCH_RETRY_BASE_DELAY_SECONDS = 30
|
||||
|
||||
@shared_task(time_limit=FETCH_FEED_TIME_LIMIT, soft_time_limit=FETCH_FEED_SOFT_TIME_LIMIT)
|
||||
def fetch_feed(feed_id: int):
|
||||
|
||||
def _record_unexpected_failure(feed: Feed) -> None:
|
||||
logger.exception("Feed %s: unexpected error during fetch", feed.feed_url)
|
||||
record_fetch_error(feed, "Unexpected error — see worker logs")
|
||||
feed.last_fetched_at = timezone.now()
|
||||
feed.claimed_at = None
|
||||
_schedule_next_fetch(feed)
|
||||
feed.save()
|
||||
|
||||
|
||||
@shared_task(
|
||||
bind=True, time_limit=FETCH_FEED_TIME_LIMIT, soft_time_limit=FETCH_FEED_SOFT_TIME_LIMIT
|
||||
)
|
||||
def fetch_feed(self, feed_id: int):
|
||||
try:
|
||||
feed = Feed.objects.get(pk=feed_id, is_active=True)
|
||||
except Feed.DoesNotExist:
|
||||
@@ -102,6 +154,20 @@ def fetch_feed(feed_id: int):
|
||||
|
||||
try:
|
||||
result = fetch_and_ingest(feed)
|
||||
except TRANSIENT_FETCH_ERRORS as exc:
|
||||
if self.request.retries < FETCH_MAX_RETRIES:
|
||||
delay = FETCH_RETRY_BASE_DELAY_SECONDS * 2**self.request.retries
|
||||
logger.warning(
|
||||
"Feed %s: transient error (%s); retry %d/%d in %ds",
|
||||
feed.feed_url,
|
||||
type(exc).__name__,
|
||||
self.request.retries + 1,
|
||||
FETCH_MAX_RETRIES,
|
||||
delay,
|
||||
)
|
||||
raise self.retry(exc=exc, countdown=delay, max_retries=FETCH_MAX_RETRIES)
|
||||
_record_unexpected_failure(feed)
|
||||
raise
|
||||
except Exception:
|
||||
# fetch_and_ingest already handles the expected failure modes
|
||||
# (network errors, SSRF-blocked targets, unparseable feeds) by
|
||||
@@ -113,12 +179,7 @@ def fetch_feed(feed_id: int):
|
||||
# CLAIM_STALE_MINUTES, and no record of what happened. Re-raised
|
||||
# afterward so Celery still marks the task FAILED (visible in
|
||||
# worker logs/monitoring) instead of swallowing it.
|
||||
logger.exception("Feed %s: unexpected error during fetch", feed.feed_url)
|
||||
record_fetch_error(feed, "Unexpected error — see worker logs")
|
||||
feed.last_fetched_at = timezone.now()
|
||||
feed.claimed_at = None
|
||||
_schedule_next_fetch(feed)
|
||||
feed.save()
|
||||
_record_unexpected_failure(feed)
|
||||
raise
|
||||
|
||||
feed.claimed_at = None
|
||||
@@ -127,6 +188,7 @@ def fetch_feed(feed_id: int):
|
||||
|
||||
changed_article_ids = result.new_article_ids + result.updated_article_ids
|
||||
if changed_article_ids:
|
||||
invalidate_feed_subscribers_sidebar_cache(feed_id)
|
||||
logger.info(
|
||||
"Feed %s: %d new article(s), %d updated",
|
||||
feed.feed_url,
|
||||
@@ -148,22 +210,36 @@ def fetch_feed(feed_id: int):
|
||||
}
|
||||
|
||||
|
||||
@shared_task
|
||||
def reset_feed_backoff(feed_id: int):
|
||||
"""Admin-triggered: clear a feed's error count and due-date backoff so
|
||||
it's retried at its normal configured interval instead of the backed-
|
||||
off one (next_fetch_at=None also makes it immediately due for the
|
||||
next poll_due_feeds tick, same as a feed that's never been fetched)."""
|
||||
def reset_backoff(feed_id: int) -> None:
|
||||
"""Clear a feed's error count and due-date backoff so it's retried at
|
||||
its normal configured interval instead of the backed-off one
|
||||
(next_fetch_at=None also makes it immediately due, same as a feed
|
||||
that's never been fetched). A plain synchronous function so a caller
|
||||
can finish the reset *before* dispatching the fetch — as an async task
|
||||
it could land after the fetch and undo its freshly scheduled
|
||||
next_fetch_at."""
|
||||
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0, next_fetch_at=None)
|
||||
|
||||
|
||||
@shared_task
|
||||
def import_opml_task(user_id: int, opml_content: str):
|
||||
def reset_feed_backoff(feed_id: int):
|
||||
reset_backoff(feed_id)
|
||||
|
||||
|
||||
@shared_task
|
||||
def import_opml_task(user_id: int, upload_key: str):
|
||||
"""Background path for large OPML imports — see OPML_ASYNC_THRESHOLD
|
||||
in apps.feeds.views. opml_content is passed as text rather than a
|
||||
file handle since Celery tasks must be JSON-serializable."""
|
||||
in apps.feeds.views. The document itself waits in the cache under
|
||||
upload_key (set by the view) rather than travelling in the broker
|
||||
message; it's removed once read, and expires on its own if this task
|
||||
never runs."""
|
||||
content = cache.get(upload_key)
|
||||
if content is None:
|
||||
logger.warning("OPML import %s expired or missing; nothing imported", upload_key)
|
||||
return None
|
||||
cache.delete(upload_key)
|
||||
try:
|
||||
user = get_user_model().objects.get(pk=user_id)
|
||||
except get_user_model().DoesNotExist:
|
||||
return None
|
||||
return import_opml(user, BytesIO(opml_content.encode("utf-8")))
|
||||
return import_opml(user, BytesIO(content))
|
||||
@@ -338,3 +338,88 @@ def test_ingest_falls_back_to_row_by_row_when_a_concurrent_insert_wins(feed):
|
||||
|
||||
assert len(result.new_article_ids) == 3
|
||||
assert Article.objects.filter(feed=feed).count() == 3
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_purged_article_is_not_resurrected_by_a_feed_that_still_lists_it(feed):
|
||||
"""#29: a feed still listing an entry the retention purge deleted must
|
||||
not bring it back as a brand-new, unread article."""
|
||||
from apps.articles.tasks import purge_old_articles
|
||||
from apps.core.models import SiteSettings
|
||||
|
||||
SiteSettings.objects.create(retention_days=30)
|
||||
responses.add(responses.GET, FEED_URL, body=_feed_xml(3), status=200)
|
||||
fetch_and_ingest(feed)
|
||||
Article.objects.filter(feed=feed, guid="guid-0").update(
|
||||
fetched_at=timezone.now() - dt.timedelta(days=90)
|
||||
)
|
||||
assert purge_old_articles() == 1
|
||||
assert not Article.objects.filter(feed=feed, guid="guid-0").exists()
|
||||
|
||||
result = fetch_and_ingest(feed)
|
||||
|
||||
assert result.new_article_ids == []
|
||||
assert not Article.objects.filter(feed=feed, guid="guid-0").exists()
|
||||
assert Article.objects.filter(feed=feed).count() == 2
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_entries_older_than_the_retention_window_are_not_ingested(feed):
|
||||
"""With retention on, a feed's long archive isn't ingested only to be
|
||||
purged (and tombstoned) the same night."""
|
||||
from apps.core.models import SiteSettings
|
||||
|
||||
SiteSettings.objects.create(retention_days=30)
|
||||
xml = (
|
||||
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
|
||||
"<item><title>Ancient</title><guid>old</guid>"
|
||||
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate></item>"
|
||||
"<item><title>Undated</title><guid>undated</guid></item>"
|
||||
"</channel></rss>"
|
||||
)
|
||||
responses.add(responses.GET, FEED_URL, body=xml, status=200)
|
||||
|
||||
result = fetch_and_ingest(feed)
|
||||
|
||||
assert len(result.new_article_ids) == 1
|
||||
assert Article.objects.filter(feed=feed).get().guid == "undated"
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_old_entries_are_ingested_normally_when_retention_is_off(feed):
|
||||
xml = (
|
||||
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
|
||||
"<item><title>Ancient</title><guid>old</guid>"
|
||||
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate></item>"
|
||||
"</channel></rss>"
|
||||
)
|
||||
responses.add(responses.GET, FEED_URL, body=xml, status=200)
|
||||
|
||||
assert len(fetch_and_ingest(feed).new_article_ids) == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_unsafe_urls_in_a_feed_are_not_stored(feed):
|
||||
"""R2-S1: a feed's channel link and entry links are untrusted — only
|
||||
absolute http(s) URLs may be persisted (they end up in href attributes)."""
|
||||
xml = (
|
||||
'<?xml version="1.0"?><rss version="2.0"><channel><title>T</title>'
|
||||
"<link>javascript:alert(1)</link>"
|
||||
"<item><title>Bad</title><guid>bad</guid><link>javascript:alert(2)</link></item>"
|
||||
"<item><title>Relative</title><guid>rel</guid><link>/posts/1</link></item>"
|
||||
"<item><title>Good</title><guid>good</guid><link>https://example.com/good</link></item>"
|
||||
"</channel></rss>"
|
||||
)
|
||||
responses.add(responses.GET, FEED_URL, body=xml, status=200)
|
||||
|
||||
fetch_and_ingest(feed)
|
||||
|
||||
assert feed.site_url == ""
|
||||
links = dict(Article.objects.filter(feed=feed).values_list("guid", "link"))
|
||||
assert links["bad"] == ""
|
||||
assert links["rel"] == "https://example.com/posts/1"
|
||||
assert links["good"] == "https://example.com/good"
|
||||
@@ -192,3 +192,95 @@ class TestExportOPML:
|
||||
exported = export_opml(alice)
|
||||
assert b"<opml" in exported
|
||||
assert b"<outline" not in exported
|
||||
|
||||
|
||||
class TestUnsafeOPMLUrls:
|
||||
def test_non_http_feed_urls_are_skipped_and_unsafe_html_urls_blanked(self):
|
||||
from io import BytesIO
|
||||
|
||||
from apps.feeds.opml import parse_opml
|
||||
|
||||
xml = b"""<?xml version="1.0"?><opml version="2.0"><head/><body>
|
||||
<outline text="Bad" xmlUrl="javascript:alert(1)"/>
|
||||
<outline text="FTP" xmlUrl="ftp://example.com/feed.xml"/>
|
||||
<outline text="Good" xmlUrl="https://example.com/feed.xml" htmlUrl="javascript:alert(2)"/>
|
||||
</body></opml>"""
|
||||
|
||||
entries = parse_opml(BytesIO(xml))
|
||||
|
||||
assert [e["feed_url"] for e in entries] == ["https://example.com/feed.xml"]
|
||||
assert entries[0]["site_url"] == ""
|
||||
|
||||
|
||||
class TestOPMLLimits:
|
||||
def _opml(self, body):
|
||||
from io import BytesIO
|
||||
|
||||
return BytesIO(
|
||||
f'<?xml version="1.0"?><opml version="2.0"><head/><body>{body}</body></opml>'.encode()
|
||||
)
|
||||
|
||||
def test_deep_nesting_is_rejected_without_recursing(self):
|
||||
from apps.feeds.opml import MAX_OPML_DEPTH, parse_opml
|
||||
|
||||
depth = MAX_OPML_DEPTH + 50
|
||||
body = '<outline text="x">' * depth + "</outline>" * depth
|
||||
|
||||
with pytest.raises(ValueError, match="nested too deeply"):
|
||||
parse_opml(self._opml(body))
|
||||
|
||||
def test_a_pathologically_deep_file_does_not_crash_the_parser(self):
|
||||
from apps.feeds.opml import parse_opml
|
||||
|
||||
depth = 50_000
|
||||
body = "<outline>" * depth + "</outline>" * depth
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
parse_opml(self._opml(body))
|
||||
|
||||
def test_too_many_outlines_is_rejected(self, monkeypatch):
|
||||
from apps.feeds import opml
|
||||
|
||||
monkeypatch.setattr(opml, "MAX_OPML_OUTLINES", 5)
|
||||
body = '<outline text="folder"/>' * 6
|
||||
|
||||
with pytest.raises(ValueError, match="too many entries"):
|
||||
opml.parse_opml(self._opml(body))
|
||||
|
||||
def test_too_many_feeds_is_rejected(self, monkeypatch):
|
||||
from apps.feeds import opml
|
||||
|
||||
monkeypatch.setattr(opml, "MAX_OPML_FEEDS", 2)
|
||||
body = "".join(f'<outline xmlUrl="https://e.example.com/{i}.xml"/>' for i in range(3))
|
||||
|
||||
with pytest.raises(ValueError, match="too many feeds"):
|
||||
opml.parse_opml(self._opml(body))
|
||||
|
||||
def test_a_zip_with_too_many_files_is_rejected(self, monkeypatch):
|
||||
import zipfile
|
||||
from io import BytesIO
|
||||
|
||||
from apps.feeds import opml
|
||||
|
||||
monkeypatch.setattr(opml, "MAX_ZIP_ENTRIES", 3)
|
||||
buffer = BytesIO()
|
||||
with zipfile.ZipFile(buffer, "w") as archive:
|
||||
for i in range(5):
|
||||
archive.writestr(f"f{i}.txt", "x")
|
||||
archive.writestr("subs.opml", "<opml/>")
|
||||
|
||||
with pytest.raises(ValueError, match="too many files"):
|
||||
opml.extract_opml_bytes(buffer.getvalue())
|
||||
|
||||
def test_a_feed_outlines_children_are_not_imported_as_separate_feeds(self):
|
||||
from apps.feeds.opml import parse_opml
|
||||
|
||||
body = (
|
||||
'<outline xmlUrl="https://a.example.com/f.xml">'
|
||||
'<outline xmlUrl="https://child.example.com/f.xml"/></outline>'
|
||||
'<outline xmlUrl="https://b.example.com/f.xml"/>'
|
||||
)
|
||||
|
||||
urls = [e["feed_url"] for e in parse_opml(self._opml(body))]
|
||||
|
||||
assert urls == ["https://a.example.com/f.xml", "https://b.example.com/f.xml"]
|
||||
@@ -136,3 +136,74 @@ class TestSafeGet:
|
||||
)
|
||||
response = safe_get("https://example.com/feed.xml", headers={}, timeout=5)
|
||||
assert response.headers.get("ETag") == "abc123"
|
||||
|
||||
|
||||
class TestConnectionPinning:
|
||||
"""R2-S2: the address that's validated is the address that's connected
|
||||
to — a hostname that resolves differently at connect time can't slip an
|
||||
internal address past a pre-check."""
|
||||
|
||||
def test_a_rebinding_dns_answer_at_connect_time_is_blocked(self, monkeypatch):
|
||||
import socket
|
||||
|
||||
calls = []
|
||||
|
||||
def rebinding_getaddrinfo(host, *args, **kwargs):
|
||||
calls.append(host)
|
||||
ip = "93.184.216.34" if len(calls) == 1 else "127.0.0.1"
|
||||
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, 0))]
|
||||
|
||||
monkeypatch.setattr(socket, "getaddrinfo", rebinding_getaddrinfo)
|
||||
|
||||
# The up-front check sees the public answer...
|
||||
validate_fetch_url("http://rebind.example.com/feed.xml")
|
||||
# ...but the connection's own lookup gets the internal one and must refuse.
|
||||
with pytest.raises(SSRFBlocked, match="non-public"):
|
||||
safe_get("http://rebind.example.com/feed.xml", headers={}, timeout=2)
|
||||
|
||||
def test_connects_to_the_validated_address_and_keeps_the_original_host_header(
|
||||
self, monkeypatch
|
||||
):
|
||||
import socket
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
from apps.feeds.services import ssrf_guard
|
||||
|
||||
seen = {}
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
seen["host"] = self.headers["Host"]
|
||||
body = b"<rss/>"
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
server = HTTPServer(("127.0.0.1", 0), Handler)
|
||||
port = server.server_address[1]
|
||||
threading.Thread(target=server.handle_request, daemon=True).start()
|
||||
|
||||
real_getaddrinfo = socket.getaddrinfo
|
||||
monkeypatch.setattr(
|
||||
socket,
|
||||
"getaddrinfo",
|
||||
lambda host, *a, **k: (
|
||||
[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", port))]
|
||||
if host == "feed.test"
|
||||
else real_getaddrinfo(host, *a, **k)
|
||||
),
|
||||
)
|
||||
# Allow loopback for this test only, standing in for a public address.
|
||||
monkeypatch.setattr(ssrf_guard, "_is_allowed_ip", lambda ip: True)
|
||||
|
||||
response = safe_get(f"http://feed.test:{port}/feed.xml", headers={}, timeout=5)
|
||||
server.server_close()
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.content == b"<rss/>"
|
||||
assert seen["host"] == f"feed.test:{port}"
|
||||
@@ -173,9 +173,9 @@ def test_import_opml_dispatches_background_task_above_threshold(client, alice, m
|
||||
|
||||
dispatched = {}
|
||||
|
||||
def fake_delay(user_id, content):
|
||||
def fake_delay(user_id, upload_key):
|
||||
dispatched["user_id"] = user_id
|
||||
dispatched["content"] = content
|
||||
dispatched["upload_key"] = upload_key
|
||||
|
||||
from . import views as feeds_views
|
||||
|
||||
@@ -187,6 +187,11 @@ def test_import_opml_dispatches_background_task_above_threshold(client, alice, m
|
||||
|
||||
assert response.status_code == 302
|
||||
assert dispatched["user_id"] == alice.id
|
||||
# The task gets a reference to the document, not the document itself.
|
||||
from django.core.cache import cache
|
||||
|
||||
assert dispatched["upload_key"].startswith("opml-import:")
|
||||
assert cache.get(dispatched["upload_key"]) == SIMPLE_OPML
|
||||
# The background task was dispatched instead of a synchronous import.
|
||||
assert not Subscription.objects.filter(user=alice).exists()
|
||||
|
||||
@@ -204,9 +209,51 @@ def test_export_opml_view_returns_downloadable_file(client, alice, alices_subscr
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_import_opml_task_runs_import(alice):
|
||||
from django.core.cache import cache
|
||||
|
||||
from .tasks import import_opml_task
|
||||
|
||||
result = import_opml_task(alice.id, SIMPLE_OPML.decode("utf-8"))
|
||||
cache.set("opml-import:test", SIMPLE_OPML, timeout=60)
|
||||
|
||||
result = import_opml_task(alice.id, "opml-import:test")
|
||||
|
||||
assert result["created"] == 2
|
||||
assert Subscription.objects.filter(user=alice).count() == 2
|
||||
assert cache.get("opml-import:test") is None, "the stored upload is removed once read"
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_import_opml_task_does_nothing_if_the_upload_expired(alice):
|
||||
from .tasks import import_opml_task
|
||||
|
||||
assert import_opml_task(alice.id, "opml-import:gone") is None
|
||||
assert not Subscription.objects.filter(user=alice).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_import_opml_rejects_an_oversized_upload(client, alice, monkeypatch):
|
||||
from . import forms
|
||||
|
||||
monkeypatch.setattr(forms, "MAX_UPLOAD_BYTES", 100)
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
|
||||
upload = SimpleUploadedFile("big.opml", SIMPLE_OPML, content_type="text/x-opml")
|
||||
response = client.post(reverse("feeds:import_opml"), {"opml_file": upload})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "too large" in response.content.decode()
|
||||
assert not Subscription.objects.filter(user=alice).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_import_opml_reports_a_limit_error_instead_of_a_generic_one(client, alice, monkeypatch):
|
||||
from . import opml
|
||||
|
||||
monkeypatch.setattr(opml, "MAX_OPML_FEEDS", 1)
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
|
||||
upload = SimpleUploadedFile("subs.opml", SIMPLE_OPML, content_type="text/x-opml")
|
||||
response = client.post(reverse("feeds:import_opml"), {"opml_file": upload}, follow=True)
|
||||
|
||||
assert "too many feeds" in response.content.decode()
|
||||
assert not Subscription.objects.filter(user=alice).exists()
|
||||
@@ -3,6 +3,7 @@ from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import responses
|
||||
from django.urls import reverse
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.articles.models import Article
|
||||
@@ -120,6 +121,25 @@ def test_fetch_feed_task_ingests_and_saves_feed_state():
|
||||
assert Article.objects.filter(feed=feed, guid="guid-1").exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_fetch_feed_drops_subscribers_cached_sidebar_counts(django_user_model):
|
||||
"""New articles must show in the sidebar at the next refresh, not after
|
||||
the counts cache's TTL."""
|
||||
from apps.articles.queries import unread_counts_by_feed
|
||||
from apps.feeds.models import Subscription
|
||||
|
||||
user = django_user_model.objects.create_user("reader", password="pw-reader-123")
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
Subscription.objects.create(user=user, feed=feed)
|
||||
assert unread_counts_by_feed(user) == {} # now cached
|
||||
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
||||
|
||||
fetch_feed(feed.id)
|
||||
|
||||
assert unread_counts_by_feed(user) == {feed.id: 1}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_fetch_feed_task_skips_inactive_feed():
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", is_active=False)
|
||||
@@ -228,3 +248,177 @@ def test_fetch_feed_dispatches_rule_evaluation_in_bounded_chunks(monkeypatch):
|
||||
|
||||
chunk_sizes = [len(call.args[1]) for call in mock_delay.call_args_list]
|
||||
assert chunk_sizes == [2, 2, 1]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_enqueue_fetch_claims_the_feed_and_queues_once():
|
||||
from apps.feeds.tasks import enqueue_fetch
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||
first = enqueue_fetch(feed.id)
|
||||
second = enqueue_fetch(feed.id)
|
||||
|
||||
assert (first, second) == (True, False)
|
||||
mock_delay.assert_called_once_with(feed.id)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_the_poller_does_not_dispatch_a_feed_a_direct_enqueue_already_claimed():
|
||||
"""T1: a just-added feed (next_fetch_at NULL, i.e. due) is claimed by
|
||||
the add-feed path, so the periodic poller must not fetch it too."""
|
||||
from apps.feeds.tasks import enqueue_fetch
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||
enqueue_fetch(feed.id)
|
||||
dispatched = poll_due_feeds()
|
||||
|
||||
assert dispatched == 0
|
||||
assert mock_delay.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_enqueue_fetch_releases_the_claim_if_queueing_fails():
|
||||
from apps.feeds.tasks import enqueue_fetch
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
|
||||
with pytest.raises(ConnectionError):
|
||||
enqueue_fetch(feed.id)
|
||||
|
||||
feed.refresh_from_db()
|
||||
assert feed.claimed_at is None
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_poller_survives_a_broker_failure_and_leaves_feeds_unclaimed():
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
|
||||
assert poll_due_feeds() == 0
|
||||
|
||||
feed.refresh_from_db()
|
||||
assert feed.claimed_at is None
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_add_feed_claims_its_initial_fetch(client):
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
get_user_model().objects.create_user("alice", password="pw-alice-123")
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||
client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"})
|
||||
poll_due_feeds()
|
||||
|
||||
assert mock_delay.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_add_feed_still_subscribes_when_the_broker_is_down(client):
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
from apps.feeds.models import Subscription
|
||||
|
||||
user = get_user_model().objects.create_user("alice", password="pw-alice-123")
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay", side_effect=ConnectionError("broker down")):
|
||||
response = client.post(
|
||||
reverse("feeds:add_feed"), {"feed_url": "https://example.com/new.xml"}
|
||||
)
|
||||
|
||||
assert response.status_code == 302
|
||||
assert Subscription.objects.filter(
|
||||
user=user, feed__feed_url="https://example.com/new.xml"
|
||||
).exists()
|
||||
|
||||
|
||||
class TestTransientFailureRetries:
|
||||
"""R2-T5: a locked database or dropped connection is retried promptly
|
||||
with bounded backoff; everything else keeps the record-and-back-off
|
||||
behavior."""
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_a_transient_database_error_is_retried_without_penalizing_the_feed(self):
|
||||
from celery.exceptions import Retry
|
||||
from django.db import OperationalError
|
||||
|
||||
feed = Feed.objects.create(
|
||||
feed_url="https://example.com/feed.xml", claimed_at=timezone.now()
|
||||
)
|
||||
|
||||
with (
|
||||
patch("apps.feeds.tasks.fetch_and_ingest", side_effect=OperationalError("locked")),
|
||||
patch.object(fetch_feed, "retry", side_effect=Retry()) as mock_retry,
|
||||
):
|
||||
with pytest.raises(Retry):
|
||||
fetch_feed(feed.id)
|
||||
|
||||
assert mock_retry.call_args.kwargs["countdown"] == 30
|
||||
feed.refresh_from_db()
|
||||
assert feed.fetch_error_count == 0, "the feed wasn't at fault"
|
||||
assert feed.claimed_at is not None, "claim is held so the poller can't double-fetch"
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_backoff_doubles_each_attempt(self):
|
||||
from celery.exceptions import Retry
|
||||
from django.db import InterfaceError
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
delays = []
|
||||
for attempt in range(3):
|
||||
fetch_feed.push_request(retries=attempt)
|
||||
try:
|
||||
with (
|
||||
patch("apps.feeds.tasks.fetch_and_ingest", side_effect=InterfaceError("gone")),
|
||||
patch.object(fetch_feed, "retry", side_effect=Retry()) as mock_retry,
|
||||
):
|
||||
with pytest.raises(Retry):
|
||||
fetch_feed(feed.id)
|
||||
delays.append(mock_retry.call_args.kwargs["countdown"])
|
||||
finally:
|
||||
fetch_feed.pop_request()
|
||||
|
||||
assert delays == [30, 60, 120]
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_after_the_retries_are_spent_it_is_recorded_as_a_normal_failure(self):
|
||||
from django.db import OperationalError
|
||||
|
||||
from apps.feeds.tasks import FETCH_MAX_RETRIES
|
||||
|
||||
feed = Feed.objects.create(
|
||||
feed_url="https://example.com/feed.xml", claimed_at=timezone.now()
|
||||
)
|
||||
|
||||
fetch_feed.push_request(retries=FETCH_MAX_RETRIES)
|
||||
try:
|
||||
with patch("apps.feeds.tasks.fetch_and_ingest", side_effect=OperationalError("locked")):
|
||||
with pytest.raises(OperationalError):
|
||||
fetch_feed(feed.id)
|
||||
finally:
|
||||
fetch_feed.pop_request()
|
||||
|
||||
feed.refresh_from_db()
|
||||
assert feed.fetch_error_count == 1
|
||||
assert feed.claimed_at is None
|
||||
assert feed.next_fetch_at is not None
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_other_exceptions_are_not_retried(self):
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
|
||||
with (
|
||||
patch("apps.feeds.tasks.fetch_and_ingest", side_effect=RuntimeError("parser bug")),
|
||||
patch.object(fetch_feed, "retry") as mock_retry,
|
||||
):
|
||||
with pytest.raises(RuntimeError):
|
||||
fetch_feed(feed.id)
|
||||
|
||||
mock_retry.assert_not_called()
|
||||
@@ -117,3 +117,14 @@ def test_add_feed_view_rejects_duplicate_subscription(client, two_users):
|
||||
|
||||
assert response.status_code == 302
|
||||
assert Subscription.objects.filter(user=alice, feed=feed).count() == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.parametrize("url", ["ftp://example.com/feed.xml", "javascript:alert(1)"])
|
||||
def test_add_feed_rejects_non_http_urls(client, two_users, url):
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
|
||||
response = client.post(reverse("feeds:add_feed"), {"feed_url": url})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert not Feed.objects.filter(feed_url=url).exists()
|
||||
+30
-5
@@ -1,7 +1,10 @@
|
||||
import logging
|
||||
import uuid
|
||||
from io import BytesIO
|
||||
|
||||
from django.contrib import messages
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.core.cache import cache
|
||||
from django.http import HttpResponse
|
||||
from django.shortcuts import get_object_or_404, redirect, render
|
||||
from django.views.decorators.http import require_POST
|
||||
@@ -11,10 +14,13 @@ from apps.articles.models import SubscriptionTag, Tag
|
||||
|
||||
from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm
|
||||
from .models import Feed, Subscription
|
||||
from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml
|
||||
from .tasks import fetch_feed, import_opml_task
|
||||
from .opml import MAX_UPLOAD_BYTES, export_opml, extract_opml_bytes, import_opml, parse_opml
|
||||
from .tasks import enqueue_fetch, import_opml_task
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OPML_ASYNC_THRESHOLD = 50
|
||||
OPML_UPLOAD_TTL_SECONDS = 60 * 60
|
||||
|
||||
|
||||
@login_required
|
||||
@@ -42,7 +48,13 @@ def add_feed(request):
|
||||
# slow or large feed would otherwise tie up a Gunicorn worker for
|
||||
# the duration of the request; fetch_feed handles the save/error
|
||||
# bookkeeping the same way the periodic poll's dispatch does.
|
||||
fetch_feed.delay(feed.pk)
|
||||
try:
|
||||
enqueue_fetch(feed.pk)
|
||||
except Exception:
|
||||
# The subscription itself is saved; with the broker down the
|
||||
# periodic poller picks the (never-fetched) feed up once it's
|
||||
# back, so this isn't worth failing the request over.
|
||||
logger.exception("Could not queue the initial fetch for feed %s", feed.pk)
|
||||
messages.success(
|
||||
request,
|
||||
f"Subscribed to {feed_url}. Fetching its articles now — "
|
||||
@@ -140,16 +152,29 @@ def unsubscribe(request, subscription_id):
|
||||
def import_opml_view(request):
|
||||
form = OPMLImportForm(request.POST or None, request.FILES or None)
|
||||
if request.method == "POST" and form.is_valid():
|
||||
uploaded = form.cleaned_data["opml_file"].read()
|
||||
uploaded = form.cleaned_data["opml_file"].read(MAX_UPLOAD_BYTES + 1)
|
||||
try:
|
||||
content = extract_opml_bytes(uploaded)
|
||||
entry_count = len(parse_opml(BytesIO(content)))
|
||||
except ValueError as exc:
|
||||
# Our own limit/format errors carry a user-presentable message.
|
||||
messages.error(request, str(exc))
|
||||
return redirect("feeds:import_opml")
|
||||
except Exception:
|
||||
messages.error(request, "That file doesn't look like a valid OPML export.")
|
||||
return redirect("feeds:import_opml")
|
||||
|
||||
if entry_count > OPML_ASYNC_THRESHOLD:
|
||||
import_opml_task.delay(request.user.id, content.decode("utf-8"))
|
||||
# Hand the worker a reference, not the document: a multi-MB
|
||||
# file doesn't belong inside a broker message.
|
||||
upload_key = f"opml-import:{uuid.uuid4().hex}"
|
||||
try:
|
||||
cache.set(upload_key, content, timeout=OPML_UPLOAD_TTL_SECONDS)
|
||||
import_opml_task.delay(request.user.id, upload_key)
|
||||
except Exception:
|
||||
logger.exception("Could not queue the OPML import")
|
||||
messages.error(request, "Couldn't queue that import right now — try again shortly.")
|
||||
return redirect("feeds:import_opml")
|
||||
messages.success(request, f"Importing {entry_count} feeds in the background.")
|
||||
else:
|
||||
result = import_opml(request.user, BytesIO(content))
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import hashlib
|
||||
import hmac
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
AUTH_MARKER_HEADER = "X-Evection-Auth"
|
||||
ANONYMOUS_MARKER = "anon"
|
||||
|
||||
|
||||
def auth_marker(user) -> str:
|
||||
"""An opaque, stable identifier for who is logged in — "anon" if no
|
||||
one. Derived with an HMAC of the user's pk under SECRET_KEY, so it's
|
||||
constant for one user but reveals neither the pk nor anything
|
||||
guessable."""
|
||||
if user is None or not user.is_authenticated:
|
||||
return ANONYMOUS_MARKER
|
||||
digest = hmac.new(
|
||||
settings.SECRET_KEY.encode(), f"evection-sw-auth:{user.pk}".encode(), hashlib.sha256
|
||||
).hexdigest()
|
||||
return digest[:16]
|
||||
|
||||
|
||||
class AuthMarkerMiddleware:
|
||||
"""Stamps every HTML response with who it was rendered for. The service
|
||||
worker (templates/pwa/sw.js) caches authenticated pages for offline
|
||||
reading in a single cache; it compares this marker on every navigation
|
||||
and purges that cache whenever it changes — session expiry (the login
|
||||
page comes back as "anon"), logging in, switching accounts — so one
|
||||
session's pages can't be served to the next (R2-S3)."""
|
||||
|
||||
def __init__(self, get_response):
|
||||
self.get_response = get_response
|
||||
|
||||
def __call__(self, request):
|
||||
response = self.get_response(request)
|
||||
if response.get("Content-Type", "").startswith("text/html"):
|
||||
response[AUTH_MARKER_HEADER] = auth_marker(getattr(request, "user", None))
|
||||
return response
|
||||
@@ -66,3 +66,42 @@ def test_base_template_links_manifest_and_registers_service_worker(client, djang
|
||||
assert 'rel="manifest"' in content
|
||||
assert "register-sw.js" in content
|
||||
assert "apple-touch-icon" in content
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_html_responses_are_stamped_with_who_they_were_rendered_for(client, django_user_model):
|
||||
"""R2-S3: the service worker purges its page cache whenever this marker
|
||||
changes, so it has to be stable per user, different between users,
|
||||
and "anon" when logged out."""
|
||||
django_user_model.objects.create_user("alice", password="pw-alice-123")
|
||||
django_user_model.objects.create_user("bob", password="pw-bob-123")
|
||||
|
||||
anon = client.get(reverse("accounts:login"))["X-Evection-Auth"]
|
||||
assert anon == "anon"
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
alice_1 = client.get(reverse("reader:stream"))["X-Evection-Auth"]
|
||||
alice_2 = client.get(reverse("reader:stream"))["X-Evection-Auth"]
|
||||
client.logout()
|
||||
client.login(username="bob", password="pw-bob-123")
|
||||
bob = client.get(reverse("reader:stream"))["X-Evection-Auth"]
|
||||
|
||||
assert alice_1 == alice_2
|
||||
assert len({anon, alice_1, bob}) == 3
|
||||
assert "alice" not in alice_1 and not alice_1.isdigit()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_non_html_responses_are_not_stamped(client, django_user_model):
|
||||
django_user_model.objects.create_user("alice", password="pw-alice-123")
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
|
||||
assert "X-Evection-Auth" not in client.get(reverse("core:health"))
|
||||
|
||||
|
||||
def test_service_worker_purges_its_page_cache_when_the_auth_marker_changes(client):
|
||||
body = client.get(reverse("pwa:service_worker")).content.decode()
|
||||
|
||||
assert "X-Evection-Auth" in body
|
||||
assert "caches.delete(PAGES_CACHE)" in body
|
||||
assert "previous !== marker" in body
|
||||
+179
-51
@@ -188,8 +188,7 @@ def test_sidebar_shows_a_tags_total_unread_count(client, alice):
|
||||
response = client.get(reverse("reader:stream"))
|
||||
|
||||
# The only "7" on the page — the individual feeds show 3 and 4.
|
||||
content = response.content.decode()
|
||||
assert '<span class="unread-count">7</span>' in content
|
||||
assert sorted(_visible_counts(response.content.decode())) == ["3", "4", "7"]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -226,35 +225,27 @@ def test_article_rows_carry_their_utc_instant_for_client_side_date_grouping(clie
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_check_new_articles_reports_nothing_below_the_given_cursor(client, alice):
|
||||
def test_pages_render_the_banner_placeholder_and_baseline_for_any_scope(client, alice):
|
||||
"""The banner lives in the base layout with the page's baseline id
|
||||
rendered in, so it works under any feed/tag selection (and on pages
|
||||
that aren't the stream), not only the unscoped list."""
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
existing = Article.objects.create(feed=feed, guid="existing", title="Existing")
|
||||
article = Article.objects.create(feed=feed, guid="g1", title="Hello")
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
response = client.get(reverse("reader:check_new"), {"filter": "all", "since": existing.pk})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.content.decode().strip() == ""
|
||||
for query in ({"filter": "all"}, {"feed": feed.pk}):
|
||||
content = client.get(reverse("reader:stream"), query).content.decode()
|
||||
assert 'class="new-articles-banner"' in content
|
||||
assert f'data-since="{article.pk}"' in content
|
||||
other_page = client.get(reverse("feeds:subscription_list")).content.decode()
|
||||
assert 'class="new-articles-banner"' in other_page
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_check_new_articles_reports_articles_created_after_the_cursor(client, alice):
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
existing = Article.objects.create(feed=feed, guid="existing", title="Existing")
|
||||
Article.objects.create(feed=feed, guid="new-1", title="New one")
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
response = client.get(reverse("reader:check_new"), {"filter": "all", "since": existing.pk})
|
||||
|
||||
content = response.content.decode()
|
||||
assert "1 new article" in content
|
||||
assert "click to show" in content
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_check_new_articles_respects_the_active_tag_scope(client, alice):
|
||||
def test_new_since_counts_new_articles_outside_the_page_s_own_feed(client, alice):
|
||||
"""The banner for a page showing feed A still reports a new article in
|
||||
feed B — the poll isn't scoped to the selected feed or tag."""
|
||||
from apps.articles.models import SubscriptionTag, Tag
|
||||
|
||||
feed_a = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
||||
@@ -263,38 +254,93 @@ def test_check_new_articles_respects_the_active_tag_scope(client, alice):
|
||||
Subscription.objects.create(user=alice, feed=feed_b)
|
||||
tag = Tag.objects.create(user=alice, name="work")
|
||||
SubscriptionTag.objects.create(user=alice, subscription=sub_a, tag=tag)
|
||||
|
||||
cursor = Article.objects.create(feed=feed_a, guid="cursor", title="Cursor").pk
|
||||
# New in feed_b, which isn't tagged "work" — shouldn't count for this scope.
|
||||
Article.objects.create(feed=feed_b, guid="untagged-new", title="Untagged new")
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
response = client.get(
|
||||
reverse("reader:check_new"), {"filter": "all", "tag": tag.pk, "since": cursor}
|
||||
)
|
||||
body = client.get(reverse("reader:new_since"), {"since": cursor}).json()
|
||||
|
||||
assert response.content.decode().strip() == ""
|
||||
|
||||
# New in feed_a (tagged "work") — should count.
|
||||
Article.objects.create(feed=feed_a, guid="tagged-new", title="Tagged new")
|
||||
response = client.get(
|
||||
reverse("reader:check_new"), {"filter": "all", "tag": tag.pk, "since": cursor}
|
||||
)
|
||||
assert "1 new article" in response.content.decode()
|
||||
assert body["new_count"] == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_stream_exposes_the_latest_article_id_and_scope_for_the_banner(client, alice):
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
article = Article.objects.create(feed=feed, guid="g1", title="Hello")
|
||||
def test_new_since_returns_current_sidebar_counts(client, alice):
|
||||
from apps.articles.models import SubscriptionTag, Tag
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
||||
sub = Subscription.objects.create(user=alice, feed=feed)
|
||||
tag = Tag.objects.create(user=alice, name="work")
|
||||
SubscriptionTag.objects.create(user=alice, subscription=sub, tag=tag)
|
||||
for i in range(3):
|
||||
Article.objects.create(feed=feed, guid=f"g{i}", title=f"T{i}")
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
response = client.get(reverse("reader:stream"), {"filter": "all"})
|
||||
content = response.content.decode()
|
||||
counts = client.get(reverse("reader:new_since"), {"since": 0}).json()["counts"]
|
||||
|
||||
assert f"since={article.pk}" in content
|
||||
assert "filter=all" in content
|
||||
assert counts == {"feeds": {str(feed.pk): 3}, "tags": {str(tag.pk): 3}, "starred": 0}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.parametrize(
|
||||
"url_name, uses_article",
|
||||
[("reader:toggle_read", True), ("reader:toggle_star", True), ("reader:article_inline", True)],
|
||||
)
|
||||
def test_actions_that_change_counts_tell_the_page_to_refresh_them(
|
||||
client, alice, url_name, uses_article
|
||||
):
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
article = Article.objects.create(feed=feed, guid="a-1", title="Hello")
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
url = reverse(url_name, args=[article.pk])
|
||||
response = client.get(url) if url_name == "reader:article_inline" else client.post(url)
|
||||
|
||||
assert response["HX-Trigger"] == "countsChanged"
|
||||
|
||||
|
||||
def _visible_counts(content):
|
||||
"""Texts of the sidebar count badges that are actually shown."""
|
||||
return re.findall(
|
||||
r'<span class="unread-count" data-(?:feed|tag)-id="\d+">(\d+)</span>', content
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_sidebar_count_is_fresh_right_after_new_articles_are_ingested(client, alice):
|
||||
"""Ingestion drops the cached counts, so the next render (and the next
|
||||
poll) shows new articles at once instead of after the cache TTL."""
|
||||
from apps.core.cache_utils import invalidate_feed_subscribers_sidebar_cache
|
||||
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
Article.objects.create(feed=feed, guid="a-1", title="One")
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
assert _visible_counts(client.get(reverse("reader:stream")).content.decode()) == ["1"]
|
||||
|
||||
Article.objects.create(feed=feed, guid="a-2", title="Two")
|
||||
assert _visible_counts(client.get(reverse("reader:stream")).content.decode()) == ["1"]
|
||||
invalidate_feed_subscribers_sidebar_cache(feed.pk)
|
||||
assert _visible_counts(client.get(reverse("reader:stream")).content.decode()) == ["2"]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_new_subscription_shows_its_count_without_waiting_for_the_cache(client, alice):
|
||||
feed_a = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
||||
feed_b = Feed.objects.create(feed_url="https://b.example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed_a)
|
||||
Article.objects.create(feed=feed_a, guid="a", title="A")
|
||||
Article.objects.create(feed=feed_b, guid="b", title="B")
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
client.get(reverse("reader:stream"))
|
||||
Subscription.objects.create(user=alice, feed=feed_b)
|
||||
|
||||
assert sorted(_visible_counts(client.get(reverse("reader:stream")).content.decode())) == [
|
||||
"1",
|
||||
"1",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -310,12 +356,12 @@ def test_sidebar_unread_count_updates_immediately_after_reading_an_article(clien
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
|
||||
response = client.get(reverse("reader:stream"))
|
||||
assert '<span class="unread-count">1</span>' in response.content.decode()
|
||||
assert _visible_counts(response.content.decode()) == ["1"]
|
||||
|
||||
client.get(reverse("reader:article_detail", args=[article.pk]))
|
||||
|
||||
response = client.get(reverse("reader:stream"))
|
||||
assert "unread-count" not in response.content.decode()
|
||||
assert _visible_counts(response.content.decode()) == []
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -327,12 +373,12 @@ def test_sidebar_unread_count_updates_immediately_after_toggle_read(client, alic
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
response = client.get(reverse("reader:stream"))
|
||||
assert "unread-count" not in response.content.decode()
|
||||
assert _visible_counts(response.content.decode()) == []
|
||||
|
||||
client.post(reverse("reader:toggle_read", args=[article.pk]))
|
||||
|
||||
response = client.get(reverse("reader:stream"))
|
||||
assert '<span class="unread-count">1</span>' in response.content.decode()
|
||||
assert _visible_counts(response.content.decode()) == ["1"]
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -343,12 +389,12 @@ def test_sidebar_unread_count_updates_immediately_after_mark_all_read(client, al
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
response = client.get(reverse("reader:stream"))
|
||||
assert '<span class="unread-count">1</span>' in response.content.decode()
|
||||
assert _visible_counts(response.content.decode()) == ["1"]
|
||||
|
||||
client.post(reverse("reader:mark_all_read"))
|
||||
|
||||
response = client.get(reverse("reader:stream"))
|
||||
assert "unread-count" not in response.content.decode()
|
||||
assert _visible_counts(response.content.decode()) == []
|
||||
|
||||
|
||||
def _stream_article_ids(client, query):
|
||||
@@ -503,3 +549,85 @@ def test_article_with_no_publish_date_shows_when_it_was_fetched(client, alice):
|
||||
|
||||
article.refresh_from_db()
|
||||
assert article.published_at is None
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_new_since_requires_login(client):
|
||||
response = client.get(reverse("reader:new_since"))
|
||||
assert response.status_code == 302
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_new_since_without_a_cursor_returns_the_baseline_id(client, alice):
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
article = Article.objects.create(feed=feed, guid="g1", title="Hello")
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
body = client.get(reverse("reader:new_since")).json()
|
||||
|
||||
assert body["latest_id"] == article.pk
|
||||
assert body["new_count"] == 0
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_new_since_counts_only_new_unread_visible_articles_from_my_feeds(client, alice, bob):
|
||||
mine = Feed.objects.create(feed_url="https://mine.example.com/feed.xml")
|
||||
theirs = Feed.objects.create(feed_url="https://theirs.example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=mine)
|
||||
Subscription.objects.create(user=bob, feed=theirs)
|
||||
cursor = Article.objects.create(feed=mine, guid="cursor", title="Cursor").pk
|
||||
|
||||
new_unread = Article.objects.create(feed=mine, guid="n1", title="New unread")
|
||||
already_read = Article.objects.create(feed=mine, guid="n2", title="New but read")
|
||||
hidden = Article.objects.create(feed=mine, guid="n3", title="New but hidden")
|
||||
Article.objects.create(feed=theirs, guid="n4", title="Someone else's feed")
|
||||
UserArticleState.objects.create(user=alice, article=already_read, is_read=True)
|
||||
UserArticleState.objects.create(user=alice, article=hidden, is_hidden=True)
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
body = client.get(reverse("reader:new_since"), {"since": cursor}).json()
|
||||
|
||||
assert body["new_count"] == 1
|
||||
assert new_unread.pk > cursor
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_authenticated_pages_load_the_live_updates_script_with_its_endpoint(client, alice):
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
content = client.get(reverse("reader:stream")).content.decode()
|
||||
|
||||
assert "js/live-updates" in content
|
||||
assert f'data-url="{reverse("reader:new_since")}"' in content
|
||||
client.logout()
|
||||
assert "live-updates" not in client.get(reverse("accounts:login")).content.decode()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_unsafe_stored_links_are_not_rendered_as_hrefs(client, alice, django_user_model):
|
||||
"""R2-S1 render boundary: even if a bad URL is already in the database
|
||||
(stored before validation existed), it must not become a clickable
|
||||
javascript: link."""
|
||||
feed = Feed.objects.create(
|
||||
feed_url="https://example.com/feed.xml", site_url="javascript:alert(1)"
|
||||
)
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
article = Article.objects.create(
|
||||
feed=feed, guid="g1", title="Hello", link="javascript:alert(2)"
|
||||
)
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
detail = client.get(reverse("reader:article_detail", args=[article.pk])).content.decode()
|
||||
assert "javascript:" not in detail
|
||||
assert "Original" not in detail
|
||||
|
||||
article.link = "https://example.com/real"
|
||||
article.save()
|
||||
detail = client.get(reverse("reader:article_detail", args=[article.pk])).content.decode()
|
||||
assert 'href="https://example.com/real"' in detail
|
||||
|
||||
django_user_model.objects.create_user("admin", password="pw-admin-123", is_staff=True)
|
||||
client.login(username="admin", password="pw-admin-123")
|
||||
Feed.objects.filter(pk=feed.pk).update(fetch_error_count=2, last_fetch_status="error")
|
||||
dashboard = client.get(reverse("core:dashboard")).content.decode()
|
||||
assert "javascript:" not in dashboard
|
||||
+1
-1
@@ -6,7 +6,7 @@ app_name = "reader"
|
||||
|
||||
urlpatterns = [
|
||||
path("", views.stream, name="stream"),
|
||||
path("check-new/", views.check_new_articles, name="check_new"),
|
||||
path("new-since/", views.new_since, name="new_since"),
|
||||
path("mark-all-read/", views.mark_all_read, name="mark_all_read"),
|
||||
path("articles/<int:article_id>/", views.article_detail, name="article_detail"),
|
||||
path(
|
||||
|
||||
+32
-20
@@ -5,7 +5,7 @@ from urllib.parse import urlencode
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.db.models import Max, Q
|
||||
from django.db.models.functions import Coalesce
|
||||
from django.http import Http404
|
||||
from django.http import Http404, JsonResponse
|
||||
from django.shortcuts import get_object_or_404, redirect, render
|
||||
from django.urls import reverse
|
||||
from django.utils import timezone
|
||||
@@ -22,6 +22,7 @@ from apps.articles.models import (
|
||||
)
|
||||
from apps.articles.queries import with_user_state
|
||||
from apps.core.cache_utils import invalidate_sidebar_unread_cache
|
||||
from apps.core.context_processors import sidebar_counts
|
||||
from apps.feeds.models import Feed
|
||||
|
||||
FILTER_CHOICES = [
|
||||
@@ -137,7 +138,7 @@ def _resolve_active_filter(request):
|
||||
|
||||
def _filtered_queryset(request, active_filter):
|
||||
"""The stream's fully-scoped queryset (filter tab + tag/feed scope),
|
||||
unpaginated — shared by stream() and check_new_articles() so the "any
|
||||
unpaginated — used by stream() so the "any
|
||||
new articles?" check always matches exactly what the stream itself
|
||||
would show."""
|
||||
queryset = _base_queryset(request.user)
|
||||
@@ -233,29 +234,33 @@ def stream(request):
|
||||
"has_older_page": has_older_page and bool(articles),
|
||||
"newer_qs": newer_qs,
|
||||
"older_qs": older_qs,
|
||||
# Lets the "new articles" banner ask, on this exact same scope,
|
||||
# whether anything with a higher id has shown up since this page
|
||||
# was rendered — id (not published_at) because it's monotonic
|
||||
# regardless of how a feed backdates an entry's published date.
|
||||
"latest_article_id": queryset.aggregate(Max("id"))["id__max"] or 0,
|
||||
"new_check_qs": urlencode(scope_params),
|
||||
"scope_qs": urlencode(scope_params),
|
||||
}
|
||||
return render(request, "reader/stream.html", context)
|
||||
|
||||
|
||||
@login_required
|
||||
def check_new_articles(request):
|
||||
active_filter = _resolve_active_filter(request)
|
||||
queryset, _active_tag, _active_feed = _filtered_queryset(request, active_filter)
|
||||
|
||||
def new_since(request):
|
||||
"""Feeds the live updates (static/js/live-updates.js) on every
|
||||
authenticated page: how many unread, non-hidden articles from the
|
||||
user's subscriptions have appeared after the page's baseline id
|
||||
(?since), for the new-articles banner and the browser-tab dot, plus
|
||||
the sidebar's current counts. Deliberately not scoped to the stream's
|
||||
current filter/tag/feed — anything new, whichever page is open."""
|
||||
latest_id = Article.objects.aggregate(Max("id"))["id__max"] or 0
|
||||
counts = sidebar_counts(request.user)
|
||||
try:
|
||||
since_id = int(request.GET.get("since", 0))
|
||||
except ValueError:
|
||||
since_id = 0
|
||||
since_id = int(request.GET["since"])
|
||||
except (KeyError, ValueError):
|
||||
return JsonResponse({"latest_id": latest_id, "new_count": 0, "counts": counts})
|
||||
|
||||
new_count = queryset.filter(id__gt=since_id).count()
|
||||
return render(request, "reader/_new_articles_banner.html", {"new_count": new_count})
|
||||
new_count = (
|
||||
_base_queryset(request.user)
|
||||
.filter(id__gt=since_id)
|
||||
.exclude(user_states__user=request.user, user_states__is_read=True)
|
||||
.count()
|
||||
)
|
||||
return JsonResponse({"latest_id": latest_id, "new_count": new_count, "counts": counts})
|
||||
|
||||
|
||||
@login_required
|
||||
@@ -295,6 +300,13 @@ def mark_all_read(request):
|
||||
return redirect(f"{stream_url}?{query}" if query else stream_url)
|
||||
|
||||
|
||||
def _counts_changed(response):
|
||||
"""Tells the page's live-update script (static/js/live-updates.js) to
|
||||
refresh the sidebar counts right now rather than at the next poll."""
|
||||
response["HX-Trigger"] = "countsChanged"
|
||||
return response
|
||||
|
||||
|
||||
def _load_article_and_mark_read(user, article_id):
|
||||
article = get_object_or_404(_base_queryset(user), pk=article_id)
|
||||
|
||||
@@ -330,7 +342,7 @@ def article_inline(request, article_id):
|
||||
content fragment plus an out-of-band update of the row's own read/star
|
||||
icons (see templates/reader/_article_inline.html)."""
|
||||
article = _load_article_and_mark_read(request.user, article_id)
|
||||
return render(request, "reader/_article_inline.html", {"article": article})
|
||||
return _counts_changed(render(request, "reader/_article_inline.html", {"article": article}))
|
||||
|
||||
|
||||
@login_required
|
||||
@@ -346,7 +358,7 @@ def toggle_read(request, article_id):
|
||||
|
||||
article.is_read = state.is_read
|
||||
article.is_starred, article.is_saved_for_later = _article_tag_flags(request.user, article)
|
||||
return render(request, "reader/_article_actions.html", {"article": article})
|
||||
return _counts_changed(render(request, "reader/_article_actions.html", {"article": article}))
|
||||
|
||||
|
||||
@login_required
|
||||
@@ -382,4 +394,4 @@ def _toggle_system_tag(request, article_id, tag_name):
|
||||
article.is_read = state.is_read if state else False
|
||||
article.is_starred, article.is_saved_for_later = _article_tag_flags(request.user, article)
|
||||
|
||||
return render(request, "reader/_article_actions.html", {"article": article})
|
||||
return _counts_changed(render(request, "reader/_article_actions.html", {"article": article}))
|
||||
@@ -1,5 +1,12 @@
|
||||
"""Stored, GIN-indexed tsvector for Postgres search (C5), kept current by a
|
||||
trigger so no ingestion/update code path can forget to maintain it.
|
||||
"""Stored tsvector for Postgres search (C5), kept current by a trigger so
|
||||
no ingestion/update code path can forget to maintain it.
|
||||
|
||||
Stage 1 of 2, and deliberately cheap: it only adds the (nullable) column
|
||||
and installs the trigger, so new and updated rows get a vector from this
|
||||
point on. Both are metadata-only/brief-lock operations even on a huge
|
||||
table. The slow work — backfilling existing rows and building the GIN
|
||||
index — is 0003, done in batches and CONCURRENTLY so a large existing
|
||||
archive doesn't lock the table for the length of a deploy (R2-T3).
|
||||
|
||||
No-ops entirely on SQLite — its FTS5 table (0001_sqlite_fts) already plays
|
||||
this role. The column is deliberately not a Django model field: a
|
||||
@@ -17,11 +24,6 @@ VECTOR_EXPR = (
|
||||
CREATE = f"""
|
||||
ALTER TABLE articles_article ADD COLUMN IF NOT EXISTS search_vector tsvector;
|
||||
|
||||
UPDATE articles_article SET search_vector = {VECTOR_EXPR.format(row="")};
|
||||
|
||||
CREATE INDEX IF NOT EXISTS articles_article_search_vector_gin
|
||||
ON articles_article USING GIN (search_vector);
|
||||
|
||||
CREATE OR REPLACE FUNCTION articles_article_search_vector_update() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.search_vector := {VECTOR_EXPR.format(row="NEW.")};
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Stage 2 of 2 (see 0002): backfill existing rows and build the GIN index
|
||||
without holding a long lock (R2-T3).
|
||||
|
||||
Not atomic, on purpose. The backfill runs in id-range batches, each its
|
||||
own short transaction, so concurrent reads and writes keep flowing and a
|
||||
failed deploy can simply be re-run (it only touches rows whose vector is
|
||||
still NULL). CREATE INDEX CONCURRENTLY can't run inside a transaction at
|
||||
all. Rows written during the backfill already get a vector from the
|
||||
trigger installed in 0002.
|
||||
|
||||
Safe to re-run, and a cheap no-op on an install that already has the
|
||||
index and a full column (e.g. one that applied the earlier single-step
|
||||
version of 0002).
|
||||
|
||||
No-ops entirely on SQLite.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
from django.db import migrations
|
||||
|
||||
BATCH_SIZE = 5000
|
||||
|
||||
INDEX_NAME = "articles_article_search_vector_gin"
|
||||
|
||||
BACKFILL = """
|
||||
UPDATE articles_article
|
||||
SET search_vector =
|
||||
setweight(to_tsvector('english', coalesce(title, '')), 'A') ||
|
||||
setweight(to_tsvector('english', coalesce(content_html, '')), 'B')
|
||||
WHERE id >= %s AND id < %s AND search_vector IS NULL
|
||||
"""
|
||||
|
||||
|
||||
def backfill_and_index(apps, schema_editor):
|
||||
connection = schema_editor.connection
|
||||
if connection.vendor != "postgresql":
|
||||
return
|
||||
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SELECT min(id), max(id) FROM articles_article")
|
||||
low, high = cursor.fetchone()
|
||||
if low is not None:
|
||||
started = time.monotonic()
|
||||
done = 0
|
||||
for start in range(low, high + 1, BATCH_SIZE):
|
||||
cursor.execute(BACKFILL, [start, start + BATCH_SIZE])
|
||||
done += cursor.rowcount
|
||||
print(f"\n backfilled {done} search vector(s) in {time.monotonic() - started:.1f}s")
|
||||
|
||||
# A failed CONCURRENTLY build leaves an INVALID index behind, which
|
||||
# IF NOT EXISTS would then happily skip — drop it so a re-run
|
||||
# actually rebuilds.
|
||||
cursor.execute(
|
||||
"SELECT 1 FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid "
|
||||
"WHERE c.relname = %s AND NOT i.indisvalid",
|
||||
[INDEX_NAME],
|
||||
)
|
||||
if cursor.fetchone():
|
||||
cursor.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}")
|
||||
cursor.execute(
|
||||
f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {INDEX_NAME} "
|
||||
"ON articles_article USING GIN (search_vector)"
|
||||
)
|
||||
|
||||
|
||||
def drop_index(apps, schema_editor):
|
||||
if schema_editor.connection.vendor != "postgresql":
|
||||
return
|
||||
with schema_editor.connection.cursor() as cursor:
|
||||
cursor.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}")
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
atomic = False
|
||||
|
||||
dependencies = [("search", "0002_postgres_search_vector")]
|
||||
|
||||
operations = [migrations.RunPython(backfill_and_index, drop_index)]
|
||||
@@ -4,6 +4,7 @@ Base Django settings for Evection, shared by all environments.
|
||||
Environment-specific overrides live in dev.py / prod.py / test.py.
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
from pathlib import Path
|
||||
|
||||
import environ
|
||||
@@ -49,6 +50,7 @@ MIDDLEWARE = [
|
||||
"django.middleware.clickjacking.XFrameOptionsMiddleware",
|
||||
"django_htmx.middleware.HtmxMiddleware",
|
||||
"apps.accounts.middleware.SetupWizardMiddleware",
|
||||
"apps.pwa.middleware.AuthMarkerMiddleware",
|
||||
]
|
||||
|
||||
ROOT_URLCONF = "config.urls"
|
||||
@@ -121,6 +123,18 @@ EVECTION_SIGNUP_ENABLED = env.bool("EVECTION_SIGNUP_ENABLED", default=False)
|
||||
EVECTION_MIN_FETCH_INTERVAL_MINUTES = env.int("EVECTION_MIN_FETCH_INTERVAL_MINUTES", default=15)
|
||||
EVECTION_PWA_CACHE_VERSION = env.str("EVECTION_PWA_CACHE_VERSION", default="v1")
|
||||
|
||||
# How long /health/ready/ reuses its queue/feed aggregates (0 = never).
|
||||
EVECTION_HEALTH_CACHE_SECONDS = env.int("EVECTION_HEALTH_CACHE_SECONDS", default=15)
|
||||
|
||||
# IPs/CIDRs of the reverse proxy in front of gunicorn. When set, the login
|
||||
# throttle only believes X-Forwarded-For from these peers (see
|
||||
# apps.accounts.throttle.client_address). Empty keeps the legacy behavior.
|
||||
EVECTION_TRUSTED_PROXIES = [
|
||||
ipaddress.ip_network(entry.strip(), strict=False)
|
||||
for entry in env.list("EVECTION_TRUSTED_PROXIES", default=[])
|
||||
if entry.strip()
|
||||
]
|
||||
|
||||
# Redis-backed rather than Django's default per-process LocMemCache: login
|
||||
# rate limiting (apps.accounts.views) must share counters across every
|
||||
# gunicorn worker, not just whichever one happened to handle a given
|
||||
|
||||
@@ -12,3 +12,6 @@ PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
|
||||
# location is per-process but shared across tests in that process; see
|
||||
# apps.accounts.test_rate_limit's cache.clear() fixture).
|
||||
CACHES = {"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}}
|
||||
|
||||
# Tests change state between calls to /health/ready/; caching it would hide that.
|
||||
EVECTION_HEALTH_CACHE_SECONDS = 0
|
||||
+62
-12
@@ -19,7 +19,8 @@ Both SQLite and PostgreSQL are fully supported. Steps that differ are marked
|
||||
|
||||
Evection is deployed from git tags, never from a branch tip — the same pattern
|
||||
as the author's other self-hosted projects. A release tag looks like
|
||||
`v2026.10.31` (`v<year>.<month>.<day>`); `scripts/install.sh` and
|
||||
`v2026.11.1` (`v<year>.<month>.<day>`, no zero padding); a second release the same
|
||||
day is `v2026.11.1.2`, a third `v2026.11.1.3`, and so on. `scripts/install.sh` and
|
||||
`scripts/upgrade.sh` both always deploy whatever tag is newest by creation
|
||||
time, via a detached-HEAD checkout, regardless of what's checked out or
|
||||
committed on `main`.
|
||||
@@ -28,7 +29,7 @@ Before your first production install, cut and push a tag from whatever commit
|
||||
you want to ship:
|
||||
|
||||
```bash
|
||||
git tag v$(date +%Y.%m.%d)
|
||||
git tag v$(date +%Y.%-m.%-d) # a later release the same day: append .2, .3, ...
|
||||
git push --tags
|
||||
```
|
||||
|
||||
@@ -105,9 +106,17 @@ than a branch.)
|
||||
```bash
|
||||
cd /opt/evection/app
|
||||
sudo -u evection -H python3 -m venv /opt/evection/venv
|
||||
sudo -u evection -H /opt/evection/venv/bin/pip install .
|
||||
sudo -u evection -H /opt/evection/venv/bin/pip install --upgrade pip
|
||||
sudo -u evection -H /opt/evection/venv/bin/pip install --require-hashes -r requirements.txt
|
||||
sudo -u evection -H /opt/evection/venv/bin/pip install --no-deps .
|
||||
```
|
||||
|
||||
`requirements.txt` pins every dependency to an exact, hash-verified version, so
|
||||
two installs of the same release get identical dependency code and a tampered
|
||||
package fails the hash check instead of installing. The app itself goes in
|
||||
with `--no-deps` afterwards. (Don't use a bare `pip install .` — it re-resolves
|
||||
version ranges at install time, which is what the lock file exists to avoid.)
|
||||
|
||||
## 4. Configure
|
||||
|
||||
Copy the example environment file and edit it:
|
||||
@@ -142,6 +151,15 @@ At minimum, set the following in `.env`:
|
||||
- `REDIS_URL` — `redis://localhost:6379/0` is fine for a single-host install
|
||||
- `EVECTION_GUNICORN_BIND` — the `address:port` gunicorn listens on; see
|
||||
"Reverse proxy" below for how to pick this
|
||||
- `EVECTION_TRUSTED_PROXIES` — the IP address (or CIDR, comma-separated for more
|
||||
than one) of your reverse proxy, as gunicorn sees it connecting. The login
|
||||
throttle only believes `X-Forwarded-For` from these addresses, and takes the
|
||||
address your proxy actually saw rather than the first entry (which a client can
|
||||
forge). Leave it unset and it falls back to trusting that header from anyone,
|
||||
which is only safe while nothing but your proxy can reach gunicorn;
|
||||
`manage.py check --deploy` warns until it's set. With Nginx Proxy Manager on
|
||||
another host, use that host's address (check `ss -tn` on this machine while
|
||||
loading the site if you're not sure what gunicorn sees).
|
||||
|
||||
How often every feed is polled is a site-wide setting (Django admin → Site
|
||||
settings), not a `.env` value — `EVECTION_MIN_FETCH_INTERVAL_MINUTES` only sets
|
||||
@@ -314,17 +332,42 @@ misconfigured feed or an outage at the source.
|
||||
|
||||
`/health/` is an unauthenticated liveness check (DB connectivity only) —
|
||||
`scripts/upgrade.sh` uses it to confirm a deploy actually came back up.
|
||||
Point an external uptime monitor at `/health/ready/` instead: it also checks
|
||||
Redis and that the background poller has ticked in the last five minutes
|
||||
(proving Celery beat is scheduling *and* a worker is consuming), and returns
|
||||
503 with a per-check breakdown when any of them fails — so "the site loads
|
||||
but nothing is being fetched" actually alerts.
|
||||
Point an external uptime monitor at `/health/ready/` instead. It returns 503
|
||||
with a per-check breakdown when any of these fail:
|
||||
|
||||
- `database` and `redis` — reachable.
|
||||
- `poll_heartbeat` — the background poller ticked in the last five minutes
|
||||
(proves Celery beat is scheduling *and* a worker is consuming).
|
||||
- `queue_depth` — no more than 1000 tasks waiting in the Celery queue
|
||||
(workers aren't keeping up, or aren't running).
|
||||
- `recent_fetch_success` — at least one feed has been fetched successfully
|
||||
within three poll intervals (minimum 30 minutes), so "scheduled but every
|
||||
fetch is failing" is caught. A brand-new install gets that long before it
|
||||
counts as stale.
|
||||
- `feed_failures` — no more than half of your active feeds are failing, once
|
||||
you have at least four (a handful of dead feeds is normal; most of them
|
||||
failing at once points at your server, DNS, or outbound network).
|
||||
|
||||
The payload also reports `queue_depth`, `active_feeds`, `failing_feeds`, and
|
||||
the age of the last successful fetch, so you can graph them. The queue and feed
|
||||
figures are reused for 15 seconds (`EVECTION_HEALTH_CACHE_SECONDS`, 0 disables
|
||||
it) so a fast-polling monitor can't turn them into steady database load; the
|
||||
database, Redis and poller checks are always live. The endpoint is
|
||||
unauthenticated, so if you'd rather not expose those numbers publicly, restrict
|
||||
`/health/ready/` to your monitor's address at the proxy.
|
||||
|
||||
**Retention.** By default every fetched article is kept forever. In the Django
|
||||
admin, "Site settings" has a *Retention days* field: set it to delete articles
|
||||
fetched longer ago than that, nightly (03:30 UTC, via Celery beat). Articles
|
||||
anyone has starred or saved for later are always kept; read state and tags for
|
||||
deleted articles go with them. The dashboard shows the article count and the
|
||||
deleted articles go with them. A small record (feed and entry id) is kept for
|
||||
each deleted article so a feed that still lists it can't bring it back as new
|
||||
and unread — those records are themselves removed after a year, so an entry a
|
||||
feed is *still* publishing a year after it was purged (with no publish date, or
|
||||
a recent one) can come back as new. Entries whose own publish date is already
|
||||
older than the window aren't fetched in the first place. A large first purge
|
||||
works in time-boxed runs (about four minutes each) that queue their own
|
||||
continuation, so it never monopolises a worker. The dashboard shows the article count and the
|
||||
age of the oldest one, so you can see what retention is doing.
|
||||
|
||||
**SQLite.** SQLite installs run in WAL mode with a 20-second busy timeout (set
|
||||
@@ -361,12 +404,13 @@ Equivalent by hand:
|
||||
|
||||
```bash
|
||||
cd /opt/evection/app
|
||||
git fetch --tags --force
|
||||
git fetch --tags # deliberately not --force: see below
|
||||
LATEST_TAG="$(git tag --sort=-creatordate | head -n1)"
|
||||
git checkout --detach "$LATEST_TAG"
|
||||
chown -R evection:evection /opt/evection/app
|
||||
sed -i "s/^EVECTION_PWA_CACHE_VERSION=.*/EVECTION_PWA_CACHE_VERSION=$LATEST_TAG/" .env
|
||||
sudo -u evection -H /opt/evection/venv/bin/pip install .
|
||||
sudo -u evection -H /opt/evection/venv/bin/pip install --require-hashes -r requirements.txt
|
||||
sudo -u evection -H /opt/evection/venv/bin/pip install --no-deps --upgrade .
|
||||
sudo -u evection -H env DJANGO_SETTINGS_MODULE=config.settings.prod \
|
||||
/opt/evection/venv/bin/python manage.py migrate
|
||||
sudo -u evection -H env DJANGO_SETTINGS_MODULE=config.settings.prod \
|
||||
@@ -374,11 +418,17 @@ sudo -u evection -H env DJANGO_SETTINGS_MODULE=config.settings.prod \
|
||||
sudo systemctl restart evection-gunicorn evection-celery-worker evection-celery-beat
|
||||
```
|
||||
|
||||
A plain `git fetch --tags` refuses to move a tag you already have onto a
|
||||
different commit — the sign of a rewritten or tampered release. If it fails
|
||||
with "would clobber existing tag", stop and find out why before deploying;
|
||||
don't reach for `--force`, which would silently follow the rewritten tag into
|
||||
production (`scripts/upgrade.sh` stops there too).
|
||||
|
||||
To ship a new release, just cut and push a new tag on the machine you build
|
||||
from — the next `upgrade.sh` run on the server picks it up:
|
||||
|
||||
```bash
|
||||
git tag v$(date +%Y.%m.%d)
|
||||
git tag v$(date +%Y.%-m.%-d) # a later release the same day: v2026.11.1.2, .3, ...
|
||||
git push --tags
|
||||
```
|
||||
|
||||
|
||||
+1
-1
@@ -81,7 +81,7 @@ fi
|
||||
LATEST_TAG="$(git tag --sort=-creatordate | head -n1)"
|
||||
if [ -z "$LATEST_TAG" ]; then
|
||||
echo "No tags found in this repo — nothing to deploy." \
|
||||
"Cut and push a release tag first (e.g. git tag v$(date +%Y.%m.%d) && git push --tags)." >&2
|
||||
"Cut and push a release tag first (e.g. git tag v$(date +%Y.%-m.%-d) && git push --tags)." >&2
|
||||
exit 1
|
||||
fi
|
||||
CURRENT_REF="$(git describe --tags --exact-match 2>/dev/null || echo "none")"
|
||||
|
||||
@@ -113,6 +113,12 @@ form p {
|
||||
display: inline;
|
||||
}
|
||||
|
||||
.new-articles-banner[hidden],
|
||||
.unread-count[hidden],
|
||||
.starred-count[hidden] {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.new-articles-banner {
|
||||
display: block;
|
||||
width: 100%;
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
// Keeps every authenticated page current without ever moving the article
|
||||
// list under the reader. One poll (/new-since/) drives three things:
|
||||
// - the "N new articles" banner, wherever in the sidebar tree you are;
|
||||
// - a red dot on the browser tab's favicon, visible from a background tab;
|
||||
// - the sidebar's unread/starred counts, patched in place.
|
||||
// New articles are never inserted automatically: the banner click reloads
|
||||
// the current view. A full page load resets the baseline (data-since,
|
||||
// rendered into the page) — exactly when the user has seen the new articles.
|
||||
// The poll deliberately keeps running in a hidden tab (that's what the dot is
|
||||
// for); it also fires when the tab becomes visible again, and as soon as a
|
||||
// read/star action tells us the counts changed (the countsChanged event the
|
||||
// server sends via HX-Trigger).
|
||||
(function () {
|
||||
var POLL_MS = 300000;
|
||||
var MIN_GAP_MS = 15000; // coalesce bursts (rapid reading, tab switches)
|
||||
var script = document.currentScript;
|
||||
var url = script && script.dataset.url;
|
||||
var icon = document.querySelector('link[rel="icon"]');
|
||||
var banner = document.querySelector(".new-articles-banner");
|
||||
if (!url) return;
|
||||
|
||||
var originalHref = icon ? icon.href : null;
|
||||
var baseline = parseInt(script.dataset.since, 10);
|
||||
if (isNaN(baseline)) baseline = null;
|
||||
var timer = null;
|
||||
var inFlight = false;
|
||||
var rerun = false; // a read happened while a poll was in flight: its answer may predate it
|
||||
var lastPoll = 0;
|
||||
var badged = null; // the dotted icon's data: URL, drawn once on first need
|
||||
var showing = false;
|
||||
|
||||
function drawDot() {
|
||||
return new Promise(function (resolve) {
|
||||
var image = new Image();
|
||||
image.onload = function () {
|
||||
var size = 64;
|
||||
var canvas = document.createElement("canvas");
|
||||
canvas.width = canvas.height = size;
|
||||
var ctx = canvas.getContext("2d");
|
||||
ctx.drawImage(image, 0, 0, size, size);
|
||||
ctx.beginPath();
|
||||
ctx.arc(size * 0.74, size * 0.26, size * 0.23, 0, 2 * Math.PI);
|
||||
ctx.fillStyle = "#e53935";
|
||||
ctx.fill();
|
||||
ctx.lineWidth = size * 0.07;
|
||||
ctx.strokeStyle = "#ffffff";
|
||||
ctx.stroke();
|
||||
resolve(canvas.toDataURL("image/png"));
|
||||
};
|
||||
image.onerror = function () {
|
||||
resolve(null);
|
||||
};
|
||||
image.src = originalHref;
|
||||
});
|
||||
}
|
||||
|
||||
// Replace the <link> rather than editing href: some browsers ignore an
|
||||
// in-place change to an existing favicon element.
|
||||
function setIcon(href) {
|
||||
var fresh = icon.cloneNode();
|
||||
fresh.href = href;
|
||||
icon.parentNode.replaceChild(fresh, icon);
|
||||
icon = fresh;
|
||||
}
|
||||
|
||||
function showDot(on) {
|
||||
if (!icon || on === showing) return Promise.resolve();
|
||||
showing = on;
|
||||
if (!on) {
|
||||
setIcon(originalHref);
|
||||
return Promise.resolve();
|
||||
}
|
||||
var ready = badged ? Promise.resolve(badged) : drawDot();
|
||||
return ready.then(function (href) {
|
||||
badged = href;
|
||||
if (href && showing) setIcon(href);
|
||||
});
|
||||
}
|
||||
|
||||
function setCount(el, value) {
|
||||
var text = el.hasAttribute("data-starred-count") ? "(" + value + ")" : String(value);
|
||||
el.textContent = value ? text : "";
|
||||
el.hidden = !value;
|
||||
}
|
||||
|
||||
function updateCounts(counts) {
|
||||
if (!counts) return;
|
||||
document.querySelectorAll(".unread-count[data-feed-id]").forEach(function (el) {
|
||||
setCount(el, counts.feeds[el.dataset.feedId] || 0);
|
||||
});
|
||||
document.querySelectorAll(".unread-count[data-tag-id]").forEach(function (el) {
|
||||
setCount(el, counts.tags[el.dataset.tagId] || 0);
|
||||
});
|
||||
document.querySelectorAll("[data-starred-count]").forEach(function (el) {
|
||||
setCount(el, counts.starred || 0);
|
||||
});
|
||||
}
|
||||
|
||||
function updateBanner(count) {
|
||||
if (!banner) return;
|
||||
banner.hidden = !count;
|
||||
if (count) {
|
||||
banner.textContent =
|
||||
count + " new article" + (count === 1 ? "" : "s") + " \u2014 click to refresh";
|
||||
}
|
||||
}
|
||||
|
||||
function schedule() {
|
||||
clearTimeout(timer);
|
||||
timer = setTimeout(poll, POLL_MS);
|
||||
}
|
||||
|
||||
function poll() {
|
||||
if (inFlight) {
|
||||
rerun = true;
|
||||
return;
|
||||
}
|
||||
inFlight = true;
|
||||
lastPoll = Date.now();
|
||||
var query = baseline === null ? "" : "?since=" + baseline;
|
||||
fetch(url + query, { credentials: "same-origin", headers: { Accept: "application/json" } })
|
||||
.then(function (response) {
|
||||
// A redirect means the session ended (we'd be reading the login
|
||||
// page) — stop polling instead of hammering it.
|
||||
if (!response.ok || response.redirected) throw new Error("stop");
|
||||
return response.json();
|
||||
})
|
||||
.then(function (data) {
|
||||
inFlight = false;
|
||||
if (baseline === null) baseline = data.latest_id;
|
||||
updateCounts(data.counts);
|
||||
updateBanner(data.new_count);
|
||||
showDot(data.new_count > 0);
|
||||
if (rerun) {
|
||||
rerun = false;
|
||||
poll();
|
||||
} else {
|
||||
schedule();
|
||||
}
|
||||
})
|
||||
.catch(function (error) {
|
||||
inFlight = false;
|
||||
rerun = false;
|
||||
if (error.message !== "stop") schedule();
|
||||
});
|
||||
}
|
||||
|
||||
function pollSoon() {
|
||||
var wait = Math.max(0, MIN_GAP_MS - (Date.now() - lastPoll));
|
||||
clearTimeout(timer);
|
||||
timer = setTimeout(poll, wait);
|
||||
}
|
||||
|
||||
document.addEventListener("visibilitychange", function () {
|
||||
if (!document.hidden) pollSoon();
|
||||
});
|
||||
document.body.addEventListener("countsChanged", function () {
|
||||
// Reading is interactive: refresh the sidebar now, not after the gap.
|
||||
clearTimeout(timer);
|
||||
timer = setTimeout(poll, 150);
|
||||
});
|
||||
|
||||
poll();
|
||||
})();
|
||||
+8
-1
@@ -41,7 +41,14 @@ document.addEventListener("click", function (event) {
|
||||
toggleArticle(link);
|
||||
return;
|
||||
}
|
||||
if (event.target.closest(".new-articles-banner")) {
|
||||
var banner = event.target.closest(".new-articles-banner");
|
||||
if (banner) {
|
||||
// Reloading keeps the current filter/tag/feed in the URL; from any
|
||||
// other page (settings, search) go to the stream instead.
|
||||
if (location.pathname === banner.dataset.streamUrl) {
|
||||
location.reload();
|
||||
} else {
|
||||
location.href = banner.dataset.streamUrl;
|
||||
}
|
||||
}
|
||||
});
|
||||
+4
-1
@@ -38,7 +38,9 @@
|
||||
{% endif %}
|
||||
|
||||
<main class="{% if user.is_authenticated %}app-content{% else %}auth-content{% endif %}">
|
||||
{% if not user.is_authenticated %}
|
||||
{% if user.is_authenticated %}
|
||||
<button type="button" class="new-articles-banner" data-stream-url="{% url 'reader:stream' %}" hidden></button>
|
||||
{% else %}
|
||||
<div class="auth-brand">
|
||||
<img src="{% static 'img/brand-full-dark.png' %}" alt="Evection" class="brand-wordmark-dark">
|
||||
<img src="{% static 'img/brand-full-light.png' %}" alt="Evection" class="brand-wordmark-light">
|
||||
@@ -58,6 +60,7 @@
|
||||
{% if user.is_authenticated %}
|
||||
<script src="{% static 'js/theme.js' %}" defer></script>
|
||||
<script src="{% static 'js/sidebar-state.js' %}" defer></script>
|
||||
<script src="{% static 'js/live-updates.js' %}" data-url="{% url 'reader:new_since' %}" data-since="{{ latest_article_id }}" defer></script>
|
||||
{% endif %}
|
||||
<script src="{% static 'js/register-sw.js' %}" defer></script>
|
||||
</body>
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
<li>
|
||||
<a href="{% url 'reader:stream' %}?filter=starred"
|
||||
class="{% if request.GET.filter == 'starred' %}active{% endif %}">
|
||||
Favourites{% if sidebar_starred_count %} ({{ sidebar_starred_count }}){% endif %}
|
||||
Favourites <span class="starred-count" data-starred-count{% if not sidebar_starred_count %} hidden{% endif %}>({{ sidebar_starred_count }})</span>
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
@@ -36,7 +36,7 @@
|
||||
<span class="sidebar-tag-row">
|
||||
<a href="{% url 'reader:stream' %}?tag={{ entry.tag.pk }}"
|
||||
class="sidebar-tag-link{% if request.GET.tag == entry.tag.pk|stringformat:'s' %} active{% endif %}">{{ entry.tag.name }}</a>
|
||||
{% if entry.unread_count %}<span class="unread-count">{{ entry.unread_count }}</span>{% endif %}
|
||||
<span class="unread-count" data-tag-id="{{ entry.tag.pk }}"{% if not entry.unread_count %} hidden{% endif %}>{% if entry.unread_count %}{{ entry.unread_count }}{% endif %}</span>
|
||||
</span>
|
||||
</summary>
|
||||
{% if entry.subscriptions %}
|
||||
@@ -48,7 +48,7 @@
|
||||
{% if subscription.feed.favicon_url %}<img src="{{ subscription.feed.favicon_url }}" alt="" class="feed-favicon">{% endif %}
|
||||
<span>{{ subscription.display_title }}</span>
|
||||
</a>
|
||||
{% if subscription.unread_count %}<span class="unread-count">{{ subscription.unread_count }}</span>{% endif %}
|
||||
<span class="unread-count" data-feed-id="{{ subscription.feed_id }}"{% if not subscription.unread_count %} hidden{% endif %}>{% if subscription.unread_count %}{{ subscription.unread_count }}{% endif %}</span>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
@@ -67,7 +67,7 @@
|
||||
{% if subscription.feed.favicon_url %}<img src="{{ subscription.feed.favicon_url }}" alt="" class="feed-favicon">{% endif %}
|
||||
<span>{{ subscription.display_title }}</span>
|
||||
</a>
|
||||
{% if subscription.unread_count %}<span class="unread-count">{{ subscription.unread_count }}</span>{% endif %}
|
||||
<span class="unread-count" data-feed-id="{{ subscription.feed_id }}"{% if not subscription.unread_count %} hidden{% endif %}>{% if subscription.unread_count %}{{ subscription.unread_count }}{% endif %}</span>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{% extends "base.html" %}
|
||||
{% load safe_urls %}
|
||||
{% block title %}Dashboard — Evection{% endblock %}
|
||||
{% block content %}
|
||||
<h1>Dashboard</h1>
|
||||
@@ -27,7 +28,7 @@
|
||||
<tbody>
|
||||
{% for feed in erroring_feeds %}
|
||||
<tr>
|
||||
<td><a href="{{ feed.site_url|default:feed.feed_url }}">{{ feed.title|default:feed.feed_url }}</a></td>
|
||||
<td><a href="{{ feed.site_url|default:feed.feed_url|safe_http_url }}">{{ feed.title|default:feed.feed_url }}</a></td>
|
||||
<td>{{ feed.subscriber_count }}</td>
|
||||
<td>{{ feed.get_last_fetch_status_display }}</td>
|
||||
<td>{{ feed.fetch_error_count }}</td>
|
||||
|
||||
@@ -42,12 +42,38 @@ async function trimCache(cacheName, maxItems) {
|
||||
await Promise.all(toDelete.map((key) => cache.delete(key)));
|
||||
}
|
||||
|
||||
// Every HTML response carries X-Evection-Auth: an opaque id for who it was
|
||||
// rendered for ("anon" when no one is logged in — see apps.pwa.middleware).
|
||||
// PAGES_CACHE holds one person's pages; the marker it belongs to is kept in
|
||||
// the shell cache (a service worker's own memory doesn't survive being
|
||||
// stopped). Any change — session expired, logged in, switched accounts —
|
||||
// purges the pages before anything new is stored, so one session's
|
||||
// authenticated HTML can never be served to the next, whether or not the
|
||||
// logout request itself passed through this worker.
|
||||
const AUTH_MARKER_KEY = "/__evection-auth-marker__";
|
||||
|
||||
async function syncAuthMarker(response) {
|
||||
const marker = response.headers.get("X-Evection-Auth");
|
||||
if (marker === null) return null; // not an HTML page we can attribute
|
||||
const shell = await caches.open(SHELL_CACHE);
|
||||
const stored = await shell.match(AUTH_MARKER_KEY);
|
||||
const previous = stored ? await stored.text() : null;
|
||||
if (previous !== marker) {
|
||||
await caches.delete(PAGES_CACHE);
|
||||
await shell.put(AUTH_MARKER_KEY, new Response(marker));
|
||||
}
|
||||
return marker;
|
||||
}
|
||||
|
||||
async function networkFirstForNavigation(request) {
|
||||
try {
|
||||
const response = await fetch(request);
|
||||
const marker = await syncAuthMarker(response);
|
||||
if (marker !== null && marker !== "anon") {
|
||||
const cache = await caches.open(PAGES_CACHE);
|
||||
cache.put(request, response.clone());
|
||||
trimCache(PAGES_CACHE, MAX_CACHED_PAGES);
|
||||
}
|
||||
return response;
|
||||
} catch (err) {
|
||||
const cached = await caches.match(request);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
<div class="article-content-header">
|
||||
{% load safe_urls %}<div class="article-content-header">
|
||||
<h2 class="article-content-title">{{ article.title|default:"(untitled)" }}</h2>
|
||||
<p class="article-meta">
|
||||
{% if article.feed.favicon_url %}<img src="{{ article.feed.favicon_url }}" alt="" class="feed-favicon">{% endif %}
|
||||
@@ -6,7 +6,7 @@
|
||||
{# fetched_at stands in when the feed gave no publish date #}
|
||||
· <time datetime="{{ article.published_at|default:article.fetched_at|date:"c" }}" data-local-format="long">{{ article.published_at|default:article.fetched_at|date:"M j, Y H:i" }}</time>
|
||||
{% if article.author %} · {{ article.author }}{% endif %}
|
||||
{% if article.link %} · <a href="{{ article.link }}" target="_blank" rel="noopener noreferrer">Original ↗</a>{% endif %}
|
||||
{% with safe_link=article.link|safe_http_url %}{% if safe_link %} · <a href="{{ safe_link }}" target="_blank" rel="noopener noreferrer">Original ↗</a>{% endif %}{% endwith %}
|
||||
</p>
|
||||
|
||||
<div
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
{% if new_count %}
|
||||
<button type="button" class="new-articles-banner">
|
||||
{{ new_count }} new article{{ new_count|pluralize }} — click to show
|
||||
</button>
|
||||
{% endif %}
|
||||
@@ -23,11 +23,5 @@
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div id="new-articles-banner"
|
||||
hx-get="{% url 'reader:check_new' %}?since={{ latest_article_id }}&{{ new_check_qs }}"
|
||||
hx-trigger="every 30s"
|
||||
hx-target="this"
|
||||
hx-swap="innerHTML"></div>
|
||||
|
||||
{% include "reader/_article_list.html" %}
|
||||
{% endblock %}
|
||||
Reference in new issue
Block a user