diff --git a/.env.example b/.env.example index bd70411..6b53fc7 100644 --- a/.env.example +++ b/.env.example @@ -23,3 +23,9 @@ EVECTION_MIN_FETCH_INTERVAL_MINUTES=15 # Bump this on deploys that change cached static assets to invalidate old # service worker caches. EVECTION_PWA_CACHE_VERSION=v1 + +# IP address(es) or CIDR(s) of the reverse proxy in front of gunicorn, +# comma-separated, e.g. 192.168.1.10 or 10.0.0.0/24. When set, the login +# throttle trusts X-Forwarded-For only from these addresses. Strongly +# recommended; `manage.py check --deploy` warns until it's set. +# EVECTION_TRUSTED_PROXIES= diff --git a/apps/accounts/apps.py b/apps/accounts/apps.py index e3ceb5a..4042453 100644 --- a/apps/accounts/apps.py +++ b/apps/accounts/apps.py @@ -4,3 +4,6 @@ from django.apps import AppConfig class AccountsConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "apps.accounts" + + def ready(self): + from . import checks # noqa: F401 (registers the deploy check) diff --git a/apps/accounts/checks.py b/apps/accounts/checks.py new file mode 100644 index 0000000..2985a1f --- /dev/null +++ b/apps/accounts/checks.py @@ -0,0 +1,20 @@ +from django.conf import settings +from django.core.checks import Tags, Warning, register + + +@register(Tags.security, deploy=True) +def trusted_proxies_configured(app_configs, **kwargs): + if settings.DEBUG or settings.EVECTION_TRUSTED_PROXIES: + return [] + return [ + Warning( + "EVECTION_TRUSTED_PROXIES is not set.", + hint=( + "The login throttle then trusts the first X-Forwarded-For entry from " + "anyone, which only holds while gunicorn is reachable by nothing but a " + "proxy that overwrites that header. Set EVECTION_TRUSTED_PROXIES to your " + "reverse proxy's IP/CIDR in .env (see docs/install.md)." + ), + id="evection.W001", + ) + ] diff --git a/apps/accounts/test_rate_limit.py b/apps/accounts/test_rate_limit.py index 5112506..5fb3131 100644 --- a/apps/accounts/test_rate_limit.py +++ b/apps/accounts/test_rate_limit.py @@ -74,19 +74,144 @@ def test_too_many_failed_attempts_against_one_username_blocks_that_username(clie assert response.wsgi_request.user == bob -@pytest.mark.django_db -def test_login_still_works_when_the_throttle_cache_is_down(client, monkeypatch): +@pytest.fixture +def cache_down(monkeypatch): from unittest.mock import Mock - from django.contrib.auth import get_user_model - - get_user_model().objects.create_user("alice", password="pw-alice-123") broken = Mock(side_effect=ConnectionError("redis down")) - monkeypatch.setattr("apps.accounts.views.cache.get", broken) - monkeypatch.setattr("apps.accounts.views.cache.incr", broken) - monkeypatch.setattr("apps.accounts.views.cache.set", broken) + for name in ("get", "incr", "set"): + monkeypatch.setattr(f"apps.accounts.throttle.cache.{name}", broken) - bad = client.post(reverse("accounts:login"), {"username": "alice", "password": "wrong"}) - assert bad.status_code == 200 + +@pytest.fixture(autouse=True) +def _reset_local_limiter(): + from .views import login_throttle + + login_throttle._local.clear() + yield + login_throttle._local.clear() + + +@pytest.mark.django_db +def test_login_still_works_when_the_throttle_cache_is_down(client, alice, cache_down): + assert _bad_login(client).status_code == 200 good = client.post(reverse("accounts:login"), {"username": "alice", "password": "pw-alice-123"}) assert good.status_code == 302 + + +@pytest.mark.django_db +def test_a_redis_outage_degrades_to_a_local_limit_instead_of_no_limit(client, alice, cache_down): + """R2-S4: previously every attempt was allowed while the cache was + down, so an outage switched brute-force protection off entirely.""" + for _ in range(LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME): + _bad_login(client) + + blocked = client.post( + reverse("accounts:login"), {"username": "alice", "password": "pw-alice-123"} + ) + assert blocked.status_code == 302 + assert blocked.url == reverse("accounts:login"), "even the right password is refused" + + +class TestLocalLimiter: + def test_counts_per_key_and_expires(self, monkeypatch): + import time as time_module + + from .throttle import LoginThrottle + + throttle = LoginThrottle(window_seconds=60) + for _ in range(3): + throttle._local_record("k") + assert throttle._local_count("k") == 3 + assert throttle._local_count("other") == 0 + + real = time_module.monotonic() + monkeypatch.setattr("apps.accounts.throttle.time.monotonic", lambda: real + 61) + assert throttle._local_count("k") == 0 + + def test_memory_is_bounded(self, monkeypatch): + from . import throttle as throttle_module + + monkeypatch.setattr(throttle_module, "LOCAL_MAX_KEYS", 5) + throttle = throttle_module.LoginThrottle(window_seconds=60) + for i in range(50): + throttle._local_record(f"k{i}") + assert len(throttle._local) <= 5 + + +class TestClientAddress: + def _request(self, remote, forwarded=None): + from django.test import RequestFactory + + extra = {"REMOTE_ADDR": remote} + if forwarded is not None: + extra["HTTP_X_FORWARDED_FOR"] = forwarded + return RequestFactory().get("/", **extra) + + def test_unset_keeps_the_legacy_first_hop_behavior(self, settings): + from .throttle import client_address + + settings.EVECTION_TRUSTED_PROXIES = [] + assert client_address(self._request("10.0.0.5", "1.2.3.4, 10.0.0.5")) == "1.2.3.4" + assert client_address(self._request("10.0.0.5")) == "10.0.0.5" + + def test_trusted_proxy_uses_the_rightmost_untrusted_hop_not_the_spoofable_first(self, settings): + import ipaddress + + from .throttle import client_address + + settings.EVECTION_TRUSTED_PROXIES = [ipaddress.ip_network("10.0.0.5/32")] + # The client sent "X-Forwarded-For: 6.6.6.6"; the proxy appended the + # address it actually saw (203.0.113.9). + request = self._request("10.0.0.5", "6.6.6.6, 203.0.113.9") + assert client_address(request) == "203.0.113.9" + + def test_forwarded_for_from_an_untrusted_peer_is_ignored(self, settings): + import ipaddress + + from .throttle import client_address + + settings.EVECTION_TRUSTED_PROXIES = [ipaddress.ip_network("10.0.0.5/32")] + # Someone reaching gunicorn directly and claiming to be 1.2.3.4. + assert client_address(self._request("198.51.100.7", "1.2.3.4")) == "198.51.100.7" + + def test_chained_trusted_proxies_are_skipped(self, settings): + import ipaddress + + from .throttle import client_address + + settings.EVECTION_TRUSTED_PROXIES = [ipaddress.ip_network("10.0.0.0/24")] + request = self._request("10.0.0.5", "203.0.113.9, 10.0.0.7") + assert client_address(request) == "203.0.113.9" + + def test_trusted_peer_with_no_forwarded_header_falls_back_to_the_peer(self, settings): + import ipaddress + + from .throttle import client_address + + settings.EVECTION_TRUSTED_PROXIES = [ipaddress.ip_network("10.0.0.5/32")] + assert client_address(self._request("10.0.0.5")) == "10.0.0.5" + + +class TestDeployCheck: + def test_warns_when_trusted_proxies_is_unset_in_production(self, settings): + from .checks import trusted_proxies_configured + + settings.DEBUG = False + settings.EVECTION_TRUSTED_PROXIES = [] + warnings = trusted_proxies_configured(None) + + assert [w.id for w in warnings] == ["evection.W001"] + + def test_quiet_when_configured_or_in_debug(self, settings): + import ipaddress + + from .checks import trusted_proxies_configured + + settings.DEBUG = False + settings.EVECTION_TRUSTED_PROXIES = [ipaddress.ip_network("10.0.0.5/32")] + assert trusted_proxies_configured(None) == [] + + settings.DEBUG = True + settings.EVECTION_TRUSTED_PROXIES = [] + assert trusted_proxies_configured(None) == [] diff --git a/apps/accounts/throttle.py b/apps/accounts/throttle.py new file mode 100644 index 0000000..4064017 --- /dev/null +++ b/apps/accounts/throttle.py @@ -0,0 +1,106 @@ +"""Login attempt throttling: counters live in the shared cache (Redis) so +every gunicorn worker sees them, with an in-process fallback when the +cache is unreachable (R2-S4) — so a Redis outage degrades throttling to +per-worker limits instead of switching it off for anyone who happens to +choose that moment.""" + +import ipaddress +import logging +import threading +import time + +from django.conf import settings +from django.core.cache import cache + +logger = logging.getLogger(__name__) + +LOCAL_MAX_KEYS = 10_000 + + +class LoginThrottle: + def __init__(self, window_seconds: int): + self.window = window_seconds + self._lock = threading.Lock() + self._local: dict[str, tuple[int, float]] = {} + + def is_blocked(self, key: str, limit: int) -> bool: + try: + return cache.get(key, 0) >= limit + except Exception: + logger.warning("Login throttle cache unavailable; using local limiter") + return self._local_count(key) >= limit + + def record(self, key: str) -> None: + try: + try: + cache.incr(key) + except ValueError: + cache.set(key, 1, self.window) + except Exception: + logger.warning("Login throttle cache unavailable; using local limiter") + self._local_record(key) + + def _local_count(self, key: str) -> int: + with self._lock: + entry = self._local.get(key) + if entry is None or entry[1] <= time.monotonic(): + return 0 + return entry[0] + + def _local_record(self, key: str) -> None: + now = time.monotonic() + with self._lock: + if len(self._local) >= LOCAL_MAX_KEYS: + self._local = {k: v for k, v in self._local.items() if v[1] > now} + if len(self._local) >= LOCAL_MAX_KEYS: + self._local.clear() # under a flood, bounded memory beats perfect memory + count, expires = self._local.get(key, (0, 0.0)) + if expires <= now: + count, expires = 0, now + self.window + self._local[key] = (count + 1, expires) + + +def _parse_ip(value: str): + try: + return ipaddress.ip_address(value.strip()) + except ValueError: + return None + + +def _is_trusted_proxy(value: str, trusted) -> bool: + ip = _parse_ip(value) + return ip is not None and any(ip in network for network in trusted) + + +def client_address(request) -> str: + """The address to throttle a login attempt by. + + EVECTION_TRUSTED_PROXIES (comma-separated IPs/CIDRs of your reverse + proxy) makes this strict: X-Forwarded-For is only believed when the + TCP peer is one of those proxies, and then the *rightmost* entry that + isn't itself a trusted proxy is used — the address your proxy actually + saw — rather than the first entry, which is whatever the client chose + to send. A client reaching gunicorn directly, or a proxy outside the + list, gets its own peer address. + + Unset (the default), it keeps the previous behavior — trust the first + X-Forwarded-For entry — which is only safe while gunicorn is reachable + by nothing but a proxy that overwrites that header (docs/install.md); + `manage.py check --deploy` warns until it's set. + """ + remote = request.META.get("REMOTE_ADDR", "unknown") + forwarded = [ + part.strip() + for part in request.META.get("HTTP_X_FORWARDED_FOR", "").split(",") + if part.strip() + ] + trusted = settings.EVECTION_TRUSTED_PROXIES + + if not trusted: + return forwarded[0] if forwarded else remote + if not _is_trusted_proxy(remote, trusted): + return remote + for address in reversed(forwarded): + if not _is_trusted_proxy(address, trusted): + return address + return forwarded[0] if forwarded else remote diff --git a/apps/accounts/views.py b/apps/accounts/views.py index 28f5770..4dd3d28 100644 --- a/apps/accounts/views.py +++ b/apps/accounts/views.py @@ -4,13 +4,13 @@ from django.contrib import messages from django.contrib.auth import login from django.contrib.auth.decorators import user_passes_test from django.contrib.auth.views import LoginView as DjangoLoginView -from django.core.cache import cache from django.db import IntegrityError, transaction from django.shortcuts import redirect, render from .forms import CreateUserForm, SetupForm from .middleware import SetupWizardMiddleware from .models import SetupLock, User +from .throttle import LoginThrottle, client_address logger = logging.getLogger(__name__) @@ -27,44 +27,13 @@ LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_ADDRESS = 15 LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME = 8 -def _client_address(request) -> str: - # gunicorn is always run behind a reverse proxy in every deployment - # (see docs/install.md) and its bind address is never meant to be - # reachable from outside that proxy — the same trust boundary - # config.settings.prod already relies on for X-Forwarded-Proto. A - # direct request (e.g. local dev) has no X-Forwarded-For and falls - # back to REMOTE_ADDR. - forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR", "") - if forwarded_for: - return forwarded_for.split(",")[0].strip() - return request.META.get("REMOTE_ADDR", "unknown") +login_throttle = LoginThrottle(LOGIN_RATE_LIMIT_WINDOW_SECONDS) def _attempts_key(kind: str, value: str) -> str: return f"login-attempts:{kind}:{value}" -def _too_many_attempts(key: str, limit: int) -> bool: - # Fails open if the cache (Redis) is unreachable: logging in must keep - # working through a cache outage, at the cost of no throttling until - # it's back — an outage shouldn't lock every user out of the app. - try: - return cache.get(key, 0) >= limit - except Exception: - logger.warning("Login throttle cache unavailable; not rate limiting", exc_info=True) - return False - - -def _record_failed_attempt(key: str) -> None: - try: - try: - cache.incr(key) - except ValueError: - cache.set(key, 1, LOGIN_RATE_LIMIT_WINDOW_SECONDS) - except Exception: - logger.warning("Login throttle cache unavailable; attempt not recorded", exc_info=True) - - class LoginView(DjangoLoginView): """Wraps Django's own LoginView with the throttle above — everything else (form, template, redirect behavior) is unchanged.""" @@ -72,11 +41,11 @@ class LoginView(DjangoLoginView): template_name = "accounts/login.html" def post(self, request, *args, **kwargs): - address_key = _attempts_key("addr", _client_address(request)) + address_key = _attempts_key("addr", client_address(request)) username_key = _attempts_key("user", request.POST.get("username", "")[:150]) - if _too_many_attempts( + if login_throttle.is_blocked( address_key, LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_ADDRESS - ) or _too_many_attempts(username_key, LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME): + ) or login_throttle.is_blocked(username_key, LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME): messages.error( request, "Too many login attempts. Please wait a few minutes and try again." ) @@ -84,8 +53,8 @@ class LoginView(DjangoLoginView): return super().post(request, *args, **kwargs) def form_invalid(self, form): - _record_failed_attempt(_attempts_key("addr", _client_address(self.request))) - _record_failed_attempt(_attempts_key("user", self.request.POST.get("username", "")[:150])) + login_throttle.record(_attempts_key("addr", client_address(self.request))) + login_throttle.record(_attempts_key("user", self.request.POST.get("username", "")[:150])) return super().form_invalid(form) diff --git a/apps/core/health.py b/apps/core/health.py index 31110ff..8669ea9 100644 --- a/apps/core/health.py +++ b/apps/core/health.py @@ -2,6 +2,7 @@ from datetime import timedelta import redis from django.conf import settings +from django.core.cache import cache from django.db.models import Max, Min from django.utils import timezone @@ -42,6 +43,33 @@ def _success_window() -> timedelta: return timedelta(minutes=max(3 * base, MIN_SUCCESS_WINDOW_MINUTES)) +PIPELINE_CACHE_KEY = "health-ready-pipeline" + + +def cached_fetch_pipeline_status() -> tuple[dict[str, bool], dict]: + """fetch_pipeline_status, reused for EVECTION_HEALTH_CACHE_SECONDS + (R2-C5): the queue/feed aggregates behind it cost real queries, and a + monitor that polls /health/ready/ every second shouldn't turn that + into recurring database and Redis load. The cheap liveness probes + (database, Redis, poll heartbeat) are not cached — those stay live. + A cache failure just means computing it fresh.""" + ttl = settings.EVECTION_HEALTH_CACHE_SECONDS + if ttl > 0: + try: + cached = cache.get(PIPELINE_CACHE_KEY) + except Exception: + cached = None + if cached is not None: + return cached + result = fetch_pipeline_status() + if ttl > 0: + try: + cache.set(PIPELINE_CACHE_KEY, result, timeout=ttl) + except Exception: + pass + return result + + def fetch_pipeline_status(now=None) -> tuple[dict[str, bool], dict]: """Checks that fetching is actually *succeeding*, not just scheduled (the poll heartbeat only proves beat and a worker are alive): returns diff --git a/apps/core/tests.py b/apps/core/tests.py index 81dfaa1..405b56c 100644 --- a/apps/core/tests.py +++ b/apps/core/tests.py @@ -265,3 +265,45 @@ def test_health_ready_flags_most_feeds_failing_but_not_a_single_broken_one(clien assert body["checks"]["feed_failures"] is False assert body["failing_feeds"] == 3 assert healthy.pk and broken + + +@pytest.mark.django_db +def test_readiness_reuses_the_pipeline_aggregates_for_a_short_time(client, broker_queue, settings): + """R2-C5: the expensive part of the readiness check is cached briefly, + while the cheap liveness probes stay live.""" + from django.core.cache import cache + from django.utils import timezone + + from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY + + settings.EVECTION_HEALTH_CACHE_SECONDS = 30 + cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now()) + Feed.objects.create(feed_url="https://a.example.com/feed.xml") + + first = client.get(reverse("core:health_ready")).json() + Feed.objects.create(feed_url="https://b.example.com/feed.xml") + second = client.get(reverse("core:health_ready")).json() + + assert first["active_feeds"] == 1 + assert second["active_feeds"] == 1, "served from the short-lived cache" + + cache.delete("health-ready-pipeline") + third = client.get(reverse("core:health_ready")).json() + assert third["active_feeds"] == 2 + + +@pytest.mark.django_db +def test_readiness_liveness_probes_are_never_cached(client, broker_queue, settings): + from django.core.cache import cache + + settings.EVECTION_HEALTH_CACHE_SECONDS = 30 + first = client.get(reverse("core:health_ready")) + assert first.json()["checks"]["poll_heartbeat"] is False + + from django.utils import timezone + + from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY + + cache.set(POLL_HEARTBEAT_CACHE_KEY, timezone.now()) + second = client.get(reverse("core:health_ready")) + assert second.json()["checks"]["poll_heartbeat"] is True diff --git a/apps/core/views.py b/apps/core/views.py index 988f1c5..1af8d3a 100644 --- a/apps/core/views.py +++ b/apps/core/views.py @@ -12,7 +12,7 @@ from apps.articles.models import Article from apps.feeds.models import Feed, Subscription from apps.feeds.tasks import POLL_HEARTBEAT_CACHE_KEY, enqueue_fetch, reset_backoff -from .health import fetch_pipeline_status +from .health import cached_fetch_pipeline_status is_admin = user_passes_test(lambda u: u.is_authenticated and u.is_staff) @@ -70,7 +70,7 @@ def health_ready(request): checks["poll_heartbeat"] = age <= POLL_HEARTBEAT_MAX_AGE_SECONDS if checks["database"]: - pipeline_checks, pipeline_detail = fetch_pipeline_status() + pipeline_checks, pipeline_detail = cached_fetch_pipeline_status() checks.update(pipeline_checks) detail.update(pipeline_detail) diff --git a/config/settings/base.py b/config/settings/base.py index 5869f19..d92bd47 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -4,6 +4,7 @@ Base Django settings for Evection, shared by all environments. Environment-specific overrides live in dev.py / prod.py / test.py. """ +import ipaddress from pathlib import Path import environ @@ -122,6 +123,18 @@ EVECTION_SIGNUP_ENABLED = env.bool("EVECTION_SIGNUP_ENABLED", default=False) EVECTION_MIN_FETCH_INTERVAL_MINUTES = env.int("EVECTION_MIN_FETCH_INTERVAL_MINUTES", default=15) EVECTION_PWA_CACHE_VERSION = env.str("EVECTION_PWA_CACHE_VERSION", default="v1") +# How long /health/ready/ reuses its queue/feed aggregates (0 = never). +EVECTION_HEALTH_CACHE_SECONDS = env.int("EVECTION_HEALTH_CACHE_SECONDS", default=15) + +# IPs/CIDRs of the reverse proxy in front of gunicorn. When set, the login +# throttle only believes X-Forwarded-For from these peers (see +# apps.accounts.throttle.client_address). Empty keeps the legacy behavior. +EVECTION_TRUSTED_PROXIES = [ + ipaddress.ip_network(entry.strip(), strict=False) + for entry in env.list("EVECTION_TRUSTED_PROXIES", default=[]) + if entry.strip() +] + # Redis-backed rather than Django's default per-process LocMemCache: login # rate limiting (apps.accounts.views) must share counters across every # gunicorn worker, not just whichever one happened to handle a given diff --git a/config/settings/test.py b/config/settings/test.py index b1fe71e..b0c0659 100644 --- a/config/settings/test.py +++ b/config/settings/test.py @@ -12,3 +12,6 @@ PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"] # location is per-process but shared across tests in that process; see # apps.accounts.test_rate_limit's cache.clear() fixture). CACHES = {"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}} + +# Tests change state between calls to /health/ready/; caching it would hide that. +EVECTION_HEALTH_CACHE_SECONDS = 0 diff --git a/docs/install.md b/docs/install.md index 87fa4f0..9038ef5 100644 --- a/docs/install.md +++ b/docs/install.md @@ -150,6 +150,15 @@ At minimum, set the following in `.env`: - `REDIS_URL` — `redis://localhost:6379/0` is fine for a single-host install - `EVECTION_GUNICORN_BIND` — the `address:port` gunicorn listens on; see "Reverse proxy" below for how to pick this +- `EVECTION_TRUSTED_PROXIES` — the IP address (or CIDR, comma-separated for more + than one) of your reverse proxy, as gunicorn sees it connecting. The login + throttle only believes `X-Forwarded-For` from these addresses, and takes the + address your proxy actually saw rather than the first entry (which a client can + forge). Leave it unset and it falls back to trusting that header from anyone, + which is only safe while nothing but your proxy can reach gunicorn; + `manage.py check --deploy` warns until it's set. With Nginx Proxy Manager on + another host, use that host's address (check `ss -tn` on this machine while + loading the site if you're not sure what gunicorn sees). How often every feed is polled is a site-wide setting (Django admin → Site settings), not a `.env` value — `EVECTION_MIN_FETCH_INTERVAL_MINUTES` only sets @@ -339,7 +348,12 @@ with a per-check breakdown when any of these fail: failing at once points at your server, DNS, or outbound network). The payload also reports `queue_depth`, `active_feeds`, `failing_feeds`, and -the age of the last successful fetch, so you can graph them. +the age of the last successful fetch, so you can graph them. The queue and feed +figures are reused for 15 seconds (`EVECTION_HEALTH_CACHE_SECONDS`, 0 disables +it) so a fast-polling monitor can't turn them into steady database load; the +database, Redis and poller checks are always live. The endpoint is +unauthenticated, so if you'd rather not expose those numbers publicly, restrict +`/health/ready/` to your monitor's address at the proxy. **Retention.** By default every fetched article is kept forever. In the Django admin, "Site settings" has a *Retention days* field: set it to delete articles @@ -347,8 +361,12 @@ fetched longer ago than that, nightly (03:30 UTC, via Celery beat). Articles anyone has starred or saved for later are always kept; read state and tags for deleted articles go with them. A small record (feed and entry id) is kept for each deleted article so a feed that still lists it can't bring it back as new -and unread, and entries whose own publish date is already older than the window -aren't fetched in the first place. The dashboard shows the article count and the +and unread — those records are themselves removed after a year, so an entry a +feed is *still* publishing a year after it was purged (with no publish date, or +a recent one) can come back as new. Entries whose own publish date is already +older than the window aren't fetched in the first place. A large first purge +works in time-boxed runs (about four minutes each) that queue their own +continuation, so it never monopolises a worker. The dashboard shows the article count and the age of the oldest one, so you can see what retention is doing. **SQLite.** SQLite installs run in WAL mode with a 20-second busy timeout (set