Private
Public Access
Compare commits
18
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7bb981b545 | ||
|
|
fd68400822 | ||
|
|
90837b339c | ||
|
|
dc8b9ff83b | ||
|
|
a1ea2e5662 | ||
|
|
9241a54958 | ||
|
|
64f25789f5 | ||
|
|
0466ccc334 | ||
|
|
e760eeda1d | ||
|
|
9439f9cc59 | ||
|
|
fd8eeb605e | ||
|
|
91c0b1dd26 | ||
|
|
f2ee793cf8 | ||
|
|
b7d9ee672c | ||
|
|
d7b433e580 | ||
|
|
b52ba2be8a | ||
|
|
56eb2bc498 | ||
|
|
9d91da6fc6 |
No files matched your search
@@ -0,0 +1,22 @@
|
|||||||
|
# Generated by Django 5.2.18 on 2026-10-31 18:00
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
("accounts", "0001_initial"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.CreateModel(
|
||||||
|
name="SetupLock",
|
||||||
|
fields=[
|
||||||
|
(
|
||||||
|
"id",
|
||||||
|
models.PositiveSmallIntegerField(default=1, primary_key=True, serialize=False),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
]
|
||||||
@@ -5,3 +5,15 @@ from django.db import models
|
|||||||
class User(AbstractUser):
|
class User(AbstractUser):
|
||||||
timezone = models.CharField(max_length=64, default="UTC")
|
timezone = models.CharField(max_length=64, default="UTC")
|
||||||
created_at = models.DateTimeField(auto_now_add=True)
|
created_at = models.DateTimeField(auto_now_add=True)
|
||||||
|
|
||||||
|
|
||||||
|
class SetupLock(models.Model):
|
||||||
|
"""Singleton row that gates first-run setup against a race between
|
||||||
|
concurrent requests. The setup view always tries to INSERT the row
|
||||||
|
with pk=1 in the same transaction as creating the first admin user:
|
||||||
|
the database's primary-key uniqueness makes exactly one of those
|
||||||
|
INSERTs win even under concurrent requests, so only one admin account
|
||||||
|
can ever be created this way, instead of the User.objects.exists()
|
||||||
|
check alone (which has a window between the check and the save)."""
|
||||||
|
|
||||||
|
id = models.PositiveSmallIntegerField(primary_key=True, default=1)
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
import pytest
|
||||||
|
from django.contrib.auth import get_user_model
|
||||||
|
from django.core.cache import cache
|
||||||
|
from django.urls import reverse
|
||||||
|
|
||||||
|
from .views import (
|
||||||
|
LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_ADDRESS,
|
||||||
|
LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME,
|
||||||
|
)
|
||||||
|
|
||||||
|
User = get_user_model()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _clear_rate_limit_cache():
|
||||||
|
cache.clear()
|
||||||
|
yield
|
||||||
|
cache.clear()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def alice(db):
|
||||||
|
return User.objects.create_user("alice", password="pw-alice-123")
|
||||||
|
|
||||||
|
|
||||||
|
def _bad_login(client, username="alice"):
|
||||||
|
return client.post(reverse("accounts:login"), {"username": username, "password": "wrong"})
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_a_single_failed_login_still_shows_the_normal_error(client, alice):
|
||||||
|
response = _bad_login(client)
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert b"Please enter a correct" in response.content or response.context["form"].errors
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_correct_credentials_succeed_before_the_limit_is_hit(client, alice):
|
||||||
|
for _ in range(3):
|
||||||
|
_bad_login(client)
|
||||||
|
|
||||||
|
response = client.post(
|
||||||
|
reverse("accounts:login"), {"username": "alice", "password": "pw-alice-123"}
|
||||||
|
)
|
||||||
|
assert response.status_code == 302
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_too_many_failed_attempts_from_one_address_blocks_further_tries(client, alice):
|
||||||
|
for _ in range(LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_ADDRESS):
|
||||||
|
_bad_login(client)
|
||||||
|
|
||||||
|
# Even the correct password is now refused — the throttle runs
|
||||||
|
# before the form is ever validated.
|
||||||
|
response = client.post(
|
||||||
|
reverse("accounts:login"), {"username": "alice", "password": "pw-alice-123"}
|
||||||
|
)
|
||||||
|
assert response.status_code == 302
|
||||||
|
assert response.url == reverse("accounts:login")
|
||||||
|
assert not response.wsgi_request.user.is_authenticated
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_too_many_failed_attempts_against_one_username_blocks_that_username(client, alice):
|
||||||
|
bob = User.objects.create_user("bob", password="pw-bob-123")
|
||||||
|
|
||||||
|
for _ in range(LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME):
|
||||||
|
_bad_login(client, username="alice")
|
||||||
|
|
||||||
|
# Same client address, different username — the per-username
|
||||||
|
# throttle for "alice" shouldn't affect bob logging in.
|
||||||
|
response = client.post(reverse("accounts:login"), {"username": "bob", "password": "pw-bob-123"})
|
||||||
|
assert response.status_code == 302
|
||||||
|
assert response.wsgi_request.user == bob
|
||||||
+23
-1
@@ -1,7 +1,7 @@
|
|||||||
import pytest
|
import pytest
|
||||||
from django.urls import reverse
|
from django.urls import reverse
|
||||||
|
|
||||||
from .models import User
|
from .models import SetupLock, User
|
||||||
|
|
||||||
# SetupWizardMiddleware's cache is reset around every test by the autouse
|
# SetupWizardMiddleware's cache is reset around every test by the autouse
|
||||||
# fixture in the top-level conftest.py, not just here.
|
# fixture in the top-level conftest.py, not just here.
|
||||||
@@ -43,6 +43,28 @@ def test_setup_creates_admin_and_logs_in(client):
|
|||||||
assert stream_response.status_code == 200
|
assert stream_response.status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_setup_refuses_a_second_concurrent_submission(client):
|
||||||
|
# Simulates two requests racing the setup form: the first claims the
|
||||||
|
# SetupLock row directly (standing in for a request that's already
|
||||||
|
# past the User.objects.exists() check), so a second submission's
|
||||||
|
# INSERT must lose instead of also creating an admin account.
|
||||||
|
SetupLock.objects.create()
|
||||||
|
|
||||||
|
response = client.post(
|
||||||
|
reverse("accounts:setup"),
|
||||||
|
{
|
||||||
|
"username": "raced",
|
||||||
|
"email": "raced@example.com",
|
||||||
|
"password": "correct-horse-battery",
|
||||||
|
"password_confirm": "correct-horse-battery",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert response.status_code == 302
|
||||||
|
assert response.url == reverse("accounts:login")
|
||||||
|
assert not User.objects.filter(username="raced").exists()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
def test_setup_is_unreachable_once_a_user_exists(client):
|
def test_setup_is_unreachable_once_a_user_exists(client):
|
||||||
User.objects.create_user("existing", password="irrelevant")
|
User.objects.create_user("existing", password="irrelevant")
|
||||||
|
|||||||
@@ -7,11 +7,7 @@ app_name = "accounts"
|
|||||||
|
|
||||||
urlpatterns = [
|
urlpatterns = [
|
||||||
path("setup/", views.setup, name="setup"),
|
path("setup/", views.setup, name="setup"),
|
||||||
path(
|
path("login/", views.LoginView.as_view(), name="login"),
|
||||||
"login/",
|
|
||||||
auth_views.LoginView.as_view(template_name="accounts/login.html"),
|
|
||||||
name="login",
|
|
||||||
),
|
|
||||||
path("logout/", auth_views.LogoutView.as_view(), name="logout"),
|
path("logout/", auth_views.LogoutView.as_view(), name="logout"),
|
||||||
path("users/", views.manage_users, name="manage_users"),
|
path("users/", views.manage_users, name="manage_users"),
|
||||||
path("users/create/", views.create_user, name="create_user"),
|
path("users/create/", views.create_user, name="create_user"),
|
||||||
|
|||||||
+81
-6
@@ -1,14 +1,79 @@
|
|||||||
from django.contrib import messages
|
from django.contrib import messages
|
||||||
from django.contrib.auth import login
|
from django.contrib.auth import login
|
||||||
from django.contrib.auth.decorators import user_passes_test
|
from django.contrib.auth.decorators import user_passes_test
|
||||||
|
from django.contrib.auth.views import LoginView as DjangoLoginView
|
||||||
|
from django.core.cache import cache
|
||||||
|
from django.db import IntegrityError, transaction
|
||||||
from django.shortcuts import redirect, render
|
from django.shortcuts import redirect, render
|
||||||
|
|
||||||
from .forms import CreateUserForm, SetupForm
|
from .forms import CreateUserForm, SetupForm
|
||||||
from .middleware import SetupWizardMiddleware
|
from .middleware import SetupWizardMiddleware
|
||||||
from .models import User
|
from .models import SetupLock, User
|
||||||
|
|
||||||
is_admin = user_passes_test(lambda u: u.is_authenticated and u.is_staff)
|
is_admin = user_passes_test(lambda u: u.is_authenticated and u.is_staff)
|
||||||
|
|
||||||
|
# Two independent throttles, either of which can block an attempt: by
|
||||||
|
# client address (catches one attacker spraying many usernames) and by
|
||||||
|
# the submitted username (catches a distributed/low-and-slow attack on
|
||||||
|
# one specific account from many addresses). Both are generous enough
|
||||||
|
# not to lock out a user who's just fat-fingered their own password a
|
||||||
|
# few times.
|
||||||
|
LOGIN_RATE_LIMIT_WINDOW_SECONDS = 5 * 60
|
||||||
|
LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_ADDRESS = 15
|
||||||
|
LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME = 8
|
||||||
|
|
||||||
|
|
||||||
|
def _client_address(request) -> str:
|
||||||
|
# gunicorn is always run behind a reverse proxy in every deployment
|
||||||
|
# (see docs/install.md) and its bind address is never meant to be
|
||||||
|
# reachable from outside that proxy — the same trust boundary
|
||||||
|
# config.settings.prod already relies on for X-Forwarded-Proto. A
|
||||||
|
# direct request (e.g. local dev) has no X-Forwarded-For and falls
|
||||||
|
# back to REMOTE_ADDR.
|
||||||
|
forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR", "")
|
||||||
|
if forwarded_for:
|
||||||
|
return forwarded_for.split(",")[0].strip()
|
||||||
|
return request.META.get("REMOTE_ADDR", "unknown")
|
||||||
|
|
||||||
|
|
||||||
|
def _attempts_key(kind: str, value: str) -> str:
|
||||||
|
return f"login-attempts:{kind}:{value}"
|
||||||
|
|
||||||
|
|
||||||
|
def _too_many_attempts(key: str, limit: int) -> bool:
|
||||||
|
return cache.get(key, 0) >= limit
|
||||||
|
|
||||||
|
|
||||||
|
def _record_failed_attempt(key: str) -> None:
|
||||||
|
try:
|
||||||
|
cache.incr(key)
|
||||||
|
except ValueError:
|
||||||
|
cache.set(key, 1, LOGIN_RATE_LIMIT_WINDOW_SECONDS)
|
||||||
|
|
||||||
|
|
||||||
|
class LoginView(DjangoLoginView):
|
||||||
|
"""Wraps Django's own LoginView with the throttle above — everything
|
||||||
|
else (form, template, redirect behavior) is unchanged."""
|
||||||
|
|
||||||
|
template_name = "accounts/login.html"
|
||||||
|
|
||||||
|
def post(self, request, *args, **kwargs):
|
||||||
|
address_key = _attempts_key("addr", _client_address(request))
|
||||||
|
username_key = _attempts_key("user", request.POST.get("username", "")[:150])
|
||||||
|
if _too_many_attempts(
|
||||||
|
address_key, LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_ADDRESS
|
||||||
|
) or _too_many_attempts(username_key, LOGIN_RATE_LIMIT_MAX_ATTEMPTS_PER_USERNAME):
|
||||||
|
messages.error(
|
||||||
|
request, "Too many login attempts. Please wait a few minutes and try again."
|
||||||
|
)
|
||||||
|
return redirect("accounts:login")
|
||||||
|
return super().post(request, *args, **kwargs)
|
||||||
|
|
||||||
|
def form_invalid(self, form):
|
||||||
|
_record_failed_attempt(_attempts_key("addr", _client_address(self.request)))
|
||||||
|
_record_failed_attempt(_attempts_key("user", self.request.POST.get("username", "")[:150]))
|
||||||
|
return super().form_invalid(form)
|
||||||
|
|
||||||
|
|
||||||
def setup(request):
|
def setup(request):
|
||||||
if User.objects.exists():
|
if User.objects.exists():
|
||||||
@@ -16,11 +81,21 @@ def setup(request):
|
|||||||
|
|
||||||
form = SetupForm(request.POST or None)
|
form = SetupForm(request.POST or None)
|
||||||
if request.method == "POST" and form.is_valid():
|
if request.method == "POST" and form.is_valid():
|
||||||
user = form.save(commit=False)
|
try:
|
||||||
user.set_password(form.cleaned_data["password"])
|
with transaction.atomic():
|
||||||
user.is_staff = True
|
# Claims the one-time setup slot: the first of any
|
||||||
user.is_superuser = True
|
# concurrent requests to reach this INSERT wins it, and
|
||||||
user.save()
|
# every other one raises IntegrityError instead of also
|
||||||
|
# creating an admin account. See SetupLock's docstring.
|
||||||
|
SetupLock.objects.create()
|
||||||
|
user = form.save(commit=False)
|
||||||
|
user.set_password(form.cleaned_data["password"])
|
||||||
|
user.is_staff = True
|
||||||
|
user.is_superuser = True
|
||||||
|
user.save()
|
||||||
|
except IntegrityError:
|
||||||
|
messages.info(request, "Setup was already completed. Log in with that account.")
|
||||||
|
return redirect("accounts:login")
|
||||||
SetupWizardMiddleware._a_user_exists = True
|
SetupWizardMiddleware._a_user_exists = True
|
||||||
login(request, user)
|
login(request, user)
|
||||||
messages.success(request, "Welcome to Evection! Your admin account is ready.")
|
messages.success(request, "Welcome to Evection! Your admin account is ready.")
|
||||||
|
|||||||
+12
-12
@@ -1,5 +1,4 @@
|
|||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
from django.db.models import Q
|
|
||||||
|
|
||||||
from apps.feeds.models import Subscription
|
from apps.feeds.models import Subscription
|
||||||
from apps.rules.engine import evaluate_rules_for_article
|
from apps.rules.engine import evaluate_rules_for_article
|
||||||
@@ -14,6 +13,10 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
|
|||||||
FilterRules (scoped to this feed/tag or unscoped) against the given
|
FilterRules (scoped to this feed/tag or unscoped) against the given
|
||||||
articles. Dispatched by the fetch pipeline after new/updated articles
|
articles. Dispatched by the fetch pipeline after new/updated articles
|
||||||
are committed for a feed.
|
are committed for a feed.
|
||||||
|
|
||||||
|
Per-rule scoping is left to evaluate_rules_for_article/rule_in_scope
|
||||||
|
rather than filtered into this query, so this and the backlog
|
||||||
|
re-apply task (apps.rules.tasks) can't scope rules differently.
|
||||||
"""
|
"""
|
||||||
if not article_ids:
|
if not article_ids:
|
||||||
return 0
|
return 0
|
||||||
@@ -26,24 +29,21 @@ def apply_filter_rules_for_feed(feed_id: int, article_ids: list[int]):
|
|||||||
|
|
||||||
evaluated = 0
|
evaluated = 0
|
||||||
for subscription in subscriptions:
|
for subscription in subscriptions:
|
||||||
rules = FilterRule.objects.filter(user=subscription.user, is_enabled=True).filter(
|
rules = list(
|
||||||
Q(feed_id=feed_id) | Q(feed__isnull=True)
|
FilterRule.objects.filter(user=subscription.user, is_enabled=True).order_by(
|
||||||
|
"priority", "id"
|
||||||
|
)
|
||||||
)
|
)
|
||||||
subscription_tag_ids = list(
|
if not rules:
|
||||||
|
continue
|
||||||
|
subscription_tag_ids = set(
|
||||||
SubscriptionTag.objects.filter(
|
SubscriptionTag.objects.filter(
|
||||||
user=subscription.user, subscription=subscription
|
user=subscription.user, subscription=subscription
|
||||||
).values_list("tag_id", flat=True)
|
).values_list("tag_id", flat=True)
|
||||||
)
|
)
|
||||||
if subscription_tag_ids:
|
|
||||||
rules = rules.filter(Q(tag_id__in=subscription_tag_ids) | Q(tag__isnull=True))
|
|
||||||
else:
|
|
||||||
rules = rules.filter(tag__isnull=True)
|
|
||||||
rules = list(rules.order_by("priority", "id"))
|
|
||||||
if not rules:
|
|
||||||
continue
|
|
||||||
|
|
||||||
for article in articles:
|
for article in articles:
|
||||||
evaluate_rules_for_article(subscription.user, article, rules)
|
evaluate_rules_for_article(subscription.user, article, rules, subscription_tag_ids)
|
||||||
evaluated += 1
|
evaluated += 1
|
||||||
|
|
||||||
return evaluated
|
return evaluated
|
||||||
+28
-1
@@ -1 +1,28 @@
|
|||||||
# Register your models here.
|
from django.contrib import admin
|
||||||
|
from django.shortcuts import redirect
|
||||||
|
from django.urls import reverse
|
||||||
|
|
||||||
|
from .models import SiteSettings
|
||||||
|
|
||||||
|
|
||||||
|
@admin.register(SiteSettings)
|
||||||
|
class SiteSettingsAdmin(admin.ModelAdmin):
|
||||||
|
"""A true singleton, not a collection — browsing a changelist to pick
|
||||||
|
the one row and click into it is an extra, pointless step, so
|
||||||
|
"Core > Site settings" in the sidebar goes straight to the (only)
|
||||||
|
change form instead, creating the row first if this is a fresh
|
||||||
|
install. No changelist/add/delete views exist at all here.
|
||||||
|
"""
|
||||||
|
|
||||||
|
fields = ["fetch_interval_minutes"]
|
||||||
|
|
||||||
|
def has_add_permission(self, request):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def has_delete_permission(self, request, obj=None):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def changelist_view(self, request, extra_context=None):
|
||||||
|
settings_row = SiteSettings.load()
|
||||||
|
url = reverse("admin:core_sitesettings_change", args=[settings_row.pk])
|
||||||
|
return redirect(url)
|
||||||
@@ -44,7 +44,19 @@ def sidebar(request):
|
|||||||
|
|
||||||
return {
|
return {
|
||||||
"sidebar_tag_groups": [
|
"sidebar_tag_groups": [
|
||||||
{"tag": tag, "subscriptions": feeds_by_tag.get(tag.pk, [])} for tag in tags
|
{
|
||||||
|
"tag": tag,
|
||||||
|
"subscriptions": feeds_by_tag.get(tag.pk, []),
|
||||||
|
# A feed can carry more than one tag, so this is a sum
|
||||||
|
# over whichever of the user's feeds this tag happens to
|
||||||
|
# cover right now — not a separately-tracked counter — and
|
||||||
|
# reuses the per-feed counts already computed above rather
|
||||||
|
# than a second query.
|
||||||
|
"unread_count": sum(
|
||||||
|
subscription.unread_count for subscription in feeds_by_tag.get(tag.pk, [])
|
||||||
|
),
|
||||||
|
}
|
||||||
|
for tag in tags
|
||||||
],
|
],
|
||||||
"sidebar_untagged": untagged,
|
"sidebar_untagged": untagged,
|
||||||
"sidebar_starred_count": ArticleTag.objects.filter(
|
"sidebar_starred_count": ArticleTag.objects.filter(
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# Generated by Django 5.2.18 on 2026-10-31 18:56
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
initial = True
|
||||||
|
|
||||||
|
dependencies = []
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.CreateModel(
|
||||||
|
name="SiteSettings",
|
||||||
|
fields=[
|
||||||
|
(
|
||||||
|
"id",
|
||||||
|
models.PositiveSmallIntegerField(default=1, primary_key=True, serialize=False),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"fetch_interval_minutes",
|
||||||
|
models.PositiveIntegerField(
|
||||||
|
default=60,
|
||||||
|
help_text="How often, in minutes, every feed is polled for new articles. Can't be set lower than EVECTION_MIN_FETCH_INTERVAL_MINUTES (see .env).",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
options={
|
||||||
|
"verbose_name": "Site settings",
|
||||||
|
"verbose_name_plural": "Site settings",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
]
|
||||||
+35
-1
@@ -1 +1,35 @@
|
|||||||
# Create your models here.
|
from django.db import models
|
||||||
|
|
||||||
|
|
||||||
|
class SiteSettings(models.Model):
|
||||||
|
"""Singleton row (pk=1, same pattern as accounts.SetupLock) holding
|
||||||
|
instance-wide configuration an operator reasonably wants to change
|
||||||
|
without touching the server or .env — editable via Django admin.
|
||||||
|
Currently just the feed-poll interval, which used to be a per-feed
|
||||||
|
field; it's site-wide now (see apps.feeds.tasks.poll_due_feeds).
|
||||||
|
"""
|
||||||
|
|
||||||
|
id = models.PositiveSmallIntegerField(primary_key=True, default=1)
|
||||||
|
fetch_interval_minutes = models.PositiveIntegerField(
|
||||||
|
default=60,
|
||||||
|
help_text=(
|
||||||
|
"How often, in minutes, every feed is polled for new articles. "
|
||||||
|
"Can't be set lower than EVECTION_MIN_FETCH_INTERVAL_MINUTES (see .env)."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
verbose_name = "Site settings"
|
||||||
|
verbose_name_plural = "Site settings"
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return "Site settings"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def load(cls):
|
||||||
|
"""Returns the singleton row, creating it with defaults on first
|
||||||
|
use so callers never need to special-case "no row exists yet"
|
||||||
|
(e.g. a fresh install whose first poll_due_feeds run hasn't
|
||||||
|
created it, and whose admin hasn't visited the settings page)."""
|
||||||
|
obj, _ = cls.objects.get_or_create(pk=1)
|
||||||
|
return obj
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
import pytest
|
||||||
|
from django.contrib.auth import get_user_model
|
||||||
|
from django.test import RequestFactory
|
||||||
|
from django.urls import reverse
|
||||||
|
|
||||||
|
from apps.articles.models import Article, SubscriptionTag, Tag
|
||||||
|
from apps.feeds.models import Feed, Subscription
|
||||||
|
|
||||||
|
from .context_processors import sidebar
|
||||||
|
|
||||||
|
User = get_user_model()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def alice(db):
|
||||||
|
return User.objects.create_user("alice", password="pw-alice-123")
|
||||||
|
|
||||||
|
|
||||||
|
def _sidebar_context(user):
|
||||||
|
request = RequestFactory().get("/")
|
||||||
|
request.user = user
|
||||||
|
return sidebar(request)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_tag_unread_count_sums_its_feeds(alice):
|
||||||
|
feed_a = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
||||||
|
feed_b = Feed.objects.create(feed_url="https://b.example.com/feed.xml")
|
||||||
|
sub_a = Subscription.objects.create(user=alice, feed=feed_a)
|
||||||
|
sub_b = Subscription.objects.create(user=alice, feed=feed_b)
|
||||||
|
tag = Tag.objects.create(user=alice, name="work")
|
||||||
|
SubscriptionTag.objects.create(user=alice, subscription=sub_a, tag=tag)
|
||||||
|
SubscriptionTag.objects.create(user=alice, subscription=sub_b, tag=tag)
|
||||||
|
|
||||||
|
for i in range(3):
|
||||||
|
Article.objects.create(feed=feed_a, guid=f"a-{i}", title=f"A{i}")
|
||||||
|
for i in range(4):
|
||||||
|
Article.objects.create(feed=feed_b, guid=f"b-{i}", title=f"B{i}")
|
||||||
|
|
||||||
|
context = _sidebar_context(alice)
|
||||||
|
[group] = context["sidebar_tag_groups"]
|
||||||
|
assert group["tag"] == tag
|
||||||
|
assert group["unread_count"] == 7
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_tag_unread_count_is_zero_with_no_unread_articles(alice):
|
||||||
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||||
|
sub = Subscription.objects.create(user=alice, feed=feed)
|
||||||
|
tag = Tag.objects.create(user=alice, name="empty")
|
||||||
|
SubscriptionTag.objects.create(user=alice, subscription=sub, tag=tag)
|
||||||
|
|
||||||
|
context = _sidebar_context(alice)
|
||||||
|
[group] = context["sidebar_tag_groups"]
|
||||||
|
assert group["unread_count"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_sidebar_sections_carry_a_stable_collapse_state_key(client, alice):
|
||||||
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||||
|
sub = Subscription.objects.create(user=alice, feed=feed)
|
||||||
|
tag = Tag.objects.create(user=alice, name="work")
|
||||||
|
SubscriptionTag.objects.create(user=alice, subscription=sub, tag=tag)
|
||||||
|
|
||||||
|
other_feed = Feed.objects.create(feed_url="https://other.example.com/feed.xml")
|
||||||
|
Subscription.objects.create(user=alice, feed=other_feed) # untagged
|
||||||
|
|
||||||
|
client.login(username="alice", password="pw-alice-123")
|
||||||
|
response = client.get(reverse("reader:stream"))
|
||||||
|
content = response.content.decode()
|
||||||
|
|
||||||
|
# static/js/sidebar-state.js reads these to restore/persist each
|
||||||
|
# section's collapsed state across page loads.
|
||||||
|
assert f'data-sidebar-key="tag-{tag.pk}"' in content
|
||||||
|
assert 'data-sidebar-key="untagged"' in content
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_base_template_loads_the_sidebar_state_script(client, alice):
|
||||||
|
client.login(username="alice", password="pw-alice-123")
|
||||||
|
response = client.get(reverse("reader:stream"))
|
||||||
|
assert "sidebar-state.js" in response.content.decode()
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
from datetime import timedelta
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from django.contrib.auth import get_user_model
|
||||||
|
from django.urls import reverse
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
from apps.feeds.models import Feed
|
||||||
|
from apps.feeds.tasks import poll_due_feeds
|
||||||
|
|
||||||
|
from .models import SiteSettings
|
||||||
|
|
||||||
|
User = get_user_model()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_load_creates_the_singleton_row_with_defaults():
|
||||||
|
assert not SiteSettings.objects.exists()
|
||||||
|
settings_row = SiteSettings.load()
|
||||||
|
assert settings_row.fetch_interval_minutes == 60
|
||||||
|
assert SiteSettings.objects.count() == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_load_returns_the_existing_row_without_creating_a_second_one():
|
||||||
|
SiteSettings.objects.create(fetch_interval_minutes=30)
|
||||||
|
settings_row = SiteSettings.load()
|
||||||
|
assert settings_row.fetch_interval_minutes == 30
|
||||||
|
assert SiteSettings.objects.count() == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_poll_due_feeds_uses_the_site_wide_interval_for_every_feed():
|
||||||
|
SiteSettings.objects.create(fetch_interval_minutes=120)
|
||||||
|
due = Feed.objects.create(
|
||||||
|
feed_url="https://a.example.com/feed.xml",
|
||||||
|
last_fetched_at=timezone.now() - timedelta(minutes=150),
|
||||||
|
)
|
||||||
|
not_due = Feed.objects.create(
|
||||||
|
feed_url="https://b.example.com/feed.xml",
|
||||||
|
last_fetched_at=timezone.now() - timedelta(minutes=90),
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||||
|
poll_due_feeds()
|
||||||
|
|
||||||
|
dispatched_ids = {call.args[0] for call in mock_delay.call_args_list}
|
||||||
|
assert dispatched_ids == {due.id}
|
||||||
|
assert not_due.id not in dispatched_ids
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_admin_changelist_redirects_straight_to_the_change_form(client):
|
||||||
|
assert not SiteSettings.objects.exists()
|
||||||
|
User.objects.create_superuser("admin", password="pw-admin-123")
|
||||||
|
client.login(username="admin", password="pw-admin-123")
|
||||||
|
|
||||||
|
changelist_url = reverse("admin:core_sitesettings_changelist")
|
||||||
|
response = client.get(changelist_url)
|
||||||
|
|
||||||
|
settings_row = SiteSettings.objects.get() # created by the redirect itself
|
||||||
|
assert response.status_code == 302
|
||||||
|
assert response.url == reverse("admin:core_sitesettings_change", args=[settings_row.pk])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_admin_never_offers_add_or_delete(client):
|
||||||
|
# ModelAdmin.has_add_permission/has_delete_permission returning False
|
||||||
|
# makes Django raise PermissionDenied rather than build a 403 response
|
||||||
|
# directly — the test client's default behavior re-raises that for
|
||||||
|
# debugging convenience instead of returning it as a response.
|
||||||
|
client.raise_request_exception = False
|
||||||
|
|
||||||
|
User.objects.create_superuser("admin", password="pw-admin-123")
|
||||||
|
client.login(username="admin", password="pw-admin-123")
|
||||||
|
|
||||||
|
response = client.get(reverse("admin:core_sitesettings_add"))
|
||||||
|
assert response.status_code == 403
|
||||||
|
|
||||||
|
settings_row = SiteSettings.load()
|
||||||
|
delete_url = reverse("admin:core_sitesettings_delete", args=[settings_row.pk])
|
||||||
|
response = client.get(delete_url)
|
||||||
|
assert response.status_code == 403
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
import socket
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _fake_public_dns(monkeypatch):
|
||||||
|
"""apps.feeds.services.ssrf_guard resolves a feed's real hostname via
|
||||||
|
socket.getaddrinfo before every fetch. Tests use made-up hostnames
|
||||||
|
like example.com/a.example.com purely as fixture data — they must
|
||||||
|
not depend on real DNS/network access to pass that check, and must
|
||||||
|
resolve deterministically regardless of this sandbox's network
|
||||||
|
access. Every lookup here resolves to a fixed, definitely-public IP;
|
||||||
|
tests that specifically exercise the "resolves to a non-public
|
||||||
|
address" rejection path patch socket.getaddrinfo themselves instead.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _fake_getaddrinfo(host, *args, **kwargs):
|
||||||
|
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0))]
|
||||||
|
|
||||||
|
monkeypatch.setattr(socket, "getaddrinfo", _fake_getaddrinfo)
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
# Generated by Django 5.2.18 on 2026-10-31 18:00
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
("feeds", "0002_remove_folder"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.AddField(
|
||||||
|
model_name="feed",
|
||||||
|
name="claimed_at",
|
||||||
|
field=models.DateTimeField(blank=True, null=True),
|
||||||
|
),
|
||||||
|
]
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
# Generated by Django 5.2.18 on 2026-10-31 18:56
|
||||||
|
|
||||||
|
from django.db import migrations
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
|
||||||
|
dependencies = [
|
||||||
|
("feeds", "0003_feed_claimed_at"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.RemoveField(
|
||||||
|
model_name="feed",
|
||||||
|
name="fetch_interval_minutes",
|
||||||
|
),
|
||||||
|
]
|
||||||
+13
-2
@@ -35,11 +35,22 @@ class Feed(models.Model):
|
|||||||
)
|
)
|
||||||
last_error_message = models.TextField(blank=True)
|
last_error_message = models.TextField(blank=True)
|
||||||
|
|
||||||
# Instance-wide, not a per-user preference — see class docstring.
|
# The poll interval itself is site-wide now (apps.core.models.
|
||||||
fetch_interval_minutes = models.PositiveIntegerField(default=60)
|
# SiteSettings), not per-feed — only the error-driven backoff on top
|
||||||
|
# of it still varies per feed (see apps.feeds.services.fetcher.
|
||||||
|
# effective_interval_minutes).
|
||||||
fetch_error_count = models.PositiveIntegerField(default=0)
|
fetch_error_count = models.PositiveIntegerField(default=0)
|
||||||
is_active = models.BooleanField(default=True)
|
is_active = models.BooleanField(default=True)
|
||||||
|
|
||||||
|
# Set by poll_due_feeds' atomic claim (an UPDATE ... WHERE clause, not
|
||||||
|
# select_for_update — Celery Beat and fetch_feed run in separate
|
||||||
|
# transactions) and cleared once fetch_feed finishes, so overlapping
|
||||||
|
# beat ticks or a fetch that runs long can't both dispatch the same
|
||||||
|
# feed. A claim older than CLAIM_STALE_MINUTES (tasks.py) is treated as
|
||||||
|
# abandoned and can be reclaimed, so a worker dying mid-fetch doesn't
|
||||||
|
# wedge the feed forever.
|
||||||
|
claimed_at = models.DateTimeField(null=True, blank=True)
|
||||||
|
|
||||||
created_at = models.DateTimeField(auto_now_add=True)
|
created_at = models.DateTimeField(auto_now_add=True)
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
|
|||||||
@@ -23,12 +23,23 @@ from django.utils import timezone
|
|||||||
from apps.articles.models import Article
|
from apps.articles.models import Article
|
||||||
from apps.feeds.models import Feed
|
from apps.feeds.models import Feed
|
||||||
|
|
||||||
|
from .ssrf_guard import SSRFBlocked, safe_get
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
FETCH_TIMEOUT_SECONDS = 15
|
FETCH_TIMEOUT_SECONDS = 15
|
||||||
MAX_BACKOFF_MINUTES = 24 * 60
|
MAX_BACKOFF_MINUTES = 24 * 60
|
||||||
USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)"
|
USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)"
|
||||||
|
|
||||||
|
# ssrf_guard.MAX_RESPONSE_BYTES already bounds the raw download; these
|
||||||
|
# bound what a single poll does *with* that response — a feed within the
|
||||||
|
# byte cap can still carry an unreasonable number of entries, or one
|
||||||
|
# entry with an unreasonably large body, and both cost real CPU/DB time
|
||||||
|
# (nh3 sanitization, an Article row/UPDATE) per entry processed.
|
||||||
|
MAX_ENTRIES_PER_POLL = 200
|
||||||
|
MAX_CONTENT_HTML_INPUT_CHARS = 200_000 # bounds sanitizer cost, not just storage
|
||||||
|
MAX_CONTENT_HTML_CHARS = 50_000 # bounds what's actually stored/rendered
|
||||||
|
|
||||||
ALLOWED_TAGS = {
|
ALLOWED_TAGS = {
|
||||||
"a",
|
"a",
|
||||||
"p",
|
"p",
|
||||||
@@ -81,7 +92,9 @@ class FetchResult:
|
|||||||
def sanitize_html(raw_html: str) -> str:
|
def sanitize_html(raw_html: str) -> str:
|
||||||
if not raw_html:
|
if not raw_html:
|
||||||
return ""
|
return ""
|
||||||
return nh3.clean(raw_html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None)
|
bounded = raw_html[:MAX_CONTENT_HTML_INPUT_CHARS]
|
||||||
|
cleaned = nh3.clean(bounded, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None)
|
||||||
|
return cleaned[:MAX_CONTENT_HTML_CHARS]
|
||||||
|
|
||||||
|
|
||||||
def _content_hash(title: str, content_html: str) -> str:
|
def _content_hash(title: str, content_html: str) -> str:
|
||||||
@@ -111,14 +124,17 @@ def _entry_content_html(entry) -> str:
|
|||||||
return getattr(entry, "summary", "") or ""
|
return getattr(entry, "summary", "") or ""
|
||||||
|
|
||||||
|
|
||||||
def effective_interval_minutes(feed: Feed) -> int:
|
def effective_interval_minutes(feed: Feed, configured_interval_minutes: int) -> int:
|
||||||
"""The actual polling interval to use, applying exponential backoff on
|
"""The actual polling interval to use for `feed`, applying exponential
|
||||||
top of the admin-configured ``fetch_interval_minutes`` for feeds that
|
backoff on top of the site-wide `configured_interval_minutes` (see
|
||||||
are currently erroring. ``fetch_interval_minutes`` itself is never
|
apps.core.models.SiteSettings) for a feed that's currently erroring.
|
||||||
mutated by the fetcher — it's the admin's chosen baseline.
|
Takes the configured value as a parameter rather than querying
|
||||||
|
SiteSettings itself, since this runs once per feed (apps.feeds.tasks
|
||||||
|
calls it from a list comprehension over every active feed) and the
|
||||||
|
site-wide value doesn't vary by feed.
|
||||||
"""
|
"""
|
||||||
floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15)
|
floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15)
|
||||||
base = max(feed.fetch_interval_minutes, floor)
|
base = max(configured_interval_minutes, floor)
|
||||||
if feed.fetch_error_count <= 0:
|
if feed.fetch_error_count <= 0:
|
||||||
return base
|
return base
|
||||||
backoff = base * (2**feed.fetch_error_count)
|
backoff = base * (2**feed.fetch_error_count)
|
||||||
@@ -139,10 +155,13 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
|
|||||||
headers["If-Modified-Since"] = feed.last_modified
|
headers["If-Modified-Since"] = feed.last_modified
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = requests.get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS)
|
response = safe_get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS)
|
||||||
|
except SSRFBlocked as exc:
|
||||||
|
feed.last_fetched_at = timezone.now()
|
||||||
|
return record_fetch_error(feed, f"Refused to fetch: {exc}")
|
||||||
except requests.RequestException as exc:
|
except requests.RequestException as exc:
|
||||||
feed.last_fetched_at = timezone.now()
|
feed.last_fetched_at = timezone.now()
|
||||||
return _record_error(feed, f"Request failed: {exc}")
|
return record_fetch_error(feed, f"Request failed: {exc}")
|
||||||
|
|
||||||
feed.last_fetched_at = timezone.now()
|
feed.last_fetched_at = timezone.now()
|
||||||
|
|
||||||
@@ -154,12 +173,12 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
|
|||||||
)
|
)
|
||||||
|
|
||||||
if response.status_code >= 400:
|
if response.status_code >= 400:
|
||||||
return _record_error(feed, f"HTTP {response.status_code}")
|
return record_fetch_error(feed, f"HTTP {response.status_code}")
|
||||||
|
|
||||||
parsed = feedparser.parse(response.content)
|
parsed = feedparser.parse(response.content)
|
||||||
|
|
||||||
if parsed.bozo and not parsed.entries:
|
if parsed.bozo and not parsed.entries:
|
||||||
return _record_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}")
|
return record_fetch_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}")
|
||||||
|
|
||||||
if parsed.bozo:
|
if parsed.bozo:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
@@ -181,7 +200,17 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
|
|||||||
new_ids: list[int] = []
|
new_ids: list[int] = []
|
||||||
updated_ids: list[int] = []
|
updated_ids: list[int] = []
|
||||||
|
|
||||||
for entry in parsed.entries:
|
entries = parsed.entries
|
||||||
|
if len(entries) > MAX_ENTRIES_PER_POLL:
|
||||||
|
logger.warning(
|
||||||
|
"Feed %s: %d entries in one poll, only processing the first %d",
|
||||||
|
feed.feed_url,
|
||||||
|
len(entries),
|
||||||
|
MAX_ENTRIES_PER_POLL,
|
||||||
|
)
|
||||||
|
entries = entries[:MAX_ENTRIES_PER_POLL]
|
||||||
|
|
||||||
|
for entry in entries:
|
||||||
guid = _entry_guid(entry)
|
guid = _entry_guid(entry)
|
||||||
title = (getattr(entry, "title", "") or "")[:1000]
|
title = (getattr(entry, "title", "") or "")[:1000]
|
||||||
content_html = sanitize_html(_entry_content_html(entry))
|
content_html = sanitize_html(_entry_content_html(entry))
|
||||||
@@ -215,7 +244,7 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _record_error(feed: Feed, message: str) -> FetchResult:
|
def record_fetch_error(feed: Feed, message: str) -> FetchResult:
|
||||||
feed.last_fetch_status = Feed.STATUS_ERROR
|
feed.last_fetch_status = Feed.STATUS_ERROR
|
||||||
feed.last_error_message = message[:2000]
|
feed.last_error_message = message[:2000]
|
||||||
feed.fetch_error_count += 1
|
feed.fetch_error_count += 1
|
||||||
|
|||||||
@@ -0,0 +1,121 @@
|
|||||||
|
"""Rejects feed-fetch targets that could turn this server/worker's own
|
||||||
|
network access into an SSRF vector.
|
||||||
|
|
||||||
|
Feed URLs are entirely attacker-controlled (any authenticated user via
|
||||||
|
"add feed", or an uploaded OPML file — see apps.feeds.opml) and this app
|
||||||
|
fetches them from the server's own network position. fetch_and_ingest
|
||||||
|
(apps.feeds.services.fetcher) is the single chokepoint every feed URL
|
||||||
|
passes through before any HTTP request is made, regardless of which
|
||||||
|
path created the Feed row, so enforcing this here covers all of them.
|
||||||
|
|
||||||
|
This validates the resolved address before connecting and again before
|
||||||
|
following each redirect hop, denying by default anything that isn't a
|
||||||
|
globally routable address (ip.is_global) — covering loopback, RFC1918
|
||||||
|
private ranges, link-local (including the 169.254.169.254 cloud
|
||||||
|
metadata address), and so on, for both IPv4 and IPv6.
|
||||||
|
|
||||||
|
What this does *not* close: the TCP connection isn't pinned to the
|
||||||
|
specific IP that was validated, so a DNS answer that changes between
|
||||||
|
the validation lookup and the actual connect (classic DNS rebinding)
|
||||||
|
could still slip through. Closing that fully needs a custom transport
|
||||||
|
that connects to the exact validated IP while still verifying TLS
|
||||||
|
against the original hostname — more machinery than this self-hosted
|
||||||
|
app's threat model currently justifies; this stops the much more
|
||||||
|
common case of a feed URL (or a redirect) simply pointing straight at
|
||||||
|
an internal address.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ipaddress
|
||||||
|
import socket
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from urllib.parse import urljoin, urlparse
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from requests.structures import CaseInsensitiveDict
|
||||||
|
|
||||||
|
ALLOWED_SCHEMES = {"http", "https"}
|
||||||
|
MAX_REDIRECTS = 5
|
||||||
|
MAX_RESPONSE_BYTES = 10 * 1024 * 1024 # 10 MB — generous for an RSS/Atom feed
|
||||||
|
READ_CHUNK_BYTES = 65536
|
||||||
|
|
||||||
|
|
||||||
|
class SSRFBlocked(Exception):
|
||||||
|
"""Raised when a fetch target — or one of its redirects — isn't safe
|
||||||
|
to connect to on a user's behalf."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class BoundedResponse:
|
||||||
|
status_code: int
|
||||||
|
# requests' own CaseInsensitiveDict — server header casing varies
|
||||||
|
# (ETag/Etag/etag) and fetcher.py looks these up by a fixed-case key.
|
||||||
|
headers: CaseInsensitiveDict
|
||||||
|
content: bytes
|
||||||
|
|
||||||
|
|
||||||
|
def validate_fetch_url(url: str) -> None:
|
||||||
|
"""Raises SSRFBlocked unless `url` has an http(s) scheme, a hostname,
|
||||||
|
and every address that hostname resolves to is globally routable."""
|
||||||
|
parsed = urlparse(url)
|
||||||
|
if parsed.scheme not in ALLOWED_SCHEMES:
|
||||||
|
raise SSRFBlocked(f"Unsupported URL scheme: {parsed.scheme!r}")
|
||||||
|
if not parsed.hostname:
|
||||||
|
raise SSRFBlocked("URL has no hostname")
|
||||||
|
|
||||||
|
try:
|
||||||
|
addrinfo = socket.getaddrinfo(parsed.hostname, None)
|
||||||
|
except OSError as exc:
|
||||||
|
raise SSRFBlocked(f"Could not resolve {parsed.hostname!r}: {exc}") from exc
|
||||||
|
|
||||||
|
for *_, sockaddr in addrinfo:
|
||||||
|
ip = ipaddress.ip_address(sockaddr[0])
|
||||||
|
if not ip.is_global:
|
||||||
|
raise SSRFBlocked(f"{parsed.hostname!r} resolves to a non-public address: {ip}")
|
||||||
|
|
||||||
|
|
||||||
|
def safe_get(url: str, *, headers: dict, timeout: float) -> BoundedResponse:
|
||||||
|
"""GETs `url` the way fetch_and_ingest needs it: every redirect hop is
|
||||||
|
validated and followed manually (never left to requests/urllib3, so
|
||||||
|
a redirect can't bypass validate_fetch_url), capped at MAX_REDIRECTS,
|
||||||
|
and the body is read in bounded chunks rather than trusting
|
||||||
|
Content-Length or buffering an unbounded response.
|
||||||
|
"""
|
||||||
|
current_url = url
|
||||||
|
with requests.Session() as session:
|
||||||
|
for _ in range(MAX_REDIRECTS + 1):
|
||||||
|
validate_fetch_url(current_url)
|
||||||
|
response = session.get(
|
||||||
|
current_url,
|
||||||
|
headers=headers,
|
||||||
|
timeout=timeout,
|
||||||
|
stream=True,
|
||||||
|
allow_redirects=False,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
if response.is_redirect or response.is_permanent_redirect:
|
||||||
|
location = response.headers.get("Location")
|
||||||
|
if not location:
|
||||||
|
raise SSRFBlocked("Redirect response had no Location header")
|
||||||
|
current_url = urljoin(current_url, location)
|
||||||
|
continue
|
||||||
|
return BoundedResponse(
|
||||||
|
status_code=response.status_code,
|
||||||
|
headers=response.headers,
|
||||||
|
content=_read_bounded(response),
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
response.close()
|
||||||
|
raise SSRFBlocked(f"Too many redirects (> {MAX_REDIRECTS})")
|
||||||
|
|
||||||
|
|
||||||
|
def _read_bounded(response: requests.Response) -> bytes:
|
||||||
|
chunks = []
|
||||||
|
total = 0
|
||||||
|
for chunk in response.iter_content(chunk_size=READ_CHUNK_BYTES):
|
||||||
|
total += len(chunk)
|
||||||
|
if total > MAX_RESPONSE_BYTES:
|
||||||
|
raise SSRFBlocked(f"Response exceeded {MAX_RESPONSE_BYTES} bytes")
|
||||||
|
chunks.append(chunk)
|
||||||
|
return b"".join(chunks)
|
||||||
+69
-11
@@ -4,50 +4,108 @@ from io import BytesIO
|
|||||||
|
|
||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
from django.contrib.auth import get_user_model
|
from django.contrib.auth import get_user_model
|
||||||
|
from django.db.models import Q
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
from apps.articles.tasks import apply_filter_rules_for_feed
|
from apps.articles.tasks import apply_filter_rules_for_feed
|
||||||
|
from apps.core.models import SiteSettings
|
||||||
|
|
||||||
from .models import Feed
|
from .models import Feed
|
||||||
from .opml import import_opml
|
from .opml import import_opml
|
||||||
from .services.fetcher import effective_interval_minutes, fetch_and_ingest
|
from .services.fetcher import effective_interval_minutes, fetch_and_ingest, record_fetch_error
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# How long a claimed-but-unfinished fetch blocks the same feed from being
|
||||||
|
# claimed again — must comfortably exceed FETCH_TIMEOUT_SECONDS plus parse/
|
||||||
|
# ingest time, so a slow-but-healthy fetch is never raced by the next tick,
|
||||||
|
# while a worker that died mid-fetch doesn't wedge the feed forever.
|
||||||
|
CLAIM_STALE_MINUTES = 10
|
||||||
|
|
||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
def poll_due_feeds():
|
def poll_due_feeds():
|
||||||
"""Find every active Feed whose polling interval has elapsed and
|
"""Find every active Feed whose polling interval has elapsed, atomically
|
||||||
dispatch a fetch_feed task for each — fan-out happens here, not via one
|
claim it (so overlapping beat ticks or a slow-running previous fetch
|
||||||
Celery Beat entry per feed.
|
can't dispatch the same feed twice), and dispatch a fetch_feed task for
|
||||||
|
each claimed feed — fan-out happens here, not via one Celery Beat entry
|
||||||
|
per feed.
|
||||||
"""
|
"""
|
||||||
|
now = timezone.now()
|
||||||
|
claim_stale_before = now - timedelta(minutes=CLAIM_STALE_MINUTES)
|
||||||
|
# Loaded once per call, not per feed — the interval is site-wide
|
||||||
|
# (apps.core.models.SiteSettings), so every feed below shares it.
|
||||||
|
configured_interval = SiteSettings.load().fetch_interval_minutes
|
||||||
due_feed_ids = [
|
due_feed_ids = [
|
||||||
feed.id
|
feed.id
|
||||||
for feed in Feed.objects.filter(is_active=True).only(
|
for feed in Feed.objects.filter(is_active=True).only(
|
||||||
"id", "last_fetched_at", "fetch_interval_minutes", "fetch_error_count"
|
"id", "last_fetched_at", "fetch_error_count"
|
||||||
)
|
)
|
||||||
if _is_due(feed)
|
if _is_due(feed, configured_interval)
|
||||||
]
|
]
|
||||||
|
claimed_ids = []
|
||||||
for feed_id in due_feed_ids:
|
for feed_id in due_feed_ids:
|
||||||
|
# The UPDATE's WHERE clause re-checks staleness at the database
|
||||||
|
# level: it only claims rows nobody else has already claimed more
|
||||||
|
# recently than CLAIM_STALE_MINUTES ago, so two workers racing this
|
||||||
|
# same loop can't both get rowcount=1 for the same feed.
|
||||||
|
claimed = (
|
||||||
|
Feed.objects.filter(id=feed_id, is_active=True)
|
||||||
|
.filter(Q(claimed_at__isnull=True) | Q(claimed_at__lt=claim_stale_before))
|
||||||
|
.update(claimed_at=now)
|
||||||
|
)
|
||||||
|
if claimed:
|
||||||
|
claimed_ids.append(feed_id)
|
||||||
|
for feed_id in claimed_ids:
|
||||||
fetch_feed.delay(feed_id)
|
fetch_feed.delay(feed_id)
|
||||||
return len(due_feed_ids)
|
return len(claimed_ids)
|
||||||
|
|
||||||
|
|
||||||
def _is_due(feed: Feed) -> bool:
|
def _is_due(feed: Feed, configured_interval_minutes: int) -> bool:
|
||||||
if feed.last_fetched_at is None:
|
if feed.last_fetched_at is None:
|
||||||
return True
|
return True
|
||||||
interval = effective_interval_minutes(feed)
|
interval = effective_interval_minutes(feed, configured_interval_minutes)
|
||||||
return feed.last_fetched_at <= timezone.now() - timedelta(minutes=interval)
|
return feed.last_fetched_at <= timezone.now() - timedelta(minutes=interval)
|
||||||
|
|
||||||
|
|
||||||
@shared_task
|
# Bounds a single feed's fetch+parse+ingest, independent of
|
||||||
|
# FETCH_TIMEOUT_SECONDS (apps.feeds.services.fetcher) which only covers
|
||||||
|
# the HTTP request itself: parsing/sanitizing/writing a large-but-under-
|
||||||
|
# the-byte-cap feed could still run long. soft_time_limit lets the
|
||||||
|
# except-Exception handler below record a normal feed error (and clear
|
||||||
|
# the claim) before the hard time_limit kills the worker process outright.
|
||||||
|
FETCH_FEED_SOFT_TIME_LIMIT = 60
|
||||||
|
FETCH_FEED_TIME_LIMIT = 90
|
||||||
|
|
||||||
|
|
||||||
|
@shared_task(time_limit=FETCH_FEED_TIME_LIMIT, soft_time_limit=FETCH_FEED_SOFT_TIME_LIMIT)
|
||||||
def fetch_feed(feed_id: int):
|
def fetch_feed(feed_id: int):
|
||||||
try:
|
try:
|
||||||
feed = Feed.objects.get(pk=feed_id, is_active=True)
|
feed = Feed.objects.get(pk=feed_id, is_active=True)
|
||||||
except Feed.DoesNotExist:
|
except Feed.DoesNotExist:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
result = fetch_and_ingest(feed)
|
try:
|
||||||
|
result = fetch_and_ingest(feed)
|
||||||
|
except Exception:
|
||||||
|
# fetch_and_ingest already handles the expected failure modes
|
||||||
|
# (network errors, SSRF-blocked targets, unparseable feeds) by
|
||||||
|
# returning an error FetchResult rather than raising — anything
|
||||||
|
# that reaches here is unexpected (a parser/sanitizer bug, a DB
|
||||||
|
# error, this task's own soft time limit), so it's recorded the
|
||||||
|
# same way a handled failure would be rather than silently
|
||||||
|
# leaving the feed's status stale, the claim held for the full
|
||||||
|
# CLAIM_STALE_MINUTES, and no record of what happened. Re-raised
|
||||||
|
# afterward so Celery still marks the task FAILED (visible in
|
||||||
|
# worker logs/monitoring) instead of swallowing it.
|
||||||
|
logger.exception("Feed %s: unexpected error during fetch", feed.feed_url)
|
||||||
|
record_fetch_error(feed, "Unexpected error — see worker logs")
|
||||||
|
feed.last_fetched_at = timezone.now()
|
||||||
|
feed.claimed_at = None
|
||||||
|
feed.save()
|
||||||
|
raise
|
||||||
|
|
||||||
|
feed.claimed_at = None
|
||||||
feed.save()
|
feed.save()
|
||||||
|
|
||||||
changed_article_ids = result.new_article_ids + result.updated_article_ids
|
changed_article_ids = result.new_article_ids + result.updated_article_ids
|
||||||
|
|||||||
@@ -5,7 +5,13 @@ from django.utils import timezone
|
|||||||
|
|
||||||
from apps.articles.models import Article, UserArticleState
|
from apps.articles.models import Article, UserArticleState
|
||||||
from apps.feeds.models import Feed
|
from apps.feeds.models import Feed
|
||||||
from apps.feeds.services.fetcher import effective_interval_minutes, fetch_and_ingest
|
from apps.feeds.services.fetcher import (
|
||||||
|
MAX_CONTENT_HTML_CHARS,
|
||||||
|
MAX_ENTRIES_PER_POLL,
|
||||||
|
effective_interval_minutes,
|
||||||
|
fetch_and_ingest,
|
||||||
|
sanitize_html,
|
||||||
|
)
|
||||||
|
|
||||||
FEED_URL = "https://example.com/feed.xml"
|
FEED_URL = "https://example.com/feed.xml"
|
||||||
|
|
||||||
@@ -130,7 +136,7 @@ def test_unparseable_feed_records_error_and_backs_off(feed):
|
|||||||
|
|
||||||
assert result.status == Feed.STATUS_ERROR
|
assert result.status == Feed.STATUS_ERROR
|
||||||
assert feed.fetch_error_count == 1
|
assert feed.fetch_error_count == 1
|
||||||
assert effective_interval_minutes(feed) > feed.fetch_interval_minutes
|
assert effective_interval_minutes(feed, 60) > 60
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@@ -196,3 +202,29 @@ def test_refetching_updated_content_does_not_reset_user_state():
|
|||||||
state.refresh_from_db()
|
state.refresh_from_db()
|
||||||
assert state.is_read is True
|
assert state.is_read is True
|
||||||
assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists()
|
assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_sanitize_html_caps_stored_output_length():
|
||||||
|
huge = "<p>" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "</p>"
|
||||||
|
cleaned = sanitize_html(huge)
|
||||||
|
assert len(cleaned) <= MAX_CONTENT_HTML_CHARS
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
@responses.activate
|
||||||
|
def test_fetch_only_processes_the_first_max_entries_per_poll(feed):
|
||||||
|
items = "".join(
|
||||||
|
f"<item><title>Item {i}</title><guid>guid-{i}</guid></item>"
|
||||||
|
for i in range(MAX_ENTRIES_PER_POLL + 5)
|
||||||
|
)
|
||||||
|
xml = f"""<?xml version="1.0"?>
|
||||||
|
<rss version="2.0"><channel><title>Big feed</title>{items}</channel></rss>
|
||||||
|
"""
|
||||||
|
responses.add(responses.GET, FEED_URL, body=xml, status=200)
|
||||||
|
|
||||||
|
result = fetch_and_ingest(feed)
|
||||||
|
feed.save()
|
||||||
|
|
||||||
|
assert len(result.new_article_ids) == MAX_ENTRIES_PER_POLL
|
||||||
|
assert Article.objects.filter(feed=feed).count() == MAX_ENTRIES_PER_POLL
|
||||||
|
assert not Article.objects.filter(feed=feed, guid=f"guid-{MAX_ENTRIES_PER_POLL}").exists()
|
||||||
@@ -0,0 +1,138 @@
|
|||||||
|
import socket
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import responses
|
||||||
|
|
||||||
|
from apps.feeds.services.ssrf_guard import (
|
||||||
|
MAX_REDIRECTS,
|
||||||
|
MAX_RESPONSE_BYTES,
|
||||||
|
SSRFBlocked,
|
||||||
|
safe_get,
|
||||||
|
validate_fetch_url,
|
||||||
|
)
|
||||||
|
|
||||||
|
# apps/feeds/conftest.py's autouse _fake_public_dns fixture pins every
|
||||||
|
# hostname to this address unless a test below overrides it directly.
|
||||||
|
PUBLIC_IP = "93.184.216.34"
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_dns(monkeypatch, ip: str):
|
||||||
|
def _fake_getaddrinfo(host, *args, **kwargs):
|
||||||
|
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, 0))]
|
||||||
|
|
||||||
|
monkeypatch.setattr(socket, "getaddrinfo", _fake_getaddrinfo)
|
||||||
|
|
||||||
|
|
||||||
|
class TestValidateFetchUrl:
|
||||||
|
def test_allows_a_public_https_url(self):
|
||||||
|
validate_fetch_url("https://example.com/feed.xml")
|
||||||
|
|
||||||
|
def test_rejects_a_non_http_scheme(self):
|
||||||
|
with pytest.raises(SSRFBlocked, match="scheme"):
|
||||||
|
validate_fetch_url("file:///etc/passwd")
|
||||||
|
|
||||||
|
def test_rejects_a_url_with_no_hostname(self):
|
||||||
|
with pytest.raises(SSRFBlocked, match="hostname"):
|
||||||
|
validate_fetch_url("https:///feed.xml")
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"ip",
|
||||||
|
[
|
||||||
|
"127.0.0.1", # loopback
|
||||||
|
"10.0.0.5", # RFC1918 private
|
||||||
|
"192.168.1.1", # RFC1918 private
|
||||||
|
"169.254.169.254", # link-local / cloud metadata
|
||||||
|
"0.0.0.0", # unspecified
|
||||||
|
"::1", # IPv6 loopback
|
||||||
|
"fc00::1", # IPv6 unique local
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_rejects_a_url_resolving_to_a_non_public_address(self, monkeypatch, ip):
|
||||||
|
_patch_dns(monkeypatch, ip)
|
||||||
|
with pytest.raises(SSRFBlocked, match="non-public"):
|
||||||
|
validate_fetch_url("https://internal.example.com/feed.xml")
|
||||||
|
|
||||||
|
def test_rejects_a_url_whose_hostname_does_not_resolve(self, monkeypatch):
|
||||||
|
def _raise(*args, **kwargs):
|
||||||
|
raise socket.gaierror("nope")
|
||||||
|
|
||||||
|
monkeypatch.setattr(socket, "getaddrinfo", _raise)
|
||||||
|
with pytest.raises(SSRFBlocked, match="Could not resolve"):
|
||||||
|
validate_fetch_url("https://nowhere.example.com/feed.xml")
|
||||||
|
|
||||||
|
|
||||||
|
class TestSafeGet:
|
||||||
|
@responses.activate
|
||||||
|
def test_returns_the_response_body_for_a_normal_feed(self):
|
||||||
|
responses.add(responses.GET, "https://example.com/feed.xml", body="<rss></rss>")
|
||||||
|
response = safe_get("https://example.com/feed.xml", headers={}, timeout=5)
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.content == b"<rss></rss>"
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_follows_a_safe_redirect(self):
|
||||||
|
responses.add(
|
||||||
|
responses.GET,
|
||||||
|
"https://example.com/feed.xml",
|
||||||
|
status=302,
|
||||||
|
headers={"Location": "https://example.com/final.xml"},
|
||||||
|
)
|
||||||
|
responses.add(responses.GET, "https://example.com/final.xml", body="<rss>final</rss>")
|
||||||
|
|
||||||
|
response = safe_get("https://example.com/feed.xml", headers={}, timeout=5)
|
||||||
|
assert response.content == b"<rss>final</rss>"
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_blocks_a_redirect_to_a_private_address(self, monkeypatch):
|
||||||
|
responses.add(
|
||||||
|
responses.GET,
|
||||||
|
"https://example.com/feed.xml",
|
||||||
|
status=302,
|
||||||
|
headers={"Location": "http://internal.example.com/secret"},
|
||||||
|
)
|
||||||
|
|
||||||
|
real_getaddrinfo = socket.getaddrinfo
|
||||||
|
|
||||||
|
def _fake_getaddrinfo(host, *args, **kwargs):
|
||||||
|
if host == "internal.example.com":
|
||||||
|
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 0))]
|
||||||
|
return real_getaddrinfo(host, *args, **kwargs)
|
||||||
|
|
||||||
|
monkeypatch.setattr(socket, "getaddrinfo", _fake_getaddrinfo)
|
||||||
|
|
||||||
|
with pytest.raises(SSRFBlocked, match="non-public"):
|
||||||
|
safe_get("https://example.com/feed.xml", headers={}, timeout=5)
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_blocks_a_redirect_loop_past_max_redirects(self):
|
||||||
|
for _ in range(MAX_REDIRECTS + 2):
|
||||||
|
responses.add(
|
||||||
|
responses.GET,
|
||||||
|
"https://example.com/feed.xml",
|
||||||
|
status=302,
|
||||||
|
headers={"Location": "https://example.com/feed.xml"},
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(SSRFBlocked, match="Too many redirects"):
|
||||||
|
safe_get("https://example.com/feed.xml", headers={}, timeout=5)
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_blocks_a_response_over_the_size_cap(self):
|
||||||
|
responses.add(
|
||||||
|
responses.GET,
|
||||||
|
"https://example.com/feed.xml",
|
||||||
|
body=b"x" * (MAX_RESPONSE_BYTES + 1),
|
||||||
|
)
|
||||||
|
with pytest.raises(SSRFBlocked, match="exceeded"):
|
||||||
|
safe_get("https://example.com/feed.xml", headers={}, timeout=5)
|
||||||
|
|
||||||
|
@responses.activate
|
||||||
|
def test_preserves_case_insensitive_header_lookup(self):
|
||||||
|
responses.add(
|
||||||
|
responses.GET,
|
||||||
|
"https://example.com/feed.xml",
|
||||||
|
body="<rss></rss>",
|
||||||
|
headers={"etag": "abc123"},
|
||||||
|
)
|
||||||
|
response = safe_get("https://example.com/feed.xml", headers={}, timeout=5)
|
||||||
|
assert response.headers.get("ETag") == "abc123"
|
||||||
@@ -16,17 +16,14 @@ def test_poll_due_feeds_dispatches_only_due_feeds():
|
|||||||
recently_fetched = Feed.objects.create(
|
recently_fetched = Feed.objects.create(
|
||||||
feed_url="https://b.example.com/feed.xml",
|
feed_url="https://b.example.com/feed.xml",
|
||||||
last_fetched_at=timezone.now(),
|
last_fetched_at=timezone.now(),
|
||||||
fetch_interval_minutes=60,
|
|
||||||
)
|
)
|
||||||
overdue = Feed.objects.create(
|
overdue = Feed.objects.create(
|
||||||
feed_url="https://c.example.com/feed.xml",
|
feed_url="https://c.example.com/feed.xml",
|
||||||
last_fetched_at=timezone.now() - timedelta(hours=2),
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
||||||
fetch_interval_minutes=60,
|
|
||||||
)
|
)
|
||||||
inactive_but_overdue = Feed.objects.create(
|
inactive_but_overdue = Feed.objects.create(
|
||||||
feed_url="https://d.example.com/feed.xml",
|
feed_url="https://d.example.com/feed.xml",
|
||||||
last_fetched_at=timezone.now() - timedelta(hours=2),
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
||||||
fetch_interval_minutes=60,
|
|
||||||
is_active=False,
|
is_active=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -40,6 +37,56 @@ def test_poll_due_feeds_dispatches_only_due_feeds():
|
|||||||
assert inactive_but_overdue.id not in dispatched_ids
|
assert inactive_but_overdue.id not in dispatched_ids
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_poll_due_feeds_does_not_reclaim_a_feed_it_just_claimed():
|
||||||
|
# Models a second beat tick (or an overlapping worker) running
|
||||||
|
# poll_due_feeds before the first claimed fetch_feed has finished —
|
||||||
|
# the feed must only be dispatched once.
|
||||||
|
overdue = Feed.objects.create(
|
||||||
|
feed_url="https://c.example.com/feed.xml",
|
||||||
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||||
|
first_count = poll_due_feeds()
|
||||||
|
second_count = poll_due_feeds()
|
||||||
|
|
||||||
|
assert first_count == 1
|
||||||
|
assert second_count == 0
|
||||||
|
assert mock_delay.call_count == 1
|
||||||
|
overdue.refresh_from_db()
|
||||||
|
assert overdue.claimed_at is not None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_poll_due_feeds_reclaims_a_stale_claim():
|
||||||
|
from apps.feeds.tasks import CLAIM_STALE_MINUTES
|
||||||
|
|
||||||
|
overdue = Feed.objects.create(
|
||||||
|
feed_url="https://c.example.com/feed.xml",
|
||||||
|
last_fetched_at=timezone.now() - timedelta(hours=2),
|
||||||
|
claimed_at=timezone.now() - timedelta(minutes=CLAIM_STALE_MINUTES + 1),
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||||
|
dispatched_count = poll_due_feeds()
|
||||||
|
|
||||||
|
assert dispatched_count == 1
|
||||||
|
assert mock_delay.call_args.args[0] == overdue.id
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
@responses.activate
|
||||||
|
def test_fetch_feed_task_clears_its_claim():
|
||||||
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", claimed_at=timezone.now())
|
||||||
|
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
||||||
|
|
||||||
|
fetch_feed(feed.id)
|
||||||
|
|
||||||
|
feed.refresh_from_db()
|
||||||
|
assert feed.claimed_at is None
|
||||||
|
|
||||||
|
|
||||||
GOOD_FEED_XML = """<?xml version="1.0"?>
|
GOOD_FEED_XML = """<?xml version="1.0"?>
|
||||||
<rss version="2.0"><channel>
|
<rss version="2.0"><channel>
|
||||||
<title>Example Blog</title>
|
<title>Example Blog</title>
|
||||||
@@ -75,3 +122,22 @@ def test_fetch_feed_task_skips_inactive_feed():
|
|||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
def test_fetch_feed_task_skips_missing_feed():
|
def test_fetch_feed_task_skips_missing_feed():
|
||||||
assert fetch_feed(999999) is None
|
assert fetch_feed(999999) is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_fetch_feed_task_records_error_and_clears_claim_on_unexpected_exception():
|
||||||
|
feed = Feed.objects.create(
|
||||||
|
feed_url="https://example.com/feed.xml",
|
||||||
|
claimed_at=timezone.now(),
|
||||||
|
fetch_error_count=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch("apps.feeds.tasks.fetch_and_ingest", side_effect=RuntimeError("boom")):
|
||||||
|
with pytest.raises(RuntimeError, match="boom"):
|
||||||
|
fetch_feed(feed.id)
|
||||||
|
|
||||||
|
feed.refresh_from_db()
|
||||||
|
assert feed.last_fetch_status == Feed.STATUS_ERROR
|
||||||
|
assert feed.fetch_error_count == 1
|
||||||
|
assert feed.claimed_at is None
|
||||||
|
assert feed.last_fetched_at is not None
|
||||||
@@ -32,6 +32,11 @@ def test_service_worker_is_served_at_root_with_correct_type_and_scope(client):
|
|||||||
assert 'self.addEventListener("fetch"' in body
|
assert 'self.addEventListener("fetch"' in body
|
||||||
# Mutations must stay network-only, never intercepted.
|
# Mutations must stay network-only, never intercepted.
|
||||||
assert 'request.method !== "GET"' in body
|
assert 'request.method !== "GET"' in body
|
||||||
|
# Logging out must purge PAGES_CACHE so a later offline navigation on
|
||||||
|
# a shared/account-switched device can't serve another user's
|
||||||
|
# previously cached authenticated pages.
|
||||||
|
assert "caches.delete(PAGES_CACHE)" in body
|
||||||
|
assert reverse("accounts:logout") in body
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import re
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from django.contrib.auth import get_user_model
|
from django.contrib.auth import get_user_model
|
||||||
from django.urls import reverse
|
from django.urls import reverse
|
||||||
@@ -164,3 +166,54 @@ def test_saved_filter_only_shows_saved_articles(client, alice, alices_article):
|
|||||||
|
|
||||||
response = client.get(reverse("reader:stream"), {"filter": "starred"})
|
response = client.get(reverse("reader:stream"), {"filter": "starred"})
|
||||||
assert alices_article.title not in response.content.decode()
|
assert alices_article.title not in response.content.decode()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_sidebar_shows_a_tags_total_unread_count(client, alice):
|
||||||
|
from apps.articles.models import SubscriptionTag, Tag
|
||||||
|
|
||||||
|
feed_a = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
||||||
|
feed_b = Feed.objects.create(feed_url="https://b.example.com/feed.xml")
|
||||||
|
sub_a = Subscription.objects.create(user=alice, feed=feed_a)
|
||||||
|
sub_b = Subscription.objects.create(user=alice, feed=feed_b)
|
||||||
|
tag = Tag.objects.create(user=alice, name="work")
|
||||||
|
SubscriptionTag.objects.create(user=alice, subscription=sub_a, tag=tag)
|
||||||
|
SubscriptionTag.objects.create(user=alice, subscription=sub_b, tag=tag)
|
||||||
|
for i in range(3):
|
||||||
|
Article.objects.create(feed=feed_a, guid=f"a-{i}", title=f"A{i}")
|
||||||
|
for i in range(4):
|
||||||
|
Article.objects.create(feed=feed_b, guid=f"b-{i}", title=f"B{i}")
|
||||||
|
|
||||||
|
client.login(username="alice", password="pw-alice-123")
|
||||||
|
response = client.get(reverse("reader:stream"))
|
||||||
|
|
||||||
|
# The only "7" on the page — the individual feeds show 3 and 4.
|
||||||
|
content = response.content.decode()
|
||||||
|
assert '<span class="unread-count">7</span>' in content
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_article_rows_carry_their_utc_instant_for_client_side_date_grouping(client, alice):
|
||||||
|
"""The server still groups date headers in UTC, but ships each row's
|
||||||
|
raw UTC instant so date-groups.js can rebuild the headers using the
|
||||||
|
viewer's local calendar day instead."""
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||||
|
Subscription.objects.create(user=alice, feed=feed)
|
||||||
|
dated = Article.objects.create(
|
||||||
|
feed=feed, guid="dated", title="Dated", published_at=timezone.now()
|
||||||
|
)
|
||||||
|
undated = Article.objects.create(feed=feed, guid="undated", title="Undated")
|
||||||
|
|
||||||
|
client.login(username="alice", password="pw-alice-123")
|
||||||
|
response = client.get(reverse("reader:stream"), {"filter": "all"})
|
||||||
|
content = response.content.decode()
|
||||||
|
|
||||||
|
expected_iso = dated.published_at.isoformat()
|
||||||
|
assert f'id="article-{dated.pk}"' in content
|
||||||
|
assert f'data-published-utc="{expected_iso}"' in content
|
||||||
|
|
||||||
|
undated_row = re.search(rf'<li id="article-{undated.pk}".*?>', content, flags=re.DOTALL)
|
||||||
|
assert undated_row is not None
|
||||||
|
assert 'data-published-utc=""' in undated_row.group(0)
|
||||||
+68
-4
@@ -9,6 +9,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
|
import signal
|
||||||
|
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
@@ -27,6 +28,42 @@ logger = logging.getLogger(__name__)
|
|||||||
|
|
||||||
_TAG_RE = re.compile(r"<[^>]+>")
|
_TAG_RE = re.compile(r"<[^>]+>")
|
||||||
|
|
||||||
|
# Bounds how much CPU/time a single user-authored regex rule can burn
|
||||||
|
# against one article. re has no built-in match timeout, so this uses
|
||||||
|
# SIGALRM to interrupt a runaway match (catastrophic backtracking) —
|
||||||
|
# only effective on the main thread of a process with SIGALRM available,
|
||||||
|
# which covers gunicorn's sync workers and Celery's (default) prefork
|
||||||
|
# pool; elsewhere it's skipped and MAX_REGEX_HAYSTACK_CHARS is the only
|
||||||
|
# bound. reapply_rules_for_user (apps.rules.tasks) also carries a Celery
|
||||||
|
# task time_limit as a second, coarser backstop.
|
||||||
|
REGEX_TIMEOUT_SECONDS = 1
|
||||||
|
MAX_REGEX_HAYSTACK_CHARS = 20_000
|
||||||
|
|
||||||
|
|
||||||
|
class _RegexTimeout(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _raise_regex_timeout(signum, frame):
|
||||||
|
raise _RegexTimeout()
|
||||||
|
|
||||||
|
|
||||||
|
def _regex_search_with_timeout(pattern: str, haystack: str, flags: int):
|
||||||
|
if not hasattr(signal, "SIGALRM"):
|
||||||
|
return re.search(pattern, haystack, flags)
|
||||||
|
try:
|
||||||
|
previous_handler = signal.signal(signal.SIGALRM, _raise_regex_timeout)
|
||||||
|
except ValueError:
|
||||||
|
# Not the main thread (e.g. a threaded dev server) — signal
|
||||||
|
# timers only work there.
|
||||||
|
return re.search(pattern, haystack, flags)
|
||||||
|
signal.setitimer(signal.ITIMER_REAL, REGEX_TIMEOUT_SECONDS)
|
||||||
|
try:
|
||||||
|
return re.search(pattern, haystack, flags)
|
||||||
|
finally:
|
||||||
|
signal.setitimer(signal.ITIMER_REAL, 0)
|
||||||
|
signal.signal(signal.SIGALRM, previous_handler)
|
||||||
|
|
||||||
|
|
||||||
def _haystack(article: Article, match_field: str) -> str:
|
def _haystack(article: Article, match_field: str) -> str:
|
||||||
if match_field == FilterRule.FIELD_TITLE:
|
if match_field == FilterRule.FIELD_TITLE:
|
||||||
@@ -47,8 +84,12 @@ def rule_matches(rule: FilterRule, article: Article) -> bool:
|
|||||||
|
|
||||||
if rule.match_type == FilterRule.MATCH_REGEX:
|
if rule.match_type == FilterRule.MATCH_REGEX:
|
||||||
flags = 0 if rule.case_sensitive else re.IGNORECASE
|
flags = 0 if rule.case_sensitive else re.IGNORECASE
|
||||||
|
haystack = raw_haystack[:MAX_REGEX_HAYSTACK_CHARS]
|
||||||
try:
|
try:
|
||||||
return re.search(rule.match_value, raw_haystack, flags) is not None
|
return _regex_search_with_timeout(rule.match_value, haystack, flags) is not None
|
||||||
|
except _RegexTimeout:
|
||||||
|
logger.warning("Regex timed out for rule %s", rule.pk)
|
||||||
|
return False
|
||||||
except re.error as exc:
|
except re.error as exc:
|
||||||
logger.warning("Invalid regex in rule %s: %s", rule.pk, exc)
|
logger.warning("Invalid regex in rule %s: %s", rule.pk, exc)
|
||||||
return False
|
return False
|
||||||
@@ -100,13 +141,36 @@ def apply_rule_action(rule: FilterRule, user, article: Article, state: UserArtic
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def evaluate_rules_for_article(user, article: Article, rules) -> None:
|
def rule_in_scope(rule: FilterRule, feed_id, subscription_tag_ids) -> bool:
|
||||||
"""Evaluates an ordered iterable of rules (already scoped/enabled)
|
"""Whether `rule`'s own feed/tag scope (see FilterRule's docstring)
|
||||||
against one article for one user, applying actions as they match."""
|
covers an article from `feed_id`, given the user's subscription-tag
|
||||||
|
ids for that feed.
|
||||||
|
|
||||||
|
Centralized here — rather than left for each caller to re-derive via
|
||||||
|
its own query filters — so ingest-time dispatch and backlog re-apply
|
||||||
|
can't drift into scoping rules differently (see apps.articles.tasks
|
||||||
|
and apps.rules.tasks).
|
||||||
|
"""
|
||||||
|
if rule.feed_id is not None and rule.feed_id != feed_id:
|
||||||
|
return False
|
||||||
|
if rule.tag_id is not None and rule.tag_id not in subscription_tag_ids:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_rules_for_article(user, article: Article, rules, subscription_tag_ids=()) -> None:
|
||||||
|
"""Evaluates an ordered iterable of a user's enabled rules against one
|
||||||
|
article, applying actions as they match. `rules` doesn't need to be
|
||||||
|
pre-scoped to this article's feed/tags — out-of-scope rules (per
|
||||||
|
rule_in_scope) are skipped here, so callers can pass one user's whole
|
||||||
|
enabled ruleset regardless of which feed(s) the article(s) belong to.
|
||||||
|
"""
|
||||||
matched_any = False
|
matched_any = False
|
||||||
state = None
|
state = None
|
||||||
|
|
||||||
for rule in rules:
|
for rule in rules:
|
||||||
|
if not rule_in_scope(rule, article.feed_id, subscription_tag_ids):
|
||||||
|
continue
|
||||||
if not rule_matches(rule, article):
|
if not rule_matches(rule, article):
|
||||||
continue
|
continue
|
||||||
if state is None:
|
if state is None:
|
||||||
|
|||||||
+24
-6
@@ -1,6 +1,6 @@
|
|||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
|
|
||||||
from apps.articles.models import Article
|
from apps.articles.models import Article, SubscriptionTag
|
||||||
from apps.feeds.models import Subscription
|
from apps.feeds.models import Subscription
|
||||||
|
|
||||||
from .engine import evaluate_rules_for_article
|
from .engine import evaluate_rules_for_article
|
||||||
@@ -12,9 +12,20 @@ def reapply_rules_for_user_backlog(rule_id: int) -> int:
|
|||||||
rule, so priority/short-circuit ordering stays correct) against their
|
rule, so priority/short-circuit ordering stays correct) against their
|
||||||
unread backlog, scoped by the triggering rule's own feed/tag scope.
|
unread backlog, scoped by the triggering rule's own feed/tag scope.
|
||||||
|
|
||||||
Exists as a plain function so it can be called synchronously for
|
The *backlog* is narrowed to the triggering rule's own feed/tag scope
|
||||||
immediate feedback (e.g. from a "Re-apply" view) or wrapped in a task
|
(re-applying a feed-scoped rule has no reason to touch other feeds'
|
||||||
for background use — see reapply_rules_for_user below.
|
articles), but which of the user's *other* rules apply to each of
|
||||||
|
those articles is still per-article/per-feed — evaluate_rules_for_article
|
||||||
|
checks that via rule_in_scope, so a feed- or tag-scoped rule for a
|
||||||
|
different feed can't fire on this backlog just because it happened to
|
||||||
|
match (see apps.rules.engine.rule_in_scope).
|
||||||
|
|
||||||
|
Exists as a plain function, separate from the reapply_rules_for_user
|
||||||
|
task below, so it's unit-testable without Celery — but every caller
|
||||||
|
should go through that task (see apps.rules.views.reapply_rule), not
|
||||||
|
call this directly from a request: a large backlog or a pathological
|
||||||
|
regex (bounded but not eliminated by apps.rules.engine's timeout
|
||||||
|
guard) could otherwise tie up a web worker for a long time.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
rule = FilterRule.objects.select_related("user").get(pk=rule_id)
|
rule = FilterRule.objects.select_related("user").get(pk=rule_id)
|
||||||
@@ -45,13 +56,20 @@ def reapply_rules_for_user_backlog(rule_id: int) -> int:
|
|||||||
|
|
||||||
rules = list(FilterRule.objects.filter(user=user, is_enabled=True).order_by("priority", "id"))
|
rules = list(FilterRule.objects.filter(user=user, is_enabled=True).order_by("priority", "id"))
|
||||||
|
|
||||||
|
subscription_tag_ids_by_feed: dict[int, set[int]] = {}
|
||||||
|
for feed_id, tag_id in SubscriptionTag.objects.filter(
|
||||||
|
user=user, subscription__feed_id__in=feed_ids
|
||||||
|
).values_list("subscription__feed_id", "tag_id"):
|
||||||
|
subscription_tag_ids_by_feed.setdefault(feed_id, set()).add(tag_id)
|
||||||
|
|
||||||
count = 0
|
count = 0
|
||||||
for article in backlog:
|
for article in backlog:
|
||||||
evaluate_rules_for_article(user, article, rules)
|
subscription_tag_ids = subscription_tag_ids_by_feed.get(article.feed_id, set())
|
||||||
|
evaluate_rules_for_article(user, article, rules, subscription_tag_ids)
|
||||||
count += 1
|
count += 1
|
||||||
return count
|
return count
|
||||||
|
|
||||||
|
|
||||||
@shared_task
|
@shared_task(time_limit=120, soft_time_limit=90)
|
||||||
def reapply_rules_for_user(rule_id: int):
|
def reapply_rules_for_user(rule_id: int):
|
||||||
return reapply_rules_for_user_backlog(rule_id)
|
return reapply_rules_for_user_backlog(rule_id)
|
||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import time
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from django.contrib.auth import get_user_model
|
from django.contrib.auth import get_user_model
|
||||||
|
|
||||||
@@ -10,7 +12,7 @@ from apps.articles.models import (
|
|||||||
)
|
)
|
||||||
from apps.feeds.models import Feed
|
from apps.feeds.models import Feed
|
||||||
|
|
||||||
from .engine import evaluate_rules_for_article, rule_matches
|
from .engine import evaluate_rules_for_article, rule_in_scope, rule_matches
|
||||||
from .models import FilterRule
|
from .models import FilterRule
|
||||||
|
|
||||||
User = get_user_model()
|
User = get_user_model()
|
||||||
@@ -87,6 +89,22 @@ class TestRuleMatches:
|
|||||||
rule = make_rule(user, match_type=FilterRule.MATCH_REGEX, match_value="(unclosed")
|
rule = make_rule(user, match_type=FilterRule.MATCH_REGEX, match_value="(unclosed")
|
||||||
assert rule_matches(rule, article) is False
|
assert rule_matches(rule, article) is False
|
||||||
|
|
||||||
|
def test_catastrophic_backtracking_regex_times_out_instead_of_hanging(self, user, article):
|
||||||
|
# (a+)+$ against a long run of a's followed by a non-matching
|
||||||
|
# character is a textbook ReDoS pattern — left unguarded this
|
||||||
|
# takes on the order of a minute (confirmed locally); the
|
||||||
|
# REGEX_TIMEOUT_SECONDS guard in apps.rules.engine must interrupt
|
||||||
|
# it well before that instead of hanging the caller.
|
||||||
|
article.title = "a" * 30 + "!"
|
||||||
|
rule = make_rule(user, match_type=FilterRule.MATCH_REGEX, match_value=r"(a+)+$")
|
||||||
|
|
||||||
|
start = time.monotonic()
|
||||||
|
result = rule_matches(rule, article)
|
||||||
|
elapsed = time.monotonic() - start
|
||||||
|
|
||||||
|
assert result is False
|
||||||
|
assert elapsed < 5
|
||||||
|
|
||||||
def test_content_field_strips_html_tags(self, user, article):
|
def test_content_field_strips_html_tags(self, user, article):
|
||||||
rule = make_rule(user, match_field=FilterRule.FIELD_CONTENT, match_value="Lots of details")
|
rule = make_rule(user, match_field=FilterRule.FIELD_CONTENT, match_value="Lots of details")
|
||||||
assert rule_matches(rule, article) is True
|
assert rule_matches(rule, article) is True
|
||||||
@@ -104,6 +122,24 @@ class TestRuleMatches:
|
|||||||
assert rule_matches(rule, article) is True
|
assert rule_matches(rule, article) is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
class TestRuleInScope:
|
||||||
|
def test_unscoped_rule_applies_to_any_feed(self, user):
|
||||||
|
rule = make_rule(user)
|
||||||
|
assert rule_in_scope(rule, feed_id=999, subscription_tag_ids=set()) is True
|
||||||
|
|
||||||
|
def test_feed_scoped_rule_only_applies_to_its_own_feed(self, user, feed):
|
||||||
|
rule = make_rule(user, feed=feed)
|
||||||
|
assert rule_in_scope(rule, feed_id=feed.id, subscription_tag_ids=set()) is True
|
||||||
|
assert rule_in_scope(rule, feed_id=feed.id + 1, subscription_tag_ids=set()) is False
|
||||||
|
|
||||||
|
def test_tag_scoped_rule_only_applies_when_the_feed_carries_that_tag(self, user):
|
||||||
|
tag = Tag.objects.create(user=user, name="work")
|
||||||
|
rule = make_rule(user, tag=tag)
|
||||||
|
assert rule_in_scope(rule, feed_id=1, subscription_tag_ids={tag.id}) is True
|
||||||
|
assert rule_in_scope(rule, feed_id=1, subscription_tag_ids=set()) is False
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
class TestApplyRuleAction:
|
class TestApplyRuleAction:
|
||||||
def test_star_action(self, user, article):
|
def test_star_action(self, user, article):
|
||||||
|
|||||||
+103
-1
@@ -1,8 +1,17 @@
|
|||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from django.contrib.auth import get_user_model
|
from django.contrib.auth import get_user_model
|
||||||
from django.urls import reverse
|
from django.urls import reverse
|
||||||
|
|
||||||
from apps.articles.models import STARRED_TAG_NAME, Article, ArticleTag, UserArticleState
|
from apps.articles.models import (
|
||||||
|
STARRED_TAG_NAME,
|
||||||
|
Article,
|
||||||
|
ArticleTag,
|
||||||
|
SubscriptionTag,
|
||||||
|
Tag,
|
||||||
|
UserArticleState,
|
||||||
|
)
|
||||||
from apps.feeds.models import Feed, Subscription
|
from apps.feeds.models import Feed, Subscription
|
||||||
|
|
||||||
from .models import FilterRule
|
from .models import FilterRule
|
||||||
@@ -119,6 +128,28 @@ def test_reapply_rule_view_applies_to_unread_backlog(client, alice, feed):
|
|||||||
assert is_starred(alice, article) is True
|
assert is_starred(alice, article) is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_reapply_rule_view_always_queues_through_celery_never_runs_inline(client, alice, feed):
|
||||||
|
# A large backlog or a pathological regex must not be able to tie up
|
||||||
|
# the request/response cycle — the view should dispatch a task, not
|
||||||
|
# call reapply_rules_for_user_backlog directly.
|
||||||
|
Subscription.objects.create(user=alice, feed=feed)
|
||||||
|
rule = FilterRule.objects.create(
|
||||||
|
user=alice,
|
||||||
|
match_field=FilterRule.FIELD_TITLE,
|
||||||
|
match_type=FilterRule.MATCH_CONTAINS,
|
||||||
|
match_value="Python",
|
||||||
|
action=FilterRule.ACTION_STAR,
|
||||||
|
)
|
||||||
|
client.login(username="alice", password="pw-alice-123")
|
||||||
|
|
||||||
|
with patch("apps.rules.views.reapply_rules_for_user.delay") as mock_delay:
|
||||||
|
response = client.post(reverse("rules:reapply_rule", args=[rule.pk]))
|
||||||
|
|
||||||
|
assert response.status_code == 302
|
||||||
|
mock_delay.assert_called_once_with(rule.pk)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
def test_reapply_skips_already_read_articles(alice, feed):
|
def test_reapply_skips_already_read_articles(alice, feed):
|
||||||
article = Article.objects.create(feed=feed, guid="guid-1", title="Python 4 released")
|
article = Article.objects.create(feed=feed, guid="guid-1", title="Python 4 released")
|
||||||
@@ -160,3 +191,74 @@ def test_reapply_respects_feed_scope(alice):
|
|||||||
|
|
||||||
assert is_starred(alice, article_a) is True
|
assert is_starred(alice, article_a) is True
|
||||||
assert not UserArticleState.objects.filter(user=alice, article=article_b).exists()
|
assert not UserArticleState.objects.filter(user=alice, article=article_b).exists()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_reapply_does_not_apply_a_different_feeds_scoped_rule_to_the_backlog():
|
||||||
|
# Regression test for a bug where re-applying one feed-scoped rule
|
||||||
|
# loaded *every* enabled rule and ran it against the whole backlog
|
||||||
|
# with no scope check — so a same-matching rule scoped to a feed
|
||||||
|
# the user isn't re-applying would still fire on this backlog.
|
||||||
|
feed_a = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
||||||
|
feed_b = Feed.objects.create(feed_url="https://b.example.com/feed.xml")
|
||||||
|
alice = User.objects.create_user("alice", password="pw-alice-123")
|
||||||
|
Subscription.objects.create(user=alice, feed=feed_a)
|
||||||
|
Subscription.objects.create(user=alice, feed=feed_b)
|
||||||
|
article_a = Article.objects.create(feed=feed_a, guid="guid-1", title="Python in A")
|
||||||
|
|
||||||
|
triggering_rule = FilterRule.objects.create(
|
||||||
|
user=alice,
|
||||||
|
feed=feed_a,
|
||||||
|
match_field=FilterRule.FIELD_TITLE,
|
||||||
|
match_type=FilterRule.MATCH_CONTAINS,
|
||||||
|
match_value="Python",
|
||||||
|
action=FilterRule.ACTION_STAR,
|
||||||
|
)
|
||||||
|
FilterRule.objects.create(
|
||||||
|
user=alice,
|
||||||
|
feed=feed_b,
|
||||||
|
match_field=FilterRule.FIELD_TITLE,
|
||||||
|
match_type=FilterRule.MATCH_CONTAINS,
|
||||||
|
match_value="Python",
|
||||||
|
action=FilterRule.ACTION_TAG,
|
||||||
|
action_value="feed-b-only",
|
||||||
|
)
|
||||||
|
|
||||||
|
reapply_rules_for_user_backlog(triggering_rule.pk)
|
||||||
|
|
||||||
|
assert is_starred(alice, article_a) is True
|
||||||
|
assert not Tag.objects.filter(user=alice, name="feed-b-only").exists()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
def test_reapply_applies_a_tag_scoped_rule_only_to_that_tags_feeds():
|
||||||
|
feed_a = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
||||||
|
feed_b = Feed.objects.create(feed_url="https://b.example.com/feed.xml")
|
||||||
|
alice = User.objects.create_user("alice", password="pw-alice-123")
|
||||||
|
sub_a = Subscription.objects.create(user=alice, feed=feed_a)
|
||||||
|
Subscription.objects.create(user=alice, feed=feed_b)
|
||||||
|
tag = Tag.objects.create(user=alice, name="work")
|
||||||
|
SubscriptionTag.objects.create(user=alice, subscription=sub_a, tag=tag)
|
||||||
|
article_a = Article.objects.create(feed=feed_a, guid="guid-1", title="Python in A")
|
||||||
|
article_b = Article.objects.create(feed=feed_b, guid="guid-2", title="Python in B")
|
||||||
|
|
||||||
|
unscoped_trigger = FilterRule.objects.create(
|
||||||
|
user=alice,
|
||||||
|
match_field=FilterRule.FIELD_TITLE,
|
||||||
|
match_type=FilterRule.MATCH_CONTAINS,
|
||||||
|
match_value="unrelated-trigger-value",
|
||||||
|
action=FilterRule.ACTION_MARK_READ,
|
||||||
|
)
|
||||||
|
FilterRule.objects.create(
|
||||||
|
user=alice,
|
||||||
|
tag=tag,
|
||||||
|
match_field=FilterRule.FIELD_TITLE,
|
||||||
|
match_type=FilterRule.MATCH_CONTAINS,
|
||||||
|
match_value="Python",
|
||||||
|
action=FilterRule.ACTION_STAR,
|
||||||
|
)
|
||||||
|
|
||||||
|
reapply_rules_for_user_backlog(unscoped_trigger.pk)
|
||||||
|
|
||||||
|
assert is_starred(alice, article_a) is True
|
||||||
|
assert is_starred(alice, article_b) is False
|
||||||
+6
-5
@@ -5,7 +5,7 @@ from django.views.decorators.http import require_POST
|
|||||||
|
|
||||||
from .forms import FilterRuleForm
|
from .forms import FilterRuleForm
|
||||||
from .models import FilterRule
|
from .models import FilterRule
|
||||||
from .tasks import reapply_rules_for_user_backlog
|
from .tasks import reapply_rules_for_user
|
||||||
|
|
||||||
|
|
||||||
@login_required
|
@login_required
|
||||||
@@ -48,8 +48,9 @@ def delete_rule(request, rule_id):
|
|||||||
@require_POST
|
@require_POST
|
||||||
def reapply_rule(request, rule_id):
|
def reapply_rule(request, rule_id):
|
||||||
rule = get_object_or_404(FilterRule, user=request.user, pk=rule_id)
|
rule = get_object_or_404(FilterRule, user=request.user, pk=rule_id)
|
||||||
# Run synchronously for immediate feedback, same pattern as add_feed —
|
# Always queued, never run inline from the request/response cycle —
|
||||||
# a Celery worker isn't required for this to work.
|
# a large backlog or a pathological regex could otherwise tie up a
|
||||||
count = reapply_rules_for_user_backlog(rule.pk)
|
# web worker for a long time (see apps.rules.tasks).
|
||||||
messages.success(request, f"Re-applied your rules to {count} unread article(s).")
|
reapply_rules_for_user.delay(rule.pk)
|
||||||
|
messages.success(request, "Re-applying your rules to your unread backlog…")
|
||||||
return redirect("rules:rule_list")
|
return redirect("rules:rule_list")
|
||||||
@@ -106,9 +106,34 @@ EVECTION_SIGNUP_ENABLED = env.bool("EVECTION_SIGNUP_ENABLED", default=False)
|
|||||||
EVECTION_MIN_FETCH_INTERVAL_MINUTES = env.int("EVECTION_MIN_FETCH_INTERVAL_MINUTES", default=15)
|
EVECTION_MIN_FETCH_INTERVAL_MINUTES = env.int("EVECTION_MIN_FETCH_INTERVAL_MINUTES", default=15)
|
||||||
EVECTION_PWA_CACHE_VERSION = env.str("EVECTION_PWA_CACHE_VERSION", default="v1")
|
EVECTION_PWA_CACHE_VERSION = env.str("EVECTION_PWA_CACHE_VERSION", default="v1")
|
||||||
|
|
||||||
|
# Redis-backed rather than Django's default per-process LocMemCache: login
|
||||||
|
# rate limiting (apps.accounts.views) must share counters across every
|
||||||
|
# gunicorn worker, not just whichever one happened to handle a given
|
||||||
|
# request. Redis is already a hard install requirement for Celery, so
|
||||||
|
# this adds no new infrastructure — just a distinct key prefix so cache
|
||||||
|
# keys can't collide with anything else sharing that Redis instance.
|
||||||
|
CACHES = {
|
||||||
|
"default": {
|
||||||
|
"BACKEND": "django.core.cache.backends.redis.RedisCache",
|
||||||
|
"LOCATION": env.str("REDIS_URL", default="redis://localhost:6379/0"),
|
||||||
|
"KEY_PREFIX": "evection-cache",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
# Celery
|
# Celery
|
||||||
CELERY_BROKER_URL = env.str("REDIS_URL", default="redis://localhost:6379/0")
|
CELERY_BROKER_URL = env.str("REDIS_URL", default="redis://localhost:6379/0")
|
||||||
CELERY_RESULT_BACKEND = None
|
CELERY_RESULT_BACKEND = None
|
||||||
CELERY_TASK_SERIALIZER = "json"
|
CELERY_TASK_SERIALIZER = "json"
|
||||||
CELERY_ACCEPT_CONTENT = ["json"]
|
CELERY_ACCEPT_CONTENT = ["json"]
|
||||||
CELERY_TIMEZONE = TIME_ZONE
|
CELERY_TIMEZONE = TIME_ZONE
|
||||||
|
|
||||||
|
# Without these, a worker process killed mid-task (OOM, deploy restart,
|
||||||
|
# crash) silently loses that task: the default is to ack a message the
|
||||||
|
# moment a worker *starts* it, not when it finishes. acks_late + reject-
|
||||||
|
# on-worker-lost means the message is only consumed on completion (success
|
||||||
|
# or a handled failure) and is redelivered to another worker otherwise.
|
||||||
|
# prefetch_multiplier=1 keeps a worker from grabbing a batch of tasks it
|
||||||
|
# could similarly lose all at once.
|
||||||
|
CELERY_TASK_ACKS_LATE = True
|
||||||
|
CELERY_TASK_REJECT_ON_WORKER_LOST = True
|
||||||
|
CELERY_WORKER_PREFETCH_MULTIPLIER = 1
|
||||||
+28
-1
@@ -1,8 +1,30 @@
|
|||||||
|
from django.core.exceptions import ImproperlyConfigured
|
||||||
|
|
||||||
from .base import * # noqa: F403
|
from .base import * # noqa: F403
|
||||||
from .base import env
|
from .base import SECRET_KEY, env
|
||||||
|
|
||||||
DEBUG = False
|
DEBUG = False
|
||||||
|
|
||||||
|
# base.py falls back to a publicly known placeholder if SECRET_KEY isn't
|
||||||
|
# set — fine for a throwaway dev DB, but production must never sign
|
||||||
|
# sessions/CSRF tokens with a key anyone can read in this repo's history.
|
||||||
|
# Fail startup rather than quietly running with a known key.
|
||||||
|
if not SECRET_KEY or SECRET_KEY == "django-insecure-change-me-in-production":
|
||||||
|
raise ImproperlyConfigured(
|
||||||
|
"SECRET_KEY is missing or still the placeholder default. Set a unique, "
|
||||||
|
"random SECRET_KEY in .env before running with prod settings — generate "
|
||||||
|
'one with: python -c "from django.core.management.utils import '
|
||||||
|
'get_random_secret_key; print(get_random_secret_key())"'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Health checks (scripts/upgrade.sh, an external uptime monitor, ...) hit
|
||||||
|
# gunicorn directly over loopback, bypassing the reverse proxy entirely —
|
||||||
|
# that request's Host header is "127.0.0.1:<port>", not the public domain
|
||||||
|
# in ALLOWED_HOSTS, so without this Django 400s it. Harmless to always
|
||||||
|
# allow: gunicorn's bind address is never reachable from outside this host
|
||||||
|
# in the first place (see docs/install.md's "Reverse proxy" section).
|
||||||
|
ALLOWED_HOSTS = [*ALLOWED_HOSTS, "localhost", "127.0.0.1", "::1"] # noqa: F405
|
||||||
|
|
||||||
# A reverse proxy (nginx, Caddy, Traefik, Nginx Proxy Manager on another
|
# A reverse proxy (nginx, Caddy, Traefik, Nginx Proxy Manager on another
|
||||||
# host, ...) is required in front of gunicorn in every deployment — see
|
# host, ...) is required in front of gunicorn in every deployment — see
|
||||||
# docs/install.md — and it's always the one terminating TLS. This tells
|
# docs/install.md — and it's always the one terminating TLS. This tells
|
||||||
@@ -10,6 +32,11 @@ DEBUG = False
|
|||||||
# sets, and the redirect/cookie/HSTS settings below assume it's set.
|
# sets, and the redirect/cookie/HSTS settings below assume it's set.
|
||||||
SECURE_SSL_REDIRECT = env.bool("SECURE_SSL_REDIRECT", default=True)
|
SECURE_SSL_REDIRECT = env.bool("SECURE_SSL_REDIRECT", default=True)
|
||||||
SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
|
SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
|
||||||
|
# /health/ is also exempted from the HTTPS redirect above: hit directly
|
||||||
|
# over loopback (see ALLOWED_HOSTS above) it has no X-Forwarded-Proto
|
||||||
|
# header to prove it's "secure", and there's no TLS listener on gunicorn's
|
||||||
|
# own port to redirect to anyway.
|
||||||
|
SECURE_REDIRECT_EXEMPT = [r"^health/$"]
|
||||||
|
|
||||||
# WhiteNoise serves /static/ directly from gunicorn instead of relying on
|
# WhiteNoise serves /static/ directly from gunicorn instead of relying on
|
||||||
# the reverse proxy to alias the staticfiles directory on disk — the proxy
|
# the reverse proxy to alias the staticfiles directory on disk — the proxy
|
||||||
|
|||||||
@@ -5,3 +5,10 @@ CELERY_TASK_ALWAYS_EAGER = True
|
|||||||
CELERY_TASK_EAGER_PROPAGATES = True
|
CELERY_TASK_EAGER_PROPAGATES = True
|
||||||
|
|
||||||
PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
|
PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
|
||||||
|
|
||||||
|
# base.py's CACHES needs a real Redis instance — tests must not depend on
|
||||||
|
# one being available, and a plain per-process cache is also what keeps
|
||||||
|
# login-rate-limit tests isolated from each other (LocMemCache's default
|
||||||
|
# location is per-process but shared across tests in that process; see
|
||||||
|
# apps.accounts.test_rate_limit's cache.clear() fixture).
|
||||||
|
CACHES = {"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}}
|
||||||
+10
-5
@@ -66,7 +66,7 @@ and 4 below first — prerequisites and what `.env` controls.
|
|||||||
|
|
||||||
- Python 3.11 or 3.12, with its `venv` module (Debian/Ubuntu split this into a
|
- Python 3.11 or 3.12, with its `venv` module (Debian/Ubuntu split this into a
|
||||||
separate `python3.X-venv` package — see below)
|
separate `python3.X-venv` package — see below)
|
||||||
- Redis (Celery's broker)
|
- Redis (Celery's broker, and the app's cache — e.g. login rate limiting)
|
||||||
- **Postgres only**: a PostgreSQL server (12+) and its client headers if you're not
|
- **Postgres only**: a PostgreSQL server (12+) and its client headers if you're not
|
||||||
using the bundled `psycopg[binary]` wheel (most platforms are covered by the wheel,
|
using the bundled `psycopg[binary]` wheel (most platforms are covered by the wheel,
|
||||||
so this is rarely needed)
|
so this is rarely needed)
|
||||||
@@ -143,6 +143,10 @@ At minimum, set the following in `.env`:
|
|||||||
- `EVECTION_GUNICORN_BIND` — the `address:port` gunicorn listens on; see
|
- `EVECTION_GUNICORN_BIND` — the `address:port` gunicorn listens on; see
|
||||||
"Reverse proxy" below for how to pick this
|
"Reverse proxy" below for how to pick this
|
||||||
|
|
||||||
|
How often every feed is polled is a site-wide setting (Django admin → Site
|
||||||
|
settings), not a `.env` value — `EVECTION_MIN_FETCH_INTERVAL_MINUTES` only sets
|
||||||
|
a floor under it, so it can't be configured lower than that without editing `.env`.
|
||||||
|
|
||||||
Evection reads `config.settings.prod` when run via the commands below (see each
|
Evection reads `config.settings.prod` when run via the commands below (see each
|
||||||
systemd unit's `DJANGO_SETTINGS_MODULE`). That module turns on HTTPS-only cookies,
|
systemd unit's `DJANGO_SETTINGS_MODULE`). That module turns on HTTPS-only cookies,
|
||||||
HSTS, and a Content-Security-Policy header on top of the base settings, and
|
HSTS, and a Content-Security-Policy header on top of the base settings, and
|
||||||
@@ -295,6 +299,7 @@ git fetch --tags --force
|
|||||||
LATEST_TAG="$(git tag --sort=-creatordate | head -n1)"
|
LATEST_TAG="$(git tag --sort=-creatordate | head -n1)"
|
||||||
git checkout --detach "$LATEST_TAG"
|
git checkout --detach "$LATEST_TAG"
|
||||||
chown -R evection:evection /opt/evection/app
|
chown -R evection:evection /opt/evection/app
|
||||||
|
sed -i "s/^EVECTION_PWA_CACHE_VERSION=.*/EVECTION_PWA_CACHE_VERSION=$LATEST_TAG/" .env
|
||||||
sudo -u evection -H /opt/evection/venv/bin/pip install .
|
sudo -u evection -H /opt/evection/venv/bin/pip install .
|
||||||
sudo -u evection -H env DJANGO_SETTINGS_MODULE=config.settings.prod \
|
sudo -u evection -H env DJANGO_SETTINGS_MODULE=config.settings.prod \
|
||||||
/opt/evection/venv/bin/python manage.py migrate
|
/opt/evection/venv/bin/python manage.py migrate
|
||||||
@@ -311,7 +316,7 @@ git tag v$(date +%Y.%m.%d)
|
|||||||
git push --tags
|
git push --tags
|
||||||
```
|
```
|
||||||
|
|
||||||
Bump `EVECTION_PWA_CACHE_VERSION` in `.env` whenever a release changes cached
|
`upgrade.sh` sets `EVECTION_PWA_CACHE_VERSION` in `.env` to the deployed tag on every
|
||||||
static assets (CSS/JS/icons) — it's baked into the service worker's cache name, so
|
run — it's baked into the service worker's cache name, so this forces every installed
|
||||||
changing it forces every installed PWA to drop its old cache instead of serving
|
PWA to drop its old cache instead of serving stale assets after the upgrade. You don't
|
||||||
stale assets after the upgrade.
|
need to bump it by hand.
|
||||||
+1
-1
@@ -204,7 +204,7 @@ REDIS_URL=${EVECTION_REDIS_URL:-redis://localhost:6379/0}
|
|||||||
EVECTION_GUNICORN_BIND=$GUNICORN_BIND
|
EVECTION_GUNICORN_BIND=$GUNICORN_BIND
|
||||||
EVECTION_SIGNUP_ENABLED=False
|
EVECTION_SIGNUP_ENABLED=False
|
||||||
EVECTION_MIN_FETCH_INTERVAL_MINUTES=15
|
EVECTION_MIN_FETCH_INTERVAL_MINUTES=15
|
||||||
EVECTION_PWA_CACHE_VERSION=v1
|
EVECTION_PWA_CACHE_VERSION=${LATEST_TAG:-v1}
|
||||||
EOF
|
EOF
|
||||||
chown "$EVECTION_USER:$EVECTION_USER" "$ENV_FILE"
|
chown "$EVECTION_USER:$EVECTION_USER" "$ENV_FILE"
|
||||||
chmod 600 "$ENV_FILE"
|
chmod 600 "$ENV_FILE"
|
||||||
|
|||||||
+12
-3
@@ -72,6 +72,18 @@ echo "Deploying $LATEST_TAG (currently on: $CURRENT_REF)"
|
|||||||
git checkout --quiet --detach "$LATEST_TAG"
|
git checkout --quiet --detach "$LATEST_TAG"
|
||||||
chown -R "$EVECTION_USER:$EVECTION_USER" "$APP_DIR"
|
chown -R "$EVECTION_USER:$EVECTION_USER" "$APP_DIR"
|
||||||
|
|
||||||
|
# Bump the PWA's cache-busting version to this release's tag on every
|
||||||
|
# deploy, not just ones that happened to touch static assets — a missed
|
||||||
|
# manual bump otherwise leaves installed PWAs serving a stale cached
|
||||||
|
# shell/pages indefinitely. Cheap even when nothing cacheable changed:
|
||||||
|
# it just makes the service worker open a fresh cache name next time it
|
||||||
|
# activates.
|
||||||
|
if grep -q '^EVECTION_PWA_CACHE_VERSION=' "$APP_DIR/.env"; then
|
||||||
|
sed -i "s/^EVECTION_PWA_CACHE_VERSION=.*/EVECTION_PWA_CACHE_VERSION=$LATEST_TAG/" "$APP_DIR/.env"
|
||||||
|
else
|
||||||
|
echo "EVECTION_PWA_CACHE_VERSION=$LATEST_TAG" >> "$APP_DIR/.env"
|
||||||
|
fi
|
||||||
|
|
||||||
echo "==> Installing updated dependencies"
|
echo "==> Installing updated dependencies"
|
||||||
sudo -u "$EVECTION_USER" -H "$VENV_DIR/bin/pip" install --upgrade "$APP_DIR"
|
sudo -u "$EVECTION_USER" -H "$VENV_DIR/bin/pip" install --upgrade "$APP_DIR"
|
||||||
|
|
||||||
@@ -131,7 +143,4 @@ systemctl status evection-gunicorn --no-pager -l | head -10
|
|||||||
|
|
||||||
echo
|
echo
|
||||||
echo "==> Deployed $LATEST_TAG."
|
echo "==> Deployed $LATEST_TAG."
|
||||||
echo " If this release changed cached static assets (CSS/JS/icons), bump"
|
|
||||||
echo " EVECTION_PWA_CACHE_VERSION in $APP_DIR/.env and restart evection-gunicorn"
|
|
||||||
echo " again so installed PWAs drop their old cache instead of serving stale assets."
|
|
||||||
echo " journalctl -u evection-gunicorn -f to watch logs."
|
echo " journalctl -u evection-gunicorn -f to watch logs."
|
||||||
@@ -290,13 +290,48 @@ form p {
|
|||||||
padding: 0 0.5rem;
|
padding: 0 0.5rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* `summary`'s native disclosure triangle is a ::marker, positioned for
|
||||||
|
list-item/inline content — once the tag-name + unread-count row below
|
||||||
|
needed to be a flex box (block-level), the marker got pushed onto its
|
||||||
|
own line above instead of sitting beside it. Replacing it with a
|
||||||
|
plain ::before triangle lets `summary` itself be a flex row with that
|
||||||
|
triangle as a normal flex item, so it stays inline no matter what the
|
||||||
|
rest of the row contains. */
|
||||||
.sidebar-section summary {
|
.sidebar-section summary {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.4rem;
|
||||||
padding: 0.4rem 0.75rem;
|
padding: 0.4rem 0.75rem;
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
color: var(--text-dim);
|
color: var(--text-dim);
|
||||||
font-size: 0.75rem;
|
font-size: 0.75rem;
|
||||||
text-transform: uppercase;
|
text-transform: uppercase;
|
||||||
letter-spacing: 0.04em;
|
letter-spacing: 0.04em;
|
||||||
|
list-style: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.sidebar-section summary::-webkit-details-marker {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.sidebar-section summary::before {
|
||||||
|
content: "▶";
|
||||||
|
flex: 0 0 auto;
|
||||||
|
font-size: 0.6rem;
|
||||||
|
transition: transform 0.1s ease;
|
||||||
|
}
|
||||||
|
|
||||||
|
.sidebar-section[open] > summary::before {
|
||||||
|
transform: rotate(90deg);
|
||||||
|
}
|
||||||
|
|
||||||
|
.sidebar-tag-row {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 0.4rem;
|
||||||
|
flex: 1 1 auto;
|
||||||
|
min-width: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
.sidebar-tag-link {
|
.sidebar-tag-link {
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
(function () {
|
||||||
|
// The server groups articles into "Received today/yesterday/<date>"
|
||||||
|
// headers using UTC (config.settings.base TIME_ZONE), so an article
|
||||||
|
// published near UTC midnight can land in the wrong day group from the
|
||||||
|
// viewer's own perspective. Each article row carries its UTC instant in
|
||||||
|
// data-published-utc; this throws away the server's date-header <li>s
|
||||||
|
// and rebuilds them using the browser's local calendar day instead.
|
||||||
|
var MONTHS = [
|
||||||
|
"January", "February", "March", "April", "May", "June",
|
||||||
|
"July", "August", "September", "October", "November", "December",
|
||||||
|
];
|
||||||
|
|
||||||
|
function localDateOf(iso) {
|
||||||
|
if (!iso) return null;
|
||||||
|
var date = new Date(iso);
|
||||||
|
if (isNaN(date.getTime())) return null;
|
||||||
|
date.setHours(0, 0, 0, 0);
|
||||||
|
return date;
|
||||||
|
}
|
||||||
|
|
||||||
|
function labelFor(day, today) {
|
||||||
|
if (day === null) return "Undated";
|
||||||
|
var formatted = day.getDate() + " " + MONTHS[day.getMonth()] + " " + day.getFullYear();
|
||||||
|
var diffDays = Math.round((today - day) / 86400000);
|
||||||
|
if (diffDays === 0) return "Received today — " + formatted;
|
||||||
|
if (diffDays === 1) return "Received yesterday — " + formatted;
|
||||||
|
return "Received " + formatted;
|
||||||
|
}
|
||||||
|
|
||||||
|
function regroupDateHeaders(root) {
|
||||||
|
if (!root) return;
|
||||||
|
var list = root.matches && root.matches("ul.article-list") ? root : root.querySelector("ul.article-list");
|
||||||
|
if (!list || !list.querySelector("li.article-row[data-published-utc], li.article-row")) return;
|
||||||
|
|
||||||
|
list.querySelectorAll("li.date-header").forEach(function (el) {
|
||||||
|
el.parentNode.removeChild(el);
|
||||||
|
});
|
||||||
|
|
||||||
|
var today = new Date();
|
||||||
|
today.setHours(0, 0, 0, 0);
|
||||||
|
|
||||||
|
var currentKey;
|
||||||
|
list.querySelectorAll("li.article-row").forEach(function (row) {
|
||||||
|
var day = localDateOf(row.getAttribute("data-published-utc"));
|
||||||
|
var key = day === null ? "undated" : day.getTime();
|
||||||
|
if (key !== currentKey) {
|
||||||
|
var header = document.createElement("li");
|
||||||
|
header.className = "date-header";
|
||||||
|
header.textContent = labelFor(day, today);
|
||||||
|
list.insertBefore(header, row);
|
||||||
|
currentKey = key;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
regroupDateHeaders(document);
|
||||||
|
|
||||||
|
document.body.addEventListener("htmx:afterSwap", function (event) {
|
||||||
|
regroupDateHeaders(event.target);
|
||||||
|
});
|
||||||
|
})();
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
(function () {
|
||||||
|
// Everything is stored and server-rendered in UTC (config.settings.base
|
||||||
|
// TIME_ZONE) — these <time datetime="..."> elements carry that UTC
|
||||||
|
// instant plus a UTC-formatted fallback as their initial text, and this
|
||||||
|
// reformats them using the browser's own timezone, client-side, so the
|
||||||
|
// server never needs to know (or guess) where the viewer actually is.
|
||||||
|
var SHORT_OPTIONS = { hour: "numeric", minute: "2-digit" };
|
||||||
|
var LONG_OPTIONS = {
|
||||||
|
year: "numeric",
|
||||||
|
month: "short",
|
||||||
|
day: "numeric",
|
||||||
|
hour: "2-digit",
|
||||||
|
minute: "2-digit",
|
||||||
|
};
|
||||||
|
|
||||||
|
function localizeTimesWithin(root) {
|
||||||
|
if (!root || typeof root.querySelectorAll !== "function") return;
|
||||||
|
root.querySelectorAll("time[data-local-format]").forEach(function (el) {
|
||||||
|
var iso = el.getAttribute("datetime");
|
||||||
|
if (!iso) return;
|
||||||
|
var date = new Date(iso);
|
||||||
|
if (isNaN(date.getTime())) return;
|
||||||
|
var options = el.dataset.localFormat === "short" ? SHORT_OPTIONS : LONG_OPTIONS;
|
||||||
|
try {
|
||||||
|
el.textContent = date.toLocaleString(undefined, options);
|
||||||
|
} catch (e) {
|
||||||
|
/* Intl unavailable or threw — leave the server-rendered UTC text as-is */
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
localizeTimesWithin(document);
|
||||||
|
|
||||||
|
// Article rows/content are frequently swapped in by htmx (pagination,
|
||||||
|
// the inline expand-article view, the dashboard's retry action) without
|
||||||
|
// a full page load, so every swap needs the same pass over just the
|
||||||
|
// new content.
|
||||||
|
document.body.addEventListener("htmx:afterSwap", function (event) {
|
||||||
|
localizeTimesWithin(event.target);
|
||||||
|
});
|
||||||
|
})();
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
(function () {
|
||||||
|
// Remembers which sidebar <details> sections (one per tag, plus
|
||||||
|
// "Untagged" — see templates/core/_sidebar.html's data-sidebar-key)
|
||||||
|
// the user has collapsed, so a full page navigation (this isn't an
|
||||||
|
// SPA — every click re-renders the sidebar server-side) doesn't keep
|
||||||
|
// snapping them back open. Per-browser via localStorage, same pattern
|
||||||
|
// as theme.js/theme-init.js — not synced across devices/accounts.
|
||||||
|
var STORAGE_KEY = "evection-sidebar-collapsed";
|
||||||
|
|
||||||
|
function readCollapsed() {
|
||||||
|
try {
|
||||||
|
var raw = localStorage.getItem(STORAGE_KEY);
|
||||||
|
return raw ? JSON.parse(raw) : [];
|
||||||
|
} catch (e) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeCollapsed(keys) {
|
||||||
|
try {
|
||||||
|
localStorage.setItem(STORAGE_KEY, JSON.stringify(keys));
|
||||||
|
} catch (e) {
|
||||||
|
/* localStorage unavailable (private mode, etc.) — state just won't persist */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var collapsed = readCollapsed();
|
||||||
|
|
||||||
|
document.querySelectorAll(".sidebar-section[data-sidebar-key]").forEach(function (details) {
|
||||||
|
var key = details.dataset.sidebarKey;
|
||||||
|
if (collapsed.indexOf(key) !== -1) {
|
||||||
|
details.open = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
details.addEventListener("toggle", function () {
|
||||||
|
var current = readCollapsed();
|
||||||
|
var index = current.indexOf(key);
|
||||||
|
if (details.open) {
|
||||||
|
if (index !== -1) current.splice(index, 1);
|
||||||
|
} else if (index === -1) {
|
||||||
|
current.push(key);
|
||||||
|
}
|
||||||
|
writeCollapsed(current);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
})();
|
||||||
+6
-1
@@ -53,7 +53,12 @@
|
|||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
<script src="{% static 'js/stream.js' %}" defer></script>
|
<script src="{% static 'js/stream.js' %}" defer></script>
|
||||||
{% if user.is_authenticated %}<script src="{% static 'js/theme.js' %}" defer></script>{% endif %}
|
<script src="{% static 'js/localize-times.js' %}" defer></script>
|
||||||
|
<script src="{% static 'js/date-groups.js' %}" defer></script>
|
||||||
|
{% if user.is_authenticated %}
|
||||||
|
<script src="{% static 'js/theme.js' %}" defer></script>
|
||||||
|
<script src="{% static 'js/sidebar-state.js' %}" defer></script>
|
||||||
|
{% endif %}
|
||||||
<script src="{% static 'js/register-sw.js' %}" defer></script>
|
<script src="{% static 'js/register-sw.js' %}" defer></script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
@@ -31,10 +31,13 @@
|
|||||||
</ul>
|
</ul>
|
||||||
|
|
||||||
{% for entry in sidebar_tag_groups %}
|
{% for entry in sidebar_tag_groups %}
|
||||||
<details class="sidebar-section" open>
|
<details class="sidebar-section" data-sidebar-key="tag-{{ entry.tag.pk }}" open>
|
||||||
<summary>
|
<summary>
|
||||||
<a href="{% url 'reader:stream' %}?tag={{ entry.tag.pk }}"
|
<span class="sidebar-tag-row">
|
||||||
class="sidebar-tag-link{% if request.GET.tag == entry.tag.pk|stringformat:'s' %} active{% endif %}">{{ entry.tag.name }}</a>
|
<a href="{% url 'reader:stream' %}?tag={{ entry.tag.pk }}"
|
||||||
|
class="sidebar-tag-link{% if request.GET.tag == entry.tag.pk|stringformat:'s' %} active{% endif %}">{{ entry.tag.name }}</a>
|
||||||
|
{% if entry.unread_count %}<span class="unread-count">{{ entry.unread_count }}</span>{% endif %}
|
||||||
|
</span>
|
||||||
</summary>
|
</summary>
|
||||||
{% if entry.subscriptions %}
|
{% if entry.subscriptions %}
|
||||||
<ul class="sidebar-item-list">
|
<ul class="sidebar-item-list">
|
||||||
@@ -54,7 +57,7 @@
|
|||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
|
||||||
{% if sidebar_untagged %}
|
{% if sidebar_untagged %}
|
||||||
<details class="sidebar-section" open>
|
<details class="sidebar-section" data-sidebar-key="untagged" open>
|
||||||
<summary>Untagged</summary>
|
<summary>Untagged</summary>
|
||||||
<ul class="sidebar-item-list">
|
<ul class="sidebar-item-list">
|
||||||
{% for subscription in sidebar_untagged %}
|
{% for subscription in sidebar_untagged %}
|
||||||
|
|||||||
@@ -31,7 +31,13 @@
|
|||||||
<td>{{ feed.subscriber_count }}</td>
|
<td>{{ feed.subscriber_count }}</td>
|
||||||
<td>{{ feed.get_last_fetch_status_display }}</td>
|
<td>{{ feed.get_last_fetch_status_display }}</td>
|
||||||
<td>{{ feed.fetch_error_count }}</td>
|
<td>{{ feed.fetch_error_count }}</td>
|
||||||
<td>{{ feed.last_fetched_at|default:"never" }}</td>
|
<td>
|
||||||
|
{% if feed.last_fetched_at %}
|
||||||
|
<time datetime="{{ feed.last_fetched_at|date:"c" }}" data-local-format="long">{{ feed.last_fetched_at|date:"M j, Y H:i" }}</time>
|
||||||
|
{% else %}
|
||||||
|
never
|
||||||
|
{% endif %}
|
||||||
|
</td>
|
||||||
<td>{{ feed.last_error_message|default:"—" }}</td>
|
<td>{{ feed.last_error_message|default:"—" }}</td>
|
||||||
<td>
|
<td>
|
||||||
<form method="post" action="{% url 'core:retry_feed' feed.id %}">
|
<form method="post" action="{% url 'core:retry_feed' feed.id %}">
|
||||||
|
|||||||
+15
-1
@@ -66,8 +66,23 @@ async function cacheFirstForAsset(request) {
|
|||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const LOGOUT_PATH = "{% url 'accounts:logout' %}";
|
||||||
|
|
||||||
self.addEventListener("fetch", (event) => {
|
self.addEventListener("fetch", (event) => {
|
||||||
const { request } = event;
|
const { request } = event;
|
||||||
|
const url = new URL(request.url);
|
||||||
|
|
||||||
|
// Logging out ends this device's session, so any authenticated HTML
|
||||||
|
// sitting in PAGES_CACHE must not outlive it — otherwise a later
|
||||||
|
// offline navigation (shared device, different account logging in
|
||||||
|
// next) could still serve a previous user's cached article pages.
|
||||||
|
// Purging it synchronously with the logout request itself, rather
|
||||||
|
// than via a postMessage from the page, means there's no race with
|
||||||
|
// the navigation that follows.
|
||||||
|
if (request.method === "POST" && url.pathname === LOGOUT_PATH) {
|
||||||
|
event.respondWith(caches.delete(PAGES_CACHE).then(() => fetch(request)));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Mutations (mark-read, star, OPML upload, forms...) are always
|
// Mutations (mark-read, star, OPML upload, forms...) are always
|
||||||
// network-only — never intercept anything but GET.
|
// network-only — never intercept anything but GET.
|
||||||
@@ -78,7 +93,6 @@ self.addEventListener("fetch", (event) => {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const url = new URL(request.url);
|
|
||||||
if (url.pathname.startsWith("/static/")) {
|
if (url.pathname.startsWith("/static/")) {
|
||||||
event.respondWith(cacheFirstForAsset(request));
|
event.respondWith(cacheFirstForAsset(request));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,10 @@
|
|||||||
<h2 class="article-content-title">{{ article.title|default:"(untitled)" }}</h2>
|
<h2 class="article-content-title">{{ article.title|default:"(untitled)" }}</h2>
|
||||||
<p class="article-meta">
|
<p class="article-meta">
|
||||||
{% if article.feed.favicon_url %}<img src="{{ article.feed.favicon_url }}" alt="" class="feed-favicon">{% endif %}
|
{% if article.feed.favicon_url %}<img src="{{ article.feed.favicon_url }}" alt="" class="feed-favicon">{% endif %}
|
||||||
{{ article.feed.title }} · {{ article.published_at|date:"M j, Y H:i" }}
|
{{ article.feed.title }}
|
||||||
|
{% if article.published_at %}
|
||||||
|
· <time datetime="{{ article.published_at|date:"c" }}" data-local-format="long">{{ article.published_at|date:"M j, Y H:i" }}</time>
|
||||||
|
{% endif %}
|
||||||
{% if article.author %} · {{ article.author }}{% endif %}
|
{% if article.author %} · {{ article.author }}{% endif %}
|
||||||
{% if article.link %} · <a href="{{ article.link }}" target="_blank" rel="noopener noreferrer">Original ↗</a>{% endif %}
|
{% if article.link %} · <a href="{{ article.link }}" target="_blank" rel="noopener noreferrer">Original ↗</a>{% endif %}
|
||||||
</p>
|
</p>
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
<li id="article-{{ article.pk }}" class="article-row{% if article.is_read %} is-read{% endif %}">
|
<li id="article-{{ article.pk }}" class="article-row{% if article.is_read %} is-read{% endif %}"
|
||||||
|
data-published-utc="{% if article.published_at %}{{ article.published_at|date:"c" }}{% endif %}">
|
||||||
<div class="article-row-main">
|
<div class="article-row-main">
|
||||||
{% include "reader/_article_actions.html" %}
|
{% include "reader/_article_actions.html" %}
|
||||||
{% if article.feed.favicon_url %}
|
{% if article.feed.favicon_url %}
|
||||||
@@ -8,7 +9,9 @@
|
|||||||
<a href="{% url 'reader:article_detail' article.pk %}" class="article-title"
|
<a href="{% url 'reader:article_detail' article.pk %}" class="article-title"
|
||||||
data-inline-url="{% url 'reader:article_inline' article.pk %}"
|
data-inline-url="{% url 'reader:article_inline' article.pk %}"
|
||||||
onclick="return toggleArticle(event, {{ article.pk }})">{{ article.title|default:"(untitled)" }}</a>
|
onclick="return toggleArticle(event, {{ article.pk }})">{{ article.title|default:"(untitled)" }}</a>
|
||||||
<span class="article-time">{{ article.published_at|date:"g:i A" }}</span>
|
{% if article.published_at %}
|
||||||
|
<time class="article-time" datetime="{{ article.published_at|date:"c" }}" data-local-format="short">{{ article.published_at|date:"g:i A" }}</time>
|
||||||
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div id="article-body-{{ article.pk }}" class="article-inline-body" hidden></div>
|
<div id="article-body-{{ article.pk }}" class="article-inline-body" hidden></div>
|
||||||
</li>
|
</li>
|
||||||
Reference in new issue
Block a user