Add core data models and the feed fetch pipeline

Implements Folder/Feed/Subscription (feeds app) and Article/
UserArticleState/Tag/ArticleTag (articles app), preserving the
shared-Feed/shared-Article dedup design: a feed is fetched once
regardless of subscriber count, and per-user read/star/save/hide state
is created lazily rather than eagerly per subscriber.

Adds the fetch pipeline: feedparser-based parsing with conditional GET
(ETag/Last-Modified), nh3 HTML sanitization, guid-based dedup with
content-hash change detection that never resets a user's read/star
state, and exponential backoff on fetch errors respecting a configured
minimum interval floor. Wired into Celery via poll_due_feeds/fetch_feed
tasks, plus a minimal "add feed by URL" view that fetches synchronously
so new subscribers see results immediately.

Verified live against a real feed (hnrss.org) for ingestion, 304
dedup, and error backoff, in addition to the unit/integration test
suite (28 tests, all passing).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5 committed 2026-10-30 16:03:20 -06:00
1 parent c635cd9330
commit d200ab80f7
19 files changed
+1355 -8

No files matched your search

+27 -1
View File
@@ -1 +1,27 @@
# Register your models here.
from django.contrib import admin
from .models import Article, ArticleTag, Tag, UserArticleState
@admin.register(Article)
class ArticleAdmin(admin.ModelAdmin):
list_display = ["title", "feed", "published_at", "fetched_at"]
list_filter = ["feed"]
search_fields = ["title", "guid", "link"]
@admin.register(UserArticleState)
class UserArticleStateAdmin(admin.ModelAdmin):
list_display = ["user", "article", "is_read", "is_starred", "is_saved_for_later", "is_hidden"]
list_filter = ["is_read", "is_starred", "is_saved_for_later", "is_hidden"]
@admin.register(Tag)
class TagAdmin(admin.ModelAdmin):
list_display = ["name", "user", "color"]
list_filter = ["user"]
@admin.register(ArticleTag)
class ArticleTagAdmin(admin.ModelAdmin):
list_display = ["user", "article", "tag"]
+171
View File
@@ -0,0 +1,171 @@
# Generated by Django 5.2.18 on 2026-10-30 21:57
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
("feeds", "0001_initial"),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name="Article",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("guid", models.CharField(max_length=1000)),
("link", models.URLField(blank=True, max_length=2000)),
("title", models.CharField(blank=True, max_length=1000)),
("author", models.CharField(blank=True, max_length=500)),
("content_html", models.TextField(blank=True)),
("summary", models.TextField(blank=True)),
("content_hash", models.CharField(blank=True, max_length=64)),
("published_at", models.DateTimeField(blank=True, null=True)),
("fetched_at", models.DateTimeField(auto_now_add=True)),
(
"feed",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="articles",
to="feeds.feed",
),
),
],
options={
"ordering": ["-published_at", "-fetched_at"],
},
),
migrations.CreateModel(
name="Tag",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("name", models.CharField(max_length=100)),
("color", models.CharField(blank=True, max_length=7)),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="tags",
to=settings.AUTH_USER_MODEL,
),
),
],
options={
"ordering": ["name"],
},
),
migrations.CreateModel(
name="ArticleTag",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
(
"article",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="article_tags",
to="articles.article",
),
),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
),
),
(
"tag",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="article_tags",
to="articles.tag",
),
),
],
),
migrations.CreateModel(
name="UserArticleState",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("is_read", models.BooleanField(default=False)),
("is_starred", models.BooleanField(default=False)),
("is_saved_for_later", models.BooleanField(default=False)),
("is_hidden", models.BooleanField(default=False)),
("read_at", models.DateTimeField(blank=True, null=True)),
("starred_at", models.DateTimeField(blank=True, null=True)),
(
"article",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="user_states",
to="articles.article",
),
),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="article_states",
to=settings.AUTH_USER_MODEL,
),
),
],
),
migrations.AddIndex(
model_name="article",
index=models.Index(
fields=["feed", "published_at"], name="articles_ar_feed_id_67b4fb_idx"
),
),
migrations.AlterUniqueTogether(
name="article",
unique_together={("feed", "guid")},
),
migrations.AlterUniqueTogether(
name="tag",
unique_together={("user", "name")},
),
migrations.AlterUniqueTogether(
name="articletag",
unique_together={("user", "article", "tag")},
),
migrations.AddIndex(
model_name="userarticlestate",
index=models.Index(fields=["user", "is_read"], name="articles_us_user_id_ded063_idx"),
),
migrations.AddIndex(
model_name="userarticlestate",
index=models.Index(
fields=["user", "is_starred"], name="articles_us_user_id_cb0761_idx"
),
),
migrations.AlterUniqueTogether(
name="userarticlestate",
unique_together={("user", "article")},
),
]
+91 -1
View File
@@ -1 +1,91 @@
# Create your models here.
from django.conf import settings
from django.db import models
from apps.feeds.models import Feed
class Article(models.Model):
"""A single fetched entry, shared across every subscriber of its feed.
Per-user state (read/starred/saved/hidden) lives in UserArticleState,
never on this model — editing an article's content on re-fetch must
never touch anyone's read/star state.
"""
feed = models.ForeignKey(Feed, on_delete=models.CASCADE, related_name="articles")
guid = models.CharField(max_length=1000)
link = models.URLField(max_length=2000, blank=True)
title = models.CharField(max_length=1000, blank=True)
author = models.CharField(max_length=500, blank=True)
content_html = models.TextField(blank=True)
summary = models.TextField(blank=True)
content_hash = models.CharField(max_length=64, blank=True)
published_at = models.DateTimeField(null=True, blank=True)
fetched_at = models.DateTimeField(auto_now_add=True)
class Meta:
unique_together = [("feed", "guid")]
indexes = [models.Index(fields=["feed", "published_at"])]
ordering = ["-published_at", "-fetched_at"]
def __str__(self):
return self.title or self.guid
class UserArticleState(models.Model):
"""Per-user read/star/save/hide state for an Article.
Rows are created lazily, only on first interaction — a row with no
explicit True flags is equivalent to "no row exists" (unread, not
starred). Never bulk-create one of these per subscriber on ingest.
"""
user = models.ForeignKey(
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="article_states"
)
article = models.ForeignKey(Article, on_delete=models.CASCADE, related_name="user_states")
is_read = models.BooleanField(default=False)
is_starred = models.BooleanField(default=False)
is_saved_for_later = models.BooleanField(default=False)
is_hidden = models.BooleanField(default=False)
read_at = models.DateTimeField(null=True, blank=True)
starred_at = models.DateTimeField(null=True, blank=True)
class Meta:
unique_together = [("user", "article")]
indexes = [
models.Index(fields=["user", "is_read"]),
models.Index(fields=["user", "is_starred"]),
]
def __str__(self):
return f"{self.user}: {self.article}"
class Tag(models.Model):
user = models.ForeignKey(
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="tags"
)
name = models.CharField(max_length=100)
color = models.CharField(max_length=7, blank=True)
class Meta:
unique_together = [("user", "name")]
ordering = ["name"]
def __str__(self):
return self.name
class ArticleTag(models.Model):
user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
article = models.ForeignKey(Article, on_delete=models.CASCADE, related_name="article_tags")
tag = models.ForeignKey(Tag, on_delete=models.CASCADE, related_name="article_tags")
class Meta:
unique_together = [("user", "article", "tag")]
def __str__(self):
return f"{self.article} -> {self.tag}"
+81 -1
View File
@@ -1 +1,81 @@
# Create your tests here.
import pytest
from django.contrib.auth import get_user_model
from apps.feeds.models import Feed
from .models import Article, Tag, UserArticleState
User = get_user_model()
@pytest.fixture
def feed(db):
return Feed.objects.create(feed_url="https://example.com/feed.xml", title="Example")
@pytest.fixture
def two_users(db):
alice = User.objects.create_user("alice", password="pw-alice-123")
bob = User.objects.create_user("bob", password="pw-bob-123")
return alice, bob
@pytest.mark.django_db
def test_articles_dedup_by_feed_and_guid(feed):
article, created = Article.objects.get_or_create(
feed=feed, guid="guid-1", defaults={"title": "First version"}
)
assert created
# Re-ingesting the same guid (as the fetch pipeline would on a repeat
# poll) must not create a second row.
article_again, created_again = Article.objects.get_or_create(
feed=feed, guid="guid-1", defaults={"title": "Should not be used"}
)
assert not created_again
assert article_again.pk == article.pk
assert Article.objects.filter(feed=feed, guid="guid-1").count() == 1
@pytest.mark.django_db
def test_user_article_state_is_lazy_and_per_user(feed, two_users):
alice, bob = two_users
article = Article.objects.create(feed=feed, guid="guid-1", title="Hello")
# No state rows exist until someone interacts with the article.
assert UserArticleState.objects.filter(article=article).count() == 0
UserArticleState.objects.create(user=alice, article=article, is_read=True)
assert UserArticleState.objects.filter(article=article).count() == 1
alice_state = UserArticleState.objects.get(user=alice, article=article)
assert alice_state.is_read is True
# Bob has no row at all — he should be treated as unread, not as sharing
# Alice's state.
assert not UserArticleState.objects.filter(user=bob, article=article).exists()
@pytest.mark.django_db
def test_tags_are_scoped_per_user(two_users):
alice, bob = two_users
Tag.objects.create(user=alice, name="golang")
Tag.objects.create(user=bob, name="golang")
assert Tag.objects.filter(name="golang").count() == 2
with pytest.raises(Exception):
Tag.objects.create(user=alice, name="golang")
@pytest.mark.django_db
def test_editing_article_content_does_not_touch_user_state(feed, two_users):
alice, _ = two_users
article = Article.objects.create(feed=feed, guid="guid-1", title="Original title")
state = UserArticleState.objects.create(user=alice, article=article, is_read=True)
# Simulate the fetch pipeline updating content on a re-fetch.
article.title = "Updated title"
article.save()
state.refresh_from_db()
assert state.is_read is True
+29 -1
View File
@@ -1 +1,29 @@
# Register your models here.
from django.contrib import admin
from .models import Feed, Folder, Subscription
@admin.register(Folder)
class FolderAdmin(admin.ModelAdmin):
list_display = ["name", "user", "order"]
list_filter = ["user"]
@admin.register(Feed)
class FeedAdmin(admin.ModelAdmin):
list_display = [
"title",
"feed_url",
"last_fetch_status",
"last_fetched_at",
"fetch_error_count",
"is_active",
]
list_filter = ["last_fetch_status", "is_active"]
search_fields = ["title", "feed_url"]
@admin.register(Subscription)
class SubscriptionAdmin(admin.ModelAdmin):
list_display = ["user", "feed", "folder", "custom_title"]
list_filter = ["user", "folder"]
+13
View File
@@ -0,0 +1,13 @@
from django import forms
from .models import Folder
class AddFeedForm(forms.Form):
feed_url = forms.URLField(label="Feed URL", max_length=1000, assume_scheme="https")
folder = forms.ModelChoiceField(queryset=Folder.objects.none(), required=False)
def __init__(self, *args, user=None, **kwargs):
super().__init__(*args, **kwargs)
if user is not None:
self.fields["folder"].queryset = Folder.objects.filter(user=user)
+122
View File
@@ -0,0 +1,122 @@
# Generated by Django 5.2.18 on 2026-10-30 21:57
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name="Feed",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("feed_url", models.URLField(max_length=1000, unique=True)),
("site_url", models.URLField(blank=True, max_length=1000)),
("title", models.CharField(blank=True, max_length=500)),
("favicon_url", models.URLField(blank=True, max_length=1000)),
("etag", models.CharField(blank=True, max_length=255)),
("last_modified", models.CharField(blank=True, max_length=255)),
("last_fetched_at", models.DateTimeField(blank=True, null=True)),
(
"last_fetch_status",
models.CharField(
choices=[
("pending", "Pending"),
("ok", "OK"),
("not_modified", "Not modified"),
("error", "Error"),
],
default="pending",
max_length=20,
),
),
("last_error_message", models.TextField(blank=True)),
("fetch_interval_minutes", models.PositiveIntegerField(default=60)),
("fetch_error_count", models.PositiveIntegerField(default=0)),
("is_active", models.BooleanField(default=True)),
("created_at", models.DateTimeField(auto_now_add=True)),
],
),
migrations.CreateModel(
name="Folder",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("name", models.CharField(max_length=255)),
("order", models.PositiveIntegerField(default=0)),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="folders",
to=settings.AUTH_USER_MODEL,
),
),
],
options={
"ordering": ["order", "name"],
"unique_together": {("user", "name")},
},
),
migrations.CreateModel(
name="Subscription",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("custom_title", models.CharField(blank=True, max_length=500)),
("order", models.PositiveIntegerField(default=0)),
("created_at", models.DateTimeField(auto_now_add=True)),
(
"feed",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="subscriptions",
to="feeds.feed",
),
),
(
"folder",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="subscriptions",
to="feeds.folder",
),
),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="subscriptions",
to=settings.AUTH_USER_MODEL,
),
),
],
options={
"ordering": ["order", "id"],
"unique_together": {("user", "feed")},
},
),
]
+87 -1
View File
@@ -1 +1,87 @@
# Create your models here.
from django.conf import settings
from django.db import models
class Folder(models.Model):
user = models.ForeignKey(
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="folders"
)
name = models.CharField(max_length=255)
order = models.PositiveIntegerField(default=0)
class Meta:
unique_together = [("user", "name")]
ordering = ["order", "name"]
def __str__(self):
return self.name
class Feed(models.Model):
"""A single polled RSS/Atom source, shared across every subscriber.
One row per feed_url regardless of how many users subscribe to it — the
fetch pipeline polls each feed exactly once and every Subscription below
just points at the same Feed/Article data. Never add per-user fetching.
"""
STATUS_PENDING = "pending"
STATUS_OK = "ok"
STATUS_NOT_MODIFIED = "not_modified"
STATUS_ERROR = "error"
STATUS_CHOICES = [
(STATUS_PENDING, "Pending"),
(STATUS_OK, "OK"),
(STATUS_NOT_MODIFIED, "Not modified"),
(STATUS_ERROR, "Error"),
]
feed_url = models.URLField(max_length=1000, unique=True)
site_url = models.URLField(max_length=1000, blank=True)
title = models.CharField(max_length=500, blank=True)
favicon_url = models.URLField(max_length=1000, blank=True)
etag = models.CharField(max_length=255, blank=True)
last_modified = models.CharField(max_length=255, blank=True)
last_fetched_at = models.DateTimeField(null=True, blank=True)
last_fetch_status = models.CharField(
max_length=20, choices=STATUS_CHOICES, default=STATUS_PENDING
)
last_error_message = models.TextField(blank=True)
# Instance-wide, not a per-user preference — see class docstring.
fetch_interval_minutes = models.PositiveIntegerField(default=60)
fetch_error_count = models.PositiveIntegerField(default=0)
is_active = models.BooleanField(default=True)
created_at = models.DateTimeField(auto_now_add=True)
def __str__(self):
return self.title or self.feed_url
class Subscription(models.Model):
"""Per-user join between a User and a shared Feed."""
user = models.ForeignKey(
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="subscriptions"
)
feed = models.ForeignKey(Feed, on_delete=models.CASCADE, related_name="subscriptions")
folder = models.ForeignKey(
Folder, null=True, blank=True, on_delete=models.SET_NULL, related_name="subscriptions"
)
custom_title = models.CharField(max_length=500, blank=True)
order = models.PositiveIntegerField(default=0)
created_at = models.DateTimeField(auto_now_add=True)
class Meta:
unique_together = [("user", "feed")]
ordering = ["order", "id"]
def __str__(self):
return self.custom_title or self.feed.title or self.feed.feed_url
@property
def display_title(self):
return self.custom_title or self.feed.title or self.feed.feed_url
View File
Whitespace-only changes.
+225
View File
@@ -0,0 +1,225 @@
"""HTTP fetching + feedparser glue for polling a single Feed.
Kept separate from tasks.py so the actual fetch/parse/dedup logic can be
unit-tested with plain function calls (no Celery/broker involved) and
exercised from a shell for manual verification.
"""
from __future__ import annotations
import hashlib
import logging
from calendar import timegm
from dataclasses import dataclass
from datetime import datetime
from datetime import timezone as dt_timezone
import feedparser
import nh3
import requests
from django.conf import settings
from django.utils import timezone
from apps.articles.models import Article
from apps.feeds.models import Feed
logger = logging.getLogger(__name__)
FETCH_TIMEOUT_SECONDS = 15
MAX_BACKOFF_MINUTES = 24 * 60
USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)"
ALLOWED_TAGS = {
"a",
"p",
"br",
"hr",
"strong",
"em",
"b",
"i",
"u",
"s",
"blockquote",
"code",
"pre",
"ul",
"ol",
"li",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"img",
"figure",
"figcaption",
"table",
"thead",
"tbody",
"tr",
"th",
"td",
"span",
"div",
}
ALLOWED_ATTRIBUTES = {
"a": {"href", "title", "rel"},
"img": {"src", "alt", "title", "width", "height"},
}
@dataclass
class FetchResult:
status: str # Feed.STATUS_OK / STATUS_NOT_MODIFIED / STATUS_ERROR
new_article_ids: list[int]
updated_article_ids: list[int]
error_message: str = ""
def sanitize_html(raw_html: str) -> str:
if not raw_html:
return ""
return nh3.clean(raw_html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None)
def _content_hash(title: str, content_html: str) -> str:
return hashlib.sha256(f"{title}\n{content_html}".encode("utf-8")).hexdigest()
def _entry_guid(entry) -> str:
if getattr(entry, "id", None):
return entry.id
if getattr(entry, "link", None):
return entry.link
title = getattr(entry, "title", "")
published = getattr(entry, "published", "")
return hashlib.sha256(f"{title}|{published}".encode("utf-8")).hexdigest()
def _entry_published_at(entry):
parsed = getattr(entry, "published_parsed", None) or getattr(entry, "updated_parsed", None)
if not parsed:
return None
return datetime.fromtimestamp(timegm(parsed), tz=dt_timezone.utc)
def _entry_content_html(entry) -> str:
if getattr(entry, "content", None):
return entry.content[0].value
return getattr(entry, "summary", "") or ""
def effective_interval_minutes(feed: Feed) -> int:
"""The actual polling interval to use, applying exponential backoff on
top of the admin-configured ``fetch_interval_minutes`` for feeds that
are currently erroring. ``fetch_interval_minutes`` itself is never
mutated by the fetcher — it's the admin's chosen baseline.
"""
floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15)
base = max(feed.fetch_interval_minutes, floor)
if feed.fetch_error_count <= 0:
return base
backoff = base * (2**feed.fetch_error_count)
return min(backoff, MAX_BACKOFF_MINUTES)
def fetch_and_ingest(feed: Feed) -> FetchResult:
"""Fetch one Feed over HTTP, parse it, and dedup/insert its entries.
Pure function over a Feed instance — callers are responsible for
saving it afterwards. Safe to call directly from a shell or a test,
with no Celery broker required.
"""
headers = {"User-Agent": USER_AGENT}
if feed.etag:
headers["If-None-Match"] = feed.etag
if feed.last_modified:
headers["If-Modified-Since"] = feed.last_modified
try:
response = requests.get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS)
except requests.RequestException as exc:
feed.last_fetched_at = timezone.now()
return _record_error(feed, f"Request failed: {exc}")
feed.last_fetched_at = timezone.now()
if response.status_code == 304:
feed.last_fetch_status = Feed.STATUS_NOT_MODIFIED
feed.fetch_error_count = 0
return FetchResult(
status=Feed.STATUS_NOT_MODIFIED, new_article_ids=[], updated_article_ids=[]
)
if response.status_code >= 400:
return _record_error(feed, f"HTTP {response.status_code}")
parsed = feedparser.parse(response.content)
if parsed.bozo and not parsed.entries:
return _record_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}")
if parsed.bozo:
logger.warning(
"Feed %s parsed with warnings: %s", feed.feed_url, parsed.get("bozo_exception")
)
feed.etag = response.headers.get("ETag", "")
feed.last_modified = response.headers.get("Last-Modified", "")
feed.last_fetch_status = Feed.STATUS_OK
feed.fetch_error_count = 0
channel_title = parsed.feed.get("title")
if channel_title and not feed.title:
feed.title = channel_title[:500]
channel_link = parsed.feed.get("link")
if channel_link and not feed.site_url:
feed.site_url = channel_link[:1000]
new_ids: list[int] = []
updated_ids: list[int] = []
for entry in parsed.entries:
guid = _entry_guid(entry)
title = (getattr(entry, "title", "") or "")[:1000]
content_html = sanitize_html(_entry_content_html(entry))
content_hash = _content_hash(title, content_html)
article, created = Article.objects.get_or_create(
feed=feed,
guid=guid,
defaults={
"link": (getattr(entry, "link", "") or "")[:2000],
"title": title,
"author": (getattr(entry, "author", "") or "")[:500],
"content_html": content_html,
"summary": (getattr(entry, "summary", "") or "")[:5000],
"content_hash": content_hash,
"published_at": _entry_published_at(entry),
},
)
if created:
new_ids.append(article.pk)
elif article.content_hash != content_hash:
article.title = title
article.content_html = content_html
article.content_hash = content_hash
article.save(update_fields=["title", "content_html", "content_hash"])
updated_ids.append(article.pk)
return FetchResult(
status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids
)
def _record_error(feed: Feed, message: str) -> FetchResult:
feed.last_fetch_status = Feed.STATUS_ERROR
feed.last_error_message = message[:2000]
feed.fetch_error_count += 1
logger.warning("Feed %s fetch error: %s", feed.feed_url, message)
return FetchResult(
status=Feed.STATUS_ERROR, new_article_ids=[], updated_article_ids=[], error_message=message
)
+69
View File
@@ -0,0 +1,69 @@
import logging
from datetime import timedelta
from celery import shared_task
from django.utils import timezone
from .models import Feed
from .services.fetcher import effective_interval_minutes, fetch_and_ingest
logger = logging.getLogger(__name__)
@shared_task
def poll_due_feeds():
"""Find every active Feed whose polling interval has elapsed and
dispatch a fetch_feed task for each — fan-out happens here, not via one
Celery Beat entry per feed.
"""
due_feed_ids = [
feed.id
for feed in Feed.objects.filter(is_active=True).only(
"id", "last_fetched_at", "fetch_interval_minutes", "fetch_error_count"
)
if _is_due(feed)
]
for feed_id in due_feed_ids:
fetch_feed.delay(feed_id)
return len(due_feed_ids)
def _is_due(feed: Feed) -> bool:
if feed.last_fetched_at is None:
return True
interval = effective_interval_minutes(feed)
return feed.last_fetched_at <= timezone.now() - timedelta(minutes=interval)
@shared_task
def fetch_feed(feed_id: int):
try:
feed = Feed.objects.get(pk=feed_id, is_active=True)
except Feed.DoesNotExist:
return None
result = fetch_and_ingest(feed)
feed.save()
if result.new_article_ids:
logger.info(
"Feed %s: %d new article(s), %d updated",
feed.feed_url,
len(result.new_article_ids),
len(result.updated_article_ids),
)
# Rule application (tag/star/hide/mark_read on new articles for each
# subscriber) is wired in once apps.rules exists — see milestone 6.
return {
"status": result.status,
"new": len(result.new_article_ids),
"updated": len(result.updated_article_ids),
}
@shared_task
def reset_feed_backoff(feed_id: int):
"""Admin-triggered: clear a feed's error count so it's retried at its
normal configured interval instead of the backed-off one."""
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0)
+196
View File
@@ -0,0 +1,196 @@
import pytest
import requests
import responses
from django.utils import timezone
from apps.articles.models import Article, UserArticleState
from apps.feeds.models import Feed
from apps.feeds.services.fetcher import effective_interval_minutes, fetch_and_ingest
FEED_URL = "https://example.com/feed.xml"
GOOD_FEED_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>Example Blog</title>
<link>https://example.com</link>
<item>
<title>First post</title>
<link>https://example.com/1</link>
<guid>https://example.com/1</guid>
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
<description>&lt;p&gt;Hello &lt;script&gt;alert(1)&lt;/script&gt;world&lt;/p&gt;</description>
</item>
</channel></rss>
"""
NO_GUID_FEED_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>No Guid Blog</title>
<item>
<title>Only a link</title>
<link>https://example.com/only-link</link>
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
<description>Body</description>
</item>
</channel></rss>
"""
MALFORMED_BUT_SALVAGEABLE_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>Slightly Broken</title>
<item>
<title>Unescaped & ampersand but still has entries</title>
<guid>guid-1</guid>
</item>
</channel></rss>
"""
TOTAL_GARBAGE = "this is not xml at all {{{"
@pytest.fixture
def feed(db):
return Feed.objects.create(feed_url=FEED_URL)
@pytest.mark.django_db
@responses.activate
def test_fetch_ingests_new_articles_and_sanitizes_html(feed):
responses.add(
responses.GET,
FEED_URL,
body=GOOD_FEED_XML,
status=200,
headers={"ETag": '"abc123"', "Last-Modified": "Mon, 01 Jan 2024 00:00:00 GMT"},
)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_OK
assert len(result.new_article_ids) == 1
assert feed.title == "Example Blog"
assert feed.etag == '"abc123"'
assert feed.last_modified == "Mon, 01 Jan 2024 00:00:00 GMT"
article = Article.objects.get(feed=feed, guid="https://example.com/1")
assert "<script>" not in article.content_html
assert "world" in article.content_html
@pytest.mark.django_db
@responses.activate
def test_second_poll_with_304_does_not_duplicate_or_error(feed):
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
fetch_and_ingest(feed)
feed.save()
assert Article.objects.filter(feed=feed).count() == 1
responses.add(responses.GET, FEED_URL, status=304)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_NOT_MODIFIED
assert Article.objects.filter(feed=feed).count() == 1
assert feed.fetch_error_count == 0
@pytest.mark.django_db
@responses.activate
def test_entry_without_guid_falls_back_to_link(feed):
responses.add(responses.GET, FEED_URL, body=NO_GUID_FEED_XML, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert len(result.new_article_ids) == 1
article = Article.objects.get(pk=result.new_article_ids[0])
assert article.guid == "https://example.com/only-link"
@pytest.mark.django_db
@responses.activate
def test_bozo_feed_with_entries_still_ingests(feed):
responses.add(responses.GET, FEED_URL, body=MALFORMED_BUT_SALVAGEABLE_XML, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_OK
assert len(result.new_article_ids) == 1
@pytest.mark.django_db
@responses.activate
def test_unparseable_feed_records_error_and_backs_off(feed):
responses.add(responses.GET, FEED_URL, body=TOTAL_GARBAGE, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_ERROR
assert feed.fetch_error_count == 1
assert effective_interval_minutes(feed) > feed.fetch_interval_minutes
@pytest.mark.django_db
@responses.activate
def test_http_500_records_error(feed):
responses.add(responses.GET, FEED_URL, status=500)
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_ERROR
assert feed.last_fetch_status == Feed.STATUS_ERROR
assert feed.fetch_error_count == 1
@pytest.mark.django_db
@responses.activate
def test_connection_error_still_updates_last_fetched_at(feed):
responses.add(
responses.GET,
FEED_URL,
body=requests.exceptions.ConnectionError("boom"),
)
before = timezone.now()
result = fetch_and_ingest(feed)
feed.save()
assert result.status == Feed.STATUS_ERROR
assert feed.last_fetched_at is not None
assert feed.last_fetched_at >= before
@pytest.mark.django_db
@responses.activate
def test_refetching_updated_content_does_not_reset_user_state():
from django.contrib.auth import get_user_model
User = get_user_model()
user = User.objects.create_user("alice", password="pw-alice-123")
feed = Feed.objects.create(feed_url=FEED_URL)
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
fetch_and_ingest(feed)
feed.save()
article = Article.objects.get(feed=feed)
state = UserArticleState.objects.create(
user=user, article=article, is_read=True, is_starred=True
)
updated_xml = GOOD_FEED_XML.replace("First post", "First post (edited)")
responses.add(responses.GET, FEED_URL, body=updated_xml, status=200)
result = fetch_and_ingest(feed)
feed.save()
assert result.updated_article_ids == [article.pk]
article.refresh_from_db()
assert article.title == "First post (edited)"
state.refresh_from_db()
assert state.is_read is True
assert state.is_starred is True
+77
View File
@@ -0,0 +1,77 @@
from datetime import timedelta
from unittest.mock import patch
import pytest
import responses
from django.utils import timezone
from apps.articles.models import Article
from apps.feeds.models import Feed
from apps.feeds.tasks import fetch_feed, poll_due_feeds
@pytest.mark.django_db
def test_poll_due_feeds_dispatches_only_due_feeds():
never_fetched = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
recently_fetched = Feed.objects.create(
feed_url="https://b.example.com/feed.xml",
last_fetched_at=timezone.now(),
fetch_interval_minutes=60,
)
overdue = Feed.objects.create(
feed_url="https://c.example.com/feed.xml",
last_fetched_at=timezone.now() - timedelta(hours=2),
fetch_interval_minutes=60,
)
inactive_but_overdue = Feed.objects.create(
feed_url="https://d.example.com/feed.xml",
last_fetched_at=timezone.now() - timedelta(hours=2),
fetch_interval_minutes=60,
is_active=False,
)
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
dispatched_count = poll_due_feeds()
dispatched_ids = {call.args[0] for call in mock_delay.call_args_list}
assert dispatched_ids == {never_fetched.id, overdue.id}
assert dispatched_count == 2
assert recently_fetched.id not in dispatched_ids
assert inactive_but_overdue.id not in dispatched_ids
GOOD_FEED_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>Example Blog</title>
<item>
<title>First post</title>
<guid>guid-1</guid>
</item>
</channel></rss>
"""
@pytest.mark.django_db
@responses.activate
def test_fetch_feed_task_ingests_and_saves_feed_state():
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
result = fetch_feed(feed.id)
assert result["status"] == Feed.STATUS_OK
assert result["new"] == 1
feed.refresh_from_db()
assert feed.last_fetch_status == Feed.STATUS_OK
assert Article.objects.filter(feed=feed, guid="guid-1").exists()
@pytest.mark.django_db
def test_fetch_feed_task_skips_inactive_feed():
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", is_active=False)
assert fetch_feed(feed.id) is None
@pytest.mark.django_db
def test_fetch_feed_task_skips_missing_feed():
assert fetch_feed(999999) is None
+100 -1
View File
@@ -1 +1,100 @@
# Create your tests here.
import pytest
import responses
from django.contrib.auth import get_user_model
from django.urls import reverse
from .models import Feed, Folder, Subscription
User = get_user_model()
@pytest.fixture
def two_users(db):
alice = User.objects.create_user("alice", password="pw-alice-123")
bob = User.objects.create_user("bob", password="pw-bob-123")
return alice, bob
@pytest.mark.django_db
def test_subscribing_to_same_feed_url_reuses_one_feed_row(two_users):
alice, bob = two_users
feed, created = Feed.objects.get_or_create(feed_url="https://example.com/feed.xml")
assert created
Subscription.objects.create(user=alice, feed=feed)
feed_again, created_again = Feed.objects.get_or_create(feed_url="https://example.com/feed.xml")
assert not created_again
assert feed_again.pk == feed.pk
Subscription.objects.create(user=bob, feed=feed_again)
assert Feed.objects.count() == 1
assert Subscription.objects.filter(feed=feed).count() == 2
@pytest.mark.django_db
def test_a_user_cannot_subscribe_to_the_same_feed_twice(two_users):
alice, _ = two_users
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
Subscription.objects.create(user=alice, feed=feed)
with pytest.raises(Exception):
Subscription.objects.create(user=alice, feed=feed)
@pytest.mark.django_db
def test_folder_names_are_scoped_per_user(two_users):
alice, bob = two_users
Folder.objects.create(user=alice, name="Tech")
# Same folder name for a different user must be allowed.
Folder.objects.create(user=bob, name="Tech")
assert Folder.objects.filter(name="Tech").count() == 2
with pytest.raises(Exception):
Folder.objects.create(user=alice, name="Tech")
GOOD_FEED_XML = """<?xml version="1.0"?>
<rss version="2.0"><channel>
<title>Example Blog</title>
<item><title>First post</title><guid>guid-1</guid></item>
</channel></rss>
"""
@pytest.mark.django_db
@responses.activate
def test_add_feed_view_subscribes_and_fetches_immediately(client, two_users):
alice, _ = two_users
client.login(username="alice", password="pw-alice-123")
responses.add(responses.GET, "https://example.com/feed.xml", body=GOOD_FEED_XML, status=200)
response = client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/feed.xml"})
assert response.status_code == 302
assert Subscription.objects.filter(
user=alice, feed__feed_url="https://example.com/feed.xml"
).exists()
@pytest.mark.django_db
def test_add_feed_view_requires_login(client):
response = client.get(reverse("feeds:add_feed"))
assert response.status_code == 302
@pytest.mark.django_db
@responses.activate
def test_add_feed_view_rejects_duplicate_subscription(client, two_users):
alice, _ = two_users
client.login(username="alice", password="pw-alice-123")
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
Subscription.objects.create(user=alice, feed=feed)
response = client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/feed.xml"})
assert response.status_code == 302
assert Subscription.objects.filter(user=alice, feed=feed).count() == 1
+9
View File
@@ -0,0 +1,9 @@
from django.urls import path
from . import views
app_name = "feeds"
urlpatterns = [
path("feeds/add/", views.add_feed, name="add_feed"),
]
+45 -1
View File
@@ -1 +1,45 @@
# Create your views here.
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.shortcuts import redirect, render
from .forms import AddFeedForm
from .models import Feed, Subscription
from .services.fetcher import fetch_and_ingest
@login_required
def add_feed(request):
form = AddFeedForm(request.POST or None, user=request.user)
if request.method == "POST" and form.is_valid():
feed_url = form.cleaned_data["feed_url"]
feed, _created = Feed.objects.get_or_create(feed_url=feed_url)
subscription, created = Subscription.objects.get_or_create(
user=request.user,
feed=feed,
defaults={"folder": form.cleaned_data["folder"]},
)
if not created:
messages.info(request, "You're already subscribed to that feed.")
return redirect("reader:stream")
# Fetch synchronously so the user sees results immediately; the
# periodic Celery poll takes over from here for subsequent fetches.
result = fetch_and_ingest(feed)
feed.save()
if result.status == Feed.STATUS_ERROR:
messages.warning(
request,
f"Subscribed to {feed_url}, but the initial fetch failed: {result.error_message}",
)
else:
article_count = len(result.new_article_ids)
messages.success(
request,
f"Subscribed to '{subscription.display_title}' — "
f"{article_count} article(s) fetched.",
)
return redirect("reader:stream")
return render(request, "feeds/add_feed.html", {"form": form})