Add core data models and the feed fetch pipeline

Implements Folder/Feed/Subscription (feeds app) and Article/
UserArticleState/Tag/ArticleTag (articles app), preserving the
shared-Feed/shared-Article dedup design: a feed is fetched once
regardless of subscriber count, and per-user read/star/save/hide state
is created lazily rather than eagerly per subscriber.

Adds the fetch pipeline: feedparser-based parsing with conditional GET
(ETag/Last-Modified), nh3 HTML sanitization, guid-based dedup with
content-hash change detection that never resets a user's read/star
state, and exponential backoff on fetch errors respecting a configured
minimum interval floor. Wired into Celery via poll_due_feeds/fetch_feed
tasks, plus a minimal "add feed by URL" view that fetches synchronously
so new subscribers see results immediately.

Verified live against a real feed (hnrss.org) for ingestion, 304
dedup, and error backoff, in addition to the unit/integration test
suite (28 tests, all passing).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5 committed 2026-10-30 16:03:20 -06:00
1 parent c635cd9330
commit d200ab80f7
19 files changed
+1355 -8

No files matched your search

+27 -1
View File
@@ -1 +1,27 @@
# Register your models here.
from django.contrib import admin
from .models import Article, ArticleTag, Tag, UserArticleState
@admin.register(Article)
class ArticleAdmin(admin.ModelAdmin):
list_display = ["title", "feed", "published_at", "fetched_at"]
list_filter = ["feed"]
search_fields = ["title", "guid", "link"]
@admin.register(UserArticleState)
class UserArticleStateAdmin(admin.ModelAdmin):
list_display = ["user", "article", "is_read", "is_starred", "is_saved_for_later", "is_hidden"]
list_filter = ["is_read", "is_starred", "is_saved_for_later", "is_hidden"]
@admin.register(Tag)
class TagAdmin(admin.ModelAdmin):
list_display = ["name", "user", "color"]
list_filter = ["user"]
@admin.register(ArticleTag)
class ArticleTagAdmin(admin.ModelAdmin):
list_display = ["user", "article", "tag"]
+171
View File
@@ -0,0 +1,171 @@
# Generated by Django 5.2.18 on 2026-10-30 21:57
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
("feeds", "0001_initial"),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name="Article",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("guid", models.CharField(max_length=1000)),
("link", models.URLField(blank=True, max_length=2000)),
("title", models.CharField(blank=True, max_length=1000)),
("author", models.CharField(blank=True, max_length=500)),
("content_html", models.TextField(blank=True)),
("summary", models.TextField(blank=True)),
("content_hash", models.CharField(blank=True, max_length=64)),
("published_at", models.DateTimeField(blank=True, null=True)),
("fetched_at", models.DateTimeField(auto_now_add=True)),
(
"feed",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="articles",
to="feeds.feed",
),
),
],
options={
"ordering": ["-published_at", "-fetched_at"],
},
),
migrations.CreateModel(
name="Tag",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("name", models.CharField(max_length=100)),
("color", models.CharField(blank=True, max_length=7)),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="tags",
to=settings.AUTH_USER_MODEL,
),
),
],
options={
"ordering": ["name"],
},
),
migrations.CreateModel(
name="ArticleTag",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
(
"article",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="article_tags",
to="articles.article",
),
),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
),
),
(
"tag",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="article_tags",
to="articles.tag",
),
),
],
),
migrations.CreateModel(
name="UserArticleState",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("is_read", models.BooleanField(default=False)),
("is_starred", models.BooleanField(default=False)),
("is_saved_for_later", models.BooleanField(default=False)),
("is_hidden", models.BooleanField(default=False)),
("read_at", models.DateTimeField(blank=True, null=True)),
("starred_at", models.DateTimeField(blank=True, null=True)),
(
"article",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="user_states",
to="articles.article",
),
),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="article_states",
to=settings.AUTH_USER_MODEL,
),
),
],
),
migrations.AddIndex(
model_name="article",
index=models.Index(
fields=["feed", "published_at"], name="articles_ar_feed_id_67b4fb_idx"
),
),
migrations.AlterUniqueTogether(
name="article",
unique_together={("feed", "guid")},
),
migrations.AlterUniqueTogether(
name="tag",
unique_together={("user", "name")},
),
migrations.AlterUniqueTogether(
name="articletag",
unique_together={("user", "article", "tag")},
),
migrations.AddIndex(
model_name="userarticlestate",
index=models.Index(fields=["user", "is_read"], name="articles_us_user_id_ded063_idx"),
),
migrations.AddIndex(
model_name="userarticlestate",
index=models.Index(
fields=["user", "is_starred"], name="articles_us_user_id_cb0761_idx"
),
),
migrations.AlterUniqueTogether(
name="userarticlestate",
unique_together={("user", "article")},
),
]
+91 -1
View File
@@ -1 +1,91 @@
# Create your models here.
from django.conf import settings
from django.db import models
from apps.feeds.models import Feed
class Article(models.Model):
"""A single fetched entry, shared across every subscriber of its feed.
Per-user state (read/starred/saved/hidden) lives in UserArticleState,
never on this model — editing an article's content on re-fetch must
never touch anyone's read/star state.
"""
feed = models.ForeignKey(Feed, on_delete=models.CASCADE, related_name="articles")
guid = models.CharField(max_length=1000)
link = models.URLField(max_length=2000, blank=True)
title = models.CharField(max_length=1000, blank=True)
author = models.CharField(max_length=500, blank=True)
content_html = models.TextField(blank=True)
summary = models.TextField(blank=True)
content_hash = models.CharField(max_length=64, blank=True)
published_at = models.DateTimeField(null=True, blank=True)
fetched_at = models.DateTimeField(auto_now_add=True)
class Meta:
unique_together = [("feed", "guid")]
indexes = [models.Index(fields=["feed", "published_at"])]
ordering = ["-published_at", "-fetched_at"]
def __str__(self):
return self.title or self.guid
class UserArticleState(models.Model):
"""Per-user read/star/save/hide state for an Article.
Rows are created lazily, only on first interaction — a row with no
explicit True flags is equivalent to "no row exists" (unread, not
starred). Never bulk-create one of these per subscriber on ingest.
"""
user = models.ForeignKey(
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="article_states"
)
article = models.ForeignKey(Article, on_delete=models.CASCADE, related_name="user_states")
is_read = models.BooleanField(default=False)
is_starred = models.BooleanField(default=False)
is_saved_for_later = models.BooleanField(default=False)
is_hidden = models.BooleanField(default=False)
read_at = models.DateTimeField(null=True, blank=True)
starred_at = models.DateTimeField(null=True, blank=True)
class Meta:
unique_together = [("user", "article")]
indexes = [
models.Index(fields=["user", "is_read"]),
models.Index(fields=["user", "is_starred"]),
]
def __str__(self):
return f"{self.user}: {self.article}"
class Tag(models.Model):
user = models.ForeignKey(
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="tags"
)
name = models.CharField(max_length=100)
color = models.CharField(max_length=7, blank=True)
class Meta:
unique_together = [("user", "name")]
ordering = ["name"]
def __str__(self):
return self.name
class ArticleTag(models.Model):
user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
article = models.ForeignKey(Article, on_delete=models.CASCADE, related_name="article_tags")
tag = models.ForeignKey(Tag, on_delete=models.CASCADE, related_name="article_tags")
class Meta:
unique_together = [("user", "article", "tag")]
def __str__(self):
return f"{self.article} -> {self.tag}"
+81 -1
View File
@@ -1 +1,81 @@
# Create your tests here.
import pytest
from django.contrib.auth import get_user_model
from apps.feeds.models import Feed
from .models import Article, Tag, UserArticleState
User = get_user_model()
@pytest.fixture
def feed(db):
return Feed.objects.create(feed_url="https://example.com/feed.xml", title="Example")
@pytest.fixture
def two_users(db):
alice = User.objects.create_user("alice", password="pw-alice-123")
bob = User.objects.create_user("bob", password="pw-bob-123")
return alice, bob
@pytest.mark.django_db
def test_articles_dedup_by_feed_and_guid(feed):
article, created = Article.objects.get_or_create(
feed=feed, guid="guid-1", defaults={"title": "First version"}
)
assert created
# Re-ingesting the same guid (as the fetch pipeline would on a repeat
# poll) must not create a second row.
article_again, created_again = Article.objects.get_or_create(
feed=feed, guid="guid-1", defaults={"title": "Should not be used"}
)
assert not created_again
assert article_again.pk == article.pk
assert Article.objects.filter(feed=feed, guid="guid-1").count() == 1
@pytest.mark.django_db
def test_user_article_state_is_lazy_and_per_user(feed, two_users):
alice, bob = two_users
article = Article.objects.create(feed=feed, guid="guid-1", title="Hello")
# No state rows exist until someone interacts with the article.
assert UserArticleState.objects.filter(article=article).count() == 0
UserArticleState.objects.create(user=alice, article=article, is_read=True)
assert UserArticleState.objects.filter(article=article).count() == 1
alice_state = UserArticleState.objects.get(user=alice, article=article)
assert alice_state.is_read is True
# Bob has no row at all — he should be treated as unread, not as sharing
# Alice's state.
assert not UserArticleState.objects.filter(user=bob, article=article).exists()
@pytest.mark.django_db
def test_tags_are_scoped_per_user(two_users):
alice, bob = two_users
Tag.objects.create(user=alice, name="golang")
Tag.objects.create(user=bob, name="golang")
assert Tag.objects.filter(name="golang").count() == 2
with pytest.raises(Exception):
Tag.objects.create(user=alice, name="golang")
@pytest.mark.django_db
def test_editing_article_content_does_not_touch_user_state(feed, two_users):
alice, _ = two_users
article = Article.objects.create(feed=feed, guid="guid-1", title="Original title")
state = UserArticleState.objects.create(user=alice, article=article, is_read=True)
# Simulate the fetch pipeline updating content on a re-fetch.
article.title = "Updated title"
article.save()
state.refresh_from_db()
assert state.is_read is True