Add core data models and the feed fetch pipeline

Implements Folder/Feed/Subscription (feeds app) and Article/
UserArticleState/Tag/ArticleTag (articles app), preserving the
shared-Feed/shared-Article dedup design: a feed is fetched once
regardless of subscriber count, and per-user read/star/save/hide state
is created lazily rather than eagerly per subscriber.

Adds the fetch pipeline: feedparser-based parsing with conditional GET
(ETag/Last-Modified), nh3 HTML sanitization, guid-based dedup with
content-hash change detection that never resets a user's read/star
state, and exponential backoff on fetch errors respecting a configured
minimum interval floor. Wired into Celery via poll_due_feeds/fetch_feed
tasks, plus a minimal "add feed by URL" view that fetches synchronously
so new subscribers see results immediately.

Verified live against a real feed (hnrss.org) for ingestion, 304
dedup, and error backoff, in addition to the unit/integration test
suite (28 tests, all passing).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
ksmithandClaude Sonnet 5 committed 2026-10-30 16:03:20 -06:00
1 parent c635cd9330
commit d200ab80f7
19 files changed
+1355 -8

No files matched your search

+171
View File
@@ -0,0 +1,171 @@
# Generated by Django 5.2.18 on 2026-10-30 21:57
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
("feeds", "0001_initial"),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name="Article",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("guid", models.CharField(max_length=1000)),
("link", models.URLField(blank=True, max_length=2000)),
("title", models.CharField(blank=True, max_length=1000)),
("author", models.CharField(blank=True, max_length=500)),
("content_html", models.TextField(blank=True)),
("summary", models.TextField(blank=True)),
("content_hash", models.CharField(blank=True, max_length=64)),
("published_at", models.DateTimeField(blank=True, null=True)),
("fetched_at", models.DateTimeField(auto_now_add=True)),
(
"feed",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="articles",
to="feeds.feed",
),
),
],
options={
"ordering": ["-published_at", "-fetched_at"],
},
),
migrations.CreateModel(
name="Tag",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("name", models.CharField(max_length=100)),
("color", models.CharField(blank=True, max_length=7)),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="tags",
to=settings.AUTH_USER_MODEL,
),
),
],
options={
"ordering": ["name"],
},
),
migrations.CreateModel(
name="ArticleTag",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
(
"article",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="article_tags",
to="articles.article",
),
),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
),
),
(
"tag",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="article_tags",
to="articles.tag",
),
),
],
),
migrations.CreateModel(
name="UserArticleState",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("is_read", models.BooleanField(default=False)),
("is_starred", models.BooleanField(default=False)),
("is_saved_for_later", models.BooleanField(default=False)),
("is_hidden", models.BooleanField(default=False)),
("read_at", models.DateTimeField(blank=True, null=True)),
("starred_at", models.DateTimeField(blank=True, null=True)),
(
"article",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="user_states",
to="articles.article",
),
),
(
"user",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="article_states",
to=settings.AUTH_USER_MODEL,
),
),
],
),
migrations.AddIndex(
model_name="article",
index=models.Index(
fields=["feed", "published_at"], name="articles_ar_feed_id_67b4fb_idx"
),
),
migrations.AlterUniqueTogether(
name="article",
unique_together={("feed", "guid")},
),
migrations.AlterUniqueTogether(
name="tag",
unique_together={("user", "name")},
),
migrations.AlterUniqueTogether(
name="articletag",
unique_together={("user", "article", "tag")},
),
migrations.AddIndex(
model_name="userarticlestate",
index=models.Index(fields=["user", "is_read"], name="articles_us_user_id_ded063_idx"),
),
migrations.AddIndex(
model_name="userarticlestate",
index=models.Index(
fields=["user", "is_starred"], name="articles_us_user_id_cb0761_idx"
),
),
migrations.AlterUniqueTogether(
name="userarticlestate",
unique_together={("user", "article")},
),
]