Private
Public Access
Add core data models and the feed fetch pipeline
Implements Folder/Feed/Subscription (feeds app) and Article/ UserArticleState/Tag/ArticleTag (articles app), preserving the shared-Feed/shared-Article dedup design: a feed is fetched once regardless of subscriber count, and per-user read/star/save/hide state is created lazily rather than eagerly per subscriber. Adds the fetch pipeline: feedparser-based parsing with conditional GET (ETag/Last-Modified), nh3 HTML sanitization, guid-based dedup with content-hash change detection that never resets a user's read/star state, and exponential backoff on fetch errors respecting a configured minimum interval floor. Wired into Celery via poll_due_feeds/fetch_feed tasks, plus a minimal "add feed by URL" view that fetches synchronously so new subscribers see results immediately. Verified live against a real feed (hnrss.org) for ingestion, 304 dedup, and error backoff, in addition to the unit/integration test suite (28 tests, all passing). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c635cd9330
commit
d200ab80f7
19 files changed
+1355
-8
No files matched your search
@@ -0,0 +1,171 @@
|
||||
# Generated by Django 5.2.18 on 2026-10-30 21:57
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
("feeds", "0001_initial"),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="Article",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("guid", models.CharField(max_length=1000)),
|
||||
("link", models.URLField(blank=True, max_length=2000)),
|
||||
("title", models.CharField(blank=True, max_length=1000)),
|
||||
("author", models.CharField(blank=True, max_length=500)),
|
||||
("content_html", models.TextField(blank=True)),
|
||||
("summary", models.TextField(blank=True)),
|
||||
("content_hash", models.CharField(blank=True, max_length=64)),
|
||||
("published_at", models.DateTimeField(blank=True, null=True)),
|
||||
("fetched_at", models.DateTimeField(auto_now_add=True)),
|
||||
(
|
||||
"feed",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="articles",
|
||||
to="feeds.feed",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ["-published_at", "-fetched_at"],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Tag",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("name", models.CharField(max_length=100)),
|
||||
("color", models.CharField(blank=True, max_length=7)),
|
||||
(
|
||||
"user",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="tags",
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ["name"],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="ArticleTag",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
(
|
||||
"article",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="article_tags",
|
||||
to="articles.article",
|
||||
),
|
||||
),
|
||||
(
|
||||
"user",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
|
||||
),
|
||||
),
|
||||
(
|
||||
"tag",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="article_tags",
|
||||
to="articles.tag",
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="UserArticleState",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("is_read", models.BooleanField(default=False)),
|
||||
("is_starred", models.BooleanField(default=False)),
|
||||
("is_saved_for_later", models.BooleanField(default=False)),
|
||||
("is_hidden", models.BooleanField(default=False)),
|
||||
("read_at", models.DateTimeField(blank=True, null=True)),
|
||||
("starred_at", models.DateTimeField(blank=True, null=True)),
|
||||
(
|
||||
"article",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="user_states",
|
||||
to="articles.article",
|
||||
),
|
||||
),
|
||||
(
|
||||
"user",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="article_states",
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="article",
|
||||
index=models.Index(
|
||||
fields=["feed", "published_at"], name="articles_ar_feed_id_67b4fb_idx"
|
||||
),
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name="article",
|
||||
unique_together={("feed", "guid")},
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name="tag",
|
||||
unique_together={("user", "name")},
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name="articletag",
|
||||
unique_together={("user", "article", "tag")},
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="userarticlestate",
|
||||
index=models.Index(fields=["user", "is_read"], name="articles_us_user_id_ded063_idx"),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="userarticlestate",
|
||||
index=models.Index(
|
||||
fields=["user", "is_starred"], name="articles_us_user_id_cb0761_idx"
|
||||
),
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name="userarticlestate",
|
||||
unique_together={("user", "article")},
|
||||
),
|
||||
]
|
||||
Reference in new issue
Block a user