Private
Public Access
Add core data models and the feed fetch pipeline
Implements Folder/Feed/Subscription (feeds app) and Article/ UserArticleState/Tag/ArticleTag (articles app), preserving the shared-Feed/shared-Article dedup design: a feed is fetched once regardless of subscriber count, and per-user read/star/save/hide state is created lazily rather than eagerly per subscriber. Adds the fetch pipeline: feedparser-based parsing with conditional GET (ETag/Last-Modified), nh3 HTML sanitization, guid-based dedup with content-hash change detection that never resets a user's read/star state, and exponential backoff on fetch errors respecting a configured minimum interval floor. Wired into Celery via poll_due_feeds/fetch_feed tasks, plus a minimal "add feed by URL" view that fetches synchronously so new subscribers see results immediately. Verified live against a real feed (hnrss.org) for ingestion, 304 dedup, and error backoff, in addition to the unit/integration test suite (28 tests, all passing). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c635cd9330
commit
d200ab80f7
19 files changed
+1355
-8
No files matched your search
@@ -0,0 +1,122 @@
|
||||
# Generated by Django 5.2.18 on 2026-10-30 21:57
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="Feed",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("feed_url", models.URLField(max_length=1000, unique=True)),
|
||||
("site_url", models.URLField(blank=True, max_length=1000)),
|
||||
("title", models.CharField(blank=True, max_length=500)),
|
||||
("favicon_url", models.URLField(blank=True, max_length=1000)),
|
||||
("etag", models.CharField(blank=True, max_length=255)),
|
||||
("last_modified", models.CharField(blank=True, max_length=255)),
|
||||
("last_fetched_at", models.DateTimeField(blank=True, null=True)),
|
||||
(
|
||||
"last_fetch_status",
|
||||
models.CharField(
|
||||
choices=[
|
||||
("pending", "Pending"),
|
||||
("ok", "OK"),
|
||||
("not_modified", "Not modified"),
|
||||
("error", "Error"),
|
||||
],
|
||||
default="pending",
|
||||
max_length=20,
|
||||
),
|
||||
),
|
||||
("last_error_message", models.TextField(blank=True)),
|
||||
("fetch_interval_minutes", models.PositiveIntegerField(default=60)),
|
||||
("fetch_error_count", models.PositiveIntegerField(default=0)),
|
||||
("is_active", models.BooleanField(default=True)),
|
||||
("created_at", models.DateTimeField(auto_now_add=True)),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Folder",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("name", models.CharField(max_length=255)),
|
||||
("order", models.PositiveIntegerField(default=0)),
|
||||
(
|
||||
"user",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="folders",
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ["order", "name"],
|
||||
"unique_together": {("user", "name")},
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Subscription",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("custom_title", models.CharField(blank=True, max_length=500)),
|
||||
("order", models.PositiveIntegerField(default=0)),
|
||||
("created_at", models.DateTimeField(auto_now_add=True)),
|
||||
(
|
||||
"feed",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="subscriptions",
|
||||
to="feeds.feed",
|
||||
),
|
||||
),
|
||||
(
|
||||
"folder",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="subscriptions",
|
||||
to="feeds.folder",
|
||||
),
|
||||
),
|
||||
(
|
||||
"user",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="subscriptions",
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ["order", "id"],
|
||||
"unique_together": {("user", "feed")},
|
||||
},
|
||||
),
|
||||
]
|
||||
Reference in new issue
Block a user