Private
Public Access
Add core data models and the feed fetch pipeline
Implements Folder/Feed/Subscription (feeds app) and Article/ UserArticleState/Tag/ArticleTag (articles app), preserving the shared-Feed/shared-Article dedup design: a feed is fetched once regardless of subscriber count, and per-user read/star/save/hide state is created lazily rather than eagerly per subscriber. Adds the fetch pipeline: feedparser-based parsing with conditional GET (ETag/Last-Modified), nh3 HTML sanitization, guid-based dedup with content-hash change detection that never resets a user's read/star state, and exponential backoff on fetch errors respecting a configured minimum interval floor. Wired into Celery via poll_due_feeds/fetch_feed tasks, plus a minimal "add feed by URL" view that fetches synchronously so new subscribers see results immediately. Verified live against a real feed (hnrss.org) for ingestion, 304 dedup, and error backoff, in addition to the unit/integration test suite (28 tests, all passing). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c635cd9330
commit
d200ab80f7
19 files changed
+1355
-8
No files matched your search
+27
-1
@@ -1 +1,27 @@
|
||||
# Register your models here.
|
||||
from django.contrib import admin
|
||||
|
||||
from .models import Article, ArticleTag, Tag, UserArticleState
|
||||
|
||||
|
||||
@admin.register(Article)
|
||||
class ArticleAdmin(admin.ModelAdmin):
|
||||
list_display = ["title", "feed", "published_at", "fetched_at"]
|
||||
list_filter = ["feed"]
|
||||
search_fields = ["title", "guid", "link"]
|
||||
|
||||
|
||||
@admin.register(UserArticleState)
|
||||
class UserArticleStateAdmin(admin.ModelAdmin):
|
||||
list_display = ["user", "article", "is_read", "is_starred", "is_saved_for_later", "is_hidden"]
|
||||
list_filter = ["is_read", "is_starred", "is_saved_for_later", "is_hidden"]
|
||||
|
||||
|
||||
@admin.register(Tag)
|
||||
class TagAdmin(admin.ModelAdmin):
|
||||
list_display = ["name", "user", "color"]
|
||||
list_filter = ["user"]
|
||||
|
||||
|
||||
@admin.register(ArticleTag)
|
||||
class ArticleTagAdmin(admin.ModelAdmin):
|
||||
list_display = ["user", "article", "tag"]
|
||||
@@ -0,0 +1,171 @@
|
||||
# Generated by Django 5.2.18 on 2026-10-30 21:57
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
("feeds", "0001_initial"),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="Article",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("guid", models.CharField(max_length=1000)),
|
||||
("link", models.URLField(blank=True, max_length=2000)),
|
||||
("title", models.CharField(blank=True, max_length=1000)),
|
||||
("author", models.CharField(blank=True, max_length=500)),
|
||||
("content_html", models.TextField(blank=True)),
|
||||
("summary", models.TextField(blank=True)),
|
||||
("content_hash", models.CharField(blank=True, max_length=64)),
|
||||
("published_at", models.DateTimeField(blank=True, null=True)),
|
||||
("fetched_at", models.DateTimeField(auto_now_add=True)),
|
||||
(
|
||||
"feed",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="articles",
|
||||
to="feeds.feed",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ["-published_at", "-fetched_at"],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Tag",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("name", models.CharField(max_length=100)),
|
||||
("color", models.CharField(blank=True, max_length=7)),
|
||||
(
|
||||
"user",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="tags",
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ["name"],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="ArticleTag",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
(
|
||||
"article",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="article_tags",
|
||||
to="articles.article",
|
||||
),
|
||||
),
|
||||
(
|
||||
"user",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
|
||||
),
|
||||
),
|
||||
(
|
||||
"tag",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="article_tags",
|
||||
to="articles.tag",
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="UserArticleState",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("is_read", models.BooleanField(default=False)),
|
||||
("is_starred", models.BooleanField(default=False)),
|
||||
("is_saved_for_later", models.BooleanField(default=False)),
|
||||
("is_hidden", models.BooleanField(default=False)),
|
||||
("read_at", models.DateTimeField(blank=True, null=True)),
|
||||
("starred_at", models.DateTimeField(blank=True, null=True)),
|
||||
(
|
||||
"article",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="user_states",
|
||||
to="articles.article",
|
||||
),
|
||||
),
|
||||
(
|
||||
"user",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="article_states",
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="article",
|
||||
index=models.Index(
|
||||
fields=["feed", "published_at"], name="articles_ar_feed_id_67b4fb_idx"
|
||||
),
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name="article",
|
||||
unique_together={("feed", "guid")},
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name="tag",
|
||||
unique_together={("user", "name")},
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name="articletag",
|
||||
unique_together={("user", "article", "tag")},
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="userarticlestate",
|
||||
index=models.Index(fields=["user", "is_read"], name="articles_us_user_id_ded063_idx"),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="userarticlestate",
|
||||
index=models.Index(
|
||||
fields=["user", "is_starred"], name="articles_us_user_id_cb0761_idx"
|
||||
),
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name="userarticlestate",
|
||||
unique_together={("user", "article")},
|
||||
),
|
||||
]
|
||||
+91
-1
@@ -1 +1,91 @@
|
||||
# Create your models here.
|
||||
from django.conf import settings
|
||||
from django.db import models
|
||||
|
||||
from apps.feeds.models import Feed
|
||||
|
||||
|
||||
class Article(models.Model):
|
||||
"""A single fetched entry, shared across every subscriber of its feed.
|
||||
|
||||
Per-user state (read/starred/saved/hidden) lives in UserArticleState,
|
||||
never on this model — editing an article's content on re-fetch must
|
||||
never touch anyone's read/star state.
|
||||
"""
|
||||
|
||||
feed = models.ForeignKey(Feed, on_delete=models.CASCADE, related_name="articles")
|
||||
guid = models.CharField(max_length=1000)
|
||||
link = models.URLField(max_length=2000, blank=True)
|
||||
title = models.CharField(max_length=1000, blank=True)
|
||||
author = models.CharField(max_length=500, blank=True)
|
||||
content_html = models.TextField(blank=True)
|
||||
summary = models.TextField(blank=True)
|
||||
content_hash = models.CharField(max_length=64, blank=True)
|
||||
published_at = models.DateTimeField(null=True, blank=True)
|
||||
fetched_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
unique_together = [("feed", "guid")]
|
||||
indexes = [models.Index(fields=["feed", "published_at"])]
|
||||
ordering = ["-published_at", "-fetched_at"]
|
||||
|
||||
def __str__(self):
|
||||
return self.title or self.guid
|
||||
|
||||
|
||||
class UserArticleState(models.Model):
|
||||
"""Per-user read/star/save/hide state for an Article.
|
||||
|
||||
Rows are created lazily, only on first interaction — a row with no
|
||||
explicit True flags is equivalent to "no row exists" (unread, not
|
||||
starred). Never bulk-create one of these per subscriber on ingest.
|
||||
"""
|
||||
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="article_states"
|
||||
)
|
||||
article = models.ForeignKey(Article, on_delete=models.CASCADE, related_name="user_states")
|
||||
|
||||
is_read = models.BooleanField(default=False)
|
||||
is_starred = models.BooleanField(default=False)
|
||||
is_saved_for_later = models.BooleanField(default=False)
|
||||
is_hidden = models.BooleanField(default=False)
|
||||
|
||||
read_at = models.DateTimeField(null=True, blank=True)
|
||||
starred_at = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
class Meta:
|
||||
unique_together = [("user", "article")]
|
||||
indexes = [
|
||||
models.Index(fields=["user", "is_read"]),
|
||||
models.Index(fields=["user", "is_starred"]),
|
||||
]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.user}: {self.article}"
|
||||
|
||||
|
||||
class Tag(models.Model):
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="tags"
|
||||
)
|
||||
name = models.CharField(max_length=100)
|
||||
color = models.CharField(max_length=7, blank=True)
|
||||
|
||||
class Meta:
|
||||
unique_together = [("user", "name")]
|
||||
ordering = ["name"]
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
|
||||
class ArticleTag(models.Model):
|
||||
user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
|
||||
article = models.ForeignKey(Article, on_delete=models.CASCADE, related_name="article_tags")
|
||||
tag = models.ForeignKey(Tag, on_delete=models.CASCADE, related_name="article_tags")
|
||||
|
||||
class Meta:
|
||||
unique_together = [("user", "article", "tag")]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.article} -> {self.tag}"
|
||||
+81
-1
@@ -1 +1,81 @@
|
||||
# Create your tests here.
|
||||
import pytest
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
from apps.feeds.models import Feed
|
||||
|
||||
from .models import Article, Tag, UserArticleState
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def feed(db):
|
||||
return Feed.objects.create(feed_url="https://example.com/feed.xml", title="Example")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def two_users(db):
|
||||
alice = User.objects.create_user("alice", password="pw-alice-123")
|
||||
bob = User.objects.create_user("bob", password="pw-bob-123")
|
||||
return alice, bob
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_articles_dedup_by_feed_and_guid(feed):
|
||||
article, created = Article.objects.get_or_create(
|
||||
feed=feed, guid="guid-1", defaults={"title": "First version"}
|
||||
)
|
||||
assert created
|
||||
|
||||
# Re-ingesting the same guid (as the fetch pipeline would on a repeat
|
||||
# poll) must not create a second row.
|
||||
article_again, created_again = Article.objects.get_or_create(
|
||||
feed=feed, guid="guid-1", defaults={"title": "Should not be used"}
|
||||
)
|
||||
assert not created_again
|
||||
assert article_again.pk == article.pk
|
||||
assert Article.objects.filter(feed=feed, guid="guid-1").count() == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_user_article_state_is_lazy_and_per_user(feed, two_users):
|
||||
alice, bob = two_users
|
||||
article = Article.objects.create(feed=feed, guid="guid-1", title="Hello")
|
||||
|
||||
# No state rows exist until someone interacts with the article.
|
||||
assert UserArticleState.objects.filter(article=article).count() == 0
|
||||
|
||||
UserArticleState.objects.create(user=alice, article=article, is_read=True)
|
||||
|
||||
assert UserArticleState.objects.filter(article=article).count() == 1
|
||||
alice_state = UserArticleState.objects.get(user=alice, article=article)
|
||||
assert alice_state.is_read is True
|
||||
# Bob has no row at all — he should be treated as unread, not as sharing
|
||||
# Alice's state.
|
||||
assert not UserArticleState.objects.filter(user=bob, article=article).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_tags_are_scoped_per_user(two_users):
|
||||
alice, bob = two_users
|
||||
Tag.objects.create(user=alice, name="golang")
|
||||
Tag.objects.create(user=bob, name="golang")
|
||||
|
||||
assert Tag.objects.filter(name="golang").count() == 2
|
||||
|
||||
with pytest.raises(Exception):
|
||||
Tag.objects.create(user=alice, name="golang")
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_editing_article_content_does_not_touch_user_state(feed, two_users):
|
||||
alice, _ = two_users
|
||||
article = Article.objects.create(feed=feed, guid="guid-1", title="Original title")
|
||||
state = UserArticleState.objects.create(user=alice, article=article, is_read=True)
|
||||
|
||||
# Simulate the fetch pipeline updating content on a re-fetch.
|
||||
article.title = "Updated title"
|
||||
article.save()
|
||||
|
||||
state.refresh_from_db()
|
||||
assert state.is_read is True
|
||||
Reference in new issue
Block a user