Private
Public Access
Add core data models and the feed fetch pipeline
Implements Folder/Feed/Subscription (feeds app) and Article/ UserArticleState/Tag/ArticleTag (articles app), preserving the shared-Feed/shared-Article dedup design: a feed is fetched once regardless of subscriber count, and per-user read/star/save/hide state is created lazily rather than eagerly per subscriber. Adds the fetch pipeline: feedparser-based parsing with conditional GET (ETag/Last-Modified), nh3 HTML sanitization, guid-based dedup with content-hash change detection that never resets a user's read/star state, and exponential backoff on fetch errors respecting a configured minimum interval floor. Wired into Celery via poll_due_feeds/fetch_feed tasks, plus a minimal "add feed by URL" view that fetches synchronously so new subscribers see results immediately. Verified live against a real feed (hnrss.org) for ingestion, 304 dedup, and error backoff, in addition to the unit/integration test suite (28 tests, all passing). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c635cd9330
commit
d200ab80f7
19 files changed
+1355
-8
No files matched your search
+91
-1
@@ -1 +1,91 @@
|
||||
# Create your models here.
|
||||
from django.conf import settings
|
||||
from django.db import models
|
||||
|
||||
from apps.feeds.models import Feed
|
||||
|
||||
|
||||
class Article(models.Model):
|
||||
"""A single fetched entry, shared across every subscriber of its feed.
|
||||
|
||||
Per-user state (read/starred/saved/hidden) lives in UserArticleState,
|
||||
never on this model — editing an article's content on re-fetch must
|
||||
never touch anyone's read/star state.
|
||||
"""
|
||||
|
||||
feed = models.ForeignKey(Feed, on_delete=models.CASCADE, related_name="articles")
|
||||
guid = models.CharField(max_length=1000)
|
||||
link = models.URLField(max_length=2000, blank=True)
|
||||
title = models.CharField(max_length=1000, blank=True)
|
||||
author = models.CharField(max_length=500, blank=True)
|
||||
content_html = models.TextField(blank=True)
|
||||
summary = models.TextField(blank=True)
|
||||
content_hash = models.CharField(max_length=64, blank=True)
|
||||
published_at = models.DateTimeField(null=True, blank=True)
|
||||
fetched_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
unique_together = [("feed", "guid")]
|
||||
indexes = [models.Index(fields=["feed", "published_at"])]
|
||||
ordering = ["-published_at", "-fetched_at"]
|
||||
|
||||
def __str__(self):
|
||||
return self.title or self.guid
|
||||
|
||||
|
||||
class UserArticleState(models.Model):
|
||||
"""Per-user read/star/save/hide state for an Article.
|
||||
|
||||
Rows are created lazily, only on first interaction — a row with no
|
||||
explicit True flags is equivalent to "no row exists" (unread, not
|
||||
starred). Never bulk-create one of these per subscriber on ingest.
|
||||
"""
|
||||
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="article_states"
|
||||
)
|
||||
article = models.ForeignKey(Article, on_delete=models.CASCADE, related_name="user_states")
|
||||
|
||||
is_read = models.BooleanField(default=False)
|
||||
is_starred = models.BooleanField(default=False)
|
||||
is_saved_for_later = models.BooleanField(default=False)
|
||||
is_hidden = models.BooleanField(default=False)
|
||||
|
||||
read_at = models.DateTimeField(null=True, blank=True)
|
||||
starred_at = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
class Meta:
|
||||
unique_together = [("user", "article")]
|
||||
indexes = [
|
||||
models.Index(fields=["user", "is_read"]),
|
||||
models.Index(fields=["user", "is_starred"]),
|
||||
]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.user}: {self.article}"
|
||||
|
||||
|
||||
class Tag(models.Model):
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="tags"
|
||||
)
|
||||
name = models.CharField(max_length=100)
|
||||
color = models.CharField(max_length=7, blank=True)
|
||||
|
||||
class Meta:
|
||||
unique_together = [("user", "name")]
|
||||
ordering = ["name"]
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
|
||||
class ArticleTag(models.Model):
|
||||
user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
|
||||
article = models.ForeignKey(Article, on_delete=models.CASCADE, related_name="article_tags")
|
||||
tag = models.ForeignKey(Tag, on_delete=models.CASCADE, related_name="article_tags")
|
||||
|
||||
class Meta:
|
||||
unique_together = [("user", "article", "tag")]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.article} -> {self.tag}"
|
||||
Reference in new issue
Block a user