Private
Public Access
Add core data models and the feed fetch pipeline
Implements Folder/Feed/Subscription (feeds app) and Article/ UserArticleState/Tag/ArticleTag (articles app), preserving the shared-Feed/shared-Article dedup design: a feed is fetched once regardless of subscriber count, and per-user read/star/save/hide state is created lazily rather than eagerly per subscriber. Adds the fetch pipeline: feedparser-based parsing with conditional GET (ETag/Last-Modified), nh3 HTML sanitization, guid-based dedup with content-hash change detection that never resets a user's read/star state, and exponential backoff on fetch errors respecting a configured minimum interval floor. Wired into Celery via poll_due_feeds/fetch_feed tasks, plus a minimal "add feed by URL" view that fetches synchronously so new subscribers see results immediately. Verified live against a real feed (hnrss.org) for ingestion, 304 dedup, and error backoff, in addition to the unit/integration test suite (28 tests, all passing). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c635cd9330
commit
d200ab80f7
19 files changed
+1355
-8
No files matched your search
+87
-1
@@ -1 +1,87 @@
|
||||
# Create your models here.
|
||||
from django.conf import settings
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Folder(models.Model):
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="folders"
|
||||
)
|
||||
name = models.CharField(max_length=255)
|
||||
order = models.PositiveIntegerField(default=0)
|
||||
|
||||
class Meta:
|
||||
unique_together = [("user", "name")]
|
||||
ordering = ["order", "name"]
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
|
||||
class Feed(models.Model):
|
||||
"""A single polled RSS/Atom source, shared across every subscriber.
|
||||
|
||||
One row per feed_url regardless of how many users subscribe to it — the
|
||||
fetch pipeline polls each feed exactly once and every Subscription below
|
||||
just points at the same Feed/Article data. Never add per-user fetching.
|
||||
"""
|
||||
|
||||
STATUS_PENDING = "pending"
|
||||
STATUS_OK = "ok"
|
||||
STATUS_NOT_MODIFIED = "not_modified"
|
||||
STATUS_ERROR = "error"
|
||||
STATUS_CHOICES = [
|
||||
(STATUS_PENDING, "Pending"),
|
||||
(STATUS_OK, "OK"),
|
||||
(STATUS_NOT_MODIFIED, "Not modified"),
|
||||
(STATUS_ERROR, "Error"),
|
||||
]
|
||||
|
||||
feed_url = models.URLField(max_length=1000, unique=True)
|
||||
site_url = models.URLField(max_length=1000, blank=True)
|
||||
title = models.CharField(max_length=500, blank=True)
|
||||
favicon_url = models.URLField(max_length=1000, blank=True)
|
||||
|
||||
etag = models.CharField(max_length=255, blank=True)
|
||||
last_modified = models.CharField(max_length=255, blank=True)
|
||||
|
||||
last_fetched_at = models.DateTimeField(null=True, blank=True)
|
||||
last_fetch_status = models.CharField(
|
||||
max_length=20, choices=STATUS_CHOICES, default=STATUS_PENDING
|
||||
)
|
||||
last_error_message = models.TextField(blank=True)
|
||||
|
||||
# Instance-wide, not a per-user preference — see class docstring.
|
||||
fetch_interval_minutes = models.PositiveIntegerField(default=60)
|
||||
fetch_error_count = models.PositiveIntegerField(default=0)
|
||||
is_active = models.BooleanField(default=True)
|
||||
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
def __str__(self):
|
||||
return self.title or self.feed_url
|
||||
|
||||
|
||||
class Subscription(models.Model):
|
||||
"""Per-user join between a User and a shared Feed."""
|
||||
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="subscriptions"
|
||||
)
|
||||
feed = models.ForeignKey(Feed, on_delete=models.CASCADE, related_name="subscriptions")
|
||||
folder = models.ForeignKey(
|
||||
Folder, null=True, blank=True, on_delete=models.SET_NULL, related_name="subscriptions"
|
||||
)
|
||||
custom_title = models.CharField(max_length=500, blank=True)
|
||||
order = models.PositiveIntegerField(default=0)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
unique_together = [("user", "feed")]
|
||||
ordering = ["order", "id"]
|
||||
|
||||
def __str__(self):
|
||||
return self.custom_title or self.feed.title or self.feed.feed_url
|
||||
|
||||
@property
|
||||
def display_title(self):
|
||||
return self.custom_title or self.feed.title or self.feed.feed_url
|
||||
Reference in new issue
Block a user