Private
Public Access
Add core data models and the feed fetch pipeline
Implements Folder/Feed/Subscription (feeds app) and Article/ UserArticleState/Tag/ArticleTag (articles app), preserving the shared-Feed/shared-Article dedup design: a feed is fetched once regardless of subscriber count, and per-user read/star/save/hide state is created lazily rather than eagerly per subscriber. Adds the fetch pipeline: feedparser-based parsing with conditional GET (ETag/Last-Modified), nh3 HTML sanitization, guid-based dedup with content-hash change detection that never resets a user's read/star state, and exponential backoff on fetch errors respecting a configured minimum interval floor. Wired into Celery via poll_due_feeds/fetch_feed tasks, plus a minimal "add feed by URL" view that fetches synchronously so new subscribers see results immediately. Verified live against a real feed (hnrss.org) for ingestion, 304 dedup, and error backoff, in addition to the unit/integration test suite (28 tests, all passing). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c635cd9330
commit
d200ab80f7
19 files changed
+1355
-8
No files matched your search
+27
-1
@@ -1 +1,27 @@
|
||||
# Register your models here.
|
||||
from django.contrib import admin
|
||||
|
||||
from .models import Article, ArticleTag, Tag, UserArticleState
|
||||
|
||||
|
||||
@admin.register(Article)
|
||||
class ArticleAdmin(admin.ModelAdmin):
|
||||
list_display = ["title", "feed", "published_at", "fetched_at"]
|
||||
list_filter = ["feed"]
|
||||
search_fields = ["title", "guid", "link"]
|
||||
|
||||
|
||||
@admin.register(UserArticleState)
|
||||
class UserArticleStateAdmin(admin.ModelAdmin):
|
||||
list_display = ["user", "article", "is_read", "is_starred", "is_saved_for_later", "is_hidden"]
|
||||
list_filter = ["is_read", "is_starred", "is_saved_for_later", "is_hidden"]
|
||||
|
||||
|
||||
@admin.register(Tag)
|
||||
class TagAdmin(admin.ModelAdmin):
|
||||
list_display = ["name", "user", "color"]
|
||||
list_filter = ["user"]
|
||||
|
||||
|
||||
@admin.register(ArticleTag)
|
||||
class ArticleTagAdmin(admin.ModelAdmin):
|
||||
list_display = ["user", "article", "tag"]
|
||||
@@ -0,0 +1,171 @@
|
||||
# Generated by Django 5.2.18 on 2026-10-30 21:57
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
("feeds", "0001_initial"),
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="Article",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("guid", models.CharField(max_length=1000)),
|
||||
("link", models.URLField(blank=True, max_length=2000)),
|
||||
("title", models.CharField(blank=True, max_length=1000)),
|
||||
("author", models.CharField(blank=True, max_length=500)),
|
||||
("content_html", models.TextField(blank=True)),
|
||||
("summary", models.TextField(blank=True)),
|
||||
("content_hash", models.CharField(blank=True, max_length=64)),
|
||||
("published_at", models.DateTimeField(blank=True, null=True)),
|
||||
("fetched_at", models.DateTimeField(auto_now_add=True)),
|
||||
(
|
||||
"feed",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="articles",
|
||||
to="feeds.feed",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ["-published_at", "-fetched_at"],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Tag",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("name", models.CharField(max_length=100)),
|
||||
("color", models.CharField(blank=True, max_length=7)),
|
||||
(
|
||||
"user",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="tags",
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ["name"],
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="ArticleTag",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
(
|
||||
"article",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="article_tags",
|
||||
to="articles.article",
|
||||
),
|
||||
),
|
||||
(
|
||||
"user",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL
|
||||
),
|
||||
),
|
||||
(
|
||||
"tag",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="article_tags",
|
||||
to="articles.tag",
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="UserArticleState",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("is_read", models.BooleanField(default=False)),
|
||||
("is_starred", models.BooleanField(default=False)),
|
||||
("is_saved_for_later", models.BooleanField(default=False)),
|
||||
("is_hidden", models.BooleanField(default=False)),
|
||||
("read_at", models.DateTimeField(blank=True, null=True)),
|
||||
("starred_at", models.DateTimeField(blank=True, null=True)),
|
||||
(
|
||||
"article",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="user_states",
|
||||
to="articles.article",
|
||||
),
|
||||
),
|
||||
(
|
||||
"user",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="article_states",
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="article",
|
||||
index=models.Index(
|
||||
fields=["feed", "published_at"], name="articles_ar_feed_id_67b4fb_idx"
|
||||
),
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name="article",
|
||||
unique_together={("feed", "guid")},
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name="tag",
|
||||
unique_together={("user", "name")},
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name="articletag",
|
||||
unique_together={("user", "article", "tag")},
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="userarticlestate",
|
||||
index=models.Index(fields=["user", "is_read"], name="articles_us_user_id_ded063_idx"),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="userarticlestate",
|
||||
index=models.Index(
|
||||
fields=["user", "is_starred"], name="articles_us_user_id_cb0761_idx"
|
||||
),
|
||||
),
|
||||
migrations.AlterUniqueTogether(
|
||||
name="userarticlestate",
|
||||
unique_together={("user", "article")},
|
||||
),
|
||||
]
|
||||
+91
-1
@@ -1 +1,91 @@
|
||||
# Create your models here.
|
||||
from django.conf import settings
|
||||
from django.db import models
|
||||
|
||||
from apps.feeds.models import Feed
|
||||
|
||||
|
||||
class Article(models.Model):
|
||||
"""A single fetched entry, shared across every subscriber of its feed.
|
||||
|
||||
Per-user state (read/starred/saved/hidden) lives in UserArticleState,
|
||||
never on this model — editing an article's content on re-fetch must
|
||||
never touch anyone's read/star state.
|
||||
"""
|
||||
|
||||
feed = models.ForeignKey(Feed, on_delete=models.CASCADE, related_name="articles")
|
||||
guid = models.CharField(max_length=1000)
|
||||
link = models.URLField(max_length=2000, blank=True)
|
||||
title = models.CharField(max_length=1000, blank=True)
|
||||
author = models.CharField(max_length=500, blank=True)
|
||||
content_html = models.TextField(blank=True)
|
||||
summary = models.TextField(blank=True)
|
||||
content_hash = models.CharField(max_length=64, blank=True)
|
||||
published_at = models.DateTimeField(null=True, blank=True)
|
||||
fetched_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
unique_together = [("feed", "guid")]
|
||||
indexes = [models.Index(fields=["feed", "published_at"])]
|
||||
ordering = ["-published_at", "-fetched_at"]
|
||||
|
||||
def __str__(self):
|
||||
return self.title or self.guid
|
||||
|
||||
|
||||
class UserArticleState(models.Model):
|
||||
"""Per-user read/star/save/hide state for an Article.
|
||||
|
||||
Rows are created lazily, only on first interaction — a row with no
|
||||
explicit True flags is equivalent to "no row exists" (unread, not
|
||||
starred). Never bulk-create one of these per subscriber on ingest.
|
||||
"""
|
||||
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="article_states"
|
||||
)
|
||||
article = models.ForeignKey(Article, on_delete=models.CASCADE, related_name="user_states")
|
||||
|
||||
is_read = models.BooleanField(default=False)
|
||||
is_starred = models.BooleanField(default=False)
|
||||
is_saved_for_later = models.BooleanField(default=False)
|
||||
is_hidden = models.BooleanField(default=False)
|
||||
|
||||
read_at = models.DateTimeField(null=True, blank=True)
|
||||
starred_at = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
class Meta:
|
||||
unique_together = [("user", "article")]
|
||||
indexes = [
|
||||
models.Index(fields=["user", "is_read"]),
|
||||
models.Index(fields=["user", "is_starred"]),
|
||||
]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.user}: {self.article}"
|
||||
|
||||
|
||||
class Tag(models.Model):
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="tags"
|
||||
)
|
||||
name = models.CharField(max_length=100)
|
||||
color = models.CharField(max_length=7, blank=True)
|
||||
|
||||
class Meta:
|
||||
unique_together = [("user", "name")]
|
||||
ordering = ["name"]
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
|
||||
class ArticleTag(models.Model):
|
||||
user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
|
||||
article = models.ForeignKey(Article, on_delete=models.CASCADE, related_name="article_tags")
|
||||
tag = models.ForeignKey(Tag, on_delete=models.CASCADE, related_name="article_tags")
|
||||
|
||||
class Meta:
|
||||
unique_together = [("user", "article", "tag")]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.article} -> {self.tag}"
|
||||
+81
-1
@@ -1 +1,81 @@
|
||||
# Create your tests here.
|
||||
import pytest
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
from apps.feeds.models import Feed
|
||||
|
||||
from .models import Article, Tag, UserArticleState
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def feed(db):
|
||||
return Feed.objects.create(feed_url="https://example.com/feed.xml", title="Example")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def two_users(db):
|
||||
alice = User.objects.create_user("alice", password="pw-alice-123")
|
||||
bob = User.objects.create_user("bob", password="pw-bob-123")
|
||||
return alice, bob
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_articles_dedup_by_feed_and_guid(feed):
|
||||
article, created = Article.objects.get_or_create(
|
||||
feed=feed, guid="guid-1", defaults={"title": "First version"}
|
||||
)
|
||||
assert created
|
||||
|
||||
# Re-ingesting the same guid (as the fetch pipeline would on a repeat
|
||||
# poll) must not create a second row.
|
||||
article_again, created_again = Article.objects.get_or_create(
|
||||
feed=feed, guid="guid-1", defaults={"title": "Should not be used"}
|
||||
)
|
||||
assert not created_again
|
||||
assert article_again.pk == article.pk
|
||||
assert Article.objects.filter(feed=feed, guid="guid-1").count() == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_user_article_state_is_lazy_and_per_user(feed, two_users):
|
||||
alice, bob = two_users
|
||||
article = Article.objects.create(feed=feed, guid="guid-1", title="Hello")
|
||||
|
||||
# No state rows exist until someone interacts with the article.
|
||||
assert UserArticleState.objects.filter(article=article).count() == 0
|
||||
|
||||
UserArticleState.objects.create(user=alice, article=article, is_read=True)
|
||||
|
||||
assert UserArticleState.objects.filter(article=article).count() == 1
|
||||
alice_state = UserArticleState.objects.get(user=alice, article=article)
|
||||
assert alice_state.is_read is True
|
||||
# Bob has no row at all — he should be treated as unread, not as sharing
|
||||
# Alice's state.
|
||||
assert not UserArticleState.objects.filter(user=bob, article=article).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_tags_are_scoped_per_user(two_users):
|
||||
alice, bob = two_users
|
||||
Tag.objects.create(user=alice, name="golang")
|
||||
Tag.objects.create(user=bob, name="golang")
|
||||
|
||||
assert Tag.objects.filter(name="golang").count() == 2
|
||||
|
||||
with pytest.raises(Exception):
|
||||
Tag.objects.create(user=alice, name="golang")
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_editing_article_content_does_not_touch_user_state(feed, two_users):
|
||||
alice, _ = two_users
|
||||
article = Article.objects.create(feed=feed, guid="guid-1", title="Original title")
|
||||
state = UserArticleState.objects.create(user=alice, article=article, is_read=True)
|
||||
|
||||
# Simulate the fetch pipeline updating content on a re-fetch.
|
||||
article.title = "Updated title"
|
||||
article.save()
|
||||
|
||||
state.refresh_from_db()
|
||||
assert state.is_read is True
|
||||
+29
-1
@@ -1 +1,29 @@
|
||||
# Register your models here.
|
||||
from django.contrib import admin
|
||||
|
||||
from .models import Feed, Folder, Subscription
|
||||
|
||||
|
||||
@admin.register(Folder)
|
||||
class FolderAdmin(admin.ModelAdmin):
|
||||
list_display = ["name", "user", "order"]
|
||||
list_filter = ["user"]
|
||||
|
||||
|
||||
@admin.register(Feed)
|
||||
class FeedAdmin(admin.ModelAdmin):
|
||||
list_display = [
|
||||
"title",
|
||||
"feed_url",
|
||||
"last_fetch_status",
|
||||
"last_fetched_at",
|
||||
"fetch_error_count",
|
||||
"is_active",
|
||||
]
|
||||
list_filter = ["last_fetch_status", "is_active"]
|
||||
search_fields = ["title", "feed_url"]
|
||||
|
||||
|
||||
@admin.register(Subscription)
|
||||
class SubscriptionAdmin(admin.ModelAdmin):
|
||||
list_display = ["user", "feed", "folder", "custom_title"]
|
||||
list_filter = ["user", "folder"]
|
||||
@@ -0,0 +1,13 @@
|
||||
from django import forms
|
||||
|
||||
from .models import Folder
|
||||
|
||||
|
||||
class AddFeedForm(forms.Form):
|
||||
feed_url = forms.URLField(label="Feed URL", max_length=1000, assume_scheme="https")
|
||||
folder = forms.ModelChoiceField(queryset=Folder.objects.none(), required=False)
|
||||
|
||||
def __init__(self, *args, user=None, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
if user is not None:
|
||||
self.fields["folder"].queryset = Folder.objects.filter(user=user)
|
||||
@@ -0,0 +1,122 @@
|
||||
# Generated by Django 5.2.18 on 2026-10-30 21:57
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.conf import settings
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="Feed",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("feed_url", models.URLField(max_length=1000, unique=True)),
|
||||
("site_url", models.URLField(blank=True, max_length=1000)),
|
||||
("title", models.CharField(blank=True, max_length=500)),
|
||||
("favicon_url", models.URLField(blank=True, max_length=1000)),
|
||||
("etag", models.CharField(blank=True, max_length=255)),
|
||||
("last_modified", models.CharField(blank=True, max_length=255)),
|
||||
("last_fetched_at", models.DateTimeField(blank=True, null=True)),
|
||||
(
|
||||
"last_fetch_status",
|
||||
models.CharField(
|
||||
choices=[
|
||||
("pending", "Pending"),
|
||||
("ok", "OK"),
|
||||
("not_modified", "Not modified"),
|
||||
("error", "Error"),
|
||||
],
|
||||
default="pending",
|
||||
max_length=20,
|
||||
),
|
||||
),
|
||||
("last_error_message", models.TextField(blank=True)),
|
||||
("fetch_interval_minutes", models.PositiveIntegerField(default=60)),
|
||||
("fetch_error_count", models.PositiveIntegerField(default=0)),
|
||||
("is_active", models.BooleanField(default=True)),
|
||||
("created_at", models.DateTimeField(auto_now_add=True)),
|
||||
],
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Folder",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("name", models.CharField(max_length=255)),
|
||||
("order", models.PositiveIntegerField(default=0)),
|
||||
(
|
||||
"user",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="folders",
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ["order", "name"],
|
||||
"unique_together": {("user", "name")},
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Subscription",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("custom_title", models.CharField(blank=True, max_length=500)),
|
||||
("order", models.PositiveIntegerField(default=0)),
|
||||
("created_at", models.DateTimeField(auto_now_add=True)),
|
||||
(
|
||||
"feed",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="subscriptions",
|
||||
to="feeds.feed",
|
||||
),
|
||||
),
|
||||
(
|
||||
"folder",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="subscriptions",
|
||||
to="feeds.folder",
|
||||
),
|
||||
),
|
||||
(
|
||||
"user",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="subscriptions",
|
||||
to=settings.AUTH_USER_MODEL,
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ["order", "id"],
|
||||
"unique_together": {("user", "feed")},
|
||||
},
|
||||
),
|
||||
]
|
||||
+87
-1
@@ -1 +1,87 @@
|
||||
# Create your models here.
|
||||
from django.conf import settings
|
||||
from django.db import models
|
||||
|
||||
|
||||
class Folder(models.Model):
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="folders"
|
||||
)
|
||||
name = models.CharField(max_length=255)
|
||||
order = models.PositiveIntegerField(default=0)
|
||||
|
||||
class Meta:
|
||||
unique_together = [("user", "name")]
|
||||
ordering = ["order", "name"]
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
|
||||
class Feed(models.Model):
|
||||
"""A single polled RSS/Atom source, shared across every subscriber.
|
||||
|
||||
One row per feed_url regardless of how many users subscribe to it — the
|
||||
fetch pipeline polls each feed exactly once and every Subscription below
|
||||
just points at the same Feed/Article data. Never add per-user fetching.
|
||||
"""
|
||||
|
||||
STATUS_PENDING = "pending"
|
||||
STATUS_OK = "ok"
|
||||
STATUS_NOT_MODIFIED = "not_modified"
|
||||
STATUS_ERROR = "error"
|
||||
STATUS_CHOICES = [
|
||||
(STATUS_PENDING, "Pending"),
|
||||
(STATUS_OK, "OK"),
|
||||
(STATUS_NOT_MODIFIED, "Not modified"),
|
||||
(STATUS_ERROR, "Error"),
|
||||
]
|
||||
|
||||
feed_url = models.URLField(max_length=1000, unique=True)
|
||||
site_url = models.URLField(max_length=1000, blank=True)
|
||||
title = models.CharField(max_length=500, blank=True)
|
||||
favicon_url = models.URLField(max_length=1000, blank=True)
|
||||
|
||||
etag = models.CharField(max_length=255, blank=True)
|
||||
last_modified = models.CharField(max_length=255, blank=True)
|
||||
|
||||
last_fetched_at = models.DateTimeField(null=True, blank=True)
|
||||
last_fetch_status = models.CharField(
|
||||
max_length=20, choices=STATUS_CHOICES, default=STATUS_PENDING
|
||||
)
|
||||
last_error_message = models.TextField(blank=True)
|
||||
|
||||
# Instance-wide, not a per-user preference — see class docstring.
|
||||
fetch_interval_minutes = models.PositiveIntegerField(default=60)
|
||||
fetch_error_count = models.PositiveIntegerField(default=0)
|
||||
is_active = models.BooleanField(default=True)
|
||||
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
def __str__(self):
|
||||
return self.title or self.feed_url
|
||||
|
||||
|
||||
class Subscription(models.Model):
|
||||
"""Per-user join between a User and a shared Feed."""
|
||||
|
||||
user = models.ForeignKey(
|
||||
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="subscriptions"
|
||||
)
|
||||
feed = models.ForeignKey(Feed, on_delete=models.CASCADE, related_name="subscriptions")
|
||||
folder = models.ForeignKey(
|
||||
Folder, null=True, blank=True, on_delete=models.SET_NULL, related_name="subscriptions"
|
||||
)
|
||||
custom_title = models.CharField(max_length=500, blank=True)
|
||||
order = models.PositiveIntegerField(default=0)
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
|
||||
class Meta:
|
||||
unique_together = [("user", "feed")]
|
||||
ordering = ["order", "id"]
|
||||
|
||||
def __str__(self):
|
||||
return self.custom_title or self.feed.title or self.feed.feed_url
|
||||
|
||||
@property
|
||||
def display_title(self):
|
||||
return self.custom_title or self.feed.title or self.feed.feed_url
|
||||
Whitespace-only changes.
@@ -0,0 +1,225 @@
|
||||
"""HTTP fetching + feedparser glue for polling a single Feed.
|
||||
|
||||
Kept separate from tasks.py so the actual fetch/parse/dedup logic can be
|
||||
unit-tested with plain function calls (no Celery/broker involved) and
|
||||
exercised from a shell for manual verification.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
from calendar import timegm
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from datetime import timezone as dt_timezone
|
||||
|
||||
import feedparser
|
||||
import nh3
|
||||
import requests
|
||||
from django.conf import settings
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.articles.models import Article
|
||||
from apps.feeds.models import Feed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FETCH_TIMEOUT_SECONDS = 15
|
||||
MAX_BACKOFF_MINUTES = 24 * 60
|
||||
USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)"
|
||||
|
||||
ALLOWED_TAGS = {
|
||||
"a",
|
||||
"p",
|
||||
"br",
|
||||
"hr",
|
||||
"strong",
|
||||
"em",
|
||||
"b",
|
||||
"i",
|
||||
"u",
|
||||
"s",
|
||||
"blockquote",
|
||||
"code",
|
||||
"pre",
|
||||
"ul",
|
||||
"ol",
|
||||
"li",
|
||||
"h1",
|
||||
"h2",
|
||||
"h3",
|
||||
"h4",
|
||||
"h5",
|
||||
"h6",
|
||||
"img",
|
||||
"figure",
|
||||
"figcaption",
|
||||
"table",
|
||||
"thead",
|
||||
"tbody",
|
||||
"tr",
|
||||
"th",
|
||||
"td",
|
||||
"span",
|
||||
"div",
|
||||
}
|
||||
ALLOWED_ATTRIBUTES = {
|
||||
"a": {"href", "title", "rel"},
|
||||
"img": {"src", "alt", "title", "width", "height"},
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class FetchResult:
|
||||
status: str # Feed.STATUS_OK / STATUS_NOT_MODIFIED / STATUS_ERROR
|
||||
new_article_ids: list[int]
|
||||
updated_article_ids: list[int]
|
||||
error_message: str = ""
|
||||
|
||||
|
||||
def sanitize_html(raw_html: str) -> str:
|
||||
if not raw_html:
|
||||
return ""
|
||||
return nh3.clean(raw_html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None)
|
||||
|
||||
|
||||
def _content_hash(title: str, content_html: str) -> str:
|
||||
return hashlib.sha256(f"{title}\n{content_html}".encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _entry_guid(entry) -> str:
|
||||
if getattr(entry, "id", None):
|
||||
return entry.id
|
||||
if getattr(entry, "link", None):
|
||||
return entry.link
|
||||
title = getattr(entry, "title", "")
|
||||
published = getattr(entry, "published", "")
|
||||
return hashlib.sha256(f"{title}|{published}".encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _entry_published_at(entry):
|
||||
parsed = getattr(entry, "published_parsed", None) or getattr(entry, "updated_parsed", None)
|
||||
if not parsed:
|
||||
return None
|
||||
return datetime.fromtimestamp(timegm(parsed), tz=dt_timezone.utc)
|
||||
|
||||
|
||||
def _entry_content_html(entry) -> str:
|
||||
if getattr(entry, "content", None):
|
||||
return entry.content[0].value
|
||||
return getattr(entry, "summary", "") or ""
|
||||
|
||||
|
||||
def effective_interval_minutes(feed: Feed) -> int:
|
||||
"""The actual polling interval to use, applying exponential backoff on
|
||||
top of the admin-configured ``fetch_interval_minutes`` for feeds that
|
||||
are currently erroring. ``fetch_interval_minutes`` itself is never
|
||||
mutated by the fetcher — it's the admin's chosen baseline.
|
||||
"""
|
||||
floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15)
|
||||
base = max(feed.fetch_interval_minutes, floor)
|
||||
if feed.fetch_error_count <= 0:
|
||||
return base
|
||||
backoff = base * (2**feed.fetch_error_count)
|
||||
return min(backoff, MAX_BACKOFF_MINUTES)
|
||||
|
||||
|
||||
def fetch_and_ingest(feed: Feed) -> FetchResult:
|
||||
"""Fetch one Feed over HTTP, parse it, and dedup/insert its entries.
|
||||
|
||||
Pure function over a Feed instance — callers are responsible for
|
||||
saving it afterwards. Safe to call directly from a shell or a test,
|
||||
with no Celery broker required.
|
||||
"""
|
||||
headers = {"User-Agent": USER_AGENT}
|
||||
if feed.etag:
|
||||
headers["If-None-Match"] = feed.etag
|
||||
if feed.last_modified:
|
||||
headers["If-Modified-Since"] = feed.last_modified
|
||||
|
||||
try:
|
||||
response = requests.get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS)
|
||||
except requests.RequestException as exc:
|
||||
feed.last_fetched_at = timezone.now()
|
||||
return _record_error(feed, f"Request failed: {exc}")
|
||||
|
||||
feed.last_fetched_at = timezone.now()
|
||||
|
||||
if response.status_code == 304:
|
||||
feed.last_fetch_status = Feed.STATUS_NOT_MODIFIED
|
||||
feed.fetch_error_count = 0
|
||||
return FetchResult(
|
||||
status=Feed.STATUS_NOT_MODIFIED, new_article_ids=[], updated_article_ids=[]
|
||||
)
|
||||
|
||||
if response.status_code >= 400:
|
||||
return _record_error(feed, f"HTTP {response.status_code}")
|
||||
|
||||
parsed = feedparser.parse(response.content)
|
||||
|
||||
if parsed.bozo and not parsed.entries:
|
||||
return _record_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}")
|
||||
|
||||
if parsed.bozo:
|
||||
logger.warning(
|
||||
"Feed %s parsed with warnings: %s", feed.feed_url, parsed.get("bozo_exception")
|
||||
)
|
||||
|
||||
feed.etag = response.headers.get("ETag", "")
|
||||
feed.last_modified = response.headers.get("Last-Modified", "")
|
||||
feed.last_fetch_status = Feed.STATUS_OK
|
||||
feed.fetch_error_count = 0
|
||||
|
||||
channel_title = parsed.feed.get("title")
|
||||
if channel_title and not feed.title:
|
||||
feed.title = channel_title[:500]
|
||||
channel_link = parsed.feed.get("link")
|
||||
if channel_link and not feed.site_url:
|
||||
feed.site_url = channel_link[:1000]
|
||||
|
||||
new_ids: list[int] = []
|
||||
updated_ids: list[int] = []
|
||||
|
||||
for entry in parsed.entries:
|
||||
guid = _entry_guid(entry)
|
||||
title = (getattr(entry, "title", "") or "")[:1000]
|
||||
content_html = sanitize_html(_entry_content_html(entry))
|
||||
content_hash = _content_hash(title, content_html)
|
||||
|
||||
article, created = Article.objects.get_or_create(
|
||||
feed=feed,
|
||||
guid=guid,
|
||||
defaults={
|
||||
"link": (getattr(entry, "link", "") or "")[:2000],
|
||||
"title": title,
|
||||
"author": (getattr(entry, "author", "") or "")[:500],
|
||||
"content_html": content_html,
|
||||
"summary": (getattr(entry, "summary", "") or "")[:5000],
|
||||
"content_hash": content_hash,
|
||||
"published_at": _entry_published_at(entry),
|
||||
},
|
||||
)
|
||||
|
||||
if created:
|
||||
new_ids.append(article.pk)
|
||||
elif article.content_hash != content_hash:
|
||||
article.title = title
|
||||
article.content_html = content_html
|
||||
article.content_hash = content_hash
|
||||
article.save(update_fields=["title", "content_html", "content_hash"])
|
||||
updated_ids.append(article.pk)
|
||||
|
||||
return FetchResult(
|
||||
status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids
|
||||
)
|
||||
|
||||
|
||||
def _record_error(feed: Feed, message: str) -> FetchResult:
|
||||
feed.last_fetch_status = Feed.STATUS_ERROR
|
||||
feed.last_error_message = message[:2000]
|
||||
feed.fetch_error_count += 1
|
||||
logger.warning("Feed %s fetch error: %s", feed.feed_url, message)
|
||||
return FetchResult(
|
||||
status=Feed.STATUS_ERROR, new_article_ids=[], updated_article_ids=[], error_message=message
|
||||
)
|
||||
@@ -0,0 +1,69 @@
|
||||
import logging
|
||||
from datetime import timedelta
|
||||
|
||||
from celery import shared_task
|
||||
from django.utils import timezone
|
||||
|
||||
from .models import Feed
|
||||
from .services.fetcher import effective_interval_minutes, fetch_and_ingest
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@shared_task
|
||||
def poll_due_feeds():
|
||||
"""Find every active Feed whose polling interval has elapsed and
|
||||
dispatch a fetch_feed task for each — fan-out happens here, not via one
|
||||
Celery Beat entry per feed.
|
||||
"""
|
||||
due_feed_ids = [
|
||||
feed.id
|
||||
for feed in Feed.objects.filter(is_active=True).only(
|
||||
"id", "last_fetched_at", "fetch_interval_minutes", "fetch_error_count"
|
||||
)
|
||||
if _is_due(feed)
|
||||
]
|
||||
for feed_id in due_feed_ids:
|
||||
fetch_feed.delay(feed_id)
|
||||
return len(due_feed_ids)
|
||||
|
||||
|
||||
def _is_due(feed: Feed) -> bool:
|
||||
if feed.last_fetched_at is None:
|
||||
return True
|
||||
interval = effective_interval_minutes(feed)
|
||||
return feed.last_fetched_at <= timezone.now() - timedelta(minutes=interval)
|
||||
|
||||
|
||||
@shared_task
|
||||
def fetch_feed(feed_id: int):
|
||||
try:
|
||||
feed = Feed.objects.get(pk=feed_id, is_active=True)
|
||||
except Feed.DoesNotExist:
|
||||
return None
|
||||
|
||||
result = fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
if result.new_article_ids:
|
||||
logger.info(
|
||||
"Feed %s: %d new article(s), %d updated",
|
||||
feed.feed_url,
|
||||
len(result.new_article_ids),
|
||||
len(result.updated_article_ids),
|
||||
)
|
||||
# Rule application (tag/star/hide/mark_read on new articles for each
|
||||
# subscriber) is wired in once apps.rules exists — see milestone 6.
|
||||
|
||||
return {
|
||||
"status": result.status,
|
||||
"new": len(result.new_article_ids),
|
||||
"updated": len(result.updated_article_ids),
|
||||
}
|
||||
|
||||
|
||||
@shared_task
|
||||
def reset_feed_backoff(feed_id: int):
|
||||
"""Admin-triggered: clear a feed's error count so it's retried at its
|
||||
normal configured interval instead of the backed-off one."""
|
||||
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0)
|
||||
@@ -0,0 +1,196 @@
|
||||
import pytest
|
||||
import requests
|
||||
import responses
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.articles.models import Article, UserArticleState
|
||||
from apps.feeds.models import Feed
|
||||
from apps.feeds.services.fetcher import effective_interval_minutes, fetch_and_ingest
|
||||
|
||||
FEED_URL = "https://example.com/feed.xml"
|
||||
|
||||
GOOD_FEED_XML = """<?xml version="1.0"?>
|
||||
<rss version="2.0"><channel>
|
||||
<title>Example Blog</title>
|
||||
<link>https://example.com</link>
|
||||
<item>
|
||||
<title>First post</title>
|
||||
<link>https://example.com/1</link>
|
||||
<guid>https://example.com/1</guid>
|
||||
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
||||
<description><p>Hello <script>alert(1)</script>world</p></description>
|
||||
</item>
|
||||
</channel></rss>
|
||||
"""
|
||||
|
||||
NO_GUID_FEED_XML = """<?xml version="1.0"?>
|
||||
<rss version="2.0"><channel>
|
||||
<title>No Guid Blog</title>
|
||||
<item>
|
||||
<title>Only a link</title>
|
||||
<link>https://example.com/only-link</link>
|
||||
<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>
|
||||
<description>Body</description>
|
||||
</item>
|
||||
</channel></rss>
|
||||
"""
|
||||
|
||||
MALFORMED_BUT_SALVAGEABLE_XML = """<?xml version="1.0"?>
|
||||
<rss version="2.0"><channel>
|
||||
<title>Slightly Broken</title>
|
||||
<item>
|
||||
<title>Unescaped & ampersand but still has entries</title>
|
||||
<guid>guid-1</guid>
|
||||
</item>
|
||||
</channel></rss>
|
||||
"""
|
||||
|
||||
TOTAL_GARBAGE = "this is not xml at all {{{"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def feed(db):
|
||||
return Feed.objects.create(feed_url=FEED_URL)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_fetch_ingests_new_articles_and_sanitizes_html(feed):
|
||||
responses.add(
|
||||
responses.GET,
|
||||
FEED_URL,
|
||||
body=GOOD_FEED_XML,
|
||||
status=200,
|
||||
headers={"ETag": '"abc123"', "Last-Modified": "Mon, 01 Jan 2024 00:00:00 GMT"},
|
||||
)
|
||||
|
||||
result = fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
assert result.status == Feed.STATUS_OK
|
||||
assert len(result.new_article_ids) == 1
|
||||
assert feed.title == "Example Blog"
|
||||
assert feed.etag == '"abc123"'
|
||||
assert feed.last_modified == "Mon, 01 Jan 2024 00:00:00 GMT"
|
||||
|
||||
article = Article.objects.get(feed=feed, guid="https://example.com/1")
|
||||
assert "<script>" not in article.content_html
|
||||
assert "world" in article.content_html
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_second_poll_with_304_does_not_duplicate_or_error(feed):
|
||||
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
|
||||
fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
assert Article.objects.filter(feed=feed).count() == 1
|
||||
|
||||
responses.add(responses.GET, FEED_URL, status=304)
|
||||
result = fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
assert result.status == Feed.STATUS_NOT_MODIFIED
|
||||
assert Article.objects.filter(feed=feed).count() == 1
|
||||
assert feed.fetch_error_count == 0
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_entry_without_guid_falls_back_to_link(feed):
|
||||
responses.add(responses.GET, FEED_URL, body=NO_GUID_FEED_XML, status=200)
|
||||
|
||||
result = fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
assert len(result.new_article_ids) == 1
|
||||
article = Article.objects.get(pk=result.new_article_ids[0])
|
||||
assert article.guid == "https://example.com/only-link"
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_bozo_feed_with_entries_still_ingests(feed):
|
||||
responses.add(responses.GET, FEED_URL, body=MALFORMED_BUT_SALVAGEABLE_XML, status=200)
|
||||
|
||||
result = fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
assert result.status == Feed.STATUS_OK
|
||||
assert len(result.new_article_ids) == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_unparseable_feed_records_error_and_backs_off(feed):
|
||||
responses.add(responses.GET, FEED_URL, body=TOTAL_GARBAGE, status=200)
|
||||
|
||||
result = fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
assert result.status == Feed.STATUS_ERROR
|
||||
assert feed.fetch_error_count == 1
|
||||
assert effective_interval_minutes(feed) > feed.fetch_interval_minutes
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_http_500_records_error(feed):
|
||||
responses.add(responses.GET, FEED_URL, status=500)
|
||||
|
||||
result = fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
assert result.status == Feed.STATUS_ERROR
|
||||
assert feed.last_fetch_status == Feed.STATUS_ERROR
|
||||
assert feed.fetch_error_count == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_connection_error_still_updates_last_fetched_at(feed):
|
||||
responses.add(
|
||||
responses.GET,
|
||||
FEED_URL,
|
||||
body=requests.exceptions.ConnectionError("boom"),
|
||||
)
|
||||
|
||||
before = timezone.now()
|
||||
result = fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
assert result.status == Feed.STATUS_ERROR
|
||||
assert feed.last_fetched_at is not None
|
||||
assert feed.last_fetched_at >= before
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_refetching_updated_content_does_not_reset_user_state():
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
User = get_user_model()
|
||||
user = User.objects.create_user("alice", password="pw-alice-123")
|
||||
feed = Feed.objects.create(feed_url=FEED_URL)
|
||||
|
||||
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
|
||||
fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
article = Article.objects.get(feed=feed)
|
||||
state = UserArticleState.objects.create(
|
||||
user=user, article=article, is_read=True, is_starred=True
|
||||
)
|
||||
|
||||
updated_xml = GOOD_FEED_XML.replace("First post", "First post (edited)")
|
||||
responses.add(responses.GET, FEED_URL, body=updated_xml, status=200)
|
||||
result = fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
assert result.updated_article_ids == [article.pk]
|
||||
article.refresh_from_db()
|
||||
assert article.title == "First post (edited)"
|
||||
|
||||
state.refresh_from_db()
|
||||
assert state.is_read is True
|
||||
assert state.is_starred is True
|
||||
@@ -0,0 +1,77 @@
|
||||
from datetime import timedelta
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import responses
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.articles.models import Article
|
||||
from apps.feeds.models import Feed
|
||||
from apps.feeds.tasks import fetch_feed, poll_due_feeds
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_poll_due_feeds_dispatches_only_due_feeds():
|
||||
never_fetched = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
||||
recently_fetched = Feed.objects.create(
|
||||
feed_url="https://b.example.com/feed.xml",
|
||||
last_fetched_at=timezone.now(),
|
||||
fetch_interval_minutes=60,
|
||||
)
|
||||
overdue = Feed.objects.create(
|
||||
feed_url="https://c.example.com/feed.xml",
|
||||
last_fetched_at=timezone.now() - timedelta(hours=2),
|
||||
fetch_interval_minutes=60,
|
||||
)
|
||||
inactive_but_overdue = Feed.objects.create(
|
||||
feed_url="https://d.example.com/feed.xml",
|
||||
last_fetched_at=timezone.now() - timedelta(hours=2),
|
||||
fetch_interval_minutes=60,
|
||||
is_active=False,
|
||||
)
|
||||
|
||||
with patch("apps.feeds.tasks.fetch_feed.delay") as mock_delay:
|
||||
dispatched_count = poll_due_feeds()
|
||||
|
||||
dispatched_ids = {call.args[0] for call in mock_delay.call_args_list}
|
||||
assert dispatched_ids == {never_fetched.id, overdue.id}
|
||||
assert dispatched_count == 2
|
||||
assert recently_fetched.id not in dispatched_ids
|
||||
assert inactive_but_overdue.id not in dispatched_ids
|
||||
|
||||
|
||||
GOOD_FEED_XML = """<?xml version="1.0"?>
|
||||
<rss version="2.0"><channel>
|
||||
<title>Example Blog</title>
|
||||
<item>
|
||||
<title>First post</title>
|
||||
<guid>guid-1</guid>
|
||||
</item>
|
||||
</channel></rss>
|
||||
"""
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_fetch_feed_task_ingests_and_saves_feed_state():
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
responses.add(responses.GET, feed.feed_url, body=GOOD_FEED_XML, status=200)
|
||||
|
||||
result = fetch_feed(feed.id)
|
||||
|
||||
assert result["status"] == Feed.STATUS_OK
|
||||
assert result["new"] == 1
|
||||
feed.refresh_from_db()
|
||||
assert feed.last_fetch_status == Feed.STATUS_OK
|
||||
assert Article.objects.filter(feed=feed, guid="guid-1").exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_fetch_feed_task_skips_inactive_feed():
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", is_active=False)
|
||||
assert fetch_feed(feed.id) is None
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_fetch_feed_task_skips_missing_feed():
|
||||
assert fetch_feed(999999) is None
|
||||
+100
-1
@@ -1 +1,100 @@
|
||||
# Create your tests here.
|
||||
import pytest
|
||||
import responses
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.urls import reverse
|
||||
|
||||
from .models import Feed, Folder, Subscription
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def two_users(db):
|
||||
alice = User.objects.create_user("alice", password="pw-alice-123")
|
||||
bob = User.objects.create_user("bob", password="pw-bob-123")
|
||||
return alice, bob
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_subscribing_to_same_feed_url_reuses_one_feed_row(two_users):
|
||||
alice, bob = two_users
|
||||
|
||||
feed, created = Feed.objects.get_or_create(feed_url="https://example.com/feed.xml")
|
||||
assert created
|
||||
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
|
||||
feed_again, created_again = Feed.objects.get_or_create(feed_url="https://example.com/feed.xml")
|
||||
assert not created_again
|
||||
assert feed_again.pk == feed.pk
|
||||
|
||||
Subscription.objects.create(user=bob, feed=feed_again)
|
||||
|
||||
assert Feed.objects.count() == 1
|
||||
assert Subscription.objects.filter(feed=feed).count() == 2
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_a_user_cannot_subscribe_to_the_same_feed_twice(two_users):
|
||||
alice, _ = two_users
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_folder_names_are_scoped_per_user(two_users):
|
||||
alice, bob = two_users
|
||||
Folder.objects.create(user=alice, name="Tech")
|
||||
# Same folder name for a different user must be allowed.
|
||||
Folder.objects.create(user=bob, name="Tech")
|
||||
|
||||
assert Folder.objects.filter(name="Tech").count() == 2
|
||||
|
||||
with pytest.raises(Exception):
|
||||
Folder.objects.create(user=alice, name="Tech")
|
||||
|
||||
|
||||
GOOD_FEED_XML = """<?xml version="1.0"?>
|
||||
<rss version="2.0"><channel>
|
||||
<title>Example Blog</title>
|
||||
<item><title>First post</title><guid>guid-1</guid></item>
|
||||
</channel></rss>
|
||||
"""
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_add_feed_view_subscribes_and_fetches_immediately(client, two_users):
|
||||
alice, _ = two_users
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
responses.add(responses.GET, "https://example.com/feed.xml", body=GOOD_FEED_XML, status=200)
|
||||
|
||||
response = client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/feed.xml"})
|
||||
|
||||
assert response.status_code == 302
|
||||
assert Subscription.objects.filter(
|
||||
user=alice, feed__feed_url="https://example.com/feed.xml"
|
||||
).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_add_feed_view_requires_login(client):
|
||||
response = client.get(reverse("feeds:add_feed"))
|
||||
assert response.status_code == 302
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_add_feed_view_rejects_duplicate_subscription(client, two_users):
|
||||
alice, _ = two_users
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
|
||||
response = client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/feed.xml"})
|
||||
|
||||
assert response.status_code == 302
|
||||
assert Subscription.objects.filter(user=alice, feed=feed).count() == 1
|
||||
@@ -0,0 +1,9 @@
|
||||
from django.urls import path
|
||||
|
||||
from . import views
|
||||
|
||||
app_name = "feeds"
|
||||
|
||||
urlpatterns = [
|
||||
path("feeds/add/", views.add_feed, name="add_feed"),
|
||||
]
|
||||
+45
-1
@@ -1 +1,45 @@
|
||||
# Create your views here.
|
||||
from django.contrib import messages
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.shortcuts import redirect, render
|
||||
|
||||
from .forms import AddFeedForm
|
||||
from .models import Feed, Subscription
|
||||
from .services.fetcher import fetch_and_ingest
|
||||
|
||||
|
||||
@login_required
|
||||
def add_feed(request):
|
||||
form = AddFeedForm(request.POST or None, user=request.user)
|
||||
if request.method == "POST" and form.is_valid():
|
||||
feed_url = form.cleaned_data["feed_url"]
|
||||
feed, _created = Feed.objects.get_or_create(feed_url=feed_url)
|
||||
|
||||
subscription, created = Subscription.objects.get_or_create(
|
||||
user=request.user,
|
||||
feed=feed,
|
||||
defaults={"folder": form.cleaned_data["folder"]},
|
||||
)
|
||||
if not created:
|
||||
messages.info(request, "You're already subscribed to that feed.")
|
||||
return redirect("reader:stream")
|
||||
|
||||
# Fetch synchronously so the user sees results immediately; the
|
||||
# periodic Celery poll takes over from here for subsequent fetches.
|
||||
result = fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
if result.status == Feed.STATUS_ERROR:
|
||||
messages.warning(
|
||||
request,
|
||||
f"Subscribed to {feed_url}, but the initial fetch failed: {result.error_message}",
|
||||
)
|
||||
else:
|
||||
article_count = len(result.new_article_ids)
|
||||
messages.success(
|
||||
request,
|
||||
f"Subscribed to '{subscription.display_title}' — "
|
||||
f"{article_count} article(s) fetched.",
|
||||
)
|
||||
return redirect("reader:stream")
|
||||
|
||||
return render(request, "feeds/add_feed.html", {"form": form})
|
||||
@@ -4,5 +4,6 @@ from django.urls import include, path
|
||||
urlpatterns = [
|
||||
path("admin/", admin.site.urls),
|
||||
path("", include("apps.accounts.urls")),
|
||||
path("", include("apps.feeds.urls")),
|
||||
path("", include("apps.reader.urls")),
|
||||
]
|
||||
@@ -0,0 +1,11 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Add feed — Evection{% endblock %}
|
||||
{% block content %}
|
||||
<h1>Add a feed</h1>
|
||||
<form method="post">
|
||||
{% csrf_token %}
|
||||
{{ form.as_p }}
|
||||
<button type="submit">Subscribe</button>
|
||||
</form>
|
||||
<p><a href="{% url 'reader:stream' %}">Back to stream</a></p>
|
||||
{% endblock %}
|
||||
@@ -3,5 +3,5 @@
|
||||
{% block content %}
|
||||
<h1>Your stream</h1>
|
||||
<p>No feeds yet. The reading UI lands in a later milestone.</p>
|
||||
<p><a href="{% url 'accounts:logout' %}">Log out</a></p>
|
||||
<p><a href="{% url 'feeds:add_feed' %}">Add a feed</a> · <a href="{% url 'accounts:logout' %}">Log out</a></p>
|
||||
{% endblock %}
|
||||
Reference in new issue
Block a user