diff --git a/apps/articles/admin.py b/apps/articles/admin.py index 846f6b4..bc66ae1 100644 --- a/apps/articles/admin.py +++ b/apps/articles/admin.py @@ -1 +1,27 @@ -# Register your models here. +from django.contrib import admin + +from .models import Article, ArticleTag, Tag, UserArticleState + + +@admin.register(Article) +class ArticleAdmin(admin.ModelAdmin): + list_display = ["title", "feed", "published_at", "fetched_at"] + list_filter = ["feed"] + search_fields = ["title", "guid", "link"] + + +@admin.register(UserArticleState) +class UserArticleStateAdmin(admin.ModelAdmin): + list_display = ["user", "article", "is_read", "is_starred", "is_saved_for_later", "is_hidden"] + list_filter = ["is_read", "is_starred", "is_saved_for_later", "is_hidden"] + + +@admin.register(Tag) +class TagAdmin(admin.ModelAdmin): + list_display = ["name", "user", "color"] + list_filter = ["user"] + + +@admin.register(ArticleTag) +class ArticleTagAdmin(admin.ModelAdmin): + list_display = ["user", "article", "tag"] diff --git a/apps/articles/migrations/0001_initial.py b/apps/articles/migrations/0001_initial.py new file mode 100644 index 0000000..3e67758 --- /dev/null +++ b/apps/articles/migrations/0001_initial.py @@ -0,0 +1,171 @@ +# Generated by Django 5.2.18 on 2026-10-30 21:57 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("feeds", "0001_initial"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Article", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("guid", models.CharField(max_length=1000)), + ("link", models.URLField(blank=True, max_length=2000)), + ("title", models.CharField(blank=True, max_length=1000)), + ("author", models.CharField(blank=True, max_length=500)), + ("content_html", models.TextField(blank=True)), + ("summary", models.TextField(blank=True)), + ("content_hash", models.CharField(blank=True, max_length=64)), + ("published_at", models.DateTimeField(blank=True, null=True)), + ("fetched_at", models.DateTimeField(auto_now_add=True)), + ( + "feed", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="articles", + to="feeds.feed", + ), + ), + ], + options={ + "ordering": ["-published_at", "-fetched_at"], + }, + ), + migrations.CreateModel( + name="Tag", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("name", models.CharField(max_length=100)), + ("color", models.CharField(blank=True, max_length=7)), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="tags", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "ordering": ["name"], + }, + ), + migrations.CreateModel( + name="ArticleTag", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "article", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="article_tags", + to="articles.article", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL + ), + ), + ( + "tag", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="article_tags", + to="articles.tag", + ), + ), + ], + ), + migrations.CreateModel( + name="UserArticleState", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("is_read", models.BooleanField(default=False)), + ("is_starred", models.BooleanField(default=False)), + ("is_saved_for_later", models.BooleanField(default=False)), + ("is_hidden", models.BooleanField(default=False)), + ("read_at", models.DateTimeField(blank=True, null=True)), + ("starred_at", models.DateTimeField(blank=True, null=True)), + ( + "article", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="user_states", + to="articles.article", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="article_states", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + migrations.AddIndex( + model_name="article", + index=models.Index( + fields=["feed", "published_at"], name="articles_ar_feed_id_67b4fb_idx" + ), + ), + migrations.AlterUniqueTogether( + name="article", + unique_together={("feed", "guid")}, + ), + migrations.AlterUniqueTogether( + name="tag", + unique_together={("user", "name")}, + ), + migrations.AlterUniqueTogether( + name="articletag", + unique_together={("user", "article", "tag")}, + ), + migrations.AddIndex( + model_name="userarticlestate", + index=models.Index(fields=["user", "is_read"], name="articles_us_user_id_ded063_idx"), + ), + migrations.AddIndex( + model_name="userarticlestate", + index=models.Index( + fields=["user", "is_starred"], name="articles_us_user_id_cb0761_idx" + ), + ), + migrations.AlterUniqueTogether( + name="userarticlestate", + unique_together={("user", "article")}, + ), + ] diff --git a/apps/articles/models.py b/apps/articles/models.py index 6b20219..2146979 100644 --- a/apps/articles/models.py +++ b/apps/articles/models.py @@ -1 +1,91 @@ -# Create your models here. +from django.conf import settings +from django.db import models + +from apps.feeds.models import Feed + + +class Article(models.Model): + """A single fetched entry, shared across every subscriber of its feed. + + Per-user state (read/starred/saved/hidden) lives in UserArticleState, + never on this model — editing an article's content on re-fetch must + never touch anyone's read/star state. + """ + + feed = models.ForeignKey(Feed, on_delete=models.CASCADE, related_name="articles") + guid = models.CharField(max_length=1000) + link = models.URLField(max_length=2000, blank=True) + title = models.CharField(max_length=1000, blank=True) + author = models.CharField(max_length=500, blank=True) + content_html = models.TextField(blank=True) + summary = models.TextField(blank=True) + content_hash = models.CharField(max_length=64, blank=True) + published_at = models.DateTimeField(null=True, blank=True) + fetched_at = models.DateTimeField(auto_now_add=True) + + class Meta: + unique_together = [("feed", "guid")] + indexes = [models.Index(fields=["feed", "published_at"])] + ordering = ["-published_at", "-fetched_at"] + + def __str__(self): + return self.title or self.guid + + +class UserArticleState(models.Model): + """Per-user read/star/save/hide state for an Article. + + Rows are created lazily, only on first interaction — a row with no + explicit True flags is equivalent to "no row exists" (unread, not + starred). Never bulk-create one of these per subscriber on ingest. + """ + + user = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="article_states" + ) + article = models.ForeignKey(Article, on_delete=models.CASCADE, related_name="user_states") + + is_read = models.BooleanField(default=False) + is_starred = models.BooleanField(default=False) + is_saved_for_later = models.BooleanField(default=False) + is_hidden = models.BooleanField(default=False) + + read_at = models.DateTimeField(null=True, blank=True) + starred_at = models.DateTimeField(null=True, blank=True) + + class Meta: + unique_together = [("user", "article")] + indexes = [ + models.Index(fields=["user", "is_read"]), + models.Index(fields=["user", "is_starred"]), + ] + + def __str__(self): + return f"{self.user}: {self.article}" + + +class Tag(models.Model): + user = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="tags" + ) + name = models.CharField(max_length=100) + color = models.CharField(max_length=7, blank=True) + + class Meta: + unique_together = [("user", "name")] + ordering = ["name"] + + def __str__(self): + return self.name + + +class ArticleTag(models.Model): + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + article = models.ForeignKey(Article, on_delete=models.CASCADE, related_name="article_tags") + tag = models.ForeignKey(Tag, on_delete=models.CASCADE, related_name="article_tags") + + class Meta: + unique_together = [("user", "article", "tag")] + + def __str__(self): + return f"{self.article} -> {self.tag}" diff --git a/apps/articles/tests.py b/apps/articles/tests.py index a39b155..e322f57 100644 --- a/apps/articles/tests.py +++ b/apps/articles/tests.py @@ -1 +1,81 @@ -# Create your tests here. +import pytest +from django.contrib.auth import get_user_model + +from apps.feeds.models import Feed + +from .models import Article, Tag, UserArticleState + +User = get_user_model() + + +@pytest.fixture +def feed(db): + return Feed.objects.create(feed_url="https://example.com/feed.xml", title="Example") + + +@pytest.fixture +def two_users(db): + alice = User.objects.create_user("alice", password="pw-alice-123") + bob = User.objects.create_user("bob", password="pw-bob-123") + return alice, bob + + +@pytest.mark.django_db +def test_articles_dedup_by_feed_and_guid(feed): + article, created = Article.objects.get_or_create( + feed=feed, guid="guid-1", defaults={"title": "First version"} + ) + assert created + + # Re-ingesting the same guid (as the fetch pipeline would on a repeat + # poll) must not create a second row. + article_again, created_again = Article.objects.get_or_create( + feed=feed, guid="guid-1", defaults={"title": "Should not be used"} + ) + assert not created_again + assert article_again.pk == article.pk + assert Article.objects.filter(feed=feed, guid="guid-1").count() == 1 + + +@pytest.mark.django_db +def test_user_article_state_is_lazy_and_per_user(feed, two_users): + alice, bob = two_users + article = Article.objects.create(feed=feed, guid="guid-1", title="Hello") + + # No state rows exist until someone interacts with the article. + assert UserArticleState.objects.filter(article=article).count() == 0 + + UserArticleState.objects.create(user=alice, article=article, is_read=True) + + assert UserArticleState.objects.filter(article=article).count() == 1 + alice_state = UserArticleState.objects.get(user=alice, article=article) + assert alice_state.is_read is True + # Bob has no row at all — he should be treated as unread, not as sharing + # Alice's state. + assert not UserArticleState.objects.filter(user=bob, article=article).exists() + + +@pytest.mark.django_db +def test_tags_are_scoped_per_user(two_users): + alice, bob = two_users + Tag.objects.create(user=alice, name="golang") + Tag.objects.create(user=bob, name="golang") + + assert Tag.objects.filter(name="golang").count() == 2 + + with pytest.raises(Exception): + Tag.objects.create(user=alice, name="golang") + + +@pytest.mark.django_db +def test_editing_article_content_does_not_touch_user_state(feed, two_users): + alice, _ = two_users + article = Article.objects.create(feed=feed, guid="guid-1", title="Original title") + state = UserArticleState.objects.create(user=alice, article=article, is_read=True) + + # Simulate the fetch pipeline updating content on a re-fetch. + article.title = "Updated title" + article.save() + + state.refresh_from_db() + assert state.is_read is True diff --git a/apps/feeds/admin.py b/apps/feeds/admin.py index 846f6b4..b8cbf3e 100644 --- a/apps/feeds/admin.py +++ b/apps/feeds/admin.py @@ -1 +1,29 @@ -# Register your models here. +from django.contrib import admin + +from .models import Feed, Folder, Subscription + + +@admin.register(Folder) +class FolderAdmin(admin.ModelAdmin): + list_display = ["name", "user", "order"] + list_filter = ["user"] + + +@admin.register(Feed) +class FeedAdmin(admin.ModelAdmin): + list_display = [ + "title", + "feed_url", + "last_fetch_status", + "last_fetched_at", + "fetch_error_count", + "is_active", + ] + list_filter = ["last_fetch_status", "is_active"] + search_fields = ["title", "feed_url"] + + +@admin.register(Subscription) +class SubscriptionAdmin(admin.ModelAdmin): + list_display = ["user", "feed", "folder", "custom_title"] + list_filter = ["user", "folder"] diff --git a/apps/feeds/forms.py b/apps/feeds/forms.py new file mode 100644 index 0000000..8ac12db --- /dev/null +++ b/apps/feeds/forms.py @@ -0,0 +1,13 @@ +from django import forms + +from .models import Folder + + +class AddFeedForm(forms.Form): + feed_url = forms.URLField(label="Feed URL", max_length=1000, assume_scheme="https") + folder = forms.ModelChoiceField(queryset=Folder.objects.none(), required=False) + + def __init__(self, *args, user=None, **kwargs): + super().__init__(*args, **kwargs) + if user is not None: + self.fields["folder"].queryset = Folder.objects.filter(user=user) diff --git a/apps/feeds/migrations/0001_initial.py b/apps/feeds/migrations/0001_initial.py new file mode 100644 index 0000000..f5e677c --- /dev/null +++ b/apps/feeds/migrations/0001_initial.py @@ -0,0 +1,122 @@ +# Generated by Django 5.2.18 on 2026-10-30 21:57 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Feed", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("feed_url", models.URLField(max_length=1000, unique=True)), + ("site_url", models.URLField(blank=True, max_length=1000)), + ("title", models.CharField(blank=True, max_length=500)), + ("favicon_url", models.URLField(blank=True, max_length=1000)), + ("etag", models.CharField(blank=True, max_length=255)), + ("last_modified", models.CharField(blank=True, max_length=255)), + ("last_fetched_at", models.DateTimeField(blank=True, null=True)), + ( + "last_fetch_status", + models.CharField( + choices=[ + ("pending", "Pending"), + ("ok", "OK"), + ("not_modified", "Not modified"), + ("error", "Error"), + ], + default="pending", + max_length=20, + ), + ), + ("last_error_message", models.TextField(blank=True)), + ("fetch_interval_minutes", models.PositiveIntegerField(default=60)), + ("fetch_error_count", models.PositiveIntegerField(default=0)), + ("is_active", models.BooleanField(default=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ], + ), + migrations.CreateModel( + name="Folder", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("name", models.CharField(max_length=255)), + ("order", models.PositiveIntegerField(default=0)), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="folders", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "ordering": ["order", "name"], + "unique_together": {("user", "name")}, + }, + ), + migrations.CreateModel( + name="Subscription", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("custom_title", models.CharField(blank=True, max_length=500)), + ("order", models.PositiveIntegerField(default=0)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ( + "feed", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="subscriptions", + to="feeds.feed", + ), + ), + ( + "folder", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="subscriptions", + to="feeds.folder", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="subscriptions", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "ordering": ["order", "id"], + "unique_together": {("user", "feed")}, + }, + ), + ] diff --git a/apps/feeds/models.py b/apps/feeds/models.py index 6b20219..c858d5d 100644 --- a/apps/feeds/models.py +++ b/apps/feeds/models.py @@ -1 +1,87 @@ -# Create your models here. +from django.conf import settings +from django.db import models + + +class Folder(models.Model): + user = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="folders" + ) + name = models.CharField(max_length=255) + order = models.PositiveIntegerField(default=0) + + class Meta: + unique_together = [("user", "name")] + ordering = ["order", "name"] + + def __str__(self): + return self.name + + +class Feed(models.Model): + """A single polled RSS/Atom source, shared across every subscriber. + + One row per feed_url regardless of how many users subscribe to it — the + fetch pipeline polls each feed exactly once and every Subscription below + just points at the same Feed/Article data. Never add per-user fetching. + """ + + STATUS_PENDING = "pending" + STATUS_OK = "ok" + STATUS_NOT_MODIFIED = "not_modified" + STATUS_ERROR = "error" + STATUS_CHOICES = [ + (STATUS_PENDING, "Pending"), + (STATUS_OK, "OK"), + (STATUS_NOT_MODIFIED, "Not modified"), + (STATUS_ERROR, "Error"), + ] + + feed_url = models.URLField(max_length=1000, unique=True) + site_url = models.URLField(max_length=1000, blank=True) + title = models.CharField(max_length=500, blank=True) + favicon_url = models.URLField(max_length=1000, blank=True) + + etag = models.CharField(max_length=255, blank=True) + last_modified = models.CharField(max_length=255, blank=True) + + last_fetched_at = models.DateTimeField(null=True, blank=True) + last_fetch_status = models.CharField( + max_length=20, choices=STATUS_CHOICES, default=STATUS_PENDING + ) + last_error_message = models.TextField(blank=True) + + # Instance-wide, not a per-user preference — see class docstring. + fetch_interval_minutes = models.PositiveIntegerField(default=60) + fetch_error_count = models.PositiveIntegerField(default=0) + is_active = models.BooleanField(default=True) + + created_at = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return self.title or self.feed_url + + +class Subscription(models.Model): + """Per-user join between a User and a shared Feed.""" + + user = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.CASCADE, related_name="subscriptions" + ) + feed = models.ForeignKey(Feed, on_delete=models.CASCADE, related_name="subscriptions") + folder = models.ForeignKey( + Folder, null=True, blank=True, on_delete=models.SET_NULL, related_name="subscriptions" + ) + custom_title = models.CharField(max_length=500, blank=True) + order = models.PositiveIntegerField(default=0) + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + unique_together = [("user", "feed")] + ordering = ["order", "id"] + + def __str__(self): + return self.custom_title or self.feed.title or self.feed.feed_url + + @property + def display_title(self): + return self.custom_title or self.feed.title or self.feed.feed_url diff --git a/apps/feeds/services/__init__.py b/apps/feeds/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/feeds/services/fetcher.py b/apps/feeds/services/fetcher.py new file mode 100644 index 0000000..017b715 --- /dev/null +++ b/apps/feeds/services/fetcher.py @@ -0,0 +1,225 @@ +"""HTTP fetching + feedparser glue for polling a single Feed. + +Kept separate from tasks.py so the actual fetch/parse/dedup logic can be +unit-tested with plain function calls (no Celery/broker involved) and +exercised from a shell for manual verification. +""" + +from __future__ import annotations + +import hashlib +import logging +from calendar import timegm +from dataclasses import dataclass +from datetime import datetime +from datetime import timezone as dt_timezone + +import feedparser +import nh3 +import requests +from django.conf import settings +from django.utils import timezone + +from apps.articles.models import Article +from apps.feeds.models import Feed + +logger = logging.getLogger(__name__) + +FETCH_TIMEOUT_SECONDS = 15 +MAX_BACKOFF_MINUTES = 24 * 60 +USER_AGENT = "Evection/0.1 (+https://github.com/; self-hosted RSS reader)" + +ALLOWED_TAGS = { + "a", + "p", + "br", + "hr", + "strong", + "em", + "b", + "i", + "u", + "s", + "blockquote", + "code", + "pre", + "ul", + "ol", + "li", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "img", + "figure", + "figcaption", + "table", + "thead", + "tbody", + "tr", + "th", + "td", + "span", + "div", +} +ALLOWED_ATTRIBUTES = { + "a": {"href", "title", "rel"}, + "img": {"src", "alt", "title", "width", "height"}, +} + + +@dataclass +class FetchResult: + status: str # Feed.STATUS_OK / STATUS_NOT_MODIFIED / STATUS_ERROR + new_article_ids: list[int] + updated_article_ids: list[int] + error_message: str = "" + + +def sanitize_html(raw_html: str) -> str: + if not raw_html: + return "" + return nh3.clean(raw_html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES, link_rel=None) + + +def _content_hash(title: str, content_html: str) -> str: + return hashlib.sha256(f"{title}\n{content_html}".encode("utf-8")).hexdigest() + + +def _entry_guid(entry) -> str: + if getattr(entry, "id", None): + return entry.id + if getattr(entry, "link", None): + return entry.link + title = getattr(entry, "title", "") + published = getattr(entry, "published", "") + return hashlib.sha256(f"{title}|{published}".encode("utf-8")).hexdigest() + + +def _entry_published_at(entry): + parsed = getattr(entry, "published_parsed", None) or getattr(entry, "updated_parsed", None) + if not parsed: + return None + return datetime.fromtimestamp(timegm(parsed), tz=dt_timezone.utc) + + +def _entry_content_html(entry) -> str: + if getattr(entry, "content", None): + return entry.content[0].value + return getattr(entry, "summary", "") or "" + + +def effective_interval_minutes(feed: Feed) -> int: + """The actual polling interval to use, applying exponential backoff on + top of the admin-configured ``fetch_interval_minutes`` for feeds that + are currently erroring. ``fetch_interval_minutes`` itself is never + mutated by the fetcher — it's the admin's chosen baseline. + """ + floor = getattr(settings, "EVECTION_MIN_FETCH_INTERVAL_MINUTES", 15) + base = max(feed.fetch_interval_minutes, floor) + if feed.fetch_error_count <= 0: + return base + backoff = base * (2**feed.fetch_error_count) + return min(backoff, MAX_BACKOFF_MINUTES) + + +def fetch_and_ingest(feed: Feed) -> FetchResult: + """Fetch one Feed over HTTP, parse it, and dedup/insert its entries. + + Pure function over a Feed instance — callers are responsible for + saving it afterwards. Safe to call directly from a shell or a test, + with no Celery broker required. + """ + headers = {"User-Agent": USER_AGENT} + if feed.etag: + headers["If-None-Match"] = feed.etag + if feed.last_modified: + headers["If-Modified-Since"] = feed.last_modified + + try: + response = requests.get(feed.feed_url, headers=headers, timeout=FETCH_TIMEOUT_SECONDS) + except requests.RequestException as exc: + feed.last_fetched_at = timezone.now() + return _record_error(feed, f"Request failed: {exc}") + + feed.last_fetched_at = timezone.now() + + if response.status_code == 304: + feed.last_fetch_status = Feed.STATUS_NOT_MODIFIED + feed.fetch_error_count = 0 + return FetchResult( + status=Feed.STATUS_NOT_MODIFIED, new_article_ids=[], updated_article_ids=[] + ) + + if response.status_code >= 400: + return _record_error(feed, f"HTTP {response.status_code}") + + parsed = feedparser.parse(response.content) + + if parsed.bozo and not parsed.entries: + return _record_error(feed, f"Unparseable feed: {parsed.get('bozo_exception')}") + + if parsed.bozo: + logger.warning( + "Feed %s parsed with warnings: %s", feed.feed_url, parsed.get("bozo_exception") + ) + + feed.etag = response.headers.get("ETag", "") + feed.last_modified = response.headers.get("Last-Modified", "") + feed.last_fetch_status = Feed.STATUS_OK + feed.fetch_error_count = 0 + + channel_title = parsed.feed.get("title") + if channel_title and not feed.title: + feed.title = channel_title[:500] + channel_link = parsed.feed.get("link") + if channel_link and not feed.site_url: + feed.site_url = channel_link[:1000] + + new_ids: list[int] = [] + updated_ids: list[int] = [] + + for entry in parsed.entries: + guid = _entry_guid(entry) + title = (getattr(entry, "title", "") or "")[:1000] + content_html = sanitize_html(_entry_content_html(entry)) + content_hash = _content_hash(title, content_html) + + article, created = Article.objects.get_or_create( + feed=feed, + guid=guid, + defaults={ + "link": (getattr(entry, "link", "") or "")[:2000], + "title": title, + "author": (getattr(entry, "author", "") or "")[:500], + "content_html": content_html, + "summary": (getattr(entry, "summary", "") or "")[:5000], + "content_hash": content_hash, + "published_at": _entry_published_at(entry), + }, + ) + + if created: + new_ids.append(article.pk) + elif article.content_hash != content_hash: + article.title = title + article.content_html = content_html + article.content_hash = content_hash + article.save(update_fields=["title", "content_html", "content_hash"]) + updated_ids.append(article.pk) + + return FetchResult( + status=Feed.STATUS_OK, new_article_ids=new_ids, updated_article_ids=updated_ids + ) + + +def _record_error(feed: Feed, message: str) -> FetchResult: + feed.last_fetch_status = Feed.STATUS_ERROR + feed.last_error_message = message[:2000] + feed.fetch_error_count += 1 + logger.warning("Feed %s fetch error: %s", feed.feed_url, message) + return FetchResult( + status=Feed.STATUS_ERROR, new_article_ids=[], updated_article_ids=[], error_message=message + ) diff --git a/apps/feeds/tasks.py b/apps/feeds/tasks.py new file mode 100644 index 0000000..2d0ff0c --- /dev/null +++ b/apps/feeds/tasks.py @@ -0,0 +1,69 @@ +import logging +from datetime import timedelta + +from celery import shared_task +from django.utils import timezone + +from .models import Feed +from .services.fetcher import effective_interval_minutes, fetch_and_ingest + +logger = logging.getLogger(__name__) + + +@shared_task +def poll_due_feeds(): + """Find every active Feed whose polling interval has elapsed and + dispatch a fetch_feed task for each — fan-out happens here, not via one + Celery Beat entry per feed. + """ + due_feed_ids = [ + feed.id + for feed in Feed.objects.filter(is_active=True).only( + "id", "last_fetched_at", "fetch_interval_minutes", "fetch_error_count" + ) + if _is_due(feed) + ] + for feed_id in due_feed_ids: + fetch_feed.delay(feed_id) + return len(due_feed_ids) + + +def _is_due(feed: Feed) -> bool: + if feed.last_fetched_at is None: + return True + interval = effective_interval_minutes(feed) + return feed.last_fetched_at <= timezone.now() - timedelta(minutes=interval) + + +@shared_task +def fetch_feed(feed_id: int): + try: + feed = Feed.objects.get(pk=feed_id, is_active=True) + except Feed.DoesNotExist: + return None + + result = fetch_and_ingest(feed) + feed.save() + + if result.new_article_ids: + logger.info( + "Feed %s: %d new article(s), %d updated", + feed.feed_url, + len(result.new_article_ids), + len(result.updated_article_ids), + ) + # Rule application (tag/star/hide/mark_read on new articles for each + # subscriber) is wired in once apps.rules exists — see milestone 6. + + return { + "status": result.status, + "new": len(result.new_article_ids), + "updated": len(result.updated_article_ids), + } + + +@shared_task +def reset_feed_backoff(feed_id: int): + """Admin-triggered: clear a feed's error count so it's retried at its + normal configured interval instead of the backed-off one.""" + Feed.objects.filter(pk=feed_id).update(fetch_error_count=0) diff --git a/apps/feeds/test_fetcher.py b/apps/feeds/test_fetcher.py new file mode 100644 index 0000000..1916956 --- /dev/null +++ b/apps/feeds/test_fetcher.py @@ -0,0 +1,196 @@ +import pytest +import requests +import responses +from django.utils import timezone + +from apps.articles.models import Article, UserArticleState +from apps.feeds.models import Feed +from apps.feeds.services.fetcher import effective_interval_minutes, fetch_and_ingest + +FEED_URL = "https://example.com/feed.xml" + +GOOD_FEED_XML = """ + + Example Blog + https://example.com + + First post + https://example.com/1 + https://example.com/1 + Mon, 01 Jan 2024 00:00:00 GMT + <p>Hello <script>alert(1)</script>world</p> + + +""" + +NO_GUID_FEED_XML = """ + + No Guid Blog + + Only a link + https://example.com/only-link + Mon, 01 Jan 2024 00:00:00 GMT + Body + + +""" + +MALFORMED_BUT_SALVAGEABLE_XML = """ + + Slightly Broken + + Unescaped & ampersand but still has entries + guid-1 + + +""" + +TOTAL_GARBAGE = "this is not xml at all {{{" + + +@pytest.fixture +def feed(db): + return Feed.objects.create(feed_url=FEED_URL) + + +@pytest.mark.django_db +@responses.activate +def test_fetch_ingests_new_articles_and_sanitizes_html(feed): + responses.add( + responses.GET, + FEED_URL, + body=GOOD_FEED_XML, + status=200, + headers={"ETag": '"abc123"', "Last-Modified": "Mon, 01 Jan 2024 00:00:00 GMT"}, + ) + + result = fetch_and_ingest(feed) + feed.save() + + assert result.status == Feed.STATUS_OK + assert len(result.new_article_ids) == 1 + assert feed.title == "Example Blog" + assert feed.etag == '"abc123"' + assert feed.last_modified == "Mon, 01 Jan 2024 00:00:00 GMT" + + article = Article.objects.get(feed=feed, guid="https://example.com/1") + assert "