Private
Public Access
Add OPML import/export and a subscriptions management page
Implements OPML parsing with defusedxml (untrusted upload, XXE risk) and export with the stdlib's plain ElementTree (trusted, app-generated output). Nesting deeper than one level flattens into its nearest enclosing folder, matching the app's flat-folder model. Import reuses existing Feed/Subscription rows via get_or_create, so re-importing the same file is idempotent and cross-user Feed dedup is preserved. Large imports (>50 entries) dispatch to a Celery task instead of blocking the request; newly imported feeds aren't fetched immediately since they have no last_fetched_at, so the next poll_due_feeds beat cycle picks them up naturally. Adds a Subscriptions page (replacing the standalone "Add feed" link in nav) listing current subscriptions with unsubscribe, plus Import/ Export OPML actions. Verified live: imported a 2-feed/1-folder OPML file against the running dev server (via an authenticated session, since this browser tool has no file-picker automation), confirmed the resulting subscriptions and folder in the UI, exported and inspected the resulting OPML structure, and unsubscribed from one feed. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8653dbeaf9
commit
ea214dd9ce
11 files changed
+533
-3
No files matched your search
@@ -11,3 +11,7 @@ class AddFeedForm(forms.Form):
|
||||
super().__init__(*args, **kwargs)
|
||||
if user is not None:
|
||||
self.fields["folder"].queryset = Folder.objects.filter(user=user)
|
||||
|
||||
|
||||
class OPMLImportForm(forms.Form):
|
||||
opml_file = forms.FileField(label="OPML file")
|
||||
@@ -0,0 +1,135 @@
|
||||
"""OPML import/export.
|
||||
|
||||
Import parses with defusedxml since an uploaded .opml file is untrusted
|
||||
user input (XXE risk with a plain XML parser). Export uses the stdlib's
|
||||
plain ElementTree since the output is entirely app-generated — no
|
||||
untrusted input, no XXE concern there.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime
|
||||
from datetime import timezone as dt_timezone
|
||||
|
||||
import defusedxml.ElementTree as DefusedET
|
||||
|
||||
from .models import Feed, Folder, Subscription
|
||||
|
||||
|
||||
def parse_opml(file_obj) -> list[dict]:
|
||||
"""Returns a flat list of {feed_url, title, site_url, folder_name}.
|
||||
|
||||
Folders are kept to a single flat level, matching the app's folder
|
||||
model — nesting deeper than one level collapses into its nearest
|
||||
enclosing folder rather than being represented as a path.
|
||||
"""
|
||||
tree = DefusedET.parse(file_obj)
|
||||
root = tree.getroot()
|
||||
body = root.find("body")
|
||||
entries: list[dict] = []
|
||||
if body is None:
|
||||
return entries
|
||||
|
||||
def walk(container, folder_name):
|
||||
for outline in container.findall("outline"):
|
||||
xml_url = outline.get("xmlUrl")
|
||||
if xml_url:
|
||||
entries.append(
|
||||
{
|
||||
"feed_url": xml_url,
|
||||
"title": outline.get("title") or outline.get("text") or "",
|
||||
"site_url": outline.get("htmlUrl") or "",
|
||||
"folder_name": folder_name,
|
||||
}
|
||||
)
|
||||
else:
|
||||
nested_name = outline.get("title") or outline.get("text") or folder_name
|
||||
walk(outline, nested_name)
|
||||
|
||||
walk(body, None)
|
||||
return entries
|
||||
|
||||
|
||||
def import_opml(user, file_obj) -> dict:
|
||||
"""Creates/reuses Feeds and Subscriptions for every entry. Does not
|
||||
fetch any feed content — newly created Feeds have no last_fetched_at,
|
||||
so the next poll_due_feeds beat cycle picks them up naturally."""
|
||||
entries = parse_opml(file_obj)
|
||||
folder_cache: dict[str, Folder] = {}
|
||||
created_count = 0
|
||||
already_subscribed_count = 0
|
||||
|
||||
for entry in entries:
|
||||
folder = None
|
||||
folder_name = entry["folder_name"]
|
||||
if folder_name:
|
||||
folder = folder_cache.get(folder_name)
|
||||
if folder is None:
|
||||
folder, _ = Folder.objects.get_or_create(user=user, name=folder_name[:255])
|
||||
folder_cache[folder_name] = folder
|
||||
|
||||
feed, _ = Feed.objects.get_or_create(
|
||||
feed_url=entry["feed_url"][:1000],
|
||||
defaults={
|
||||
"title": entry["title"][:500],
|
||||
"site_url": entry["site_url"][:1000],
|
||||
},
|
||||
)
|
||||
_, created = Subscription.objects.get_or_create(
|
||||
user=user,
|
||||
feed=feed,
|
||||
defaults={"folder": folder, "custom_title": ""},
|
||||
)
|
||||
if created:
|
||||
created_count += 1
|
||||
else:
|
||||
already_subscribed_count += 1
|
||||
|
||||
return {
|
||||
"total": len(entries),
|
||||
"created": created_count,
|
||||
"already_subscribed": already_subscribed_count,
|
||||
}
|
||||
|
||||
|
||||
def export_opml(user) -> bytes:
|
||||
root = ET.Element("opml", version="2.0")
|
||||
head = ET.SubElement(root, "head")
|
||||
ET.SubElement(head, "title").text = "Evection subscriptions"
|
||||
ET.SubElement(head, "dateCreated").text = datetime.now(dt_timezone.utc).strftime(
|
||||
"%a, %d %b %Y %H:%M:%S %z"
|
||||
)
|
||||
body = ET.SubElement(root, "body")
|
||||
|
||||
subscriptions = (
|
||||
Subscription.objects.filter(user=user)
|
||||
.select_related("feed", "folder")
|
||||
.order_by("folder__name", "order", "id")
|
||||
)
|
||||
|
||||
folder_elements: dict[str, ET.Element] = {}
|
||||
for subscription in subscriptions:
|
||||
parent = body
|
||||
if subscription.folder:
|
||||
parent = folder_elements.get(subscription.folder.name)
|
||||
if parent is None:
|
||||
parent = ET.SubElement(
|
||||
body,
|
||||
"outline",
|
||||
title=subscription.folder.name,
|
||||
text=subscription.folder.name,
|
||||
)
|
||||
folder_elements[subscription.folder.name] = parent
|
||||
|
||||
ET.SubElement(
|
||||
parent,
|
||||
"outline",
|
||||
type="rss",
|
||||
text=subscription.display_title,
|
||||
title=subscription.display_title,
|
||||
xmlUrl=subscription.feed.feed_url,
|
||||
htmlUrl=subscription.feed.site_url or "",
|
||||
)
|
||||
|
||||
return ET.tostring(root, encoding="utf-8", xml_declaration=True)
|
||||
@@ -1,12 +1,15 @@
|
||||
import logging
|
||||
from datetime import timedelta
|
||||
from io import BytesIO
|
||||
|
||||
from celery import shared_task
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.articles.tasks import apply_filter_rules_for_feed
|
||||
|
||||
from .models import Feed
|
||||
from .opml import import_opml
|
||||
from .services.fetcher import effective_interval_minutes, fetch_and_ingest
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -69,3 +72,15 @@ def reset_feed_backoff(feed_id: int):
|
||||
"""Admin-triggered: clear a feed's error count so it's retried at its
|
||||
normal configured interval instead of the backed-off one."""
|
||||
Feed.objects.filter(pk=feed_id).update(fetch_error_count=0)
|
||||
|
||||
|
||||
@shared_task
|
||||
def import_opml_task(user_id: int, opml_content: str):
|
||||
"""Background path for large OPML imports — see OPML_ASYNC_THRESHOLD
|
||||
in apps.feeds.views. opml_content is passed as text rather than a
|
||||
file handle since Celery tasks must be JSON-serializable."""
|
||||
try:
|
||||
user = get_user_model().objects.get(pk=user_id)
|
||||
except get_user_model().DoesNotExist:
|
||||
return None
|
||||
return import_opml(user, BytesIO(opml_content.encode("utf-8")))
|
||||
@@ -0,0 +1,128 @@
|
||||
from io import BytesIO
|
||||
|
||||
import pytest
|
||||
from django.contrib.auth import get_user_model
|
||||
|
||||
from .models import Feed, Folder, Subscription
|
||||
from .opml import export_opml, import_opml, parse_opml
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
SIMPLE_OPML = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<opml version="2.0">
|
||||
<head><title>Subscriptions</title></head>
|
||||
<body>
|
||||
<outline text="Tech">
|
||||
<outline type="rss" text="Example Blog" title="Example Blog"
|
||||
xmlUrl="https://example.com/feed.xml" htmlUrl="https://example.com"/>
|
||||
</outline>
|
||||
<outline type="rss" text="Unfoldered Feed" title="Unfoldered Feed"
|
||||
xmlUrl="https://unfoldered.example.com/feed.xml"/>
|
||||
</body>
|
||||
</opml>
|
||||
"""
|
||||
|
||||
NESTED_OPML = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<opml version="2.0">
|
||||
<body>
|
||||
<outline text="Parent">
|
||||
<outline text="Child">
|
||||
<outline type="rss" text="Deeply nested feed"
|
||||
xmlUrl="https://nested.example.com/feed.xml"/>
|
||||
</outline>
|
||||
</outline>
|
||||
</body>
|
||||
</opml>
|
||||
"""
|
||||
|
||||
MALFORMED_XML = b"<opml><body><outline"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def alice(db):
|
||||
return User.objects.create_user("alice", password="pw-alice-123")
|
||||
|
||||
|
||||
class TestParseOPML:
|
||||
def test_parses_foldered_and_unfoldered_entries(self):
|
||||
entries = parse_opml(BytesIO(SIMPLE_OPML))
|
||||
|
||||
assert len(entries) == 2
|
||||
foldered = next(e for e in entries if e["feed_url"] == "https://example.com/feed.xml")
|
||||
assert foldered["folder_name"] == "Tech"
|
||||
assert foldered["title"] == "Example Blog"
|
||||
|
||||
unfoldered = next(
|
||||
e for e in entries if e["feed_url"] == "https://unfoldered.example.com/feed.xml"
|
||||
)
|
||||
assert unfoldered["folder_name"] is None
|
||||
|
||||
def test_flattens_nesting_deeper_than_one_level(self):
|
||||
entries = parse_opml(BytesIO(NESTED_OPML))
|
||||
|
||||
assert len(entries) == 1
|
||||
# Deepest enclosing folder name wins rather than a path.
|
||||
assert entries[0]["folder_name"] == "Child"
|
||||
|
||||
def test_malformed_xml_raises(self):
|
||||
with pytest.raises(Exception):
|
||||
parse_opml(BytesIO(MALFORMED_XML))
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestImportOPML:
|
||||
def test_creates_feeds_folders_and_subscriptions(self, alice):
|
||||
result = import_opml(alice, BytesIO(SIMPLE_OPML))
|
||||
|
||||
assert result == {"total": 2, "created": 2, "already_subscribed": 0}
|
||||
assert Folder.objects.filter(user=alice, name="Tech").exists()
|
||||
assert Subscription.objects.filter(
|
||||
user=alice, feed__feed_url="https://example.com/feed.xml"
|
||||
).exists()
|
||||
unfoldered_sub = Subscription.objects.get(
|
||||
user=alice, feed__feed_url="https://unfoldered.example.com/feed.xml"
|
||||
)
|
||||
assert unfoldered_sub.folder is None
|
||||
|
||||
def test_reimporting_the_same_file_does_not_duplicate(self, alice):
|
||||
import_opml(alice, BytesIO(SIMPLE_OPML))
|
||||
result = import_opml(alice, BytesIO(SIMPLE_OPML))
|
||||
|
||||
assert result == {"total": 2, "created": 0, "already_subscribed": 2}
|
||||
assert Subscription.objects.filter(user=alice).count() == 2
|
||||
assert Folder.objects.filter(user=alice, name="Tech").count() == 1
|
||||
|
||||
def test_reuses_existing_feed_across_users(self, alice):
|
||||
bob = User.objects.create_user("bob", password="pw-bob-123")
|
||||
import_opml(alice, BytesIO(SIMPLE_OPML))
|
||||
import_opml(bob, BytesIO(SIMPLE_OPML))
|
||||
|
||||
assert Feed.objects.filter(feed_url="https://example.com/feed.xml").count() == 1
|
||||
assert (
|
||||
Subscription.objects.filter(feed__feed_url="https://example.com/feed.xml").count() == 2
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestExportOPML:
|
||||
def test_export_round_trips_through_import(self, alice):
|
||||
import_opml(alice, BytesIO(SIMPLE_OPML))
|
||||
|
||||
exported = export_opml(alice)
|
||||
|
||||
bob = User.objects.create_user("bob", password="pw-bob-123")
|
||||
result = import_opml(bob, BytesIO(exported))
|
||||
|
||||
assert result["created"] == 2
|
||||
assert Folder.objects.filter(user=bob, name="Tech").exists()
|
||||
assert Subscription.objects.filter(
|
||||
user=bob, feed__feed_url="https://example.com/feed.xml", folder__name="Tech"
|
||||
).exists()
|
||||
assert Subscription.objects.filter(
|
||||
user=bob, feed__feed_url="https://unfoldered.example.com/feed.xml", folder__isnull=True
|
||||
).exists()
|
||||
|
||||
def test_export_with_no_subscriptions(self, alice):
|
||||
exported = export_opml(alice)
|
||||
assert b"<opml" in exported
|
||||
assert b"<outline" not in exported
|
||||
@@ -0,0 +1,123 @@
|
||||
import pytest
|
||||
from django.contrib.auth import get_user_model
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from django.urls import reverse
|
||||
|
||||
from .models import Feed, Subscription
|
||||
from .test_opml import SIMPLE_OPML
|
||||
|
||||
User = get_user_model()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def alice(db):
|
||||
return User.objects.create_user("alice", password="pw-alice-123")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bob(db):
|
||||
return User.objects.create_user("bob", password="pw-bob-123")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def alices_subscription(alice):
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml", title="Example")
|
||||
return Subscription.objects.create(user=alice, feed=feed)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_subscription_list_requires_login(client):
|
||||
response = client.get(reverse("feeds:subscription_list"))
|
||||
assert response.status_code == 302
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_subscription_list_shows_only_own_subscriptions(client, alice, bob, alices_subscription):
|
||||
client.login(username="bob", password="pw-bob-123")
|
||||
response = client.get(reverse("feeds:subscription_list"))
|
||||
assert "Example" not in response.content.decode()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_unsubscribe(client, alice, alices_subscription):
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
response = client.post(reverse("feeds:unsubscribe", args=[alices_subscription.pk]))
|
||||
assert response.status_code == 302
|
||||
assert not Subscription.objects.filter(pk=alices_subscription.pk).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_user_cannot_unsubscribe_another_users_subscription(
|
||||
client, alice, bob, alices_subscription
|
||||
):
|
||||
client.login(username="bob", password="pw-bob-123")
|
||||
response = client.post(reverse("feeds:unsubscribe", args=[alices_subscription.pk]))
|
||||
assert response.status_code == 404
|
||||
assert Subscription.objects.filter(pk=alices_subscription.pk).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_import_opml_view_creates_subscriptions(client, alice):
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
upload = SimpleUploadedFile("subs.opml", SIMPLE_OPML, content_type="text/x-opml")
|
||||
|
||||
response = client.post(reverse("feeds:import_opml"), {"opml_file": upload})
|
||||
|
||||
assert response.status_code == 302
|
||||
assert Subscription.objects.filter(user=alice).count() == 2
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_import_opml_view_rejects_malformed_file(client, alice):
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
upload = SimpleUploadedFile("subs.opml", b"not xml at all", content_type="text/x-opml")
|
||||
|
||||
response = client.post(reverse("feeds:import_opml"), {"opml_file": upload})
|
||||
|
||||
assert response.status_code == 302
|
||||
assert not Subscription.objects.filter(user=alice).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_import_opml_dispatches_background_task_above_threshold(client, alice, monkeypatch):
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
|
||||
dispatched = {}
|
||||
|
||||
def fake_delay(user_id, content):
|
||||
dispatched["user_id"] = user_id
|
||||
dispatched["content"] = content
|
||||
|
||||
from . import views as feeds_views
|
||||
|
||||
monkeypatch.setattr(feeds_views.import_opml_task, "delay", fake_delay)
|
||||
monkeypatch.setattr(feeds_views, "OPML_ASYNC_THRESHOLD", 1)
|
||||
|
||||
upload = SimpleUploadedFile("subs.opml", SIMPLE_OPML, content_type="text/x-opml")
|
||||
response = client.post(reverse("feeds:import_opml"), {"opml_file": upload})
|
||||
|
||||
assert response.status_code == 302
|
||||
assert dispatched["user_id"] == alice.id
|
||||
# The background task was dispatched instead of a synchronous import.
|
||||
assert not Subscription.objects.filter(user=alice).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_export_opml_view_returns_downloadable_file(client, alice, alices_subscription):
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
response = client.get(reverse("feeds:export_opml"))
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response["Content-Type"] == "text/x-opml"
|
||||
assert "attachment" in response["Content-Disposition"]
|
||||
assert b"example.com/feed.xml" in response.content
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_import_opml_task_runs_import(alice):
|
||||
from .tasks import import_opml_task
|
||||
|
||||
result = import_opml_task(alice.id, SIMPLE_OPML.decode("utf-8"))
|
||||
|
||||
assert result["created"] == 2
|
||||
assert Subscription.objects.filter(user=alice).count() == 2
|
||||
@@ -6,4 +6,12 @@ app_name = "feeds"
|
||||
|
||||
urlpatterns = [
|
||||
path("feeds/add/", views.add_feed, name="add_feed"),
|
||||
path("subscriptions/", views.subscription_list, name="subscription_list"),
|
||||
path(
|
||||
"subscriptions/<int:subscription_id>/unsubscribe/",
|
||||
views.unsubscribe,
|
||||
name="unsubscribe",
|
||||
),
|
||||
path("subscriptions/import/", views.import_opml_view, name="import_opml"),
|
||||
path("subscriptions/export/", views.export_opml_view, name="export_opml"),
|
||||
]
|
||||
+64
-2
@@ -1,10 +1,18 @@
|
||||
from io import BytesIO
|
||||
|
||||
from django.contrib import messages
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.shortcuts import redirect, render
|
||||
from django.http import HttpResponse
|
||||
from django.shortcuts import get_object_or_404, redirect, render
|
||||
from django.views.decorators.http import require_POST
|
||||
|
||||
from .forms import AddFeedForm
|
||||
from .forms import AddFeedForm, OPMLImportForm
|
||||
from .models import Feed, Subscription
|
||||
from .opml import export_opml, import_opml, parse_opml
|
||||
from .services.fetcher import fetch_and_ingest
|
||||
from .tasks import import_opml_task
|
||||
|
||||
OPML_ASYNC_THRESHOLD = 50
|
||||
|
||||
|
||||
@login_required
|
||||
@@ -43,3 +51,57 @@ def add_feed(request):
|
||||
return redirect("reader:stream")
|
||||
|
||||
return render(request, "feeds/add_feed.html", {"form": form})
|
||||
|
||||
|
||||
@login_required
|
||||
def subscription_list(request):
|
||||
subscriptions = (
|
||||
Subscription.objects.filter(user=request.user)
|
||||
.select_related("feed", "folder")
|
||||
.order_by("folder__name", "order", "id")
|
||||
)
|
||||
return render(request, "feeds/subscriptions.html", {"subscriptions": subscriptions})
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def unsubscribe(request, subscription_id):
|
||||
subscription = get_object_or_404(Subscription, user=request.user, pk=subscription_id)
|
||||
title = subscription.display_title
|
||||
subscription.delete()
|
||||
messages.success(request, f"Unsubscribed from '{title}'.")
|
||||
return redirect("feeds:subscription_list")
|
||||
|
||||
|
||||
@login_required
|
||||
def import_opml_view(request):
|
||||
form = OPMLImportForm(request.POST or None, request.FILES or None)
|
||||
if request.method == "POST" and form.is_valid():
|
||||
content = form.cleaned_data["opml_file"].read()
|
||||
try:
|
||||
entry_count = len(parse_opml(BytesIO(content)))
|
||||
except Exception:
|
||||
messages.error(request, "That file doesn't look like a valid OPML export.")
|
||||
return redirect("feeds:import_opml")
|
||||
|
||||
if entry_count > OPML_ASYNC_THRESHOLD:
|
||||
import_opml_task.delay(request.user.id, content.decode("utf-8"))
|
||||
messages.success(request, f"Importing {entry_count} feeds in the background.")
|
||||
else:
|
||||
result = import_opml(request.user, BytesIO(content))
|
||||
messages.success(
|
||||
request,
|
||||
f"Imported {result['created']} feed(s) "
|
||||
f"({result['already_subscribed']} already subscribed).",
|
||||
)
|
||||
return redirect("feeds:subscription_list")
|
||||
|
||||
return render(request, "feeds/import_opml.html", {"form": form})
|
||||
|
||||
|
||||
@login_required
|
||||
def export_opml_view(request):
|
||||
content = export_opml(request.user)
|
||||
response = HttpResponse(content, content_type="text/x-opml")
|
||||
response["Content-Disposition"] = 'attachment; filename="evection-subscriptions.opml"'
|
||||
return response
|
||||
Reference in new issue
Block a user