Private
Public Access
FreshRSS's Export feature produces a .zip (an .opml.xml plus a starred- articles .json this app doesn't import) rather than a bare .opml file — uploading that zip as-is failed with "That file doesn't look like a valid OPML export" since defusedxml can't parse a zip as XML. apps.feeds.opml.extract_opml_bytes detects a zip upload and pulls the .opml/.opml.xml member out of it before parsing; a bare .opml upload is returned unchanged, so existing behavior is untouched. Verified against the reporter's actual FreshRSS export (18 feeds across 6 categories) both directly through extract_opml_bytes/parse_opml and as a real upload through the running app. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
173 lines
6.5 KiB
Python
173 lines
6.5 KiB
Python
"""OPML import/export.
|
|
|
|
Import parses with defusedxml since an uploaded .opml file is untrusted
|
|
user input (XXE risk with a plain XML parser). Export uses the stdlib's
|
|
plain ElementTree since the output is entirely app-generated — no
|
|
untrusted input, no XXE concern there.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import xml.etree.ElementTree as ET
|
|
import zipfile
|
|
from datetime import datetime
|
|
from datetime import timezone as dt_timezone
|
|
from io import BytesIO
|
|
|
|
import defusedxml.ElementTree as DefusedET
|
|
|
|
from apps.articles.models import SubscriptionTag, Tag
|
|
|
|
from .models import Feed, Subscription
|
|
|
|
# FreshRSS's own "Export" produces a .zip (an .opml.xml plus a starred-
|
|
# articles .json we don't import) rather than a bare .opml file — accept
|
|
# that directly instead of making people unzip it themselves first.
|
|
MAX_ZIPPED_OPML_SIZE = 20 * 1024 * 1024
|
|
|
|
|
|
def extract_opml_bytes(uploaded_bytes: bytes) -> bytes:
|
|
"""If the upload is a zip (e.g. FreshRSS's export), returns the bytes
|
|
of the .opml/.opml.xml member inside it; otherwise returns the input
|
|
unchanged, assuming it's already a bare OPML file."""
|
|
if not zipfile.is_zipfile(BytesIO(uploaded_bytes)):
|
|
return uploaded_bytes
|
|
|
|
with zipfile.ZipFile(BytesIO(uploaded_bytes)) as archive:
|
|
candidates = [
|
|
info
|
|
for info in archive.infolist()
|
|
if not info.is_dir() and info.filename.lower().endswith((".opml", ".opml.xml"))
|
|
]
|
|
if not candidates:
|
|
raise ValueError("No .opml file found inside the uploaded zip.")
|
|
member = min(candidates, key=lambda info: info.file_size)
|
|
if member.file_size > MAX_ZIPPED_OPML_SIZE:
|
|
raise ValueError("The .opml file inside the zip is implausibly large.")
|
|
return archive.read(member)
|
|
|
|
|
|
def parse_opml(file_obj) -> list[dict]:
|
|
"""Returns a flat list of {feed_url, title, site_url, tag_names}.
|
|
|
|
A feed's tags come from two places, combined: every enclosing
|
|
category/folder outline's name (so a feed nested two levels deep picks
|
|
up both as separate tags — tags aren't limited to one level the way
|
|
the old folder scheme was), and a comma-separated "category" attribute
|
|
on the feed's own outline, the convention export_opml below writes and
|
|
many other readers use for a flat multi-category representation.
|
|
"""
|
|
tree = DefusedET.parse(file_obj)
|
|
root = tree.getroot()
|
|
body = root.find("body")
|
|
entries: list[dict] = []
|
|
if body is None:
|
|
return entries
|
|
|
|
def walk(container, ancestor_names):
|
|
for outline in container.findall("outline"):
|
|
xml_url = outline.get("xmlUrl")
|
|
if xml_url:
|
|
tag_names = list(ancestor_names)
|
|
category_attr = outline.get("category")
|
|
if category_attr:
|
|
for name in category_attr.split(","):
|
|
name = name.strip()
|
|
if name and name not in tag_names:
|
|
tag_names.append(name)
|
|
entries.append(
|
|
{
|
|
"feed_url": xml_url,
|
|
"title": outline.get("title") or outline.get("text") or "",
|
|
"site_url": outline.get("htmlUrl") or "",
|
|
"tag_names": tag_names,
|
|
}
|
|
)
|
|
else:
|
|
nested_name = outline.get("title") or outline.get("text")
|
|
next_ancestors = ancestor_names + [nested_name] if nested_name else ancestor_names
|
|
walk(outline, next_ancestors)
|
|
|
|
walk(body, [])
|
|
return entries
|
|
|
|
|
|
def import_opml(user, file_obj) -> dict:
|
|
"""Creates/reuses Feeds and Subscriptions for every entry, tagging each
|
|
subscription per its entry's tag_names. Does not fetch any feed
|
|
content — newly created Feeds have no last_fetched_at, so the next
|
|
poll_due_feeds beat cycle picks them up naturally."""
|
|
entries = parse_opml(file_obj)
|
|
tag_cache: dict[str, Tag] = {}
|
|
created_count = 0
|
|
already_subscribed_count = 0
|
|
|
|
for entry in entries:
|
|
feed, _ = Feed.objects.get_or_create(
|
|
feed_url=entry["feed_url"][:1000],
|
|
defaults={
|
|
"title": entry["title"][:500],
|
|
"site_url": entry["site_url"][:1000],
|
|
},
|
|
)
|
|
subscription, created = Subscription.objects.get_or_create(
|
|
user=user,
|
|
feed=feed,
|
|
defaults={"custom_title": ""},
|
|
)
|
|
if created:
|
|
created_count += 1
|
|
else:
|
|
already_subscribed_count += 1
|
|
|
|
for tag_name in entry["tag_names"]:
|
|
tag = tag_cache.get(tag_name)
|
|
if tag is None:
|
|
tag, _ = Tag.objects.get_or_create(user=user, name=tag_name[:100])
|
|
tag_cache[tag_name] = tag
|
|
SubscriptionTag.objects.get_or_create(user=user, subscription=subscription, tag=tag)
|
|
|
|
return {
|
|
"total": len(entries),
|
|
"created": created_count,
|
|
"already_subscribed": already_subscribed_count,
|
|
}
|
|
|
|
|
|
def export_opml(user) -> bytes:
|
|
"""A flat list of outlines (no folder nesting) with each feed's tags
|
|
joined into a comma-separated "category" attribute — the standard OPML
|
|
way to represent multi-category membership without picking one tag to
|
|
nest under, which a feed with several tags wouldn't fit."""
|
|
root = ET.Element("opml", version="2.0")
|
|
head = ET.SubElement(root, "head")
|
|
ET.SubElement(head, "title").text = "Evection subscriptions"
|
|
ET.SubElement(head, "dateCreated").text = datetime.now(dt_timezone.utc).strftime(
|
|
"%a, %d %b %Y %H:%M:%S %z"
|
|
)
|
|
body = ET.SubElement(root, "body")
|
|
|
|
subscriptions = (
|
|
Subscription.objects.filter(user=user).select_related("feed").order_by("order", "id")
|
|
)
|
|
tag_names_by_subscription: dict[int, list[str]] = {}
|
|
for row in (
|
|
SubscriptionTag.objects.filter(user=user).select_related("tag").order_by("tag__name")
|
|
):
|
|
tag_names_by_subscription.setdefault(row.subscription_id, []).append(row.tag.name)
|
|
|
|
for subscription in subscriptions:
|
|
attrs = {
|
|
"type": "rss",
|
|
"text": subscription.display_title,
|
|
"title": subscription.display_title,
|
|
"xmlUrl": subscription.feed.feed_url,
|
|
"htmlUrl": subscription.feed.site_url or "",
|
|
}
|
|
tag_names = tag_names_by_subscription.get(subscription.pk)
|
|
if tag_names:
|
|
attrs["category"] = ",".join(tag_names)
|
|
ET.SubElement(body, "outline", attrs)
|
|
|
|
return ET.tostring(root, encoding="utf-8", xml_declaration=True)
|