Private
Public Access
An OPML upload was read fully into memory, parsed into a whole tree, walked recursively, and (for large imports) embedded in a Celery message. Nothing bounded raw size, ZIP entries, nesting depth, outline count or feed count. Uploads are capped at 5 MB (checked on the reported size before reading) and ZIPs at 50 entries, with the decompressed member bounded by what it actually yields rather than only its header. Parsing now streams with iterparse and aborts the moment a limit is crossed — 24 levels deep, 20,000 outlines, 5,000 feeds — so a hostile file never builds a huge or deep tree; a 50,000-level nesting bomb is rejected cleanly. Limit errors are shown to the user. Large imports go to the worker by cache reference (deleted once read, expires after an hour) instead of as the whole document inside the broker message. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
220 lines
8.9 KiB
Python
220 lines
8.9 KiB
Python
"""OPML import/export.
|
|
|
|
Import parses with defusedxml since an uploaded .opml file is untrusted
|
|
user input (XXE risk with a plain XML parser). Export uses the stdlib's
|
|
plain ElementTree since the output is entirely app-generated — no
|
|
untrusted input, no XXE concern there.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import xml.etree.ElementTree as ET
|
|
import zipfile
|
|
from datetime import datetime
|
|
from datetime import timezone as dt_timezone
|
|
from io import BytesIO
|
|
|
|
import defusedxml.ElementTree as DefusedET
|
|
|
|
from apps.articles.models import SubscriptionTag, Tag
|
|
from apps.core.urlutils import http_url_or_empty
|
|
|
|
from .models import Feed, Subscription
|
|
|
|
# FreshRSS's own "Export" produces a .zip (an .opml.xml plus a starred-
|
|
# articles .json we don't import) rather than a bare .opml file — accept
|
|
# that directly instead of making people unzip it themselves first.
|
|
MAX_ZIPPED_OPML_SIZE = 20 * 1024 * 1024
|
|
|
|
# An upload is untrusted: bound everything an attacker controls (R2-S5).
|
|
# A real export of thousands of feeds is well under a megabyte.
|
|
MAX_UPLOAD_BYTES = 5 * 1024 * 1024
|
|
MAX_ZIP_ENTRIES = 50
|
|
MAX_OPML_DEPTH = 24
|
|
MAX_OPML_OUTLINES = 20_000
|
|
MAX_OPML_FEEDS = 5_000
|
|
|
|
|
|
def extract_opml_bytes(uploaded_bytes: bytes) -> bytes:
|
|
"""If the upload is a zip (e.g. FreshRSS's export), returns the bytes
|
|
of the .opml/.opml.xml member inside it; otherwise returns the input
|
|
unchanged, assuming it's already a bare OPML file."""
|
|
if not zipfile.is_zipfile(BytesIO(uploaded_bytes)):
|
|
return uploaded_bytes
|
|
|
|
with zipfile.ZipFile(BytesIO(uploaded_bytes)) as archive:
|
|
if len(archive.infolist()) > MAX_ZIP_ENTRIES:
|
|
raise ValueError("The uploaded zip contains too many files.")
|
|
candidates = [
|
|
info
|
|
for info in archive.infolist()
|
|
if not info.is_dir() and info.filename.lower().endswith((".opml", ".opml.xml"))
|
|
]
|
|
if not candidates:
|
|
raise ValueError("No .opml file found inside the uploaded zip.")
|
|
member = min(candidates, key=lambda info: info.file_size)
|
|
if member.file_size > MAX_ZIPPED_OPML_SIZE:
|
|
raise ValueError("The .opml file inside the zip is implausibly large.")
|
|
# file_size is just a header field an attacker controls, so bound
|
|
# what actually decompresses too, not only what it claims.
|
|
with archive.open(member) as handle:
|
|
data = handle.read(MAX_ZIPPED_OPML_SIZE + 1)
|
|
if len(data) > MAX_ZIPPED_OPML_SIZE:
|
|
raise ValueError("The .opml file inside the zip is implausibly large.")
|
|
return data
|
|
|
|
|
|
def parse_opml(file_obj) -> list[dict]:
|
|
"""Returns a flat list of {feed_url, title, site_url, tag_names}.
|
|
|
|
A feed's tags come from two places, combined: every enclosing
|
|
category/folder outline's name (so a feed nested two levels deep picks
|
|
up both as separate tags — tags aren't limited to one level the way
|
|
the old folder scheme was), and a comma-separated "category" attribute
|
|
on the feed's own outline, the convention export_opml below writes and
|
|
many other readers use for a flat multi-category representation.
|
|
"""
|
|
entries: list[dict] = []
|
|
# Streamed with iterparse instead of building the whole tree first, so
|
|
# a hostile file (absurd nesting, millions of outlines) is rejected as
|
|
# soon as it crosses a limit rather than after it has used the memory.
|
|
in_body = False
|
|
depth = 0
|
|
outline_count = 0
|
|
open_feed_outlines = 0 # >0 while inside a feed outline: its children are ignored
|
|
pushed_name: list[bool] = [] # per open category outline: did it add a tag name?
|
|
ancestor_names: list[str] = []
|
|
|
|
for event, elem in DefusedET.iterparse(file_obj, events=("start", "end")):
|
|
if event == "start":
|
|
depth += 1
|
|
if depth > MAX_OPML_DEPTH:
|
|
raise ValueError("The OPML file is nested too deeply.")
|
|
if elem.tag == "body":
|
|
in_body = True
|
|
elif elem.tag == "outline" and in_body:
|
|
outline_count += 1
|
|
if outline_count > MAX_OPML_OUTLINES:
|
|
raise ValueError("The OPML file contains too many entries.")
|
|
if open_feed_outlines:
|
|
open_feed_outlines += 1
|
|
continue
|
|
raw_xml_url = elem.get("xmlUrl")
|
|
if raw_xml_url:
|
|
open_feed_outlines = 1
|
|
xml_url = http_url_or_empty(raw_xml_url)
|
|
if not xml_url:
|
|
continue # a feed URL that isn't http(s) can never be fetched
|
|
tag_names = list(ancestor_names)
|
|
for name in (elem.get("category") or "").split(","):
|
|
name = name.strip()
|
|
if name and name not in tag_names:
|
|
tag_names.append(name)
|
|
entries.append(
|
|
{
|
|
"feed_url": xml_url,
|
|
"title": elem.get("title") or elem.get("text") or "",
|
|
"site_url": http_url_or_empty(elem.get("htmlUrl")),
|
|
"tag_names": tag_names,
|
|
}
|
|
)
|
|
if len(entries) > MAX_OPML_FEEDS:
|
|
raise ValueError("The OPML file contains too many feeds.")
|
|
else:
|
|
nested_name = elem.get("title") or elem.get("text")
|
|
pushed_name.append(bool(nested_name))
|
|
if nested_name:
|
|
ancestor_names.append(nested_name)
|
|
else:
|
|
depth -= 1
|
|
if elem.tag == "body":
|
|
in_body = False
|
|
elif elem.tag == "outline" and in_body:
|
|
if open_feed_outlines:
|
|
open_feed_outlines -= 1
|
|
elif pushed_name:
|
|
if pushed_name.pop():
|
|
ancestor_names.pop()
|
|
elem.clear()
|
|
return entries
|
|
|
|
|
|
def import_opml(user, file_obj) -> dict:
|
|
"""Creates/reuses Feeds and Subscriptions for every entry, tagging each
|
|
subscription per its entry's tag_names. Does not fetch any feed
|
|
content — newly created Feeds have no last_fetched_at, so the next
|
|
poll_due_feeds beat cycle picks them up naturally."""
|
|
entries = parse_opml(file_obj)
|
|
tag_cache: dict[str, Tag] = {}
|
|
created_count = 0
|
|
already_subscribed_count = 0
|
|
|
|
for entry in entries:
|
|
feed, _ = Feed.objects.get_or_create(
|
|
feed_url=entry["feed_url"][:1000],
|
|
defaults={
|
|
"title": entry["title"][:500],
|
|
"site_url": entry["site_url"][:1000],
|
|
},
|
|
)
|
|
subscription, created = Subscription.objects.get_or_create(
|
|
user=user,
|
|
feed=feed,
|
|
defaults={"custom_title": ""},
|
|
)
|
|
if created:
|
|
created_count += 1
|
|
else:
|
|
already_subscribed_count += 1
|
|
|
|
for tag_name in entry["tag_names"]:
|
|
tag = tag_cache.get(tag_name)
|
|
if tag is None:
|
|
tag, _ = Tag.objects.get_or_create(user=user, name=tag_name[:100])
|
|
tag_cache[tag_name] = tag
|
|
SubscriptionTag.objects.get_or_create(user=user, subscription=subscription, tag=tag)
|
|
|
|
return {
|
|
"total": len(entries),
|
|
"created": created_count,
|
|
"already_subscribed": already_subscribed_count,
|
|
}
|
|
|
|
|
|
def export_opml(user) -> bytes:
|
|
"""A flat list of outlines (no folder nesting) with each feed's tags
|
|
joined into a comma-separated "category" attribute — the standard OPML
|
|
way to represent multi-category membership without picking one tag to
|
|
nest under, which a feed with several tags wouldn't fit."""
|
|
root = ET.Element("opml", version="2.0")
|
|
head = ET.SubElement(root, "head")
|
|
ET.SubElement(head, "title").text = "Evection subscriptions"
|
|
ET.SubElement(head, "dateCreated").text = datetime.now(dt_timezone.utc).strftime(
|
|
"%a, %d %b %Y %H:%M:%S %z"
|
|
)
|
|
body = ET.SubElement(root, "body")
|
|
|
|
subscriptions = (
|
|
Subscription.objects.filter(user=user).select_related("feed").order_by("order", "id")
|
|
)
|
|
tag_names_by_subscription: dict[int, list[str]] = {}
|
|
for row in (
|
|
SubscriptionTag.objects.filter(user=user).select_related("tag").order_by("tag__name")
|
|
):
|
|
tag_names_by_subscription.setdefault(row.subscription_id, []).append(row.tag.name)
|
|
|
|
for subscription in subscriptions:
|
|
attrs = {
|
|
"type": "rss",
|
|
"text": subscription.display_title,
|
|
"title": subscription.display_title,
|
|
"xmlUrl": subscription.feed.feed_url,
|
|
"htmlUrl": subscription.feed.site_url or "",
|
|
}
|
|
tag_names = tag_names_by_subscription.get(subscription.pk)
|
|
if tag_names:
|
|
attrs["category"] = ",".join(tag_names)
|
|
ET.SubElement(body, "outline", attrs)
|
|
|
|
return ET.tostring(root, encoding="utf-8", xml_declaration=True)
|