Private
Public Access
Add SSRF protection to feed fetching: validate scheme/address, bound redirects and response size (S1)
fetch_and_ingest fetched a feed_url supplied straight from the "add feed" form or an uploaded OPML file, with requests following redirects on its own and no check that the target (or a redirect's target) was actually a public, fetchable feed rather than an internal service, localhost, or a cloud metadata endpoint — reachable from wherever this app runs, on any authenticated user's behalf. apps.feeds.services.ssrf_guard.safe_get() replaces the direct requests.get() call: it resolves and validates the hostname (rejecting anything that isn't a globally routable address) before the initial request and before following each redirect hop manually, caps the number of hops, and bounds the response body size by streaming it instead of buffering it unbounded. OPML-imported feeds go through the same check the first time poll_due_feeds fetches them, since this is the one chokepoint every feed URL passes through regardless of which path created the Feed row. Known residual gap, documented in ssrf_guard.py: this validates then connects rather than pinning the TCP connection to the exact address it validated, so a narrow DNS-rebinding race isn't closed. Fixing that needs a custom transport and felt like more machinery than this self-hosted app's threat model currently justifies. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fd8eeb605e
commit
9439f9cc59
4 files changed
+286
-1
No files matched your search
@@ -0,0 +1,121 @@
|
||||
"""Rejects feed-fetch targets that could turn this server/worker's own
|
||||
network access into an SSRF vector.
|
||||
|
||||
Feed URLs are entirely attacker-controlled (any authenticated user via
|
||||
"add feed", or an uploaded OPML file — see apps.feeds.opml) and this app
|
||||
fetches them from the server's own network position. fetch_and_ingest
|
||||
(apps.feeds.services.fetcher) is the single chokepoint every feed URL
|
||||
passes through before any HTTP request is made, regardless of which
|
||||
path created the Feed row, so enforcing this here covers all of them.
|
||||
|
||||
This validates the resolved address before connecting and again before
|
||||
following each redirect hop, denying by default anything that isn't a
|
||||
globally routable address (ip.is_global) — covering loopback, RFC1918
|
||||
private ranges, link-local (including the 169.254.169.254 cloud
|
||||
metadata address), and so on, for both IPv4 and IPv6.
|
||||
|
||||
What this does *not* close: the TCP connection isn't pinned to the
|
||||
specific IP that was validated, so a DNS answer that changes between
|
||||
the validation lookup and the actual connect (classic DNS rebinding)
|
||||
could still slip through. Closing that fully needs a custom transport
|
||||
that connects to the exact validated IP while still verifying TLS
|
||||
against the original hostname — more machinery than this self-hosted
|
||||
app's threat model currently justifies; this stops the much more
|
||||
common case of a feed URL (or a redirect) simply pointing straight at
|
||||
an internal address.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import requests
|
||||
from requests.structures import CaseInsensitiveDict
|
||||
|
||||
ALLOWED_SCHEMES = {"http", "https"}
|
||||
MAX_REDIRECTS = 5
|
||||
MAX_RESPONSE_BYTES = 10 * 1024 * 1024 # 10 MB — generous for an RSS/Atom feed
|
||||
READ_CHUNK_BYTES = 65536
|
||||
|
||||
|
||||
class SSRFBlocked(Exception):
|
||||
"""Raised when a fetch target — or one of its redirects — isn't safe
|
||||
to connect to on a user's behalf."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class BoundedResponse:
|
||||
status_code: int
|
||||
# requests' own CaseInsensitiveDict — server header casing varies
|
||||
# (ETag/Etag/etag) and fetcher.py looks these up by a fixed-case key.
|
||||
headers: CaseInsensitiveDict
|
||||
content: bytes
|
||||
|
||||
|
||||
def validate_fetch_url(url: str) -> None:
|
||||
"""Raises SSRFBlocked unless `url` has an http(s) scheme, a hostname,
|
||||
and every address that hostname resolves to is globally routable."""
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ALLOWED_SCHEMES:
|
||||
raise SSRFBlocked(f"Unsupported URL scheme: {parsed.scheme!r}")
|
||||
if not parsed.hostname:
|
||||
raise SSRFBlocked("URL has no hostname")
|
||||
|
||||
try:
|
||||
addrinfo = socket.getaddrinfo(parsed.hostname, None)
|
||||
except OSError as exc:
|
||||
raise SSRFBlocked(f"Could not resolve {parsed.hostname!r}: {exc}") from exc
|
||||
|
||||
for *_, sockaddr in addrinfo:
|
||||
ip = ipaddress.ip_address(sockaddr[0])
|
||||
if not ip.is_global:
|
||||
raise SSRFBlocked(f"{parsed.hostname!r} resolves to a non-public address: {ip}")
|
||||
|
||||
|
||||
def safe_get(url: str, *, headers: dict, timeout: float) -> BoundedResponse:
|
||||
"""GETs `url` the way fetch_and_ingest needs it: every redirect hop is
|
||||
validated and followed manually (never left to requests/urllib3, so
|
||||
a redirect can't bypass validate_fetch_url), capped at MAX_REDIRECTS,
|
||||
and the body is read in bounded chunks rather than trusting
|
||||
Content-Length or buffering an unbounded response.
|
||||
"""
|
||||
current_url = url
|
||||
with requests.Session() as session:
|
||||
for _ in range(MAX_REDIRECTS + 1):
|
||||
validate_fetch_url(current_url)
|
||||
response = session.get(
|
||||
current_url,
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
stream=True,
|
||||
allow_redirects=False,
|
||||
)
|
||||
try:
|
||||
if response.is_redirect or response.is_permanent_redirect:
|
||||
location = response.headers.get("Location")
|
||||
if not location:
|
||||
raise SSRFBlocked("Redirect response had no Location header")
|
||||
current_url = urljoin(current_url, location)
|
||||
continue
|
||||
return BoundedResponse(
|
||||
status_code=response.status_code,
|
||||
headers=response.headers,
|
||||
content=_read_bounded(response),
|
||||
)
|
||||
finally:
|
||||
response.close()
|
||||
raise SSRFBlocked(f"Too many redirects (> {MAX_REDIRECTS})")
|
||||
|
||||
|
||||
def _read_bounded(response: requests.Response) -> bytes:
|
||||
chunks = []
|
||||
total = 0
|
||||
for chunk in response.iter_content(chunk_size=READ_CHUNK_BYTES):
|
||||
total += len(chunk)
|
||||
if total > MAX_RESPONSE_BYTES:
|
||||
raise SSRFBlocked(f"Response exceeded {MAX_RESPONSE_BYTES} bytes")
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
Reference in new issue
Block a user