Private
Public Access
The SSRF guard checked that a hostname resolved to public addresses, then let requests resolve it again to connect — so a DNS answer that changed in between (DNS rebinding) could point the connection at an internal address. Connections now go through a pinned adapter whose socket factory resolves, checks and connects inside one function: whatever the name resolves to at connect time is what gets validated and used. Only the socket is replaced, so TLS still verifies against the original hostname. The up-front validate_fetch_url check stays as defense in depth. Tests cover a rebinding answer at connect time being refused and a real loopback round trip with the original Host header preserved; checked live: a real HTTPS feed fetches, 127.0.0.1/169.254.169.254/ localhost are blocked, and a wrong-hostname certificate is still rejected. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
204 lines
7.8 KiB
Python
204 lines
7.8 KiB
Python
"""Rejects feed-fetch targets that could turn this server/worker's own
|
|
network access into an SSRF vector.
|
|
|
|
Feed URLs are entirely attacker-controlled (any authenticated user via
|
|
"add feed", or an uploaded OPML file — see apps.feeds.opml) and this app
|
|
fetches them from the server's own network position. fetch_and_ingest
|
|
(apps.feeds.services.fetcher) is the single chokepoint every feed URL
|
|
passes through before any HTTP request is made, regardless of which
|
|
path created the Feed row, so enforcing this here covers all of them.
|
|
|
|
This validates the resolved address before connecting and again before
|
|
following each redirect hop, denying by default anything that isn't a
|
|
globally routable address (ip.is_global) — covering loopback, RFC1918
|
|
private ranges, link-local (including the 169.254.169.254 cloud
|
|
metadata address), and so on, for both IPv4 and IPv6.
|
|
|
|
The connection is pinned too (R2-S2): the address is resolved, checked and
|
|
connected to inside one function (_connect_validated, used by the adapter
|
|
below as the connection's socket factory), so a DNS answer that changes
|
|
between a separate validation lookup and the connect (DNS rebinding) can't
|
|
steer the connection to an internal address — whatever it resolves to *at
|
|
connect time* is what gets validated. TLS still verifies against the
|
|
original hostname, because only the socket is replaced, not the host the
|
|
TLS layer is told about. The up-front validate_fetch_url check stays as
|
|
defense in depth and gives a clear error before any connection attempt.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ipaddress
|
|
import socket
|
|
from dataclasses import dataclass
|
|
from urllib.parse import urljoin, urlparse
|
|
|
|
import requests
|
|
from requests.adapters import HTTPAdapter
|
|
from requests.structures import CaseInsensitiveDict
|
|
from urllib3.connection import HTTPConnection, HTTPSConnection
|
|
from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool
|
|
|
|
ALLOWED_SCHEMES = {"http", "https"}
|
|
MAX_REDIRECTS = 5
|
|
MAX_RESPONSE_BYTES = 10 * 1024 * 1024 # 10 MB — generous for an RSS/Atom feed
|
|
READ_CHUNK_BYTES = 65536
|
|
|
|
|
|
class SSRFBlocked(Exception):
|
|
"""Raised when a fetch target — or one of its redirects — isn't safe
|
|
to connect to on a user's behalf."""
|
|
|
|
|
|
@dataclass
|
|
class BoundedResponse:
|
|
status_code: int
|
|
# requests' own CaseInsensitiveDict — server header casing varies
|
|
# (ETag/Etag/etag) and fetcher.py looks these up by a fixed-case key.
|
|
headers: CaseInsensitiveDict
|
|
content: bytes
|
|
|
|
|
|
def _is_allowed_ip(ip) -> bool:
|
|
return ip.is_global
|
|
|
|
|
|
def validate_fetch_url(url: str) -> None:
|
|
"""Raises SSRFBlocked unless `url` has an http(s) scheme, a hostname,
|
|
and every address that hostname resolves to is globally routable."""
|
|
parsed = urlparse(url)
|
|
if parsed.scheme not in ALLOWED_SCHEMES:
|
|
raise SSRFBlocked(f"Unsupported URL scheme: {parsed.scheme!r}")
|
|
if not parsed.hostname:
|
|
raise SSRFBlocked("URL has no hostname")
|
|
|
|
try:
|
|
addrinfo = socket.getaddrinfo(parsed.hostname, None)
|
|
except OSError as exc:
|
|
raise SSRFBlocked(f"Could not resolve {parsed.hostname!r}: {exc}") from exc
|
|
|
|
for *_, sockaddr in addrinfo:
|
|
ip = ipaddress.ip_address(sockaddr[0])
|
|
if not _is_allowed_ip(ip):
|
|
raise SSRFBlocked(f"{parsed.hostname!r} resolves to a non-public address: {ip}")
|
|
|
|
|
|
def _connect_validated(host, port, timeout, source_address, socket_options):
|
|
"""Resolves `host`, requires every address to be globally routable, and
|
|
connects to one of those exact addresses — the same lookup result is
|
|
both validated and used, so there is nothing for a rebinding DNS
|
|
answer to change in between."""
|
|
try:
|
|
addrinfo = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
|
except OSError as exc:
|
|
raise SSRFBlocked(f"Could not resolve {host!r}: {exc}") from exc
|
|
|
|
candidates = []
|
|
for family, socktype, proto, _canonname, sockaddr in addrinfo:
|
|
ip = ipaddress.ip_address(sockaddr[0])
|
|
if not _is_allowed_ip(ip):
|
|
raise SSRFBlocked(f"{host!r} resolves to a non-public address: {ip}")
|
|
candidates.append((family, socktype, proto, sockaddr))
|
|
if not candidates:
|
|
raise SSRFBlocked(f"{host!r} did not resolve to any address")
|
|
|
|
last_error: OSError | None = None
|
|
for family, socktype, proto, sockaddr in candidates:
|
|
sock = None
|
|
try:
|
|
sock = socket.socket(family, socktype, proto)
|
|
for level, option, value in socket_options or ():
|
|
sock.setsockopt(level, option, value)
|
|
if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
|
|
sock.settimeout(timeout)
|
|
if source_address:
|
|
sock.bind(source_address)
|
|
sock.connect(sockaddr)
|
|
return sock
|
|
except OSError as exc:
|
|
last_error = exc
|
|
if sock is not None:
|
|
sock.close()
|
|
raise last_error or OSError(f"Could not connect to {host!r}")
|
|
|
|
|
|
class _PinnedHTTPConnection(HTTPConnection):
|
|
def _new_conn(self):
|
|
return _connect_validated(
|
|
self._dns_host, self.port, self.timeout, self.source_address, self.socket_options
|
|
)
|
|
|
|
|
|
class _PinnedHTTPSConnection(HTTPSConnection):
|
|
def _new_conn(self):
|
|
return _connect_validated(
|
|
self._dns_host, self.port, self.timeout, self.source_address, self.socket_options
|
|
)
|
|
|
|
|
|
class _PinnedHTTPConnectionPool(HTTPConnectionPool):
|
|
ConnectionCls = _PinnedHTTPConnection
|
|
|
|
|
|
class _PinnedHTTPSConnectionPool(HTTPSConnectionPool):
|
|
ConnectionCls = _PinnedHTTPSConnection
|
|
|
|
|
|
class PinnedAdapter(HTTPAdapter):
|
|
"""requests adapter whose connections go through _connect_validated."""
|
|
|
|
def init_poolmanager(self, *args, **kwargs):
|
|
super().init_poolmanager(*args, **kwargs)
|
|
self.poolmanager.pool_classes_by_scheme = {
|
|
"http": _PinnedHTTPConnectionPool,
|
|
"https": _PinnedHTTPSConnectionPool,
|
|
}
|
|
|
|
|
|
def safe_get(url: str, *, headers: dict, timeout: float) -> BoundedResponse:
|
|
"""GETs `url` the way fetch_and_ingest needs it: every redirect hop is
|
|
validated and followed manually (never left to requests/urllib3, so
|
|
a redirect can't bypass validate_fetch_url), capped at MAX_REDIRECTS,
|
|
and the body is read in bounded chunks rather than trusting
|
|
Content-Length or buffering an unbounded response.
|
|
"""
|
|
current_url = url
|
|
with requests.Session() as session:
|
|
adapter = PinnedAdapter()
|
|
session.mount("http://", adapter)
|
|
session.mount("https://", adapter)
|
|
for _ in range(MAX_REDIRECTS + 1):
|
|
validate_fetch_url(current_url)
|
|
response = session.get(
|
|
current_url,
|
|
headers=headers,
|
|
timeout=timeout,
|
|
stream=True,
|
|
allow_redirects=False,
|
|
)
|
|
try:
|
|
if response.is_redirect or response.is_permanent_redirect:
|
|
location = response.headers.get("Location")
|
|
if not location:
|
|
raise SSRFBlocked("Redirect response had no Location header")
|
|
current_url = urljoin(current_url, location)
|
|
continue
|
|
return BoundedResponse(
|
|
status_code=response.status_code,
|
|
headers=response.headers,
|
|
content=_read_bounded(response),
|
|
)
|
|
finally:
|
|
response.close()
|
|
raise SSRFBlocked(f"Too many redirects (> {MAX_REDIRECTS})")
|
|
|
|
|
|
def _read_bounded(response: requests.Response) -> bytes:
|
|
chunks = []
|
|
total = 0
|
|
for chunk in response.iter_content(chunk_size=READ_CHUNK_BYTES):
|
|
total += len(chunk)
|
|
if total > MAX_RESPONSE_BYTES:
|
|
raise SSRFBlocked(f"Response exceeded {MAX_RESPONSE_BYTES} bytes")
|
|
chunks.append(chunk)
|
|
return b"".join(chunks)
|