"""Rejects feed-fetch targets that could turn this server/worker's own network access into an SSRF vector. Feed URLs are entirely attacker-controlled (any authenticated user via "add feed", or an uploaded OPML file — see apps.feeds.opml) and this app fetches them from the server's own network position. fetch_and_ingest (apps.feeds.services.fetcher) is the single chokepoint every feed URL passes through before any HTTP request is made, regardless of which path created the Feed row, so enforcing this here covers all of them. This validates the resolved address before connecting and again before following each redirect hop, denying by default anything that isn't a globally routable address (ip.is_global) — covering loopback, RFC1918 private ranges, link-local (including the 169.254.169.254 cloud metadata address), and so on, for both IPv4 and IPv6. The connection is pinned too (R2-S2): the address is resolved, checked and connected to inside one function (_connect_validated, used by the adapter below as the connection's socket factory), so a DNS answer that changes between a separate validation lookup and the connect (DNS rebinding) can't steer the connection to an internal address — whatever it resolves to *at connect time* is what gets validated. TLS still verifies against the original hostname, because only the socket is replaced, not the host the TLS layer is told about. The up-front validate_fetch_url check stays as defense in depth and gives a clear error before any connection attempt. """ from __future__ import annotations import ipaddress import socket from dataclasses import dataclass from urllib.parse import urljoin, urlparse import requests from requests.adapters import HTTPAdapter from requests.structures import CaseInsensitiveDict from urllib3.connection import HTTPConnection, HTTPSConnection from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool ALLOWED_SCHEMES = {"http", "https"} MAX_REDIRECTS = 5 MAX_RESPONSE_BYTES = 10 * 1024 * 1024 # 10 MB — generous for an RSS/Atom feed READ_CHUNK_BYTES = 65536 class SSRFBlocked(Exception): """Raised when a fetch target — or one of its redirects — isn't safe to connect to on a user's behalf.""" @dataclass class BoundedResponse: status_code: int # requests' own CaseInsensitiveDict — server header casing varies # (ETag/Etag/etag) and fetcher.py looks these up by a fixed-case key. headers: CaseInsensitiveDict content: bytes def _is_allowed_ip(ip) -> bool: return ip.is_global def validate_fetch_url(url: str) -> None: """Raises SSRFBlocked unless `url` has an http(s) scheme, a hostname, and every address that hostname resolves to is globally routable.""" parsed = urlparse(url) if parsed.scheme not in ALLOWED_SCHEMES: raise SSRFBlocked(f"Unsupported URL scheme: {parsed.scheme!r}") if not parsed.hostname: raise SSRFBlocked("URL has no hostname") try: addrinfo = socket.getaddrinfo(parsed.hostname, None) except OSError as exc: raise SSRFBlocked(f"Could not resolve {parsed.hostname!r}: {exc}") from exc for *_, sockaddr in addrinfo: ip = ipaddress.ip_address(sockaddr[0]) if not _is_allowed_ip(ip): raise SSRFBlocked(f"{parsed.hostname!r} resolves to a non-public address: {ip}") def _connect_validated(host, port, timeout, source_address, socket_options): """Resolves `host`, requires every address to be globally routable, and connects to one of those exact addresses — the same lookup result is both validated and used, so there is nothing for a rebinding DNS answer to change in between.""" try: addrinfo = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) except OSError as exc: raise SSRFBlocked(f"Could not resolve {host!r}: {exc}") from exc candidates = [] for family, socktype, proto, _canonname, sockaddr in addrinfo: ip = ipaddress.ip_address(sockaddr[0]) if not _is_allowed_ip(ip): raise SSRFBlocked(f"{host!r} resolves to a non-public address: {ip}") candidates.append((family, socktype, proto, sockaddr)) if not candidates: raise SSRFBlocked(f"{host!r} did not resolve to any address") last_error: OSError | None = None for family, socktype, proto, sockaddr in candidates: sock = None try: sock = socket.socket(family, socktype, proto) for level, option, value in socket_options or (): sock.setsockopt(level, option, value) if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: sock.settimeout(timeout) if source_address: sock.bind(source_address) sock.connect(sockaddr) return sock except OSError as exc: last_error = exc if sock is not None: sock.close() raise last_error or OSError(f"Could not connect to {host!r}") class _PinnedHTTPConnection(HTTPConnection): def _new_conn(self): return _connect_validated( self._dns_host, self.port, self.timeout, self.source_address, self.socket_options ) class _PinnedHTTPSConnection(HTTPSConnection): def _new_conn(self): return _connect_validated( self._dns_host, self.port, self.timeout, self.source_address, self.socket_options ) class _PinnedHTTPConnectionPool(HTTPConnectionPool): ConnectionCls = _PinnedHTTPConnection class _PinnedHTTPSConnectionPool(HTTPSConnectionPool): ConnectionCls = _PinnedHTTPSConnection class PinnedAdapter(HTTPAdapter): """requests adapter whose connections go through _connect_validated.""" def init_poolmanager(self, *args, **kwargs): super().init_poolmanager(*args, **kwargs) self.poolmanager.pool_classes_by_scheme = { "http": _PinnedHTTPConnectionPool, "https": _PinnedHTTPSConnectionPool, } def safe_get(url: str, *, headers: dict, timeout: float) -> BoundedResponse: """GETs `url` the way fetch_and_ingest needs it: every redirect hop is validated and followed manually (never left to requests/urllib3, so a redirect can't bypass validate_fetch_url), capped at MAX_REDIRECTS, and the body is read in bounded chunks rather than trusting Content-Length or buffering an unbounded response. """ current_url = url with requests.Session() as session: adapter = PinnedAdapter() session.mount("http://", adapter) session.mount("https://", adapter) for _ in range(MAX_REDIRECTS + 1): validate_fetch_url(current_url) response = session.get( current_url, headers=headers, timeout=timeout, stream=True, allow_redirects=False, ) try: if response.is_redirect or response.is_permanent_redirect: location = response.headers.get("Location") if not location: raise SSRFBlocked("Redirect response had no Location header") current_url = urljoin(current_url, location) continue return BoundedResponse( status_code=response.status_code, headers=response.headers, content=_read_bounded(response), ) finally: response.close() raise SSRFBlocked(f"Too many redirects (> {MAX_REDIRECTS})") def _read_bounded(response: requests.Response) -> bytes: chunks = [] total = 0 for chunk in response.iter_content(chunk_size=READ_CHUNK_BYTES): total += len(chunk) if total > MAX_RESPONSE_BYTES: raise SSRFBlocked(f"Response exceeded {MAX_RESPONSE_BYTES} bytes") chunks.append(chunk) return b"".join(chunks)