From a44b8a618c6a2a5d89a2bcb242336a0eaf7dd946 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 12:23:36 +0200 Subject: [PATCH 01/31] ref(helpers): Extract repository checks from initialize_provider The git relay needs the same not-found, inactive and organization checks on a repository, but it gets a token instead of a provider, so it cannot go through initialize_provider. Co-Authored-By: Claude Opus 5.5 --- src/scm/helpers.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/scm/helpers.py b/src/scm/helpers.py index 67c3d7db..fb24811a 100644 --- a/src/scm/helpers.py +++ b/src/scm/helpers.py @@ -35,13 +35,13 @@ def iter_all_pages[T]( cursor = next_cursor -def initialize_provider( +def resolve_repository( organization_id: int, repository_id: RepositoryId, *, fetch_repository: Callable[[int, RepositoryId], Repository | None], - fetch_provider: Callable[[int, Repository], Provider | None], -) -> Provider: +) -> Repository: + """Return the repository only if it exists, is active and belongs to the organization.""" repository = fetch_repository(organization_id, repository_id) if not repository: raise RepositoryNotFound(organization_id=organization_id, repository_id=repository_id) @@ -49,6 +49,17 @@ def initialize_provider( raise RepositoryInactive(organization_id=organization_id, repository_id=repository_id) if repository["organization_id"] != organization_id: raise RepositoryOrganizationMismatch(organization_id=organization_id, repository_id=repository_id) + return repository + + +def initialize_provider( + organization_id: int, + repository_id: RepositoryId, + *, + fetch_repository: Callable[[int, RepositoryId], Repository | None], + fetch_provider: Callable[[int, Repository], Provider | None], +) -> Provider: + repository = resolve_repository(organization_id, repository_id, fetch_repository=fetch_repository) provider = fetch_provider(organization_id, repository) if provider is None: From f95a5704b75f787b4c41e42a11df82de9754f7a8 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 12:23:36 +0200 Subject: [PATCH 02/31] feat(git): Add git grants and a streaming git relay Coding-job sandboxes need to clone and push without holding provider credentials. Seer and Sentry both depend on sentry-scm, so the pieces both sides of the git endpoint share live here: Seer signs grants with scm.git.grant, and Sentry's view hands each request to GitServer, which verifies the grant and checks the route and repository against it before it looks up the repository or asks for a token. The relay streams both bodies without parsing them and reuses a keep-alive connection per provider host, because a new connection per request cost 0.4-0.8s per clone in the prototype (getsentry/sandbox-router#145). It blocks cookies, never follows redirects, and drops requests' exceptions so the Authorization header cannot reach an error report. Refs CW-2221 Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- src/scm/errors.py | 18 ++ src/scm/git/grant.py | 75 +++++++ src/scm/git/operations.py | 33 +++ src/scm/git/relay.py | 171 ++++++++++++++ src/scm/git/remote.py | 33 +++ src/scm/git/server.py | 98 +++++++++ src/scm/git/types.py | 59 +++++ src/scm/rpc/errors.py | 2 + tests/unit/git_upstream.py | 121 ++++++++++ tests/unit/test_git_grant.py | 161 ++++++++++++++ tests/unit/test_git_operations.py | 95 ++++++++ tests/unit/test_git_relay.py | 355 ++++++++++++++++++++++++++++++ tests/unit/test_git_remote.py | 62 ++++++ tests/unit/test_git_server.py | 257 +++++++++++++++++++++ 15 files changed, 1542 insertions(+), 1 deletion(-) create mode 100644 src/scm/git/grant.py create mode 100644 src/scm/git/operations.py create mode 100644 src/scm/git/relay.py create mode 100644 src/scm/git/remote.py create mode 100644 src/scm/git/server.py create mode 100644 src/scm/git/types.py create mode 100644 tests/unit/git_upstream.py create mode 100644 tests/unit/test_git_grant.py create mode 100644 tests/unit/test_git_operations.py create mode 100644 tests/unit/test_git_relay.py create mode 100644 tests/unit/test_git_remote.py create mode 100644 tests/unit/test_git_server.py diff --git a/README.md b/README.md index 1777685b..2547b6f5 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,11 @@ The SCM (Source Code Management) platform is a vendor-agnostic abstraction layer ### Features -The platform exposes three subsystems: +The platform exposes four subsystems: - **Actions** — outbound SCM operations initiated by Sentry code. The `SourceCodeManager` class provides 70+ methods covering comments, reactions, pull requests, branches, git objects, reviews, and check runs. With more actions planned to be added as we port more use cases. - **Actions RPC** - outbound SCM commands exposed over the network. The `SourceCodeManager` is fully available over the network. This enables usage of the SCM for services outside the monolith. +- **Git relay** - `git` fetches and pushes for services that hold no provider credentials. Seer signs a grant naming the organization, operation and repositories with `scm.git.grant.sign_grant`, and `scm.git.server.GitServer` checks each git smart-HTTP request against it before relaying the request to the provider with a token, streaming both bodies. - **Event Stream** — inbound webhook processing. SCM service-providers push events which are deserialized into typed, provider-neutral dataclasses (`CheckRunEvent`, `CommentEvent`, `PullRequestEvent`) which are then dispatched to registered listener functions. ### Why diff --git a/src/scm/errors.py b/src/scm/errors.py index c4691d3e..e0c357f0 100644 --- a/src/scm/errors.py +++ b/src/scm/errors.py @@ -1,6 +1,9 @@ from typing import ClassVar, Literal type ErrorCode = Literal[ + "git_invalid_grant", + "git_provider_not_supported", + "git_route_not_allowed", "malformed_external_id", "path_is_directory", "path_is_not_directory", @@ -37,6 +40,9 @@ ] ERROR_CODES: dict[ErrorCode, str] = { + "git_invalid_grant": "The git grant was missing, malformed, forged or expired.", + "git_provider_not_supported": "The repository's provider has no git remote.", + "git_route_not_allowed": "The git grant does not allow this request.", "malformed_external_id": "The repository's external ID was malformed.", "path_is_directory": "The requested path is a directory, not a file.", "path_is_not_directory": "The requested path is not a directory.", @@ -123,6 +129,18 @@ def from_code(cls, code: ErrorCode, detail: str | None = None) -> "SCMCodedError return error_cls(detail=detail) +class GitInvalidGrant(SCMCodedError): + code = "git_invalid_grant" + + +class GitProviderNotSupported(SCMCodedError): + code = "git_provider_not_supported" + + +class GitRouteNotAllowed(SCMCodedError): + code = "git_route_not_allowed" + + class MalformedExternalId(SCMCodedError): code = "malformed_external_id" diff --git a/src/scm/git/grant.py b/src/scm/git/grant.py new file mode 100644 index 00000000..562b7bfb --- /dev/null +++ b/src/scm/git/grant.py @@ -0,0 +1,75 @@ +"""Signing and verifying git grants. + +The wire format is ``.``, where the payload is the grant's JSON and the +signature is an HMAC-SHA256 over a fixed prefix and the payload. The prefix keeps a grant signature from ever +matching a signature the RPC endpoint makes with the same secret over some other message. +""" + +import base64 +import hashlib +import hmac +import re +import time + +import msgspec + +from scm.errors import GitInvalidGrant +from scm.git.types import Grant + +_SIGNATURE_DOMAIN = b"scm-git-grant." +_BASE64URL = re.compile(r"[A-Za-z0-9_-]+") + + +def sign_grant(secret: str, grant: Grant) -> str: + payload = msgspec.json.encode(grant) + return f"{_b64encode(payload)}.{_b64encode(_signature(secret, payload))}" + + +def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> Grant: + """Return the grant if any of ``secrets`` signed it and it has not expired. + + Verification takes a list of secrets, as ``RpcServer`` does, so a new secret can be added before signers + switch to it and the old one removed after their grants expire. + """ + if not token: + raise GitInvalidGrant(detail="The git grant is missing.") + try: + payload_b64, signature_b64 = token.split(".") + payload = _b64decode(payload_b64) + signature = _b64decode(signature_b64) + except ValueError: # binascii.Error is a ValueError too + raise GitInvalidGrant(detail="The git grant is malformed.") from None + + # Check every secret, not just until one matches, so the time taken does not reveal which one signed it. + matches = [hmac.compare_digest(signature, _signature(secret, payload)) for secret in secrets] + if not any(matches): + raise GitInvalidGrant(detail="The git grant's signature does not match.") + + try: + grant = msgspec.json.decode(payload, type=Grant) + except msgspec.ValidationError as e: + # Only a holder of the secret can get here, so the reason is safe to return and saves a debugging round. + raise GitInvalidGrant(detail=f"The git grant's payload is invalid: {e}") from None + except msgspec.DecodeError: + raise GitInvalidGrant(detail="The git grant's payload is not JSON.") from None + + if grant.expires_at <= (time.time() if now is None else now): + raise GitInvalidGrant(detail="The git grant has expired.") + if not grant.repositories: + raise GitInvalidGrant(detail="The git grant names no repositories.") + return grant + + +def _signature(secret: str, payload: bytes) -> bytes: + return hmac.new(secret.encode(), _SIGNATURE_DOMAIN + payload, hashlib.sha256).digest() + + +def _b64encode(data: bytes) -> str: + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") + + +def _b64decode(data: str) -> bytes: + # urlsafe_b64decode skips characters outside its alphabet; refusing them keeps one token to one decoding. + if not _BASE64URL.fullmatch(data): + raise ValueError("not base64url") + return base64.urlsafe_b64decode(data + "=" * (-len(data) % 4)) diff --git a/src/scm/git/operations.py b/src/scm/git/operations.py new file mode 100644 index 00000000..7826f3b7 --- /dev/null +++ b/src/scm/git/operations.py @@ -0,0 +1,33 @@ +from scm.errors import GitRouteNotAllowed +from scm.git.types import GitOperation, Grant +from scm.types import RepositoryId + +# The git smart-HTTP routes, keyed by (method, path below .git/, query string). git sends these exact query +# strings, so an exact match leaves nothing for the provider to read differently than this table does. +_ROUTES: dict[tuple[str, str, str], GitOperation] = { + ("GET", "info/refs", "service=git-upload-pack"): "fetch", + ("POST", "git-upload-pack", ""): "fetch", + ("GET", "info/refs", "service=git-receive-pack"): "push", + ("POST", "git-receive-pack", ""): "push", +} + + +def route_operation(method: str, route: str, query: str) -> GitOperation | None: + """Return the operation a git smart-HTTP request belongs to, or None if it is not one.""" + return _ROUTES.get((method, route, query)) + + +def check_request(grant: Grant, repository_id: RepositoryId, method: str, route: str, query: str) -> GitOperation: + """Refuse a request the grant does not reach, before anything is sent to the provider. + + A fetch grant reaches only upload-pack and a push grant only receive-pack, and the repository must be one the + grant lists, named the same way. + """ + operation = route_operation(method, route, query) + if operation is None: + raise GitRouteNotAllowed(detail=f"{method} {route} is not a git fetch or push route.") + if operation != grant.operation: + raise GitRouteNotAllowed(detail=f"The git grant allows {grant.operation}, not {operation}.") + if repository_id not in grant.repositories: + raise GitRouteNotAllowed(detail="The git grant does not cover this repository.") + return operation diff --git a/src/scm/git/relay.py b/src/scm/git/relay.py new file mode 100644 index 00000000..4826e0ee --- /dev/null +++ b/src/scm/git/relay.py @@ -0,0 +1,171 @@ +from collections.abc import Iterator, Mapping +from http.cookiejar import DefaultCookiePolicy + +import requests +from requests.adapters import HTTPAdapter +from requests.auth import HTTPBasicAuth +from requests.structures import CaseInsensitiveDict + +from scm.errors import ( + ClientClosedRequest, + GitRouteNotAllowed, + ResourceGatewayTimeout, + ResourceServiceUnavailable, + RpcMalformedRequestHeaders, +) +from scm.git.operations import route_operation +from scm.git.types import GitRemote, Readable +from scm.rpc.types import StreamResponse + +DEFAULT_CHUNK_SIZE = 64 * 1024 +# A large clone streams its pack for minutes, but the provider sends something well within the read timeout. +DEFAULT_TIMEOUT = (10.0, 300.0) + +# What git sends that the provider needs: protocol v2 is negotiated with Git-Protocol, and git gzips large fetch +# requests. Everything else stays behind, including the caller's own credentials and the grant. +_REQUEST_HEADERS = frozenset( + {"accept", "accept-encoding", "content-encoding", "content-type", "git-protocol", "user-agent"} +) +# Bodies pass through still encoded, so Content-Encoding and Content-Length describe what the client receives. +# Location is left out because a redirect would send the client to the provider directly. +_RESPONSE_HEADERS = frozenset( + {"cache-control", "content-encoding", "content-length", "content-type", "expires", "pragma"} +) + + +class GitRelay: + """Forwards git smart-HTTP requests to a provider with a token, streaming the body in both directions. + + Nothing buffers a request or a response, and neither body is parsed. Build one per process and reuse it: it + keeps a pool of keep-alive connections per provider host, because a clone makes three requests and a new + connection for each cost 0.4-0.8s per clone in the prototype. + """ + + def __init__( + self, + *, + max_hosts: int = 32, + max_connections_per_host: int = 64, + timeout: tuple[float, float] = DEFAULT_TIMEOUT, + chunk_size: int = DEFAULT_CHUNK_SIZE, + ) -> None: + self.timeout = timeout + self.chunk_size = chunk_size + self.session = requests.Session() + # The session is shared by every organization's requests, so it must not carry a provider's cookies from + # one request to the next. + self.session.cookies.set_policy(DefaultCookiePolicy(allowed_domains=[])) + adapter = HTTPAdapter(pool_connections=max_hosts, pool_maxsize=max_connections_per_host, max_retries=0) + self.session.mount("https://", adapter) + self.session.mount("http://", adapter) + + def forward( + self, + remote: GitRemote, + token: str, + *, + method: str, + route: str, + query: str, + headers: Mapping[str, str], + body: Readable | None, + ) -> StreamResponse: + """Send one request to ``remote`` and return the provider's response as it arrives. + + ``route`` is the path below ``.git/``. Check the request against a grant before calling this. + """ + # The caller has checked the operation; this only keeps anything but a git route away from the token. + if route_operation(method, route, query) is None: + raise GitRouteNotAllowed(detail=f"{method} {route} is not a git fetch or push route.") + + client_headers = CaseInsensitiveDict(headers) + upstream_headers = CaseInsensitiveDict( + {name: value for name, value in client_headers.items() if name.lower() in _REQUEST_HEADERS} + ) + # requests asks for gzip and more by default, which the client may not decode once the body is passed on. + upstream_headers.setdefault("Accept-Encoding", "identity") + + url = f"{remote.url}/{route}?{query}" if query else f"{remote.url}/{route}" + try: + upstream = self.session.request( + method, + url, + headers=upstream_headers, + data=_request_body(body, _content_length(client_headers), self.chunk_size), + # Passing auth also stops requests from replacing it with credentials from a .netrc file. + auth=HTTPBasicAuth(remote.username, token), + stream=True, + # A redirect would carry the token to wherever the provider points. + allow_redirects=False, + timeout=self.timeout, + ) + # "from None" drops requests' exception, whose request holds the Authorization header. + except requests.Timeout: + raise ResourceGatewayTimeout(detail=f"{remote.url} did not respond in time.") from None + except requests.ConnectionError: + raise ResourceServiceUnavailable(detail=f"Could not connect to {remote.url}.") from None + + return StreamResponse( + status_code=upstream.status_code, + headers={name: value for name, value in upstream.headers.items() if name.lower() in _RESPONSE_HEADERS}, + content=self._stream(upstream), + ) + + def close(self) -> None: + self.session.close() + + def _stream(self, upstream: requests.Response) -> Iterator[bytes]: + # Closing the response once it is read to the end returns its connection to the pool; closing it early, + # because the client went away, drops the connection instead. + with upstream: + yield from filter(bool, upstream.raw.stream(self.chunk_size, decode_content=False)) + + +def _content_length(headers: CaseInsensitiveDict[str]) -> int | None: + raw = headers.get("Content-Length") + if raw is None: + return None + if not raw.isdigit(): + raise RpcMalformedRequestHeaders(detail="Could not parse Content-Length header") + return int(raw) + + +def _request_body( + body: Readable | None, content_length: int | None, chunk_size: int +) -> "Iterator[bytes] | _SizedBody | None": + if body is None or content_length == 0: + return None + if content_length is None: + # requests sends a generator with chunked transfer encoding, the way git sends a large push. + return _read_until_eof(body, chunk_size) + return _SizedBody(body, content_length, chunk_size) + + +def _read_until_eof(body: Readable, chunk_size: int) -> Iterator[bytes]: + while chunk := body.read(chunk_size): + yield chunk + + +class _SizedBody: + """A body of known length, read as it is sent. + + requests sends Content-Length for anything with a ``__len__`` and streams anything iterable, so this passes + the client's length on without reading the body first. Reading stops at the declared length. + """ + + def __init__(self, body: Readable, length: int, chunk_size: int) -> None: + self.body = body + self.length = length + self.chunk_size = chunk_size + + def __len__(self) -> int: + return self.length + + def __iter__(self) -> Iterator[bytes]: + remaining = self.length + while remaining > 0: + chunk = self.body.read(min(self.chunk_size, remaining)) + if not chunk: + raise ClientClosedRequest(detail="The client closed the request before sending its whole body.") + remaining -= len(chunk) + yield chunk diff --git a/src/scm/git/remote.py b/src/scm/git/remote.py new file mode 100644 index 00000000..a80d52bd --- /dev/null +++ b/src/scm/git/remote.py @@ -0,0 +1,33 @@ +from scm.errors import GitProviderNotSupported, MalformedExternalId +from scm.git.types import GitRemote +from scm.types import Repository + +GITHUB_GIT_BASE_URL = "https://github.com" + + +def git_remote(repository: Repository) -> GitRemote: + """Return the git endpoint and basic-auth username for a repository's provider. + + Bitbucket Cloud (``x-token-auth``) and Azure DevOps (a bearer header) follow the same pattern once the + platform supports them as providers. + """ + match repository["provider_name"]: + case "github": + return GitRemote(url=f"{GITHUB_GIT_BASE_URL}/{repository['name']}.git", username="x-access-token") + case "github_enterprise": + if not repository["web_base_url"]: + raise GitProviderNotSupported(detail="The GitHub Enterprise repository has no web_base_url.") + base_url = repository["web_base_url"].rstrip("/") + return GitRemote(url=f"{base_url}/{repository['name']}.git", username="x-access-token") + case "gitlab": + # Sentry leaves web_base_url unset for GitLab; the host is in the external id, ":", + # and the name is the namespace path with spaces around its slashes. GitLabProvider reads both the same + # way. + external_id = repository["external_id"] + if external_id is None or ":" not in external_id: + raise MalformedExternalId() + netloc = external_id.rsplit(":", maxsplit=1)[0] + path = repository["name"].replace(" ", "") + return GitRemote(url=f"https://{netloc}/{path}.git", username="oauth2") + case provider_name: + raise GitProviderNotSupported(detail=f"The {provider_name} provider has no git remote.") diff --git a/src/scm/git/server.py b/src/scm/git/server.py new file mode 100644 index 00000000..a3c4f331 --- /dev/null +++ b/src/scm/git/server.py @@ -0,0 +1,98 @@ +from collections.abc import Callable, Mapping + +from requests.structures import CaseInsensitiveDict + +from scm.errors import ProviderNotFound, SCMCodedError +from scm.git.grant import verify_grant +from scm.git.operations import check_request +from scm.git.relay import GitRelay +from scm.git.remote import git_remote +from scm.git.types import GRANT_HEADER, GetToken, Readable +from scm.helpers import resolve_repository +from scm.rpc.errors import SPECIAL_STATUS_MAP +from scm.rpc.types import StreamResponse +from scm.types import Repository, RepositoryId + + +class GitServer: + """Serves a git endpoint: checks each request against its grant, then relays it with a provider token. + + The order is the security boundary. The grant is verified and the request's route and repository checked + against it before the repository is looked up or a token fetched, so a request the grant does not reach + never touches the provider. Build one per process: its relay pools upstream connections. + """ + + def __init__( + self, + secrets: list[str], + fetch_repository: Callable[[int, RepositoryId], Repository | None], + get_token: GetToken, + record_count: Callable[[str, int, dict[str, str]], None], + emit_error: Callable[[Exception], None], + relay: GitRelay | None = None, + ) -> None: + self.secrets = secrets + self.fetch_repository = fetch_repository + self.get_token = get_token + self.record_count = record_count + self.emit_error = emit_error + self.relay = relay or GitRelay() + + def handle( + self, + *, + method: str, + repository_id: RepositoryId, + route: str, + query: str, + headers: Mapping[str, str], + body: Readable | None, + ) -> StreamResponse: + """Answer one git request. + + ``route`` is the path below ``.git/`` and ``query`` the raw query string. ``body`` is read only + while the request is sent upstream, so pass the unread request stream. + """ + try: + grant = verify_grant(self.secrets, CaseInsensitiveDict(headers).get(GRANT_HEADER, "")) + operation = check_request(grant, repository_id, method, route, query) + repository = resolve_repository( + grant.organization_id, repository_id, fetch_repository=self.fetch_repository + ) + remote = git_remote(repository) + token = self.get_token(repository, operation) + if token is None: + raise ProviderNotFound(detail="No git credentials are available for this repository.") + response = self.relay.forward( + remote, token, method=method, route=route, query=query, headers=headers, body=body + ) + except SCMCodedError as e: + self.record_count("sentry.scm.git.request.failed", 1, {"code": e.code}) + if e.code == "unhandled_exception": + self.emit_error(e) + return _error_response(e) + + self.record_count( + "sentry.scm.git.request.succeeded", + 1, + { + "operation": operation, + "provider": repository["provider_name"], + "status": str(response.status_code), + }, + ) + return response + + +def _error_response(error: SCMCodedError) -> StreamResponse: + # git asks for credentials on a 401, which a sandboxed client must never have, so a refusal is a 403. + status = SPECIAL_STATUS_MAP.get(error.code, 400) + if status == 401: + status = 403 + # When its first request fails, git shows a text/plain body to the user as "remote: ". + content = f"{error.detail}\n".encode() + return StreamResponse( + status_code=status, + headers={"Content-Type": "text/plain; charset=utf-8", "Content-Length": str(len(content))}, + content=iter([content]), + ) diff --git a/src/scm/git/types.py b/src/scm/git/types.py new file mode 100644 index 00000000..0e431c97 --- /dev/null +++ b/src/scm/git/types.py @@ -0,0 +1,59 @@ +import dataclasses +from collections.abc import Callable +from typing import Literal, Protocol + +import msgspec + +from scm.types import Repository, RepositoryId + +type GitOperation = Literal["fetch", "push"] +"""What a grant lets its holder do: fetch reaches git-upload-pack, push reaches git-receive-pack.""" + +GRANT_VERSION: Literal[1] = 1 + +GRANT_HEADER = "X-Scm-Git-Grant" +"""The request header that carries a signed grant to the git endpoint.""" + + +class Grant(msgspec.Struct, frozen=True, kw_only=True, forbid_unknown_fields=True): + """Permission to run one git operation against a fixed set of an organization's repositories. + + Seer signs a grant per coding job and Sentry's git endpoint verifies it on every request. Each repository + is named the way the RPC endpoint names one, as a ``RepositoryId`` under the grant's organization. A fetch + grant holds one repository today; the list leaves room for submodules and git-based dependencies without a + new format. + + Unknown fields are refused rather than ignored, so a field a newer signer adds to narrow a grant can never + be dropped by an older verifier that would then allow more than the signer meant. + """ + + organization_id: int + operation: GitOperation + repositories: tuple[RepositoryId, ...] + expires_at: int + """Unix time in seconds. Every grant expires.""" + version: Literal[1] = GRANT_VERSION + + +@dataclasses.dataclass(frozen=True) +class GitRemote: + """Where a repository's git smart-HTTP endpoint lives and how a token authenticates against it.""" + + url: str + """The repository's clone URL, e.g. ``https://github.com/getsentry/sentry.git``.""" + username: str + """The basic-auth username the provider pairs with a token, e.g. ``x-access-token`` on GitHub.""" + + +type GetToken = Callable[[Repository, GitOperation], str | None] +"""Return a provider token that can run ``operation`` on ``repository``, or None if there is none. + +The relay never builds credentials itself, so Sentry's integration token getters today and a central token +mint later plug in without changing it. The operation lets a mint scope the token, e.g. read-only for a fetch. +""" + + +class Readable(Protocol): + """A request body the relay pulls from as it sends upstream, e.g. a WSGI input or a Django request.""" + + def read(self, size: int = -1, /) -> bytes: ... diff --git a/src/scm/rpc/errors.py b/src/scm/rpc/errors.py index 10b7a6df..a8ccf239 100644 --- a/src/scm/rpc/errors.py +++ b/src/scm/rpc/errors.py @@ -4,6 +4,8 @@ from scm.rpc.types import Error, ErrorResponse SPECIAL_STATUS_MAP: dict[ErrorCode, int] = { + "git_invalid_grant": 403, + "git_route_not_allowed": 403, "provider_not_found": 404, "rate_limit_exceeded": 429, "repository_inactive": 404, diff --git a/tests/unit/git_upstream.py b/tests/unit/git_upstream.py new file mode 100644 index 00000000..3b5acefc --- /dev/null +++ b/tests/unit/git_upstream.py @@ -0,0 +1,121 @@ +"""A local HTTP/1.1 server standing in for a provider's git endpoint in relay tests.""" + +import base64 +import threading +from collections.abc import Callable, Iterator +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from requests.structures import CaseInsensitiveDict + + +@dataclass +class ReceivedRequest: + method: str + path: str + headers: CaseInsensitiveDict[str] + body: bytes + + +type Handler = Callable[["UpstreamHandler"], None] + + +@dataclass +class Upstream: + url: str + requests: list[ReceivedRequest] = field(default_factory=list) + connections: int = 0 + handler: Handler | None = None + + +class UpstreamHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + server: "_Server" + + def setup(self) -> None: + super().setup() + self.server.upstream.connections += 1 + + def do_GET(self) -> None: + self._dispatch() + + def do_POST(self) -> None: + self._dispatch() + + def _dispatch(self) -> None: + handler = self.server.upstream.handler or echo + handler(self) + + def body_chunks(self) -> Iterator[bytes]: + """Yield the request body as it arrives, one HTTP chunk or one read at a time.""" + if self.headers.get("Transfer-Encoding") == "chunked": + while (size := int(self.rfile.readline().strip(), 16)) > 0: + yield self.rfile.read(size) + self.rfile.readline() + self.rfile.readline() + else: + remaining = int(self.headers.get("Content-Length", "0")) + while remaining > 0: + chunk = self.rfile.read1(remaining) # type: ignore[attr-defined] + remaining -= len(chunk) + yield chunk + + def record(self, body: bytes) -> None: + self.server.upstream.requests.append( + ReceivedRequest( + method=self.command, path=self.path, headers=CaseInsensitiveDict(self.headers.items()), body=body + ) + ) + + def respond(self, status: int, body: bytes, headers: dict[str, str] | None = None) -> None: + self.send_response(status) + for name, value in (headers or {}).items(): + self.send_header(name, value) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def start_chunked(self, status: int, headers: dict[str, str] | None = None) -> None: + self.send_response(status) + for name, value in (headers or {}).items(): + self.send_header(name, value) + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + + def write_chunk(self, data: bytes) -> None: + self.wfile.write(f"{len(data):x}\r\n".encode() + data + b"\r\n") + self.wfile.flush() + + def log_message(self, format: str, *args: object) -> None: + pass + + +class _Server(ThreadingHTTPServer): + daemon_threads = True + upstream: Upstream + + +def echo(handler: UpstreamHandler) -> None: + body = b"".join(handler.body_chunks()) + handler.record(body) + handler.respond(200, body, {"Content-Type": "application/x-git-upload-pack-result"}) + + +def basic_auth(username: str, password: str) -> str: + return "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode() + + +class running_upstream: + """Context manager that serves an ``Upstream`` on a free local port.""" + + def __enter__(self) -> Upstream: + self._server = _Server(("127.0.0.1", 0), UpstreamHandler) + host, port = self._server.server_address[:2] + self._server.upstream = Upstream(url=f"http://{host!s}:{port}") + self._thread = threading.Thread(target=self._server.serve_forever, args=(0.01,), daemon=True) + self._thread.start() + return self._server.upstream + + def __exit__(self, *exc: object) -> None: + self._server.shutdown() + self._server.server_close() diff --git a/tests/unit/test_git_grant.py b/tests/unit/test_git_grant.py new file mode 100644 index 00000000..a5f5401b --- /dev/null +++ b/tests/unit/test_git_grant.py @@ -0,0 +1,161 @@ +import base64 +import hashlib +import hmac +import json + +import msgspec +import pytest + +from scm.errors import GitInvalidGrant +from scm.git.grant import sign_grant, verify_grant +from scm.git.types import Grant +from scm.rpc.helpers import sign_post + +NOW = 1_800_000_000 +SECRET = "secret" + + +def make_grant(**overrides) -> Grant: + fields = { + "organization_id": 1, + "operation": "fetch", + "repositories": (5, ("github", "123")), + "expires_at": NOW + 600, + } + fields.update(overrides) + return Grant(**fields) # type: ignore[arg-type] + + +def b64(data: bytes) -> str: + return base64.urlsafe_b64encode(data).rstrip(b"=").decode() + + +def sign_payload(payload: bytes, secret: str = SECRET) -> str: + """Sign arbitrary bytes the way sign_grant does, to reach the checks that run after the signature.""" + signature = hmac.new(secret.encode(), b"scm-git-grant." + payload, hashlib.sha256).digest() + return f"{b64(payload)}.{b64(signature)}" + + +def payload_of(grant: Grant) -> dict: + return json.loads(msgspec.json.encode(grant)) + + +def assert_invalid(token: str, detail: str, secrets: list[str] | None = None) -> None: + with pytest.raises(GitInvalidGrant) as exc_info: + verify_grant(secrets or [SECRET], token, now=NOW) + assert detail in exc_info.value.detail + + +class TestRoundTrip: + def test_verifies_what_it_signed(self): + grant = make_grant() + assert verify_grant([SECRET], sign_grant(SECRET, grant), now=NOW) == grant + + def test_keeps_both_repository_id_forms(self): + grant = verify_grant([SECRET], sign_grant(SECRET, make_grant()), now=NOW) + assert grant.repositories == (5, ("github", "123")) + + def test_push_grant(self): + grant = make_grant(operation="push") + assert verify_grant([SECRET], sign_grant(SECRET, grant), now=NOW).operation == "push" + + def test_payload_carries_the_version(self): + token = sign_grant(SECRET, make_grant()) + payload = json.loads(base64.urlsafe_b64decode(token.split(".")[0] + "==")) + assert payload["version"] == 1 + + +class TestExpiry: + def test_expired(self): + assert_invalid(sign_grant(SECRET, make_grant(expires_at=NOW - 1)), "expired") + + def test_expires_at_its_expiry_time(self): + assert_invalid(sign_grant(SECRET, make_grant(expires_at=NOW)), "expired") + + def test_valid_until_its_expiry_time(self): + token = sign_grant(SECRET, make_grant(expires_at=NOW + 1)) + assert verify_grant([SECRET], token, now=NOW).expires_at == NOW + 1 + + def test_expiry_is_required(self): + payload = payload_of(make_grant()) + del payload["expires_at"] + assert_invalid(sign_payload(json.dumps(payload).encode()), "expires_at") + + +class TestTampering: + def test_changed_payload(self): + token = sign_grant(SECRET, make_grant()) + _, signature = token.split(".") + forged = msgspec.json.encode(make_grant(organization_id=2)) + assert_invalid(f"{b64(forged)}.{signature}", "signature does not match") + + def test_changed_signature(self): + payload, signature = sign_grant(SECRET, make_grant()).split(".") + flipped = bytearray(base64.urlsafe_b64decode(signature + "=")) + flipped[0] ^= 1 + assert_invalid(f"{payload}.{b64(bytes(flipped))}", "signature does not match") + + def test_other_secret(self): + assert_invalid(sign_grant("other", make_grant()), "signature does not match") + + def test_no_secrets(self): + with pytest.raises(GitInvalidGrant): + verify_grant([], sign_grant(SECRET, make_grant()), now=NOW) + + def test_rpc_signature_over_the_same_payload_is_not_a_grant_signature(self): + payload = msgspec.json.encode(make_grant()) + rpc_signature = bytes.fromhex(sign_post(SECRET, payload).removeprefix("rpc0:")) + assert_invalid(f"{b64(payload)}.{b64(rpc_signature)}", "signature does not match") + + +class TestRotatedSecrets: + def test_old_secret_verifies_during_rotation(self): + assert verify_grant(["new", "old"], sign_grant("old", make_grant()), now=NOW) == make_grant() + + def test_new_secret_verifies_during_rotation(self): + assert verify_grant(["new", "old"], sign_grant("new", make_grant()), now=NOW) == make_grant() + + def test_old_secret_refused_after_rotation(self): + assert_invalid(sign_grant("old", make_grant()), "signature does not match", secrets=["new"]) + + +class TestMalformed: + @pytest.mark.parametrize( + "token", + [ + "nodot", + "a.b.c", + "abc.d+f", + "ab=c.def", + "abc.dé", + ".", + ], + ) + def test_malformed_token(self, token): + assert_invalid(token, "malformed") + + def test_missing(self): + assert_invalid("", "missing") + + def test_signed_but_not_json(self): + assert_invalid(sign_payload(b"not json"), "not JSON") + + @pytest.mark.parametrize( + "change", + [ + {"organization_id": True}, + {"organization_id": "1"}, + {"operation": "delete"}, + {"version": 2}, + {"repositories": [["github"]]}, + {"repositories": [["unknown_provider", "123"]]}, + {"expires_at": 1.5}, + {"narrower_than_you_think": True}, + ], + ) + def test_signed_but_invalid_payload(self, change): + payload = payload_of(make_grant()) | change + assert_invalid(sign_payload(json.dumps(payload).encode()), "payload is invalid") + + def test_no_repositories(self): + assert_invalid(sign_grant(SECRET, make_grant(repositories=())), "names no repositories") diff --git a/tests/unit/test_git_operations.py b/tests/unit/test_git_operations.py new file mode 100644 index 00000000..d6216104 --- /dev/null +++ b/tests/unit/test_git_operations.py @@ -0,0 +1,95 @@ +import pytest + +from scm.errors import GitRouteNotAllowed +from scm.git.operations import check_request, route_operation +from scm.git.types import Grant + + +def make_grant(**overrides) -> Grant: + fields = {"organization_id": 1, "operation": "fetch", "repositories": (5,), "expires_at": 0} + fields.update(overrides) + return Grant(**fields) # type: ignore[arg-type] + + +class TestRouteOperation: + @pytest.mark.parametrize( + "method,route,query,operation", + [ + ("GET", "info/refs", "service=git-upload-pack", "fetch"), + ("POST", "git-upload-pack", "", "fetch"), + ("GET", "info/refs", "service=git-receive-pack", "push"), + ("POST", "git-receive-pack", "", "push"), + ], + ) + def test_git_routes(self, method, route, query, operation): + assert route_operation(method, route, query) == operation + + @pytest.mark.parametrize( + "method,route,query", + [ + ("GET", "info/refs", ""), + ("GET", "info/refs", "service=git-upload-pack&x=1"), + ("GET", "info/refs", "service=git-upload-archive"), + ("POST", "info/refs", "service=git-upload-pack"), + ("GET", "git-upload-pack", ""), + ("POST", "git-upload-pack", "service=git-upload-pack"), + ("post", "git-upload-pack", ""), + ("POST", "git-upload-archive", ""), + ("POST", "info/lfs/objects/batch", ""), + ("GET", "HEAD", ""), + ("GET", "objects/info/packs", ""), + ("GET", "../info/refs", "service=git-upload-pack"), + ("POST", "/git-upload-pack", ""), + ], + ) + def test_other_routes(self, method, route, query): + assert route_operation(method, route, query) is None + + +class TestCheckRequest: + def test_fetch_grant_reaches_fetch_routes(self): + grant = make_grant() + assert check_request(grant, 5, "GET", "info/refs", "service=git-upload-pack") == "fetch" + assert check_request(grant, 5, "POST", "git-upload-pack", "") == "fetch" + + def test_push_grant_reaches_push_routes(self): + grant = make_grant(operation="push") + assert check_request(grant, 5, "GET", "info/refs", "service=git-receive-pack") == "push" + assert check_request(grant, 5, "POST", "git-receive-pack", "") == "push" + + @pytest.mark.parametrize( + "method,route,query", + [("GET", "info/refs", "service=git-receive-pack"), ("POST", "git-receive-pack", "")], + ) + def test_fetch_grant_cannot_push(self, method, route, query): + with pytest.raises(GitRouteNotAllowed, match="allows fetch, not push"): + check_request(make_grant(), 5, method, route, query) + + @pytest.mark.parametrize( + "method,route,query", + [("GET", "info/refs", "service=git-upload-pack"), ("POST", "git-upload-pack", "")], + ) + def test_push_grant_cannot_fetch(self, method, route, query): + with pytest.raises(GitRouteNotAllowed, match="allows push, not fetch"): + check_request(make_grant(operation="push"), 5, method, route, query) + + def test_unknown_route(self): + with pytest.raises(GitRouteNotAllowed, match="not a git fetch or push route"): + check_request(make_grant(), 5, "POST", "info/lfs/objects/batch", "") + + def test_repository_not_in_grant(self): + with pytest.raises(GitRouteNotAllowed, match="does not cover this repository"): + check_request(make_grant(), 6, "POST", "git-upload-pack", "") + + def test_repository_named_another_way_is_not_in_grant(self): + # The grant and the request name repositories the same way; matching a database id to a provider id + # would mean resolving every repository the grant lists. + grant = make_grant(repositories=(("github", "123"),)) + with pytest.raises(GitRouteNotAllowed): + check_request(grant, 5, "POST", "git-upload-pack", "") + assert check_request(grant, ("github", "123"), "POST", "git-upload-pack", "") == "fetch" + + def test_any_listed_repository(self): + grant = make_grant(repositories=(5, 6, ("gitlab", "gitlab.com:7"))) + assert check_request(grant, 6, "POST", "git-upload-pack", "") == "fetch" + assert check_request(grant, ("gitlab", "gitlab.com:7"), "POST", "git-upload-pack", "") == "fetch" diff --git a/tests/unit/test_git_relay.py b/tests/unit/test_git_relay.py new file mode 100644 index 00000000..befa3ccd --- /dev/null +++ b/tests/unit/test_git_relay.py @@ -0,0 +1,355 @@ +import gzip +import io +import socket +import threading +import time +from collections.abc import Iterator + +import pytest + +from scm.errors import GitRouteNotAllowed, ResourceGatewayTimeout, ResourceServiceUnavailable, SCMCodedError +from scm.git.relay import GitRelay +from scm.git.types import GitRemote +from tests.unit.git_upstream import UpstreamHandler, basic_auth, running_upstream + +TOKEN = "ghs_secrettoken" +# Long enough that a relay that buffers fails the test instead of hanging it. +WAIT = 5 + + +@pytest.fixture +def upstream(): + with running_upstream() as upstream: + yield upstream + + +@pytest.fixture +def relay(): + relay = GitRelay() + yield relay + relay.close() + + +def remote_for(upstream) -> GitRemote: + return GitRemote(url=f"{upstream.url}/getsentry/sentry.git", username="x-access-token") + + +def fetch_refs(relay: GitRelay, upstream, headers: dict[str, str] | None = None): + return relay.forward( + remote_for(upstream), + TOKEN, + method="GET", + route="info/refs", + query="service=git-upload-pack", + headers=headers or {}, + body=None, + ) + + +def upload_pack(relay: GitRelay, upstream, body, headers: dict[str, str] | None = None): + return relay.forward( + remote_for(upstream), + TOKEN, + method="POST", + route="git-upload-pack", + query="", + headers={"Content-Type": "application/x-git-upload-pack-request", **(headers or {})}, + body=body, + ) + + +class TestRequest: + def test_url_and_auth(self, relay, upstream): + b"".join(fetch_refs(relay, upstream).content) + [request] = upstream.requests + assert request.method == "GET" + assert request.path == "/getsentry/sentry.git/info/refs?service=git-upload-pack" + assert request.headers["Authorization"] == basic_auth("x-access-token", TOKEN) + + def test_git_headers_pass_through(self, relay, upstream): + body = gzip.compress(b"0000") + response = upload_pack( + relay, + upstream, + io.BytesIO(body), + headers={ + "Git-Protocol": "version=2", + "Content-Encoding": "gzip", + "Accept": "application/x-git-upload-pack-result", + "Accept-Encoding": "deflate, gzip", + "User-Agent": "git/2.47.0", + "Content-Length": str(len(body)), + }, + ) + b"".join(response.content) + [request] = upstream.requests + assert request.headers["Git-Protocol"] == "version=2" + assert request.headers["Content-Encoding"] == "gzip" + assert request.headers["Content-Type"] == "application/x-git-upload-pack-request" + assert request.headers["Accept"] == "application/x-git-upload-pack-result" + assert request.headers["Accept-Encoding"] == "deflate, gzip" + assert request.headers["User-Agent"] == "git/2.47.0" + # The body is passed on as it came, still compressed. + assert request.body == body + + def test_other_headers_stay_behind(self, relay, upstream): + b"".join( + fetch_refs( + relay, + upstream, + headers={ + "Authorization": "Bearer router-credential", + "X-Scm-Git-Grant": "grant", + "Cookie": "session=1", + "X-Forwarded-For": "10.0.0.1", + "Host": "sentry.io", + }, + ).content + ) + [request] = upstream.requests + assert request.headers["Authorization"] == basic_auth("x-access-token", TOKEN) + assert request.headers["Host"] == upstream.url.removeprefix("http://") + for name in ("X-Scm-Git-Grant", "Cookie", "X-Forwarded-For"): + assert name not in request.headers + + def test_header_names_are_case_insensitive(self, relay, upstream): + b"".join(fetch_refs(relay, upstream, headers={"git-protocol": "version=2", "accept-encoding": "gzip"}).content) + [request] = upstream.requests + assert request.headers["Git-Protocol"] == "version=2" + assert request.headers["Accept-Encoding"] == "gzip" + + def test_asks_for_an_unencoded_response_unless_the_client_says_otherwise(self, relay, upstream): + b"".join(fetch_refs(relay, upstream).content) + assert upstream.requests[0].headers["Accept-Encoding"] == "identity" + + def test_refuses_routes_that_are_not_git(self, relay, upstream): + with pytest.raises(GitRouteNotAllowed): + relay.forward( + remote_for(upstream), + TOKEN, + method="POST", + route="info/lfs/objects/batch", + query="", + headers={}, + body=None, + ) + assert upstream.requests == [] + + def test_known_length_body_is_sent_with_its_length(self, relay, upstream): + b"".join(upload_pack(relay, upstream, io.BytesIO(b"0000"), headers={"Content-Length": "4"}).content) + [request] = upstream.requests + assert request.headers["Content-Length"] == "4" + assert "Transfer-Encoding" not in request.headers + assert request.body == b"0000" + + def test_unknown_length_body_is_sent_chunked(self, relay, upstream): + b"".join(upload_pack(relay, upstream, io.BytesIO(b"0000")).content) + [request] = upstream.requests + assert request.headers["Transfer-Encoding"] == "chunked" + assert request.body == b"0000" + + def test_reads_no_further_than_the_declared_length(self, relay, upstream): + body = io.BytesIO(b"0000trailing") + b"".join(upload_pack(relay, upstream, body, headers={"Content-Length": "4"}).content) + assert upstream.requests[0].body == b"0000" + assert body.read() == b"trailing" + + def test_body_shorter_than_its_length(self, relay, upstream): + with pytest.raises(SCMCodedError) as exc_info: + upload_pack(relay, upstream, io.BytesIO(b"00"), headers={"Content-Length": "4"}) + assert exc_info.value.code == "client_closed_request" + + @pytest.mark.parametrize("content_length", ["abc", "-1", "1.5"]) + def test_malformed_content_length(self, relay, upstream, content_length): + with pytest.raises(SCMCodedError) as exc_info: + upload_pack(relay, upstream, io.BytesIO(b"0000"), headers={"Content-Length": content_length}) + assert exc_info.value.code == "rpc_malformed_request_headers" + assert upstream.requests == [] + + +class TestResponse: + def test_status_body_and_headers(self, relay, upstream): + def handler(h: UpstreamHandler) -> None: + h.record(b"") + h.respond( + 200, + b"001e# service=git-upload-pack\n0000", + { + "Content-Type": "application/x-git-upload-pack-advertisement", + "Cache-Control": "no-cache", + "Set-Cookie": "_gh_sess=abc", + "X-GitHub-Request-Id": "ABC", + "WWW-Authenticate": 'Basic realm="GitHub"', + }, + ) + + upstream.handler = handler + response = fetch_refs(relay, upstream) + assert response.status_code == 200 + assert b"".join(response.content) == b"001e# service=git-upload-pack\n0000" + assert response.headers == { + "Content-Type": "application/x-git-upload-pack-advertisement", + "Cache-Control": "no-cache", + "Content-Length": "34", + } + + def test_error_status_passes_through(self, relay, upstream): + upstream.handler = lambda h: h.respond(404, b"Repository not found.", {"Content-Type": "text/plain"}) + response = fetch_refs(relay, upstream) + assert response.status_code == 404 + assert b"".join(response.content) == b"Repository not found." + + def test_encoded_body_passes_through_undecoded(self, relay, upstream): + compressed = gzip.compress(b"0008NAK\n" * 100) + upstream.handler = lambda h: h.respond(200, compressed, {"Content-Encoding": "gzip"}) + response = fetch_refs(relay, upstream, headers={"Accept-Encoding": "gzip"}) + assert response.headers["Content-Encoding"] == "gzip" + assert response.headers["Content-Length"] == str(len(compressed)) + assert b"".join(response.content) == compressed + + def test_redirect_is_not_followed(self, relay, upstream): + def handler(h: UpstreamHandler) -> None: + h.record(b"") + h.respond(301, b"", {"Location": "https://elsewhere.example.com/repo.git/info/refs"}) + + upstream.handler = handler + response = fetch_refs(relay, upstream) + assert response.status_code == 301 + assert "Location" not in response.headers + b"".join(response.content) + assert len(upstream.requests) == 1 + + +class TestStreaming: + def test_response_streams_before_the_upstream_finishes(self, relay, upstream): + client_got_first_chunk = threading.Event() + upstream_waited = [] + + def handler(h: UpstreamHandler) -> None: + h.record(b"".join(h.body_chunks())) + h.start_chunked(200, {"Content-Type": "application/x-git-upload-pack-result"}) + h.write_chunk(b"first") + upstream_waited.append(client_got_first_chunk.wait(WAIT)) + h.write_chunk(b"second") + h.write_chunk(b"") + + upstream.handler = handler + content = upload_pack(relay, upstream, io.BytesIO(b"0000"), headers={"Content-Length": "4"}).content + assert next(content) == b"first" + client_got_first_chunk.set() + assert b"".join(content) == b"second" + assert upstream_waited == [True] + + @pytest.mark.parametrize("known_length", [True, False]) + def test_request_streams_before_the_client_finishes(self, relay, upstream, known_length): + upstream_got_first_chunk = threading.Event() + received: list[bytes] = [] + + def handler(h: UpstreamHandler) -> None: + chunks = h.body_chunks() + received.append(next(chunks)) + upstream_got_first_chunk.set() + received.extend(chunks) + h.respond(200, b"ok") + + class SlowClient: + """Holds the rest of its body back until the upstream has the first part.""" + + def __init__(self) -> None: + self.parts: Iterator[bytes] = iter([b"first", b"second"]) + self.waited: list[bool] = [] + + def read(self, size: int = -1, /) -> bytes: + part = next(self.parts, b"") + if part == b"second": + self.waited.append(upstream_got_first_chunk.wait(WAIT)) + return part + + upstream.handler = handler + client = SlowClient() + headers = {"Content-Length": str(len(b"firstsecond"))} if known_length else {} + response = upload_pack(relay, upstream, client, headers=headers) + assert b"".join(response.content) == b"ok" + assert client.waited == [True] + assert b"".join(received) == b"firstsecond" + + +class TestConnections: + def test_reuses_the_upstream_connection(self, relay, upstream): + b"".join(fetch_refs(relay, upstream).content) + b"".join(upload_pack(relay, upstream, io.BytesIO(b"0000"), headers={"Content-Length": "4"}).content) + b"".join(upload_pack(relay, upstream, io.BytesIO(b"0000")).content) + assert len(upstream.requests) == 3 + assert upstream.connections == 1 + + def test_does_not_carry_cookies_between_requests(self, relay, upstream): + def handler(h: UpstreamHandler) -> None: + h.record(b"") + h.respond(200, b"", {"Set-Cookie": "_gh_sess=abc; Path=/"}) + + upstream.handler = handler + b"".join(fetch_refs(relay, upstream).content) + b"".join(fetch_refs(relay, upstream).content) + assert [r.headers.get("Cookie") for r in upstream.requests] == [None, None] + + +class TestUpstreamFailures: + def test_connection_refused(self, relay): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + remote = GitRemote(url=f"http://127.0.0.1:{port}/getsentry/sentry.git", username="x-access-token") + + with pytest.raises(ResourceServiceUnavailable) as exc_info: + relay.forward( + remote, TOKEN, method="GET", route="info/refs", query="service=git-upload-pack", headers={}, body=None + ) + assert_token_not_in(exc_info.value) + + def test_read_timeout(self, upstream): + release = threading.Event() + + def handler(h: UpstreamHandler) -> None: + release.wait(WAIT) + try: + h.respond(200, b"") + except OSError: # The relay stopped waiting and closed the connection. + h.close_connection = True + + upstream.handler = handler + relay = GitRelay(timeout=(1, 0.1)) + try: + with pytest.raises(ResourceGatewayTimeout) as exc_info: + fetch_refs(relay, upstream) + assert_token_not_in(exc_info.value) + finally: + release.set() + relay.close() + + +def assert_token_not_in(error: Exception) -> None: + # The requests exception holds the prepared request and its Authorization header, so it must not be chained. + assert error.__cause__ is None + assert error.__suppress_context__ + assert TOKEN not in str(error) + assert basic_auth("x-access-token", TOKEN) not in repr(error) + + +class TestClose: + def test_closing_an_unread_response_drops_the_upstream_connection(self, relay, upstream): + upstream_saw_close = threading.Event() + + def handler(h: UpstreamHandler) -> None: + h.start_chunked(200) + try: + for _ in range(1000): + h.write_chunk(b"x" * 64 * 1024) + time.sleep(0.005) + except OSError: + h.close_connection = True + upstream_saw_close.set() + + upstream.handler = handler + response = fetch_refs(relay, upstream) + response.content.close() # type: ignore[attr-defined] + assert upstream_saw_close.wait(WAIT) diff --git a/tests/unit/test_git_remote.py b/tests/unit/test_git_remote.py new file mode 100644 index 00000000..ced9d9a6 --- /dev/null +++ b/tests/unit/test_git_remote.py @@ -0,0 +1,62 @@ +import pytest + +from scm.errors import GitProviderNotSupported, MalformedExternalId +from scm.git.remote import git_remote +from scm.git.types import GitRemote +from scm.types import Repository + + +def make_repository(**overrides) -> Repository: + defaults: Repository = { + "id": 1, + "external_id": "123", + "integration_id": 1, + "is_active": True, + "name": "getsentry/sentry", + "organization_id": 1, + "provider_name": "github", + "web_base_url": None, + "installation_id": None, + } + return {**defaults, **overrides} # type: ignore[typeddict-item] + + +def test_github(): + assert git_remote(make_repository()) == GitRemote( + url="https://github.com/getsentry/sentry.git", username="x-access-token" + ) + + +@pytest.mark.parametrize("web_base_url", ["https://ghe.example.com", "https://ghe.example.com/"]) +def test_github_enterprise(web_base_url): + repository = make_repository(provider_name="github_enterprise", web_base_url=web_base_url) + assert git_remote(repository) == GitRemote( + url="https://ghe.example.com/getsentry/sentry.git", username="x-access-token" + ) + + +def test_github_enterprise_without_host(): + with pytest.raises(GitProviderNotSupported, match="no web_base_url"): + git_remote(make_repository(provider_name="github_enterprise")) + + +def test_gitlab(): + repository = make_repository(provider_name="gitlab", external_id="gitlab.com:42", name="group / sub / project") + assert git_remote(repository) == GitRemote(url="https://gitlab.com/group/sub/project.git", username="oauth2") + + +def test_self_hosted_gitlab_with_port(): + repository = make_repository(provider_name="gitlab", external_id="git.example.com:8443:42", name="team / app") + assert git_remote(repository) == GitRemote(url="https://git.example.com:8443/team/app.git", username="oauth2") + + +@pytest.mark.parametrize("external_id", [None, "42"]) +def test_gitlab_malformed_external_id(external_id): + with pytest.raises(MalformedExternalId): + git_remote(make_repository(provider_name="gitlab", external_id=external_id)) + + +@pytest.mark.parametrize("provider_name", ["bitbucket", "cursor_origin"]) +def test_unsupported_providers(provider_name): + with pytest.raises(GitProviderNotSupported, match=provider_name): + git_remote(make_repository(provider_name=provider_name)) diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py new file mode 100644 index 00000000..40841688 --- /dev/null +++ b/tests/unit/test_git_server.py @@ -0,0 +1,257 @@ +import io +import time +from unittest.mock import MagicMock + +import pytest + +from scm.git.grant import sign_grant +from scm.git.relay import GitRelay +from scm.git.server import GitServer +from scm.git.types import GRANT_HEADER, GitOperation, GitRemote, Grant +from scm.rpc.types import StreamResponse +from scm.types import Repository, RepositoryId +from tests.unit.git_upstream import basic_auth, running_upstream + +SECRET = "secret" +TOKEN = "ghs_secrettoken" + + +def make_repository(**overrides) -> Repository: + defaults: Repository = { + "id": 5, + "external_id": "123", + "integration_id": 1, + "is_active": True, + "name": "getsentry/sentry", + "organization_id": 1, + "provider_name": "github", + "web_base_url": None, + "installation_id": None, + } + return {**defaults, **overrides} # type: ignore[typeddict-item] + + +def make_grant(**overrides) -> str: + fields = { + "organization_id": 1, + "operation": "fetch", + "repositories": (5,), + "expires_at": int(time.time()) + 600, + } + fields.update(overrides) + return sign_grant(SECRET, Grant(**fields)) # type: ignore[arg-type] + + +class Recorder: + def __init__(self, repository: Repository | None = None, token: str | None = TOKEN) -> None: + self.repository = make_repository() if repository is None else repository + self.token = token + self.repository_lookups: list[tuple[int, RepositoryId]] = [] + self.token_lookups: list[tuple[Repository, GitOperation]] = [] + self.counts: list[tuple[str, dict[str, str]]] = [] + self.errors: list[Exception] = [] + + def fetch_repository(self, organization_id: int, repository_id: RepositoryId) -> Repository | None: + self.repository_lookups.append((organization_id, repository_id)) + return self.repository + + def get_token(self, repository: Repository, operation: GitOperation) -> str | None: + self.token_lookups.append((repository, operation)) + return self.token + + def record_count(self, name: str, value: int, tags: dict[str, str]) -> None: + self.counts.append((name, tags)) + + +def make_server(recorder: Recorder, relay: GitRelay | None = None) -> GitServer: + if relay is None: + relay = MagicMock(spec=GitRelay) + relay.forward.return_value = StreamResponse(status_code=200, headers={}, content=iter([b"refs"])) + return GitServer( + secrets=[SECRET], + fetch_repository=recorder.fetch_repository, + get_token=recorder.get_token, + record_count=recorder.record_count, + emit_error=recorder.errors.append, + relay=relay, + ) + + +def fetch_refs(server: GitServer, grant: str | None, repository_id: RepositoryId = 5) -> StreamResponse: + return server.handle( + method="GET", + repository_id=repository_id, + route="info/refs", + query="service=git-upload-pack", + headers={GRANT_HEADER: grant} if grant is not None else {}, + body=None, + ) + + +def assert_refused(response: StreamResponse, status: int, detail: str) -> None: + assert response.status_code == status + assert response.headers["Content-Type"] == "text/plain; charset=utf-8" + body = b"".join(response.content).decode() + assert detail in body + assert int(response.headers["Content-Length"]) == len(body.encode()) + + +class TestRefusedBeforeTheProvider: + """Nothing the grant does not allow may look up a repository or a token.""" + + @pytest.mark.parametrize( + "grant,detail", + [ + (None, "missing"), + ("garbage", "malformed"), + ( + sign_grant("other", Grant(organization_id=1, operation="fetch", repositories=(5,), expires_at=2**40)), + "signature", + ), + (make_grant(expires_at=int(time.time()) - 1), "expired"), + ], + ) + def test_invalid_grant(self, grant, detail): + recorder = Recorder() + server = make_server(recorder) + assert_refused(fetch_refs(server, grant), 403, detail) + assert recorder.repository_lookups == [] + assert recorder.token_lookups == [] + server.relay.forward.assert_not_called() # type: ignore[attr-defined] + assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_invalid_grant"})] + + def test_operation_the_grant_does_not_allow(self): + recorder = Recorder() + server = make_server(recorder) + response = server.handle( + method="POST", + repository_id=5, + route="git-receive-pack", + query="", + headers={GRANT_HEADER: make_grant()}, + body=io.BytesIO(b"0000"), + ) + assert_refused(response, 403, "allows fetch, not push") + assert recorder.repository_lookups == [] + assert recorder.token_lookups == [] + + def test_repository_the_grant_does_not_list(self): + recorder = Recorder() + assert_refused(fetch_refs(make_server(recorder), make_grant(), repository_id=6), 403, "does not cover") + assert recorder.repository_lookups == [] + + def test_grant_header_name_is_case_insensitive(self): + recorder = Recorder() + server = make_server(recorder) + response = server.handle( + method="GET", + repository_id=5, + route="info/refs", + query="service=git-upload-pack", + headers={GRANT_HEADER.lower(): make_grant()}, + body=None, + ) + assert response.status_code == 200 + + +class TestRepository: + def test_looked_up_under_the_grants_organization(self): + recorder = Recorder() + fetch_refs(make_server(recorder), make_grant(organization_id=1)) + assert recorder.repository_lookups == [(1, 5)] + + def test_not_found(self): + recorder = Recorder() + recorder.repository = None # type: ignore[assignment] + assert_refused(fetch_refs(make_server(recorder), make_grant()), 404, "could not be found") + assert recorder.token_lookups == [] + + def test_other_organization(self): + recorder = Recorder(repository=make_repository(organization_id=2)) + assert_refused(fetch_refs(make_server(recorder), make_grant()), 404, "did not belong") + assert recorder.token_lookups == [] + + def test_inactive(self): + recorder = Recorder(repository=make_repository(is_active=False)) + assert_refused(fetch_refs(make_server(recorder), make_grant()), 404, "inactive") + assert recorder.token_lookups == [] + + def test_provider_without_a_git_remote(self): + recorder = Recorder(repository=make_repository(provider_name="bitbucket")) + assert_refused(fetch_refs(make_server(recorder), make_grant()), 400, "no git remote") + assert recorder.token_lookups == [] + + +class TestRelay: + def test_no_token(self): + recorder = Recorder(token=None) + assert_refused(fetch_refs(make_server(recorder), make_grant()), 404, "No git credentials") + + def test_token_asked_for_the_grants_operation(self): + recorder = Recorder() + server = make_server(recorder) + server.handle( + method="POST", + repository_id=5, + route="git-receive-pack", + query="", + headers={GRANT_HEADER: make_grant(operation="push")}, + body=io.BytesIO(b"0000"), + ) + assert recorder.token_lookups == [(make_repository(), "push")] + + def test_forwards_with_remote_and_token(self): + recorder = Recorder() + server = make_server(recorder) + headers = {GRANT_HEADER: make_grant(), "Git-Protocol": "version=2"} + response = server.handle( + method="GET", + repository_id=5, + route="info/refs", + query="service=git-upload-pack", + headers=headers, + body=None, + ) + assert b"".join(response.content) == b"refs" + server.relay.forward.assert_called_once_with( # type: ignore[attr-defined] + GitRemote(url="https://github.com/getsentry/sentry.git", username="x-access-token"), + TOKEN, + method="GET", + route="info/refs", + query="service=git-upload-pack", + headers=headers, + body=None, + ) + assert recorder.counts == [ + ("sentry.scm.git.request.succeeded", {"operation": "fetch", "provider": "github", "status": "200"}) + ] + + def test_upstream_unavailable(self): + recorder = Recorder( + repository=make_repository(provider_name="github_enterprise", web_base_url="http://127.0.0.1:1") + ) + relay = GitRelay(timeout=(1, 1)) + try: + response = fetch_refs(make_server(recorder, relay), make_grant()) + finally: + relay.close() + assert_refused(response, 503, "Could not connect") + assert TOKEN not in b"".join(response.content).decode() + assert recorder.errors == [] + + def test_relays_end_to_end(self): + with running_upstream() as upstream: + recorder = Recorder( + repository=make_repository(provider_name="github_enterprise", web_base_url=upstream.url) + ) + relay = GitRelay() + try: + response = fetch_refs(make_server(recorder, relay), make_grant()) + assert response.status_code == 200 + b"".join(response.content) + finally: + relay.close() + [request] = upstream.requests + assert request.path == "/getsentry/sentry.git/info/refs?service=git-upload-pack" + assert request.headers["Authorization"] == basic_auth("x-access-token", TOKEN) + assert GRANT_HEADER not in request.headers From 1ace97f5501be4b78ac3ca150575bffe8305a2a8 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 12:27:28 +0200 Subject: [PATCH 03/31] fix(git): Send tokens only over https and clarify the git module The relay mounted a plain-http adapter so tests could reach a local server, and requests mounts one by default anyway, so nothing stopped a token from going out in cleartext if a remote ever had an http URL. The relay now refuses non-https remotes, git_remote refuses a web_base_url that is not https, and the tests serve TLS with a throwaway certificate instead. A GitHub Enterprise repository without a web_base_url is bad repository data, not an unsupported provider, so it gets its own error like MalformedExternalId. GitLab now uses web_base_url when Sentry sets it, because only that keeps the scheme and a path prefix; until then it falls back to the host in the external id. cryptography becomes an explicit dev dependency because the tests now use it directly; it was only locked through pyjwt[crypto]. Refs CW-2221 Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 1 + src/scm/errors.py | 6 ++++ src/scm/git/grant.py | 12 +++++--- src/scm/git/operations.py | 14 ++++++--- src/scm/git/relay.py | 45 ++++++++++++++++++--------- src/scm/git/remote.py | 44 +++++++++++++++++--------- src/scm/git/server.py | 9 +++--- src/scm/git/types.py | 20 ++++++------ tests/unit/git_upstream.py | 58 +++++++++++++++++++++++++++++++++-- tests/unit/test_git_relay.py | 20 +++++++----- tests/unit/test_git_remote.py | 33 +++++++++++++++++--- tests/unit/test_git_server.py | 5 +-- uv.lock | 2 ++ 13 files changed, 203 insertions(+), 66 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b4917981..5faafb76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ select = [ [dependency-groups] dev = [ + "cryptography>=48.0.1", "mypy>=1.20.0", "prek>=0.3.6", "pyjwt[crypto]>=2.9.0", diff --git a/src/scm/errors.py b/src/scm/errors.py index e0c357f0..b2a29c06 100644 --- a/src/scm/errors.py +++ b/src/scm/errors.py @@ -4,6 +4,7 @@ "git_invalid_grant", "git_provider_not_supported", "git_route_not_allowed", + "invalid_web_base_url", "malformed_external_id", "path_is_directory", "path_is_not_directory", @@ -43,6 +44,7 @@ "git_invalid_grant": "The git grant was missing, malformed, forged or expired.", "git_provider_not_supported": "The repository's provider has no git remote.", "git_route_not_allowed": "The git grant does not allow this request.", + "invalid_web_base_url": "The repository's web_base_url is missing or is not an https URL.", "malformed_external_id": "The repository's external ID was malformed.", "path_is_directory": "The requested path is a directory, not a file.", "path_is_not_directory": "The requested path is not a directory.", @@ -141,6 +143,10 @@ class GitRouteNotAllowed(SCMCodedError): code = "git_route_not_allowed" +class InvalidWebBaseUrl(SCMCodedError): + code = "invalid_web_base_url" + + class MalformedExternalId(SCMCodedError): code = "malformed_external_id" diff --git a/src/scm/git/grant.py b/src/scm/git/grant.py index 562b7bfb..e4db07ca 100644 --- a/src/scm/git/grant.py +++ b/src/scm/git/grant.py @@ -1,8 +1,8 @@ """Signing and verifying git grants. The wire format is ``.``, where the payload is the grant's JSON and the -signature is an HMAC-SHA256 over a fixed prefix and the payload. The prefix keeps a grant signature from ever -matching a signature the RPC endpoint makes with the same secret over some other message. +signature is an HMAC-SHA256 over a fixed prefix and the payload. Because of the prefix, a signature the RPC +endpoint makes with the same secret can never pass as a grant signature. """ import base64 @@ -37,7 +37,7 @@ def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> payload_b64, signature_b64 = token.split(".") payload = _b64decode(payload_b64) signature = _b64decode(signature_b64) - except ValueError: # binascii.Error is a ValueError too + except ValueError: # Raised both for the wrong number of parts and for bad base64 (binascii.Error). raise GitInvalidGrant(detail="The git grant is malformed.") from None # Check every secret, not just until one matches, so the time taken does not reveal which one signed it. @@ -48,7 +48,8 @@ def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> try: grant = msgspec.json.decode(payload, type=Grant) except msgspec.ValidationError as e: - # Only a holder of the secret can get here, so the reason is safe to return and saves a debugging round. + # The signature matched, so a holder of the secret wrote this payload; telling them why it is invalid + # gives nothing away. raise GitInvalidGrant(detail=f"The git grant's payload is invalid: {e}") from None except msgspec.DecodeError: raise GitInvalidGrant(detail="The git grant's payload is not JSON.") from None @@ -69,7 +70,8 @@ def _b64encode(data: bytes) -> str: def _b64decode(data: str) -> bytes: - # urlsafe_b64decode skips characters outside its alphabet; refusing them keeps one token to one decoding. + # urlsafe_b64decode silently drops characters outside its alphabet. Refuse them instead, so a grant has only + # one valid spelling. if not _BASE64URL.fullmatch(data): raise ValueError("not base64url") return base64.urlsafe_b64decode(data + "=" * (-len(data) % 4)) diff --git a/src/scm/git/operations.py b/src/scm/git/operations.py index 7826f3b7..76010ded 100644 --- a/src/scm/git/operations.py +++ b/src/scm/git/operations.py @@ -2,8 +2,13 @@ from scm.git.types import GitOperation, Grant from scm.types import RepositoryId -# The git smart-HTTP routes, keyed by (method, path below .git/, query string). git sends these exact query -# strings, so an exact match leaves nothing for the provider to read differently than this table does. +# The git smart-HTTP routes and the operation each belongs to, keyed by (method, path below .git/, query +# string). git names its two services from the server's side: in a fetch the server uploads a pack to the client +# (git-upload-pack), and in a push it receives a pack from the client (git-receive-pack). Each service has a GET +# that lists the refs and a POST that does the transfer. +# +# The query string is compared whole instead of parsed. git sends exactly these strings, so a request with extra +# or repeated parameters is refused rather than passed on to a provider that might read them differently. _ROUTES: dict[tuple[str, str, str], GitOperation] = { ("GET", "info/refs", "service=git-upload-pack"): "fetch", ("POST", "git-upload-pack", ""): "fetch", @@ -20,8 +25,9 @@ def route_operation(method: str, route: str, query: str) -> GitOperation | None: def check_request(grant: Grant, repository_id: RepositoryId, method: str, route: str, query: str) -> GitOperation: """Refuse a request the grant does not reach, before anything is sent to the provider. - A fetch grant reaches only upload-pack and a push grant only receive-pack, and the repository must be one the - grant lists, named the same way. + A fetch grant reaches only upload-pack and a push grant only receive-pack. The repository must be listed in + the grant in the same form the request uses: a database id does not match the same repository's + ``(provider, external id)`` pair. """ operation = route_operation(method, route, query) if operation is None: diff --git a/src/scm/git/relay.py b/src/scm/git/relay.py index 4826e0ee..668bbb37 100644 --- a/src/scm/git/relay.py +++ b/src/scm/git/relay.py @@ -17,17 +17,26 @@ from scm.git.types import GitRemote, Readable from scm.rpc.types import StreamResponse +# The most bytes the relay reads at once, from the client's body or from the provider's response, so also the +# most it holds per request in each direction. Smaller chunks cost more Python per byte of a multi-gigabyte pack; +# larger ones hold more memory per request. 64 KiB is what RpcServer streams with and what the prototype ran; no +# other size was measured. DEFAULT_CHUNK_SIZE = 64 * 1024 -# A large clone streams its pack for minutes, but the provider sends something well within the read timeout. +# (connect, read) in seconds. The read timeout bounds a silence, not a transfer: it is the longest the provider may +# go without sending a byte, so a clone that streams for minutes never hits it. 300s leaves room for a provider that +# assembles a large pack before it sends anything. Both values come from the prototype. DEFAULT_TIMEOUT = (10.0, 300.0) -# What git sends that the provider needs: protocol v2 is negotiated with Git-Protocol, and git gzips large fetch -# requests. Everything else stays behind, including the caller's own credentials and the grant. +# The headers git sends that the provider needs. Accept and Content-Type name the git service, Git-Protocol +# negotiates protocol v2, Content-Encoding marks a gzipped request body (git gzips large fetch requests), +# Accept-Encoding says what the client can decode, and User-Agent names the git client. Everything else stays +# behind, including the caller's own credentials and the grant. _REQUEST_HEADERS = frozenset( {"accept", "accept-encoding", "content-encoding", "content-type", "git-protocol", "user-agent"} ) -# Bodies pass through still encoded, so Content-Encoding and Content-Length describe what the client receives. -# Location is left out because a redirect would send the client to the provider directly. +# Response bodies are passed on without decoding, so Content-Encoding and Content-Length still describe the bytes +# the client receives. Location is dropped because following a redirect would take the client to the provider +# directly, past the relay. _RESPONSE_HEADERS = frozenset( {"cache-control", "content-encoding", "content-length", "content-type", "expires", "pragma"} ) @@ -37,8 +46,10 @@ class GitRelay: """Forwards git smart-HTTP requests to a provider with a token, streaming the body in both directions. Nothing buffers a request or a response, and neither body is parsed. Build one per process and reuse it: it - keeps a pool of keep-alive connections per provider host, because a clone makes three requests and a new - connection for each cost 0.4-0.8s per clone in the prototype. + keeps a pool of keep-alive connections per provider host, because a clone makes three requests and opening a + new connection for each added hundreds of milliseconds per clone in the prototype. + + Only https remotes are accepted, because every request carries a provider token. """ def __init__( @@ -52,12 +63,11 @@ def __init__( self.timeout = timeout self.chunk_size = chunk_size self.session = requests.Session() - # The session is shared by every organization's requests, so it must not carry a provider's cookies from - # one request to the next. + # Every organization's requests share this session, so a cookie a provider sets on one request must not + # be sent with the next. self.session.cookies.set_policy(DefaultCookiePolicy(allowed_domains=[])) adapter = HTTPAdapter(pool_connections=max_hosts, pool_maxsize=max_connections_per_host, max_retries=0) self.session.mount("https://", adapter) - self.session.mount("http://", adapter) def forward( self, @@ -74,7 +84,11 @@ def forward( ``route`` is the path below ``.git/``. Check the request against a grant before calling this. """ - # The caller has checked the operation; this only keeps anything but a git route away from the token. + if not remote.url.startswith("https://"): + # git_remote only builds https URLs, so this is a caller's bug rather than a request to refuse. + raise ValueError(f"Refusing to send a token to {remote.url}, which is not https.") + # Callers check the request against a grant first; this only makes sure the token never goes to anything + # but a git route. if route_operation(method, route, query) is None: raise GitRouteNotAllowed(detail=f"{method} {route} is not a git fetch or push route.") @@ -82,7 +96,8 @@ def forward( upstream_headers = CaseInsensitiveDict( {name: value for name, value in client_headers.items() if name.lower() in _REQUEST_HEADERS} ) - # requests asks for gzip and more by default, which the client may not decode once the body is passed on. + # Otherwise requests asks for gzip, deflate and more, and since the response is passed on undecoded the + # client could get an encoding it cannot read. upstream_headers.setdefault("Accept-Encoding", "identity") url = f"{remote.url}/{route}?{query}" if query else f"{remote.url}/{route}" @@ -95,7 +110,7 @@ def forward( # Passing auth also stops requests from replacing it with credentials from a .netrc file. auth=HTTPBasicAuth(remote.username, token), stream=True, - # A redirect would carry the token to wherever the provider points. + # Following a redirect would send the request somewhere other than the repository's remote. allow_redirects=False, timeout=self.timeout, ) @@ -115,8 +130,8 @@ def close(self) -> None: self.session.close() def _stream(self, upstream: requests.Response) -> Iterator[bytes]: - # Closing the response once it is read to the end returns its connection to the pool; closing it early, - # because the client went away, drops the connection instead. + # Once the response is read to the end its connection goes back to the pool. Closing it before then, + # because the client went away, drops the connection instead, since it is mid-response. with upstream: yield from filter(bool, upstream.raw.stream(self.chunk_size, decode_content=False)) diff --git a/src/scm/git/remote.py b/src/scm/git/remote.py index a80d52bd..298ad9a1 100644 --- a/src/scm/git/remote.py +++ b/src/scm/git/remote.py @@ -1,4 +1,6 @@ -from scm.errors import GitProviderNotSupported, MalformedExternalId +from urllib.parse import urlsplit + +from scm.errors import GitProviderNotSupported, InvalidWebBaseUrl, MalformedExternalId from scm.git.types import GitRemote from scm.types import Repository @@ -8,26 +10,40 @@ def git_remote(repository: Repository) -> GitRemote: """Return the git endpoint and basic-auth username for a repository's provider. - Bitbucket Cloud (``x-token-auth``) and Azure DevOps (a bearer header) follow the same pattern once the - platform supports them as providers. + Bitbucket Cloud (basic auth as ``x-token-auth``) can be added the same way once the platform has a provider + for it. Azure DevOps authenticates git with a bearer header, which ``GitRemote`` cannot express yet. """ match repository["provider_name"]: case "github": return GitRemote(url=f"{GITHUB_GIT_BASE_URL}/{repository['name']}.git", username="x-access-token") case "github_enterprise": - if not repository["web_base_url"]: - raise GitProviderNotSupported(detail="The GitHub Enterprise repository has no web_base_url.") - base_url = repository["web_base_url"].rstrip("/") + if repository["web_base_url"] is None: + raise InvalidWebBaseUrl(detail="The GitHub Enterprise repository has no web_base_url.") + base_url = _https_base_url(repository["web_base_url"]) return GitRemote(url=f"{base_url}/{repository['name']}.git", username="x-access-token") case "gitlab": - # Sentry leaves web_base_url unset for GitLab; the host is in the external id, ":", - # and the name is the namespace path with spaces around its slashes. GitLabProvider reads both the same - # way. - external_id = repository["external_id"] - if external_id is None or ":" not in external_id: - raise MalformedExternalId() - netloc = external_id.rsplit(":", maxsplit=1)[0] + # Sentry's repository name for GitLab is the namespace path with spaces around each slash. path = repository["name"].replace(" ", "") - return GitRemote(url=f"https://{netloc}/{path}.git", username="oauth2") + return GitRemote(url=f"{_gitlab_base_url(repository)}/{path}.git", username="oauth2") case provider_name: raise GitProviderNotSupported(detail=f"The {provider_name} provider has no git remote.") + + +def _gitlab_base_url(repository: Repository) -> str: + if repository["web_base_url"] is not None: + return _https_base_url(repository["web_base_url"]) + # Sentry does not set web_base_url for GitLab yet. Until it does, take the host from the external id, + # ":", and assume https, as GitLabProvider does. That loses an instance served under a + # path prefix, which only web_base_url carries. + external_id = repository["external_id"] + if external_id is None or ":" not in external_id: + raise MalformedExternalId() + return f"https://{external_id.rsplit(':', maxsplit=1)[0]}" + + +def _https_base_url(web_base_url: str) -> str: + # The token goes to this URL, so it must be https. + parts = urlsplit(web_base_url) + if parts.scheme != "https" or not parts.netloc or parts.query or parts.fragment: + raise InvalidWebBaseUrl(detail=f"The repository's web_base_url {web_base_url!r} is not an https URL.") + return web_base_url.rstrip("/") diff --git a/src/scm/git/server.py b/src/scm/git/server.py index a3c4f331..ce994022 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -17,9 +17,10 @@ class GitServer: """Serves a git endpoint: checks each request against its grant, then relays it with a provider token. - The order is the security boundary. The grant is verified and the request's route and repository checked - against it before the repository is looked up or a token fetched, so a request the grant does not reach - never touches the provider. Build one per process: its relay pools upstream connections. + The order of the checks matters. The grant is verified, and the request's route and repository checked + against it, before the repository is looked up or a token fetched, so a request the grant does not allow + never reaches the database, the token supplier or the provider. Build one per process: its relay pools + upstream connections. """ def __init__( @@ -85,7 +86,7 @@ def handle( def _error_response(error: SCMCodedError) -> StreamResponse: - # git asks for credentials on a 401, which a sandboxed client must never have, so a refusal is a 403. + # On a 401 git asks the user for credentials, which a sandboxed client must never have, so refuse with a 403. status = SPECIAL_STATUS_MAP.get(error.code, 400) if status == 401: status = 403 diff --git a/src/scm/git/types.py b/src/scm/git/types.py index 0e431c97..218bcbe2 100644 --- a/src/scm/git/types.py +++ b/src/scm/git/types.py @@ -7,7 +7,7 @@ from scm.types import Repository, RepositoryId type GitOperation = Literal["fetch", "push"] -"""What a grant lets its holder do: fetch reaches git-upload-pack, push reaches git-receive-pack.""" +"""What a grant lets its holder do. ``scm.git.operations`` maps each to its git routes.""" GRANT_VERSION: Literal[1] = 1 @@ -18,13 +18,13 @@ class Grant(msgspec.Struct, frozen=True, kw_only=True, forbid_unknown_fields=True): """Permission to run one git operation against a fixed set of an organization's repositories. - Seer signs a grant per coding job and Sentry's git endpoint verifies it on every request. Each repository - is named the way the RPC endpoint names one, as a ``RepositoryId`` under the grant's organization. A fetch - grant holds one repository today; the list leaves room for submodules and git-based dependencies without a - new format. + Seer signs a grant per coding job and Sentry's git endpoint verifies it on every request. Each repository is + a ``RepositoryId`` under the grant's organization, as in the RPC endpoint: Sentry's repository id or a + ``(provider, external id)`` pair. A fetch grant holds one repository today; the list leaves room for + submodules and git-based dependencies without a new format. - Unknown fields are refused rather than ignored, so a field a newer signer adds to narrow a grant can never - be dropped by an older verifier that would then allow more than the signer meant. + A grant with a field this version does not know is refused rather than read without it. If a later version + adds a field that restricts a grant, a verifier that ignored the field would allow more than the grant does. """ organization_id: int @@ -33,6 +33,7 @@ class Grant(msgspec.Struct, frozen=True, kw_only=True, forbid_unknown_fields=Tru expires_at: int """Unix time in seconds. Every grant expires.""" version: Literal[1] = GRANT_VERSION + """The format version. A verifier refuses a version it does not know.""" @dataclasses.dataclass(frozen=True) @@ -48,8 +49,9 @@ class GitRemote: type GetToken = Callable[[Repository, GitOperation], str | None] """Return a provider token that can run ``operation`` on ``repository``, or None if there is none. -The relay never builds credentials itself, so Sentry's integration token getters today and a central token -mint later plug in without changing it. The operation lets a mint scope the token, e.g. read-only for a fetch. +The git module never builds credentials itself: Sentry's integration clients supply them today, and a central +token mint can later without a change here. The operation lets the supplier scope the token, e.g. read-only for a +fetch. """ diff --git a/tests/unit/git_upstream.py b/tests/unit/git_upstream.py index 3b5acefc..9f32446d 100644 --- a/tests/unit/git_upstream.py +++ b/tests/unit/git_upstream.py @@ -1,11 +1,26 @@ -"""A local HTTP/1.1 server standing in for a provider's git endpoint in relay tests.""" +"""A local HTTPS server standing in for a provider's git endpoint in relay tests. + +The relay only sends tokens over https, so the server uses a self-signed certificate made for the test run. Point +requests at it with ``REQUESTS_CA_BUNDLE=``. +""" import base64 +import datetime +import functools +import ipaddress +import ssl +import sys +import tempfile import threading from collections.abc import Callable, Iterator from dataclasses import dataclass, field from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import NameOID from requests.structures import CaseInsensitiveDict @@ -23,6 +38,7 @@ class ReceivedRequest: @dataclass class Upstream: url: str + ca_bundle: str requests: list[ReceivedRequest] = field(default_factory=list) connections: int = 0 handler: Handler | None = None @@ -94,6 +110,13 @@ class _Server(ThreadingHTTPServer): daemon_threads = True upstream: Upstream + def handle_error(self, request: object, client_address: object) -> None: + # Some tests have the relay drop the connection mid-request on purpose (a timeout, a short body), and the + # assertions are on the relay's side, so a reset here is expected rather than worth a traceback. + if isinstance(sys.exc_info()[1], ConnectionError): + return + super().handle_error(request, client_address) # type: ignore[arg-type] + def echo(handler: UpstreamHandler) -> None: body = b"".join(handler.body_chunks()) @@ -105,13 +128,44 @@ def basic_auth(username: str, password: str) -> str: return "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode() +@functools.cache +def _certificate() -> tuple[str, str]: + """Return the paths of a self-signed certificate for 127.0.0.1 and its key, made once per test run.""" + key = ec.generate_private_key(ec.SECP256R1()) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")]) + now = datetime.datetime.now(datetime.UTC) + certificate = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(minutes=1)) + .not_valid_after(now + datetime.timedelta(days=1)) + .add_extension(x509.SubjectAlternativeName([x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), critical=False) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .sign(key, hashes.SHA256()) + ) + directory = Path(tempfile.mkdtemp(prefix="scm-git-upstream-")) + cert_path, key_path = directory / "cert.pem", directory / "key.pem" + cert_path.write_bytes(certificate.public_bytes(serialization.Encoding.PEM)) + key_path.write_bytes( + key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) + ) + return str(cert_path), str(key_path) + + class running_upstream: """Context manager that serves an ``Upstream`` on a free local port.""" def __enter__(self) -> Upstream: + cert_path, key_path = _certificate() + context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + context.load_cert_chain(cert_path, key_path) self._server = _Server(("127.0.0.1", 0), UpstreamHandler) + self._server.socket = context.wrap_socket(self._server.socket, server_side=True) host, port = self._server.server_address[:2] - self._server.upstream = Upstream(url=f"http://{host!s}:{port}") + self._server.upstream = Upstream(url=f"https://{host!s}:{port}", ca_bundle=cert_path) self._thread = threading.Thread(target=self._server.serve_forever, args=(0.01,), daemon=True) self._thread.start() return self._server.upstream diff --git a/tests/unit/test_git_relay.py b/tests/unit/test_git_relay.py index befa3ccd..3b1e9c61 100644 --- a/tests/unit/test_git_relay.py +++ b/tests/unit/test_git_relay.py @@ -18,8 +18,9 @@ @pytest.fixture -def upstream(): +def upstream(monkeypatch): with running_upstream() as upstream: + monkeypatch.setenv("REQUESTS_CA_BUNDLE", upstream.ca_bundle) yield upstream @@ -108,7 +109,7 @@ def test_other_headers_stay_behind(self, relay, upstream): ) [request] = upstream.requests assert request.headers["Authorization"] == basic_auth("x-access-token", TOKEN) - assert request.headers["Host"] == upstream.url.removeprefix("http://") + assert request.headers["Host"] == upstream.url.removeprefix("https://") for name in ("X-Scm-Git-Grant", "Cookie", "X-Forwarded-For"): assert name not in request.headers @@ -122,6 +123,14 @@ def test_asks_for_an_unencoded_response_unless_the_client_says_otherwise(self, r b"".join(fetch_refs(relay, upstream).content) assert upstream.requests[0].headers["Accept-Encoding"] == "identity" + def test_refuses_plain_http(self, relay, upstream): + remote = GitRemote(url=upstream.url.replace("https://", "http://") + "/repo.git", username="x-access-token") + with pytest.raises(ValueError, match="not https"): + relay.forward( + remote, TOKEN, method="GET", route="info/refs", query="service=git-upload-pack", headers={}, body=None + ) + assert upstream.requests == [] + def test_refuses_routes_that_are_not_git(self, relay, upstream): with pytest.raises(GitRouteNotAllowed): relay.forward( @@ -298,7 +307,7 @@ def test_connection_refused(self, relay): with socket.socket() as s: s.bind(("127.0.0.1", 0)) port = s.getsockname()[1] - remote = GitRemote(url=f"http://127.0.0.1:{port}/getsentry/sentry.git", username="x-access-token") + remote = GitRemote(url=f"https://127.0.0.1:{port}/getsentry/sentry.git", username="x-access-token") with pytest.raises(ResourceServiceUnavailable) as exc_info: relay.forward( @@ -311,10 +320,7 @@ def test_read_timeout(self, upstream): def handler(h: UpstreamHandler) -> None: release.wait(WAIT) - try: - h.respond(200, b"") - except OSError: # The relay stopped waiting and closed the connection. - h.close_connection = True + h.respond(200, b"") upstream.handler = handler relay = GitRelay(timeout=(1, 0.1)) diff --git a/tests/unit/test_git_remote.py b/tests/unit/test_git_remote.py index ced9d9a6..87c68b57 100644 --- a/tests/unit/test_git_remote.py +++ b/tests/unit/test_git_remote.py @@ -1,6 +1,6 @@ import pytest -from scm.errors import GitProviderNotSupported, MalformedExternalId +from scm.errors import GitProviderNotSupported, InvalidWebBaseUrl, MalformedExternalId from scm.git.remote import git_remote from scm.git.types import GitRemote from scm.types import Repository @@ -35,11 +35,36 @@ def test_github_enterprise(web_base_url): ) -def test_github_enterprise_without_host(): - with pytest.raises(GitProviderNotSupported, match="no web_base_url"): +def test_github_enterprise_without_web_base_url(): + with pytest.raises(InvalidWebBaseUrl, match="no web_base_url"): git_remote(make_repository(provider_name="github_enterprise")) +@pytest.mark.parametrize( + "web_base_url", + ["http://ghe.example.com", "ghe.example.com", "https://", "https://ghe.example.com?x=1", ""], +) +@pytest.mark.parametrize("provider_name", ["github_enterprise", "gitlab"]) +def test_web_base_url_must_be_https(provider_name, web_base_url): + repository = make_repository(provider_name=provider_name, external_id="gitlab.com:42", web_base_url=web_base_url) + with pytest.raises(InvalidWebBaseUrl, match="not an https URL"): + git_remote(repository) + + +@pytest.mark.parametrize( + "web_base_url,url", + [ + ("https://git.example.com", "https://git.example.com/team/app.git"), + ("https://example.com/gitlab/", "https://example.com/gitlab/team/app.git"), + ], +) +def test_gitlab_prefers_web_base_url(web_base_url, url): + repository = make_repository( + provider_name="gitlab", external_id="ignored.example.com:42", name="team / app", web_base_url=web_base_url + ) + assert git_remote(repository) == GitRemote(url=url, username="oauth2") + + def test_gitlab(): repository = make_repository(provider_name="gitlab", external_id="gitlab.com:42", name="group / sub / project") assert git_remote(repository) == GitRemote(url="https://gitlab.com/group/sub/project.git", username="oauth2") @@ -51,7 +76,7 @@ def test_self_hosted_gitlab_with_port(): @pytest.mark.parametrize("external_id", [None, "42"]) -def test_gitlab_malformed_external_id(external_id): +def test_gitlab_without_web_base_url_needs_a_host_in_the_external_id(external_id): with pytest.raises(MalformedExternalId): git_remote(make_repository(provider_name="gitlab", external_id=external_id)) diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index 40841688..dc88a6bd 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -228,7 +228,7 @@ def test_forwards_with_remote_and_token(self): def test_upstream_unavailable(self): recorder = Recorder( - repository=make_repository(provider_name="github_enterprise", web_base_url="http://127.0.0.1:1") + repository=make_repository(provider_name="github_enterprise", web_base_url="https://127.0.0.1:1") ) relay = GitRelay(timeout=(1, 1)) try: @@ -239,8 +239,9 @@ def test_upstream_unavailable(self): assert TOKEN not in b"".join(response.content).decode() assert recorder.errors == [] - def test_relays_end_to_end(self): + def test_relays_end_to_end(self, monkeypatch): with running_upstream() as upstream: + monkeypatch.setenv("REQUESTS_CA_BUNDLE", upstream.ca_bundle) recorder = Recorder( repository=make_repository(provider_name="github_enterprise", web_base_url=upstream.url) ) diff --git a/uv.lock b/uv.lock index 7d683b5c..8379d0cf 100644 --- a/uv.lock +++ b/uv.lock @@ -550,6 +550,7 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "cryptography" }, { name = "mypy" }, { name = "prek" }, { name = "pyjwt", extra = ["crypto"] }, @@ -567,6 +568,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "cryptography", specifier = ">=48.0.1" }, { name = "mypy", specifier = ">=1.20.0" }, { name = "prek", specifier = ">=0.3.6" }, { name = "pyjwt", extras = ["crypto"], specifier = ">=2.9.0" }, From 35c92d8747b821ed816de54b13e511ffd22c9515 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 14:26:38 +0200 Subject: [PATCH 04/31] fix(git): Report an unverifiable provider certificate as such requests' SSLError is a kind of ConnectionError, so a provider whose TLS certificate could not be verified was reported as "Could not connect", which points at the network rather than the certificate. The relay now says the host's certificate could not be verified. Refs CW-2221 Co-Authored-By: Claude Opus 5.5 --- src/scm/errors.py | 6 ++++++ src/scm/git/relay.py | 7 +++++++ src/scm/rpc/errors.py | 1 + tests/unit/test_git_relay.py | 15 ++++++++++++++- 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/scm/errors.py b/src/scm/errors.py index b2a29c06..9fe33cb7 100644 --- a/src/scm/errors.py +++ b/src/scm/errors.py @@ -29,6 +29,7 @@ "resource_unprocessable_content", "resource_server_error", "resource_bad_gateway", + "resource_certificate_invalid", "resource_service_unavailable", "resource_gateway_timeout", "unexpected_response_format", @@ -69,6 +70,7 @@ "resource_unprocessable_content": "Request could not be processed.", "resource_server_error": "The service-provider encountered an internal error.", "resource_bad_gateway": "The service-provider returned an invalid response from an upstream gateway.", + "resource_certificate_invalid": "The service-provider's TLS certificate could not be verified.", "resource_service_unavailable": "The service-provider is temporarily unavailable. The request can be retried.", "resource_gateway_timeout": "The service-provider did not respond in time. The request can be retried.", "unexpected_response_format": "The response format was in an unexpected format.", @@ -251,6 +253,10 @@ class ResourceBadGateway(SCMCodedError): code = "resource_bad_gateway" +class ResourceCertificateInvalid(SCMCodedError): + code = "resource_certificate_invalid" + + class ResourceServiceUnavailable(SCMCodedError): code = "resource_service_unavailable" diff --git a/src/scm/git/relay.py b/src/scm/git/relay.py index 668bbb37..a41af65c 100644 --- a/src/scm/git/relay.py +++ b/src/scm/git/relay.py @@ -1,5 +1,6 @@ from collections.abc import Iterator, Mapping from http.cookiejar import DefaultCookiePolicy +from urllib.parse import urlsplit import requests from requests.adapters import HTTPAdapter @@ -9,6 +10,7 @@ from scm.errors import ( ClientClosedRequest, GitRouteNotAllowed, + ResourceCertificateInvalid, ResourceGatewayTimeout, ResourceServiceUnavailable, RpcMalformedRequestHeaders, @@ -117,6 +119,11 @@ def forward( # "from None" drops requests' exception, whose request holds the Authorization header. except requests.Timeout: raise ResourceGatewayTimeout(detail=f"{remote.url} did not respond in time.") from None + # SSLError is a kind of ConnectionError, so catch it before that one, or a bad certificate would be + # reported as an unreachable host. + except requests.exceptions.SSLError: + host = urlsplit(remote.url).netloc + raise ResourceCertificateInvalid(detail=f"The TLS certificate of {host} could not be verified.") from None except requests.ConnectionError: raise ResourceServiceUnavailable(detail=f"Could not connect to {remote.url}.") from None diff --git a/src/scm/rpc/errors.py b/src/scm/rpc/errors.py index a8ccf239..e44fd779 100644 --- a/src/scm/rpc/errors.py +++ b/src/scm/rpc/errors.py @@ -22,6 +22,7 @@ "resource_unprocessable_content": 422, "resource_server_error": 500, "resource_bad_gateway": 502, + "resource_certificate_invalid": 502, "resource_service_unavailable": 503, "resource_gateway_timeout": 504, "unexpected_response_format": 500, diff --git a/tests/unit/test_git_relay.py b/tests/unit/test_git_relay.py index 3b1e9c61..7859588e 100644 --- a/tests/unit/test_git_relay.py +++ b/tests/unit/test_git_relay.py @@ -7,7 +7,13 @@ import pytest -from scm.errors import GitRouteNotAllowed, ResourceGatewayTimeout, ResourceServiceUnavailable, SCMCodedError +from scm.errors import ( + GitRouteNotAllowed, + ResourceCertificateInvalid, + ResourceGatewayTimeout, + ResourceServiceUnavailable, + SCMCodedError, +) from scm.git.relay import GitRelay from scm.git.types import GitRemote from tests.unit.git_upstream import UpstreamHandler, basic_auth, running_upstream @@ -315,6 +321,13 @@ def test_connection_refused(self, relay): ) assert_token_not_in(exc_info.value) + def test_untrusted_certificate(self, relay, upstream, monkeypatch): + monkeypatch.delenv("REQUESTS_CA_BUNDLE") + with pytest.raises(ResourceCertificateInvalid, match="TLS certificate of 127.0.0.1") as exc_info: + fetch_refs(relay, upstream) + assert_token_not_in(exc_info.value) + assert upstream.requests == [] + def test_read_timeout(self, upstream): release = threading.Event() From a127814e1d17f7d484bfba1ded690ef75b759726 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 15:18:09 +0200 Subject: [PATCH 05/31] fix(git): Let the caller choose the relay's connection adapter A GitHub Enterprise or self-managed GitLab host is whatever the customer configured, and the relay connected wherever it resolved, including internal addresses. Sentry's API clients refuse those through BlacklistAdapter, which the library cannot import, so the relay now takes an adapter for Sentry's endpoint to pass in. Without one it keeps its plain pooled adapter. The Grant docstring also claimed the repository list covers git dependencies without a new format; that holds only for repositories Sentry has connected for the organization. Refs CW-2221 Co-Authored-By: Claude Opus 5.5 --- src/scm/git/relay.py | 15 +++++++++++---- src/scm/git/server.py | 2 +- src/scm/git/types.py | 5 +++-- tests/unit/test_git_relay.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/scm/git/relay.py b/src/scm/git/relay.py index a41af65c..54c2217e 100644 --- a/src/scm/git/relay.py +++ b/src/scm/git/relay.py @@ -52,13 +52,19 @@ class GitRelay: new connection for each added hundreds of milliseconds per clone in the prototype. Only https remotes are accepted, because every request carries a provider token. + + ``adapter`` sends every upstream request and holds the connection pools. Pass one that refuses internal + addresses: a GitHub Enterprise or self-managed GitLab host is whatever the customer configured, and the relay + connects wherever it resolves. Sentry's ``BlacklistAdapter`` does this for its API clients. Errors the adapter + raises for a refused host reach the caller unchanged. The adapter must not retry, because a retry cannot + resend a body that has already been streamed. Without one, the relay pools connections with a plain adapter + that checks nothing. """ def __init__( self, *, - max_hosts: int = 32, - max_connections_per_host: int = 64, + adapter: HTTPAdapter | None = None, timeout: tuple[float, float] = DEFAULT_TIMEOUT, chunk_size: int = DEFAULT_CHUNK_SIZE, ) -> None: @@ -68,8 +74,9 @@ def __init__( # Every organization's requests share this session, so a cookie a provider sets on one request must not # be sent with the next. self.session.cookies.set_policy(DefaultCookiePolicy(allowed_domains=[])) - adapter = HTTPAdapter(pool_connections=max_hosts, pool_maxsize=max_connections_per_host, max_retries=0) - self.session.mount("https://", adapter) + # The default keeps pools for up to 32 provider hosts and up to 64 idle connections per host. These are + # guesses for one Sentry worker's concurrent clones, not measurements. + self.session.mount("https://", adapter or HTTPAdapter(pool_connections=32, pool_maxsize=64, max_retries=0)) def forward( self, diff --git a/src/scm/git/server.py b/src/scm/git/server.py index ce994022..7b4b2f49 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -20,7 +20,7 @@ class GitServer: The order of the checks matters. The grant is verified, and the request's route and repository checked against it, before the repository is looked up or a token fetched, so a request the grant does not allow never reaches the database, the token supplier or the provider. Build one per process: its relay pools - upstream connections. + upstream connections. Pass a relay whose adapter refuses internal addresses; see ``GitRelay``. """ def __init__( diff --git a/src/scm/git/types.py b/src/scm/git/types.py index 218bcbe2..cd08734f 100644 --- a/src/scm/git/types.py +++ b/src/scm/git/types.py @@ -20,8 +20,9 @@ class Grant(msgspec.Struct, frozen=True, kw_only=True, forbid_unknown_fields=Tru Seer signs a grant per coding job and Sentry's git endpoint verifies it on every request. Each repository is a ``RepositoryId`` under the grant's organization, as in the RPC endpoint: Sentry's repository id or a - ``(provider, external id)`` pair. A fetch grant holds one repository today; the list leaves room for - submodules and git-based dependencies without a new format. + ``(provider, external id)`` pair. A fetch grant holds one repository today. The list leaves room for + submodules and git-based dependencies, but only from repositories Sentry has connected for the organization: + naming any other repository, such as a public dependency, would need a new kind of entry and a new version. A grant with a field this version does not know is refused rather than read without it. If a later version adds a field that restricts a grant, a verifier that ignored the field would allow more than the grant does. diff --git a/tests/unit/test_git_relay.py b/tests/unit/test_git_relay.py index 7859588e..d4a688a6 100644 --- a/tests/unit/test_git_relay.py +++ b/tests/unit/test_git_relay.py @@ -6,6 +6,7 @@ from collections.abc import Iterator import pytest +from requests.adapters import HTTPAdapter from scm.errors import ( GitRouteNotAllowed, @@ -308,6 +309,34 @@ def handler(h: UpstreamHandler) -> None: assert [r.headers.get("Cookie") for r in upstream.requests] == [None, None] +class TestAdapter: + def test_uses_the_adapter_it_is_given(self, upstream): + # Sentry passes an adapter that refuses internal addresses; the relay must send through it. + class HostRefused(Exception): + pass + + class RefusingAdapter(HTTPAdapter): + def send(self, request, *args, **kwargs): + raise HostRefused(request.url) + + relay = GitRelay(adapter=RefusingAdapter()) + try: + with pytest.raises(HostRefused): + fetch_refs(relay, upstream) + finally: + relay.close() + assert upstream.requests == [] + + def test_given_adapter_also_pools_connections(self, upstream): + relay = GitRelay(adapter=HTTPAdapter(max_retries=0)) + try: + b"".join(fetch_refs(relay, upstream).content) + b"".join(fetch_refs(relay, upstream).content) + finally: + relay.close() + assert upstream.connections == 1 + + class TestUpstreamFailures: def test_connection_refused(self, relay): with socket.socket() as s: From 6d2d2bf85cc3f25508ceafa7506a36d68cae9ebc Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 16:12:13 +0200 Subject: [PATCH 06/31] fix(git): Take the GitLab host only from web_base_url The host in a GitLab repository's external id is only the instance's hostname, so the remote built from it guessed https and lost the path prefix of an instance served under one. getsentry/sentry#128164 makes Sentry fill web_base_url for GitLab from the integration's instance URL, so GitLab now requires it the way GitHub Enterprise does, and the fallback goes. Refs CW-2221 Co-Authored-By: Claude Opus 5.5 --- src/scm/git/remote.py | 29 ++++++++++------------------ tests/unit/test_git_remote.py | 36 +++++++++++------------------------ 2 files changed, 21 insertions(+), 44 deletions(-) diff --git a/src/scm/git/remote.py b/src/scm/git/remote.py index 298ad9a1..fce45893 100644 --- a/src/scm/git/remote.py +++ b/src/scm/git/remote.py @@ -1,6 +1,6 @@ from urllib.parse import urlsplit -from scm.errors import GitProviderNotSupported, InvalidWebBaseUrl, MalformedExternalId +from scm.errors import GitProviderNotSupported, InvalidWebBaseUrl from scm.git.types import GitRemote from scm.types import Repository @@ -17,32 +17,23 @@ def git_remote(repository: Repository) -> GitRemote: case "github": return GitRemote(url=f"{GITHUB_GIT_BASE_URL}/{repository['name']}.git", username="x-access-token") case "github_enterprise": - if repository["web_base_url"] is None: - raise InvalidWebBaseUrl(detail="The GitHub Enterprise repository has no web_base_url.") - base_url = _https_base_url(repository["web_base_url"]) + base_url = _https_base_url(repository) return GitRemote(url=f"{base_url}/{repository['name']}.git", username="x-access-token") case "gitlab": # Sentry's repository name for GitLab is the namespace path with spaces around each slash. path = repository["name"].replace(" ", "") - return GitRemote(url=f"{_gitlab_base_url(repository)}/{path}.git", username="oauth2") + return GitRemote(url=f"{_https_base_url(repository)}/{path}.git", username="oauth2") case provider_name: raise GitProviderNotSupported(detail=f"The {provider_name} provider has no git remote.") -def _gitlab_base_url(repository: Repository) -> str: - if repository["web_base_url"] is not None: - return _https_base_url(repository["web_base_url"]) - # Sentry does not set web_base_url for GitLab yet. Until it does, take the host from the external id, - # ":", and assume https, as GitLabProvider does. That loses an instance served under a - # path prefix, which only web_base_url carries. - external_id = repository["external_id"] - if external_id is None or ":" not in external_id: - raise MalformedExternalId() - return f"https://{external_id.rsplit(':', maxsplit=1)[0]}" - - -def _https_base_url(web_base_url: str) -> str: - # The token goes to this URL, so it must be https. +def _https_base_url(repository: Repository) -> str: + # The token goes to this URL, so it must be https. Sentry sets web_base_url for every GitHub Enterprise and + # GitLab repository from the integration's instance URL; for GitLab that keeps the path prefix of an instance + # served under one. + web_base_url = repository["web_base_url"] + if web_base_url is None: + raise InvalidWebBaseUrl(detail=f"The {repository['provider_name']} repository has no web_base_url.") parts = urlsplit(web_base_url) if parts.scheme != "https" or not parts.netloc or parts.query or parts.fragment: raise InvalidWebBaseUrl(detail=f"The repository's web_base_url {web_base_url!r} is not an https URL.") diff --git a/tests/unit/test_git_remote.py b/tests/unit/test_git_remote.py index 87c68b57..ad3ec1cf 100644 --- a/tests/unit/test_git_remote.py +++ b/tests/unit/test_git_remote.py @@ -1,6 +1,6 @@ import pytest -from scm.errors import GitProviderNotSupported, InvalidWebBaseUrl, MalformedExternalId +from scm.errors import GitProviderNotSupported, InvalidWebBaseUrl from scm.git.remote import git_remote from scm.git.types import GitRemote from scm.types import Repository @@ -35,9 +35,10 @@ def test_github_enterprise(web_base_url): ) -def test_github_enterprise_without_web_base_url(): - with pytest.raises(InvalidWebBaseUrl, match="no web_base_url"): - git_remote(make_repository(provider_name="github_enterprise")) +@pytest.mark.parametrize("provider_name", ["github_enterprise", "gitlab"]) +def test_self_hosted_providers_need_a_web_base_url(provider_name): + with pytest.raises(InvalidWebBaseUrl, match=f"{provider_name} repository has no web_base_url"): + git_remote(make_repository(provider_name=provider_name)) @pytest.mark.parametrize( @@ -46,7 +47,7 @@ def test_github_enterprise_without_web_base_url(): ) @pytest.mark.parametrize("provider_name", ["github_enterprise", "gitlab"]) def test_web_base_url_must_be_https(provider_name, web_base_url): - repository = make_repository(provider_name=provider_name, external_id="gitlab.com:42", web_base_url=web_base_url) + repository = make_repository(provider_name=provider_name, web_base_url=web_base_url) with pytest.raises(InvalidWebBaseUrl, match="not an https URL"): git_remote(repository) @@ -54,33 +55,18 @@ def test_web_base_url_must_be_https(provider_name, web_base_url): @pytest.mark.parametrize( "web_base_url,url", [ - ("https://git.example.com", "https://git.example.com/team/app.git"), - ("https://example.com/gitlab/", "https://example.com/gitlab/team/app.git"), + ("https://gitlab.com", "https://gitlab.com/group/sub/project.git"), + ("https://git.example.com:8443/", "https://git.example.com:8443/group/sub/project.git"), + ("https://example.com/gitlab", "https://example.com/gitlab/group/sub/project.git"), ], ) -def test_gitlab_prefers_web_base_url(web_base_url, url): +def test_gitlab(web_base_url, url): repository = make_repository( - provider_name="gitlab", external_id="ignored.example.com:42", name="team / app", web_base_url=web_base_url + provider_name="gitlab", external_id="gitlab.com:42", name="group / sub / project", web_base_url=web_base_url ) assert git_remote(repository) == GitRemote(url=url, username="oauth2") -def test_gitlab(): - repository = make_repository(provider_name="gitlab", external_id="gitlab.com:42", name="group / sub / project") - assert git_remote(repository) == GitRemote(url="https://gitlab.com/group/sub/project.git", username="oauth2") - - -def test_self_hosted_gitlab_with_port(): - repository = make_repository(provider_name="gitlab", external_id="git.example.com:8443:42", name="team / app") - assert git_remote(repository) == GitRemote(url="https://git.example.com:8443/team/app.git", username="oauth2") - - -@pytest.mark.parametrize("external_id", [None, "42"]) -def test_gitlab_without_web_base_url_needs_a_host_in_the_external_id(external_id): - with pytest.raises(MalformedExternalId): - git_remote(make_repository(provider_name="gitlab", external_id=external_id)) - - @pytest.mark.parametrize("provider_name", ["bitbucket", "cursor_origin"]) def test_unsupported_providers(provider_name): with pytest.raises(GitProviderNotSupported, match=provider_name): From f21a1278972f4826cd8a760968042d441df8f3dd Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 16:16:31 +0200 Subject: [PATCH 07/31] fix(deps): Require urllib3 2.8.0 for the git relay's streaming The git relay reads provider responses with urllib3's chunked streaming, and a GitHub Enterprise or GitLab host is whatever a customer configured. Before urllib3 2.8.0 such a host could make that reader buffer without limit (GHSA-vxq7-64xx-v4gw), so the floor is declared here, where consumers inherit it, rather than only in this repo's lock. The test server's TLS context now also refuses TLS 1.0 and 1.1, which CodeQL flagged. Refs CW-2221 Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 3 +++ tests/unit/git_upstream.py | 1 + uv.lock | 8 +++++--- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 91217842..aca2574b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,9 @@ requires-python = ">=3.13" dependencies = [ "msgspec>=0.19.0", "requests>=2.32.4", + # The git relay streams responses through urllib3 directly. Before 2.8.0, a server could make its chunked + # reader buffer without limit (GHSA-vxq7-64xx-v4gw). + "urllib3>=2.8.0", ] [build-system] diff --git a/tests/unit/git_upstream.py b/tests/unit/git_upstream.py index 9f32446d..a66c596b 100644 --- a/tests/unit/git_upstream.py +++ b/tests/unit/git_upstream.py @@ -161,6 +161,7 @@ class running_upstream: def __enter__(self) -> Upstream: cert_path, key_path = _certificate() context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + context.minimum_version = ssl.TLSVersion.TLSv1_2 context.load_cert_chain(cert_path, key_path) self._server = _Server(("127.0.0.1", 0), UpstreamHandler) self._server.socket = context.wrap_socket(self._server.socket, server_side=True) diff --git a/uv.lock b/uv.lock index 21135155..7bd60984 100644 --- a/uv.lock +++ b/uv.lock @@ -546,6 +546,7 @@ source = { editable = "." } dependencies = [ { name = "msgspec" }, { name = "requests" }, + { name = "urllib3" }, ] [package.dev-dependencies] @@ -564,6 +565,7 @@ dev = [ requires-dist = [ { name = "msgspec", specifier = ">=0.19.0" }, { name = "requests", specifier = ">=2.32.4" }, + { name = "urllib3", specifier = ">=2.8.0" }, ] [package.metadata.requires-dev] @@ -601,9 +603,9 @@ wheels = [ [[package]] name = "urllib3" -version = "2.6.3" +version = "2.8.0" source = { registry = "https://sfw.security.sentry.io/pypi/simple" } -sdist = { url = "https://sfw.security.sentry.io/pypi/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +sdist = { url = "https://sfw.security.sentry.io/pypi/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } wheels = [ - { url = "https://sfw.security.sentry.io/pypi/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, + { url = "https://sfw.security.sentry.io/pypi/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, ] From 747d120939a0d1077bd5c3889835c42d841b0a95 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 16:28:16 +0200 Subject: [PATCH 08/31] fix(git): Answer a provider's 401 with a 502 GitServer rewrote its own 401s to 403 so git would not ask a sandboxed client for credentials, but a 401 from the provider, from an expired or rejected token, still went through unchanged. That is Sentry's credential failing, not the client's, so it becomes a 502 with a plain-text reason, and the upstream response is closed. Refs CW-2221 Co-Authored-By: Claude Opus 5.5 --- src/scm/git/server.py | 14 +++++++++++++- tests/unit/test_git_server.py | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/scm/git/server.py b/src/scm/git/server.py index 7b4b2f49..c5326e37 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -2,7 +2,7 @@ from requests.structures import CaseInsensitiveDict -from scm.errors import ProviderNotFound, SCMCodedError +from scm.errors import ProviderNotFound, ResourceBadGateway, SCMCodedError from scm.git.grant import verify_grant from scm.git.operations import check_request from scm.git.relay import GitRelay @@ -67,6 +67,11 @@ def handle( response = self.relay.forward( remote, token, method=method, route=route, query=query, headers=headers, body=body ) + if response.status_code == 401: + # The provider rejected the token Sentry supplied. Passed on, the 401 would make git ask the client + # for credentials, which it must never have and which would not help. + _close(response) + raise ResourceBadGateway(detail="The provider rejected Sentry's credentials for this repository.") except SCMCodedError as e: self.record_count("sentry.scm.git.request.failed", 1, {"code": e.code}) if e.code == "unhandled_exception": @@ -85,6 +90,13 @@ def handle( return response +def _close(response: StreamResponse) -> None: + # Closing the relay's stream before it is read to the end drops the upstream connection. + close = getattr(response.content, "close", None) + if close is not None: + close() + + def _error_response(error: SCMCodedError) -> StreamResponse: # On a 401 git asks the user for credentials, which a sandboxed client must never have, so refuse with a 403. status = SPECIAL_STATUS_MAP.get(error.code, 400) diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index dc88a6bd..502382f6 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -239,6 +239,26 @@ def test_upstream_unavailable(self): assert TOKEN not in b"".join(response.content).decode() assert recorder.errors == [] + def test_provider_rejecting_the_token_is_a_gateway_error(self): + # Passed on, a 401 would make git ask the sandboxed client for credentials. + closed = [] + + def provider_body(): + try: + yield b"Bad credentials" + finally: + closed.append(True) + + content = provider_body() + next(content) # Started, as a relay's stream is once its first chunk is read. + relay = MagicMock(spec=GitRelay) + relay.forward.return_value = StreamResponse(status_code=401, headers={}, content=content) + recorder = Recorder() + response = fetch_refs(make_server(recorder, relay), make_grant()) + assert_refused(response, 502, "rejected Sentry's credentials") + assert closed == [True] + assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "resource_bad_gateway"})] + def test_relays_end_to_end(self, monkeypatch): with running_upstream() as upstream: monkeypatch.setenv("REQUESTS_CA_BUNDLE", upstream.ca_bundle) From 026b7c7a7dc5bf726d5dd40f8b946eed7137e234 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 16:43:32 +0200 Subject: [PATCH 09/31] feat(git): Sign grants as HS256 JWTs Sentry and Seer already sign the tokens that cross between them as HS256 JWTs (viewer contexts), and the HMAC rpcsignature scheme the first version copied is the deprecated one. A grant is now a JWT: its fields are claims next to exp and iat, an aud of scm-git-grant keeps other tokens signed with the same secret, such as viewer contexts, from passing as grants, and a kid header names the signing secret the way viewer contexts do, so secrets can still rotate. The claims still go through the strict Grant struct, so unknown fields and versions are refused as before. PyJWT becomes a runtime dependency; Sentry and Seer both use it already. Refs CW-2221 Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 1 + src/scm/git/grant.py | 83 +++++++++++---------- tests/unit/test_git_grant.py | 135 ++++++++++++++++++---------------- tests/unit/test_git_server.py | 9 ++- uv.lock | 2 + 5 files changed, 125 insertions(+), 105 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aca2574b..784193d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ authors = [ requires-python = ">=3.13" dependencies = [ "msgspec>=0.19.0", + "pyjwt>=2.9.0", "requests>=2.32.4", # The git relay streams responses through urllib3 directly. Before 2.8.0, a server could make its chunked # reader buffer without limit (GHSA-vxq7-64xx-v4gw). diff --git a/src/scm/git/grant.py b/src/scm/git/grant.py index e4db07ca..30edbe35 100644 --- a/src/scm/git/grant.py +++ b/src/scm/git/grant.py @@ -1,32 +1,38 @@ """Signing and verifying git grants. -The wire format is ``.``, where the payload is the grant's JSON and the -signature is an HMAC-SHA256 over a fixed prefix and the payload. Because of the prefix, a signature the RPC -endpoint makes with the same secret can never pass as a grant signature. +A grant travels as an HS256 JWT, the format Sentry and Seer already sign viewer contexts with. The grant's fields +are claims, alongside ``exp``, ``iat`` and an ``aud`` of ``scm-git-grant``. The verifier requires that audience, +so no other token signed with the same secret, such as a viewer context, passes as a grant. A ``kid`` header +names the secret that signed the grant, as in Sentry's viewer contexts, so the verifier can hold several secrets +while they rotate. """ -import base64 import hashlib -import hmac -import re import time +from typing import Any +import jwt import msgspec from scm.errors import GitInvalidGrant from scm.git.types import Grant -_SIGNATURE_DOMAIN = b"scm-git-grant." -_BASE64URL = re.compile(r"[A-Za-z0-9_-]+") +GRANT_AUDIENCE = "scm-git-grant" +_ALGORITHM = "HS256" +# Claims the JWT layer owns. Every other claim must be a Grant field. +_JWT_CLAIMS = frozenset({"aud", "exp", "iat"}) -def sign_grant(secret: str, grant: Grant) -> str: - payload = msgspec.json.encode(grant) - return f"{_b64encode(payload)}.{_b64encode(_signature(secret, payload))}" +def sign_grant(secret: str, grant: Grant, *, now: float | None = None) -> str: + claims: dict[str, Any] = msgspec.to_builtins(grant) + claims["exp"] = claims.pop("expires_at") + claims["iat"] = int(time.time() if now is None else now) + claims["aud"] = GRANT_AUDIENCE + return jwt.encode(claims, secret, algorithm=_ALGORITHM, headers={"kid": _key_id(secret)}) def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> Grant: - """Return the grant if any of ``secrets`` signed it and it has not expired. + """Return the grant if one of ``secrets`` signed it and it has not expired. Verification takes a list of secrets, as ``RpcServer`` does, so a new secret can be added before signers switch to it and the old one removed after their grants expire. @@ -34,25 +40,37 @@ def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> if not token: raise GitInvalidGrant(detail="The git grant is missing.") try: - payload_b64, signature_b64 = token.split(".") - payload = _b64decode(payload_b64) - signature = _b64decode(signature_b64) - except ValueError: # Raised both for the wrong number of parts and for bad base64 (binascii.Error). + kid = jwt.get_unverified_header(token).get("kid") + except jwt.DecodeError: raise GitInvalidGrant(detail="The git grant is malformed.") from None + secret = next((secret for secret in secrets if _key_id(secret) == kid), None) + if secret is None: + raise GitInvalidGrant(detail="The git grant was not signed with a known secret.") - # Check every secret, not just until one matches, so the time taken does not reveal which one signed it. - matches = [hmac.compare_digest(signature, _signature(secret, payload)) for secret in secrets] - if not any(matches): - raise GitInvalidGrant(detail="The git grant's signature does not match.") + try: + claims = jwt.decode( + token, + secret, + algorithms=[_ALGORITHM], + audience=GRANT_AUDIENCE, + # Expiry is checked below against an injectable clock, so tests can pin it. + options={"require": sorted(_JWT_CLAIMS), "verify_exp": False, "verify_iat": False}, + ) + except jwt.InvalidSignatureError: + raise GitInvalidGrant(detail="The git grant's signature does not match.") from None + except jwt.DecodeError: + raise GitInvalidGrant(detail="The git grant is malformed.") from None + except jwt.InvalidTokenError as e: + # The signature matched or was never reached, and PyJWT's reasons name only claims, not values. + raise GitInvalidGrant(detail=f"The git grant is invalid: {e}") from None + fields = {name: value for name, value in claims.items() if name not in _JWT_CLAIMS} try: - grant = msgspec.json.decode(payload, type=Grant) + grant = msgspec.convert({**fields, "expires_at": claims["exp"]}, type=Grant, strict=True) except msgspec.ValidationError as e: - # The signature matched, so a holder of the secret wrote this payload; telling them why it is invalid + # The signature matched, so a holder of the secret wrote these claims; telling them why they are invalid # gives nothing away. raise GitInvalidGrant(detail=f"The git grant's payload is invalid: {e}") from None - except msgspec.DecodeError: - raise GitInvalidGrant(detail="The git grant's payload is not JSON.") from None if grant.expires_at <= (time.time() if now is None else now): raise GitInvalidGrant(detail="The git grant has expired.") @@ -61,17 +79,6 @@ def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> return grant -def _signature(secret: str, payload: bytes) -> bytes: - return hmac.new(secret.encode(), _SIGNATURE_DOMAIN + payload, hashlib.sha256).digest() - - -def _b64encode(data: bytes) -> str: - return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") - - -def _b64decode(data: str) -> bytes: - # urlsafe_b64decode silently drops characters outside its alphabet. Refuse them instead, so a grant has only - # one valid spelling. - if not _BASE64URL.fullmatch(data): - raise ValueError("not base64url") - return base64.urlsafe_b64decode(data + "=" * (-len(data) % 4)) +def _key_id(secret: str) -> str: + # The first 8 hex characters of the secret's SHA-256, the same key id Sentry's viewer contexts use. + return hashlib.sha256(secret.encode()).hexdigest()[:8] diff --git a/tests/unit/test_git_grant.py b/tests/unit/test_git_grant.py index a5f5401b..9346a65d 100644 --- a/tests/unit/test_git_grant.py +++ b/tests/unit/test_git_grant.py @@ -1,18 +1,19 @@ import base64 import hashlib -import hmac -import json +import jwt import msgspec import pytest from scm.errors import GitInvalidGrant -from scm.git.grant import sign_grant, verify_grant +from scm.git.grant import GRANT_AUDIENCE, sign_grant, verify_grant from scm.git.types import Grant -from scm.rpc.helpers import sign_post NOW = 1_800_000_000 -SECRET = "secret" +# HS256 secrets of at least 32 bytes, as PyJWT recommends. +SECRET = "test-secret-0000000000000000000000" +OLD_SECRET = "test-secret-old-000000000000000000" +NEW_SECRET = "test-secret-new-000000000000000000" def make_grant(**overrides) -> Grant: @@ -26,18 +27,19 @@ def make_grant(**overrides) -> Grant: return Grant(**fields) # type: ignore[arg-type] -def b64(data: bytes) -> str: - return base64.urlsafe_b64encode(data).rstrip(b"=").decode() +def key_id(secret: str) -> str: + return hashlib.sha256(secret.encode()).hexdigest()[:8] -def sign_payload(payload: bytes, secret: str = SECRET) -> str: - """Sign arbitrary bytes the way sign_grant does, to reach the checks that run after the signature.""" - signature = hmac.new(secret.encode(), b"scm-git-grant." + payload, hashlib.sha256).digest() - return f"{b64(payload)}.{b64(signature)}" +def claims_of(grant: Grant) -> dict: + claims = msgspec.to_builtins(grant) + claims["exp"] = claims.pop("expires_at") + return claims | {"iat": NOW, "aud": GRANT_AUDIENCE} -def payload_of(grant: Grant) -> dict: - return json.loads(msgspec.json.encode(grant)) +def sign_claims(claims: dict, secret: str = SECRET, algorithm: str = "HS256", kid: str | None = None) -> str: + """Sign arbitrary claims the way sign_grant does, to reach the checks that run after the signature.""" + return jwt.encode(claims, secret, algorithm=algorithm, headers={"kid": kid or key_id(secret)}) def assert_invalid(token: str, detail: str, secrets: list[str] | None = None) -> None: @@ -49,20 +51,24 @@ def assert_invalid(token: str, detail: str, secrets: list[str] | None = None) -> class TestRoundTrip: def test_verifies_what_it_signed(self): grant = make_grant() - assert verify_grant([SECRET], sign_grant(SECRET, grant), now=NOW) == grant - - def test_keeps_both_repository_id_forms(self): - grant = verify_grant([SECRET], sign_grant(SECRET, make_grant()), now=NOW) - assert grant.repositories == (5, ("github", "123")) + assert verify_grant([SECRET], sign_grant(SECRET, grant, now=NOW), now=NOW) == grant def test_push_grant(self): grant = make_grant(operation="push") - assert verify_grant([SECRET], sign_grant(SECRET, grant), now=NOW).operation == "push" - - def test_payload_carries_the_version(self): - token = sign_grant(SECRET, make_grant()) - payload = json.loads(base64.urlsafe_b64decode(token.split(".")[0] + "==")) - assert payload["version"] == 1 + assert verify_grant([SECRET], sign_grant(SECRET, grant, now=NOW), now=NOW).operation == "push" + + def test_token_is_a_jwt_with_the_grants_claims(self): + token = sign_grant(SECRET, make_grant(), now=NOW) + assert jwt.get_unverified_header(token) == {"alg": "HS256", "typ": "JWT", "kid": key_id(SECRET)} + assert jwt.decode(token, options={"verify_signature": False}) == { + "organization_id": 1, + "operation": "fetch", + "repositories": [5, ["github", "123"]], + "version": 1, + "exp": NOW + 600, + "iat": NOW, + "aud": GRANT_AUDIENCE, + } class TestExpiry: @@ -76,70 +82,72 @@ def test_valid_until_its_expiry_time(self): token = sign_grant(SECRET, make_grant(expires_at=NOW + 1)) assert verify_grant([SECRET], token, now=NOW).expires_at == NOW + 1 - def test_expiry_is_required(self): - payload = payload_of(make_grant()) - del payload["expires_at"] - assert_invalid(sign_payload(json.dumps(payload).encode()), "expires_at") + @pytest.mark.parametrize("claim", ["exp", "iat", "aud"]) + def test_jwt_claims_are_required(self, claim): + claims = claims_of(make_grant()) + del claims[claim] + assert_invalid(sign_claims(claims), claim) class TestTampering: def test_changed_payload(self): - token = sign_grant(SECRET, make_grant()) - _, signature = token.split(".") - forged = msgspec.json.encode(make_grant(organization_id=2)) - assert_invalid(f"{b64(forged)}.{signature}", "signature does not match") + header, _, signature = sign_grant(SECRET, make_grant()).split(".") + forged = base64.urlsafe_b64encode(msgspec.json.encode(claims_of(make_grant(organization_id=2)))) + assert_invalid(f"{header}.{forged.rstrip(b'=').decode()}.{signature}", "signature does not match") def test_changed_signature(self): - payload, signature = sign_grant(SECRET, make_grant()).split(".") + header, payload, signature = sign_grant(SECRET, make_grant()).split(".") flipped = bytearray(base64.urlsafe_b64decode(signature + "=")) flipped[0] ^= 1 - assert_invalid(f"{payload}.{b64(bytes(flipped))}", "signature does not match") + resigned = base64.urlsafe_b64encode(bytes(flipped)).rstrip(b"=").decode() + assert_invalid(f"{header}.{payload}.{resigned}", "signature does not match") + + def test_unknown_secret(self): + assert_invalid(sign_grant("other-secret-0000000000000000000000", make_grant()), "not signed with a known") - def test_other_secret(self): - assert_invalid(sign_grant("other", make_grant()), "signature does not match") + def test_other_secret_claiming_a_known_key_id(self): + token = sign_claims(claims_of(make_grant()), secret="other-secret-0000000000000000000000", kid=key_id(SECRET)) + assert_invalid(token, "signature does not match") def test_no_secrets(self): with pytest.raises(GitInvalidGrant): verify_grant([], sign_grant(SECRET, make_grant()), now=NOW) - def test_rpc_signature_over_the_same_payload_is_not_a_grant_signature(self): - payload = msgspec.json.encode(make_grant()) - rpc_signature = bytes.fromhex(sign_post(SECRET, payload).removeprefix("rpc0:")) - assert_invalid(f"{b64(payload)}.{b64(rpc_signature)}", "signature does not match") + def test_token_without_an_audience(self): + # A viewer context, for one, is an HS256 JWT signed with a shared secret and carries no audience. + claims = claims_of(make_grant()) + del claims["aud"] + assert_invalid(sign_claims(claims), '"aud"') + def test_token_for_another_audience(self): + assert_invalid(sign_claims(claims_of(make_grant()) | {"aud": "scm-rpc"}), "Audience doesn't match") + + def test_unsigned_token(self): + token = jwt.encode(claims_of(make_grant()), None, algorithm="none", headers={"kid": key_id(SECRET)}) + assert_invalid(token, "invalid") + + @pytest.mark.filterwarnings("ignore::jwt.warnings.InsecureKeyLengthWarning") + def test_other_algorithm(self): + assert_invalid(sign_claims(claims_of(make_grant()), algorithm="HS512"), "invalid") -class TestRotatedSecrets: - def test_old_secret_verifies_during_rotation(self): - assert verify_grant(["new", "old"], sign_grant("old", make_grant()), now=NOW) == make_grant() - def test_new_secret_verifies_during_rotation(self): - assert verify_grant(["new", "old"], sign_grant("new", make_grant()), now=NOW) == make_grant() +class TestRotatedSecrets: + @pytest.mark.parametrize("signer", [OLD_SECRET, NEW_SECRET]) + def test_both_secrets_verify_during_rotation(self, signer): + assert verify_grant([NEW_SECRET, OLD_SECRET], sign_grant(signer, make_grant()), now=NOW) == make_grant() def test_old_secret_refused_after_rotation(self): - assert_invalid(sign_grant("old", make_grant()), "signature does not match", secrets=["new"]) + assert_invalid(sign_grant(OLD_SECRET, make_grant()), "not signed with a known", secrets=[NEW_SECRET]) class TestMalformed: - @pytest.mark.parametrize( - "token", - [ - "nodot", - "a.b.c", - "abc.d+f", - "ab=c.def", - "abc.dé", - ".", - ], - ) + @pytest.mark.parametrize("token", ["nodot", "a.b.c", "a.b", "...", "é.é.é"]) def test_malformed_token(self, token): assert_invalid(token, "malformed") def test_missing(self): assert_invalid("", "missing") - def test_signed_but_not_json(self): - assert_invalid(sign_payload(b"not json"), "not JSON") - @pytest.mark.parametrize( "change", [ @@ -149,13 +157,12 @@ def test_signed_but_not_json(self): {"version": 2}, {"repositories": [["github"]]}, {"repositories": [["unknown_provider", "123"]]}, - {"expires_at": 1.5}, + {"exp": NOW + 600.5}, {"narrower_than_you_think": True}, ], ) - def test_signed_but_invalid_payload(self, change): - payload = payload_of(make_grant()) | change - assert_invalid(sign_payload(json.dumps(payload).encode()), "payload is invalid") + def test_signed_but_invalid_claims(self, change): + assert_invalid(sign_claims(claims_of(make_grant()) | change), "payload is invalid") def test_no_repositories(self): assert_invalid(sign_grant(SECRET, make_grant(repositories=())), "names no repositories") diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index 502382f6..bb7f3dd3 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -12,7 +12,7 @@ from scm.types import Repository, RepositoryId from tests.unit.git_upstream import basic_auth, running_upstream -SECRET = "secret" +SECRET = "test-secret-0000000000000000000000" TOKEN = "ghs_secrettoken" @@ -105,8 +105,11 @@ class TestRefusedBeforeTheProvider: (None, "missing"), ("garbage", "malformed"), ( - sign_grant("other", Grant(organization_id=1, operation="fetch", repositories=(5,), expires_at=2**40)), - "signature", + sign_grant( + "other-secret-0000000000000000000000", + Grant(organization_id=1, operation="fetch", repositories=(5,), expires_at=2**40), + ), + "not signed with a known secret", ), (make_grant(expires_at=int(time.time()) - 1), "expired"), ], diff --git a/uv.lock b/uv.lock index 7bd60984..ba6a3bcc 100644 --- a/uv.lock +++ b/uv.lock @@ -545,6 +545,7 @@ version = "1.9.0" source = { editable = "." } dependencies = [ { name = "msgspec" }, + { name = "pyjwt" }, { name = "requests" }, { name = "urllib3" }, ] @@ -564,6 +565,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "msgspec", specifier = ">=0.19.0" }, + { name = "pyjwt", specifier = ">=2.9.0" }, { name = "requests", specifier = ">=2.32.4" }, { name = "urllib3", specifier = ">=2.8.0" }, ] From bcf1f899ccdb9d88ed224b800baf28fb96b05da6 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 16:54:05 +0200 Subject: [PATCH 10/31] ref(git): Simplify GitServer's contract and the git tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From a simplification review of the PR: - GitServer requires a relay. Its default was the plain adapter that connects to internal addresses, which the docstring warned against. - GitServer drops emit_error. Nothing on the git path raises UnhandledException, so it never ran; other exceptions reach the caller, which can answer them as it does elsewhere (RestrictedIPAddress as ApiHostError in Sentry). - One require_route() serves check_request and the relay's backstop. - Content-Length accepted digits like "²", which int() refuses, and "٣", which it reads as 3; it now accepts ASCII digits only. The tests share their repository and grant builders, use closing() for relays, and send requests through one helper per file. Refs CW-2221 Co-Authored-By: Claude Opus 5.5 --- src/scm/git/operations.py | 12 +- src/scm/git/relay.py | 9 +- src/scm/git/server.py | 14 +-- tests/unit/git_builders.py | 34 +++++ tests/unit/test_git_grant.py | 14 +-- tests/unit/test_git_operations.py | 48 ++++--- tests/unit/test_git_relay.py | 33 ++--- tests/unit/test_git_remote.py | 17 +-- tests/unit/test_git_server.py | 202 ++++++++++++------------------ 9 files changed, 173 insertions(+), 210 deletions(-) create mode 100644 tests/unit/git_builders.py diff --git a/src/scm/git/operations.py b/src/scm/git/operations.py index 76010ded..2329452c 100644 --- a/src/scm/git/operations.py +++ b/src/scm/git/operations.py @@ -22,6 +22,14 @@ def route_operation(method: str, route: str, query: str) -> GitOperation | None: return _ROUTES.get((method, route, query)) +def require_route(method: str, route: str, query: str) -> GitOperation: + """Return the operation a git smart-HTTP request belongs to, refusing any other request.""" + operation = route_operation(method, route, query) + if operation is None: + raise GitRouteNotAllowed(detail=f"{method} {route} is not a git fetch or push route.") + return operation + + def check_request(grant: Grant, repository_id: RepositoryId, method: str, route: str, query: str) -> GitOperation: """Refuse a request the grant does not reach, before anything is sent to the provider. @@ -29,9 +37,7 @@ def check_request(grant: Grant, repository_id: RepositoryId, method: str, route: the grant in the same form the request uses: a database id does not match the same repository's ``(provider, external id)`` pair. """ - operation = route_operation(method, route, query) - if operation is None: - raise GitRouteNotAllowed(detail=f"{method} {route} is not a git fetch or push route.") + operation = require_route(method, route, query) if operation != grant.operation: raise GitRouteNotAllowed(detail=f"The git grant allows {grant.operation}, not {operation}.") if repository_id not in grant.repositories: diff --git a/src/scm/git/relay.py b/src/scm/git/relay.py index 54c2217e..db8ecab6 100644 --- a/src/scm/git/relay.py +++ b/src/scm/git/relay.py @@ -9,13 +9,12 @@ from scm.errors import ( ClientClosedRequest, - GitRouteNotAllowed, ResourceCertificateInvalid, ResourceGatewayTimeout, ResourceServiceUnavailable, RpcMalformedRequestHeaders, ) -from scm.git.operations import route_operation +from scm.git.operations import require_route from scm.git.types import GitRemote, Readable from scm.rpc.types import StreamResponse @@ -98,8 +97,7 @@ def forward( raise ValueError(f"Refusing to send a token to {remote.url}, which is not https.") # Callers check the request against a grant first; this only makes sure the token never goes to anything # but a git route. - if route_operation(method, route, query) is None: - raise GitRouteNotAllowed(detail=f"{method} {route} is not a git fetch or push route.") + require_route(method, route, query) client_headers = CaseInsensitiveDict(headers) upstream_headers = CaseInsensitiveDict( @@ -154,7 +152,8 @@ def _content_length(headers: CaseInsensitiveDict[str]) -> int | None: raw = headers.get("Content-Length") if raw is None: return None - if not raw.isdigit(): + # isdigit alone also accepts digits such as "²", which int() refuses, and "٣", which it reads as 3. + if not (raw.isascii() and raw.isdigit()): raise RpcMalformedRequestHeaders(detail="Could not parse Content-Length header") return int(raw) diff --git a/src/scm/git/server.py b/src/scm/git/server.py index c5326e37..12f41dd3 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -20,7 +20,11 @@ class GitServer: The order of the checks matters. The grant is verified, and the request's route and repository checked against it, before the repository is looked up or a token fetched, so a request the grant does not allow never reaches the database, the token supplier or the provider. Build one per process: its relay pools - upstream connections. Pass a relay whose adapter refuses internal addresses; see ``GitRelay``. + upstream connections. Give it a relay whose adapter refuses internal addresses; see ``GitRelay``. + + Refusals and provider failures come back as plain-text responses. Any other exception, such as an adapter + refusing a host or the token supplier failing, reaches the caller unchanged, so the caller can answer it the + way it answers the same error elsewhere. """ def __init__( @@ -29,15 +33,13 @@ def __init__( fetch_repository: Callable[[int, RepositoryId], Repository | None], get_token: GetToken, record_count: Callable[[str, int, dict[str, str]], None], - emit_error: Callable[[Exception], None], - relay: GitRelay | None = None, + relay: GitRelay, ) -> None: self.secrets = secrets self.fetch_repository = fetch_repository self.get_token = get_token self.record_count = record_count - self.emit_error = emit_error - self.relay = relay or GitRelay() + self.relay = relay def handle( self, @@ -74,8 +76,6 @@ def handle( raise ResourceBadGateway(detail="The provider rejected Sentry's credentials for this repository.") except SCMCodedError as e: self.record_count("sentry.scm.git.request.failed", 1, {"code": e.code}) - if e.code == "unhandled_exception": - self.emit_error(e) return _error_response(e) self.record_count( diff --git a/tests/unit/git_builders.py b/tests/unit/git_builders.py new file mode 100644 index 00000000..44876115 --- /dev/null +++ b/tests/unit/git_builders.py @@ -0,0 +1,34 @@ +"""Builders shared by the git module's tests.""" + +from scm.git.types import Grant +from scm.types import Repository + +# Unix seconds long after any test runs, for grants a test doesn't expect to expire. +FAR_FUTURE = 2**40 + + +def make_repository(**overrides) -> Repository: + defaults: Repository = { + "id": 5, + "external_id": "123", + "integration_id": 1, + "is_active": True, + "name": "getsentry/sentry", + "organization_id": 1, + "provider_name": "github", + "web_base_url": None, + "installation_id": None, + } + return {**defaults, **overrides} # type: ignore[typeddict-item] + + +def make_grant(**overrides) -> Grant: + """A fetch grant for the default repository, named both ways a grant can name it.""" + fields = { + "organization_id": 1, + "operation": "fetch", + "repositories": (5, ("github", "123")), + "expires_at": FAR_FUTURE, + } + fields.update(overrides) + return Grant(**fields) # type: ignore[arg-type] diff --git a/tests/unit/test_git_grant.py b/tests/unit/test_git_grant.py index 9346a65d..bd452302 100644 --- a/tests/unit/test_git_grant.py +++ b/tests/unit/test_git_grant.py @@ -8,6 +8,7 @@ from scm.errors import GitInvalidGrant from scm.git.grant import GRANT_AUDIENCE, sign_grant, verify_grant from scm.git.types import Grant +from tests.unit.git_builders import FAR_FUTURE, make_grant NOW = 1_800_000_000 # HS256 secrets of at least 32 bytes, as PyJWT recommends. @@ -16,17 +17,6 @@ NEW_SECRET = "test-secret-new-000000000000000000" -def make_grant(**overrides) -> Grant: - fields = { - "organization_id": 1, - "operation": "fetch", - "repositories": (5, ("github", "123")), - "expires_at": NOW + 600, - } - fields.update(overrides) - return Grant(**fields) # type: ignore[arg-type] - - def key_id(secret: str) -> str: return hashlib.sha256(secret.encode()).hexdigest()[:8] @@ -65,7 +55,7 @@ def test_token_is_a_jwt_with_the_grants_claims(self): "operation": "fetch", "repositories": [5, ["github", "123"]], "version": 1, - "exp": NOW + 600, + "exp": FAR_FUTURE, "iat": NOW, "aud": GRANT_AUDIENCE, } diff --git a/tests/unit/test_git_operations.py b/tests/unit/test_git_operations.py index d6216104..5d0d496a 100644 --- a/tests/unit/test_git_operations.py +++ b/tests/unit/test_git_operations.py @@ -1,14 +1,11 @@ import pytest from scm.errors import GitRouteNotAllowed -from scm.git.operations import check_request, route_operation -from scm.git.types import Grant +from scm.git.operations import check_request, require_route, route_operation +from tests.unit.git_builders import make_grant - -def make_grant(**overrides) -> Grant: - fields = {"organization_id": 1, "operation": "fetch", "repositories": (5,), "expires_at": 0} - fields.update(overrides) - return Grant(**fields) # type: ignore[arg-type] +FETCH_ROUTES = [("GET", "info/refs", "service=git-upload-pack"), ("POST", "git-upload-pack", "")] +PUSH_ROUTES = [("GET", "info/refs", "service=git-receive-pack"), ("POST", "git-receive-pack", "")] class TestRouteOperation: @@ -46,32 +43,31 @@ def test_other_routes(self, method, route, query): assert route_operation(method, route, query) is None -class TestCheckRequest: - def test_fetch_grant_reaches_fetch_routes(self): - grant = make_grant() - assert check_request(grant, 5, "GET", "info/refs", "service=git-upload-pack") == "fetch" - assert check_request(grant, 5, "POST", "git-upload-pack", "") == "fetch" +class TestRequireRoute: + def test_returns_the_operation(self): + assert require_route("POST", "git-receive-pack", "") == "push" + + def test_refuses_other_routes(self): + with pytest.raises(GitRouteNotAllowed, match="not a git fetch or push route"): + require_route("POST", "info/lfs/objects/batch", "") - def test_push_grant_reaches_push_routes(self): - grant = make_grant(operation="push") - assert check_request(grant, 5, "GET", "info/refs", "service=git-receive-pack") == "push" - assert check_request(grant, 5, "POST", "git-receive-pack", "") == "push" +class TestCheckRequest: @pytest.mark.parametrize( - "method,route,query", - [("GET", "info/refs", "service=git-receive-pack"), ("POST", "git-receive-pack", "")], + "operation,route", + [("fetch", route) for route in FETCH_ROUTES] + [("push", route) for route in PUSH_ROUTES], ) - def test_fetch_grant_cannot_push(self, method, route, query): - with pytest.raises(GitRouteNotAllowed, match="allows fetch, not push"): - check_request(make_grant(), 5, method, route, query) + def test_grant_reaches_its_operations_routes(self, operation, route): + assert check_request(make_grant(operation=operation), 5, *route) == operation @pytest.mark.parametrize( - "method,route,query", - [("GET", "info/refs", "service=git-upload-pack"), ("POST", "git-upload-pack", "")], + "operation,route", + [("fetch", route) for route in PUSH_ROUTES] + [("push", route) for route in FETCH_ROUTES], ) - def test_push_grant_cannot_fetch(self, method, route, query): - with pytest.raises(GitRouteNotAllowed, match="allows push, not fetch"): - check_request(make_grant(operation="push"), 5, method, route, query) + def test_grant_cannot_reach_the_other_operations_routes(self, operation, route): + other = "push" if operation == "fetch" else "fetch" + with pytest.raises(GitRouteNotAllowed, match=f"allows {operation}, not {other}"): + check_request(make_grant(operation=operation), 5, *route) def test_unknown_route(self): with pytest.raises(GitRouteNotAllowed, match="not a git fetch or push route"): diff --git a/tests/unit/test_git_relay.py b/tests/unit/test_git_relay.py index d4a688a6..c9564990 100644 --- a/tests/unit/test_git_relay.py +++ b/tests/unit/test_git_relay.py @@ -4,6 +4,7 @@ import threading import time from collections.abc import Iterator +from contextlib import closing import pytest from requests.adapters import HTTPAdapter @@ -42,9 +43,10 @@ def remote_for(upstream) -> GitRemote: return GitRemote(url=f"{upstream.url}/getsentry/sentry.git", username="x-access-token") -def fetch_refs(relay: GitRelay, upstream, headers: dict[str, str] | None = None): +def fetch_refs(relay: GitRelay, upstream=None, headers: dict[str, str] | None = None, *, remote=None): + """Fetch the refs advertisement from the fake upstream, or from ``remote`` when given.""" return relay.forward( - remote_for(upstream), + remote or remote_for(upstream), TOKEN, method="GET", route="info/refs", @@ -133,9 +135,7 @@ def test_asks_for_an_unencoded_response_unless_the_client_says_otherwise(self, r def test_refuses_plain_http(self, relay, upstream): remote = GitRemote(url=upstream.url.replace("https://", "http://") + "/repo.git", username="x-access-token") with pytest.raises(ValueError, match="not https"): - relay.forward( - remote, TOKEN, method="GET", route="info/refs", query="service=git-upload-pack", headers={}, body=None - ) + fetch_refs(relay, remote=remote) assert upstream.requests == [] def test_refuses_routes_that_are_not_git(self, relay, upstream): @@ -175,7 +175,7 @@ def test_body_shorter_than_its_length(self, relay, upstream): upload_pack(relay, upstream, io.BytesIO(b"00"), headers={"Content-Length": "4"}) assert exc_info.value.code == "client_closed_request" - @pytest.mark.parametrize("content_length", ["abc", "-1", "1.5"]) + @pytest.mark.parametrize("content_length", ["abc", "-1", "1.5", "²", "٣"]) def test_malformed_content_length(self, relay, upstream, content_length): with pytest.raises(SCMCodedError) as exc_info: upload_pack(relay, upstream, io.BytesIO(b"0000"), headers={"Content-Length": content_length}) @@ -319,21 +319,14 @@ class RefusingAdapter(HTTPAdapter): def send(self, request, *args, **kwargs): raise HostRefused(request.url) - relay = GitRelay(adapter=RefusingAdapter()) - try: - with pytest.raises(HostRefused): - fetch_refs(relay, upstream) - finally: - relay.close() + with closing(GitRelay(adapter=RefusingAdapter())) as relay, pytest.raises(HostRefused): + fetch_refs(relay, upstream) assert upstream.requests == [] def test_given_adapter_also_pools_connections(self, upstream): - relay = GitRelay(adapter=HTTPAdapter(max_retries=0)) - try: + with closing(GitRelay(adapter=HTTPAdapter(max_retries=0))) as relay: b"".join(fetch_refs(relay, upstream).content) b"".join(fetch_refs(relay, upstream).content) - finally: - relay.close() assert upstream.connections == 1 @@ -345,9 +338,7 @@ def test_connection_refused(self, relay): remote = GitRemote(url=f"https://127.0.0.1:{port}/getsentry/sentry.git", username="x-access-token") with pytest.raises(ResourceServiceUnavailable) as exc_info: - relay.forward( - remote, TOKEN, method="GET", route="info/refs", query="service=git-upload-pack", headers={}, body=None - ) + fetch_refs(relay, remote=remote) assert_token_not_in(exc_info.value) def test_untrusted_certificate(self, relay, upstream, monkeypatch): @@ -365,14 +356,12 @@ def handler(h: UpstreamHandler) -> None: h.respond(200, b"") upstream.handler = handler - relay = GitRelay(timeout=(1, 0.1)) try: - with pytest.raises(ResourceGatewayTimeout) as exc_info: + with closing(GitRelay(timeout=(1, 0.1))) as relay, pytest.raises(ResourceGatewayTimeout) as exc_info: fetch_refs(relay, upstream) assert_token_not_in(exc_info.value) finally: release.set() - relay.close() def assert_token_not_in(error: Exception) -> None: diff --git a/tests/unit/test_git_remote.py b/tests/unit/test_git_remote.py index ad3ec1cf..7204f035 100644 --- a/tests/unit/test_git_remote.py +++ b/tests/unit/test_git_remote.py @@ -3,22 +3,7 @@ from scm.errors import GitProviderNotSupported, InvalidWebBaseUrl from scm.git.remote import git_remote from scm.git.types import GitRemote -from scm.types import Repository - - -def make_repository(**overrides) -> Repository: - defaults: Repository = { - "id": 1, - "external_id": "123", - "integration_id": 1, - "is_active": True, - "name": "getsentry/sentry", - "organization_id": 1, - "provider_name": "github", - "web_base_url": None, - "installation_id": None, - } - return {**defaults, **overrides} # type: ignore[typeddict-item] +from tests.unit.git_builders import make_repository def test_github(): diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index bb7f3dd3..f0ea512f 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -1,5 +1,6 @@ import io import time +from contextlib import closing from unittest.mock import MagicMock import pytest @@ -7,39 +8,18 @@ from scm.git.grant import sign_grant from scm.git.relay import GitRelay from scm.git.server import GitServer -from scm.git.types import GRANT_HEADER, GitOperation, GitRemote, Grant +from scm.git.types import GRANT_HEADER, GitOperation, GitRemote, Readable from scm.rpc.types import StreamResponse from scm.types import Repository, RepositoryId +from tests.unit.git_builders import make_grant, make_repository from tests.unit.git_upstream import basic_auth, running_upstream SECRET = "test-secret-0000000000000000000000" TOKEN = "ghs_secrettoken" -def make_repository(**overrides) -> Repository: - defaults: Repository = { - "id": 5, - "external_id": "123", - "integration_id": 1, - "is_active": True, - "name": "getsentry/sentry", - "organization_id": 1, - "provider_name": "github", - "web_base_url": None, - "installation_id": None, - } - return {**defaults, **overrides} # type: ignore[typeddict-item] - - -def make_grant(**overrides) -> str: - fields = { - "organization_id": 1, - "operation": "fetch", - "repositories": (5,), - "expires_at": int(time.time()) + 600, - } - fields.update(overrides) - return sign_grant(SECRET, Grant(**fields)) # type: ignore[arg-type] +def signed_grant(**overrides) -> str: + return sign_grant(SECRET, make_grant(**overrides)) class Recorder: @@ -49,7 +29,6 @@ def __init__(self, repository: Repository | None = None, token: str | None = TOK self.repository_lookups: list[tuple[int, RepositoryId]] = [] self.token_lookups: list[tuple[Repository, GitOperation]] = [] self.counts: list[tuple[str, dict[str, str]]] = [] - self.errors: list[Exception] = [] def fetch_repository(self, organization_id: int, repository_id: RepositoryId) -> Repository | None: self.repository_lookups.append((organization_id, repository_id)) @@ -72,22 +51,37 @@ def make_server(recorder: Recorder, relay: GitRelay | None = None) -> GitServer: fetch_repository=recorder.fetch_repository, get_token=recorder.get_token, record_count=recorder.record_count, - emit_error=recorder.errors.append, relay=relay, ) -def fetch_refs(server: GitServer, grant: str | None, repository_id: RepositoryId = 5) -> StreamResponse: +def request( + server: GitServer, + grant: str | None, + *, + method: str = "GET", + route: str = "info/refs", + query: str = "service=git-upload-pack", + repository_id: RepositoryId = 5, + headers: dict[str, str] | None = None, + body: Readable | None = None, +) -> StreamResponse: + """Send a request to the server, an info/refs fetch unless told otherwise.""" + grant_headers = {GRANT_HEADER: grant} if grant is not None else {} return server.handle( - method="GET", + method=method, repository_id=repository_id, - route="info/refs", - query="service=git-upload-pack", - headers={GRANT_HEADER: grant} if grant is not None else {}, - body=None, + route=route, + query=query, + headers=grant_headers | (headers or {}), + body=body, ) +def push(server: GitServer, grant: str) -> StreamResponse: + return request(server, grant, method="POST", route="git-receive-pack", query="", body=io.BytesIO(b"0000")) + + def assert_refused(response: StreamResponse, status: int, detail: str) -> None: assert response.status_code == status assert response.headers["Content-Type"] == "text/plain; charset=utf-8" @@ -96,6 +90,17 @@ def assert_refused(response: StreamResponse, status: int, detail: str) -> None: assert int(response.headers["Content-Length"]) == len(body.encode()) +def test_requires_a_relay(): + # A default relay would connect to internal addresses, so the caller has to choose its adapter. + with pytest.raises(TypeError, match="relay"): + GitServer( # type: ignore[call-arg] + secrets=[SECRET], + fetch_repository=lambda organization_id, repository_id: None, + get_token=lambda repository, operation: None, + record_count=lambda name, value, tags: None, + ) + + class TestRefusedBeforeTheProvider: """Nothing the grant does not allow may look up a repository or a token.""" @@ -104,20 +109,14 @@ class TestRefusedBeforeTheProvider: [ (None, "missing"), ("garbage", "malformed"), - ( - sign_grant( - "other-secret-0000000000000000000000", - Grant(organization_id=1, operation="fetch", repositories=(5,), expires_at=2**40), - ), - "not signed with a known secret", - ), - (make_grant(expires_at=int(time.time()) - 1), "expired"), + (sign_grant("other-secret-0000000000000000000000", make_grant()), "not signed with a known secret"), + (signed_grant(expires_at=int(time.time()) - 1), "expired"), ], ) def test_invalid_grant(self, grant, detail): recorder = Recorder() server = make_server(recorder) - assert_refused(fetch_refs(server, grant), 403, detail) + assert_refused(request(server, grant), 403, detail) assert recorder.repository_lookups == [] assert recorder.token_lookups == [] server.relay.forward.assert_not_called() # type: ignore[attr-defined] @@ -125,96 +124,56 @@ def test_invalid_grant(self, grant, detail): def test_operation_the_grant_does_not_allow(self): recorder = Recorder() - server = make_server(recorder) - response = server.handle( - method="POST", - repository_id=5, - route="git-receive-pack", - query="", - headers={GRANT_HEADER: make_grant()}, - body=io.BytesIO(b"0000"), - ) - assert_refused(response, 403, "allows fetch, not push") + assert_refused(push(make_server(recorder), signed_grant()), 403, "allows fetch, not push") assert recorder.repository_lookups == [] assert recorder.token_lookups == [] def test_repository_the_grant_does_not_list(self): recorder = Recorder() - assert_refused(fetch_refs(make_server(recorder), make_grant(), repository_id=6), 403, "does not cover") + assert_refused(request(make_server(recorder), signed_grant(), repository_id=6), 403, "does not cover") assert recorder.repository_lookups == [] def test_grant_header_name_is_case_insensitive(self): - recorder = Recorder() - server = make_server(recorder) - response = server.handle( - method="GET", - repository_id=5, - route="info/refs", - query="service=git-upload-pack", - headers={GRANT_HEADER.lower(): make_grant()}, - body=None, - ) + response = request(make_server(Recorder()), None, headers={GRANT_HEADER.lower(): signed_grant()}) assert response.status_code == 200 class TestRepository: def test_looked_up_under_the_grants_organization(self): recorder = Recorder() - fetch_refs(make_server(recorder), make_grant(organization_id=1)) + request(make_server(recorder), signed_grant(organization_id=1)) assert recorder.repository_lookups == [(1, 5)] - def test_not_found(self): + @pytest.mark.parametrize( + "repository,status,detail", + [ + (None, 404, "could not be found"), + (make_repository(organization_id=2), 404, "did not belong"), + (make_repository(is_active=False), 404, "inactive"), + (make_repository(provider_name="bitbucket"), 400, "no git remote"), + ], + ) + def test_refused_before_the_token(self, repository, status, detail): recorder = Recorder() - recorder.repository = None # type: ignore[assignment] - assert_refused(fetch_refs(make_server(recorder), make_grant()), 404, "could not be found") - assert recorder.token_lookups == [] - - def test_other_organization(self): - recorder = Recorder(repository=make_repository(organization_id=2)) - assert_refused(fetch_refs(make_server(recorder), make_grant()), 404, "did not belong") - assert recorder.token_lookups == [] - - def test_inactive(self): - recorder = Recorder(repository=make_repository(is_active=False)) - assert_refused(fetch_refs(make_server(recorder), make_grant()), 404, "inactive") - assert recorder.token_lookups == [] - - def test_provider_without_a_git_remote(self): - recorder = Recorder(repository=make_repository(provider_name="bitbucket")) - assert_refused(fetch_refs(make_server(recorder), make_grant()), 400, "no git remote") + recorder.repository = repository + assert_refused(request(make_server(recorder), signed_grant()), status, detail) assert recorder.token_lookups == [] class TestRelay: def test_no_token(self): - recorder = Recorder(token=None) - assert_refused(fetch_refs(make_server(recorder), make_grant()), 404, "No git credentials") + assert_refused(request(make_server(Recorder(token=None)), signed_grant()), 404, "No git credentials") def test_token_asked_for_the_grants_operation(self): recorder = Recorder() - server = make_server(recorder) - server.handle( - method="POST", - repository_id=5, - route="git-receive-pack", - query="", - headers={GRANT_HEADER: make_grant(operation="push")}, - body=io.BytesIO(b"0000"), - ) + push(make_server(recorder), signed_grant(operation="push")) assert recorder.token_lookups == [(make_repository(), "push")] def test_forwards_with_remote_and_token(self): recorder = Recorder() server = make_server(recorder) - headers = {GRANT_HEADER: make_grant(), "Git-Protocol": "version=2"} - response = server.handle( - method="GET", - repository_id=5, - route="info/refs", - query="service=git-upload-pack", - headers=headers, - body=None, - ) + grant = signed_grant() + response = request(server, grant, headers={"Git-Protocol": "version=2"}) assert b"".join(response.content) == b"refs" server.relay.forward.assert_called_once_with( # type: ignore[attr-defined] GitRemote(url="https://github.com/getsentry/sentry.git", username="x-access-token"), @@ -222,7 +181,7 @@ def test_forwards_with_remote_and_token(self): method="GET", route="info/refs", query="service=git-upload-pack", - headers=headers, + headers={GRANT_HEADER: grant, "Git-Protocol": "version=2"}, body=None, ) assert recorder.counts == [ @@ -233,14 +192,10 @@ def test_upstream_unavailable(self): recorder = Recorder( repository=make_repository(provider_name="github_enterprise", web_base_url="https://127.0.0.1:1") ) - relay = GitRelay(timeout=(1, 1)) - try: - response = fetch_refs(make_server(recorder, relay), make_grant()) - finally: - relay.close() + with closing(GitRelay(timeout=(1, 1))) as relay: + response = request(make_server(recorder, relay), signed_grant()) assert_refused(response, 503, "Could not connect") assert TOKEN not in b"".join(response.content).decode() - assert recorder.errors == [] def test_provider_rejecting_the_token_is_a_gateway_error(self): # Passed on, a 401 would make git ask the sandboxed client for credentials. @@ -257,25 +212,34 @@ def provider_body(): relay = MagicMock(spec=GitRelay) relay.forward.return_value = StreamResponse(status_code=401, headers={}, content=content) recorder = Recorder() - response = fetch_refs(make_server(recorder, relay), make_grant()) + response = request(make_server(recorder, relay), signed_grant()) assert_refused(response, 502, "rejected Sentry's credentials") assert closed == [True] assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "resource_bad_gateway"})] + def test_other_errors_reach_the_caller(self): + # Sentry's view answers these the way it answers them elsewhere, e.g. a refused host as ApiHostError. + class HostRefused(Exception): + pass + + relay = MagicMock(spec=GitRelay) + relay.forward.side_effect = HostRefused() + recorder = Recorder() + with pytest.raises(HostRefused): + request(make_server(recorder, relay), signed_grant()) + assert recorder.counts == [] + def test_relays_end_to_end(self, monkeypatch): with running_upstream() as upstream: monkeypatch.setenv("REQUESTS_CA_BUNDLE", upstream.ca_bundle) recorder = Recorder( repository=make_repository(provider_name="github_enterprise", web_base_url=upstream.url) ) - relay = GitRelay() - try: - response = fetch_refs(make_server(recorder, relay), make_grant()) + with closing(GitRelay()) as relay: + response = request(make_server(recorder, relay), signed_grant()) assert response.status_code == 200 b"".join(response.content) - finally: - relay.close() - [request] = upstream.requests - assert request.path == "/getsentry/sentry.git/info/refs?service=git-upload-pack" - assert request.headers["Authorization"] == basic_auth("x-access-token", TOKEN) - assert GRANT_HEADER not in request.headers + [received] = upstream.requests + assert received.path == "/getsentry/sentry.git/info/refs?service=git-upload-pack" + assert received.headers["Authorization"] == basic_auth("x-access-token", TOKEN) + assert GRANT_HEADER not in received.headers From a340f539d71cfcd57451347cf5cf778118c83a38 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 16:56:30 +0200 Subject: [PATCH 11/31] fix(git): Fail fast on a client reset and on empty secrets From a second review of the PR: - urllib3 ignores ECONNRESET and EPIPE while sending a request body, taking them for the provider closing the connection, so a client that reset mid-push left the relay waiting out the 300s read timeout, then reported as a provider 504 (or as "could not connect"). A failed read of the client's body now raises ClientClosedRequest, which urllib3 passes on at once. - A GET is sent upstream with no body. A view passing its request stream for every method sent info/refs as an empty chunked request. - An empty secret would let anyone sign a grant, since HMAC accepts an empty key. Signing, verifying and building a GitServer refuse one. - Exceptions that are not SCM errors still reach the caller unchanged, but are now counted as failed requests. - The Readable docstring no longer suggests a Django request, which reads a chunked body as empty. Refs CW-2221 Co-Authored-By: Claude Opus 5.5 --- src/scm/git/grant.py | 8 +++++++ src/scm/git/relay.py | 20 +++++++++++++++--- src/scm/git/server.py | 6 ++++++ src/scm/git/types.py | 7 +++++- tests/unit/test_git_grant.py | 13 ++++++++++++ tests/unit/test_git_relay.py | 40 +++++++++++++++++++++++++++++++++++ tests/unit/test_git_server.py | 14 +++++++++++- 7 files changed, 103 insertions(+), 5 deletions(-) diff --git a/src/scm/git/grant.py b/src/scm/git/grant.py index 30edbe35..06d07283 100644 --- a/src/scm/git/grant.py +++ b/src/scm/git/grant.py @@ -24,6 +24,7 @@ def sign_grant(secret: str, grant: Grant, *, now: float | None = None) -> str: + _check_secrets([secret]) claims: dict[str, Any] = msgspec.to_builtins(grant) claims["exp"] = claims.pop("expires_at") claims["iat"] = int(time.time() if now is None else now) @@ -37,6 +38,7 @@ def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> Verification takes a list of secrets, as ``RpcServer`` does, so a new secret can be added before signers switch to it and the old one removed after their grants expire. """ + _check_secrets(secrets) if not token: raise GitInvalidGrant(detail="The git grant is missing.") try: @@ -79,6 +81,12 @@ def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> return grant +def _check_secrets(secrets: list[str]) -> None: + # HMAC accepts an empty key, so an empty secret, say from an unset setting, would let anyone sign a grant. + if not all(secrets): + raise ValueError("A git grant secret is empty.") + + def _key_id(secret: str) -> str: # The first 8 hex characters of the secret's SHA-256, the same key id Sentry's viewer contexts use. return hashlib.sha256(secret.encode()).hexdigest()[:8] diff --git a/src/scm/git/relay.py b/src/scm/git/relay.py index db8ecab6..c20de544 100644 --- a/src/scm/git/relay.py +++ b/src/scm/git/relay.py @@ -113,7 +113,11 @@ def forward( method, url, headers=upstream_headers, - data=_request_body(body, _content_length(client_headers), self.chunk_size), + # Of the git routes only the POSTs carry a body. A caller may pass its request stream for a GET too, + # and sent on, it would go upstream as an empty chunked body. + data=_request_body(body, _content_length(client_headers), self.chunk_size) + if method == "POST" + else None, # Passing auth also stops requests from replacing it with credentials from a .netrc file. auth=HTTPBasicAuth(remote.username, token), stream=True, @@ -170,10 +174,20 @@ def _request_body( def _read_until_eof(body: Readable, chunk_size: int) -> Iterator[bytes]: - while chunk := body.read(chunk_size): + while chunk := _read(body, chunk_size): yield chunk +def _read(body: Readable, size: int) -> bytes: + try: + return body.read(size) + except OSError: + # urllib3 ignores ECONNRESET and EPIPE while it sends a body, taking them for the provider closing the + # connection, and would then wait out the read timeout for a response to the cut-off request. Raised as + # anything other than an OSError, the client's reset ends the request at once. + raise ClientClosedRequest(detail="The client's connection failed while it sent its body.") from None + + class _SizedBody: """A body of known length, read as it is sent. @@ -192,7 +206,7 @@ def __len__(self) -> int: def __iter__(self) -> Iterator[bytes]: remaining = self.length while remaining > 0: - chunk = self.body.read(min(self.chunk_size, remaining)) + chunk = _read(self.body, min(self.chunk_size, remaining)) if not chunk: raise ClientClosedRequest(detail="The client closed the request before sending its whole body.") remaining -= len(chunk) diff --git a/src/scm/git/server.py b/src/scm/git/server.py index 12f41dd3..e9ba2023 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -35,6 +35,9 @@ def __init__( record_count: Callable[[str, int, dict[str, str]], None], relay: GitRelay, ) -> None: + # Fail when the server is built rather than on every request, because no grant could ever verify. + if not secrets or not all(secrets): + raise ValueError("GitServer needs at least one secret, and none may be empty.") self.secrets = secrets self.fetch_repository = fetch_repository self.get_token = get_token @@ -77,6 +80,9 @@ def handle( except SCMCodedError as e: self.record_count("sentry.scm.git.request.failed", 1, {"code": e.code}) return _error_response(e) + except Exception: + self.record_count("sentry.scm.git.request.failed", 1, {"code": "unhandled_exception"}) + raise self.record_count( "sentry.scm.git.request.succeeded", diff --git a/src/scm/git/types.py b/src/scm/git/types.py index cd08734f..3500cad1 100644 --- a/src/scm/git/types.py +++ b/src/scm/git/types.py @@ -57,6 +57,11 @@ class GitRemote: class Readable(Protocol): - """A request body the relay pulls from as it sends upstream, e.g. a WSGI input or a Django request.""" + """A request body the relay pulls from as it sends upstream. + + It must yield the body already decoded from its transfer encoding, ending where the body ends. A Django + request under WSGI reads a chunked body as empty, and git sends a push over about 1 MiB chunked, so a view + needs to pass a stream that has the whole body. + """ def read(self, size: int = -1, /) -> bytes: ... diff --git a/tests/unit/test_git_grant.py b/tests/unit/test_git_grant.py index bd452302..34c5cda6 100644 --- a/tests/unit/test_git_grant.py +++ b/tests/unit/test_git_grant.py @@ -121,6 +121,19 @@ def test_other_algorithm(self): assert_invalid(sign_claims(claims_of(make_grant()), algorithm="HS512"), "invalid") +class TestSecrets: + # PyJWT signs with an empty key, so an empty secret would let anyone sign grants. + def test_refuses_to_sign_with_an_empty_secret(self): + with pytest.raises(ValueError, match="empty"): + sign_grant("", make_grant()) + + @pytest.mark.filterwarnings("ignore::jwt.warnings.InsecureKeyLengthWarning") + def test_refuses_to_verify_with_an_empty_secret(self): + token = jwt.encode(claims_of(make_grant()), "", algorithm="HS256", headers={"kid": key_id("")}) + with pytest.raises(ValueError, match="empty"): + verify_grant([SECRET, ""], token, now=NOW) + + class TestRotatedSecrets: @pytest.mark.parametrize("signer", [OLD_SECRET, NEW_SECRET]) def test_both_secrets_verify_during_rotation(self, signer): diff --git a/tests/unit/test_git_relay.py b/tests/unit/test_git_relay.py index c9564990..a1702ea2 100644 --- a/tests/unit/test_git_relay.py +++ b/tests/unit/test_git_relay.py @@ -151,6 +151,24 @@ def test_refuses_routes_that_are_not_git(self, relay, upstream): ) assert upstream.requests == [] + def test_get_sends_no_body(self, relay, upstream): + # A view passes its request stream for every method; a GET's must not go upstream as an empty chunked body. + body = io.BytesIO(b"") + b"".join( + relay.forward( + remote_for(upstream), + TOKEN, + method="GET", + route="info/refs", + query="service=git-upload-pack", + headers={}, + body=body, + ).content + ) + [request] = upstream.requests + assert "Transfer-Encoding" not in request.headers + assert "Content-Length" not in request.headers + def test_known_length_body_is_sent_with_its_length(self, relay, upstream): b"".join(upload_pack(relay, upstream, io.BytesIO(b"0000"), headers={"Content-Length": "4"}).content) [request] = upstream.requests @@ -175,6 +193,28 @@ def test_body_shorter_than_its_length(self, relay, upstream): upload_pack(relay, upstream, io.BytesIO(b"00"), headers={"Content-Length": "4"}) assert exc_info.value.code == "client_closed_request" + @pytest.mark.parametrize("known_length", [True, False]) + @pytest.mark.parametrize("error", [ConnectionResetError, BrokenPipeError]) + def test_client_resetting_mid_body_is_reported_at_once(self, upstream, known_length, error): + # urllib3 ignores ECONNRESET and EPIPE while sending a body, taking them for the provider closing the + # connection, and then waits for a response to the cut-off request until its read timeout. + upstream.handler = lambda h: (b"".join(h.body_chunks()), h.respond(200, b"ok")) + + class ResettingClient: + def __init__(self) -> None: + self.reads = 0 + + def read(self, size: int = -1, /) -> bytes: + self.reads += 1 + if self.reads > 1: + raise error() + return b"first" + + headers = {"Content-Length": "100"} if known_length else {} + with closing(GitRelay(timeout=(1, 2))) as relay, pytest.raises(SCMCodedError) as exc_info: + upload_pack(relay, upstream, ResettingClient(), headers=headers) + assert exc_info.value.code == "client_closed_request" + @pytest.mark.parametrize("content_length", ["abc", "-1", "1.5", "²", "٣"]) def test_malformed_content_length(self, relay, upstream, content_length): with pytest.raises(SCMCodedError) as exc_info: diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index f0ea512f..e890b47b 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -101,6 +101,18 @@ def test_requires_a_relay(): ) +@pytest.mark.parametrize("secrets", [[], [SECRET, ""]]) +def test_requires_non_empty_secrets(secrets): + with pytest.raises(ValueError, match="secret"): + GitServer( + secrets=secrets, + fetch_repository=lambda organization_id, repository_id: None, + get_token=lambda repository, operation: None, + record_count=lambda name, value, tags: None, + relay=MagicMock(spec=GitRelay), + ) + + class TestRefusedBeforeTheProvider: """Nothing the grant does not allow may look up a repository or a token.""" @@ -227,7 +239,7 @@ class HostRefused(Exception): recorder = Recorder() with pytest.raises(HostRefused): request(make_server(recorder, relay), signed_grant()) - assert recorder.counts == [] + assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "unhandled_exception"})] def test_relays_end_to_end(self, monkeypatch): with running_upstream() as upstream: From a0c232aed198e33413351de0cbb75412ad2b4860 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 17:01:54 +0200 Subject: [PATCH 12/31] fix(git): Clone GitLab repositories from their path, not their name Sentry stores a GitLab repository's display name (name_with_namespace) as its name, so "GitLab.org / GitLab FOSS" became a clone URL of GitLab.org/GitLabFOSS.git, which GitLab refuses. The real path is only in Sentry's repo.config["path"]. Repository gains an optional path, carried through the RPC serialization (absent from older servers, so it defaults to None), and git_remote builds GitLab URLs from it. Without one it refuses with missing_repository_path rather than guessing from the name. Sentry's fetch_repository needs to fill it in. Refs CW-2221 Co-Authored-By: Claude Opus 5.5 --- src/scm/errors.py | 6 ++++++ src/scm/git/remote.py | 7 ++++--- src/scm/rpc/helpers.py | 2 ++ src/scm/rpc/types.py | 1 + src/scm/types.py | 3 +++ tests/unit/test_git_remote.py | 29 +++++++++++++++++++++++++---- tests/unit/test_rpc_server.py | 11 +++++++++++ 7 files changed, 52 insertions(+), 7 deletions(-) diff --git a/src/scm/errors.py b/src/scm/errors.py index 9fe33cb7..20cb22f4 100644 --- a/src/scm/errors.py +++ b/src/scm/errors.py @@ -6,6 +6,7 @@ "git_route_not_allowed", "invalid_web_base_url", "malformed_external_id", + "missing_repository_path", "path_is_directory", "path_is_not_directory", "provider_not_found", @@ -47,6 +48,7 @@ "git_route_not_allowed": "The git grant does not allow this request.", "invalid_web_base_url": "The repository's web_base_url is missing or is not an https URL.", "malformed_external_id": "The repository's external ID was malformed.", + "missing_repository_path": "The repository's path on its provider is unknown.", "path_is_directory": "The requested path is a directory, not a file.", "path_is_not_directory": "The requested path is not a directory.", "provider_not_found": "An unsupported integration provider was found.", @@ -153,6 +155,10 @@ class MalformedExternalId(SCMCodedError): code = "malformed_external_id" +class MissingRepositoryPath(SCMCodedError): + code = "missing_repository_path" + + class PathIsDirectory(SCMCodedError): code = "path_is_directory" diff --git a/src/scm/git/remote.py b/src/scm/git/remote.py index fce45893..329a6dd5 100644 --- a/src/scm/git/remote.py +++ b/src/scm/git/remote.py @@ -1,6 +1,6 @@ from urllib.parse import urlsplit -from scm.errors import GitProviderNotSupported, InvalidWebBaseUrl +from scm.errors import GitProviderNotSupported, InvalidWebBaseUrl, MissingRepositoryPath from scm.git.types import GitRemote from scm.types import Repository @@ -20,8 +20,9 @@ def git_remote(repository: Repository) -> GitRemote: base_url = _https_base_url(repository) return GitRemote(url=f"{base_url}/{repository['name']}.git", username="x-access-token") case "gitlab": - # Sentry's repository name for GitLab is the namespace path with spaces around each slash. - path = repository["name"].replace(" ", "") + # Sentry's name for a GitLab repository is its display name, so the path has to come separately. + if not (path := repository.get("path")): + raise MissingRepositoryPath(detail="The GitLab repository has no path.") return GitRemote(url=f"{_https_base_url(repository)}/{path}.git", username="oauth2") case provider_name: raise GitProviderNotSupported(detail=f"The {provider_name} provider has no git remote.") diff --git a/src/scm/rpc/helpers.py b/src/scm/rpc/helpers.py index 46c1d895..33f5b44b 100644 --- a/src/scm/rpc/helpers.py +++ b/src/scm/rpc/helpers.py @@ -49,6 +49,7 @@ def deserialize_repository(content: bytes) -> Repository: "provider_name": repository.attributes.provider_name, "web_base_url": repository.attributes.web_base_url, "installation_id": repository.attributes.installation_id, + "path": repository.attributes.path, } return result @@ -69,6 +70,7 @@ def serialize_repository(repository: Repository) -> bytes: provider_name=repository["provider_name"], web_base_url=repository["web_base_url"], installation_id=repository["installation_id"], + path=repository.get("path"), ), ) ) diff --git a/src/scm/rpc/types.py b/src/scm/rpc/types.py index 77a7ea41..fd182c23 100644 --- a/src/scm/rpc/types.py +++ b/src/scm/rpc/types.py @@ -32,6 +32,7 @@ class RepositoryAttributes(msgspec.Struct): provider_name: ProviderName web_base_url: str | None = None installation_id: str | None = None + path: str | None = None class JsonApiData[Type, Attributes](msgspec.Struct): diff --git a/src/scm/types.py b/src/scm/types.py index 0f53ad60..5a17f0b7 100644 --- a/src/scm/types.py +++ b/src/scm/types.py @@ -342,6 +342,9 @@ class Repository(TypedDict): web_base_url: str | None # The SCM app's external id. installation_id: str | None + # The repository's path on its provider's host, e.g. "gitlab-org/gitlab-foss". Only Sentry's GitLab + # repositories set it today, because their name is a display name ("GitLab.org / GitLab FOSS"). + path: NotRequired[str | None] class GitRepository(TypedDict): diff --git a/tests/unit/test_git_remote.py b/tests/unit/test_git_remote.py index 7204f035..1714107d 100644 --- a/tests/unit/test_git_remote.py +++ b/tests/unit/test_git_remote.py @@ -1,6 +1,6 @@ import pytest -from scm.errors import GitProviderNotSupported, InvalidWebBaseUrl +from scm.errors import GitProviderNotSupported, InvalidWebBaseUrl, MissingRepositoryPath from scm.git.remote import git_remote from scm.git.types import GitRemote from tests.unit.git_builders import make_repository @@ -23,7 +23,7 @@ def test_github_enterprise(web_base_url): @pytest.mark.parametrize("provider_name", ["github_enterprise", "gitlab"]) def test_self_hosted_providers_need_a_web_base_url(provider_name): with pytest.raises(InvalidWebBaseUrl, match=f"{provider_name} repository has no web_base_url"): - git_remote(make_repository(provider_name=provider_name)) + git_remote(make_repository(provider_name=provider_name, path="getsentry/sentry")) @pytest.mark.parametrize( @@ -32,7 +32,7 @@ def test_self_hosted_providers_need_a_web_base_url(provider_name): ) @pytest.mark.parametrize("provider_name", ["github_enterprise", "gitlab"]) def test_web_base_url_must_be_https(provider_name, web_base_url): - repository = make_repository(provider_name=provider_name, web_base_url=web_base_url) + repository = make_repository(provider_name=provider_name, path="getsentry/sentry", web_base_url=web_base_url) with pytest.raises(InvalidWebBaseUrl, match="not an https URL"): git_remote(repository) @@ -47,11 +47,32 @@ def test_web_base_url_must_be_https(provider_name, web_base_url): ) def test_gitlab(web_base_url, url): repository = make_repository( - provider_name="gitlab", external_id="gitlab.com:42", name="group / sub / project", web_base_url=web_base_url + provider_name="gitlab", + external_id="gitlab.com:42", + name="Group / Sub / Project", + path="group/sub/project", + web_base_url=web_base_url, ) assert git_remote(repository) == GitRemote(url=url, username="oauth2") +@pytest.mark.parametrize("path", [None, ""]) +def test_gitlab_needs_the_repository_path(path): + # The name is GitLab's display name, e.g. "GitLab.org / GitLab FOSS" for gitlab-org/gitlab-foss, so it can't + # stand in for the path. + repository = make_repository( + provider_name="gitlab", name="GitLab.org / GitLab FOSS", path=path, web_base_url="https://gitlab.com" + ) + with pytest.raises(MissingRepositoryPath): + git_remote(repository) + + +def test_gitlab_without_a_path_from_an_older_sentry(): + repository = make_repository(provider_name="gitlab", web_base_url="https://gitlab.com") + with pytest.raises(MissingRepositoryPath): + git_remote(repository) + + @pytest.mark.parametrize("provider_name", ["bitbucket", "cursor_origin"]) def test_unsupported_providers(provider_name): with pytest.raises(GitProviderNotSupported, match=provider_name): diff --git a/tests/unit/test_rpc_server.py b/tests/unit/test_rpc_server.py index 62fea8dc..6dd01498 100644 --- a/tests/unit/test_rpc_server.py +++ b/tests/unit/test_rpc_server.py @@ -313,6 +313,17 @@ def test_web_base_url_none_when_not_set(self): decoded = deserialize_repository(serialize_repository(make_repository())) assert decoded["web_base_url"] is None + def test_path_round_trips(self): + repo = make_repository(provider_name="gitlab", path="gitlab-org/gitlab-foss") + decoded = deserialize_repository(serialize_repository(repo)) + assert decoded["path"] == "gitlab-org/gitlab-foss" + + def test_path_none_when_the_server_omits_it(self): + payload = json.loads(serialize_repository(make_repository())) + del payload["data"]["attributes"]["path"] + decoded = deserialize_repository(json.dumps(payload).encode()) + assert decoded["path"] is None + def test_installation_id_round_trips(self): repo = make_repository(provider_name="cursor_origin", installation_id="inst_01example") decoded = deserialize_repository(serialize_repository(repo)) From 754a3fbe7b3ac46e9dafb7c60953618927391543 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 17:03:28 +0200 Subject: [PATCH 13/31] fix(git): Count a git request when its response stream ends GitServer counted a relayed request as succeeded when the provider's headers arrived, so a clone that broke halfway, or that the client gave up on, was never recorded as failed. The response stream is now counted when it ends: succeeded when it is read to the end, failed with stream_interrupted when the provider's stream breaks, and failed with client_closed_request when it is closed early. Bytes sent each way are counted at the same point, tagged by operation and provider. Refs CW-2221 Co-Authored-By: Claude Opus 5.5 --- src/scm/git/server.py | 84 ++++++++++++++++++++++++++++++----- tests/unit/test_git_server.py | 74 +++++++++++++++++++++++++++++- 2 files changed, 146 insertions(+), 12 deletions(-) diff --git a/src/scm/git/server.py b/src/scm/git/server.py index e9ba2023..2c7a6b83 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -1,4 +1,4 @@ -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterator, Mapping from requests.structures import CaseInsensitiveDict @@ -25,6 +25,10 @@ class GitServer: Refusals and provider failures come back as plain-text responses. Any other exception, such as an adapter refusing a host or the token supplier failing, reaches the caller unchanged, so the caller can answer it the way it answers the same error elsewhere. + + A relayed request is counted when its response stream ends, not when its headers arrive, so a clone that + fails halfway is counted as failed: ``stream_interrupted`` when the provider's stream breaks and + ``client_closed_request`` when the client goes away first. The bytes sent each way are counted then too. """ def __init__( @@ -59,6 +63,7 @@ def handle( ``route`` is the path below ``.git/`` and ``query`` the raw query string. ``body`` is read only while the request is sent upstream, so pass the unread request stream. """ + counted_body = _CountingReader(body) if body is not None else None try: grant = verify_grant(self.secrets, CaseInsensitiveDict(headers).get(GRANT_HEADER, "")) operation = check_request(grant, repository_id, method, route, query) @@ -70,7 +75,7 @@ def handle( if token is None: raise ProviderNotFound(detail="No git credentials are available for this repository.") response = self.relay.forward( - remote, token, method=method, route=route, query=query, headers=headers, body=body + remote, token, method=method, route=route, query=query, headers=headers, body=counted_body ) if response.status_code == 401: # The provider rejected the token Sentry supplied. Passed on, the 401 would make git ask the client @@ -84,16 +89,73 @@ def handle( self.record_count("sentry.scm.git.request.failed", 1, {"code": "unhandled_exception"}) raise - self.record_count( - "sentry.scm.git.request.succeeded", - 1, - { - "operation": operation, - "provider": repository["provider_name"], - "status": str(response.status_code), - }, + tags: dict[str, str] = {"operation": operation, "provider": repository["provider_name"]} + + def finish(failure: str | None, response_bytes: int) -> None: + if failure is None: + self.record_count("sentry.scm.git.request.succeeded", 1, tags | {"status": str(response.status_code)}) + else: + self.record_count("sentry.scm.git.request.failed", 1, {"code": failure}) + if counted_body is not None and counted_body.bytes_read: + self.record_count("sentry.scm.git.request_bytes", counted_body.bytes_read, tags) + if response_bytes: + self.record_count("sentry.scm.git.response_bytes", response_bytes, tags) + + return StreamResponse( + status_code=response.status_code, + headers=response.headers, + content=_MeteredStream(response.content, finish), ) - return response + + +class _CountingReader: + """The client's request body, counting the bytes the relay reads from it.""" + + def __init__(self, body: Readable) -> None: + self.body = body + self.bytes_read = 0 + + def read(self, size: int = -1, /) -> bytes: + data = self.body.read(size) + self.bytes_read += len(data) + return data + + +class _MeteredStream: + """The relay's response stream, reporting once how it ended and how many bytes it passed on.""" + + def __init__(self, content: Iterator[bytes], finish: Callable[[str | None, int], None]) -> None: + self.content = content + self.finish = finish + self.bytes_sent = 0 + self.finished = False + + def __iter__(self) -> "_MeteredStream": + return self + + def __next__(self) -> bytes: + try: + chunk = next(self.content) + except StopIteration: + self._end(None) + raise + except Exception: + self._end("stream_interrupted") + raise + self.bytes_sent += len(chunk) + return chunk + + def close(self) -> None: + # The server calls this after the last chunk too, when the stream has already ended. + self._end("client_closed_request") + close = getattr(self.content, "close", None) + if close is not None: + close() + + def _end(self, failure: str | None) -> None: + if not self.finished: + self.finished = True + self.finish(failure, self.bytes_sent) def _close(response: StreamResponse) -> None: diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index e890b47b..dc4e6a73 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -29,6 +29,7 @@ def __init__(self, repository: Repository | None = None, token: str | None = TOK self.repository_lookups: list[tuple[int, RepositoryId]] = [] self.token_lookups: list[tuple[Repository, GitOperation]] = [] self.counts: list[tuple[str, dict[str, str]]] = [] + self.bytes: dict[tuple[str, str], int] = {} def fetch_repository(self, organization_id: int, repository_id: RepositoryId) -> Repository | None: self.repository_lookups.append((organization_id, repository_id)) @@ -39,7 +40,11 @@ def get_token(self, repository: Repository, operation: GitOperation) -> str | No return self.token def record_count(self, name: str, value: int, tags: dict[str, str]) -> None: - self.counts.append((name, tags)) + if name.endswith("_bytes"): + key = (name, tags["operation"]) + self.bytes[key] = self.bytes.get(key, 0) + value + else: + self.counts.append((name, tags)) def make_server(recorder: Recorder, relay: GitRelay | None = None) -> GitServer: @@ -200,6 +205,51 @@ def test_forwards_with_remote_and_token(self): ("sentry.scm.git.request.succeeded", {"operation": "fetch", "provider": "github", "status": "200"}) ] + def test_counted_when_the_response_ends_not_when_it_starts(self): + recorder = Recorder() + response = request(make_server(recorder), signed_grant()) + assert recorder.counts == [] + assert b"".join(response.content) == b"refs" + response.content.close() # type: ignore[attr-defined] + assert recorder.counts == [ + ("sentry.scm.git.request.succeeded", {"operation": "fetch", "provider": "github", "status": "200"}) + ] + assert recorder.bytes == {("sentry.scm.git.response_bytes", "fetch"): 4} + + def test_provider_stream_breaking_is_a_failure(self): + def broken_stream(): + yield b"half a pack" + raise ConnectionError("provider went away") + + relay = MagicMock(spec=GitRelay) + relay.forward.return_value = StreamResponse(status_code=200, headers={}, content=broken_stream()) + recorder = Recorder() + content = request(make_server(recorder, relay), signed_grant()).content + assert next(content) == b"half a pack" + with pytest.raises(ConnectionError): + next(content) + assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "stream_interrupted"})] + assert recorder.bytes == {("sentry.scm.git.response_bytes", "fetch"): 11} + + def test_client_leaving_early_is_a_failure_and_closes_the_provider_stream(self): + closed = [] + + def provider_stream(): + try: + yield b"first" + yield b"second" + finally: + closed.append(True) + + relay = MagicMock(spec=GitRelay) + relay.forward.return_value = StreamResponse(status_code=200, headers={}, content=provider_stream()) + recorder = Recorder() + content = request(make_server(recorder, relay), signed_grant()).content + assert next(content) == b"first" + content.close() # type: ignore[attr-defined] + assert closed == [True] + assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "client_closed_request"})] + def test_upstream_unavailable(self): recorder = Recorder( repository=make_repository(provider_name="github_enterprise", web_base_url="https://127.0.0.1:1") @@ -241,6 +291,28 @@ class HostRefused(Exception): request(make_server(recorder, relay), signed_grant()) assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "unhandled_exception"})] + def test_counts_bytes_each_way_end_to_end(self, monkeypatch): + with running_upstream() as upstream: + monkeypatch.setenv("REQUESTS_CA_BUNDLE", upstream.ca_bundle) + recorder = Recorder( + repository=make_repository(provider_name="github_enterprise", web_base_url=upstream.url) + ) + with closing(GitRelay()) as relay: + response = request( + make_server(recorder, relay), + signed_grant(operation="push"), + method="POST", + route="git-receive-pack", + query="", + headers={"Content-Length": "8"}, + body=io.BytesIO(b"00000000"), + ) + assert b"".join(response.content) == b"00000000" # The fake upstream echoes the body. + assert recorder.bytes == { + ("sentry.scm.git.request_bytes", "push"): 8, + ("sentry.scm.git.response_bytes", "push"): 8, + } + def test_relays_end_to_end(self, monkeypatch): with running_upstream() as upstream: monkeypatch.setenv("REQUESTS_CA_BUNDLE", upstream.ca_bundle) From c3957a27a3af93b93a4c46317b09ff6865292a91 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 17:06:18 +0200 Subject: [PATCH 14/31] test(git): Treat a client closing mid-body as a reset in the fake upstream The relay now gives up on a request when the client's body fails, so the fake upstream can see end of stream partway through a chunked body. That is the client going away, which the server already ignores, rather than a chunk-size line to parse. Co-Authored-By: Claude Opus 5.5 --- tests/unit/git_upstream.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/unit/git_upstream.py b/tests/unit/git_upstream.py index a66c596b..994193b2 100644 --- a/tests/unit/git_upstream.py +++ b/tests/unit/git_upstream.py @@ -65,7 +65,7 @@ def _dispatch(self) -> None: def body_chunks(self) -> Iterator[bytes]: """Yield the request body as it arrives, one HTTP chunk or one read at a time.""" if self.headers.get("Transfer-Encoding") == "chunked": - while (size := int(self.rfile.readline().strip(), 16)) > 0: + while (size := int(self._chunk_size_line(), 16)) > 0: yield self.rfile.read(size) self.rfile.readline() self.rfile.readline() @@ -76,6 +76,13 @@ def body_chunks(self) -> Iterator[bytes]: remaining -= len(chunk) yield chunk + def _chunk_size_line(self) -> bytes: + line = self.rfile.readline() + if not line: + # The relay gave up on the request partway through its body, as some tests make it do. + raise ConnectionResetError("the client closed the connection mid-body") + return line.strip() + def record(self, body: bytes) -> None: self.server.upstream.requests.append( ReceivedRequest( From 0b62f41d783ecc7acf560756be892b25e8e458a7 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 17:14:15 +0200 Subject: [PATCH 15/31] fix(git): Take no proxy or CA settings from the environment requests reads HTTPS_PROXY and friends from the environment by default. Through such a proxy, an adapter that refuses internal addresses, like Sentry's BlacklistAdapter, checks the proxy's address rather than the provider's, so a customer-configured host pointing inside the network would get past it. The relay now sets trust_env to False, which also drops REQUESTS_CA_BUNDLE and .netrc, and takes an explicit ca_bundle instead. Refs CW-2221 Co-Authored-By: Claude Opus 5.5 --- src/scm/git/relay.py | 9 ++++++- tests/unit/git_upstream.py | 9 +++++-- tests/unit/test_git_relay.py | 44 ++++++++++++++++++++++++++--------- tests/unit/test_git_server.py | 10 ++++---- 4 files changed, 52 insertions(+), 20 deletions(-) diff --git a/src/scm/git/relay.py b/src/scm/git/relay.py index c20de544..f1973e68 100644 --- a/src/scm/git/relay.py +++ b/src/scm/git/relay.py @@ -58,18 +58,26 @@ class GitRelay: raises for a refused host reach the caller unchanged. The adapter must not retry, because a retry cannot resend a body that has already been streamed. Without one, the relay pools connections with a plain adapter that checks nothing. + + The relay takes no settings from the environment. Through a proxy from ``HTTPS_PROXY``, the adapter would + check the proxy's address rather than the provider's. ``ca_bundle`` names a CA file to trust instead of the + system's, in place of ``REQUESTS_CA_BUNDLE``. """ def __init__( self, *, adapter: HTTPAdapter | None = None, + ca_bundle: str | None = None, timeout: tuple[float, float] = DEFAULT_TIMEOUT, chunk_size: int = DEFAULT_CHUNK_SIZE, ) -> None: self.timeout = timeout self.chunk_size = chunk_size self.session = requests.Session() + # Without this, requests reads proxies, CA files and .netrc credentials from the environment. + self.session.trust_env = False + self.session.verify = ca_bundle or True # Every organization's requests share this session, so a cookie a provider sets on one request must not # be sent with the next. self.session.cookies.set_policy(DefaultCookiePolicy(allowed_domains=[])) @@ -118,7 +126,6 @@ def forward( data=_request_body(body, _content_length(client_headers), self.chunk_size) if method == "POST" else None, - # Passing auth also stops requests from replacing it with credentials from a .netrc file. auth=HTTPBasicAuth(remote.username, token), stream=True, # Following a redirect would send the request somewhere other than the repository's remote. diff --git a/tests/unit/git_upstream.py b/tests/unit/git_upstream.py index 994193b2..f38e067b 100644 --- a/tests/unit/git_upstream.py +++ b/tests/unit/git_upstream.py @@ -1,7 +1,7 @@ """A local HTTPS server standing in for a provider's git endpoint in relay tests. -The relay only sends tokens over https, so the server uses a self-signed certificate made for the test run. Point -requests at it with ``REQUESTS_CA_BUNDLE=``. +The relay only sends tokens over https, so the server uses a self-signed certificate made for the test run. A relay +trusts it with ``GitRelay(ca_bundle=ca_bundle())``. """ import base64 @@ -162,6 +162,11 @@ def _certificate() -> tuple[str, str]: return str(cert_path), str(key_path) +def ca_bundle() -> str: + """The path of the certificate every fake upstream in this test run serves.""" + return _certificate()[0] + + class running_upstream: """Context manager that serves an ``Upstream`` on a free local port.""" diff --git a/tests/unit/test_git_relay.py b/tests/unit/test_git_relay.py index a1702ea2..232a3f3c 100644 --- a/tests/unit/test_git_relay.py +++ b/tests/unit/test_git_relay.py @@ -18,7 +18,7 @@ ) from scm.git.relay import GitRelay from scm.git.types import GitRemote -from tests.unit.git_upstream import UpstreamHandler, basic_auth, running_upstream +from tests.unit.git_upstream import UpstreamHandler, basic_auth, ca_bundle, running_upstream TOKEN = "ghs_secrettoken" # Long enough that a relay that buffers fails the test instead of hanging it. @@ -26,15 +26,14 @@ @pytest.fixture -def upstream(monkeypatch): +def upstream(): with running_upstream() as upstream: - monkeypatch.setenv("REQUESTS_CA_BUNDLE", upstream.ca_bundle) yield upstream @pytest.fixture def relay(): - relay = GitRelay() + relay = GitRelay(ca_bundle=ca_bundle()) yield relay relay.close() @@ -211,7 +210,10 @@ def read(self, size: int = -1, /) -> bytes: return b"first" headers = {"Content-Length": "100"} if known_length else {} - with closing(GitRelay(timeout=(1, 2))) as relay, pytest.raises(SCMCodedError) as exc_info: + with ( + closing(GitRelay(ca_bundle=ca_bundle(), timeout=(1, 2))) as relay, + pytest.raises(SCMCodedError) as exc_info, + ): upload_pack(relay, upstream, ResettingClient(), headers=headers) assert exc_info.value.code == "client_closed_request" @@ -359,17 +361,32 @@ class RefusingAdapter(HTTPAdapter): def send(self, request, *args, **kwargs): raise HostRefused(request.url) - with closing(GitRelay(adapter=RefusingAdapter())) as relay, pytest.raises(HostRefused): + with closing(GitRelay(adapter=RefusingAdapter(), ca_bundle=ca_bundle())) as relay, pytest.raises(HostRefused): fetch_refs(relay, upstream) assert upstream.requests == [] def test_given_adapter_also_pools_connections(self, upstream): - with closing(GitRelay(adapter=HTTPAdapter(max_retries=0))) as relay: + with closing(GitRelay(adapter=HTTPAdapter(max_retries=0), ca_bundle=ca_bundle())) as relay: b"".join(fetch_refs(relay, upstream).content) b"".join(fetch_refs(relay, upstream).content) assert upstream.connections == 1 +class TestEnvironment: + # Behind a proxy, an adapter that refuses internal addresses would check the proxy's address rather than the + # provider's, so the relay takes nothing from the environment. + @pytest.mark.parametrize("variable", ["HTTPS_PROXY", "https_proxy", "ALL_PROXY"]) + def test_ignores_proxy_settings(self, relay, upstream, monkeypatch, variable): + monkeypatch.setenv(variable, "http://127.0.0.1:1") + b"".join(fetch_refs(relay, upstream).content) + assert len(upstream.requests) == 1 + + def test_ignores_ca_settings(self, upstream, monkeypatch): + monkeypatch.setenv("REQUESTS_CA_BUNDLE", upstream.ca_bundle) + with closing(GitRelay()) as relay, pytest.raises(ResourceCertificateInvalid): + fetch_refs(relay, upstream) + + class TestUpstreamFailures: def test_connection_refused(self, relay): with socket.socket() as s: @@ -381,9 +398,11 @@ def test_connection_refused(self, relay): fetch_refs(relay, remote=remote) assert_token_not_in(exc_info.value) - def test_untrusted_certificate(self, relay, upstream, monkeypatch): - monkeypatch.delenv("REQUESTS_CA_BUNDLE") - with pytest.raises(ResourceCertificateInvalid, match="TLS certificate of 127.0.0.1") as exc_info: + def test_untrusted_certificate(self, upstream): + with ( + closing(GitRelay()) as relay, + pytest.raises(ResourceCertificateInvalid, match="TLS certificate of 127.0.0.1") as exc_info, + ): fetch_refs(relay, upstream) assert_token_not_in(exc_info.value) assert upstream.requests == [] @@ -397,7 +416,10 @@ def handler(h: UpstreamHandler) -> None: upstream.handler = handler try: - with closing(GitRelay(timeout=(1, 0.1))) as relay, pytest.raises(ResourceGatewayTimeout) as exc_info: + with ( + closing(GitRelay(ca_bundle=ca_bundle(), timeout=(1, 0.1))) as relay, + pytest.raises(ResourceGatewayTimeout) as exc_info, + ): fetch_refs(relay, upstream) assert_token_not_in(exc_info.value) finally: diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index dc4e6a73..e543dea7 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -291,13 +291,12 @@ class HostRefused(Exception): request(make_server(recorder, relay), signed_grant()) assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "unhandled_exception"})] - def test_counts_bytes_each_way_end_to_end(self, monkeypatch): + def test_counts_bytes_each_way_end_to_end(self): with running_upstream() as upstream: - monkeypatch.setenv("REQUESTS_CA_BUNDLE", upstream.ca_bundle) recorder = Recorder( repository=make_repository(provider_name="github_enterprise", web_base_url=upstream.url) ) - with closing(GitRelay()) as relay: + with closing(GitRelay(ca_bundle=upstream.ca_bundle)) as relay: response = request( make_server(recorder, relay), signed_grant(operation="push"), @@ -313,13 +312,12 @@ def test_counts_bytes_each_way_end_to_end(self, monkeypatch): ("sentry.scm.git.response_bytes", "push"): 8, } - def test_relays_end_to_end(self, monkeypatch): + def test_relays_end_to_end(self): with running_upstream() as upstream: - monkeypatch.setenv("REQUESTS_CA_BUNDLE", upstream.ca_bundle) recorder = Recorder( repository=make_repository(provider_name="github_enterprise", web_base_url=upstream.url) ) - with closing(GitRelay()) as relay: + with closing(GitRelay(ca_bundle=upstream.ca_bundle)) as relay: response = request(make_server(recorder, relay), signed_grant()) assert response.status_code == 200 b"".join(response.content) From 54ed24878374f2e51d41b311a802fa22a380fb61 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 17:22:30 +0200 Subject: [PATCH 16/31] fix(gitlab): Build web links from the repository path GitLabProvider built file, commit and merge request links from Sentry's name for the repository with its spaces removed, but that name is GitLab's display name ("GitLab.org / GitLab FOSS"), which differs from the path (gitlab-org/gitlab-foss) in more than spaces. The links now use Repository.path, which getsentry/sentry#128198 fills in, and refuse with missing_repository_path without it. API calls address projects by id and are unaffected. Refs CW-2221 Co-Authored-By: Claude Opus 5.5 --- src/scm/providers/gitlab/provider.py | 12 ++++++---- tests/unit/provider/test_gitlab.py | 33 +++++++++++++++------------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/scm/providers/gitlab/provider.py b/src/scm/providers/gitlab/provider.py index dd10f6ce..7f7e62c2 100644 --- a/src/scm/providers/gitlab/provider.py +++ b/src/scm/providers/gitlab/provider.py @@ -9,6 +9,7 @@ from scm.errors import ( InvalidCheckRunStateTransition, MalformedExternalId, + MissingRepositoryPath, PathIsNotDirectory, ReadmeNotFound, ResourceBadRequest, @@ -243,11 +244,14 @@ def __init__(self, client: ApiClient, organization_id: int, repository: Reposito @property def web_repository_path(self) -> str: + """The repository's path on the instance (e.g. "gitlab-org/gitlab-foss"), for web URLs. + + Sentry's name for a GitLab repository is its display name ("GitLab.org / GitLab FOSS"), which differs from + the path in more than spaces, so the path has to come from the repository. """ - Normalize Sentry's GitLab full repo name (e.g., "my-group / my-team / my-subteam / my-project" - --> "my-group/my-team/my-subteam/my-project") for use in web URLs. - """ - return self.repository["name"].replace(" ", "") + if not (path := self.repository.get("path")): + raise MissingRepositoryPath(detail="The GitLab repository has no path.") + return path def request( self, diff --git a/tests/unit/provider/test_gitlab.py b/tests/unit/provider/test_gitlab.py index 09d6b67f..35c560cd 100644 --- a/tests/unit/provider/test_gitlab.py +++ b/tests/unit/provider/test_gitlab.py @@ -7,6 +7,7 @@ import pytest from scm.errors import ( + MissingRepositoryPath, RateLimitExceeded, ResourceBadGateway, ResourceBadRequest, @@ -47,7 +48,9 @@ def client() -> ApiClient: return unittest.mock.MagicMock(_name="client") -def _make_gitlab_provider(client: ApiClient, *, name: str = "test-repo") -> GitLabProvider: +def _make_gitlab_provider( + client: ApiClient, *, name: str = "Test Repo", path: str | None = "test-repo" +) -> GitLabProvider: return GitLabProvider( client=client, organization_id=1, @@ -55,6 +58,7 @@ def _make_gitlab_provider(client: ApiClient, *, name: str = "test-repo") -> GitL id=1, integration_id=1, name=name, + path=path, organization_id=1, is_active=True, external_id="gitlab.com:79787061", @@ -13939,24 +13943,23 @@ def test_download_archive_zip_uses_zip_extension(client, provider: GitLabProvide assert call.kwargs["path"] == "/projects/79787061/repository/archive.zip" -@pytest.mark.parametrize( - ("repository_name", "expected_path"), - [ - ("test-org/test-repo", "test-org/test-repo"), - ("my-group / my-project", "my-group/my-project"), - ("my-group / my-subteam / my-project", "my-group/my-subteam/my-project"), - ], -) -def test_web_repository_path_normalizes_namespaced_name( - client: ApiClient, repository_name: str, expected_path: str -) -> None: - provider = _make_gitlab_provider(client, name=repository_name) +def test_web_repository_path_is_the_repository_path_not_its_name(client: ApiClient) -> None: + # Sentry's name for a GitLab repository is its display name, which differs from the path in more than spaces. + provider = _make_gitlab_provider(client, name="GitLab.org / GitLab FOSS", path="gitlab-org/gitlab-foss") + + assert provider.web_repository_path == "gitlab-org/gitlab-foss" + + +@pytest.mark.parametrize("path", [None, ""]) +def test_web_repository_path_needs_the_repository_path(client: ApiClient, path: str | None) -> None: + provider = _make_gitlab_provider(client, name="GitLab.org / GitLab FOSS", path=path) - assert provider.web_repository_path == expected_path + with pytest.raises(MissingRepositoryPath): + provider.get_commit_url("abc123") def test_web_repository_path_used_in_commit_and_mr_urls(client: ApiClient) -> None: - provider = _make_gitlab_provider(client, name="my-group / my-project") + provider = _make_gitlab_provider(client, name="My Group / My Project", path="my-group/my-project") assert provider.get_commit_url("abc123") == "https://gitlab.com/my-group/my-project/-/commit/abc123" assert provider.get_commits_url("abc123") == "https://gitlab.com/my-group/my-project/-/commits/abc123" From a6802f0e686dbc3cb56ba33b7d0d0e79f1c48488 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 20:03:05 +0200 Subject: [PATCH 17/31] ref(git): Share stream closing and the fake relay setup The server closed a relay stream in two places with the same getattr dance, and four server tests built the same mock relay by hand. One helper each keeps the next change to either from touching only some copies. The relay's request body moves out of the session.request call so the call reads as a list of arguments rather than a conditional. Co-Authored-By: Claude Opus 5.5 --- src/scm/git/relay.py | 9 ++++----- src/scm/git/server.py | 10 ++++------ tests/unit/test_git_server.py | 23 ++++++++++++----------- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/src/scm/git/relay.py b/src/scm/git/relay.py index f1973e68..cb2a8e10 100644 --- a/src/scm/git/relay.py +++ b/src/scm/git/relay.py @@ -115,17 +115,16 @@ def forward( # client could get an encoding it cannot read. upstream_headers.setdefault("Accept-Encoding", "identity") + # Of the git routes only the POSTs carry a body. A caller may pass its request stream for a GET too, and sent + # on, it would go upstream as an empty chunked body. + data = _request_body(body, _content_length(client_headers), self.chunk_size) if method == "POST" else None url = f"{remote.url}/{route}?{query}" if query else f"{remote.url}/{route}" try: upstream = self.session.request( method, url, headers=upstream_headers, - # Of the git routes only the POSTs carry a body. A caller may pass its request stream for a GET too, - # and sent on, it would go upstream as an empty chunked body. - data=_request_body(body, _content_length(client_headers), self.chunk_size) - if method == "POST" - else None, + data=data, auth=HTTPBasicAuth(remote.username, token), stream=True, # Following a redirect would send the request somewhere other than the repository's remote. diff --git a/src/scm/git/server.py b/src/scm/git/server.py index 2c7a6b83..643e72cd 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -80,7 +80,7 @@ def handle( if response.status_code == 401: # The provider rejected the token Sentry supplied. Passed on, the 401 would make git ask the client # for credentials, which it must never have and which would not help. - _close(response) + _close_stream(response.content) raise ResourceBadGateway(detail="The provider rejected Sentry's credentials for this repository.") except SCMCodedError as e: self.record_count("sentry.scm.git.request.failed", 1, {"code": e.code}) @@ -148,9 +148,7 @@ def __next__(self) -> bytes: def close(self) -> None: # The server calls this after the last chunk too, when the stream has already ended. self._end("client_closed_request") - close = getattr(self.content, "close", None) - if close is not None: - close() + _close_stream(self.content) def _end(self, failure: str | None) -> None: if not self.finished: @@ -158,9 +156,9 @@ def _end(self, failure: str | None) -> None: self.finish(failure, self.bytes_sent) -def _close(response: StreamResponse) -> None: +def _close_stream(content: Iterator[bytes]) -> None: # Closing the relay's stream before it is read to the end drops the upstream connection. - close = getattr(response.content, "close", None) + close = getattr(content, "close", None) if close is not None: close() diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index e543dea7..63bb3464 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -1,5 +1,6 @@ import io import time +from collections.abc import Iterator from contextlib import closing from unittest.mock import MagicMock @@ -47,10 +48,16 @@ def record_count(self, name: str, value: int, tags: dict[str, str]) -> None: self.counts.append((name, tags)) +def relay_returning(content: Iterator[bytes], status_code: int = 200) -> GitRelay: + """A stand-in relay whose every forward returns ``content`` as the provider's response.""" + relay = MagicMock(spec=GitRelay) + relay.forward.return_value = StreamResponse(status_code=status_code, headers={}, content=content) + return relay + + def make_server(recorder: Recorder, relay: GitRelay | None = None) -> GitServer: if relay is None: - relay = MagicMock(spec=GitRelay) - relay.forward.return_value = StreamResponse(status_code=200, headers={}, content=iter([b"refs"])) + relay = relay_returning(iter([b"refs"])) return GitServer( secrets=[SECRET], fetch_repository=recorder.fetch_repository, @@ -221,10 +228,8 @@ def broken_stream(): yield b"half a pack" raise ConnectionError("provider went away") - relay = MagicMock(spec=GitRelay) - relay.forward.return_value = StreamResponse(status_code=200, headers={}, content=broken_stream()) recorder = Recorder() - content = request(make_server(recorder, relay), signed_grant()).content + content = request(make_server(recorder, relay_returning(broken_stream())), signed_grant()).content assert next(content) == b"half a pack" with pytest.raises(ConnectionError): next(content) @@ -241,10 +246,8 @@ def provider_stream(): finally: closed.append(True) - relay = MagicMock(spec=GitRelay) - relay.forward.return_value = StreamResponse(status_code=200, headers={}, content=provider_stream()) recorder = Recorder() - content = request(make_server(recorder, relay), signed_grant()).content + content = request(make_server(recorder, relay_returning(provider_stream())), signed_grant()).content assert next(content) == b"first" content.close() # type: ignore[attr-defined] assert closed == [True] @@ -271,10 +274,8 @@ def provider_body(): content = provider_body() next(content) # Started, as a relay's stream is once its first chunk is read. - relay = MagicMock(spec=GitRelay) - relay.forward.return_value = StreamResponse(status_code=401, headers={}, content=content) recorder = Recorder() - response = request(make_server(recorder, relay), signed_grant()) + response = request(make_server(recorder, relay_returning(content, status_code=401)), signed_grant()) assert_refused(response, 502, "rejected Sentry's credentials") assert closed == [True] assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "resource_bad_gateway"})] From 4711964769e4a4cc5c601ff81821a862664c339e Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 20:36:09 +0200 Subject: [PATCH 18/31] ref(git): Open a connection per relayed request A pooled session shared across organizations needed a blocked cookie jar and relied on urllib3 never returning a half-read connection, and each is one more thing to get right for every change to the relay. The handshakes pooling saved were measured in the prototype over a slow local link; from Sentry's servers to the providers they should cost little. With a session per request, nothing one organization's request leaves on a connection can reach another's. The adapter argument becomes a factory, since an adapter instance holds its own pool. Sentry can pass the BlacklistAdapter class as it is. Co-Authored-By: Claude Opus 5.5 --- src/scm/git/relay.py | 67 +++++++++++++++-------------- src/scm/git/server.py | 4 +- tests/unit/test_git_relay.py | 79 ++++++++++++++++++++++------------- tests/unit/test_git_server.py | 34 +++++++-------- 4 files changed, 103 insertions(+), 81 deletions(-) diff --git a/src/scm/git/relay.py b/src/scm/git/relay.py index cb2a8e10..a2be7a13 100644 --- a/src/scm/git/relay.py +++ b/src/scm/git/relay.py @@ -1,5 +1,4 @@ -from collections.abc import Iterator, Mapping -from http.cookiejar import DefaultCookiePolicy +from collections.abc import Callable, Iterator, Mapping from urllib.parse import urlsplit import requests @@ -46,18 +45,21 @@ class GitRelay: """Forwards git smart-HTTP requests to a provider with a token, streaming the body in both directions. - Nothing buffers a request or a response, and neither body is parsed. Build one per process and reuse it: it - keeps a pool of keep-alive connections per provider host, because a clone makes three requests and opening a - new connection for each added hundreds of milliseconds per clone in the prototype. + Nothing buffers a request or a response, and neither body is parsed. + + Each request opens its own connection and closes it when the response ends, so nothing one organization's + request leaves behind, such as a cookie or an unread response, can reach another's. A clone's three requests + each pay a TCP and TLS handshake. Over the prototype's slow local link that added hundreds of milliseconds per + clone; from Sentry's servers it should be small. Only https remotes are accepted, because every request carries a provider token. - ``adapter`` sends every upstream request and holds the connection pools. Pass one that refuses internal - addresses: a GitHub Enterprise or self-managed GitLab host is whatever the customer configured, and the relay - connects wherever it resolves. Sentry's ``BlacklistAdapter`` does this for its API clients. Errors the adapter - raises for a refused host reach the caller unchanged. The adapter must not retry, because a retry cannot - resend a body that has already been streamed. Without one, the relay pools connections with a plain adapter - that checks nothing. + ``adapter`` builds the adapter that sends each upstream request. Pass one that refuses internal addresses: a + GitHub Enterprise or self-managed GitLab host is whatever the customer configured, and the relay connects + wherever it resolves. Sentry's ``BlacklistAdapter`` does this for its API clients, and the class itself can be + passed. Errors the adapter raises for a refused host reach the caller unchanged. The adapter must not retry, + because a retry cannot resend a body that has already been streamed. Without one, the relay uses a plain + ``HTTPAdapter``, which checks nothing. The relay takes no settings from the environment. Through a proxy from ``HTTPS_PROXY``, the adapter would check the proxy's address rather than the provider's. ``ca_bundle`` names a CA file to trust instead of the @@ -67,23 +69,15 @@ class GitRelay: def __init__( self, *, - adapter: HTTPAdapter | None = None, + adapter: Callable[[], HTTPAdapter] = HTTPAdapter, ca_bundle: str | None = None, timeout: tuple[float, float] = DEFAULT_TIMEOUT, chunk_size: int = DEFAULT_CHUNK_SIZE, ) -> None: + self.adapter = adapter + self.ca_bundle = ca_bundle self.timeout = timeout self.chunk_size = chunk_size - self.session = requests.Session() - # Without this, requests reads proxies, CA files and .netrc credentials from the environment. - self.session.trust_env = False - self.session.verify = ca_bundle or True - # Every organization's requests share this session, so a cookie a provider sets on one request must not - # be sent with the next. - self.session.cookies.set_policy(DefaultCookiePolicy(allowed_domains=[])) - # The default keeps pools for up to 32 provider hosts and up to 64 idle connections per host. These are - # guesses for one Sentry worker's concurrent clones, not measurements. - self.session.mount("https://", adapter or HTTPAdapter(pool_connections=32, pool_maxsize=64, max_retries=0)) def forward( self, @@ -119,8 +113,10 @@ def forward( # on, it would go upstream as an empty chunked body. data = _request_body(body, _content_length(client_headers), self.chunk_size) if method == "POST" else None url = f"{remote.url}/{route}?{query}" if query else f"{remote.url}/{route}" + session = self._session() + upstream = None try: - upstream = self.session.request( + upstream = session.request( method, url, headers=upstream_headers, @@ -141,20 +137,29 @@ def forward( raise ResourceCertificateInvalid(detail=f"The TLS certificate of {host} could not be verified.") from None except requests.ConnectionError: raise ResourceServiceUnavailable(detail=f"Could not connect to {remote.url}.") from None + finally: + # Once a response arrives, _stream closes the session when the response ends. + if upstream is None: + session.close() return StreamResponse( status_code=upstream.status_code, headers={name: value for name, value in upstream.headers.items() if name.lower() in _RESPONSE_HEADERS}, - content=self._stream(upstream), + content=self._stream(session, upstream), ) - def close(self) -> None: - self.session.close() - - def _stream(self, upstream: requests.Response) -> Iterator[bytes]: - # Once the response is read to the end its connection goes back to the pool. Closing it before then, - # because the client went away, drops the connection instead, since it is mid-response. - with upstream: + def _session(self) -> requests.Session: + session = requests.Session() + # Without this, requests reads proxies, CA files and .netrc credentials from the environment. + session.trust_env = False + session.verify = self.ca_bundle or True + session.mount("https://", self.adapter()) + return session + + def _stream(self, session: requests.Session, upstream: requests.Response) -> Iterator[bytes]: + # Closing the response first drops its connection if the client went away mid-response; closing the + # session then closes a connection the finished response returned to it. + with session, upstream: yield from filter(bool, upstream.raw.stream(self.chunk_size, decode_content=False)) diff --git a/src/scm/git/server.py b/src/scm/git/server.py index 643e72cd..39057979 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -19,8 +19,8 @@ class GitServer: The order of the checks matters. The grant is verified, and the request's route and repository checked against it, before the repository is looked up or a token fetched, so a request the grant does not allow - never reaches the database, the token supplier or the provider. Build one per process: its relay pools - upstream connections. Give it a relay whose adapter refuses internal addresses; see ``GitRelay``. + never reaches the database, the token supplier or the provider. Give it a relay whose adapter refuses internal + addresses; see ``GitRelay``. Refusals and provider failures come back as plain-text responses. Any other exception, such as an adapter refusing a host or the token supplier failing, reaches the caller unchanged, so the caller can answer it the diff --git a/tests/unit/test_git_relay.py b/tests/unit/test_git_relay.py index 232a3f3c..363f7679 100644 --- a/tests/unit/test_git_relay.py +++ b/tests/unit/test_git_relay.py @@ -4,7 +4,6 @@ import threading import time from collections.abc import Iterator -from contextlib import closing import pytest from requests.adapters import HTTPAdapter @@ -33,9 +32,7 @@ def upstream(): @pytest.fixture def relay(): - relay = GitRelay(ca_bundle=ca_bundle()) - yield relay - relay.close() + return GitRelay(ca_bundle=ca_bundle()) def remote_for(upstream) -> GitRemote: @@ -210,10 +207,8 @@ def read(self, size: int = -1, /) -> bytes: return b"first" headers = {"Content-Length": "100"} if known_length else {} - with ( - closing(GitRelay(ca_bundle=ca_bundle(), timeout=(1, 2))) as relay, - pytest.raises(SCMCodedError) as exc_info, - ): + relay = GitRelay(ca_bundle=ca_bundle(), timeout=(1, 2)) + with pytest.raises(SCMCodedError) as exc_info: upload_pack(relay, upstream, ResettingClient(), headers=headers) assert exc_info.value.code == "client_closed_request" @@ -332,13 +327,49 @@ def read(self, size: int = -1, /) -> bytes: assert b"".join(received) == b"firstsecond" +class ClosingAdapter(HTTPAdapter): + """An adapter that records whether it was closed, which closes every connection it opened.""" + + def __init__(self) -> None: + super().__init__() + self.closed = False + + def close(self) -> None: + self.closed = True + super().close() + + class TestConnections: - def test_reuses_the_upstream_connection(self, relay, upstream): + def test_opens_a_connection_per_request(self, relay, upstream): b"".join(fetch_refs(relay, upstream).content) b"".join(upload_pack(relay, upstream, io.BytesIO(b"0000"), headers={"Content-Length": "4"}).content) b"".join(upload_pack(relay, upstream, io.BytesIO(b"0000")).content) assert len(upstream.requests) == 3 - assert upstream.connections == 1 + assert upstream.connections == 3 + + def test_closes_the_connection_when_the_response_ends(self, upstream): + adapters: list[ClosingAdapter] = [] + + def adapter() -> ClosingAdapter: + adapters.append(ClosingAdapter()) + return adapters[-1] + + response = fetch_refs(GitRelay(adapter=adapter, ca_bundle=ca_bundle()), upstream) + assert [a.closed for a in adapters] == [False] + b"".join(response.content) + assert [a.closed for a in adapters] == [True] + + def test_closes_the_connection_when_the_request_fails(self): + adapters: list[ClosingAdapter] = [] + + def adapter() -> ClosingAdapter: + adapters.append(ClosingAdapter()) + return adapters[-1] + + remote = GitRemote(url="https://127.0.0.1:1/getsentry/sentry.git", username="x-access-token") + with pytest.raises(ResourceServiceUnavailable): + fetch_refs(GitRelay(adapter=adapter, timeout=(1, 1)), remote=remote) + assert [a.closed for a in adapters] == [True] def test_does_not_carry_cookies_between_requests(self, relay, upstream): def handler(h: UpstreamHandler) -> None: @@ -361,16 +392,10 @@ class RefusingAdapter(HTTPAdapter): def send(self, request, *args, **kwargs): raise HostRefused(request.url) - with closing(GitRelay(adapter=RefusingAdapter(), ca_bundle=ca_bundle())) as relay, pytest.raises(HostRefused): - fetch_refs(relay, upstream) + with pytest.raises(HostRefused): + fetch_refs(GitRelay(adapter=RefusingAdapter, ca_bundle=ca_bundle()), upstream) assert upstream.requests == [] - def test_given_adapter_also_pools_connections(self, upstream): - with closing(GitRelay(adapter=HTTPAdapter(max_retries=0), ca_bundle=ca_bundle())) as relay: - b"".join(fetch_refs(relay, upstream).content) - b"".join(fetch_refs(relay, upstream).content) - assert upstream.connections == 1 - class TestEnvironment: # Behind a proxy, an adapter that refuses internal addresses would check the proxy's address rather than the @@ -383,8 +408,8 @@ def test_ignores_proxy_settings(self, relay, upstream, monkeypatch, variable): def test_ignores_ca_settings(self, upstream, monkeypatch): monkeypatch.setenv("REQUESTS_CA_BUNDLE", upstream.ca_bundle) - with closing(GitRelay()) as relay, pytest.raises(ResourceCertificateInvalid): - fetch_refs(relay, upstream) + with pytest.raises(ResourceCertificateInvalid): + fetch_refs(GitRelay(), upstream) class TestUpstreamFailures: @@ -399,11 +424,8 @@ def test_connection_refused(self, relay): assert_token_not_in(exc_info.value) def test_untrusted_certificate(self, upstream): - with ( - closing(GitRelay()) as relay, - pytest.raises(ResourceCertificateInvalid, match="TLS certificate of 127.0.0.1") as exc_info, - ): - fetch_refs(relay, upstream) + with pytest.raises(ResourceCertificateInvalid, match="TLS certificate of 127.0.0.1") as exc_info: + fetch_refs(GitRelay(), upstream) assert_token_not_in(exc_info.value) assert upstream.requests == [] @@ -416,11 +438,8 @@ def handler(h: UpstreamHandler) -> None: upstream.handler = handler try: - with ( - closing(GitRelay(ca_bundle=ca_bundle(), timeout=(1, 0.1))) as relay, - pytest.raises(ResourceGatewayTimeout) as exc_info, - ): - fetch_refs(relay, upstream) + with pytest.raises(ResourceGatewayTimeout) as exc_info: + fetch_refs(GitRelay(ca_bundle=ca_bundle(), timeout=(1, 0.1)), upstream) assert_token_not_in(exc_info.value) finally: release.set() diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index 63bb3464..deba41bf 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -1,7 +1,6 @@ import io import time from collections.abc import Iterator -from contextlib import closing from unittest.mock import MagicMock import pytest @@ -257,8 +256,7 @@ def test_upstream_unavailable(self): recorder = Recorder( repository=make_repository(provider_name="github_enterprise", web_base_url="https://127.0.0.1:1") ) - with closing(GitRelay(timeout=(1, 1))) as relay: - response = request(make_server(recorder, relay), signed_grant()) + response = request(make_server(recorder, GitRelay(timeout=(1, 1))), signed_grant()) assert_refused(response, 503, "Could not connect") assert TOKEN not in b"".join(response.content).decode() @@ -297,17 +295,17 @@ def test_counts_bytes_each_way_end_to_end(self): recorder = Recorder( repository=make_repository(provider_name="github_enterprise", web_base_url=upstream.url) ) - with closing(GitRelay(ca_bundle=upstream.ca_bundle)) as relay: - response = request( - make_server(recorder, relay), - signed_grant(operation="push"), - method="POST", - route="git-receive-pack", - query="", - headers={"Content-Length": "8"}, - body=io.BytesIO(b"00000000"), - ) - assert b"".join(response.content) == b"00000000" # The fake upstream echoes the body. + relay = GitRelay(ca_bundle=upstream.ca_bundle) + response = request( + make_server(recorder, relay), + signed_grant(operation="push"), + method="POST", + route="git-receive-pack", + query="", + headers={"Content-Length": "8"}, + body=io.BytesIO(b"00000000"), + ) + assert b"".join(response.content) == b"00000000" # The fake upstream echoes the body. assert recorder.bytes == { ("sentry.scm.git.request_bytes", "push"): 8, ("sentry.scm.git.response_bytes", "push"): 8, @@ -318,10 +316,10 @@ def test_relays_end_to_end(self): recorder = Recorder( repository=make_repository(provider_name="github_enterprise", web_base_url=upstream.url) ) - with closing(GitRelay(ca_bundle=upstream.ca_bundle)) as relay: - response = request(make_server(recorder, relay), signed_grant()) - assert response.status_code == 200 - b"".join(response.content) + relay = GitRelay(ca_bundle=upstream.ca_bundle) + response = request(make_server(recorder, relay), signed_grant()) + assert response.status_code == 200 + b"".join(response.content) [received] = upstream.requests assert received.path == "/getsentry/sentry.git/info/refs?service=git-upload-pack" assert received.headers["Authorization"] == basic_auth("x-access-token", TOKEN) From 88ab8e56d45f1c0305e5fffcf6785fc5c769e30f Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 20 Oct 2026 20:49:28 +0200 Subject: [PATCH 19/31] ref(git): Ask the provider for a repository's git remote Clone URLs lived in a match on provider_name in scm.git.remote, apart from the providers that already build every other URL for a repository, and the two had drifted: the GitLab provider took its host from the external ID while the remote took it from web_base_url. A GetGitRemoteProtocol puts the clone URL and basic-auth username next to the web-link protocols, and GitServer builds the provider with initialize_provider to ask for it. A provider without the capability, such as Cursor Origin's, is refused. Both providers now fall back to the repository's web_base_url when built without one. Sentry's fetch_provider builds GitHubProvider for GitHub Enterprise without it, which pointed web links at github.com and would have sent a GitHub Enterprise token there. GitLab keeps the external ID's host when Sentry sends no web_base_url, and uses the instance's full URL, with any path prefix, when it does. The relay's https check becomes a coded error, since a self-managed instance served over http is a configuration to refuse rather than a bug. Co-Authored-By: Claude Opus 5.5 --- bin/github-client | 8 +++ bin/gitlab-client | 8 +++ src/scm/actions.py | 8 +++ src/scm/errors.py | 12 ++--- src/scm/git/relay.py | 20 +++---- src/scm/git/remote.py | 41 --------------- src/scm/git/server.py | 30 +++++++---- src/scm/git/types.py | 11 ---- src/scm/providers/github/provider.py | 10 +++- src/scm/providers/gitlab/provider.py | 8 ++- src/scm/test_fixtures.py | 4 ++ src/scm/types.py | 15 ++++++ tests/unit/git_builders.py | 21 +++++++- tests/unit/provider/test_github.py | 36 +++++++++++++ tests/unit/provider/test_gitlab.py | 37 +++++++++++++ tests/unit/test_actions.py | 7 +++ tests/unit/test_git_relay.py | 5 +- tests/unit/test_git_remote.py | 79 ---------------------------- tests/unit/test_git_server.py | 37 +++++++++++-- tests/unit/test_rpc_integration.py | 5 ++ 20 files changed, 236 insertions(+), 166 deletions(-) delete mode 100644 src/scm/git/remote.py delete mode 100644 tests/unit/test_git_remote.py diff --git a/bin/github-client b/bin/github-client index 8b96aa83..38ce0da0 100755 --- a/bin/github-client +++ b/bin/github-client @@ -21,6 +21,7 @@ Commands: get-pull-request-review get-review-comments [--cursor ] [--per-page ] get-pull-request-url + get-git-remote mark-pull-request-ready-for-review mark-pull-request-as-draft get-issue @@ -129,6 +130,7 @@ from scm.types import ( GetFileContentProtocol, GetFileUrlProtocol, GetGitRefProtocol, + GetGitRemoteProtocol, GetIssueCommentsProtocol, GetIssueProtocol, GetPullRequestCommentsProtocol, @@ -283,6 +285,8 @@ def main() -> None: p = sub.add_parser("get-pull-request-url") p.add_argument("id") + sub.add_parser("get-git-remote") + p = sub.add_parser("mark-pull-request-ready-for-review") p.add_argument("id") @@ -567,6 +571,10 @@ def main() -> None: assert isinstance(scm, GetPullRequestUrlProtocol) dump(scm.get_pull_request_url(args.id)) + elif args.command == "get-git-remote": + assert isinstance(scm, GetGitRemoteProtocol) + dump(scm.get_git_remote()) + elif args.command == "mark-pull-request-ready-for-review": assert isinstance(scm, MarkPullRequestDraftStateProtocol) scm.mark_pull_request_ready_for_review(args.id) diff --git a/bin/gitlab-client b/bin/gitlab-client index d9fcb90e..4a5a52f5 100755 --- a/bin/gitlab-client +++ b/bin/gitlab-client @@ -23,6 +23,7 @@ Commands: get-pull-request-diff get-pull-request-review-threads [--include-reactions] get-pull-request-url + get-git-remote get-issue create-issue <body> [--assignee <user_id> ...] [--label <name> ...] update-issue <issue_id> [--state open|closed] [--assignee <user_id> ...] [--label <name> ...] @@ -117,6 +118,7 @@ from scm.types import ( GetDirectoryContentsProtocol, GetFileContentProtocol, GetFileUrlProtocol, + GetGitRemoteProtocol, GetIssueCommentsProtocol, GetIssueProtocol, GetPullRequestCommentsProtocol, @@ -248,6 +250,8 @@ def main(): p = sub.add_parser("get-pull-request-url") p.add_argument("id") + sub.add_parser("get-git-remote") + p = sub.add_parser("get-issue") p.add_argument("id") @@ -453,6 +457,10 @@ def main(): assert isinstance(scm, GetPullRequestUrlProtocol) dump(scm.get_pull_request_url(args.id)) + elif args.command == "get-git-remote": + assert isinstance(scm, GetGitRemoteProtocol) + dump(scm.get_git_remote()) + elif args.command == "get-issue": assert isinstance(scm, GetIssueProtocol) dump(scm.get_issue(args.id)) diff --git a/src/scm/actions.py b/src/scm/actions.py index dfa2071a..1ed97067 100644 --- a/src/scm/actions.py +++ b/src/scm/actions.py @@ -78,6 +78,7 @@ GetFullTreeProtocol, GetGitCommitProtocol, GetGitRefProtocol, + GetGitRemoteProtocol, GetIssueCommentReactionsProtocol, GetIssueCommentsProtocol, GetIssueProtocol, @@ -106,6 +107,7 @@ GitBlob, GitCommitObject, GitRef, + GitRemote, GitRepository, GitTree, InputTreeEntry, @@ -528,6 +530,11 @@ def get_pull_request_url(scm: GetPullRequestUrlProtocol, pull_request_id: str) - return scm.get_pull_request_url(pull_request_id) +def get_git_remote(scm: GetGitRemoteProtocol) -> GitRemote: + """Return the repository's git smart-HTTP endpoint and the basic-auth username its provider pairs with a token.""" + return scm.get_git_remote() + + def create_git_blob(scm: CreateGitBlobProtocol, content: str, encoding: str) -> ActionResult[GitBlob]: """Create a git blob object.""" return scm.create_git_blob(content, encoding) @@ -1202,6 +1209,7 @@ def download_archive( "get_file_url", "get_git_commit", "get_git_ref", + "get_git_remote", "get_issue_comment_reactions", "get_issue_comments", "get_issue_reactions", diff --git a/src/scm/errors.py b/src/scm/errors.py index 20cb22f4..1a2a88ec 100644 --- a/src/scm/errors.py +++ b/src/scm/errors.py @@ -3,8 +3,8 @@ type ErrorCode = Literal[ "git_invalid_grant", "git_provider_not_supported", + "git_remote_not_https", "git_route_not_allowed", - "invalid_web_base_url", "malformed_external_id", "missing_repository_path", "path_is_directory", @@ -45,8 +45,8 @@ ERROR_CODES: dict[ErrorCode, str] = { "git_invalid_grant": "The git grant was missing, malformed, forged or expired.", "git_provider_not_supported": "The repository's provider has no git remote.", + "git_remote_not_https": "The repository's git remote is not an https URL.", "git_route_not_allowed": "The git grant does not allow this request.", - "invalid_web_base_url": "The repository's web_base_url is missing or is not an https URL.", "malformed_external_id": "The repository's external ID was malformed.", "missing_repository_path": "The repository's path on its provider is unknown.", "path_is_directory": "The requested path is a directory, not a file.", @@ -143,12 +143,12 @@ class GitProviderNotSupported(SCMCodedError): code = "git_provider_not_supported" -class GitRouteNotAllowed(SCMCodedError): - code = "git_route_not_allowed" +class GitRemoteNotHttps(SCMCodedError): + code = "git_remote_not_https" -class InvalidWebBaseUrl(SCMCodedError): - code = "invalid_web_base_url" +class GitRouteNotAllowed(SCMCodedError): + code = "git_route_not_allowed" class MalformedExternalId(SCMCodedError): diff --git a/src/scm/git/relay.py b/src/scm/git/relay.py index a2be7a13..79896b89 100644 --- a/src/scm/git/relay.py +++ b/src/scm/git/relay.py @@ -8,14 +8,16 @@ from scm.errors import ( ClientClosedRequest, + GitRemoteNotHttps, ResourceCertificateInvalid, ResourceGatewayTimeout, ResourceServiceUnavailable, RpcMalformedRequestHeaders, ) from scm.git.operations import require_route -from scm.git.types import GitRemote, Readable +from scm.git.types import Readable from scm.rpc.types import StreamResponse +from scm.types import GitRemote # The most bytes the relay reads at once, from the client's body or from the provider's response, so also the # most it holds per request in each direction. Smaller chunks cost more Python per byte of a multi-gigabyte pack; @@ -94,9 +96,9 @@ def forward( ``route`` is the path below ``<repo>.git/``. Check the request against a grant before calling this. """ - if not remote.url.startswith("https://"): - # git_remote only builds https URLs, so this is a caller's bug rather than a request to refuse. - raise ValueError(f"Refusing to send a token to {remote.url}, which is not https.") + if not remote["url"].startswith("https://"): + # The token would cross the network in the clear, e.g. to a self-managed instance served over http. + raise GitRemoteNotHttps(detail=f"Refusing to send a token to {remote['url']}, which is not https.") # Callers check the request against a grant first; this only makes sure the token never goes to anything # but a git route. require_route(method, route, query) @@ -112,7 +114,7 @@ def forward( # Of the git routes only the POSTs carry a body. A caller may pass its request stream for a GET too, and sent # on, it would go upstream as an empty chunked body. data = _request_body(body, _content_length(client_headers), self.chunk_size) if method == "POST" else None - url = f"{remote.url}/{route}?{query}" if query else f"{remote.url}/{route}" + url = f"{remote['url']}/{route}?{query}" if query else f"{remote['url']}/{route}" session = self._session() upstream = None try: @@ -121,7 +123,7 @@ def forward( url, headers=upstream_headers, data=data, - auth=HTTPBasicAuth(remote.username, token), + auth=HTTPBasicAuth(remote["username"], token), stream=True, # Following a redirect would send the request somewhere other than the repository's remote. allow_redirects=False, @@ -129,14 +131,14 @@ def forward( ) # "from None" drops requests' exception, whose request holds the Authorization header. except requests.Timeout: - raise ResourceGatewayTimeout(detail=f"{remote.url} did not respond in time.") from None + raise ResourceGatewayTimeout(detail=f"{remote['url']} did not respond in time.") from None # SSLError is a kind of ConnectionError, so catch it before that one, or a bad certificate would be # reported as an unreachable host. except requests.exceptions.SSLError: - host = urlsplit(remote.url).netloc + host = urlsplit(remote["url"]).netloc raise ResourceCertificateInvalid(detail=f"The TLS certificate of {host} could not be verified.") from None except requests.ConnectionError: - raise ResourceServiceUnavailable(detail=f"Could not connect to {remote.url}.") from None + raise ResourceServiceUnavailable(detail=f"Could not connect to {remote['url']}.") from None finally: # Once a response arrives, _stream closes the session when the response ends. if upstream is None: diff --git a/src/scm/git/remote.py b/src/scm/git/remote.py deleted file mode 100644 index 329a6dd5..00000000 --- a/src/scm/git/remote.py +++ /dev/null @@ -1,41 +0,0 @@ -from urllib.parse import urlsplit - -from scm.errors import GitProviderNotSupported, InvalidWebBaseUrl, MissingRepositoryPath -from scm.git.types import GitRemote -from scm.types import Repository - -GITHUB_GIT_BASE_URL = "https://github.com" - - -def git_remote(repository: Repository) -> GitRemote: - """Return the git endpoint and basic-auth username for a repository's provider. - - Bitbucket Cloud (basic auth as ``x-token-auth``) can be added the same way once the platform has a provider - for it. Azure DevOps authenticates git with a bearer header, which ``GitRemote`` cannot express yet. - """ - match repository["provider_name"]: - case "github": - return GitRemote(url=f"{GITHUB_GIT_BASE_URL}/{repository['name']}.git", username="x-access-token") - case "github_enterprise": - base_url = _https_base_url(repository) - return GitRemote(url=f"{base_url}/{repository['name']}.git", username="x-access-token") - case "gitlab": - # Sentry's name for a GitLab repository is its display name, so the path has to come separately. - if not (path := repository.get("path")): - raise MissingRepositoryPath(detail="The GitLab repository has no path.") - return GitRemote(url=f"{_https_base_url(repository)}/{path}.git", username="oauth2") - case provider_name: - raise GitProviderNotSupported(detail=f"The {provider_name} provider has no git remote.") - - -def _https_base_url(repository: Repository) -> str: - # The token goes to this URL, so it must be https. Sentry sets web_base_url for every GitHub Enterprise and - # GitLab repository from the integration's instance URL; for GitLab that keeps the path prefix of an instance - # served under one. - web_base_url = repository["web_base_url"] - if web_base_url is None: - raise InvalidWebBaseUrl(detail=f"The {repository['provider_name']} repository has no web_base_url.") - parts = urlsplit(web_base_url) - if parts.scheme != "https" or not parts.netloc or parts.query or parts.fragment: - raise InvalidWebBaseUrl(detail=f"The repository's web_base_url {web_base_url!r} is not an https URL.") - return web_base_url.rstrip("/") diff --git a/src/scm/git/server.py b/src/scm/git/server.py index 39057979..205cd66c 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -2,25 +2,27 @@ from requests.structures import CaseInsensitiveDict -from scm.errors import ProviderNotFound, ResourceBadGateway, SCMCodedError +from scm.errors import GitProviderNotSupported, ProviderNotFound, ResourceBadGateway, SCMCodedError from scm.git.grant import verify_grant from scm.git.operations import check_request from scm.git.relay import GitRelay -from scm.git.remote import git_remote from scm.git.types import GRANT_HEADER, GetToken, Readable -from scm.helpers import resolve_repository +from scm.helpers import initialize_provider from scm.rpc.errors import SPECIAL_STATUS_MAP from scm.rpc.types import StreamResponse -from scm.types import Repository, RepositoryId +from scm.types import GetGitRemoteProtocol, Provider, Repository, RepositoryId class GitServer: """Serves a git endpoint: checks each request against its grant, then relays it with a provider token. The order of the checks matters. The grant is verified, and the request's route and repository checked - against it, before the repository is looked up or a token fetched, so a request the grant does not allow - never reaches the database, the token supplier or the provider. Give it a relay whose adapter refuses internal - addresses; see ``GitRelay``. + against it, before the repository and its provider are looked up or a token fetched, so a request the grant + does not allow never reaches the database, the token supplier or the provider. Give it a relay whose adapter + refuses internal addresses; see ``GitRelay``. + + The repository's provider says where its git endpoint is, through ``GetGitRemoteProtocol``. A repository whose + provider has no git remote, such as Cursor Origin's, is refused. Refusals and provider failures come back as plain-text responses. Any other exception, such as an adapter refusing a host or the token supplier failing, reaches the caller unchanged, so the caller can answer it the @@ -35,6 +37,7 @@ def __init__( self, secrets: list[str], fetch_repository: Callable[[int, RepositoryId], Repository | None], + fetch_provider: Callable[[int, Repository], Provider | None], get_token: GetToken, record_count: Callable[[str, int, dict[str, str]], None], relay: GitRelay, @@ -44,6 +47,7 @@ def __init__( raise ValueError("GitServer needs at least one secret, and none may be empty.") self.secrets = secrets self.fetch_repository = fetch_repository + self.fetch_provider = fetch_provider self.get_token = get_token self.record_count = record_count self.relay = relay @@ -67,10 +71,16 @@ def handle( try: grant = verify_grant(self.secrets, CaseInsensitiveDict(headers).get(GRANT_HEADER, "")) operation = check_request(grant, repository_id, method, route, query) - repository = resolve_repository( - grant.organization_id, repository_id, fetch_repository=self.fetch_repository + provider = initialize_provider( + grant.organization_id, + repository_id, + fetch_repository=self.fetch_repository, + fetch_provider=self.fetch_provider, ) - remote = git_remote(repository) + repository = provider.repository + if not isinstance(provider, GetGitRemoteProtocol): + raise GitProviderNotSupported(detail=f"The {repository['provider_name']} provider has no git remote.") + remote = provider.get_git_remote() token = self.get_token(repository, operation) if token is None: raise ProviderNotFound(detail="No git credentials are available for this repository.") diff --git a/src/scm/git/types.py b/src/scm/git/types.py index 3500cad1..9f998b76 100644 --- a/src/scm/git/types.py +++ b/src/scm/git/types.py @@ -1,4 +1,3 @@ -import dataclasses from collections.abc import Callable from typing import Literal, Protocol @@ -37,16 +36,6 @@ class Grant(msgspec.Struct, frozen=True, kw_only=True, forbid_unknown_fields=Tru """The format version. A verifier refuses a version it does not know.""" -@dataclasses.dataclass(frozen=True) -class GitRemote: - """Where a repository's git smart-HTTP endpoint lives and how a token authenticates against it.""" - - url: str - """The repository's clone URL, e.g. ``https://github.com/getsentry/sentry.git``.""" - username: str - """The basic-auth username the provider pairs with a token, e.g. ``x-access-token`` on GitHub.""" - - type GetToken = Callable[[Repository, GitOperation], str | None] """Return a provider token that can run ``operation`` on ``repository``, or None if there is none. diff --git a/src/scm/providers/github/provider.py b/src/scm/providers/github/provider.py index 64b79527..b133dec3 100644 --- a/src/scm/providers/github/provider.py +++ b/src/scm/providers/github/provider.py @@ -54,6 +54,7 @@ GitCommitObject, GitCommitTree, GitRef, + GitRemote, GitRepository, GitTree, InputTreeEntry, @@ -437,12 +438,14 @@ def __init__( client: ApiClient, organization_id: int, repository: Repository, - web_base_url: str = GITHUB_WEB_BASE_URL, + web_base_url: str | None = None, ) -> None: self.client = client self.organization_id = organization_id self.repository = repository - self._web_base_url = web_base_url + # A GitHub Enterprise repository carries its instance's URL. Falling back to it keeps a caller that builds + # the provider without one from pointing web links, and the git remote a token is sent to, at github.com. + self._web_base_url = web_base_url or repository["web_base_url"] or GITHUB_WEB_BASE_URL def request( self, @@ -1031,6 +1034,9 @@ def get_commits_url( def get_pull_request_url(self, pull_request_id: str) -> str: return f"{self._web_base_url}/{self.repository['name']}/pull/{pull_request_id}" + def get_git_remote(self) -> GitRemote: + return GitRemote(url=f"{self._web_base_url}/{self.repository['name']}.git", username="x-access-token") + def create_git_blob(self, content: str, encoding: str) -> ActionResult[GitBlob]: response = self.post( f"/repos/{self.repository['name']}/git/blobs", diff --git a/src/scm/providers/gitlab/provider.py b/src/scm/providers/gitlab/provider.py index 7f7e62c2..91bdf81a 100644 --- a/src/scm/providers/gitlab/provider.py +++ b/src/scm/providers/gitlab/provider.py @@ -50,6 +50,7 @@ GitCommitObject, GitCommitTree, GitRef, + GitRemote, GitRepository, GitTree, Issue, @@ -240,7 +241,9 @@ def __init__(self, client: ApiClient, organization_id: int, repository: Reposito raise MalformedExternalId() netloc, self.project_id = repository["external_id"].rsplit(":", maxsplit=1) - self.web_base_url = f"https://{netloc}" + # Sentry sends the instance's URL with the scheme and any path prefix it is served under. The external ID + # holds only the host, which is enough for an instance served at the root over https. + self.web_base_url = repository["web_base_url"] or f"https://{netloc}" @property def web_repository_path(self) -> str: @@ -744,6 +747,9 @@ def get_commits_url( def get_pull_request_url(self, pull_request_id: str) -> str: return f"{self.web_base_url}/{self.web_repository_path}/-/merge_requests/{pull_request_id}" + def get_git_remote(self) -> GitRemote: + return GitRemote(url=f"{self.web_base_url}/{self.web_repository_path}.git", username="oauth2") + def get_tree( self, tree_sha: SHA, diff --git a/src/scm/test_fixtures.py b/src/scm/test_fixtures.py index ad6c0bca..d0c92b5e 100644 --- a/src/scm/test_fixtures.py +++ b/src/scm/test_fixtures.py @@ -28,6 +28,7 @@ GitCommitObject, GitCommitTree, GitRef, + GitRemote, GitTree, InputTreeEntry, Issue, @@ -1190,6 +1191,9 @@ def get_commits_url( def get_pull_request_url(self, pull_request_id: str) -> str: return f"https://github.com/test-org/test-repo/pull/{pull_request_id}" + def get_git_remote(self) -> GitRemote: + return GitRemote(url="https://github.com/test-org/test-repo.git", username="x-access-token") + # Git blob operations def create_git_blob(self, content: str, encoding: str) -> ActionResult[GitBlob]: diff --git a/src/scm/types.py b/src/scm/types.py index 5a17f0b7..35c84f9a 100644 --- a/src/scm/types.py +++ b/src/scm/types.py @@ -65,6 +65,15 @@ class ArchiveLink(TypedDict): headers: dict[str, str] +class GitRemote(TypedDict): + """Where a repository's git smart-HTTP endpoint lives and how a token authenticates against it.""" + + url: str + """The repository's clone URL, e.g. ``https://github.com/getsentry/sentry.git``.""" + username: str + """The basic-auth username the provider pairs with a token, e.g. ``x-access-token`` on GitHub.""" + + type BuildStatus = Literal["pending", "running", "completed"] """The lifecycle stage of a CI build. @@ -1100,6 +1109,11 @@ class GetPullRequestUrlProtocol(Protocol): def get_pull_request_url(self, pull_request_id: str) -> str: ... +@runtime_checkable +class GetGitRemoteProtocol(Protocol): + def get_git_remote(self) -> GitRemote: ... + + # Commit Protocols @@ -1675,6 +1689,7 @@ def update_and_collapse_pull_request_comment( GetFileUrlProtocol, GetGitCommitProtocol, GetGitRefProtocol, + GetGitRemoteProtocol, GetIssueCommentReactionsProtocol, GetIssueCommentsProtocol, GetIssueProtocol, diff --git a/tests/unit/git_builders.py b/tests/unit/git_builders.py index 44876115..57f58c8c 100644 --- a/tests/unit/git_builders.py +++ b/tests/unit/git_builders.py @@ -1,7 +1,12 @@ """Builders shared by the git module's tests.""" +from unittest.mock import MagicMock + from scm.git.types import Grant -from scm.types import Repository +from scm.providers.cursor_origin.provider import CursorOriginProvider +from scm.providers.github.provider import GitHubProvider +from scm.providers.gitlab.provider import GitLabProvider +from scm.types import ApiClient, Provider, Repository # Unix seconds long after any test runs, for grants a test doesn't expect to expire. FAR_FUTURE = 2**40 @@ -22,6 +27,20 @@ def make_repository(**overrides) -> Repository: return {**defaults, **overrides} # type: ignore[typeddict-item] +def make_provider(organization_id: int, repository: Repository) -> Provider | None: + """Build the repository's provider the way Sentry's ``fetch_provider`` does, with a client nothing calls.""" + client = MagicMock(spec=ApiClient) + match repository["provider_name"]: + case "github" | "github_enterprise": + return GitHubProvider(client, organization_id, repository) + case "gitlab": + return GitLabProvider(client, organization_id, repository) + case "cursor_origin": + return CursorOriginProvider(client, organization_id, repository, installation_id="1") + case _: + return None + + def make_grant(**overrides) -> Grant: """A fetch grant for the default repository, named both ways a grant can name it.""" fields = { diff --git a/tests/unit/provider/test_github.py b/tests/unit/provider/test_github.py index 299797de..1fb61c79 100644 --- a/tests/unit/provider/test_github.py +++ b/tests/unit/provider/test_github.py @@ -74,6 +74,7 @@ CredentialsSet, DeleteCommitAction, DiffLine, + GitRemote, MoveCommitAction, Repository, WriteCommitAction, @@ -2081,6 +2082,40 @@ def test_get_pull_request_url_builds_pr_url() -> None: assert provider.get_pull_request_url("42") == "https://github.com/test-org/test-repo/pull/42" +def test_get_git_remote_builds_clone_url() -> None: + provider = GitHubProvider(MagicMock(spec=ApiClient), organization_id=1, repository=make_repository()) + + assert provider.get_git_remote() == GitRemote( + url="https://github.com/test-org/test-repo.git", username="x-access-token" + ) + + +def test_ghe_web_base_url_used_in_git_remote() -> None: + provider = GitHubProvider( + MagicMock(spec=ApiClient), + organization_id=1, + repository=make_repository(), + web_base_url="https://github.example.com", + ) + + assert provider.get_git_remote() == GitRemote( + url="https://github.example.com/test-org/test-repo.git", username="x-access-token" + ) + + +def test_ghe_web_base_url_defaults_to_the_repositorys() -> None: + # Built without a web_base_url, a GitHub Enterprise provider would otherwise send its token to github.com. + repository = { + **make_repository(), + "provider_name": "github_enterprise", + "web_base_url": "https://github.example.com", + } + provider = GitHubProvider(MagicMock(spec=ApiClient), organization_id=1, repository=repository) # type: ignore[arg-type] + + assert provider.get_git_remote()["url"] == "https://github.example.com/test-org/test-repo.git" + assert provider.get_pull_request_url("42") == "https://github.example.com/test-org/test-repo/pull/42" + + def test_ghe_web_base_url_used_in_url_methods() -> None: provider = GitHubProvider( MagicMock(spec=ApiClient), @@ -3658,6 +3693,7 @@ def test_public_methods_are_accounted_for() -> None: "get_commit_url", "get_commits_url", "get_pull_request_url", + "get_git_remote", "create_commit", "update_issue", "get_thread_id_from_review_comment_unique_id", diff --git a/tests/unit/provider/test_gitlab.py b/tests/unit/provider/test_gitlab.py index 35c560cd..d4ad1f89 100644 --- a/tests/unit/provider/test_gitlab.py +++ b/tests/unit/provider/test_gitlab.py @@ -37,6 +37,7 @@ ChmodCommitAction, DeleteCommitAction, DiffLine, + GitRemote, MoveCommitAction, Repository, WriteCommitAction, @@ -13969,6 +13970,42 @@ def test_web_repository_path_used_in_commit_and_mr_urls(client: ApiClient) -> No ) +def test_get_git_remote_is_the_repository_path_on_its_instance(client: ApiClient) -> None: + provider = _make_gitlab_provider(client, name="GitLab.org / GitLab FOSS", path="gitlab-org/gitlab-foss") + + assert provider.get_git_remote() == GitRemote( + url="https://gitlab.com/gitlab-org/gitlab-foss.git", username="oauth2" + ) + + +def test_get_git_remote_on_a_self_managed_instance(client: ApiClient) -> None: + provider = GitLabProvider( + client=client, + organization_id=1, + repository={**_make_gitlab_provider(client).repository, "external_id": "gitlab.example.com:8443:7"}, + ) + + assert provider.get_git_remote() == GitRemote( + url="https://gitlab.example.com:8443/test-repo.git", username="oauth2" + ) + + +def test_get_git_remote_keeps_the_instances_path_prefix(client: ApiClient) -> None: + repository = {**_make_gitlab_provider(client).repository, "web_base_url": "https://example.com/gitlab"} + provider = GitLabProvider(client=client, organization_id=1, repository=repository) # type: ignore[arg-type] + + assert provider.get_git_remote()["url"] == "https://example.com/gitlab/test-repo.git" + assert provider.get_commit_url("abc123") == "https://example.com/gitlab/test-repo/-/commit/abc123" + + +@pytest.mark.parametrize("path", [None, ""]) +def test_get_git_remote_needs_the_repository_path(client: ApiClient, path: str | None) -> None: + provider = _make_gitlab_provider(client, name="GitLab.org / GitLab FOSS", path=path) + + with pytest.raises(MissingRepositoryPath): + provider.get_git_remote() + + def test_get_file_url_builds_blob_url(provider: GitLabProvider): assert provider.get_file_url("src/main.py", "abc123") == ("https://gitlab.com/test-repo/-/blob/abc123/src/main.py") assert provider.get_file_url("src/main.py", "abc123", start_line=10) == ( diff --git a/tests/unit/test_actions.py b/tests/unit/test_actions.py index 19706e52..e3687288 100644 --- a/tests/unit/test_actions.py +++ b/tests/unit/test_actions.py @@ -53,6 +53,7 @@ get_full_tree, get_git_commit, get_git_ref, + get_git_remote, get_issue, get_issue_comment_reactions, get_issue_comments, @@ -194,6 +195,7 @@ def fetch_repository(oid, rid) -> Repository: (get_commit_url, {"commit_sha": "abc123"}), (get_commits_url, {"commit_sha": "abc123"}), (get_pull_request_url, {"pull_request_id": "1"}), + (get_git_remote, {}), # Git blob operations (create_git_blob, {"content": "hello", "encoding": "utf-8"}), # File content operations @@ -475,6 +477,10 @@ def _check_get_pull_request_url(result: Any) -> None: assert result == "https://github.com/test-org/test-repo/pull/1" +def _check_get_git_remote(result: Any) -> None: + assert result == {"url": "https://github.com/test-org/test-repo.git", "username": "x-access-token"} + + def _check_create_git_blob(result: Any) -> None: assert result["data"]["sha"] == "blob123abc" assert result["type"] == "github" @@ -831,6 +837,7 @@ def _check_download_archive(result: Any) -> None: (get_commit_url, {"commit_sha": "abc123"}, _check_get_commit_url), (get_commits_url, {"commit_sha": "abc123"}, _check_get_commits_url), (get_pull_request_url, {"pull_request_id": "1"}, _check_get_pull_request_url), + (get_git_remote, {}, _check_get_git_remote), ( create_git_blob, {"content": "hello", "encoding": "utf-8"}, diff --git a/tests/unit/test_git_relay.py b/tests/unit/test_git_relay.py index 363f7679..211ddeed 100644 --- a/tests/unit/test_git_relay.py +++ b/tests/unit/test_git_relay.py @@ -9,6 +9,7 @@ from requests.adapters import HTTPAdapter from scm.errors import ( + GitRemoteNotHttps, GitRouteNotAllowed, ResourceCertificateInvalid, ResourceGatewayTimeout, @@ -16,7 +17,7 @@ SCMCodedError, ) from scm.git.relay import GitRelay -from scm.git.types import GitRemote +from scm.types import GitRemote from tests.unit.git_upstream import UpstreamHandler, basic_auth, ca_bundle, running_upstream TOKEN = "ghs_secrettoken" @@ -130,7 +131,7 @@ def test_asks_for_an_unencoded_response_unless_the_client_says_otherwise(self, r def test_refuses_plain_http(self, relay, upstream): remote = GitRemote(url=upstream.url.replace("https://", "http://") + "/repo.git", username="x-access-token") - with pytest.raises(ValueError, match="not https"): + with pytest.raises(GitRemoteNotHttps, match="not https"): fetch_refs(relay, remote=remote) assert upstream.requests == [] diff --git a/tests/unit/test_git_remote.py b/tests/unit/test_git_remote.py deleted file mode 100644 index 1714107d..00000000 --- a/tests/unit/test_git_remote.py +++ /dev/null @@ -1,79 +0,0 @@ -import pytest - -from scm.errors import GitProviderNotSupported, InvalidWebBaseUrl, MissingRepositoryPath -from scm.git.remote import git_remote -from scm.git.types import GitRemote -from tests.unit.git_builders import make_repository - - -def test_github(): - assert git_remote(make_repository()) == GitRemote( - url="https://github.com/getsentry/sentry.git", username="x-access-token" - ) - - -@pytest.mark.parametrize("web_base_url", ["https://ghe.example.com", "https://ghe.example.com/"]) -def test_github_enterprise(web_base_url): - repository = make_repository(provider_name="github_enterprise", web_base_url=web_base_url) - assert git_remote(repository) == GitRemote( - url="https://ghe.example.com/getsentry/sentry.git", username="x-access-token" - ) - - -@pytest.mark.parametrize("provider_name", ["github_enterprise", "gitlab"]) -def test_self_hosted_providers_need_a_web_base_url(provider_name): - with pytest.raises(InvalidWebBaseUrl, match=f"{provider_name} repository has no web_base_url"): - git_remote(make_repository(provider_name=provider_name, path="getsentry/sentry")) - - -@pytest.mark.parametrize( - "web_base_url", - ["http://ghe.example.com", "ghe.example.com", "https://", "https://ghe.example.com?x=1", ""], -) -@pytest.mark.parametrize("provider_name", ["github_enterprise", "gitlab"]) -def test_web_base_url_must_be_https(provider_name, web_base_url): - repository = make_repository(provider_name=provider_name, path="getsentry/sentry", web_base_url=web_base_url) - with pytest.raises(InvalidWebBaseUrl, match="not an https URL"): - git_remote(repository) - - -@pytest.mark.parametrize( - "web_base_url,url", - [ - ("https://gitlab.com", "https://gitlab.com/group/sub/project.git"), - ("https://git.example.com:8443/", "https://git.example.com:8443/group/sub/project.git"), - ("https://example.com/gitlab", "https://example.com/gitlab/group/sub/project.git"), - ], -) -def test_gitlab(web_base_url, url): - repository = make_repository( - provider_name="gitlab", - external_id="gitlab.com:42", - name="Group / Sub / Project", - path="group/sub/project", - web_base_url=web_base_url, - ) - assert git_remote(repository) == GitRemote(url=url, username="oauth2") - - -@pytest.mark.parametrize("path", [None, ""]) -def test_gitlab_needs_the_repository_path(path): - # The name is GitLab's display name, e.g. "GitLab.org / GitLab FOSS" for gitlab-org/gitlab-foss, so it can't - # stand in for the path. - repository = make_repository( - provider_name="gitlab", name="GitLab.org / GitLab FOSS", path=path, web_base_url="https://gitlab.com" - ) - with pytest.raises(MissingRepositoryPath): - git_remote(repository) - - -def test_gitlab_without_a_path_from_an_older_sentry(): - repository = make_repository(provider_name="gitlab", web_base_url="https://gitlab.com") - with pytest.raises(MissingRepositoryPath): - git_remote(repository) - - -@pytest.mark.parametrize("provider_name", ["bitbucket", "cursor_origin"]) -def test_unsupported_providers(provider_name): - with pytest.raises(GitProviderNotSupported, match=provider_name): - git_remote(make_repository(provider_name=provider_name)) diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index deba41bf..db620aa2 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -8,10 +8,10 @@ from scm.git.grant import sign_grant from scm.git.relay import GitRelay from scm.git.server import GitServer -from scm.git.types import GRANT_HEADER, GitOperation, GitRemote, Readable +from scm.git.types import GRANT_HEADER, GitOperation, Readable from scm.rpc.types import StreamResponse -from scm.types import Repository, RepositoryId -from tests.unit.git_builders import make_grant, make_repository +from scm.types import GitRemote, Provider, Repository, RepositoryId +from tests.unit.git_builders import make_grant, make_provider, make_repository from tests.unit.git_upstream import basic_auth, running_upstream SECRET = "test-secret-0000000000000000000000" @@ -27,6 +27,7 @@ def __init__(self, repository: Repository | None = None, token: str | None = TOK self.repository = make_repository() if repository is None else repository self.token = token self.repository_lookups: list[tuple[int, RepositoryId]] = [] + self.provider_lookups: list[tuple[int, Repository]] = [] self.token_lookups: list[tuple[Repository, GitOperation]] = [] self.counts: list[tuple[str, dict[str, str]]] = [] self.bytes: dict[tuple[str, str], int] = {} @@ -35,6 +36,10 @@ def fetch_repository(self, organization_id: int, repository_id: RepositoryId) -> self.repository_lookups.append((organization_id, repository_id)) return self.repository + def fetch_provider(self, organization_id: int, repository: Repository) -> Provider | None: + self.provider_lookups.append((organization_id, repository)) + return make_provider(organization_id, repository) + def get_token(self, repository: Repository, operation: GitOperation) -> str | None: self.token_lookups.append((repository, operation)) return self.token @@ -60,6 +65,7 @@ def make_server(recorder: Recorder, relay: GitRelay | None = None) -> GitServer: return GitServer( secrets=[SECRET], fetch_repository=recorder.fetch_repository, + fetch_provider=recorder.fetch_provider, get_token=recorder.get_token, record_count=recorder.record_count, relay=relay, @@ -107,6 +113,7 @@ def test_requires_a_relay(): GitServer( # type: ignore[call-arg] secrets=[SECRET], fetch_repository=lambda organization_id, repository_id: None, + fetch_provider=lambda organization_id, repository: None, get_token=lambda repository, operation: None, record_count=lambda name, value, tags: None, ) @@ -118,6 +125,7 @@ def test_requires_non_empty_secrets(secrets): GitServer( secrets=secrets, fetch_repository=lambda organization_id, repository_id: None, + fetch_provider=lambda organization_id, repository: None, get_token=lambda repository, operation: None, record_count=lambda name, value, tags: None, relay=MagicMock(spec=GitRelay), @@ -141,6 +149,7 @@ def test_invalid_grant(self, grant, detail): server = make_server(recorder) assert_refused(request(server, grant), 403, detail) assert recorder.repository_lookups == [] + assert recorder.provider_lookups == [] assert recorder.token_lookups == [] server.relay.forward.assert_not_called() # type: ignore[attr-defined] assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_invalid_grant"})] @@ -166,6 +175,15 @@ def test_looked_up_under_the_grants_organization(self): recorder = Recorder() request(make_server(recorder), signed_grant(organization_id=1)) assert recorder.repository_lookups == [(1, 5)] + assert recorder.provider_lookups == [(1, make_repository())] + + def test_remote_comes_from_the_provider(self): + repository = make_repository(provider_name="gitlab", external_id="gitlab.example.com:7", path="group/project") + recorder = Recorder(repository=repository) + server = make_server(recorder) + request(server, signed_grant()) + remote = server.relay.forward.call_args.args[0] # type: ignore[attr-defined] + assert remote == GitRemote(url="https://gitlab.example.com/group/project.git", username="oauth2") @pytest.mark.parametrize( "repository,status,detail", @@ -173,7 +191,9 @@ def test_looked_up_under_the_grants_organization(self): (None, 404, "could not be found"), (make_repository(organization_id=2), 404, "did not belong"), (make_repository(is_active=False), 404, "inactive"), - (make_repository(provider_name="bitbucket"), 400, "no git remote"), + (make_repository(provider_name="bitbucket"), 404, "unsupported integration provider"), + (make_repository(provider_name="cursor_origin"), 400, "cursor_origin provider has no git remote"), + (make_repository(provider_name="gitlab", external_id="gitlab.com:7"), 400, "has no path"), ], ) def test_refused_before_the_token(self, repository, status, detail): @@ -260,6 +280,15 @@ def test_upstream_unavailable(self): assert_refused(response, 503, "Could not connect") assert TOKEN not in b"".join(response.content).decode() + def test_plain_http_remote(self): + # A self-managed instance served over http; the token must not cross the network in the clear. + recorder = Recorder( + repository=make_repository(provider_name="github_enterprise", web_base_url="http://127.0.0.1:1") + ) + response = request(make_server(recorder, GitRelay()), signed_grant()) + assert_refused(response, 400, "not https") + assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_remote_not_https"})] + def test_provider_rejecting_the_token_is_a_gateway_error(self): # Passed on, a 401 would make git ask the sandboxed client for credentials. closed = [] diff --git a/tests/unit/test_rpc_integration.py b/tests/unit/test_rpc_integration.py index ec266701..f2caa522 100644 --- a/tests/unit/test_rpc_integration.py +++ b/tests/unit/test_rpc_integration.py @@ -965,6 +965,11 @@ def make_client_scm(organization_id, repository_id, server): lambda scm: actions.get_pull_request_url(scm, "42"), "https://github.com/org/repo/pull/42", ), + ( + "get_git_remote", + lambda scm: actions.get_git_remote(scm), + {"url": "https://github.com/org/repo.git", "username": "x-access-token"}, + ), ] From 2662dfffcafc5526536abe517d1c837083192557 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos <dlugos.ivan@gmail.com> Date: Tue, 20 Oct 2026 20:53:10 +0200 Subject: [PATCH 20/31] feat(cursor-origin): Give Origin repositories a git remote Origin's docs (cursor.com/docs/api/origin, "Git HTTPS authentication") describe the same basic-auth scheme as GitHub: an installation token as the password with x-access-token as the username, on origin.cursor.com rather than the web UI's host. Its git endpoint refuses a bearer header. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/scm/providers/cursor_origin/provider.py | 7 +++++++ tests/unit/provider/test_cursor_origin.py | 7 +++++++ tests/unit/test_git_server.py | 10 +++++++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/scm/providers/cursor_origin/provider.py b/src/scm/providers/cursor_origin/provider.py index 6606f2c1..f0299dc5 100644 --- a/src/scm/providers/cursor_origin/provider.py +++ b/src/scm/providers/cursor_origin/provider.py @@ -48,6 +48,7 @@ GitCommitObject, GitCommitTree, GitRef, + GitRemote, GitRepository, GitTree, MoveCommitAction, @@ -79,6 +80,8 @@ PULL_REQUEST_TEMPLATE_FILENAME = "pull_request_template.md" PULL_REQUEST_TEMPLATE_DIRNAME = "pull_request_template" CURSOR_ORIGIN_WEB_BASE_URL = "https://cursor.com/codebase" +# Git over HTTPS is served from its own host, not the web UI's. +CURSOR_ORIGIN_GIT_BASE_URL = "https://origin.cursor.com" # Origin has no no-reply address for an app. CURSOR_ORIGIN_APP_COMMIT_EMAIL = "noreply@sentry.io" PAGE_TOKEN_PARAM = "pageToken" @@ -354,6 +357,10 @@ def get_commits_url( def get_pull_request_url(self, pull_request_id: str) -> str: return f"{self._web_base_url}/{self.repository_path}/pull/{pull_request_id}" + def get_git_remote(self) -> GitRemote: + # Origin's git endpoint takes an installation token as the basic-auth password and refuses a bearer header. + return GitRemote(url=f"{CURSOR_ORIGIN_GIT_BASE_URL}/{self.repository_path}.git", username="x-access-token") + def get_file_content( self, path: str, diff --git a/tests/unit/provider/test_cursor_origin.py b/tests/unit/provider/test_cursor_origin.py index a317a74a..d4be281e 100644 --- a/tests/unit/provider/test_cursor_origin.py +++ b/tests/unit/provider/test_cursor_origin.py @@ -34,6 +34,7 @@ ChmodCommitAction, CommitAuthorParam, DeleteCommitAction, + GitRemote, MoveCommitAction, Repository, WriteCommitAction, @@ -1083,6 +1084,12 @@ def test_a_pull_request_url(self, provider: CursorOriginProvider) -> None: def test_a_commit_url(self, provider: CursorOriginProvider) -> None: assert provider.get_commit_url("abc123") == f"https://cursor.com/codebase/{REPO}/commit/abc123" + def test_a_git_remote(self, provider: CursorOriginProvider) -> None: + # Git is served from origin.cursor.com rather than the web UI's cursor.com/codebase. + assert provider.get_git_remote() == GitRemote( + url=f"https://origin.cursor.com/{REPO}.git", username="x-access-token" + ) + def _commit_raw(sha: str, date: str = "2026-08-01T09:30:00Z") -> dict[str, Any]: identity = {"name": "Jane Doe", "email": "jane@example.com", "date": date} diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index db620aa2..af5b5a1d 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -1,6 +1,7 @@ import io import time from collections.abc import Iterator +from types import SimpleNamespace from unittest.mock import MagicMock import pytest @@ -177,6 +178,14 @@ def test_looked_up_under_the_grants_organization(self): assert recorder.repository_lookups == [(1, 5)] assert recorder.provider_lookups == [(1, make_repository())] + def test_provider_without_a_git_remote(self): + recorder = Recorder() + recorder.fetch_provider = lambda organization_id, repository: SimpleNamespace( # type: ignore[method-assign] + organization_id=organization_id, repository=repository + ) + assert_refused(request(make_server(recorder), signed_grant()), 400, "github provider has no git remote") + assert recorder.token_lookups == [] + def test_remote_comes_from_the_provider(self): repository = make_repository(provider_name="gitlab", external_id="gitlab.example.com:7", path="group/project") recorder = Recorder(repository=repository) @@ -192,7 +201,6 @@ def test_remote_comes_from_the_provider(self): (make_repository(organization_id=2), 404, "did not belong"), (make_repository(is_active=False), 404, "inactive"), (make_repository(provider_name="bitbucket"), 404, "unsupported integration provider"), - (make_repository(provider_name="cursor_origin"), 400, "cursor_origin provider has no git remote"), (make_repository(provider_name="gitlab", external_id="gitlab.com:7"), 400, "has no path"), ], ) From 023be64dda9e7a18c9bda421d011eb3fedce98f7 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos <dlugos.ivan@gmail.com> Date: Tue, 20 Oct 2026 21:01:55 +0200 Subject: [PATCH 21/31] ref(git): Drop the resolve_repository extraction It was pulled out of initialize_provider so GitServer could check a repository without building its provider. GitServer now builds the provider to ask it for the git remote, through initialize_provider, so nothing else calls it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/scm/helpers.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/scm/helpers.py b/src/scm/helpers.py index fb24811a..67c3d7db 100644 --- a/src/scm/helpers.py +++ b/src/scm/helpers.py @@ -35,13 +35,13 @@ def iter_all_pages[T]( cursor = next_cursor -def resolve_repository( +def initialize_provider( organization_id: int, repository_id: RepositoryId, *, fetch_repository: Callable[[int, RepositoryId], Repository | None], -) -> Repository: - """Return the repository only if it exists, is active and belongs to the organization.""" + fetch_provider: Callable[[int, Repository], Provider | None], +) -> Provider: repository = fetch_repository(organization_id, repository_id) if not repository: raise RepositoryNotFound(organization_id=organization_id, repository_id=repository_id) @@ -49,17 +49,6 @@ def resolve_repository( raise RepositoryInactive(organization_id=organization_id, repository_id=repository_id) if repository["organization_id"] != organization_id: raise RepositoryOrganizationMismatch(organization_id=organization_id, repository_id=repository_id) - return repository - - -def initialize_provider( - organization_id: int, - repository_id: RepositoryId, - *, - fetch_repository: Callable[[int, RepositoryId], Repository | None], - fetch_provider: Callable[[int, Repository], Provider | None], -) -> Provider: - repository = resolve_repository(organization_id, repository_id, fetch_repository=fetch_repository) provider = fetch_provider(organization_id, repository) if provider is None: From 1b024abafcaee0f88f1d7b00aa915debb89ac952 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos <dlugos.ivan@gmail.com> Date: Tue, 20 Oct 2026 23:14:18 +0200 Subject: [PATCH 22/31] fix(git): Close unread provider responses and keep redirect bodies streaming Review on #156 found that the relay's response generator never ran its cleanup when GitServer closed it unread, as it does with a 401, so the provider connection stayed open. And with redirects off, requests still read a redirect's whole body into memory to prepare Response.next, so a provider could make the relay buffer as much as it liked. Also from review: - GitServer keeps its own status map instead of adding git codes to the RPC one, with an explicit status for every code it answers; unknown codes are 500s. - A missing token is git_token_unavailable (502), not provider_not_found. - Any requests exception is replaced, not just the three expected ones, so none can carry the token. - verify_grant refuses a grant issued more than a minute in the future. - Failed requests that reached the provider keep their operation and provider tags. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/scm/errors.py | 6 ++++ src/scm/git/grant.py | 13 ++++++-- src/scm/git/relay.py | 60 ++++++++++++++++++++++++++++----- src/scm/git/server.py | 42 ++++++++++++++++++----- src/scm/rpc/errors.py | 3 -- tests/unit/git_upstream.py | 13 ++++++++ tests/unit/test_git_grant.py | 37 ++++++++++++++------ tests/unit/test_git_relay.py | 47 ++++++++++++++++++-------- tests/unit/test_git_server.py | 63 ++++++++++++++++++++++++----------- 9 files changed, 219 insertions(+), 65 deletions(-) diff --git a/src/scm/errors.py b/src/scm/errors.py index 1a2a88ec..6823a58c 100644 --- a/src/scm/errors.py +++ b/src/scm/errors.py @@ -5,6 +5,7 @@ "git_provider_not_supported", "git_remote_not_https", "git_route_not_allowed", + "git_token_unavailable", "malformed_external_id", "missing_repository_path", "path_is_directory", @@ -47,6 +48,7 @@ "git_provider_not_supported": "The repository's provider has no git remote.", "git_remote_not_https": "The repository's git remote is not an https URL.", "git_route_not_allowed": "The git grant does not allow this request.", + "git_token_unavailable": "No provider token could be obtained for the repository.", "malformed_external_id": "The repository's external ID was malformed.", "missing_repository_path": "The repository's path on its provider is unknown.", "path_is_directory": "The requested path is a directory, not a file.", @@ -151,6 +153,10 @@ class GitRouteNotAllowed(SCMCodedError): code = "git_route_not_allowed" +class GitTokenUnavailable(SCMCodedError): + code = "git_token_unavailable" + + class MalformedExternalId(SCMCodedError): code = "malformed_external_id" diff --git a/src/scm/git/grant.py b/src/scm/git/grant.py index 06d07283..dd57efd5 100644 --- a/src/scm/git/grant.py +++ b/src/scm/git/grant.py @@ -21,6 +21,9 @@ _ALGORITHM = "HS256" # Claims the JWT layer owns. Every other claim must be a Grant field. _JWT_CLAIMS = frozenset({"aud", "exp", "iat"}) +# How far in the future a grant's iat may be. Seer signs grants on other hosts than the one verifying them, and a +# clock running a little ahead should not get its grants refused. +_CLOCK_SKEW = 60 def sign_grant(secret: str, grant: Grant, *, now: float | None = None) -> str: @@ -39,6 +42,7 @@ def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> switch to it and the old one removed after their grants expire. """ _check_secrets(secrets) + now = time.time() if now is None else now if not token: raise GitInvalidGrant(detail="The git grant is missing.") try: @@ -55,7 +59,7 @@ def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> secret, algorithms=[_ALGORITHM], audience=GRANT_AUDIENCE, - # Expiry is checked below against an injectable clock, so tests can pin it. + # exp and iat are checked below against an injectable clock, so tests can pin it. options={"require": sorted(_JWT_CLAIMS), "verify_exp": False, "verify_iat": False}, ) except jwt.InvalidSignatureError: @@ -74,8 +78,13 @@ def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> # gives nothing away. raise GitInvalidGrant(detail=f"The git grant's payload is invalid: {e}") from None - if grant.expires_at <= (time.time() if now is None else now): + if grant.expires_at <= now: raise GitInvalidGrant(detail="The git grant has expired.") + # bool is an int to isinstance, and PyJWT would refuse it as an iat. + if not isinstance(claims["iat"], int) or isinstance(claims["iat"], bool): + raise GitInvalidGrant(detail="The git grant's iat is not an integer.") + if claims["iat"] > now + _CLOCK_SKEW: + raise GitInvalidGrant(detail="The git grant was issued in the future.") if not grant.repositories: raise GitInvalidGrant(detail="The git grant names no repositories.") return grant diff --git a/src/scm/git/relay.py b/src/scm/git/relay.py index 79896b89..bb99dddf 100644 --- a/src/scm/git/relay.py +++ b/src/scm/git/relay.py @@ -9,6 +9,7 @@ from scm.errors import ( ClientClosedRequest, GitRemoteNotHttps, + ResourceBadGateway, ResourceCertificateInvalid, ResourceGatewayTimeout, ResourceServiceUnavailable, @@ -59,8 +60,9 @@ class GitRelay: ``adapter`` builds the adapter that sends each upstream request. Pass one that refuses internal addresses: a GitHub Enterprise or self-managed GitLab host is whatever the customer configured, and the relay connects wherever it resolves. Sentry's ``BlacklistAdapter`` does this for its API clients, and the class itself can be - passed. Errors the adapter raises for a refused host reach the caller unchanged. The adapter must not retry, - because a retry cannot resend a body that has already been streamed. Without one, the relay uses a plain + passed. An error the adapter raises for a refused host reaches the caller unchanged, unless it is a requests + exception, which the relay replaces like any other so that it cannot carry the token. The adapter must not + retry, because a retry cannot resend a body that has already been streamed. Without one, the relay uses a plain ``HTTPAdapter``, which checks nothing. The relay takes no settings from the environment. Through a proxy from ``HTTPS_PROXY``, the adapter would @@ -139,6 +141,9 @@ def forward( raise ResourceCertificateInvalid(detail=f"The TLS certificate of {host} could not be verified.") from None except requests.ConnectionError: raise ResourceServiceUnavailable(detail=f"Could not connect to {remote['url']}.") from None + except requests.RequestException: + # Any other failure of the request, e.g. a malformed response. Like those above, it holds the request. + raise ResourceBadGateway(detail=f"The request to {remote['url']} failed.") from None finally: # Once a response arrives, _stream closes the session when the response ends. if upstream is None: @@ -147,22 +152,59 @@ def forward( return StreamResponse( status_code=upstream.status_code, headers={name: value for name, value in upstream.headers.items() if name.lower() in _RESPONSE_HEADERS}, - content=self._stream(session, upstream), + content=_UpstreamStream(session, upstream, self.chunk_size), ) def _session(self) -> requests.Session: - session = requests.Session() + session = _NoRedirectSession() # Without this, requests reads proxies, CA files and .netrc credentials from the environment. session.trust_env = False session.verify = self.ca_bundle or True session.mount("https://", self.adapter()) return session - def _stream(self, session: requests.Session, upstream: requests.Response) -> Iterator[bytes]: - # Closing the response first drops its connection if the client went away mid-response; closing the - # session then closes a connection the finished response returned to it. - with session, upstream: - yield from filter(bool, upstream.raw.stream(self.chunk_size, decode_content=False)) + +class _NoRedirectSession(requests.Session): + """A session that never looks for a redirect. + + Even with redirects off, requests prepares the next request of a redirect response for ``Response.next``, and + reads the whole response body into memory first, however large the provider makes it. + """ + + def get_redirect_target(self, resp: requests.Response) -> None: + return None + + +class _UpstreamStream: + """The provider's response body, which closes the response and its session when it ends or is closed. + + A generator holding them in a ``with`` would not do: closing a generator that never started does not run its + body, so a response nobody read, such as a 401 that GitServer replaces, would keep its connection open. + """ + + def __init__(self, session: requests.Session, upstream: requests.Response, chunk_size: int) -> None: + self.session = session + self.upstream = upstream + self.chunks = upstream.raw.stream(chunk_size, decode_content=False) + + def __iter__(self) -> "_UpstreamStream": + return self + + def __next__(self) -> bytes: + try: + while not (chunk := next(self.chunks)): + pass + except BaseException: + # The end of the body, or the provider's stream breaking. + self.close() + raise + return chunk + + def close(self) -> None: + # Closing the response first drops its connection if the body was not read to the end; closing the session + # then closes a connection the finished response returned to it. + self.upstream.close() + self.session.close() def _content_length(headers: CaseInsensitiveDict[str]) -> int | None: diff --git a/src/scm/git/server.py b/src/scm/git/server.py index 205cd66c..1b91e4d5 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -2,16 +2,42 @@ from requests.structures import CaseInsensitiveDict -from scm.errors import GitProviderNotSupported, ProviderNotFound, ResourceBadGateway, SCMCodedError +from scm.errors import ErrorCode, GitProviderNotSupported, GitTokenUnavailable, ResourceBadGateway, SCMCodedError from scm.git.grant import verify_grant from scm.git.operations import check_request from scm.git.relay import GitRelay from scm.git.types import GRANT_HEADER, GetToken, Readable from scm.helpers import initialize_provider -from scm.rpc.errors import SPECIAL_STATUS_MAP from scm.rpc.types import StreamResponse from scm.types import GetGitRemoteProtocol, Provider, Repository, RepositoryId +# The status of each refusal GitServer answers itself. None is a 401: on a 401 git asks the user for credentials, +# which a sandboxed client must never have and which would not help. +_STATUS: dict[ErrorCode, int] = { + # The client's own request is malformed. + "rpc_malformed_request_headers": 400, + # The grant does not allow the request. + "git_invalid_grant": 403, + "git_route_not_allowed": 403, + # The grant allows it, but there is no repository to serve. + "repository_not_found": 404, + "repository_inactive": 404, + "repository_organization_mismatch": 404, + "provider_not_found": 404, + "git_provider_not_supported": 404, + "client_closed_request": 499, + # Sentry's record of the repository is incomplete. + "malformed_external_id": 500, + "missing_repository_path": 500, + # The provider cannot be reached safely, or failed. + "git_remote_not_https": 502, + "git_token_unavailable": 502, + "resource_bad_gateway": 502, + "resource_certificate_invalid": 502, + "resource_service_unavailable": 503, + "resource_gateway_timeout": 504, +} + class GitServer: """Serves a git endpoint: checks each request against its grant, then relays it with a provider token. @@ -83,7 +109,9 @@ def handle( remote = provider.get_git_remote() token = self.get_token(repository, operation) if token is None: - raise ProviderNotFound(detail="No git credentials are available for this repository.") + raise GitTokenUnavailable( + detail=f"Sentry could not get a {repository['provider_name']} token for this repository." + ) response = self.relay.forward( remote, token, method=method, route=route, query=query, headers=headers, body=counted_body ) @@ -105,7 +133,7 @@ def finish(failure: str | None, response_bytes: int) -> None: if failure is None: self.record_count("sentry.scm.git.request.succeeded", 1, tags | {"status": str(response.status_code)}) else: - self.record_count("sentry.scm.git.request.failed", 1, {"code": failure}) + self.record_count("sentry.scm.git.request.failed", 1, tags | {"code": failure}) if counted_body is not None and counted_body.bytes_read: self.record_count("sentry.scm.git.request_bytes", counted_body.bytes_read, tags) if response_bytes: @@ -174,10 +202,8 @@ def _close_stream(content: Iterator[bytes]) -> None: def _error_response(error: SCMCodedError) -> StreamResponse: - # On a 401 git asks the user for credentials, which a sandboxed client must never have, so refuse with a 403. - status = SPECIAL_STATUS_MAP.get(error.code, 400) - if status == 401: - status = 403 + # A code missing from the map comes from a failure nobody planned for, which is the server's fault. + status = _STATUS.get(error.code, 500) # When its first request fails, git shows a text/plain body to the user as "remote: <line>". content = f"{error.detail}\n".encode() return StreamResponse( diff --git a/src/scm/rpc/errors.py b/src/scm/rpc/errors.py index e44fd779..10b7a6df 100644 --- a/src/scm/rpc/errors.py +++ b/src/scm/rpc/errors.py @@ -4,8 +4,6 @@ from scm.rpc.types import Error, ErrorResponse SPECIAL_STATUS_MAP: dict[ErrorCode, int] = { - "git_invalid_grant": 403, - "git_route_not_allowed": 403, "provider_not_found": 404, "rate_limit_exceeded": 429, "repository_inactive": 404, @@ -22,7 +20,6 @@ "resource_unprocessable_content": 422, "resource_server_error": 500, "resource_bad_gateway": 502, - "resource_certificate_invalid": 502, "resource_service_unavailable": 503, "resource_gateway_timeout": 504, "unexpected_response_format": 500, diff --git a/tests/unit/git_upstream.py b/tests/unit/git_upstream.py index f38e067b..71cdb302 100644 --- a/tests/unit/git_upstream.py +++ b/tests/unit/git_upstream.py @@ -21,6 +21,7 @@ from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import ec from cryptography.x509.oid import NameOID +from requests.adapters import HTTPAdapter from requests.structures import CaseInsensitiveDict @@ -125,6 +126,18 @@ def handle_error(self, request: object, client_address: object) -> None: super().handle_error(request, client_address) # type: ignore[arg-type] +class ClosingAdapter(HTTPAdapter): + """An adapter that records whether it was closed, which closes every connection it opened.""" + + def __init__(self) -> None: + super().__init__() + self.closed = False + + def close(self) -> None: + self.closed = True + super().close() + + def echo(handler: UpstreamHandler) -> None: body = b"".join(handler.body_chunks()) handler.record(body) diff --git a/tests/unit/test_git_grant.py b/tests/unit/test_git_grant.py index 34c5cda6..8c7cb7e9 100644 --- a/tests/unit/test_git_grant.py +++ b/tests/unit/test_git_grant.py @@ -63,15 +63,26 @@ def test_token_is_a_jwt_with_the_grants_claims(self): class TestExpiry: def test_expired(self): - assert_invalid(sign_grant(SECRET, make_grant(expires_at=NOW - 1)), "expired") + assert_invalid(sign_grant(SECRET, make_grant(expires_at=NOW - 1), now=NOW), "expired") def test_expires_at_its_expiry_time(self): - assert_invalid(sign_grant(SECRET, make_grant(expires_at=NOW)), "expired") + assert_invalid(sign_grant(SECRET, make_grant(expires_at=NOW), now=NOW), "expired") def test_valid_until_its_expiry_time(self): - token = sign_grant(SECRET, make_grant(expires_at=NOW + 1)) + token = sign_grant(SECRET, make_grant(expires_at=NOW + 1), now=NOW) assert verify_grant([SECRET], token, now=NOW).expires_at == NOW + 1 + def test_issued_in_the_future(self): + assert_invalid(sign_grant(SECRET, make_grant(), now=NOW + 61), "issued in the future") + + def test_issued_slightly_in_the_future(self): + # The signer's clock may run a little ahead of the verifier's. + assert verify_grant([SECRET], sign_grant(SECRET, make_grant(), now=NOW + 60), now=NOW) == make_grant() + + @pytest.mark.parametrize("iat", [float(NOW), str(NOW), True]) + def test_iat_must_be_an_integer(self, iat): + assert_invalid(sign_claims(claims_of(make_grant()) | {"iat": iat}), "iat is not an integer") + @pytest.mark.parametrize("claim", ["exp", "iat", "aud"]) def test_jwt_claims_are_required(self, claim): claims = claims_of(make_grant()) @@ -81,19 +92,21 @@ def test_jwt_claims_are_required(self, claim): class TestTampering: def test_changed_payload(self): - header, _, signature = sign_grant(SECRET, make_grant()).split(".") + header, _, signature = sign_grant(SECRET, make_grant(), now=NOW).split(".") forged = base64.urlsafe_b64encode(msgspec.json.encode(claims_of(make_grant(organization_id=2)))) assert_invalid(f"{header}.{forged.rstrip(b'=').decode()}.{signature}", "signature does not match") def test_changed_signature(self): - header, payload, signature = sign_grant(SECRET, make_grant()).split(".") + header, payload, signature = sign_grant(SECRET, make_grant(), now=NOW).split(".") flipped = bytearray(base64.urlsafe_b64decode(signature + "=")) flipped[0] ^= 1 resigned = base64.urlsafe_b64encode(bytes(flipped)).rstrip(b"=").decode() assert_invalid(f"{header}.{payload}.{resigned}", "signature does not match") def test_unknown_secret(self): - assert_invalid(sign_grant("other-secret-0000000000000000000000", make_grant()), "not signed with a known") + assert_invalid( + sign_grant("other-secret-0000000000000000000000", make_grant(), now=NOW), "not signed with a known" + ) def test_other_secret_claiming_a_known_key_id(self): token = sign_claims(claims_of(make_grant()), secret="other-secret-0000000000000000000000", kid=key_id(SECRET)) @@ -101,7 +114,7 @@ def test_other_secret_claiming_a_known_key_id(self): def test_no_secrets(self): with pytest.raises(GitInvalidGrant): - verify_grant([], sign_grant(SECRET, make_grant()), now=NOW) + verify_grant([], sign_grant(SECRET, make_grant(), now=NOW), now=NOW) def test_token_without_an_audience(self): # A viewer context, for one, is an HS256 JWT signed with a shared secret and carries no audience. @@ -125,7 +138,7 @@ class TestSecrets: # PyJWT signs with an empty key, so an empty secret would let anyone sign grants. def test_refuses_to_sign_with_an_empty_secret(self): with pytest.raises(ValueError, match="empty"): - sign_grant("", make_grant()) + sign_grant("", make_grant(), now=NOW) @pytest.mark.filterwarnings("ignore::jwt.warnings.InsecureKeyLengthWarning") def test_refuses_to_verify_with_an_empty_secret(self): @@ -137,10 +150,12 @@ def test_refuses_to_verify_with_an_empty_secret(self): class TestRotatedSecrets: @pytest.mark.parametrize("signer", [OLD_SECRET, NEW_SECRET]) def test_both_secrets_verify_during_rotation(self, signer): - assert verify_grant([NEW_SECRET, OLD_SECRET], sign_grant(signer, make_grant()), now=NOW) == make_grant() + assert ( + verify_grant([NEW_SECRET, OLD_SECRET], sign_grant(signer, make_grant(), now=NOW), now=NOW) == make_grant() + ) def test_old_secret_refused_after_rotation(self): - assert_invalid(sign_grant(OLD_SECRET, make_grant()), "not signed with a known", secrets=[NEW_SECRET]) + assert_invalid(sign_grant(OLD_SECRET, make_grant(), now=NOW), "not signed with a known", secrets=[NEW_SECRET]) class TestMalformed: @@ -168,4 +183,4 @@ def test_signed_but_invalid_claims(self, change): assert_invalid(sign_claims(claims_of(make_grant()) | change), "payload is invalid") def test_no_repositories(self): - assert_invalid(sign_grant(SECRET, make_grant(repositories=())), "names no repositories") + assert_invalid(sign_grant(SECRET, make_grant(repositories=()), now=NOW), "names no repositories") diff --git a/tests/unit/test_git_relay.py b/tests/unit/test_git_relay.py index 211ddeed..c71dafc1 100644 --- a/tests/unit/test_git_relay.py +++ b/tests/unit/test_git_relay.py @@ -6,11 +6,13 @@ from collections.abc import Iterator import pytest +import requests from requests.adapters import HTTPAdapter from scm.errors import ( GitRemoteNotHttps, GitRouteNotAllowed, + ResourceBadGateway, ResourceCertificateInvalid, ResourceGatewayTimeout, ResourceServiceUnavailable, @@ -18,7 +20,7 @@ ) from scm.git.relay import GitRelay from scm.types import GitRemote -from tests.unit.git_upstream import UpstreamHandler, basic_auth, ca_bundle, running_upstream +from tests.unit.git_upstream import ClosingAdapter, UpstreamHandler, basic_auth, ca_bundle, running_upstream TOKEN = "ghs_secrettoken" # Long enough that a relay that buffers fails the test instead of hanging it. @@ -273,6 +275,15 @@ def handler(h: UpstreamHandler) -> None: b"".join(response.content) assert len(upstream.requests) == 1 + def test_redirect_body_streams(self, relay, upstream): + # requests reads a redirect's whole body into memory to prepare the next request, even when it will not + # follow it. + body = b"x" * 1024 * 1024 + upstream.handler = lambda h: h.respond(302, body, {"Location": "https://elsewhere.example.com/"}) + response = fetch_refs(relay, upstream) + assert response.content.upstream._content is False # type: ignore[attr-defined] + assert b"".join(response.content) == body + class TestStreaming: def test_response_streams_before_the_upstream_finishes(self, relay, upstream): @@ -328,18 +339,6 @@ def read(self, size: int = -1, /) -> bytes: assert b"".join(received) == b"firstsecond" -class ClosingAdapter(HTTPAdapter): - """An adapter that records whether it was closed, which closes every connection it opened.""" - - def __init__(self) -> None: - super().__init__() - self.closed = False - - def close(self) -> None: - self.closed = True - super().close() - - class TestConnections: def test_opens_a_connection_per_request(self, relay, upstream): b"".join(fetch_refs(relay, upstream).content) @@ -360,6 +359,19 @@ def adapter() -> ClosingAdapter: b"".join(response.content) assert [a.closed for a in adapters] == [True] + def test_closes_the_connection_when_an_unread_response_is_closed(self, upstream): + # As GitServer does with a 401 it replaces. + adapters: list[ClosingAdapter] = [] + + def adapter() -> ClosingAdapter: + adapters.append(ClosingAdapter()) + return adapters[-1] + + upstream.handler = lambda h: h.respond(401, b"Bad credentials") + response = fetch_refs(GitRelay(adapter=adapter, ca_bundle=ca_bundle()), upstream) + response.content.close() # type: ignore[attr-defined] + assert [a.closed for a in adapters] == [True] + def test_closes_the_connection_when_the_request_fails(self): adapters: list[ClosingAdapter] = [] @@ -445,6 +457,15 @@ def handler(h: UpstreamHandler) -> None: finally: release.set() + def test_other_request_failures(self, upstream): + class FailingAdapter(HTTPAdapter): + def send(self, request, *args, **kwargs): + raise requests.exceptions.InvalidHeader("bad header", request=request) + + with pytest.raises(ResourceBadGateway, match="failed") as exc_info: + fetch_refs(GitRelay(adapter=FailingAdapter, ca_bundle=ca_bundle()), upstream) + assert_token_not_in(exc_info.value) + def assert_token_not_in(error: Exception) -> None: # The requests exception holds the prepared request and its Authorization header, so it must not be chained. diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index af5b5a1d..63f9d304 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -6,14 +6,15 @@ import pytest +from scm.errors import RateLimitExceeded from scm.git.grant import sign_grant from scm.git.relay import GitRelay -from scm.git.server import GitServer +from scm.git.server import _STATUS, GitServer from scm.git.types import GRANT_HEADER, GitOperation, Readable from scm.rpc.types import StreamResponse from scm.types import GitRemote, Provider, Repository, RepositoryId from tests.unit.git_builders import make_grant, make_provider, make_repository -from tests.unit.git_upstream import basic_auth, running_upstream +from tests.unit.git_upstream import ClosingAdapter, basic_auth, running_upstream SECRET = "test-secret-0000000000000000000000" TOKEN = "ghs_secrettoken" @@ -183,7 +184,7 @@ def test_provider_without_a_git_remote(self): recorder.fetch_provider = lambda organization_id, repository: SimpleNamespace( # type: ignore[method-assign] organization_id=organization_id, repository=repository ) - assert_refused(request(make_server(recorder), signed_grant()), 400, "github provider has no git remote") + assert_refused(request(make_server(recorder), signed_grant()), 404, "github provider has no git remote") assert recorder.token_lookups == [] def test_remote_comes_from_the_provider(self): @@ -201,7 +202,7 @@ def test_remote_comes_from_the_provider(self): (make_repository(organization_id=2), 404, "did not belong"), (make_repository(is_active=False), 404, "inactive"), (make_repository(provider_name="bitbucket"), 404, "unsupported integration provider"), - (make_repository(provider_name="gitlab", external_id="gitlab.com:7"), 400, "has no path"), + (make_repository(provider_name="gitlab", external_id="gitlab.com:7"), 500, "has no path"), ], ) def test_refused_before_the_token(self, repository, status, detail): @@ -213,7 +214,10 @@ def test_refused_before_the_token(self, repository, status, detail): class TestRelay: def test_no_token(self): - assert_refused(request(make_server(Recorder(token=None)), signed_grant()), 404, "No git credentials") + recorder = Recorder(token=None) + response = request(make_server(recorder), signed_grant()) + assert_refused(response, 502, "could not get a github token") + assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_token_unavailable"})] def test_token_asked_for_the_grants_operation(self): recorder = Recorder() @@ -260,7 +264,12 @@ def broken_stream(): assert next(content) == b"half a pack" with pytest.raises(ConnectionError): next(content) - assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "stream_interrupted"})] + assert recorder.counts == [ + ( + "sentry.scm.git.request.failed", + {"operation": "fetch", "provider": "github", "code": "stream_interrupted"}, + ) + ] assert recorder.bytes == {("sentry.scm.git.response_bytes", "fetch"): 11} def test_client_leaving_early_is_a_failure_and_closes_the_provider_stream(self): @@ -278,7 +287,12 @@ def provider_stream(): assert next(content) == b"first" content.close() # type: ignore[attr-defined] assert closed == [True] - assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "client_closed_request"})] + assert recorder.counts == [ + ( + "sentry.scm.git.request.failed", + {"operation": "fetch", "provider": "github", "code": "client_closed_request"}, + ) + ] def test_upstream_unavailable(self): recorder = Recorder( @@ -294,27 +308,38 @@ def test_plain_http_remote(self): repository=make_repository(provider_name="github_enterprise", web_base_url="http://127.0.0.1:1") ) response = request(make_server(recorder, GitRelay()), signed_grant()) - assert_refused(response, 400, "not https") + assert_refused(response, 502, "not https") assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_remote_not_https"})] def test_provider_rejecting_the_token_is_a_gateway_error(self): # Passed on, a 401 would make git ask the sandboxed client for credentials. - closed = [] + adapters: list[ClosingAdapter] = [] - def provider_body(): - try: - yield b"Bad credentials" - finally: - closed.append(True) + def adapter() -> ClosingAdapter: + adapters.append(ClosingAdapter()) + return adapters[-1] - content = provider_body() - next(content) # Started, as a relay's stream is once its first chunk is read. - recorder = Recorder() - response = request(make_server(recorder, relay_returning(content, status_code=401)), signed_grant()) + with running_upstream() as upstream: + upstream.handler = lambda h: h.respond(401, b"Bad credentials") + recorder = Recorder( + repository=make_repository(provider_name="github_enterprise", web_base_url=upstream.url) + ) + relay = GitRelay(adapter=adapter, ca_bundle=upstream.ca_bundle) + response = request(make_server(recorder, relay), signed_grant()) assert_refused(response, 502, "rejected Sentry's credentials") - assert closed == [True] + # The provider's unread response is closed with its connection. + assert [a.closed for a in adapters] == [True] assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "resource_bad_gateway"})] + def test_unexpected_error_code_is_a_server_error(self): + relay = MagicMock(spec=GitRelay) + relay.forward.side_effect = RateLimitExceeded() + assert_refused(request(make_server(Recorder(), relay), signed_grant()), 500, "") + + def test_no_refusal_is_a_401(self): + # On a 401 git asks the sandboxed client for credentials. + assert 401 not in _STATUS.values() + def test_other_errors_reach_the_caller(self): # Sentry's view answers these the way it answers them elsewhere, e.g. a refused host as ApiHostError. class HostRefused(Exception): From f593ce2327e8f45278b3f642419074462ebfa250 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos <dlugos.ivan@gmail.com> Date: Wed, 21 Oct 2026 08:15:27 +0200 Subject: [PATCH 23/31] fix(git): Keep urllib3 errors and unencodable credentials from escaping the relay HTTPAdapter.send re-raises the urllib3 errors it does not translate, and those are not RequestExceptions, so they reached the caller as raw urllib3 errors instead of a 502. Credentials that are not Latin-1 failed inside requests with a UnicodeEncodeError whose .object is the credentials themselves. The relay now catches urllib3's HTTPError with requests' own, and refuses non-ASCII credentials before building the request, with a message that leaves them out. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/scm/git/relay.py | 17 ++++++++++++----- tests/unit/test_git_relay.py | 37 ++++++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/src/scm/git/relay.py b/src/scm/git/relay.py index bb99dddf..56377fcb 100644 --- a/src/scm/git/relay.py +++ b/src/scm/git/relay.py @@ -2,6 +2,7 @@ from urllib.parse import urlsplit import requests +import urllib3 from requests.adapters import HTTPAdapter from requests.auth import HTTPBasicAuth from requests.structures import CaseInsensitiveDict @@ -60,9 +61,9 @@ class GitRelay: ``adapter`` builds the adapter that sends each upstream request. Pass one that refuses internal addresses: a GitHub Enterprise or self-managed GitLab host is whatever the customer configured, and the relay connects wherever it resolves. Sentry's ``BlacklistAdapter`` does this for its API clients, and the class itself can be - passed. An error the adapter raises for a refused host reaches the caller unchanged, unless it is a requests - exception, which the relay replaces like any other so that it cannot carry the token. The adapter must not - retry, because a retry cannot resend a body that has already been streamed. Without one, the relay uses a plain + passed. An error the adapter raises for a refused host reaches the caller unchanged, unless it is a requests or + urllib3 exception, which the relay replaces like any other so that it cannot carry the token. The adapter must + not retry, because a retry cannot resend a body that has already been streamed. Without one, the relay uses a plain ``HTTPAdapter``, which checks nothing. The relay takes no settings from the environment. Through a proxy from ``HTTPS_PROXY``, the adapter would @@ -104,6 +105,10 @@ def forward( # Callers check the request against a grant first; this only makes sure the token never goes to anything # but a git route. require_route(method, route, query) + if not f"{remote['username']}:{token}".isascii(): + # requests would fail to encode the credentials with a UnicodeEncodeError that holds them. No provider + # issues such a token, so the caller has a bug; the message leaves the token out. + raise ValueError(f"The credentials for {remote['url']} contain characters that are not ASCII.") client_headers = CaseInsensitiveDict(headers) upstream_headers = CaseInsensitiveDict( @@ -141,8 +146,10 @@ def forward( raise ResourceCertificateInvalid(detail=f"The TLS certificate of {host} could not be verified.") from None except requests.ConnectionError: raise ResourceServiceUnavailable(detail=f"Could not connect to {remote['url']}.") from None - except requests.RequestException: - # Any other failure of the request, e.g. a malformed response. Like those above, it holds the request. + # HTTPAdapter re-raises the urllib3 errors it does not translate, which are not RequestExceptions. + except (requests.RequestException, urllib3.exceptions.HTTPError): + # Any other failure of the request, e.g. a malformed response. A requests exception, like those above, + # holds the request. raise ResourceBadGateway(detail=f"The request to {remote['url']} failed.") from None finally: # Once a response arrives, _stream closes the session when the response ends. diff --git a/tests/unit/test_git_relay.py b/tests/unit/test_git_relay.py index c71dafc1..09656bba 100644 --- a/tests/unit/test_git_relay.py +++ b/tests/unit/test_git_relay.py @@ -7,6 +7,7 @@ import pytest import requests +import urllib3 from requests.adapters import HTTPAdapter from scm.errors import ( @@ -457,16 +458,48 @@ def handler(h: UpstreamHandler) -> None: finally: release.set() - def test_other_request_failures(self, upstream): + @pytest.mark.parametrize( + "make_error", + [ + pytest.param( + lambda request: requests.exceptions.InvalidHeader("bad header", request=request), id="requests" + ), + # HTTPAdapter.send re-raises the urllib3 errors it does not translate. + pytest.param(lambda request: urllib3.exceptions.HTTPError("bad response"), id="urllib3"), + ], + ) + def test_other_request_failures(self, upstream, make_error): class FailingAdapter(HTTPAdapter): def send(self, request, *args, **kwargs): - raise requests.exceptions.InvalidHeader("bad header", request=request) + raise make_error(request) with pytest.raises(ResourceBadGateway, match="failed") as exc_info: fetch_refs(GitRelay(adapter=FailingAdapter, ca_bundle=ca_bundle()), upstream) assert_token_not_in(exc_info.value) +@pytest.mark.parametrize( + ("username", "token"), + [("x-access-token", "ghs_tok\u20acn"), ("x-acc\u00e9ss-token", TOKEN)], + ids=["token", "username"], +) +def test_refuses_credentials_that_are_not_ascii(upstream, username, token): + remote = GitRemote(url=f"{upstream.url}/getsentry/sentry.git", username=username) + with pytest.raises(ValueError, match="not ASCII") as exc_info: + GitRelay(ca_bundle=ca_bundle()).forward( + remote, + token, + method="GET", + route="info/refs", + query="service=git-upload-pack", + headers={}, + body=None, + ) + assert token not in str(exc_info.value) + assert exc_info.value.__context__ is None + assert not upstream.requests + + def assert_token_not_in(error: Exception) -> None: # The requests exception holds the prepared request and its Authorization header, so it must not be chained. assert error.__cause__ is None From d1db8223b9f5a24cbbc40d11e46fe65765ddd5c6 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos <dlugos.ivan@gmail.com> Date: Wed, 21 Oct 2026 08:27:54 +0200 Subject: [PATCH 24/31] docs(git): Drop Cursor Origin as the example of a provider without a git remote It has one since Origin repositories got a git remote. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/scm/git/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scm/git/server.py b/src/scm/git/server.py index 1b91e4d5..a68df663 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -48,7 +48,7 @@ class GitServer: refuses internal addresses; see ``GitRelay``. The repository's provider says where its git endpoint is, through ``GetGitRemoteProtocol``. A repository whose - provider has no git remote, such as Cursor Origin's, is refused. + provider has no git remote is refused. Refusals and provider failures come back as plain-text responses. Any other exception, such as an adapter refusing a host or the token supplier failing, reaches the caller unchanged, so the caller can answer it the From 4c55d81076a19dd048557c6b00fd41c3d32b33cd Mon Sep 17 00:00:00 2001 From: Ivan Dlugos <dlugos.ivan@gmail.com> Date: Wed, 21 Oct 2026 11:32:05 +0200 Subject: [PATCH 25/31] ref(git): Give each git refusal its own error code git_route_not_allowed covered three unrelated refusals: a request that is not a git route at all, a grant used for the other operation, and a grant that does not name the repository. git_invalid_grant lumped expiry, which is expected for a long job, with forgery and misconfigured secrets. And a provider rejecting Sentry's token was reported as resource_bad_gateway, the same code as a malformed provider response, though it means the integration's credentials are broken. Metrics tagged with these codes could not tell any of them apart. A request that is not a git route is now a 404, since it is a path the endpoint does not serve rather than one the grant forbids. The messages git shows are unchanged. None of these codes has been released, so they change without a deprecation. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/scm/errors.py | 34 ++++++++++++++++++++++++++----- src/scm/git/grant.py | 4 ++-- src/scm/git/operations.py | 8 ++++---- src/scm/git/server.py | 12 +++++++---- tests/unit/test_git_grant.py | 13 +++++++++--- tests/unit/test_git_operations.py | 12 +++++------ tests/unit/test_git_relay.py | 4 ++-- tests/unit/test_git_server.py | 33 ++++++++++++++++++++++-------- 8 files changed, 85 insertions(+), 35 deletions(-) diff --git a/src/scm/errors.py b/src/scm/errors.py index 66a1f7c4..0764fa67 100644 --- a/src/scm/errors.py +++ b/src/scm/errors.py @@ -1,10 +1,14 @@ from typing import ClassVar, Literal type ErrorCode = Literal[ + "git_grant_expired", "git_invalid_grant", + "git_operation_not_granted", "git_provider_not_supported", "git_remote_not_https", - "git_route_not_allowed", + "git_repository_not_granted", + "git_route_not_supported", + "git_token_rejected", "git_token_unavailable", "malformed_external_id", "missing_repository_path", @@ -44,10 +48,14 @@ ] ERROR_CODES: dict[ErrorCode, str] = { - "git_invalid_grant": "The git grant was missing, malformed, forged or expired.", + "git_grant_expired": "The git grant has expired.", + "git_invalid_grant": "The git grant was missing, malformed or forged.", + "git_operation_not_granted": "The git grant does not allow this operation.", "git_provider_not_supported": "The repository's provider has no git remote.", "git_remote_not_https": "The repository's git remote is not an https URL.", - "git_route_not_allowed": "The git grant does not allow this request.", + "git_repository_not_granted": "The git grant does not cover this repository.", + "git_route_not_supported": "The request is not a git fetch or push.", + "git_token_rejected": "The provider rejected Sentry's credentials for the repository.", "git_token_unavailable": "No provider token could be obtained for the repository.", "malformed_external_id": "The repository's external ID was malformed.", "missing_repository_path": "No path was given for the repository.", @@ -137,10 +145,18 @@ def from_code(cls, code: ErrorCode, detail: str | None = None) -> "SCMCodedError return error_cls(detail=detail) +class GitGrantExpired(SCMCodedError): + code = "git_grant_expired" + + class GitInvalidGrant(SCMCodedError): code = "git_invalid_grant" +class GitOperationNotGranted(SCMCodedError): + code = "git_operation_not_granted" + + class GitProviderNotSupported(SCMCodedError): code = "git_provider_not_supported" @@ -149,8 +165,16 @@ class GitRemoteNotHttps(SCMCodedError): code = "git_remote_not_https" -class GitRouteNotAllowed(SCMCodedError): - code = "git_route_not_allowed" +class GitRepositoryNotGranted(SCMCodedError): + code = "git_repository_not_granted" + + +class GitRouteNotSupported(SCMCodedError): + code = "git_route_not_supported" + + +class GitTokenRejected(SCMCodedError): + code = "git_token_rejected" class GitTokenUnavailable(SCMCodedError): diff --git a/src/scm/git/grant.py b/src/scm/git/grant.py index dd57efd5..0428b54f 100644 --- a/src/scm/git/grant.py +++ b/src/scm/git/grant.py @@ -14,7 +14,7 @@ import jwt import msgspec -from scm.errors import GitInvalidGrant +from scm.errors import GitGrantExpired, GitInvalidGrant from scm.git.types import Grant GRANT_AUDIENCE = "scm-git-grant" @@ -79,7 +79,7 @@ def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> raise GitInvalidGrant(detail=f"The git grant's payload is invalid: {e}") from None if grant.expires_at <= now: - raise GitInvalidGrant(detail="The git grant has expired.") + raise GitGrantExpired(detail="The git grant has expired.") # bool is an int to isinstance, and PyJWT would refuse it as an iat. if not isinstance(claims["iat"], int) or isinstance(claims["iat"], bool): raise GitInvalidGrant(detail="The git grant's iat is not an integer.") diff --git a/src/scm/git/operations.py b/src/scm/git/operations.py index 2329452c..44d9e6a5 100644 --- a/src/scm/git/operations.py +++ b/src/scm/git/operations.py @@ -1,4 +1,4 @@ -from scm.errors import GitRouteNotAllowed +from scm.errors import GitOperationNotGranted, GitRepositoryNotGranted, GitRouteNotSupported from scm.git.types import GitOperation, Grant from scm.types import RepositoryId @@ -26,7 +26,7 @@ def require_route(method: str, route: str, query: str) -> GitOperation: """Return the operation a git smart-HTTP request belongs to, refusing any other request.""" operation = route_operation(method, route, query) if operation is None: - raise GitRouteNotAllowed(detail=f"{method} {route} is not a git fetch or push route.") + raise GitRouteNotSupported(detail=f"{method} {route} is not a git fetch or push route.") return operation @@ -39,7 +39,7 @@ def check_request(grant: Grant, repository_id: RepositoryId, method: str, route: """ operation = require_route(method, route, query) if operation != grant.operation: - raise GitRouteNotAllowed(detail=f"The git grant allows {grant.operation}, not {operation}.") + raise GitOperationNotGranted(detail=f"The git grant allows {grant.operation}, not {operation}.") if repository_id not in grant.repositories: - raise GitRouteNotAllowed(detail="The git grant does not cover this repository.") + raise GitRepositoryNotGranted(detail="The git grant does not cover this repository.") return operation diff --git a/src/scm/git/server.py b/src/scm/git/server.py index a68df663..fa7ccfbb 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -2,7 +2,7 @@ from requests.structures import CaseInsensitiveDict -from scm.errors import ErrorCode, GitProviderNotSupported, GitTokenUnavailable, ResourceBadGateway, SCMCodedError +from scm.errors import ErrorCode, GitProviderNotSupported, GitTokenRejected, GitTokenUnavailable, SCMCodedError from scm.git.grant import verify_grant from scm.git.operations import check_request from scm.git.relay import GitRelay @@ -14,11 +14,14 @@ # The status of each refusal GitServer answers itself. None is a 401: on a 401 git asks the user for credentials, # which a sandboxed client must never have and which would not help. _STATUS: dict[ErrorCode, int] = { - # The client's own request is malformed. + # The client's own request is malformed, or is not a git request this endpoint serves. "rpc_malformed_request_headers": 400, + "git_route_not_supported": 404, # The grant does not allow the request. "git_invalid_grant": 403, - "git_route_not_allowed": 403, + "git_grant_expired": 403, + "git_operation_not_granted": 403, + "git_repository_not_granted": 403, # The grant allows it, but there is no repository to serve. "repository_not_found": 404, "repository_inactive": 404, @@ -31,6 +34,7 @@ "missing_repository_path": 500, # The provider cannot be reached safely, or failed. "git_remote_not_https": 502, + "git_token_rejected": 502, "git_token_unavailable": 502, "resource_bad_gateway": 502, "resource_certificate_invalid": 502, @@ -119,7 +123,7 @@ def handle( # The provider rejected the token Sentry supplied. Passed on, the 401 would make git ask the client # for credentials, which it must never have and which would not help. _close_stream(response.content) - raise ResourceBadGateway(detail="The provider rejected Sentry's credentials for this repository.") + raise GitTokenRejected(detail="The provider rejected Sentry's credentials for this repository.") except SCMCodedError as e: self.record_count("sentry.scm.git.request.failed", 1, {"code": e.code}) return _error_response(e) diff --git a/tests/unit/test_git_grant.py b/tests/unit/test_git_grant.py index 8c7cb7e9..d4a8dd0a 100644 --- a/tests/unit/test_git_grant.py +++ b/tests/unit/test_git_grant.py @@ -5,7 +5,7 @@ import msgspec import pytest -from scm.errors import GitInvalidGrant +from scm.errors import GitGrantExpired, GitInvalidGrant from scm.git.grant import GRANT_AUDIENCE, sign_grant, verify_grant from scm.git.types import Grant from tests.unit.git_builders import FAR_FUTURE, make_grant @@ -63,10 +63,17 @@ def test_token_is_a_jwt_with_the_grants_claims(self): class TestExpiry: def test_expired(self): - assert_invalid(sign_grant(SECRET, make_grant(expires_at=NOW - 1), now=NOW), "expired") + with pytest.raises(GitGrantExpired, match="expired"): + verify_grant([SECRET], sign_grant(SECRET, make_grant(expires_at=NOW - 1), now=NOW), now=NOW) def test_expires_at_its_expiry_time(self): - assert_invalid(sign_grant(SECRET, make_grant(expires_at=NOW), now=NOW), "expired") + with pytest.raises(GitGrantExpired): + verify_grant([SECRET], sign_grant(SECRET, make_grant(expires_at=NOW), now=NOW), now=NOW) + + def test_forged_grant_is_invalid_even_when_expired(self): + # Expiry is checked only once the signature matches, so an expired grant is always a genuine one. + token = sign_grant("other-secret-0000000000000000000000", make_grant(expires_at=NOW - 1), now=NOW) + assert_invalid(token, "not signed with a known secret") def test_valid_until_its_expiry_time(self): token = sign_grant(SECRET, make_grant(expires_at=NOW + 1), now=NOW) diff --git a/tests/unit/test_git_operations.py b/tests/unit/test_git_operations.py index 5d0d496a..cf73827b 100644 --- a/tests/unit/test_git_operations.py +++ b/tests/unit/test_git_operations.py @@ -1,6 +1,6 @@ import pytest -from scm.errors import GitRouteNotAllowed +from scm.errors import GitOperationNotGranted, GitRepositoryNotGranted, GitRouteNotSupported from scm.git.operations import check_request, require_route, route_operation from tests.unit.git_builders import make_grant @@ -48,7 +48,7 @@ def test_returns_the_operation(self): assert require_route("POST", "git-receive-pack", "") == "push" def test_refuses_other_routes(self): - with pytest.raises(GitRouteNotAllowed, match="not a git fetch or push route"): + with pytest.raises(GitRouteNotSupported, match="not a git fetch or push route"): require_route("POST", "info/lfs/objects/batch", "") @@ -66,22 +66,22 @@ def test_grant_reaches_its_operations_routes(self, operation, route): ) def test_grant_cannot_reach_the_other_operations_routes(self, operation, route): other = "push" if operation == "fetch" else "fetch" - with pytest.raises(GitRouteNotAllowed, match=f"allows {operation}, not {other}"): + with pytest.raises(GitOperationNotGranted, match=f"allows {operation}, not {other}"): check_request(make_grant(operation=operation), 5, *route) def test_unknown_route(self): - with pytest.raises(GitRouteNotAllowed, match="not a git fetch or push route"): + with pytest.raises(GitRouteNotSupported, match="not a git fetch or push route"): check_request(make_grant(), 5, "POST", "info/lfs/objects/batch", "") def test_repository_not_in_grant(self): - with pytest.raises(GitRouteNotAllowed, match="does not cover this repository"): + with pytest.raises(GitRepositoryNotGranted, match="does not cover this repository"): check_request(make_grant(), 6, "POST", "git-upload-pack", "") def test_repository_named_another_way_is_not_in_grant(self): # The grant and the request name repositories the same way; matching a database id to a provider id # would mean resolving every repository the grant lists. grant = make_grant(repositories=(("github", "123"),)) - with pytest.raises(GitRouteNotAllowed): + with pytest.raises(GitRepositoryNotGranted): check_request(grant, 5, "POST", "git-upload-pack", "") assert check_request(grant, ("github", "123"), "POST", "git-upload-pack", "") == "fetch" diff --git a/tests/unit/test_git_relay.py b/tests/unit/test_git_relay.py index 27559d8e..2c6f8d76 100644 --- a/tests/unit/test_git_relay.py +++ b/tests/unit/test_git_relay.py @@ -12,7 +12,7 @@ from scm.errors import ( GitRemoteNotHttps, - GitRouteNotAllowed, + GitRouteNotSupported, ResourceBadGateway, ResourceCertificateInvalid, ResourceGatewayTimeout, @@ -141,7 +141,7 @@ def test_refuses_plain_http(self, relay, upstream): assert upstream.requests == [] def test_refuses_routes_that_are_not_git(self, relay, upstream): - with pytest.raises(GitRouteNotAllowed): + with pytest.raises(GitRouteNotSupported): relay.forward( remote_for(upstream), TOKEN, diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index b0398ce0..f2f96345 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -138,15 +138,19 @@ class TestRefusedBeforeTheProvider: """Nothing the grant does not allow may look up a repository or a token.""" @pytest.mark.parametrize( - "grant,detail", + "grant,detail,code", [ - (None, "missing"), - ("garbage", "malformed"), - (sign_grant("other-secret-0000000000000000000000", make_grant()), "not signed with a known secret"), - (signed_grant(expires_at=int(time.time()) - 1), "expired"), + (None, "missing", "git_invalid_grant"), + ("garbage", "malformed", "git_invalid_grant"), + ( + sign_grant("other-secret-0000000000000000000000", make_grant()), + "not signed with a known secret", + "git_invalid_grant", + ), + (signed_grant(expires_at=int(time.time()) - 1), "expired", "git_grant_expired"), ], ) - def test_invalid_grant(self, grant, detail): + def test_invalid_grant(self, grant, detail, code): recorder = Recorder() server = make_server(recorder) assert_refused(request(server, grant), 403, detail) @@ -154,18 +158,29 @@ def test_invalid_grant(self, grant, detail): assert recorder.provider_lookups == [] assert recorder.token_lookups == [] server.relay.forward.assert_not_called() # type: ignore[attr-defined] - assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_invalid_grant"})] + assert recorder.counts == [("sentry.scm.git.request.failed", {"code": code})] + + def test_route_that_is_not_git(self): + recorder = Recorder() + response = request( + make_server(recorder), signed_grant(), method="POST", route="info/lfs/objects/batch", query="" + ) + assert_refused(response, 404, "not a git fetch or push route") + assert recorder.repository_lookups == [] + assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_route_not_supported"})] def test_operation_the_grant_does_not_allow(self): recorder = Recorder() assert_refused(push(make_server(recorder), signed_grant()), 403, "allows fetch, not push") assert recorder.repository_lookups == [] assert recorder.token_lookups == [] + assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_operation_not_granted"})] def test_repository_the_grant_does_not_list(self): recorder = Recorder() assert_refused(request(make_server(recorder), signed_grant(), repository_id=6), 403, "does not cover") assert recorder.repository_lookups == [] + assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_repository_not_granted"})] def test_grant_header_name_is_case_insensitive(self): response = request(make_server(Recorder()), None, headers={GRANT_HEADER.lower(): signed_grant()}) @@ -311,7 +326,7 @@ def test_plain_http_remote(self): assert_refused(response, 502, "not https") assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_remote_not_https"})] - def test_provider_rejecting_the_token_is_a_gateway_error(self): + def test_provider_rejecting_the_token(self): # Passed on, a 401 would make git ask the sandboxed client for credentials. adapters: list[ClosingAdapter] = [] @@ -329,7 +344,7 @@ def adapter() -> ClosingAdapter: assert_refused(response, 502, "rejected Sentry's credentials") # The provider's unread response is closed with its connection. assert [a.closed for a in adapters] == [True] - assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "resource_bad_gateway"})] + assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_token_rejected"})] def test_unexpected_error_code_is_a_server_error(self): relay = MagicMock(spec=GitRelay) From 4f652947d1680ced53e1dc5a55c1657af3bbdcfa Mon Sep 17 00:00:00 2001 From: Ivan Dlugos <dlugos.ivan@gmail.com> Date: Wed, 21 Oct 2026 11:32:16 +0200 Subject: [PATCH 26/31] fix(git): Count a provider's refusal as a failed request GitServer passes a provider's own non-2xx response on to git, so the client sees the provider's message, e.g. GitHub's "Repository not found." for a repository the installation cannot see. It counted that response as request.succeeded, though git got nothing it asked for. It is now request.failed with the code provider_error. A redirect counts too: the relay neither follows it nor passes its Location on, so git cannot follow it either. Every count of a relayed request now carries the provider's status, failures included, so a dashboard can split provider_error by status and see what status a stream had when it broke. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/scm/git/server.py | 12 ++++++++++-- tests/unit/test_git_server.py | 21 +++++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/scm/git/server.py b/src/scm/git/server.py index fa7ccfbb..94bb5389 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -60,7 +60,9 @@ class GitServer: A relayed request is counted when its response stream ends, not when its headers arrive, so a clone that fails halfway is counted as failed: ``stream_interrupted`` when the provider's stream breaks and - ``client_closed_request`` when the client goes away first. The bytes sent each way are counted then too. + ``client_closed_request`` when the client goes away first. A response the provider sends with a status other + than 2xx is passed on to the client but counted as failed too, as ``provider_error``. The bytes sent each way + are counted then too. """ def __init__( @@ -134,10 +136,16 @@ def handle( tags: dict[str, str] = {"operation": operation, "provider": repository["provider_name"]} def finish(failure: str | None, response_bytes: int) -> None: + if failure is None and not 200 <= response.status_code < 300: + # The provider refused the request itself, e.g. a 404 for a repository the token cannot see. git + # shows the client the provider's message, but did not get what it asked for. + failure = "provider_error" if failure is None: self.record_count("sentry.scm.git.request.succeeded", 1, tags | {"status": str(response.status_code)}) else: - self.record_count("sentry.scm.git.request.failed", 1, tags | {"code": failure}) + self.record_count( + "sentry.scm.git.request.failed", 1, tags | {"status": str(response.status_code), "code": failure} + ) if counted_body is not None and counted_body.bytes_read: self.record_count("sentry.scm.git.request_bytes", counted_body.bytes_read, tags) if response_bytes: diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index f2f96345..97760d19 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -282,7 +282,7 @@ def broken_stream(): assert recorder.counts == [ ( "sentry.scm.git.request.failed", - {"operation": "fetch", "provider": "github", "code": "stream_interrupted"}, + {"operation": "fetch", "provider": "github", "status": "200", "code": "stream_interrupted"}, ) ] assert recorder.bytes == {("sentry.scm.git.response_bytes", "fetch"): 11} @@ -305,7 +305,7 @@ def provider_stream(): assert recorder.counts == [ ( "sentry.scm.git.request.failed", - {"operation": "fetch", "provider": "github", "code": "client_closed_request"}, + {"operation": "fetch", "provider": "github", "status": "200", "code": "client_closed_request"}, ) ] @@ -346,6 +346,23 @@ def adapter() -> ClosingAdapter: assert [a.closed for a in adapters] == [True] assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_token_rejected"})] + @pytest.mark.parametrize("status", [301, 403, 404, 500]) + def test_provider_refusing_the_request_is_passed_on_and_counted_as_failed(self, status): + # e.g. GitHub's 404 for a repository the installation cannot see; git shows the client its message. + recorder = Recorder() + response = request( + make_server(recorder, relay_returning(iter([b"Repository not found."]), status)), signed_grant() + ) + assert response.status_code == status + assert b"".join(response.content) == b"Repository not found." + response.content.close() # type: ignore[attr-defined] + assert recorder.counts == [ + ( + "sentry.scm.git.request.failed", + {"operation": "fetch", "provider": "github", "status": str(status), "code": "provider_error"}, + ) + ] + def test_unexpected_error_code_is_a_server_error(self): relay = MagicMock(spec=GitRelay) relay.forward.side_effect = RateLimitExceeded() From 451e9f8e6b7880229a59c84df5ac02eb242728ac Mon Sep 17 00:00:00 2001 From: Ivan Dlugos <dlugos.ivan@gmail.com> Date: Wed, 21 Oct 2026 17:08:17 +0200 Subject: [PATCH 27/31] ref(git): Apply review feedback on the git endpoint Refusals and provider failures are now passed to an emit_error callback, as RpcServer does, so the server reports them itself instead of relying on every client to report the message it is shown. A client closing its request is counted but not reported, since there is nothing to fix. For the same reason, provider failures no longer name the provider's URL or host: the client has no use for the address. PyJWT now checks iat, with the clock skew as leeway, instead of a hand-written check that refused floats, which are valid iat values. The rest makes the module easier to read: the route table and check_request move into relay.py, one-off helpers are inlined, the git errors share a GitSCMCodedError base so callers can catch them together, Grant.repositories is a list, and the byte metrics are named request_size and response_size. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/scm/errors.py | 22 +++--- src/scm/git/grant.py | 25 +++---- src/scm/git/operations.py | 45 ------------ src/scm/git/relay.py | 118 ++++++++++++++++++++---------- src/scm/git/server.py | 46 ++++++------ src/scm/git/types.py | 18 +---- tests/unit/git_builders.py | 2 +- tests/unit/test_git_grant.py | 20 +++-- tests/unit/test_git_operations.py | 91 ----------------------- tests/unit/test_git_relay.py | 77 +++++++++++++++++-- tests/unit/test_git_server.py | 57 +++++++++++---- 11 files changed, 261 insertions(+), 260 deletions(-) delete mode 100644 src/scm/git/operations.py delete mode 100644 tests/unit/test_git_operations.py diff --git a/src/scm/errors.py b/src/scm/errors.py index 0764fa67..cb40781d 100644 --- a/src/scm/errors.py +++ b/src/scm/errors.py @@ -145,39 +145,43 @@ def from_code(cls, code: ErrorCode, detail: str | None = None) -> "SCMCodedError return error_cls(detail=detail) -class GitGrantExpired(SCMCodedError): +class GitSCMCodedError(SCMCodedError): + """Base class for the errors of the git endpoint (``scm.git``), so a caller can catch them all at once.""" + + +class GitGrantExpired(GitSCMCodedError): code = "git_grant_expired" -class GitInvalidGrant(SCMCodedError): +class GitInvalidGrant(GitSCMCodedError): code = "git_invalid_grant" -class GitOperationNotGranted(SCMCodedError): +class GitOperationNotGranted(GitSCMCodedError): code = "git_operation_not_granted" -class GitProviderNotSupported(SCMCodedError): +class GitProviderNotSupported(GitSCMCodedError): code = "git_provider_not_supported" -class GitRemoteNotHttps(SCMCodedError): +class GitRemoteNotHttps(GitSCMCodedError): code = "git_remote_not_https" -class GitRepositoryNotGranted(SCMCodedError): +class GitRepositoryNotGranted(GitSCMCodedError): code = "git_repository_not_granted" -class GitRouteNotSupported(SCMCodedError): +class GitRouteNotSupported(GitSCMCodedError): code = "git_route_not_supported" -class GitTokenRejected(SCMCodedError): +class GitTokenRejected(GitSCMCodedError): code = "git_token_rejected" -class GitTokenUnavailable(SCMCodedError): +class GitTokenUnavailable(GitSCMCodedError): code = "git_token_unavailable" diff --git a/src/scm/git/grant.py b/src/scm/git/grant.py index 0428b54f..477e31f3 100644 --- a/src/scm/git/grant.py +++ b/src/scm/git/grant.py @@ -27,7 +27,9 @@ def sign_grant(secret: str, grant: Grant, *, now: float | None = None) -> str: - _check_secrets([secret]) + # HMAC accepts an empty key, so an empty secret, say from an unset setting, would let anyone sign a grant. + if not secret: + raise ValueError("A git grant secret is empty.") claims: dict[str, Any] = msgspec.to_builtins(grant) claims["exp"] = claims.pop("expires_at") claims["iat"] = int(time.time() if now is None else now) @@ -41,7 +43,9 @@ def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> Verification takes a list of secrets, as ``RpcServer`` does, so a new secret can be added before signers switch to it and the old one removed after their grants expire. """ - _check_secrets(secrets) + # As in sign_grant: a grant signed with an empty secret proves nothing. + if not all(secrets): + raise ValueError("A git grant secret is empty.") now = time.time() if now is None else now if not token: raise GitInvalidGrant(detail="The git grant is missing.") @@ -59,8 +63,10 @@ def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> secret, algorithms=[_ALGORITHM], audience=GRANT_AUDIENCE, - # exp and iat are checked below against an injectable clock, so tests can pin it. - options={"require": sorted(_JWT_CLAIMS), "verify_exp": False, "verify_iat": False}, + # exp is checked below against an injectable clock, so tests can pin it. PyJWT checks iat, which it + # also requires to be a number. + options={"require": sorted(_JWT_CLAIMS), "verify_exp": False}, + leeway=_CLOCK_SKEW, ) except jwt.InvalidSignatureError: raise GitInvalidGrant(detail="The git grant's signature does not match.") from None @@ -80,22 +86,11 @@ def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> if grant.expires_at <= now: raise GitGrantExpired(detail="The git grant has expired.") - # bool is an int to isinstance, and PyJWT would refuse it as an iat. - if not isinstance(claims["iat"], int) or isinstance(claims["iat"], bool): - raise GitInvalidGrant(detail="The git grant's iat is not an integer.") - if claims["iat"] > now + _CLOCK_SKEW: - raise GitInvalidGrant(detail="The git grant was issued in the future.") if not grant.repositories: raise GitInvalidGrant(detail="The git grant names no repositories.") return grant -def _check_secrets(secrets: list[str]) -> None: - # HMAC accepts an empty key, so an empty secret, say from an unset setting, would let anyone sign a grant. - if not all(secrets): - raise ValueError("A git grant secret is empty.") - - def _key_id(secret: str) -> str: # The first 8 hex characters of the secret's SHA-256, the same key id Sentry's viewer contexts use. return hashlib.sha256(secret.encode()).hexdigest()[:8] diff --git a/src/scm/git/operations.py b/src/scm/git/operations.py deleted file mode 100644 index 44d9e6a5..00000000 --- a/src/scm/git/operations.py +++ /dev/null @@ -1,45 +0,0 @@ -from scm.errors import GitOperationNotGranted, GitRepositoryNotGranted, GitRouteNotSupported -from scm.git.types import GitOperation, Grant -from scm.types import RepositoryId - -# The git smart-HTTP routes and the operation each belongs to, keyed by (method, path below <repo>.git/, query -# string). git names its two services from the server's side: in a fetch the server uploads a pack to the client -# (git-upload-pack), and in a push it receives a pack from the client (git-receive-pack). Each service has a GET -# that lists the refs and a POST that does the transfer. -# -# The query string is compared whole instead of parsed. git sends exactly these strings, so a request with extra -# or repeated parameters is refused rather than passed on to a provider that might read them differently. -_ROUTES: dict[tuple[str, str, str], GitOperation] = { - ("GET", "info/refs", "service=git-upload-pack"): "fetch", - ("POST", "git-upload-pack", ""): "fetch", - ("GET", "info/refs", "service=git-receive-pack"): "push", - ("POST", "git-receive-pack", ""): "push", -} - - -def route_operation(method: str, route: str, query: str) -> GitOperation | None: - """Return the operation a git smart-HTTP request belongs to, or None if it is not one.""" - return _ROUTES.get((method, route, query)) - - -def require_route(method: str, route: str, query: str) -> GitOperation: - """Return the operation a git smart-HTTP request belongs to, refusing any other request.""" - operation = route_operation(method, route, query) - if operation is None: - raise GitRouteNotSupported(detail=f"{method} {route} is not a git fetch or push route.") - return operation - - -def check_request(grant: Grant, repository_id: RepositoryId, method: str, route: str, query: str) -> GitOperation: - """Refuse a request the grant does not reach, before anything is sent to the provider. - - A fetch grant reaches only upload-pack and a push grant only receive-pack. The repository must be listed in - the grant in the same form the request uses: a database id does not match the same repository's - ``(provider, external id)`` pair. - """ - operation = require_route(method, route, query) - if operation != grant.operation: - raise GitOperationNotGranted(detail=f"The git grant allows {grant.operation}, not {operation}.") - if repository_id not in grant.repositories: - raise GitRepositoryNotGranted(detail="The git grant does not cover this repository.") - return operation diff --git a/src/scm/git/relay.py b/src/scm/git/relay.py index 38508636..e1a46f70 100644 --- a/src/scm/git/relay.py +++ b/src/scm/git/relay.py @@ -1,5 +1,5 @@ from collections.abc import Callable, Iterator, Mapping -from urllib.parse import urlsplit +from typing import Protocol import requests import urllib3 @@ -9,17 +9,19 @@ from scm.errors import ( ClientClosedRequest, + GitOperationNotGranted, GitRemoteNotHttps, + GitRepositoryNotGranted, + GitRouteNotSupported, ResourceBadGateway, ResourceCertificateInvalid, ResourceGatewayTimeout, ResourceServiceUnavailable, RpcMalformedRequestHeaders, ) -from scm.git.operations import require_route -from scm.git.types import Readable +from scm.git.types import GitOperation, Grant, Readable from scm.rpc.types import StreamResponse -from scm.types import GitRemote +from scm.types import GitRemote, RepositoryId # The most bytes the relay reads at once, from the client's body or from the provider's response, so also the # most it holds per request in each direction. Smaller chunks cost more Python per byte of a multi-gigabyte pack; @@ -31,6 +33,20 @@ # assembles a large pack before it sends anything. Both values come from the prototype. DEFAULT_TIMEOUT = (10.0, 300.0) +# The git smart-HTTP routes and the operation each belongs to, keyed by (method, path below <repo>.git/, query +# string). git names its two services from the server's side: in a fetch the server uploads a pack to the client +# (git-upload-pack), and in a push it receives a pack from the client (git-receive-pack). Each service has a GET +# that lists the refs and a POST that does the transfer. +# +# The query string is compared whole instead of parsed. git sends exactly these strings, so a request with extra +# or repeated parameters is refused rather than passed on to a provider that might read them differently. +_ROUTES: dict[tuple[str, str, str], GitOperation] = { + ("GET", "info/refs", "service=git-upload-pack"): "fetch", + ("POST", "git-upload-pack", ""): "fetch", + ("GET", "info/refs", "service=git-receive-pack"): "push", + ("POST", "git-receive-pack", ""): "push", +} + # The headers git sends that the provider needs. Accept and Content-Type name the git service, Git-Protocol # negotiates protocol v2, Content-Encoding marks a gzipped request body (git gzips large fetch requests), # Accept-Encoding says what the client can decode, and User-Agent names the git client. Everything else stays @@ -46,6 +62,35 @@ ) +def check_request(grant: Grant, repository_id: RepositoryId, method: str, route: str, query: str) -> GitOperation: + """Refuse a request the grant does not reach, before anything is sent to the provider. + + A fetch grant reaches only upload-pack and a push grant only receive-pack. The repository must be listed in + the grant in the same form the request uses: a database id does not match the same repository's + ``(provider, external id)`` pair. + """ + operation = _ROUTES.get((method, route, query)) + if operation is None: + raise GitRouteNotSupported(detail=f"{method} {route} is not a git fetch or push route.") + if operation != grant.operation: + raise GitOperationNotGranted(detail=f"The git grant allows {grant.operation}, not {operation}.") + if repository_id not in grant.repositories: + raise GitRepositoryNotGranted(detail="The git grant does not cover this repository.") + return operation + + +class ResponseBody(Iterator[bytes], Protocol): + """The provider's response body. Closing it before it ends drops the upstream connection.""" + + def close(self) -> None: ... + + +class RelayResponse(StreamResponse): + """The provider's response, whose body GitServer closes when it no longer needs it.""" + + content: ResponseBody + + class GitRelay: """Forwards git smart-HTTP requests to a provider with a token, streaming the body in both directions. @@ -94,17 +139,21 @@ def forward( query: str, headers: Mapping[str, str], body: Readable | None, - ) -> StreamResponse: + ) -> RelayResponse: """Send one request to ``remote`` and return the provider's response as it arrives. - ``route`` is the path below ``<repo>.git/``. Check the request against a grant before calling this. + ``route`` is the path below ``<repo>.git/``. Check the request against a grant with ``check_request`` + before calling this. + + Errors carry no URL, because the client has no use for the provider's address. """ if not remote["url"].startswith("https://"): # The token would cross the network in the clear, e.g. to a self-managed instance served over http. - raise GitRemoteNotHttps(detail=f"Refusing to send a token to {remote['url']}, which is not https.") + raise GitRemoteNotHttps() # Callers check the request against a grant first; this only makes sure the token never goes to anything # but a git route. - require_route(method, route, query) + if (method, route, query) not in _ROUTES: + raise GitRouteNotSupported(detail=f"{method} {route} is not a git fetch or push route.") if not f"{remote['basic_auth_username']}:{token}".isascii(): # requests would fail to encode the credentials with a UnicodeEncodeError that holds them. No provider # issues such a token, so the caller has a bug; the message leaves the token out. @@ -120,8 +169,25 @@ def forward( # Of the git routes only the POSTs carry a body. A caller may pass its request stream for a GET too, and sent # on, it would go upstream as an empty chunked body. - data = _request_body(body, _content_length(client_headers), self.chunk_size) if method == "POST" else None - url = f"{remote['url']}/{route}?{query}" if query else f"{remote['url']}/{route}" + data: Iterator[bytes] | _SizedBody | None = None + if method == "POST" and body is not None: + raw_length = client_headers.get("Content-Length") + if raw_length is None: + # requests sends a generator with chunked transfer encoding, the way git sends a large push. + data = _read_until_eof(body, self.chunk_size) + else: + try: + length = int(raw_length) + except ValueError: + raise RpcMalformedRequestHeaders(detail="Could not parse Content-Length header") from None + if length < 0: + raise RpcMalformedRequestHeaders(detail="Could not parse Content-Length header") + if length > 0: + data = _SizedBody(body, length, self.chunk_size) + + url = f"{remote['url']}/{route}" + if query: + url += f"?{query}" session = self._session() upstream = None try: @@ -138,25 +204,24 @@ def forward( ) # "from None" drops requests' exception, whose request holds the Authorization header. except requests.Timeout: - raise ResourceGatewayTimeout(detail=f"{remote['url']} did not respond in time.") from None + raise ResourceGatewayTimeout() from None # SSLError is a kind of ConnectionError, so catch it before that one, or a bad certificate would be # reported as an unreachable host. except requests.exceptions.SSLError: - host = urlsplit(remote["url"]).netloc - raise ResourceCertificateInvalid(detail=f"The TLS certificate of {host} could not be verified.") from None + raise ResourceCertificateInvalid() from None except requests.ConnectionError: - raise ResourceServiceUnavailable(detail=f"Could not connect to {remote['url']}.") from None + raise ResourceServiceUnavailable() from None # HTTPAdapter re-raises the urllib3 errors it does not translate, which are not RequestExceptions. except (requests.RequestException, urllib3.exceptions.HTTPError): # Any other failure of the request, e.g. a malformed response. A requests exception, like those above, # holds the request. - raise ResourceBadGateway(detail=f"The request to {remote['url']} failed.") from None + raise ResourceBadGateway() from None finally: # Once a response arrives, _stream closes the session when the response ends. if upstream is None: session.close() - return StreamResponse( + return RelayResponse( status_code=upstream.status_code, headers={name: value for name, value in upstream.headers.items() if name.lower() in _RESPONSE_HEADERS}, content=_UpstreamStream(session, upstream, self.chunk_size), @@ -214,27 +279,6 @@ def close(self) -> None: self.session.close() -def _content_length(headers: CaseInsensitiveDict[str]) -> int | None: - raw = headers.get("Content-Length") - if raw is None: - return None - # isdigit alone also accepts digits such as "²", which int() refuses, and "٣", which it reads as 3. - if not (raw.isascii() and raw.isdigit()): - raise RpcMalformedRequestHeaders(detail="Could not parse Content-Length header") - return int(raw) - - -def _request_body( - body: Readable | None, content_length: int | None, chunk_size: int -) -> "Iterator[bytes] | _SizedBody | None": - if body is None or content_length == 0: - return None - if content_length is None: - # requests sends a generator with chunked transfer encoding, the way git sends a large push. - return _read_until_eof(body, chunk_size) - return _SizedBody(body, content_length, chunk_size) - - def _read_until_eof(body: Readable, chunk_size: int) -> Iterator[bytes]: while chunk := _read(body, chunk_size): yield chunk diff --git a/src/scm/git/server.py b/src/scm/git/server.py index 94bb5389..ba984de5 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -1,11 +1,10 @@ -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Mapping from requests.structures import CaseInsensitiveDict from scm.errors import ErrorCode, GitProviderNotSupported, GitTokenRejected, GitTokenUnavailable, SCMCodedError from scm.git.grant import verify_grant -from scm.git.operations import check_request -from scm.git.relay import GitRelay +from scm.git.relay import GitRelay, ResponseBody, check_request from scm.git.types import GRANT_HEADER, GetToken, Readable from scm.helpers import initialize_provider from scm.rpc.types import StreamResponse @@ -54,15 +53,17 @@ class GitServer: The repository's provider says where its git endpoint is, through ``GetGitRemoteProtocol``. A repository whose provider has no git remote is refused. - Refusals and provider failures come back as plain-text responses. Any other exception, such as an adapter - refusing a host or the token supplier failing, reaches the caller unchanged, so the caller can answer it the - way it answers the same error elsewhere. + Refusals and provider failures come back as plain-text responses, and each is passed to ``emit_error`` as + well: the client only shows the message, so the server reports it rather than count on every client to. A + client closing its request is not reported, since there is nothing to fix. Any other exception, such as an + adapter refusing a host or the token supplier failing, reaches the caller unchanged, so the caller can answer + it the way it answers the same error elsewhere. A relayed request is counted when its response stream ends, not when its headers arrive, so a clone that fails halfway is counted as failed: ``stream_interrupted`` when the provider's stream breaks and ``client_closed_request`` when the client goes away first. A response the provider sends with a status other than 2xx is passed on to the client but counted as failed too, as ``provider_error``. The bytes sent each way - are counted then too. + are counted then too, as ``request_size`` and ``response_size``. """ def __init__( @@ -72,6 +73,7 @@ def __init__( fetch_provider: Callable[[int, Repository], Provider | None], get_token: GetToken, record_count: Callable[[str, int, dict[str, str]], None], + emit_error: Callable[[Exception], None], relay: GitRelay, ) -> None: # Fail when the server is built rather than on every request, because no grant could ever verify. @@ -82,6 +84,7 @@ def __init__( self.fetch_provider = fetch_provider self.get_token = get_token self.record_count = record_count + self.emit_error = emit_error self.relay = relay def handle( @@ -97,7 +100,10 @@ def handle( """Answer one git request. ``route`` is the path below ``<repo>.git/`` and ``query`` the raw query string. ``body`` is read only - while the request is sent upstream, so pass the unread request stream. + while the request is sent upstream, so pass the unread request stream. It must yield the body already + decoded from its transfer encoding, ending where the body ends. A Django request under WSGI reads a chunked + body as empty, and git sends a push over about 1 MiB chunked, so a Django view needs to pass a stream that + has the whole body. """ counted_body = _CountingReader(body) if body is not None else None try: @@ -124,10 +130,12 @@ def handle( if response.status_code == 401: # The provider rejected the token Sentry supplied. Passed on, the 401 would make git ask the client # for credentials, which it must never have and which would not help. - _close_stream(response.content) + response.content.close() raise GitTokenRejected(detail="The provider rejected Sentry's credentials for this repository.") except SCMCodedError as e: self.record_count("sentry.scm.git.request.failed", 1, {"code": e.code}) + if e.code != "client_closed_request": + self.emit_error(e) return _error_response(e) except Exception: self.record_count("sentry.scm.git.request.failed", 1, {"code": "unhandled_exception"}) @@ -135,7 +143,7 @@ def handle( tags: dict[str, str] = {"operation": operation, "provider": repository["provider_name"]} - def finish(failure: str | None, response_bytes: int) -> None: + def finish(failure: str | None, bytes_sent: int) -> None: if failure is None and not 200 <= response.status_code < 300: # The provider refused the request itself, e.g. a 404 for a repository the token cannot see. git # shows the client the provider's message, but did not get what it asked for. @@ -147,9 +155,9 @@ def finish(failure: str | None, response_bytes: int) -> None: "sentry.scm.git.request.failed", 1, tags | {"status": str(response.status_code), "code": failure} ) if counted_body is not None and counted_body.bytes_read: - self.record_count("sentry.scm.git.request_bytes", counted_body.bytes_read, tags) - if response_bytes: - self.record_count("sentry.scm.git.response_bytes", response_bytes, tags) + self.record_count("sentry.scm.git.request_size", counted_body.bytes_read, tags) + if bytes_sent: + self.record_count("sentry.scm.git.response_size", bytes_sent, tags) return StreamResponse( status_code=response.status_code, @@ -174,7 +182,7 @@ def read(self, size: int = -1, /) -> bytes: class _MeteredStream: """The relay's response stream, reporting once how it ended and how many bytes it passed on.""" - def __init__(self, content: Iterator[bytes], finish: Callable[[str | None, int], None]) -> None: + def __init__(self, content: ResponseBody, finish: Callable[[str | None, int], None]) -> None: self.content = content self.finish = finish self.bytes_sent = 0 @@ -198,7 +206,8 @@ def __next__(self) -> bytes: def close(self) -> None: # The server calls this after the last chunk too, when the stream has already ended. self._end("client_closed_request") - _close_stream(self.content) + # Closing the relay's stream before it is read to the end drops the upstream connection. + self.content.close() def _end(self, failure: str | None) -> None: if not self.finished: @@ -206,13 +215,6 @@ def _end(self, failure: str | None) -> None: self.finish(failure, self.bytes_sent) -def _close_stream(content: Iterator[bytes]) -> None: - # Closing the relay's stream before it is read to the end drops the upstream connection. - close = getattr(content, "close", None) - if close is not None: - close() - - def _error_response(error: SCMCodedError) -> StreamResponse: # A code missing from the map comes from a failure nobody planned for, which is the server's fault. status = _STATUS.get(error.code, 500) diff --git a/src/scm/git/types.py b/src/scm/git/types.py index 9f998b76..fe342f21 100644 --- a/src/scm/git/types.py +++ b/src/scm/git/types.py @@ -6,7 +6,7 @@ from scm.types import Repository, RepositoryId type GitOperation = Literal["fetch", "push"] -"""What a grant lets its holder do. ``scm.git.operations`` maps each to its git routes.""" +"""What a grant lets its holder do. ``scm.git.relay`` maps each to its git routes.""" GRANT_VERSION: Literal[1] = 1 @@ -29,7 +29,7 @@ class Grant(msgspec.Struct, frozen=True, kw_only=True, forbid_unknown_fields=Tru organization_id: int operation: GitOperation - repositories: tuple[RepositoryId, ...] + repositories: list[RepositoryId] expires_at: int """Unix time in seconds. Every grant expires.""" version: Literal[1] = GRANT_VERSION @@ -37,20 +37,10 @@ class Grant(msgspec.Struct, frozen=True, kw_only=True, forbid_unknown_fields=Tru type GetToken = Callable[[Repository, GitOperation], str | None] -"""Return a provider token that can run ``operation`` on ``repository``, or None if there is none. - -The git module never builds credentials itself: Sentry's integration clients supply them today, and a central -token mint can later without a change here. The operation lets the supplier scope the token, e.g. read-only for a -fetch. -""" +"""Return a provider token that can run ``operation`` on ``repository``, or None if there is none.""" class Readable(Protocol): - """A request body the relay pulls from as it sends upstream. - - It must yield the body already decoded from its transfer encoding, ending where the body ends. A Django - request under WSGI reads a chunked body as empty, and git sends a push over about 1 MiB chunked, so a view - needs to pass a stream that has the whole body. - """ + """A request body the relay pulls from as it sends upstream.""" def read(self, size: int = -1, /) -> bytes: ... diff --git a/tests/unit/git_builders.py b/tests/unit/git_builders.py index 57f58c8c..1f158127 100644 --- a/tests/unit/git_builders.py +++ b/tests/unit/git_builders.py @@ -46,7 +46,7 @@ def make_grant(**overrides) -> Grant: fields = { "organization_id": 1, "operation": "fetch", - "repositories": (5, ("github", "123")), + "repositories": [5, ("github", "123")], "expires_at": FAR_FUTURE, } fields.update(overrides) diff --git a/tests/unit/test_git_grant.py b/tests/unit/test_git_grant.py index d4a8dd0a..2d3f4b49 100644 --- a/tests/unit/test_git_grant.py +++ b/tests/unit/test_git_grant.py @@ -1,5 +1,6 @@ import base64 import hashlib +import time import jwt import msgspec @@ -10,7 +11,8 @@ from scm.git.types import Grant from tests.unit.git_builders import FAR_FUTURE, make_grant -NOW = 1_800_000_000 +# Before the real time: PyJWT checks iat against the real clock, so a grant must not be issued after it. +NOW = 1_700_000_000 # HS256 secrets of at least 32 bytes, as PyJWT recommends. SECRET = "test-secret-0000000000000000000000" OLD_SECRET = "test-secret-old-000000000000000000" @@ -79,16 +81,20 @@ def test_valid_until_its_expiry_time(self): token = sign_grant(SECRET, make_grant(expires_at=NOW + 1), now=NOW) assert verify_grant([SECRET], token, now=NOW).expires_at == NOW + 1 + # PyJWT checks iat against the real clock, so these sign against it too. def test_issued_in_the_future(self): - assert_invalid(sign_grant(SECRET, make_grant(), now=NOW + 61), "issued in the future") + assert_invalid(sign_grant(SECRET, make_grant(), now=time.time() + 600), "not yet valid (iat)") def test_issued_slightly_in_the_future(self): # The signer's clock may run a little ahead of the verifier's. - assert verify_grant([SECRET], sign_grant(SECRET, make_grant(), now=NOW + 60), now=NOW) == make_grant() + assert verify_grant([SECRET], sign_grant(SECRET, make_grant(), now=time.time() + 30), now=NOW) == make_grant() - @pytest.mark.parametrize("iat", [float(NOW), str(NOW), True]) - def test_iat_must_be_an_integer(self, iat): - assert_invalid(sign_claims(claims_of(make_grant()) | {"iat": iat}), "iat is not an integer") + @pytest.mark.parametrize("iat", [float(NOW), NOW + 0.5]) + def test_iat_may_be_a_float(self, iat): + assert verify_grant([SECRET], sign_claims(claims_of(make_grant()) | {"iat": iat}), now=NOW) == make_grant() + + def test_iat_must_be_a_number(self): + assert_invalid(sign_claims(claims_of(make_grant()) | {"iat": "yesterday"}), "iat") @pytest.mark.parametrize("claim", ["exp", "iat", "aud"]) def test_jwt_claims_are_required(self, claim): @@ -190,4 +196,4 @@ def test_signed_but_invalid_claims(self, change): assert_invalid(sign_claims(claims_of(make_grant()) | change), "payload is invalid") def test_no_repositories(self): - assert_invalid(sign_grant(SECRET, make_grant(repositories=()), now=NOW), "names no repositories") + assert_invalid(sign_grant(SECRET, make_grant(repositories=[]), now=NOW), "names no repositories") diff --git a/tests/unit/test_git_operations.py b/tests/unit/test_git_operations.py deleted file mode 100644 index cf73827b..00000000 --- a/tests/unit/test_git_operations.py +++ /dev/null @@ -1,91 +0,0 @@ -import pytest - -from scm.errors import GitOperationNotGranted, GitRepositoryNotGranted, GitRouteNotSupported -from scm.git.operations import check_request, require_route, route_operation -from tests.unit.git_builders import make_grant - -FETCH_ROUTES = [("GET", "info/refs", "service=git-upload-pack"), ("POST", "git-upload-pack", "")] -PUSH_ROUTES = [("GET", "info/refs", "service=git-receive-pack"), ("POST", "git-receive-pack", "")] - - -class TestRouteOperation: - @pytest.mark.parametrize( - "method,route,query,operation", - [ - ("GET", "info/refs", "service=git-upload-pack", "fetch"), - ("POST", "git-upload-pack", "", "fetch"), - ("GET", "info/refs", "service=git-receive-pack", "push"), - ("POST", "git-receive-pack", "", "push"), - ], - ) - def test_git_routes(self, method, route, query, operation): - assert route_operation(method, route, query) == operation - - @pytest.mark.parametrize( - "method,route,query", - [ - ("GET", "info/refs", ""), - ("GET", "info/refs", "service=git-upload-pack&x=1"), - ("GET", "info/refs", "service=git-upload-archive"), - ("POST", "info/refs", "service=git-upload-pack"), - ("GET", "git-upload-pack", ""), - ("POST", "git-upload-pack", "service=git-upload-pack"), - ("post", "git-upload-pack", ""), - ("POST", "git-upload-archive", ""), - ("POST", "info/lfs/objects/batch", ""), - ("GET", "HEAD", ""), - ("GET", "objects/info/packs", ""), - ("GET", "../info/refs", "service=git-upload-pack"), - ("POST", "/git-upload-pack", ""), - ], - ) - def test_other_routes(self, method, route, query): - assert route_operation(method, route, query) is None - - -class TestRequireRoute: - def test_returns_the_operation(self): - assert require_route("POST", "git-receive-pack", "") == "push" - - def test_refuses_other_routes(self): - with pytest.raises(GitRouteNotSupported, match="not a git fetch or push route"): - require_route("POST", "info/lfs/objects/batch", "") - - -class TestCheckRequest: - @pytest.mark.parametrize( - "operation,route", - [("fetch", route) for route in FETCH_ROUTES] + [("push", route) for route in PUSH_ROUTES], - ) - def test_grant_reaches_its_operations_routes(self, operation, route): - assert check_request(make_grant(operation=operation), 5, *route) == operation - - @pytest.mark.parametrize( - "operation,route", - [("fetch", route) for route in PUSH_ROUTES] + [("push", route) for route in FETCH_ROUTES], - ) - def test_grant_cannot_reach_the_other_operations_routes(self, operation, route): - other = "push" if operation == "fetch" else "fetch" - with pytest.raises(GitOperationNotGranted, match=f"allows {operation}, not {other}"): - check_request(make_grant(operation=operation), 5, *route) - - def test_unknown_route(self): - with pytest.raises(GitRouteNotSupported, match="not a git fetch or push route"): - check_request(make_grant(), 5, "POST", "info/lfs/objects/batch", "") - - def test_repository_not_in_grant(self): - with pytest.raises(GitRepositoryNotGranted, match="does not cover this repository"): - check_request(make_grant(), 6, "POST", "git-upload-pack", "") - - def test_repository_named_another_way_is_not_in_grant(self): - # The grant and the request name repositories the same way; matching a database id to a provider id - # would mean resolving every repository the grant lists. - grant = make_grant(repositories=(("github", "123"),)) - with pytest.raises(GitRepositoryNotGranted): - check_request(grant, 5, "POST", "git-upload-pack", "") - assert check_request(grant, ("github", "123"), "POST", "git-upload-pack", "") == "fetch" - - def test_any_listed_repository(self): - grant = make_grant(repositories=(5, 6, ("gitlab", "gitlab.com:7"))) - assert check_request(grant, 6, "POST", "git-upload-pack", "") == "fetch" - assert check_request(grant, ("gitlab", "gitlab.com:7"), "POST", "git-upload-pack", "") == "fetch" diff --git a/tests/unit/test_git_relay.py b/tests/unit/test_git_relay.py index 2c6f8d76..b3a7e167 100644 --- a/tests/unit/test_git_relay.py +++ b/tests/unit/test_git_relay.py @@ -11,7 +11,9 @@ from requests.adapters import HTTPAdapter from scm.errors import ( + GitOperationNotGranted, GitRemoteNotHttps, + GitRepositoryNotGranted, GitRouteNotSupported, ResourceBadGateway, ResourceCertificateInvalid, @@ -19,8 +21,9 @@ ResourceServiceUnavailable, SCMCodedError, ) -from scm.git.relay import GitRelay +from scm.git.relay import GitRelay, check_request from scm.types import GitRemote +from tests.unit.git_builders import make_grant from tests.unit.git_upstream import ClosingAdapter, UpstreamHandler, basic_auth, ca_bundle, running_upstream TOKEN = "ghs_secrettoken" @@ -136,7 +139,7 @@ def test_refuses_plain_http(self, relay, upstream): remote = GitRemote( url=upstream.url.replace("https://", "http://") + "/repo.git", basic_auth_username="x-access-token" ) - with pytest.raises(GitRemoteNotHttps, match="not https"): + with pytest.raises(GitRemoteNotHttps, match="not an https URL"): fetch_refs(relay, remote=remote) assert upstream.requests == [] @@ -218,7 +221,7 @@ def read(self, size: int = -1, /) -> bytes: upload_pack(relay, upstream, ResettingClient(), headers=headers) assert exc_info.value.code == "client_closed_request" - @pytest.mark.parametrize("content_length", ["abc", "-1", "1.5", "²", "٣"]) + @pytest.mark.parametrize("content_length", ["abc", "-1", "1.5", "²", ""]) def test_malformed_content_length(self, relay, upstream, content_length): with pytest.raises(SCMCodedError) as exc_info: upload_pack(relay, upstream, io.BytesIO(b"0000"), headers={"Content-Length": content_length}) @@ -438,9 +441,11 @@ def test_connection_refused(self, relay): with pytest.raises(ResourceServiceUnavailable) as exc_info: fetch_refs(relay, remote=remote) assert_token_not_in(exc_info.value) + # The client has no use for the provider's address. + assert "127.0.0.1" not in exc_info.value.detail def test_untrusted_certificate(self, upstream): - with pytest.raises(ResourceCertificateInvalid, match="TLS certificate of 127.0.0.1") as exc_info: + with pytest.raises(ResourceCertificateInvalid, match="TLS certificate") as exc_info: fetch_refs(GitRelay(), upstream) assert_token_not_in(exc_info.value) assert upstream.requests == [] @@ -475,7 +480,7 @@ class FailingAdapter(HTTPAdapter): def send(self, request, *args, **kwargs): raise make_error(request) - with pytest.raises(ResourceBadGateway, match="failed") as exc_info: + with pytest.raises(ResourceBadGateway) as exc_info: fetch_refs(GitRelay(adapter=FailingAdapter, ca_bundle=ca_bundle()), upstream) assert_token_not_in(exc_info.value) @@ -528,3 +533,65 @@ def handler(h: UpstreamHandler) -> None: response = fetch_refs(relay, upstream) response.content.close() # type: ignore[attr-defined] assert upstream_saw_close.wait(WAIT) + + +FETCH_ROUTES = [("GET", "info/refs", "service=git-upload-pack"), ("POST", "git-upload-pack", "")] +PUSH_ROUTES = [("GET", "info/refs", "service=git-receive-pack"), ("POST", "git-receive-pack", "")] + + +class TestCheckRequest: + @pytest.mark.parametrize( + "operation,route", + [("fetch", route) for route in FETCH_ROUTES] + [("push", route) for route in PUSH_ROUTES], + ) + def test_grant_reaches_its_operations_routes(self, operation, route): + assert check_request(make_grant(operation=operation), 5, *route) == operation + + @pytest.mark.parametrize( + "operation,route", + [("fetch", route) for route in PUSH_ROUTES] + [("push", route) for route in FETCH_ROUTES], + ) + def test_grant_cannot_reach_the_other_operations_routes(self, operation, route): + other = "push" if operation == "fetch" else "fetch" + with pytest.raises(GitOperationNotGranted, match=f"allows {operation}, not {other}"): + check_request(make_grant(operation=operation), 5, *route) + + @pytest.mark.parametrize( + "method,route,query", + [ + ("GET", "info/refs", ""), + ("GET", "info/refs", "service=git-upload-pack&x=1"), + ("GET", "info/refs", "service=git-upload-archive"), + ("POST", "info/refs", "service=git-upload-pack"), + ("GET", "git-upload-pack", ""), + ("POST", "git-upload-pack", "service=git-upload-pack"), + ("post", "git-upload-pack", ""), + ("POST", "git-upload-archive", ""), + ("POST", "info/lfs/objects/batch", ""), + ("GET", "HEAD", ""), + ("GET", "objects/info/packs", ""), + ("GET", "../info/refs", "service=git-upload-pack"), + ("POST", "/git-upload-pack", ""), + ], + ) + def test_routes_that_are_not_git(self, method, route, query): + for operation in ("fetch", "push"): + with pytest.raises(GitRouteNotSupported, match="not a git fetch or push route"): + check_request(make_grant(operation=operation), 5, method, route, query) + + def test_repository_not_in_grant(self): + with pytest.raises(GitRepositoryNotGranted, match="does not cover this repository"): + check_request(make_grant(), 6, "POST", "git-upload-pack", "") + + def test_repository_named_another_way_is_not_in_grant(self): + # The grant and the request name repositories the same way; matching a database id to a provider id + # would mean resolving every repository the grant lists. + grant = make_grant(repositories=[("github", "123")]) + with pytest.raises(GitRepositoryNotGranted): + check_request(grant, 5, "POST", "git-upload-pack", "") + assert check_request(grant, ("github", "123"), "POST", "git-upload-pack", "") == "fetch" + + def test_any_listed_repository(self): + grant = make_grant(repositories=[5, 6, ("gitlab", "gitlab.com:7")]) + assert check_request(grant, 6, "POST", "git-upload-pack", "") == "fetch" + assert check_request(grant, ("gitlab", "gitlab.com:7"), "POST", "git-upload-pack", "") == "fetch" diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index 97760d19..2da9bedd 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -1,14 +1,14 @@ import io import time -from collections.abc import Iterator +from collections.abc import Generator from types import SimpleNamespace from unittest.mock import MagicMock import pytest -from scm.errors import RateLimitExceeded +from scm.errors import ClientClosedRequest, RateLimitExceeded from scm.git.grant import sign_grant -from scm.git.relay import GitRelay +from scm.git.relay import GitRelay, RelayResponse, ResponseBody from scm.git.server import _STATUS, GitServer from scm.git.types import GRANT_HEADER, GitOperation, Readable from scm.rpc.types import StreamResponse @@ -32,6 +32,7 @@ def __init__(self, repository: Repository | None = None, token: str | None = TOK self.provider_lookups: list[tuple[int, Repository]] = [] self.token_lookups: list[tuple[Repository, GitOperation]] = [] self.counts: list[tuple[str, dict[str, str]]] = [] + self.errors: list[Exception] = [] self.bytes: dict[tuple[str, str], int] = {} def fetch_repository(self, organization_id: int, repository_id: RepositoryId) -> Repository | None: @@ -47,29 +48,42 @@ def get_token(self, repository: Repository, operation: GitOperation) -> str | No return self.token def record_count(self, name: str, value: int, tags: dict[str, str]) -> None: - if name.endswith("_bytes"): + if name.endswith("_size"): key = (name, tags["operation"]) self.bytes[key] = self.bytes.get(key, 0) + value else: self.counts.append((name, tags)) + def emit_error(self, error: Exception) -> None: + self.errors.append(error) -def relay_returning(content: Iterator[bytes], status_code: int = 200) -> GitRelay: + @property + def error_codes(self) -> list[str]: + return [getattr(error, "code", "") for error in self.errors] + + +def chunks(*parts: bytes) -> Generator[bytes]: + """A provider response body, which, like the relay's, can be closed.""" + yield from parts + + +def relay_returning(content: ResponseBody, status_code: int = 200) -> GitRelay: """A stand-in relay whose every forward returns ``content`` as the provider's response.""" relay = MagicMock(spec=GitRelay) - relay.forward.return_value = StreamResponse(status_code=status_code, headers={}, content=content) + relay.forward.return_value = RelayResponse(status_code=status_code, headers={}, content=content) return relay def make_server(recorder: Recorder, relay: GitRelay | None = None) -> GitServer: if relay is None: - relay = relay_returning(iter([b"refs"])) + relay = relay_returning(chunks(b"refs")) return GitServer( secrets=[SECRET], fetch_repository=recorder.fetch_repository, fetch_provider=recorder.fetch_provider, get_token=recorder.get_token, record_count=recorder.record_count, + emit_error=recorder.emit_error, relay=relay, ) @@ -118,6 +132,7 @@ def test_requires_a_relay(): fetch_provider=lambda organization_id, repository: None, get_token=lambda repository, operation: None, record_count=lambda name, value, tags: None, + emit_error=lambda error: None, ) @@ -130,6 +145,7 @@ def test_requires_non_empty_secrets(secrets): fetch_provider=lambda organization_id, repository: None, get_token=lambda repository, operation: None, record_count=lambda name, value, tags: None, + emit_error=lambda error: None, relay=MagicMock(spec=GitRelay), ) @@ -159,6 +175,7 @@ def test_invalid_grant(self, grant, detail, code): assert recorder.token_lookups == [] server.relay.forward.assert_not_called() # type: ignore[attr-defined] assert recorder.counts == [("sentry.scm.git.request.failed", {"code": code})] + assert recorder.error_codes == [code] def test_route_that_is_not_git(self): recorder = Recorder() @@ -267,7 +284,7 @@ def test_counted_when_the_response_ends_not_when_it_starts(self): assert recorder.counts == [ ("sentry.scm.git.request.succeeded", {"operation": "fetch", "provider": "github", "status": "200"}) ] - assert recorder.bytes == {("sentry.scm.git.response_bytes", "fetch"): 4} + assert recorder.bytes == {("sentry.scm.git.response_size", "fetch"): 4} def test_provider_stream_breaking_is_a_failure(self): def broken_stream(): @@ -285,7 +302,7 @@ def broken_stream(): {"operation": "fetch", "provider": "github", "status": "200", "code": "stream_interrupted"}, ) ] - assert recorder.bytes == {("sentry.scm.git.response_bytes", "fetch"): 11} + assert recorder.bytes == {("sentry.scm.git.response_size", "fetch"): 11} def test_client_leaving_early_is_a_failure_and_closes_the_provider_stream(self): closed = [] @@ -314,7 +331,8 @@ def test_upstream_unavailable(self): repository=make_repository(provider_name="github_enterprise", web_base_url="https://127.0.0.1:1") ) response = request(make_server(recorder, GitRelay(timeout=(1, 1))), signed_grant()) - assert_refused(response, 503, "Could not connect") + assert_refused(response, 503, "temporarily unavailable") + assert "127.0.0.1" not in b"".join(response.content).decode() assert TOKEN not in b"".join(response.content).decode() def test_plain_http_remote(self): @@ -323,7 +341,7 @@ def test_plain_http_remote(self): repository=make_repository(provider_name="github_enterprise", web_base_url="http://127.0.0.1:1") ) response = request(make_server(recorder, GitRelay()), signed_grant()) - assert_refused(response, 502, "not https") + assert_refused(response, 502, "not an https URL") assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_remote_not_https"})] def test_provider_rejecting_the_token(self): @@ -345,13 +363,14 @@ def adapter() -> ClosingAdapter: # The provider's unread response is closed with its connection. assert [a.closed for a in adapters] == [True] assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_token_rejected"})] + assert recorder.error_codes == ["git_token_rejected"] @pytest.mark.parametrize("status", [301, 403, 404, 500]) def test_provider_refusing_the_request_is_passed_on_and_counted_as_failed(self, status): # e.g. GitHub's 404 for a repository the installation cannot see; git shows the client its message. recorder = Recorder() response = request( - make_server(recorder, relay_returning(iter([b"Repository not found."]), status)), signed_grant() + make_server(recorder, relay_returning(chunks(b"Repository not found."), status)), signed_grant() ) assert response.status_code == status assert b"".join(response.content) == b"Repository not found." @@ -363,6 +382,14 @@ def test_provider_refusing_the_request_is_passed_on_and_counted_as_failed(self, ) ] + def test_client_closing_its_request_is_counted_but_not_reported(self): + relay = MagicMock(spec=GitRelay) + relay.forward.side_effect = ClientClosedRequest() + recorder = Recorder() + assert_refused(request(make_server(recorder, relay), signed_grant()), 499, "") + assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "client_closed_request"})] + assert recorder.errors == [] + def test_unexpected_error_code_is_a_server_error(self): relay = MagicMock(spec=GitRelay) relay.forward.side_effect = RateLimitExceeded() @@ -383,6 +410,8 @@ class HostRefused(Exception): with pytest.raises(HostRefused): request(make_server(recorder, relay), signed_grant()) assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "unhandled_exception"})] + # The caller reports it. + assert recorder.errors == [] def test_counts_bytes_each_way_end_to_end(self): with running_upstream() as upstream: @@ -401,8 +430,8 @@ def test_counts_bytes_each_way_end_to_end(self): ) assert b"".join(response.content) == b"00000000" # The fake upstream echoes the body. assert recorder.bytes == { - ("sentry.scm.git.request_bytes", "push"): 8, - ("sentry.scm.git.response_bytes", "push"): 8, + ("sentry.scm.git.request_size", "push"): 8, + ("sentry.scm.git.response_size", "push"): 8, } def test_relays_end_to_end(self): From 78833cbe57802a286774ba2b4104f22fee8bc162 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos <dlugos.ivan@gmail.com> Date: Wed, 21 Oct 2026 18:30:48 +0200 Subject: [PATCH 28/31] feat(git): Report git errors with what the request had resolved The messages GitServer reports no longer name the provider's address, and emit_error got only the exception, so an event in Sentry could not say which organization, repository, provider or host a failure was about. The provider and host are resolved inside handle, where the caller cannot see them. emit_error now also gets them as tags, each once it is known, so a bad grant reports only the requested repository while a relay failure reports all of them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/scm/git/server.py | 18 ++++++++++++++---- tests/unit/test_git_server.py | 31 +++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/scm/git/server.py b/src/scm/git/server.py index ba984de5..495ecb71 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -1,4 +1,5 @@ from collections.abc import Callable, Mapping +from urllib.parse import urlsplit from requests.structures import CaseInsensitiveDict @@ -54,8 +55,11 @@ class GitServer: provider has no git remote is refused. Refusals and provider failures come back as plain-text responses, and each is passed to ``emit_error`` as - well: the client only shows the message, so the server reports it rather than count on every client to. A - client closing its request is not reported, since there is nothing to fix. Any other exception, such as an + well: the client only shows the message, so the server reports it rather than count on every client to. The + message leaves out where the request was going, so ``emit_error`` also gets what the request had resolved + when it failed, as tags: ``repository_id`` as requested, then ``organization_id`` once the grant is verified, + ``operation``, ``provider`` and ``remote_host``. A tag is missing when the request failed before it was known. + A client closing its request is not reported, since there is nothing to fix. Any other exception, such as an adapter refusing a host or the token supplier failing, reaches the caller unchanged, so the caller can answer it the way it answers the same error elsewhere. @@ -73,7 +77,7 @@ def __init__( fetch_provider: Callable[[int, Repository], Provider | None], get_token: GetToken, record_count: Callable[[str, int, dict[str, str]], None], - emit_error: Callable[[Exception], None], + emit_error: Callable[[Exception, dict[str, str]], None], relay: GitRelay, ) -> None: # Fail when the server is built rather than on every request, because no grant could ever verify. @@ -106,9 +110,12 @@ def handle( has the whole body. """ counted_body = _CountingReader(body) if body is not None else None + context = {"repository_id": str(repository_id)} try: grant = verify_grant(self.secrets, CaseInsensitiveDict(headers).get(GRANT_HEADER, "")) + context["organization_id"] = str(grant.organization_id) operation = check_request(grant, repository_id, method, route, query) + context["operation"] = operation provider = initialize_provider( grant.organization_id, repository_id, @@ -116,9 +123,12 @@ def handle( fetch_provider=self.fetch_provider, ) repository = provider.repository + context["provider"] = repository["provider_name"] if not isinstance(provider, GetGitRemoteProtocol): raise GitProviderNotSupported(detail=f"The {repository['provider_name']} provider has no git remote.") remote = provider.get_git_remote() + # The host and port only: a URL can carry credentials before the host. + context["remote_host"] = urlsplit(remote["url"]).netloc.rpartition("@")[2] token = self.get_token(repository, operation) if token is None: raise GitTokenUnavailable( @@ -135,7 +145,7 @@ def handle( except SCMCodedError as e: self.record_count("sentry.scm.git.request.failed", 1, {"code": e.code}) if e.code != "client_closed_request": - self.emit_error(e) + self.emit_error(e, context) return _error_response(e) except Exception: self.record_count("sentry.scm.git.request.failed", 1, {"code": "unhandled_exception"}) diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index 2da9bedd..8512742f 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -3,6 +3,7 @@ from collections.abc import Generator from types import SimpleNamespace from unittest.mock import MagicMock +from urllib.parse import urlsplit import pytest @@ -32,7 +33,7 @@ def __init__(self, repository: Repository | None = None, token: str | None = TOK self.provider_lookups: list[tuple[int, Repository]] = [] self.token_lookups: list[tuple[Repository, GitOperation]] = [] self.counts: list[tuple[str, dict[str, str]]] = [] - self.errors: list[Exception] = [] + self.errors: list[tuple[Exception, dict[str, str]]] = [] self.bytes: dict[tuple[str, str], int] = {} def fetch_repository(self, organization_id: int, repository_id: RepositoryId) -> Repository | None: @@ -54,12 +55,12 @@ def record_count(self, name: str, value: int, tags: dict[str, str]) -> None: else: self.counts.append((name, tags)) - def emit_error(self, error: Exception) -> None: - self.errors.append(error) + def emit_error(self, error: Exception, context: dict[str, str]) -> None: + self.errors.append((error, context)) @property def error_codes(self) -> list[str]: - return [getattr(error, "code", "") for error in self.errors] + return [getattr(error, "code", "") for error, _ in self.errors] def chunks(*parts: bytes) -> Generator[bytes]: @@ -132,7 +133,7 @@ def test_requires_a_relay(): fetch_provider=lambda organization_id, repository: None, get_token=lambda repository, operation: None, record_count=lambda name, value, tags: None, - emit_error=lambda error: None, + emit_error=lambda error, context: None, ) @@ -145,7 +146,7 @@ def test_requires_non_empty_secrets(secrets): fetch_provider=lambda organization_id, repository: None, get_token=lambda repository, operation: None, record_count=lambda name, value, tags: None, - emit_error=lambda error: None, + emit_error=lambda error, context: None, relay=MagicMock(spec=GitRelay), ) @@ -176,6 +177,8 @@ def test_invalid_grant(self, grant, detail, code): server.relay.forward.assert_not_called() # type: ignore[attr-defined] assert recorder.counts == [("sentry.scm.git.request.failed", {"code": code})] assert recorder.error_codes == [code] + # Only the requested repository is known before the grant is verified. + assert recorder.errors[0][1] == {"repository_id": "5"} def test_route_that_is_not_git(self): recorder = Recorder() @@ -192,6 +195,7 @@ def test_operation_the_grant_does_not_allow(self): assert recorder.repository_lookups == [] assert recorder.token_lookups == [] assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_operation_not_granted"})] + assert recorder.errors[0][1] == {"repository_id": "5", "organization_id": "1"} def test_repository_the_grant_does_not_list(self): recorder = Recorder() @@ -335,6 +339,14 @@ def test_upstream_unavailable(self): assert "127.0.0.1" not in b"".join(response.content).decode() assert TOKEN not in b"".join(response.content).decode() + def test_reported_remote_host_leaves_out_credentials_in_the_url(self): + recorder = Recorder( + repository=make_repository(provider_name="github_enterprise", web_base_url="https://user:pw@127.0.0.1:1") + ) + request(make_server(recorder, GitRelay(timeout=(1, 1))), signed_grant()) + assert recorder.error_codes == ["resource_service_unavailable"] + assert recorder.errors[0][1]["remote_host"] == "127.0.0.1:1" + def test_plain_http_remote(self): # A self-managed instance served over http; the token must not cross the network in the clear. recorder = Recorder( @@ -364,6 +376,13 @@ def adapter() -> ClosingAdapter: assert [a.closed for a in adapters] == [True] assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_token_rejected"})] assert recorder.error_codes == ["git_token_rejected"] + assert recorder.errors[0][1] == { + "repository_id": "5", + "organization_id": "1", + "operation": "fetch", + "provider": "github_enterprise", + "remote_host": urlsplit(upstream.url).netloc, + } @pytest.mark.parametrize("status", [301, 403, 404, 500]) def test_provider_refusing_the_request_is_passed_on_and_counted_as_failed(self, status): From cd8d7050a122678809941e13062510bf86c4a18a Mon Sep 17 00:00:00 2001 From: Ivan Dlugos <dlugos.ivan@gmail.com> Date: Thu, 22 Oct 2026 08:25:14 +0200 Subject: [PATCH 29/31] feat(git): Add a referrer to the git grant scm-platform threads a referrer through every action, so Sentry can tell callers apart in metrics and, later, in per-caller limits and permissions. RpcServer already takes one, from the unsigned X-Referrer header, defaulting to "shared". Git requests carry none, so Sentry could not tell a coding job's clone from Seer's own fetch or push. The grant is the only thing a git request carries from its signer, so the referrer goes there, and GitServer tags its counts and reported errors with it. It is required rather than defaulting to "shared": Seer is the only signer, and a grant that does not say who it is for would defeat the point. Grant refuses unknown fields and versions, so adding a field after the first release would need grant version 2. This is unreleased, so adding it now costs nothing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/scm/git/server.py | 21 +++++++---- src/scm/git/types.py | 6 ++- tests/unit/git_builders.py | 1 + tests/unit/test_git_grant.py | 8 ++++ tests/unit/test_git_server.py | 70 ++++++++++++++++++++++++++++------- 5 files changed, 83 insertions(+), 23 deletions(-) diff --git a/src/scm/git/server.py b/src/scm/git/server.py index 495ecb71..f43457ef 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -57,11 +57,14 @@ class GitServer: Refusals and provider failures come back as plain-text responses, and each is passed to ``emit_error`` as well: the client only shows the message, so the server reports it rather than count on every client to. The message leaves out where the request was going, so ``emit_error`` also gets what the request had resolved - when it failed, as tags: ``repository_id`` as requested, then ``organization_id`` once the grant is verified, - ``operation``, ``provider`` and ``remote_host``. A tag is missing when the request failed before it was known. - A client closing its request is not reported, since there is nothing to fix. Any other exception, such as an - adapter refusing a host or the token supplier failing, reaches the caller unchanged, so the caller can answer - it the way it answers the same error elsewhere. + when it failed, as tags: ``repository_id`` as requested, then ``organization_id`` and ``referrer`` once the + grant is verified, ``operation``, ``provider`` and ``remote_host``. A tag is missing when the request failed + before it was known. A client closing its request is not reported, since there is nothing to fix. Any other + exception, such as an adapter refusing a host or the token supplier failing, reaches the caller unchanged, so + the caller can answer it the way it answers the same error elsewhere. + + Every count is tagged with the grant's ``referrer`` once the grant is verified, so Sentry can tell a coding + job's clone from Seer's own fetch or push. A relayed request is counted when its response stream ends, not when its headers arrive, so a clone that fails halfway is counted as failed: ``stream_interrupted`` when the provider's stream breaks and @@ -111,9 +114,11 @@ def handle( """ counted_body = _CountingReader(body) if body is not None else None context = {"repository_id": str(repository_id)} + referrer_tag: dict[str, str] = {} try: grant = verify_grant(self.secrets, CaseInsensitiveDict(headers).get(GRANT_HEADER, "")) context["organization_id"] = str(grant.organization_id) + referrer_tag["referrer"] = context["referrer"] = grant.referrer operation = check_request(grant, repository_id, method, route, query) context["operation"] = operation provider = initialize_provider( @@ -143,15 +148,15 @@ def handle( response.content.close() raise GitTokenRejected(detail="The provider rejected Sentry's credentials for this repository.") except SCMCodedError as e: - self.record_count("sentry.scm.git.request.failed", 1, {"code": e.code}) + self.record_count("sentry.scm.git.request.failed", 1, {"code": e.code} | referrer_tag) if e.code != "client_closed_request": self.emit_error(e, context) return _error_response(e) except Exception: - self.record_count("sentry.scm.git.request.failed", 1, {"code": "unhandled_exception"}) + self.record_count("sentry.scm.git.request.failed", 1, {"code": "unhandled_exception"} | referrer_tag) raise - tags: dict[str, str] = {"operation": operation, "provider": repository["provider_name"]} + tags: dict[str, str] = {"operation": operation, "provider": repository["provider_name"]} | referrer_tag def finish(failure: str | None, bytes_sent: int) -> None: if failure is None and not 200 <= response.status_code < 300: diff --git a/src/scm/git/types.py b/src/scm/git/types.py index fe342f21..412543b2 100644 --- a/src/scm/git/types.py +++ b/src/scm/git/types.py @@ -3,7 +3,7 @@ import msgspec -from scm.types import Repository, RepositoryId +from scm.types import Referrer, Repository, RepositoryId type GitOperation = Literal["fetch", "push"] """What a grant lets its holder do. ``scm.git.relay`` maps each to its git routes.""" @@ -28,6 +28,10 @@ class Grant(msgspec.Struct, frozen=True, kw_only=True, forbid_unknown_fields=Tru """ organization_id: int + referrer: Referrer + """Who the grant is for, e.g. a coding job or Seer itself, as ``referrer`` is elsewhere in this library. + Sentry tags the git endpoint's metrics with it. Seer sets it, because git requests carry nothing else that + tells callers apart.""" operation: GitOperation repositories: list[RepositoryId] expires_at: int diff --git a/tests/unit/git_builders.py b/tests/unit/git_builders.py index 1f158127..fab2c169 100644 --- a/tests/unit/git_builders.py +++ b/tests/unit/git_builders.py @@ -45,6 +45,7 @@ def make_grant(**overrides) -> Grant: """A fetch grant for the default repository, named both ways a grant can name it.""" fields = { "organization_id": 1, + "referrer": "seer.coding_job", "operation": "fetch", "repositories": [5, ("github", "123")], "expires_at": FAR_FUTURE, diff --git a/tests/unit/test_git_grant.py b/tests/unit/test_git_grant.py index 2d3f4b49..b126a264 100644 --- a/tests/unit/test_git_grant.py +++ b/tests/unit/test_git_grant.py @@ -54,6 +54,7 @@ def test_token_is_a_jwt_with_the_grants_claims(self): assert jwt.get_unverified_header(token) == {"alg": "HS256", "typ": "JWT", "kid": key_id(SECRET)} assert jwt.decode(token, options={"verify_signature": False}) == { "organization_id": 1, + "referrer": "seer.coding_job", "operation": "fetch", "repositories": [5, ["github", "123"]], "version": 1, @@ -184,6 +185,7 @@ def test_missing(self): [ {"organization_id": True}, {"organization_id": "1"}, + {"referrer": None}, {"operation": "delete"}, {"version": 2}, {"repositories": [["github"]]}, @@ -195,5 +197,11 @@ def test_missing(self): def test_signed_but_invalid_claims(self, change): assert_invalid(sign_claims(claims_of(make_grant()) | change), "payload is invalid") + def test_referrer_is_required(self): + # Without it Sentry could not tell a coding job's clone from Seer's own fetch. + claims = claims_of(make_grant()) + del claims["referrer"] + assert_invalid(sign_claims(claims), "referrer") + def test_no_repositories(self): assert_invalid(sign_grant(SECRET, make_grant(repositories=[]), now=NOW), "names no repositories") diff --git a/tests/unit/test_git_server.py b/tests/unit/test_git_server.py index 8512742f..332221cf 100644 --- a/tests/unit/test_git_server.py +++ b/tests/unit/test_git_server.py @@ -19,6 +19,7 @@ SECRET = "test-secret-0000000000000000000000" TOKEN = "ghs_secrettoken" +REFERRER = make_grant().referrer def signed_grant(**overrides) -> str: @@ -187,21 +188,27 @@ def test_route_that_is_not_git(self): ) assert_refused(response, 404, "not a git fetch or push route") assert recorder.repository_lookups == [] - assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_route_not_supported"})] + assert recorder.counts == [ + ("sentry.scm.git.request.failed", {"referrer": REFERRER, "code": "git_route_not_supported"}) + ] def test_operation_the_grant_does_not_allow(self): recorder = Recorder() assert_refused(push(make_server(recorder), signed_grant()), 403, "allows fetch, not push") assert recorder.repository_lookups == [] assert recorder.token_lookups == [] - assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_operation_not_granted"})] - assert recorder.errors[0][1] == {"repository_id": "5", "organization_id": "1"} + assert recorder.counts == [ + ("sentry.scm.git.request.failed", {"referrer": REFERRER, "code": "git_operation_not_granted"}) + ] + assert recorder.errors[0][1] == {"repository_id": "5", "organization_id": "1", "referrer": REFERRER} def test_repository_the_grant_does_not_list(self): recorder = Recorder() assert_refused(request(make_server(recorder), signed_grant(), repository_id=6), 403, "does not cover") assert recorder.repository_lookups == [] - assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_repository_not_granted"})] + assert recorder.counts == [ + ("sentry.scm.git.request.failed", {"referrer": REFERRER, "code": "git_repository_not_granted"}) + ] def test_grant_header_name_is_case_insensitive(self): response = request(make_server(Recorder()), None, headers={GRANT_HEADER.lower(): signed_grant()}) @@ -253,7 +260,9 @@ def test_no_token(self): recorder = Recorder(token=None) response = request(make_server(recorder), signed_grant()) assert_refused(response, 502, "could not get a github token") - assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_token_unavailable"})] + assert recorder.counts == [ + ("sentry.scm.git.request.failed", {"referrer": REFERRER, "code": "git_token_unavailable"}) + ] def test_token_asked_for_the_grants_operation(self): recorder = Recorder() @@ -276,7 +285,10 @@ def test_forwards_with_remote_and_token(self): body=None, ) assert recorder.counts == [ - ("sentry.scm.git.request.succeeded", {"operation": "fetch", "provider": "github", "status": "200"}) + ( + "sentry.scm.git.request.succeeded", + {"referrer": REFERRER, "operation": "fetch", "provider": "github", "status": "200"}, + ) ] def test_counted_when_the_response_ends_not_when_it_starts(self): @@ -286,7 +298,10 @@ def test_counted_when_the_response_ends_not_when_it_starts(self): assert b"".join(response.content) == b"refs" response.content.close() # type: ignore[attr-defined] assert recorder.counts == [ - ("sentry.scm.git.request.succeeded", {"operation": "fetch", "provider": "github", "status": "200"}) + ( + "sentry.scm.git.request.succeeded", + {"referrer": REFERRER, "operation": "fetch", "provider": "github", "status": "200"}, + ) ] assert recorder.bytes == {("sentry.scm.git.response_size", "fetch"): 4} @@ -303,7 +318,13 @@ def broken_stream(): assert recorder.counts == [ ( "sentry.scm.git.request.failed", - {"operation": "fetch", "provider": "github", "status": "200", "code": "stream_interrupted"}, + { + "referrer": REFERRER, + "operation": "fetch", + "provider": "github", + "status": "200", + "code": "stream_interrupted", + }, ) ] assert recorder.bytes == {("sentry.scm.git.response_size", "fetch"): 11} @@ -326,7 +347,13 @@ def provider_stream(): assert recorder.counts == [ ( "sentry.scm.git.request.failed", - {"operation": "fetch", "provider": "github", "status": "200", "code": "client_closed_request"}, + { + "referrer": REFERRER, + "operation": "fetch", + "provider": "github", + "status": "200", + "code": "client_closed_request", + }, ) ] @@ -354,7 +381,9 @@ def test_plain_http_remote(self): ) response = request(make_server(recorder, GitRelay()), signed_grant()) assert_refused(response, 502, "not an https URL") - assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_remote_not_https"})] + assert recorder.counts == [ + ("sentry.scm.git.request.failed", {"referrer": REFERRER, "code": "git_remote_not_https"}) + ] def test_provider_rejecting_the_token(self): # Passed on, a 401 would make git ask the sandboxed client for credentials. @@ -374,11 +403,14 @@ def adapter() -> ClosingAdapter: assert_refused(response, 502, "rejected Sentry's credentials") # The provider's unread response is closed with its connection. assert [a.closed for a in adapters] == [True] - assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "git_token_rejected"})] + assert recorder.counts == [ + ("sentry.scm.git.request.failed", {"referrer": REFERRER, "code": "git_token_rejected"}) + ] assert recorder.error_codes == ["git_token_rejected"] assert recorder.errors[0][1] == { "repository_id": "5", "organization_id": "1", + "referrer": REFERRER, "operation": "fetch", "provider": "github_enterprise", "remote_host": urlsplit(upstream.url).netloc, @@ -397,7 +429,13 @@ def test_provider_refusing_the_request_is_passed_on_and_counted_as_failed(self, assert recorder.counts == [ ( "sentry.scm.git.request.failed", - {"operation": "fetch", "provider": "github", "status": str(status), "code": "provider_error"}, + { + "referrer": REFERRER, + "operation": "fetch", + "provider": "github", + "status": str(status), + "code": "provider_error", + }, ) ] @@ -406,7 +444,9 @@ def test_client_closing_its_request_is_counted_but_not_reported(self): relay.forward.side_effect = ClientClosedRequest() recorder = Recorder() assert_refused(request(make_server(recorder, relay), signed_grant()), 499, "") - assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "client_closed_request"})] + assert recorder.counts == [ + ("sentry.scm.git.request.failed", {"referrer": REFERRER, "code": "client_closed_request"}) + ] assert recorder.errors == [] def test_unexpected_error_code_is_a_server_error(self): @@ -428,7 +468,9 @@ class HostRefused(Exception): recorder = Recorder() with pytest.raises(HostRefused): request(make_server(recorder, relay), signed_grant()) - assert recorder.counts == [("sentry.scm.git.request.failed", {"code": "unhandled_exception"})] + assert recorder.counts == [ + ("sentry.scm.git.request.failed", {"referrer": REFERRER, "code": "unhandled_exception"}) + ] # The caller reports it. assert recorder.errors == [] From 161ba134dcb04cae557ddba79d03fe1855f35eff Mon Sep 17 00:00:00 2001 From: Ivan Dlugos <dlugos.ivan@gmail.com> Date: Thu, 22 Oct 2026 15:42:50 +0200 Subject: [PATCH 30/31] feat(git): Refuse to sign a grant longer than 4096 characters The credential proxy sends the grant on in the X-Scm-Git-Grant request header, and sandbox-router's PUT /grant refuses a grant longer than 4096 characters. A grant grows with its repositories, so once submodules or dependencies are added Seer could sign one the router refuses, and the job would fail there with nothing pointing back to the limit. Refused in sign_grant, it fails in the signer's tests instead. GRANT_MAX_LENGTH matches the router's limit, which is a conservative pick rather than a measured ceiling. verify_grant does not check it: a longer grant would already have been refused on the way in. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/scm/git/grant.py | 11 +++++++++-- src/scm/git/types.py | 8 ++++++++ tests/unit/test_git_grant.py | 29 ++++++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/scm/git/grant.py b/src/scm/git/grant.py index 477e31f3..84badc2b 100644 --- a/src/scm/git/grant.py +++ b/src/scm/git/grant.py @@ -15,7 +15,7 @@ import msgspec from scm.errors import GitGrantExpired, GitInvalidGrant -from scm.git.types import Grant +from scm.git.types import GRANT_MAX_LENGTH, Grant GRANT_AUDIENCE = "scm-git-grant" _ALGORITHM = "HS256" @@ -34,7 +34,14 @@ def sign_grant(secret: str, grant: Grant, *, now: float | None = None) -> str: claims["exp"] = claims.pop("expires_at") claims["iat"] = int(time.time() if now is None else now) claims["aud"] = GRANT_AUDIENCE - return jwt.encode(claims, secret, algorithm=_ALGORITHM, headers={"kid": _key_id(secret)}) + token = jwt.encode(claims, secret, algorithm=_ALGORITHM, headers={"kid": _key_id(secret)}) + # Refused here, the signer's tests catch a grant that is too long, rather than the sandbox router in production. + if len(token) > GRANT_MAX_LENGTH: + raise ValueError( + f"The git grant is {len(token)} characters, over GRANT_MAX_LENGTH ({GRANT_MAX_LENGTH}). It travels in a " + "request header, and sandbox-router refuses a longer one." + ) + return token def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> Grant: diff --git a/src/scm/git/types.py b/src/scm/git/types.py index 412543b2..fd503ac0 100644 --- a/src/scm/git/types.py +++ b/src/scm/git/types.py @@ -13,6 +13,14 @@ GRANT_HEADER = "X-Scm-Git-Grant" """The request header that carries a signed grant to the git endpoint.""" +GRANT_MAX_LENGTH = 4096 +"""The longest signed grant, in characters. The grant travels in a request header, and sandbox-router's +``PUT /grant/{sandbox_id}`` refuses a longer one, so a change here must change the router too. A grant grows with +its repositories: a grant for one is under 400 characters, and the claims are base64-encoded, so each repository +named as ``(provider, path)`` adds about 4/3 of its path's length plus 20, e.g. 52 for a 25-character path. The +limit is a conservative pick rather than a measured ceiling: header limits further along the path are typically +around 8 KiB.""" + class Grant(msgspec.Struct, frozen=True, kw_only=True, forbid_unknown_fields=True): """Permission to run one git operation against a fixed set of an organization's repositories. diff --git a/tests/unit/test_git_grant.py b/tests/unit/test_git_grant.py index b126a264..cea9fd3d 100644 --- a/tests/unit/test_git_grant.py +++ b/tests/unit/test_git_grant.py @@ -8,7 +8,7 @@ from scm.errors import GitGrantExpired, GitInvalidGrant from scm.git.grant import GRANT_AUDIENCE, sign_grant, verify_grant -from scm.git.types import Grant +from scm.git.types import GRANT_MAX_LENGTH, Grant from tests.unit.git_builders import FAR_FUTURE, make_grant # Before the real time: PyJWT checks iat against the real clock, so a grant must not be issued after it. @@ -148,6 +148,33 @@ def test_other_algorithm(self): assert_invalid(sign_claims(claims_of(make_grant()), algorithm="HS512"), "invalid") +def grant_with_repositories(count: int) -> Grant: + # Names of one length, so each repository adds the same to the grant. + return make_grant(repositories=[("github", f"getsentry/repository-{i:04d}") for i in range(count)]) + + +class TestLength: + """The grant travels in a request header, and sandbox-router refuses one longer than GRANT_MAX_LENGTH.""" + + def test_too_long_to_sign(self): + with pytest.raises(ValueError, match=f"over GRANT_MAX_LENGTH \\({GRANT_MAX_LENGTH}\\)"): + sign_grant(SECRET, grant_with_repositories(100), now=NOW) + + def test_longest_grant_that_fits_signs(self): + # Add repositories until one more would not fit. + count = 1 + token = sign_grant(SECRET, grant_with_repositories(count), now=NOW) + while True: + try: + longer = sign_grant(SECRET, grant_with_repositories(count + 1), now=NOW) + except ValueError: + break + token, count = longer, count + 1 + # One more repository adds about 50 characters. + assert GRANT_MAX_LENGTH - 100 < len(token) <= GRANT_MAX_LENGTH + assert verify_grant([SECRET], token, now=NOW) == grant_with_repositories(count) + + class TestSecrets: # PyJWT signs with an empty key, so an empty secret would let anyone sign grants. def test_refuses_to_sign_with_an_empty_secret(self): From 7c8dd13fd8ee218fef572108bd768d117e95b10a Mon Sep 17 00:00:00 2001 From: Ivan Dlugos <dlugos.ivan@gmail.com> Date: Thu, 22 Oct 2026 17:15:15 +0200 Subject: [PATCH 31/31] ref(git): Apply the second round of review feedback - check_request is now authorize_request: it returns the request's operation but also refuses what the grant does not allow, which "check" undersold. - GitRelay.forward's docstring no longer reads as a precondition. The relay enforces what keeps the token safe (an https remote, a git route) and knows nothing of grants; GitServer authorizes first. - _UpstreamStream catches Exception rather than BaseException; StopIteration, the normal end of the body, is an Exception. - The clock skew, used once, is inlined as PyJWT's leeway. - Tests cover what coverage showed missing: empty chunks from the provider, closing the stream when the body ends, breaks or is closed unread, and a POST with Content-Length 0. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- src/scm/git/grant.py | 7 ++-- src/scm/git/relay.py | 16 ++++---- src/scm/git/server.py | 4 +- tests/unit/test_git_relay.py | 71 +++++++++++++++++++++++++++++++----- 4 files changed, 74 insertions(+), 24 deletions(-) diff --git a/src/scm/git/grant.py b/src/scm/git/grant.py index 84badc2b..89586d24 100644 --- a/src/scm/git/grant.py +++ b/src/scm/git/grant.py @@ -21,9 +21,6 @@ _ALGORITHM = "HS256" # Claims the JWT layer owns. Every other claim must be a Grant field. _JWT_CLAIMS = frozenset({"aud", "exp", "iat"}) -# How far in the future a grant's iat may be. Seer signs grants on other hosts than the one verifying them, and a -# clock running a little ahead should not get its grants refused. -_CLOCK_SKEW = 60 def sign_grant(secret: str, grant: Grant, *, now: float | None = None) -> str: @@ -73,7 +70,9 @@ def verify_grant(secrets: list[str], token: str, *, now: float | None = None) -> # exp is checked below against an injectable clock, so tests can pin it. PyJWT checks iat, which it # also requires to be a number. options={"require": sorted(_JWT_CLAIMS), "verify_exp": False}, - leeway=_CLOCK_SKEW, + # How far in the future iat may be, in seconds. Seer signs grants on other hosts than the one verifying + # them, and a clock running a little ahead should not get its grants refused. + leeway=60, ) except jwt.InvalidSignatureError: raise GitInvalidGrant(detail="The git grant's signature does not match.") from None diff --git a/src/scm/git/relay.py b/src/scm/git/relay.py index e1a46f70..1d60ed82 100644 --- a/src/scm/git/relay.py +++ b/src/scm/git/relay.py @@ -62,8 +62,8 @@ ) -def check_request(grant: Grant, repository_id: RepositoryId, method: str, route: str, query: str) -> GitOperation: - """Refuse a request the grant does not reach, before anything is sent to the provider. +def authorize_request(grant: Grant, repository_id: RepositoryId, method: str, route: str, query: str) -> GitOperation: + """Return the git operation a request belongs to, refusing the request unless the grant allows it. A fetch grant reaches only upload-pack and a push grant only receive-pack. The repository must be listed in the grant in the same form the request uses: a database id does not match the same repository's @@ -142,16 +142,16 @@ def forward( ) -> RelayResponse: """Send one request to ``remote`` and return the provider's response as it arrives. - ``route`` is the path below ``<repo>.git/``. Check the request against a grant with ``check_request`` - before calling this. + ``route`` is the path below ``<repo>.git/``. The relay enforces what keeps the token safe: an https remote + and one of the four git routes. It knows nothing of grants, so whether the request may run at all is the + caller's to decide; GitServer calls ``authorize_request`` before it looks up the repository or a token. Errors carry no URL, because the client has no use for the provider's address. """ if not remote["url"].startswith("https://"): # The token would cross the network in the clear, e.g. to a self-managed instance served over http. raise GitRemoteNotHttps() - # Callers check the request against a grant first; this only makes sure the token never goes to anything - # but a git route. + # The token never goes to anything but a git route, whatever the caller checked. if (method, route, query) not in _ROUTES: raise GitRouteNotSupported(detail=f"{method} {route} is not a git fetch or push route.") if not f"{remote['basic_auth_username']}:{token}".isascii(): @@ -266,8 +266,8 @@ def __next__(self) -> bytes: try: while not (chunk := next(self.chunks)): pass - except BaseException: - # The end of the body, or the provider's stream breaking. + except Exception: + # The end of the body (StopIteration), or the provider's stream breaking. self.close() raise return chunk diff --git a/src/scm/git/server.py b/src/scm/git/server.py index f43457ef..ba7a851c 100644 --- a/src/scm/git/server.py +++ b/src/scm/git/server.py @@ -5,7 +5,7 @@ from scm.errors import ErrorCode, GitProviderNotSupported, GitTokenRejected, GitTokenUnavailable, SCMCodedError from scm.git.grant import verify_grant -from scm.git.relay import GitRelay, ResponseBody, check_request +from scm.git.relay import GitRelay, ResponseBody, authorize_request from scm.git.types import GRANT_HEADER, GetToken, Readable from scm.helpers import initialize_provider from scm.rpc.types import StreamResponse @@ -119,7 +119,7 @@ def handle( grant = verify_grant(self.secrets, CaseInsensitiveDict(headers).get(GRANT_HEADER, "")) context["organization_id"] = str(grant.organization_id) referrer_tag["referrer"] = context["referrer"] = grant.referrer - operation = check_request(grant, repository_id, method, route, query) + operation = authorize_request(grant, repository_id, method, route, query) context["operation"] = operation provider = initialize_provider( grant.organization_id, diff --git a/tests/unit/test_git_relay.py b/tests/unit/test_git_relay.py index b3a7e167..6f54b98e 100644 --- a/tests/unit/test_git_relay.py +++ b/tests/unit/test_git_relay.py @@ -4,6 +4,7 @@ import threading import time from collections.abc import Iterator +from unittest.mock import MagicMock import pytest import requests @@ -21,7 +22,7 @@ ResourceServiceUnavailable, SCMCodedError, ) -from scm.git.relay import GitRelay, check_request +from scm.git.relay import GitRelay, _UpstreamStream, authorize_request from scm.types import GitRemote from tests.unit.git_builders import make_grant from tests.unit.git_upstream import ClosingAdapter, UpstreamHandler, basic_auth, ca_bundle, running_upstream @@ -187,6 +188,13 @@ def test_unknown_length_body_is_sent_chunked(self, relay, upstream): assert request.headers["Transfer-Encoding"] == "chunked" assert request.body == b"0000" + def test_empty_body_is_sent_as_one(self, relay, upstream): + b"".join(upload_pack(relay, upstream, io.BytesIO(b""), headers={"Content-Length": "0"}).content) + [request] = upstream.requests + assert request.headers["Content-Length"] == "0" + assert "Transfer-Encoding" not in request.headers + assert request.body == b"" + def test_reads_no_further_than_the_declared_length(self, relay, upstream): body = io.BytesIO(b"0000trailing") b"".join(upload_pack(relay, upstream, body, headers={"Content-Length": "4"}).content) @@ -515,6 +523,49 @@ def assert_token_not_in(error: Exception) -> None: assert basic_auth("x-access-token", TOKEN) not in repr(error) +def upstream_stream(*chunks: bytes | Exception) -> tuple[_UpstreamStream, MagicMock, MagicMock]: + """An _UpstreamStream over a response whose body yields ``chunks``, raising any that is an exception.""" + + def stream(chunk_size, decode_content): + for chunk in chunks: + if isinstance(chunk, Exception): + raise chunk + yield chunk + + session, response = MagicMock(spec=requests.Session), MagicMock(spec=requests.Response) + response.raw = MagicMock(stream=stream) + return _UpstreamStream(session, response, 1024), session, response + + +class TestUpstreamStream: + def test_skips_empty_chunks(self): + # An empty chunk carries nothing, so it is not passed on. + stream, _, _ = upstream_stream(b"a", b"", b"", b"b") + assert list(stream) == [b"a", b"b"] + + def test_closes_the_response_and_session_when_the_body_ends(self): + stream, session, response = upstream_stream(b"a") + assert list(stream) == [b"a"] + response.close.assert_called_once() + session.close.assert_called_once() + + def test_closes_the_response_and_session_when_the_provider_stream_breaks(self): + stream, session, response = upstream_stream(b"a", urllib3.exceptions.ProtocolError("connection broken")) + assert next(stream) == b"a" + response.close.assert_not_called() + with pytest.raises(urllib3.exceptions.ProtocolError): + next(stream) + response.close.assert_called_once() + session.close.assert_called_once() + + def test_close_before_reading(self): + # A response GitServer replaces, such as a 401, is closed without being read. + stream, session, response = upstream_stream(b"a") + stream.close() + response.close.assert_called_once() + session.close.assert_called_once() + + class TestClose: def test_closing_an_unread_response_drops_the_upstream_connection(self, relay, upstream): upstream_saw_close = threading.Event() @@ -539,13 +590,13 @@ def handler(h: UpstreamHandler) -> None: PUSH_ROUTES = [("GET", "info/refs", "service=git-receive-pack"), ("POST", "git-receive-pack", "")] -class TestCheckRequest: +class TestAuthorizeRequest: @pytest.mark.parametrize( "operation,route", [("fetch", route) for route in FETCH_ROUTES] + [("push", route) for route in PUSH_ROUTES], ) def test_grant_reaches_its_operations_routes(self, operation, route): - assert check_request(make_grant(operation=operation), 5, *route) == operation + assert authorize_request(make_grant(operation=operation), 5, *route) == operation @pytest.mark.parametrize( "operation,route", @@ -554,7 +605,7 @@ def test_grant_reaches_its_operations_routes(self, operation, route): def test_grant_cannot_reach_the_other_operations_routes(self, operation, route): other = "push" if operation == "fetch" else "fetch" with pytest.raises(GitOperationNotGranted, match=f"allows {operation}, not {other}"): - check_request(make_grant(operation=operation), 5, *route) + authorize_request(make_grant(operation=operation), 5, *route) @pytest.mark.parametrize( "method,route,query", @@ -577,21 +628,21 @@ def test_grant_cannot_reach_the_other_operations_routes(self, operation, route): def test_routes_that_are_not_git(self, method, route, query): for operation in ("fetch", "push"): with pytest.raises(GitRouteNotSupported, match="not a git fetch or push route"): - check_request(make_grant(operation=operation), 5, method, route, query) + authorize_request(make_grant(operation=operation), 5, method, route, query) def test_repository_not_in_grant(self): with pytest.raises(GitRepositoryNotGranted, match="does not cover this repository"): - check_request(make_grant(), 6, "POST", "git-upload-pack", "") + authorize_request(make_grant(), 6, "POST", "git-upload-pack", "") def test_repository_named_another_way_is_not_in_grant(self): # The grant and the request name repositories the same way; matching a database id to a provider id # would mean resolving every repository the grant lists. grant = make_grant(repositories=[("github", "123")]) with pytest.raises(GitRepositoryNotGranted): - check_request(grant, 5, "POST", "git-upload-pack", "") - assert check_request(grant, ("github", "123"), "POST", "git-upload-pack", "") == "fetch" + authorize_request(grant, 5, "POST", "git-upload-pack", "") + assert authorize_request(grant, ("github", "123"), "POST", "git-upload-pack", "") == "fetch" def test_any_listed_repository(self): grant = make_grant(repositories=[5, 6, ("gitlab", "gitlab.com:7")]) - assert check_request(grant, 6, "POST", "git-upload-pack", "") == "fetch" - assert check_request(grant, ("gitlab", "gitlab.com:7"), "POST", "git-upload-pack", "") == "fetch" + assert authorize_request(grant, 6, "POST", "git-upload-pack", "") == "fetch" + assert authorize_request(grant, ("gitlab", "gitlab.com:7"), "POST", "git-upload-pack", "") == "fetch"