diff --git a/src/sentry/api/urls.py b/src/sentry/api/urls.py index 6a9d7df94da9..84d0f61fa662 100644 --- a/src/sentry/api/urls.py +++ b/src/sentry/api/urls.py @@ -569,6 +569,7 @@ from sentry.replays.endpoints.project_replay_summary import ProjectReplaySummaryEndpoint from sentry.replays.endpoints.project_replay_video_details import ProjectReplayVideoDetailsEndpoint from sentry.replays.endpoints.project_replay_viewed_by import ProjectReplayViewedByEndpoint +from sentry.scm.endpoints.scm_git import ScmGitEndpoint from sentry.scm.endpoints.scm_rpc import ScmRpcServiceEndpoint from sentry.seer.endpoints.admin_agentic_triage_trigger import SeerAdminAgenticTriageTriggerEndpoint from sentry.seer.endpoints.admin_autofix_retry import SeerAdminAutofixRetryEndpoint @@ -3692,6 +3693,15 @@ def create_group_urls( ScmRpcServiceEndpoint.as_view(), name="sentry-api-0-scm-rpc-service", ), + # No trailing slash, because git never sends one. scm-git/hosts/ is kept free for a later route + # that names repositories by their provider host. + re_path( + # [0-9] rather than \d, which also matches other scripts' digits. 19 digits hold any + # Repository.id, and keep int() from refusing a longer id with a 500. + r"^scm-git/repositories/(?P[0-9]{1,19})\.git/(?P.+)$", + ScmGitEndpoint.as_view(), + name="sentry-api-0-scm-git", + ), re_path( r"^seer-rpc/(?P\w+)/$", SeerRpcServiceEndpoint.as_view(), diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index 5c1a38304a36..94cd79de1466 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -805,6 +805,12 @@ def SOCIAL_AUTH_DEFAULT_USERNAME() -> str: # First element is used to sign requests; request is accepted if signed with any element in the list. SCM_RPC_SHARED_SECRET: list[str] | None = None +# Ed25519 public keys, as SubjectPublicKeyInfo PEM strings, that verify the grants Seer signs for +# the SCM git endpoint. Only Seer holds a private key, so these aren't secret. A grant is accepted +# if any listed key verifies it. To rotate, add the new public key here, switch Seer to the new +# private key, wait 15 minutes for its old grants to expire, then remove the old public key. +SCM_GIT_GRANT_PUBLIC_KEYS: list[str] = [] + # Shared secret used to sign cross-region RPC requests from the launchpad microservice. LAUNCHPAD_RPC_SHARED_SECRET: list[str] | None = None if (val := os.environ.get("LAUNCHPAD_RPC_SHARED_SECRET")) is not None: @@ -2593,7 +2599,9 @@ def custom_parameter_sort(parameter: dict) -> tuple[str, int]: "http_headers": { "request": { "mode": "denylist", - "terms": ["forwarded", "-ip", "remote-", "via", "-user"], + # "grant" keeps the SCM git endpoint's X-Scm-Git-Grant out of spans and events. + # It's a credential, but none of the SDK's own terms match it. + "terms": ["forwarded", "-ip", "remote-", "via", "-user", "grant"], } }, "cookies": {"mode": "off"}, @@ -3637,6 +3645,13 @@ def custom_parameter_sort(parameter: dict) -> tuple[str, int]: if SILO_DEVSERVER or IS_DEV: LAUNCHPAD_RPC_SHARED_SECRET = ["launchpad-also-very-long-value-haha"] + # Dev only: verifies the grants a local Seer signs. Its private half is committed to Seer's + # .envrc, in a public repository, so anyone can sign grants this key accepts. + SCM_GIT_GRANT_PUBLIC_KEYS = [ + "-----BEGIN PUBLIC KEY-----\n" + "MCowBQYDK2VwAyEAq8SFj6uFiEB74zliRdruAG69shgf1FNJbj/UCnK0jFM=\n" + "-----END PUBLIC KEY-----\n" + ] if SILO_DEVSERVER: # Add connections for the region & control silo databases. diff --git a/src/sentry/features/temporary.py b/src/sentry/features/temporary.py index 6a9b425d8def..73e163a48f21 100644 --- a/src/sentry/features/temporary.py +++ b/src/sentry/features/temporary.py @@ -273,6 +273,8 @@ def register_temporary_features(manager: FeatureManager) -> None: manager.add("organizations:events-endpoint-transactions-discover-blocked", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False) # Enable detecting SDK crashes during event processing manager.add("organizations:sdk-crash-detection", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False) + # Relay git fetches and pushes for coding jobs through Sentry's git endpoint, with the org's integration credentials + manager.add("organizations:scm-git-endpoint", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=False) # Enable Seer Agent panel for AI-powered data exploration manager.add("organizations:seer-explorer", OrganizationFeature, FeatureHandlerStrategy.FLAGPOLE, api_expose=True) # Enable the Seer Automations UI and APIs diff --git a/src/sentry/scm/endpoints/scm_git.py b/src/sentry/scm/endpoints/scm_git.py new file mode 100644 index 000000000000..fd10bdb8750e --- /dev/null +++ b/src/sentry/scm/endpoints/scm_git.py @@ -0,0 +1,202 @@ +import io +import logging +from collections.abc import Generator + +import sentry_sdk +from django.conf import settings +from django.http import HttpRequest, HttpResponse, HttpResponseBase, StreamingHttpResponse +from rest_framework.request import Request +from scm.errors import GitGrantKeyError, GitRemoteRefused, SCMCodedError +from scm.git.grant import verify_grant +from scm.git.relay import GitRelay +from scm.git.server import GitServer +from scm.git.types import GRANT_HEADER, Readable +from sentry_sdk import traces + +from sentry import features +from sentry.api.api_owners import ApiOwner +from sentry.api.api_publish_status import ApiPublishStatus +from sentry.api.base import Endpoint, internal_cell_silo_endpoint +from sentry.api.negotiation import PassthroughContentNegotiation +from sentry.exceptions import RestrictedIPAddress +from sentry.models.organization import Organization +from sentry.net.http import BlacklistAdapter +from sentry.scm.private.helpers import fetch_access_token, make_fetchers, record_count_metric +from sentry.utils.http import request_body_stream + +logger = logging.getLogger(__name__) + +FAILED_METRIC = "sentry.scm.git.request.failed" + + +def report_git_error(error: Exception, context: dict[str, str]) -> None: + # An SCMCodedError's detail never holds a URL or a credential. Any other exception is named + # rather than quoted, since its message could hold either. + if isinstance(error, SCMCodedError): + _report_failure(error.code, error.detail, context) + else: + _report_failure(type(error).__name__, type(error).__name__, context) + + +def make_git_server() -> GitServer: + fetchers = make_fetchers() + return GitServer( + public_keys=_grant_public_keys(), + fetch_repository=fetchers.fetch_repository, + fetch_provider=fetchers.fetch_provider, + get_token=fetch_access_token, + record_count=record_count_metric, + emit_error=report_git_error, + # A GitHub Enterprise or self-managed GitLab host is whatever the customer configured, so + # it may resolve to one of our own addresses. + relay=GitRelay(adapter=BlacklistAdapter, map_error=_map_relay_error), + ) + + +def _map_relay_error(error: Exception) -> SCMCodedError | None: + # BlacklistAdapter refuses an internal address with Sentry's own exception, which scm-platform + # doesn't know. Mapped, GitServer counts, reports and answers it like its other refusals, and + # the report leaves out the address and the urllib3 frames that hold the token. + if isinstance(error, RestrictedIPAddress): + return GitRemoteRefused() + return None + + +@internal_cell_silo_endpoint +class ScmGitEndpoint(Endpoint): + """ + Serves git's smart-HTTP protocol to coding jobs, relaying each request to the repository's + provider with the organization's integration credentials. A grant Seer signs authorizes each + request, so the job never holds a provider credential. + + Only the credential proxy Seer's jobs go through can reach this route, so every refusal is + reported as a Sentry issue: a refusal means Sentry or Seer is misconfigured, not that a + client misbehaved. + """ + + publish_status = {"GET": ApiPublishStatus.PRIVATE, "POST": ApiPublishStatus.PRIVATE} + owner = ApiOwner.CODING_WORKFLOWS + authentication_classes = () + permission_classes = () + # git's bodies are packs and git asks for its own media types, so nothing is parsed or + # negotiated. + parser_classes = () + content_negotiation_class = PassthroughContentNegotiation + enforce_rate_limit = False + + @traces.trace + def get(self, request: Request, repository_id: str, route: str) -> HttpResponseBase: + return self._relay(request, int(repository_id), route) + + @traces.trace + def post(self, request: Request, repository_id: str, route: str) -> HttpResponseBase: + return self._relay(request, int(repository_id), route) + + def _relay(self, request: Request, repository_id: int, route: str) -> HttpResponseBase: + # The grant names the organization whose flag gates the endpoint, and it's verified before + # the organization is looked up, so a request without a valid grant never reaches the + # database. GitServer verifies the JWT again, which costs one Ed25519 signature check. + context = {"repository_id": str(repository_id)} + try: + grant = verify_grant(_grant_public_keys(), request.headers.get(GRANT_HEADER, "")) + except GitGrantKeyError as e: + # No grant could verify against the key list. That's Sentry's misconfiguration, not the + # grant's fault, so it's a 500 rather than a 403. Seer fails a checkout on a 500 at + # once, where it retries a 502 or 503, which no retry would fix. Only the report says + # which key is wrong: the client can't fix Sentry's settings. + return _refuse( + "grant_public_keys_invalid", + 500, + f"SCM_GIT_GRANT_PUBLIC_KEYS is misconfigured: {e}", + context, + tags={}, + client_detail="Sentry's git grant keys are misconfigured.", + ) + except SCMCodedError as e: + return _refuse(e.code, 403, e.detail, context, tags={}) + + context |= {"organization_id": str(grant.organization_id), "referrer": grant.referrer} + tags = {"referrer": grant.referrer} + try: + organization = Organization.objects.get_from_cache(id=grant.organization_id) + except Organization.DoesNotExist: + return _refuse( + "organization_not_found", + 404, + "The git grant's organization was not found.", + context, + tags, + ) + if not features.has("organizations:scm-git-endpoint", organization): + return _refuse( + "git_endpoint_disabled", + 404, + "The git endpoint is not enabled for this organization.", + context, + tags, + ) + + response = make_git_server().handle( + method=request.method or "", + repository_id=repository_id, + route=route, + query=request.META.get("QUERY_STRING", ""), + headers=dict(request.headers), + body=_request_body(request._request), + ) + return StreamingHttpResponse( + response.content, status=response.status_code, headers=response.headers + ) + + +def _grant_public_keys() -> list[str]: + return settings.SCM_GIT_GRANT_PUBLIC_KEYS + + +def _request_body(request: HttpRequest) -> Readable | None: + # Endpoint.dispatch has already read a body sent with a Content-Length into memory, and nginx + # sends every body that way in production, so a push is held whole up to nginx's size cap. + # Only a chunked body, as on the devserver, is still read as it streams. + body = request_body_stream(request) + if isinstance(body, bytes): + return io.BytesIO(body) + if isinstance(body, Generator): + # Only uwsgi hands over a chunked body as a generator, and Sentry serves with granian. + raise NotImplementedError("The git endpoint can't stream a chunked body under uwsgi.") + return body + + +def _refuse( + code: str, + status: int, + detail: str, + context: dict[str, str], + tags: dict[str, str], + *, + client_detail: str | None = None, +) -> HttpResponse: + # Counted and reported like the refusals GitServer answers itself. The client gets the detail + # too, unless client_detail replaces it. + record_count_metric(FAILED_METRIC, 1, {"code": code} | tags) + _report_failure(code, detail, context) + # Never a 401: on a 401 git asks for credentials, which a coding job doesn't have. + return _plain_text(detail if client_detail is None else client_detail, status) + + +def _report_failure(code: str, detail: str, context: dict[str, str]) -> None: + # git shows the client only the message, which leaves out the repository, organization and + # provider, so the log and the issue's tags carry them. The detail can vary within a code, so + # the fingerprint groups by code alone. + logger.error("scm.git.request_failed", extra={"code": code, "detail": detail, **context}) + with sentry_sdk.new_scope() as scope: + scope.fingerprint = ["scm-git", code] + scope.set_tags(context) + # A message, never the exception: Sentry's SDK sends every frame's local variables + # unscrubbed, and the frames a refusal is raised from hold the provider token, and + # requests' hold its Authorization header. + sentry_sdk.capture_message(detail, level="error") + + +def _plain_text(message: str, status: int) -> HttpResponse: + # git shows a text/plain body to the user as "remote: ". + return HttpResponse(f"{message}\n", status=status, content_type="text/plain; charset=utf-8") diff --git a/src/sentry/scm/private/helpers.py b/src/sentry/scm/private/helpers.py index fbf0883ae04b..d923ed38fbb9 100644 --- a/src/sentry/scm/private/helpers.py +++ b/src/sentry/scm/private/helpers.py @@ -4,13 +4,16 @@ import sentry_sdk from django.db.models import Q -from scm.errors import SCMCodedError, UnhandledException +from scm import Secret +from scm.errors import ResourceServiceUnavailable, SCMCodedError, UnhandledException +from scm.git.types import GitOperation from scm.providers.cursor_origin.provider import CursorOriginProvider from scm.providers.github.provider import GitHubProvider from scm.providers.gitlab.provider import GitLabProvider from scm.types import Provider, Repository, RepositoryId from sentry.constants import ObjectStatus +from sentry.hybridcloud.rpc.service import RpcRemoteException, RpcResponseException from sentry.integrations.errors import OrganizationIntegrationNotFound from sentry.integrations.services.integration.model import RpcIntegration from sentry.integrations.services.integration.service import integration_service @@ -84,6 +87,35 @@ def fetch_service_provider( return None +def fetch_access_token(repository: Repository, operation: GitOperation) -> Secret | None: + """Return a token that can run git against the repository, minted or refreshed in control. + + The token is the integration's: an installation token for GitHub and Cursor Origin, or the OAuth + token of the user who installed GitLab. It isn't scoped to the repository or the operation, so a + fetch gets the same token as a push. It's wrapped in a Secret right away, so the frames an + error report carries show it as [Filtered]. + + Raises ResourceServiceUnavailable if control can't be reached, which GitServer answers with a + 503 that Seer retries. None, which GitServer answers with a 502, means control answered but + has no token for the repository. + """ + try: + access_token = integration_service.get_access_token( + integration_id=repository["integration_id"], + organization_id=repository["organization_id"], + ) + except (RpcRemoteException, RpcResponseException): + # Control couldn't be reached, or answered with something other than an RPC response, + # such as a load balancer's error page. Both usually pass, so the request is worth + # retrying. Answered as an uncaught exception, it would be a 500 that git shows only as + # "HTTP 500". + sentry_sdk.capture_exception() + raise ResourceServiceUnavailable( + detail="Sentry could not reach its control silo for a provider token. The request can be retried." + ) from None + return Secret(access_token.token) if access_token is not None else None + + def fetch_repository( organization_id: int, repository_id: RepositoryId, @@ -136,10 +168,7 @@ def fetch_repository( base_host = domain_name.split("/", 1)[0] web_base_url = f"https://{base_host}" elif provider_name == "gitlab": - integration = integration_service.get_integration( - integration_id=repo.integration_id, - organization_id=organization_id, - ) + integration = get_integration(organization_id, repo.integration_id) if integration: # The same instance URL the GitLab API client and source links are built from, so it # keeps the scheme and any path prefix of a self-hosted instance. Installs from older diff --git a/static/app/utils/api/knownSentryApiUrls.generated.ts b/static/app/utils/api/knownSentryApiUrls.generated.ts index 468f727d7aba..ebfcb0b6c206 100644 --- a/static/app/utils/api/knownSentryApiUrls.generated.ts +++ b/static/app/utils/api/knownSentryApiUrls.generated.ts @@ -69,6 +69,7 @@ export type KnownSentryApiUrls = | '/internal/project-config/' | '/internal/projectkey-cell-mappings/' | '/internal/rpc/$serviceName/$methodName/' + | '/internal/scm-git/repositories/$repositoryId.git/$route' | '/internal/scm-rpc/' | '/internal/seer-rpc/$methodName/' | '/internal/seer/autofix/retry/' diff --git a/tests/sentry/conf/test_server.py b/tests/sentry/conf/test_server.py new file mode 100644 index 000000000000..a64bb2610c10 --- /dev/null +++ b/tests/sentry/conf/test_server.py @@ -0,0 +1,48 @@ +import os +import subprocess +import sys + +import pytest + +from sentry.utils import json + +DEV_GRANT_PUBLIC_KEY = ( + "-----BEGIN PUBLIC KEY-----\n" + "MCowBQYDK2VwAyEAq8SFj6uFiEB74zliRdruAG69shgf1FNJbj/UCnK0jFM=\n" + "-----END PUBLIC KEY-----\n" +) + + +def scm_git_grant_public_keys(**environ: str) -> list[str]: + # The settings module reads the environment once, when it's imported, so each environment + # gets its own interpreter. + env = { + name: value + for name, value in os.environ.items() + if name not in ("SENTRY_ENVIRONMENT", "SENTRY_SILO_DEVSERVER") + } + result = subprocess.run( + [ + sys.executable, + "-c", + "import json, sentry.conf.server as s; print(json.dumps(s.SCM_GIT_GRANT_PUBLIC_KEYS))", + ], + env=env | environ, + capture_output=True, + text=True, + check=True, + ) + return json.loads(result.stdout) + + +def test_production_trusts_no_git_grant_key() -> None: + # Self-hosted inherits this default, and the dev key's private half is public. + assert scm_git_grant_public_keys() == [] + + +@pytest.mark.parametrize( + "environ", + [{"SENTRY_ENVIRONMENT": "development"}, {"SENTRY_SILO_DEVSERVER": "1"}], +) +def test_dev_trusts_the_key_local_seer_signs_with(environ: dict[str, str]) -> None: + assert scm_git_grant_public_keys(**environ) == [DEV_GRANT_PUBLIC_KEY] diff --git a/tests/sentry/scm/endpoints/test_scm_git.py b/tests/sentry/scm/endpoints/test_scm_git.py new file mode 100644 index 000000000000..17f35c2be3c4 --- /dev/null +++ b/tests/sentry/scm/endpoints/test_scm_git.py @@ -0,0 +1,727 @@ +from __future__ import annotations + +import base64 +import time +from collections.abc import Callable, Sequence +from typing import Any, NamedTuple +from unittest import mock + +import pytest +import responses +import sentry_sdk +from django.conf import settings +from django.core.signals import request_finished +from django.db import close_old_connections +from django.http import HttpResponseBase +from django.test import RequestFactory, override_settings +from django.test.client import ClientHandler +from django.urls import resolve, reverse +from requests import PreparedRequest +from responses import matchers +from scm import Secret +from scm.git.grant import generate_private_key, public_key_of, sign_grant +from scm.git.types import GRANT_HEADER, GitOperation, Grant +from scm.types import RepositoryId +from sentry_sdk import traces +from sentry_sdk.data_collection import _apply_key_value_collection_filtering +from sentry_sdk.integrations.wsgi import SentryWsgiMiddleware + +from sentry.constants import ObjectStatus +from sentry.hybridcloud.rpc.service import RpcRemoteException +from sentry.scm.endpoints.scm_git import report_git_error +from sentry.testutils.cases import TestCase +from sentry.testutils.helpers.features import with_feature +from sentry.testutils.silo import cell_silo_test +from sentry.utils.sdk import traces_sampler + +TOKEN = "ghs_installation_token" +GITLAB_TOKEN = "gitlab_oauth_token" +FEATURE = "organizations:scm-git-endpoint" +REFERRER = "seer.coding_job" +# Far enough ahead that the GitHub client never refreshes the token over the network. +TOKEN_EXPIRES_AT = "3000-01-01T00:00:00" + + +class KeyPair(NamedTuple): + private: Secret + """The private key in PEM, as Seer signs grants with it.""" + public: str + """The public key in PEM, as SCM_GIT_GRANT_PUBLIC_KEYS lists it.""" + + +def make_key_pair() -> KeyPair: + # Generated, so no private key is committed. + private_key = generate_private_key() + return KeyPair(private=private_key, public=public_key_of(private_key)) + + +KEY = make_key_pair() + + +def sign( + organization_id: int, + operation: GitOperation, + repositories: Sequence[RepositoryId], + *, + expires_in: int = 300, + key: KeyPair = KEY, +) -> str: + grant = Grant( + organization_id=organization_id, + operation=operation, + repositories=list(repositories), + expires_at=int(time.time()) + expires_in, + referrer=REFERRER, + ) + return sign_grant(key.private, grant) + + +@cell_silo_test +@override_settings(SCM_GIT_GRANT_PUBLIC_KEYS=[KEY.public]) +@with_feature(FEATURE) +class ScmGitEndpointTest(TestCase): + def setUp(self) -> None: + super().setUp() + self.integration = self.create_integration( + organization=self.organization, + provider="github", + external_id="1", + metadata={"access_token": TOKEN, "expires_at": TOKEN_EXPIRES_AT}, + ) + self.repo = self.create_repo( + self.project, + name="getsentry/sentry", + provider="integrations:github", + integration_id=self.integration.id, + external_id="12345", + ) + + def fetch_grant(self) -> str: + return sign(self.organization.id, "fetch", [self.repo.id]) + + def git( + self, + method: str, + route: str, + query: str, + grant: str, + repository_id: int | None = None, + body: bytes = b"", + ) -> HttpResponseBase: + url = reverse( + "sentry-api-0-scm-git", + kwargs={"repository_id": repository_id or self.repo.id, "route": route}, + ) + return self.client.generic( + method, + f"{url}?{query}", + data=body, + content_type="application/x-git-request", + headers={GRANT_HEADER: grant}, + ) + + def fetch_refs(self, grant: str, repository_id: int | None = None) -> HttpResponseBase: + return self.git("GET", "info/refs", "service=git-upload-pack", grant, repository_id) + + def assert_refused(self, response: HttpResponseBase, status: int, message: bytes) -> None: + assert response.status_code == status + assert response["Content-Type"] == "text/plain; charset=utf-8" + assert message in b"".join(response) + assert len(responses.calls) == 0 + + def assert_reported(self, mock_sdk: mock.MagicMock, code: str, tags: dict[str, str]) -> None: + scope = mock_sdk.new_scope.return_value.__enter__.return_value + assert scope.fingerprint == ["scm-git", code] + scope.set_tags.assert_called_once_with(tags) + + def assert_reported_message(self, mock_sdk: mock.MagicMock, detail: str) -> None: + # Never the exception, whose frames hold the provider token. + mock_sdk.capture_message.assert_called_once_with(detail, level="error") + mock_sdk.capture_exception.assert_not_called() + + @responses.activate + @mock.patch("sentry.scm.endpoints.scm_git.sentry_sdk") + def test_relays_a_fetch_with_the_installation_token(self, mock_sdk: mock.MagicMock) -> None: + responses.add( + responses.GET, + "https://github.com/getsentry/sentry.git/info/refs", + match=[matchers.query_string_matcher("service=git-upload-pack")], + body=b"001e# service=git-upload-pack\n0000", + content_type="application/x-git-upload-pack-advertisement", + ) + + response = self.fetch_refs(self.fetch_grant()) + + assert response.status_code == 200 + assert response["Content-Type"] == "application/x-git-upload-pack-advertisement" + assert b"".join(response) == b"001e# service=git-upload-pack\n0000" + upstream = responses.calls[0].request + credentials = base64.b64encode(f"x-access-token:{TOKEN}".encode()).decode() + assert upstream.headers["Authorization"] == f"Basic {credentials}" + assert GRANT_HEADER not in upstream.headers + mock_sdk.capture_exception.assert_not_called() + mock_sdk.capture_message.assert_not_called() + + @responses.activate + def test_relays_a_push_body(self) -> None: + responses.add( + responses.POST, + "https://github.com/getsentry/sentry.git/git-receive-pack", + body=b"000eunpack ok\n0000", + content_type="application/x-git-receive-pack-result", + ) + grant = sign(self.organization.id, "push", [self.repo.id]) + + response = self.git("POST", "git-receive-pack", "", grant, body=b"0000PACK-bytes") + + assert response.status_code == 200 + assert b"".join(response) == b"000eunpack ok\n0000" + assert b"".join(responses.calls[0].request.body) == b"0000PACK-bytes" + + @responses.activate + @override_settings(SENTRY_BACKEND_APM_SAMPLING=0) + def test_continues_the_callers_trace(self) -> None: + # Seer and the credential proxy send sentry-trace with every git request, without baggage, + # so a fetch or push joins the trace of the step that ran it. The flag is the caller's + # sampling decision, and the endpoint's own sample rate decides without one. + incoming_trace, incoming_span = "771a43a4192642f0b136d5159a501700", "b0e6f15b45c36b12" + for flag, sampled in (("-1", True), ("-0", False), ("", False)): + seen: list[traces.StreamedSpan] = [] + + def advertise(request: PreparedRequest) -> tuple[int, dict[str, str], bytes]: + current = traces.get_current_span() + assert isinstance(current, traces.StreamedSpan) + seen.append(current) + return 200, {}, b"001e# service=git-upload-pack\n0000" + + responses.reset() + responses.add_callback( + responses.GET, + "https://github.com/getsentry/sentry.git/info/refs", + callback=advertise, + content_type="application/x-git-upload-pack-advertisement", + ) + + status, body = self.serve_through_sdk( + "info/refs", + "service=git-upload-pack", + { + GRANT_HEADER: self.fetch_grant(), + "sentry-trace": f"{incoming_trace}-{incoming_span}{flag}", + }, + ) + + assert (status, body) == ("200 OK", b"001e# service=git-upload-pack\n0000") + [span] = seen + assert span.trace_id == incoming_trace + assert span._segment._parent_span_id == incoming_span + assert span._segment.sampled is sampled + # Sentry's trace stays in Sentry. + assert "sentry-trace" not in responses.calls[0].request.headers + + def serve_through_sdk( + self, route: str, query: str, headers: dict[str, str] + ) -> tuple[str, bytes]: + """Serve a request the way a deployed Sentry does, inside the SDK's WSGI middleware. + + Tests run without an SDK client, so this one starts a streaming client like + sentry.utils.sdk's and takes it back afterwards. + """ + url = reverse( + "sentry-api-0-scm-git", kwargs={"repository_id": self.repo.id, "route": route} + ) + environ = RequestFactory().get(f"{url}?{query}", headers=headers).environ + handler = ClientHandler() + + def app(environ: dict[str, Any], start_response: Callable[..., Any]) -> HttpResponseBase: + response = handler(environ) + start_response(f"{response.status_code} {response.reason_phrase}", response.items()) + return response + + statuses: list[str] = [] + previous_client = sentry_sdk.get_client() + sentry_sdk.init( + default_integrations=False, + auto_enabling_integrations=False, + trace_lifecycle="stream", + traces_sampler=traces_sampler, + ) + try: + body = SentryWsgiMiddleware(app)(environ, lambda status, _: statuses.append(status)) + try: + return statuses[0], b"".join(body) + finally: + # A WSGI server closes the body, which sends request_finished. As in Django's test + # client, that mustn't close the test's database connection. + request_finished.disconnect(close_old_connections) + try: + body.close() + finally: + request_finished.connect(close_old_connections) + finally: + sentry_sdk.get_global_scope().set_client(previous_client) + + @responses.activate + @mock.patch("sentry.scm.endpoints.scm_git.sentry_sdk") + @mock.patch("sentry.scm.endpoints.scm_git.logger") + @mock.patch("sentry.scm.endpoints.scm_git.record_count_metric") + def test_refuses_a_request_without_a_grant( + self, + mock_record_count: mock.MagicMock, + mock_logger: mock.MagicMock, + mock_sdk: mock.MagicMock, + ) -> None: + response = self.fetch_refs("") + + self.assert_refused(response, 403, b"The git grant is missing.") + mock_record_count.assert_called_once_with( + "sentry.scm.git.request.failed", 1, {"code": "git_invalid_grant"} + ) + mock_logger.error.assert_called_once_with( + "scm.git.request_failed", + extra={ + "code": "git_invalid_grant", + "detail": "The git grant is missing.", + "repository_id": str(self.repo.id), + }, + ) + self.assert_reported_message(mock_sdk, "The git grant is missing.") + self.assert_reported(mock_sdk, "git_invalid_grant", {"repository_id": str(self.repo.id)}) + + @responses.activate + @mock.patch("sentry.scm.endpoints.scm_git.sentry_sdk") + @mock.patch("sentry.scm.endpoints.scm_git.record_count_metric") + def test_refuses_a_grant_signed_with_an_unknown_key_without_its_referrer( + self, mock_record_count: mock.MagicMock, mock_sdk: mock.MagicMock + ) -> None: + # Anyone can write a referrer into a grant they sign themselves, so only a verified one + # may tag metrics and issues. + grant = sign(self.organization.id, "fetch", [self.repo.id], key=make_key_pair()) + + response = self.fetch_refs(grant) + + self.assert_refused(response, 403, b"not signed with a known key") + mock_record_count.assert_called_once_with( + "sentry.scm.git.request.failed", 1, {"code": "git_invalid_grant"} + ) + self.assert_reported_message(mock_sdk, "The git grant was not signed with a known key.") + self.assert_reported(mock_sdk, "git_invalid_grant", {"repository_id": str(self.repo.id)}) + + @responses.activate + def test_accepts_a_grant_signed_with_any_listed_key(self) -> None: + # During a rotation Sentry lists the old and the new public key, and Seer may sign with + # either. + new_key = make_key_pair() + responses.add( + responses.GET, + "https://github.com/getsentry/sentry.git/info/refs", + body=b"0000", + content_type="application/x-git-upload-pack-advertisement", + ) + + with override_settings(SCM_GIT_GRANT_PUBLIC_KEYS=[KEY.public, new_key.public]): + response = self.fetch_refs( + sign(self.organization.id, "fetch", [self.repo.id], key=new_key) + ) + + assert response.status_code == 200 + + @responses.activate + @override_settings(SCM_GIT_GRANT_PUBLIC_KEYS=[]) + @mock.patch("sentry.scm.endpoints.scm_git.sentry_sdk") + @mock.patch("sentry.scm.endpoints.scm_git.record_count_metric") + def test_refuses_every_grant_when_no_public_key_is_configured( + self, mock_record_count: mock.MagicMock, mock_sdk: mock.MagicMock + ) -> None: + response = self.fetch_refs(self.fetch_grant()) + + self.assert_refused(response, 500, b"Sentry's git grant keys are misconfigured.") + mock_record_count.assert_called_once_with( + "sentry.scm.git.request.failed", 1, {"code": "grant_public_keys_invalid"} + ) + self.assert_reported_message( + mock_sdk, + "SCM_GIT_GRANT_PUBLIC_KEYS is misconfigured: " + "There are no git grant public keys, so no grant could verify.", + ) + self.assert_reported( + mock_sdk, "grant_public_keys_invalid", {"repository_id": str(self.repo.id)} + ) + + @responses.activate + @mock.patch("sentry.scm.endpoints.scm_git.sentry_sdk") + def test_reports_a_misplaced_private_key_without_quoting_it( + self, mock_sdk: mock.MagicMock + ) -> None: + private_pem = KEY.private.reveal() + + with override_settings(SCM_GIT_GRANT_PUBLIC_KEYS=[private_pem]): + response = self.fetch_refs(self.fetch_grant()) + + # Only the report names the bad entry, and neither quotes it. + assert b"".join(response) == b"Sentry's git grant keys are misconfigured.\n" + (detail,), _ = mock_sdk.capture_message.call_args + assert "public_keys[0] is a private key" in detail + assert private_pem.splitlines()[1] not in detail + + @responses.activate + def test_refuses_an_expired_grant(self) -> None: + grant = sign(self.organization.id, "fetch", [self.repo.id], expires_in=-1) + + response = self.fetch_refs(grant) + + self.assert_refused(response, 403, b"The git grant has expired.") + + @responses.activate + @mock.patch("sentry.scm.endpoints.scm_git.sentry_sdk") + @mock.patch("sentry.scm.endpoints.scm_git.logger") + @mock.patch("sentry.scm.endpoints.scm_git.record_count_metric") + def test_refuses_an_organization_without_the_feature( + self, + mock_record_count: mock.MagicMock, + mock_logger: mock.MagicMock, + mock_sdk: mock.MagicMock, + ) -> None: + with self.feature({FEATURE: False}): + response = self.fetch_refs(self.fetch_grant()) + + self.assert_refused(response, 404, b"not enabled for this organization") + mock_record_count.assert_called_once_with( + "sentry.scm.git.request.failed", + 1, + {"code": "git_endpoint_disabled", "referrer": REFERRER}, + ) + mock_logger.error.assert_called_once_with( + "scm.git.request_failed", + extra={ + "code": "git_endpoint_disabled", + "detail": "The git endpoint is not enabled for this organization.", + "repository_id": str(self.repo.id), + "organization_id": str(self.organization.id), + "referrer": REFERRER, + }, + ) + self.assert_reported_message( + mock_sdk, "The git endpoint is not enabled for this organization." + ) + self.assert_reported( + mock_sdk, + "git_endpoint_disabled", + { + "repository_id": str(self.repo.id), + "organization_id": str(self.organization.id), + "referrer": REFERRER, + }, + ) + + @responses.activate + def test_refuses_a_grant_for_an_organization_that_does_not_exist(self) -> None: + grant = sign(self.organization.id + 1000, "fetch", [self.repo.id]) + + response = self.fetch_refs(grant) + + self.assert_refused(response, 404, b"organization was not found") + + @responses.activate + def test_refuses_a_push_with_a_fetch_grant(self) -> None: + response = self.git("GET", "info/refs", "service=git-receive-pack", self.fetch_grant()) + + self.assert_refused(response, 403, b"The git grant allows fetch, not push.") + + @responses.activate + @mock.patch("sentry.scm.endpoints.scm_git.sentry_sdk") + @mock.patch("sentry.scm.endpoints.scm_git.logger") + def test_reports_a_refusal_with_what_the_request_resolved( + self, mock_logger: mock.MagicMock, mock_sdk: mock.MagicMock + ) -> None: + response = self.git("GET", "info/refs", "service=git-receive-pack", self.fetch_grant()) + + self.assert_refused(response, 403, b"The git grant allows fetch, not push.") + mock_logger.error.assert_called_once_with( + "scm.git.request_failed", + extra={ + "code": "git_operation_not_granted", + "detail": "The git grant allows fetch, not push.", + "repository_id": str(self.repo.id), + "organization_id": str(self.organization.id), + "referrer": REFERRER, + }, + ) + self.assert_reported_message(mock_sdk, "The git grant allows fetch, not push.") + self.assert_reported( + mock_sdk, + "git_operation_not_granted", + { + "repository_id": str(self.repo.id), + "organization_id": str(self.organization.id), + "referrer": REFERRER, + }, + ) + + @responses.activate + def test_refuses_a_fetch_with_a_push_grant(self) -> None: + grant = sign(self.organization.id, "push", [self.repo.id]) + + response = self.git("POST", "git-upload-pack", "", grant, body=b"0000") + + self.assert_refused(response, 403, b"The git grant allows push, not fetch.") + + @responses.activate + def test_refuses_a_route_git_does_not_use(self) -> None: + response = self.git("GET", "HEAD", "", self.fetch_grant()) + + self.assert_refused(response, 404, b"GET HEAD is not a git fetch or push route.") + + @responses.activate + def test_refuses_a_repository_outside_the_grant(self) -> None: + other_repo = self.create_repo( + self.project, + name="getsentry/other", + provider="integrations:github", + integration_id=self.integration.id, + external_id="67890", + ) + + response = self.fetch_refs(self.fetch_grant(), other_repo.id) + + self.assert_refused(response, 403, b"The git grant does not cover this repository.") + + @responses.activate + def test_refuses_a_grant_that_names_the_repository_by_external_id(self) -> None: + grant = sign(self.organization.id, "fetch", [("github", "12345")]) + + response = self.fetch_refs(grant) + + self.assert_refused(response, 403, b"The git grant does not cover this repository.") + + @responses.activate + def test_refuses_a_repository_the_organization_has_not_connected(self) -> None: + unconnected_id = self.repo.id + 1000 + grant = sign(self.organization.id, "fetch", [unconnected_id]) + + response = self.fetch_refs(grant, unconnected_id) + + self.assert_refused(response, 404, b"A repository could not be found.") + + @responses.activate + def test_refuses_another_organizations_repository(self) -> None: + other_org = self.create_organization() + other_integration = self.create_integration( + organization=other_org, + provider="github", + external_id="2", + metadata={"access_token": TOKEN, "expires_at": TOKEN_EXPIRES_AT}, + ) + other_repo = self.create_repo( + self.create_project(organization=other_org), + name="other-org/private", + provider="integrations:github", + integration_id=other_integration.id, + external_id="24680", + ) + grant = sign(self.organization.id, "fetch", [other_repo.id]) + + response = self.fetch_refs(grant, other_repo.id) + + self.assert_refused(response, 404, b"A repository could not be found.") + + @responses.activate + @mock.patch( + "sentry.scm.private.helpers.integration_service.get_access_token", return_value=None + ) + def test_answers_a_token_control_cannot_get_with_a_bad_gateway( + self, mock_get_access_token: mock.MagicMock + ) -> None: + response = self.fetch_refs(self.fetch_grant()) + + self.assert_refused( + response, 502, b"Sentry could not get a github token for this repository." + ) + + @responses.activate + def test_relays_a_gitlab_fetch_with_the_identity_token(self) -> None: + identity_provider = self.create_identity_provider( + type="gitlab", external_id="gitlab.com:acme" + ) + identity = self.create_identity( + self.user, identity_provider, "gitlab-user", data={"access_token": GITLAB_TOKEN} + ) + integration = self.create_integration( + organization=self.organization, + provider="gitlab", + external_id="gitlab.com:acme", + metadata={ + "instance": "gitlab.com", + "base_url": "https://gitlab.com", + "verify_ssl": True, + }, + oi_params={"default_auth_id": identity.id}, + ) + repo = self.create_repo( + self.project, + name="acme / widget", + provider="integrations:gitlab", + integration_id=integration.id, + external_id="gitlab.com:42", + ) + repo.update(config={"path": "acme/widget"}) + responses.add( + responses.GET, + "https://gitlab.com/acme/widget.git/info/refs", + match=[matchers.query_string_matcher("service=git-upload-pack")], + body=b"001e# service=git-upload-pack\n0000", + content_type="application/x-git-upload-pack-advertisement", + ) + grant = sign(self.organization.id, "fetch", [repo.id]) + + response = self.fetch_refs(grant, repo.id) + + assert response.status_code == 200 + assert b"".join(response) == b"001e# service=git-upload-pack\n0000" + credentials = base64.b64encode(f"oauth2:{GITLAB_TOKEN}".encode()).decode() + assert responses.calls[0].request.headers["Authorization"] == f"Basic {credentials}" + + @responses.activate + def test_refuses_a_route_with_a_trailing_slash(self) -> None: + response = self.git("GET", "info/refs/", "service=git-upload-pack", self.fetch_grant()) + + self.assert_refused(response, 404, b"is not a git fetch or push route.") + + @responses.activate + @mock.patch("sentry.scm.private.helpers.sentry_sdk.capture_exception") + @mock.patch( + "sentry.scm.private.helpers.integration_service.get_access_token", + side_effect=RpcRemoteException("integration", "get_access_token", "timed out"), + ) + def test_answers_with_a_retryable_503_when_control_cannot_be_reached( + self, mock_get_access_token: mock.MagicMock, mock_capture_exception: mock.MagicMock + ) -> None: + response = self.fetch_refs(self.fetch_grant()) + + self.assert_refused(response, 503, b"The request can be retried.") + + @responses.activate + def test_refuses_an_inactive_repository(self) -> None: + self.repo.update(status=ObjectStatus.DISABLED) + + response = self.fetch_refs(self.fetch_grant()) + + self.assert_refused(response, 404, b"A repository was found but it is inactive.") + + @mock.patch("sentry.scm.endpoints.scm_git.sentry_sdk") + @mock.patch("sentry.scm.endpoints.scm_git.logger") + @mock.patch("sentry.scm.endpoints.scm_git.record_count_metric") + def test_refuses_a_provider_host_on_an_internal_address( + self, + mock_record_count: mock.MagicMock, + mock_logger: mock.MagicMock, + mock_sdk: mock.MagicMock, + ) -> None: + integration = self.create_integration( + organization=self.organization, + provider="github_enterprise", + external_id="github_enterprise_external_id", + metadata={ + "domain_name": "127.0.0.1/getsentry", + "installation": {"private_key": "private_key", "id": 1, "verify_ssl": True}, + "installation_id": 1234, + "access_token": TOKEN, + "expires_at": TOKEN_EXPIRES_AT, + }, + ) + repo = self.create_repo( + self.project, + name="getsentry/internal", + provider="integrations:github_enterprise", + integration_id=integration.id, + external_id="13579", + ) + grant = sign(self.organization.id, "fetch", [repo.id]) + + response = self.fetch_refs(grant, repo.id) + + detail = "The repository's git remote is at an address Sentry does not connect to." + assert response.status_code == 500 + assert response["Content-Type"] == "text/plain; charset=utf-8" + assert b"".join(response) == f"{detail}\n".encode() + mock_record_count.assert_called_once_with( + "sentry.scm.git.request.failed", + 1, + {"code": "git_remote_refused", "referrer": REFERRER}, + ) + mock_logger.error.assert_called_once_with( + "scm.git.request_failed", + extra={ + "code": "git_remote_refused", + "detail": detail, + "repository_id": str(repo.id), + "organization_id": str(self.organization.id), + "referrer": REFERRER, + "operation": "fetch", + "provider": "github_enterprise", + "remote_host": "127.0.0.1", + }, + ) + self.assert_reported_message(mock_sdk, detail) + self.assert_reported( + mock_sdk, + "git_remote_refused", + { + "repository_id": str(repo.id), + "organization_id": str(self.organization.id), + "referrer": REFERRER, + "operation": "fetch", + "provider": "github_enterprise", + "remote_host": "127.0.0.1", + }, + ) + + +@pytest.mark.parametrize( + "path", + [ + "/api/0/internal/scm-git/repositories/getsentry/sentry.git/info/refs", + "/api/0/internal/scm-git/repositories/42/info/refs", + "/api/0/internal/scm-git/repositories/42.git/", + "/api/0/internal/scm-git/repositories/42.git", + # More digits than any Repository.id, which int() would refuse with a 500 at 4,301. + f"/api/0/internal/scm-git/repositories/{'4' * 20}.git/info/refs", + # Arabic-Indic digits, which int() would read as 42. + "/api/0/internal/scm-git/repositories/\u0664\u0662.git/info/refs", + ], + ids=[ + "name", + "no-git-suffix", + "nothing-after-git", + "no-slash-after-git", + "20-digits", + "arabic-indic", + ], +) +def test_does_not_route(path: str) -> None: + assert resolve(path).url_name == "sentry-api-catchall" + + +@mock.patch("sentry.scm.endpoints.scm_git.sentry_sdk") +@mock.patch("sentry.scm.endpoints.scm_git.logger") +def test_reports_an_uncoded_error_by_its_name_alone( + mock_logger: mock.MagicMock, mock_sdk: mock.MagicMock +) -> None: + error = ValueError("https://x-access-token:a-token@github.com/getsentry/sentry.git") + + report_git_error(error, {"repository_id": "1"}) + + mock_logger.error.assert_called_once_with( + "scm.git.request_failed", + extra={"code": "ValueError", "detail": "ValueError", "repository_id": "1"}, + ) + mock_sdk.capture_message.assert_called_once_with("ValueError", level="error") + mock_sdk.capture_exception.assert_not_called() + + +def test_the_sdk_filters_the_grant_header_out_of_telemetry() -> None: + behaviour = settings.SENTRY_SDK_CONFIG["data_collection"]["http_headers"]["request"] + + filtered = _apply_key_value_collection_filtering({GRANT_HEADER: "a.signed.grant"}, behaviour) + + assert filtered == {GRANT_HEADER: "[Filtered]"} diff --git a/tests/sentry/scm/integration/test_helpers_integration.py b/tests/sentry/scm/integration/test_helpers_integration.py index 7e18e6c98c64..9cbdf9f1dccd 100644 --- a/tests/sentry/scm/integration/test_helpers_integration.py +++ b/tests/sentry/scm/integration/test_helpers_integration.py @@ -1,14 +1,28 @@ -from unittest.mock import patch +from unittest.mock import MagicMock, patch +import pytest +from scm.errors import ResourceServiceUnavailable from scm.providers.cursor_origin.provider import CursorOriginProvider from scm.providers.github.provider import GitHubProvider +from scm.providers.gitlab.provider import GitLabProvider from scm.types import Repository from sentry.constants import ObjectStatus +from sentry.hybridcloud.rpc.service import RpcRemoteException, RpcResponseException from sentry.integrations.services.integration import integration_service from sentry.models.repository import Repository as RepositoryModel -from sentry.scm.private.helpers import fetch_repository, fetch_service_provider, make_fetchers +from sentry.scm.private.helpers import ( + fetch_access_token, + fetch_repository, + fetch_service_provider, + make_fetchers, +) from sentry.testutils.cases import TestCase +from sentry.testutils.silo import cell_silo_test + +TOKEN = "ghs_installation_token" +# Far enough ahead that the GitHub client never refreshes the token over the network. +TOKEN_EXPIRES_AT = "3000-01-01T00:00:00" class TestFetchRepository(TestCase): @@ -455,3 +469,111 @@ def test_fetchers_fetch_the_integration_once(self) -> None: assert isinstance(provider, GitHubProvider) assert service.get_integration.call_count == 1 + + def test_fetchers_fetch_a_gitlab_integration_once(self) -> None: + identity = self.create_identity( + self.user, + self.create_identity_provider(type="gitlab", external_id="gitlab.com:acme"), + "gitlab-user", + data={"access_token": "gitlab_oauth_token"}, + ) + integration = self.create_integration( + organization=self.organization, + provider="gitlab", + external_id="gitlab.com:acme", + metadata={ + "instance": "gitlab.com", + "base_url": "https://gitlab.com", + "verify_ssl": True, + }, + oi_params={"default_auth_id": identity.id}, + ) + repo = self.create_repo( + project=self.project, + name="acme / widget", + provider="integrations:gitlab", + external_id="gitlab.com:42", + integration_id=integration.id, + ) + fetchers = make_fetchers() + + with patch( + "sentry.scm.private.helpers.integration_service", wraps=integration_service + ) as service: + repository = fetchers.fetch_repository(self.organization.id, repo.id) + assert repository is not None + provider = fetchers.fetch_provider(self.organization.id, repository) + + assert isinstance(provider, GitLabProvider) + assert service.get_integration.call_count == 1 + + +@cell_silo_test +class TestFetchAccessToken(TestCase): + def scm_repository(self, integration_id: int) -> Repository: + return Repository( + external_id="1", + id=1, + integration_id=integration_id, + is_active=True, + name="getsentry/sentry", + organization_id=self.organization.id, + provider_name="github", + web_base_url=None, + installation_id=None, + ) + + def test_returns_the_installation_token(self) -> None: + integration = self.create_integration( + organization=self.organization, + provider="github", + external_id="1", + metadata={"access_token": TOKEN, "expires_at": TOKEN_EXPIRES_AT}, + ) + + token = fetch_access_token(self.scm_repository(integration.id), "fetch") + + assert token is not None + assert token.reveal() == TOKEN + + def test_returns_none_for_an_integration_the_organization_does_not_have(self) -> None: + integration = self.create_integration( + organization=self.create_organization(), + provider="github", + external_id="2", + metadata={"access_token": TOKEN, "expires_at": TOKEN_EXPIRES_AT}, + ) + + token = fetch_access_token(self.scm_repository(integration.id), "fetch") + + assert token is None + + @patch("sentry.scm.private.helpers.sentry_sdk.capture_exception") + @patch( + "sentry.scm.private.helpers.integration_service.get_access_token", + side_effect=RpcRemoteException("integration", "get_access_token", "timed out"), + ) + def test_is_retryable_when_control_cannot_be_reached( + self, mock_get_access_token: MagicMock, mock_capture_exception: MagicMock + ) -> None: + with pytest.raises(ResourceServiceUnavailable) as e: + fetch_access_token(self.scm_repository(1), "fetch") + + # Not chained: the RPC exception's frames are reported once, by capture_exception. + assert e.value.__suppress_context__ + mock_capture_exception.assert_called_once_with() + + @patch("sentry.scm.private.helpers.sentry_sdk.capture_exception") + @patch( + "sentry.scm.private.helpers.integration_service.get_access_token", + side_effect=RpcResponseException("integration", "get_access_token", "not JSON"), + ) + def test_is_retryable_when_control_answers_with_something_else( + self, mock_get_access_token: MagicMock, mock_capture_exception: MagicMock + ) -> None: + with pytest.raises(ResourceServiceUnavailable) as e: + fetch_access_token(self.scm_repository(1), "fetch") + + # Not chained: the RPC exception's frames are reported once, by capture_exception. + assert e.value.__suppress_context__ + mock_capture_exception.assert_called_once_with()