From 0edb5b15ba2fd04db5a4e6f4d846d05bbd67565b Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Tue, 20 Oct 2026 19:26:38 +0000 Subject: [PATCH 01/25] Add Loop Runtime workload for EKS Context The Helm chart had no Loop Runtime workload, so external EKS deployments could not run the Kubernetes-hosted runtime. Description Add an opt-in AWS EKS Loop Runtime Deployment, Service, ServiceAccount, and ConfigMap. Wire the runtime to the existing data-plane Secret, API URL, AWS MicroVM sandbox settings, and optional Istio route. Squashed from braintrustdata/helm#98 and rebased onto main. --- braintrust/README.md | 36 +++++ braintrust/templates/_helpers.tpl | 51 ++++++ braintrust/templates/api-configmap.yaml | 4 + .../templates/loop-runtime-configmap.yaml | 71 +++++++++ .../templates/loop-runtime-deployment.yaml | 145 ++++++++++++++++++ .../templates/loop-runtime-service.yaml | 26 ++++ .../loop-runtime-serviceaccount.yaml | 21 +++ braintrust/templates/virtualservice.yaml | 4 + .../__fixtures__/loop-runtime-values.yaml | 21 +++ braintrust/tests/api-configmap_test.yaml | 11 ++ .../tests/labels-merge-configmaps_test.yaml | 31 ++++ .../tests/labels-merge-isolation_test.yaml | 58 +++++++ .../labels-merge-serviceaccounts_test.yaml | 25 +++ .../tests/labels-merge-services_test.yaml | 31 ++++ .../tests/loop-runtime-configmap_test.yaml | 67 ++++++++ .../tests/loop-runtime-deployment_test.yaml | 91 +++++++++++ .../tests/loop-runtime-service_test.yaml | 33 ++++ .../loop-runtime-serviceaccount_test.yaml | 45 ++++++ braintrust/tests/virtualservice_test.yaml | 23 +++ braintrust/values.yaml | 101 ++++++++++++ 20 files changed, 895 insertions(+) create mode 100644 braintrust/templates/loop-runtime-configmap.yaml create mode 100644 braintrust/templates/loop-runtime-deployment.yaml create mode 100644 braintrust/templates/loop-runtime-service.yaml create mode 100644 braintrust/templates/loop-runtime-serviceaccount.yaml create mode 100644 braintrust/tests/__fixtures__/loop-runtime-values.yaml create mode 100644 braintrust/tests/loop-runtime-configmap_test.yaml create mode 100644 braintrust/tests/loop-runtime-deployment_test.yaml create mode 100644 braintrust/tests/loop-runtime-service_test.yaml create mode 100644 braintrust/tests/loop-runtime-serviceaccount_test.yaml diff --git a/braintrust/README.md b/braintrust/README.md index 5ba4bae..2b8f2b7 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -18,6 +18,42 @@ The `braintrust-secrets` secret must contain the following keys: | `GCS_ACCESS_KEY_ID` | Google HMAC Access ID string | Valid S3 API Key Id (only required if `cloud` is `google` and if `enableGcsAuth` is `false`) | | `GCS_SECRET_ACCESS_KEY` | Google HMAC Secret string | Valid S3 Secret string (only required if `cloud` is `google` and if `enableGcsAuth` is `false`) | +## Optional: Loop Runtime on AWS EKS + +Set `loopRuntime.enabled: true` with `cloud: aws` to run the Loop Runtime +service in EKS. The AWS MicroVM image, network connectors, and IAM role are +created by the Terraform data-plane module. Configure the chart with the +Terraform outputs: + +```yaml +cloud: aws + +objectStorage: + aws: + brainstoreBucket: "" + codeBundleBucket: "" + +loopRuntime: + enabled: true + image: + tag: "" + serviceAccount: + name: braintrust-loop-runtime + # Set for IRSA; leave empty when using an EKS Pod Identity association. + awsRoleArn: "" + sandbox: + imageIdentifier: "" + region: "" + ingressNetworkConnectorArns: "" + egressNetworkConnectorArns: "" +``` + +The connector values are in the Terraform +`loop_runtime_sandbox_env_vars` output. The chart reuses the existing +`braintrust-secrets` values, adds the Loop Runtime URL to the API, and adds +`/loop/runtime` routes to the chart-managed Istio VirtualService when +`virtualService.enabled` is true. + ## Azure Key Vault Driver Integration If you're using Azure, the Azure Key Vault CSI driver is default enabled and will automatically sync secrets from Azure Key Vault into Kubernetes. This eliminates the need to manually create and manage the `braintrust-secrets` Kubernetes secret. diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 4393355..c22274f 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -146,6 +146,57 @@ Internal cluster URL for the AI Gateway service. http://{{ .Values.aiGateway.service.name | default .Values.aiGateway.name }}.{{ include "braintrust.namespace" . }}:{{ .Values.aiGateway.service.port }} {{- end -}} +{{/* +Validate the AWS-only Loop Runtime configuration. Terraform supplies the +MicroVM image and connector values; Helm supplies the Kubernetes workload. +*/}} +{{- define "braintrust.loopRuntime.validate" -}} +{{- if .Values.loopRuntime.enabled -}} +{{- if ne .Values.cloud "aws" -}} +{{- fail "loopRuntime.enabled requires cloud: aws because the current sandbox backend uses AWS Lambda MicroVMs." -}} +{{- end -}} +{{- $_ := required "objectStorage.aws.brainstoreBucket is required when loopRuntime.enabled is true" .Values.objectStorage.aws.brainstoreBucket -}} +{{- $_ := required "objectStorage.aws.codeBundleBucket is required when loopRuntime.enabled is true" .Values.objectStorage.aws.codeBundleBucket -}} +{{- $_ := required "loopRuntime.sandbox.imageIdentifier is required when loopRuntime.enabled is true (use Terraform output loop_runtime_microvm_image_arn)" .Values.loopRuntime.sandbox.imageIdentifier -}} +{{- $_ := required "loopRuntime.sandbox.region is required when loopRuntime.enabled is true" .Values.loopRuntime.sandbox.region -}} +{{- $_ := required "loopRuntime.sandbox.ingressNetworkConnectorArns is required when loopRuntime.enabled is true (use Terraform sandbox output)" .Values.loopRuntime.sandbox.ingressNetworkConnectorArns -}} +{{- $_ := required "loopRuntime.sandbox.egressNetworkConnectorArns is required when loopRuntime.enabled is true (use Terraform sandbox output)" .Values.loopRuntime.sandbox.egressNetworkConnectorArns -}} +{{- $_ := required "loopRuntime.serviceAccount.name is required when loopRuntime.enabled is true" .Values.loopRuntime.serviceAccount.name -}} +{{- if .Values.loopRuntime.sandbox.allowExecutionRole -}} +{{- $_ := required "loopRuntime.sandbox.executionRoleArn is required when loopRuntime.sandbox.allowExecutionRole is true" .Values.loopRuntime.sandbox.executionRoleArn -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Internal cluster URL for the Loop Runtime service. */}} +{{- define "braintrust.loopRuntimeInternalUrl" -}} +http://{{ .Values.loopRuntime.service.name | default .Values.loopRuntime.name }}.{{ include "braintrust.namespace" . }}:{{ .Values.loopRuntime.service.port }} +{{- end -}} + +{{/* Stable Redis key namespace for a Helm-managed Loop Runtime. */}} +{{- define "braintrust.loopRuntimeRedisNamespace" -}} +{{- .Values.loopRuntime.redisNamespace | default (printf "loop-runtime:%s" .Release.Name) -}} +{{- end -}} + +{{/* Internal cluster URL for the Brainstore fast reader used by Loop Runtime. */}} +{{- define "braintrust.brainstoreFastReaderInternalUrl" -}} +http://{{ .Values.brainstore.fastreader.service.name | default .Values.brainstore.fastreader.name }}.{{ include "braintrust.namespace" . }}:{{ .Values.brainstore.fastreader.service.port }} +{{- end -}} + +{{/* Product-owned Loop Runtime routes for an Istio VirtualService. */}} +{{- define "braintrust.loopRuntimeVirtualServiceRoutes" -}} +- match: + - uri: + exact: /loop/runtime + - uri: + prefix: /loop/runtime/ + route: + - destination: + host: {{ .Values.loopRuntime.service.name | default .Values.loopRuntime.name }} + port: + number: {{ .Values.loopRuntime.service.port }} +{{- end -}} + {{/* Validate API autoscaling prerequisites (GKE + AutoscalingMetric CRD). */}} diff --git a/braintrust/templates/api-configmap.yaml b/braintrust/templates/api-configmap.yaml index 41643ba..6f8a18b 100644 --- a/braintrust/templates/api-configmap.yaml +++ b/braintrust/templates/api-configmap.yaml @@ -15,6 +15,7 @@ {{- if and (gt (len $btqlStrictOrgIds) 0) (gt (len $btqlBestEffortOrgIds) 0) -}} {{- fail "Set either btqlAuditLogsStrictOrgIds or btqlAuditLogsBestEffortOrgIds, not both." -}} {{- end -}} +{{- include "braintrust.loopRuntime.validate" . -}} {{- range $btqlStrictOrgIds -}} {{- if not (regexMatch "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" .) -}} {{- fail (printf "btqlAuditLogsStrictOrgIds: %q is not a valid UUID" .) -}} @@ -113,6 +114,9 @@ data: {{- if .Values.aiGateway.useGateway }} GATEWAY_URL: {{ include "braintrust.aiGatewayInternalUrl" . | quote }} {{- end }} + {{- if .Values.loopRuntime.enabled }} + LOOP_RUNTIME_URL: {{ include "braintrust.loopRuntimeInternalUrl" . | quote }} + {{- end }} {{- if $btqlStrictOrgIds }} BTQL_AUDIT_LOGS_STRICT_ORG_IDS: {{ $btqlStrictOrgIds | join "," | quote }} {{- end }} diff --git a/braintrust/templates/loop-runtime-configmap.yaml b/braintrust/templates/loop-runtime-configmap.yaml new file mode 100644 index 0000000..dfebfe8 --- /dev/null +++ b/braintrust/templates/loop-runtime-configmap.yaml @@ -0,0 +1,71 @@ +{{- include "braintrust.loopRuntime.validate" . -}} +{{- if .Values.loopRuntime.enabled -}} +{{- $apiUrl := default (include "braintrust.apiInternalUrl" .) .Values.loopRuntime.braintrustApiUrl -}} +{{- $aiProxyUrl := default (include "braintrust.apiAiProxyInternalUrl" .) .Values.loopRuntime.aiProxyUrl -}} +{{- $brainstoreReaderUrl := default (include "braintrust.brainstoreFastReaderInternalUrl" .) .Values.loopRuntime.brainstoreReaderUrl -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.loopRuntime.name }} + namespace: {{ include "braintrust.namespace" . }} + {{- with (mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels)) }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.loopRuntime.annotations.configmap }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +data: + BRAINTRUST_LEGACY_IDS: "true" + LOOP_RUNTIME_HOST: "0.0.0.0" + LOOP_RUNTIME_PORT: {{ .Values.loopRuntime.service.port | quote }} + LOOP_RUNTIME_LIFECYCLE_LOG: {{ .Values.loopRuntime.lifecycleLog | toString | quote }} + LOOP_RUNTIME_REDIS_NAMESPACE: {{ include "braintrust.loopRuntimeRedisNamespace" . | quote }} + LOOP_RUNTIME_STATE_DIR: {{ .Values.loopRuntime.stateDir | quote }} + ORG_NAME: {{ .Values.global.orgName | default "" | toString | trim | quote }} + BRAINTRUST_API_URL: {{ $apiUrl | quote }} + LOOP_RUNTIME_AI_PROXY_URL: {{ $aiProxyUrl | quote }} + LOOP_RUNTIME_BRAINSTORE_READER_URL: {{ $brainstoreReaderUrl | quote }} + BRAINSTORE_VERBOSE: "1" + BRAINSTORE_INDEX_URI: "s3://{{ .Values.objectStorage.aws.brainstoreBucket }}/brainstore/index" + BRAINSTORE_REALTIME_WAL_URI: "s3://{{ .Values.objectStorage.aws.brainstoreBucket }}/brainstore/wal" + BRAINSTORE_CODE_BUNDLE_URI: "s3://{{ .Values.objectStorage.aws.codeBundleBucket }}" + BRAINSTORE_CACHE_DIR: {{ .Values.loopRuntime.cacheDir | quote }} + BRAINSTORE_CONTROL_PLANE_TELEMETRY: {{ .Values.global.controlPlaneTelemetry | quote }} + BRAINSTORE_DISABLE_STATUS_UPDATES: {{ .Values.loopRuntime.brainstoreDisableStatusUpdates | toString | quote }} + NO_COLOR: "1" + AWS_DEFAULT_REGION: {{ .Values.loopRuntime.sandbox.region | quote }} + AWS_REGION: {{ .Values.loopRuntime.sandbox.region | quote }} + EXO_SANDBOX_PROVIDER: {{ .Values.loopRuntime.sandbox.provider | quote }} + EXO_SANDBOX_IMAGE: {{ .Values.loopRuntime.sandbox.image | quote }} + AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER: {{ .Values.loopRuntime.sandbox.imageIdentifier | quote }} + AWS_LAMBDA_MICROVM_REGION: {{ .Values.loopRuntime.sandbox.region | quote }} + AWS_LAMBDA_MICROVM_INGRESS_NETWORK_CONNECTOR_ARNS: {{ .Values.loopRuntime.sandbox.ingressNetworkConnectorArns | quote }} + AWS_LAMBDA_MICROVM_EGRESS_NETWORK_CONNECTOR_ARNS: {{ .Values.loopRuntime.sandbox.egressNetworkConnectorArns | quote }} + AWS_LAMBDA_MICROVM_MAX_IDLE_DURATION_SECONDS: {{ .Values.loopRuntime.sandbox.maxIdleDurationSeconds | quote }} + AWS_LAMBDA_MICROVM_SUSPENDED_DURATION_SECONDS: {{ .Values.loopRuntime.sandbox.suspendedDurationSeconds | quote }} + AWS_LAMBDA_MICROVM_AUTO_RESUME_ENABLED: {{ .Values.loopRuntime.sandbox.autoResumeEnabled | toString | quote }} + AWS_LAMBDA_MICROVM_MAXIMUM_DURATION_SECONDS: {{ .Values.loopRuntime.sandbox.maximumDurationSeconds | quote }} + AWS_LAMBDA_MICROVM_AUTH_TOKEN_EXPIRATION_MINUTES: {{ .Values.loopRuntime.sandbox.authTokenExpirationMinutes | quote }} + AWS_LAMBDA_MICROVM_RUNTIME_PORT: {{ .Values.loopRuntime.sandbox.runtimePort | quote }} + {{- if .Values.loopRuntime.sandbox.allowExecutionRole }} + AWS_LAMBDA_MICROVM_EXECUTION_ROLE_ARN: {{ .Values.loopRuntime.sandbox.executionRoleArn | quote }} + AWS_LAMBDA_MICROVM_ALLOW_EXECUTION_ROLE: "true" + {{- end }} + {{- with (.Values.global.allowedOrgIds | default "" | toString | trim) }} + ALLOWED_ORG_IDS: {{ . | quote }} + {{- end }} + {{- if eq .Values.brainstore.locksBackend "objectStorage" }} + BRAINSTORE_LOCKS_URI: "s3://{{ .Values.objectStorage.aws.brainstoreBucket }}/brainstore/locks" + {{- end }} + {{- if .Values.brainstoreWalFooterVersion }} + BRAINSTORE_WAL_FOOTER_VERSION: {{ .Values.brainstoreWalFooterVersion | quote }} + {{- end }} + {{- if .Values.skipPgForBrainstoreObjects }} + SKIP_PG_FOR_BRAINSTORE_OBJECTS: {{ .Values.skipPgForBrainstoreObjects | quote }} + {{- end }} + {{- if or .Values.brainstoreWalFooterVersion .Values.skipPgForBrainstoreObjects }} + BRAINSTORE_WAL_USE_EFFICIENT_FORMAT: "true" + {{- end }} +{{- end }} diff --git a/braintrust/templates/loop-runtime-deployment.yaml b/braintrust/templates/loop-runtime-deployment.yaml new file mode 100644 index 0000000..a705649 --- /dev/null +++ b/braintrust/templates/loop-runtime-deployment.yaml @@ -0,0 +1,145 @@ +{{- include "braintrust.loopRuntime.validate" . -}} +{{- if .Values.loopRuntime.enabled -}} +{{- $resourceLabels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) -}} +{{- $podLabels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) (deepCopy .Values.loopRuntime.podLabels) (dict "app" .Values.loopRuntime.name) -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.loopRuntime.name }} + namespace: {{ include "braintrust.namespace" . }} + {{- with $resourceLabels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.loopRuntime.annotations.deployment }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.loopRuntime.replicas }} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 100% + maxUnavailable: 0 + selector: + matchLabels: + app: {{ .Values.loopRuntime.name }} + template: + metadata: + labels: + {{- toYaml $podLabels | nindent 8 }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/loop-runtime-configmap.yaml") . | sha256sum }} + {{- with .Values.loopRuntime.annotations.pod }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.loopRuntime.serviceAccount.name }} + {{- with .Values.loopRuntime.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.loopRuntime.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.loopRuntime.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.loopRuntime.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: loop-runtime + image: "{{ .Values.loopRuntime.image.repository }}:{{ .Values.loopRuntime.image.tag }}" + imagePullPolicy: {{ .Values.loopRuntime.image.pullPolicy }} + {{- with .Values.loopRuntime.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: {{ .Values.loopRuntime.service.portName }} + containerPort: {{ .Values.loopRuntime.service.port }} + protocol: TCP + resources: + {{- toYaml .Values.loopRuntime.resources | nindent 12 }} + {{- with .Values.loopRuntime.livenessProbe }} + {{- $livenessProbe := deepCopy . -}} + {{- if and (hasKey $livenessProbe "httpGet") (not (hasKey $livenessProbe.httpGet "port")) -}} + {{- $_ := set $livenessProbe.httpGet "port" $.Values.loopRuntime.service.portName -}} + {{- end }} + livenessProbe: + {{- toYaml $livenessProbe | nindent 12 }} + {{- end }} + {{- with .Values.loopRuntime.readinessProbe }} + {{- $readinessProbe := deepCopy . -}} + {{- if and (hasKey $readinessProbe "httpGet") (not (hasKey $readinessProbe.httpGet "port")) -}} + {{- $_ := set $readinessProbe.httpGet "port" $.Values.loopRuntime.service.portName -}} + {{- end }} + readinessProbe: + {{- toYaml $readinessProbe | nindent 12 }} + {{- end }} + envFrom: + - configMapRef: + name: {{ .Values.loopRuntime.name }} + env: + - name: BRAINSTORE_METADATA_URI + valueFrom: + secretKeyRef: + name: braintrust-secrets + key: PG_URL + - name: BRAINSTORE_WAL_URI + valueFrom: + secretKeyRef: + name: braintrust-secrets + key: PG_URL + - name: BRAINSTORE_XACT_MANAGER_URI + valueFrom: + secretKeyRef: + name: braintrust-secrets + key: REDIS_URL + - name: BRAINSTORE_REDIS_URI + valueFrom: + secretKeyRef: + name: braintrust-secrets + key: REDIS_URL + - name: SERVICE_TOKEN_SECRET_KEY + valueFrom: + secretKeyRef: + name: braintrust-secrets + key: FUNCTION_SECRET_KEY + - name: BRAINSTORE_LICENSE_KEY + valueFrom: + secretKeyRef: + name: braintrust-secrets + key: BRAINSTORE_LICENSE_KEY + {{- if eq .Values.brainstore.locksBackend "redis" }} + - name: BRAINSTORE_LOCKS_URI + valueFrom: + secretKeyRef: + name: braintrust-secrets + key: REDIS_URL + {{- end }} + {{- with .Values.loopRuntime.extraEnvVars }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: state-volume + mountPath: {{ .Values.loopRuntime.stateDir }} + - name: cache-volume + mountPath: {{ .Values.loopRuntime.cacheDir }} + {{- with .Values.loopRuntime.extraContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: state-volume + emptyDir: {} + - name: cache-volume + emptyDir: {} + {{- with .Values.loopRuntime.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/braintrust/templates/loop-runtime-service.yaml b/braintrust/templates/loop-runtime-service.yaml new file mode 100644 index 0000000..9243798 --- /dev/null +++ b/braintrust/templates/loop-runtime-service.yaml @@ -0,0 +1,26 @@ +{{- include "braintrust.loopRuntime.validate" . -}} +{{- if .Values.loopRuntime.enabled -}} +{{- $resourceLabels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.loopRuntime.service.name | default .Values.loopRuntime.name }} + namespace: {{ include "braintrust.namespace" . }} + {{- with $resourceLabels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.loopRuntime.annotations.service }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + app: {{ .Values.loopRuntime.name }} + ports: + - name: {{ .Values.loopRuntime.service.portName }} + protocol: TCP + port: {{ .Values.loopRuntime.service.port }} + targetPort: {{ .Values.loopRuntime.service.port }} + type: {{ .Values.loopRuntime.service.type }} +{{- end }} diff --git a/braintrust/templates/loop-runtime-serviceaccount.yaml b/braintrust/templates/loop-runtime-serviceaccount.yaml new file mode 100644 index 0000000..57ebd09 --- /dev/null +++ b/braintrust/templates/loop-runtime-serviceaccount.yaml @@ -0,0 +1,21 @@ +{{- include "braintrust.loopRuntime.validate" . -}} +{{- if .Values.loopRuntime.enabled -}} +{{- $labels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) -}} +{{- $annotations := mergeOverwrite (deepCopy .Values.loopRuntime.annotations.serviceaccount) (deepCopy .Values.loopRuntime.serviceAccount.annotations) -}} +{{- if and (eq .Values.cloud "aws") .Values.loopRuntime.serviceAccount.awsRoleArn -}} +{{- $_ := set $annotations "eks.amazonaws.com/role-arn" .Values.loopRuntime.serviceAccount.awsRoleArn -}} +{{- end -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.loopRuntime.serviceAccount.name }} + namespace: {{ include "braintrust.namespace" . }} + {{- with $labels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/braintrust/templates/virtualservice.yaml b/braintrust/templates/virtualservice.yaml index 5b4dd8d..a979192 100644 --- a/braintrust/templates/virtualservice.yaml +++ b/braintrust/templates/virtualservice.yaml @@ -5,6 +5,7 @@ {{- if and $virtualServiceWorkloadIsolation.enabled (not .Values.api.workloadIsolation.enabled) -}} {{- fail "virtualService.workloadIsolation.enabled requires api.workloadIsolation.enabled." -}} {{- end -}} +{{- include "braintrust.loopRuntime.validate" . -}} {{- if .Values.virtualService.enabled }} apiVersion: networking.istio.io/v1 kind: VirtualService @@ -29,6 +30,9 @@ spec: {{- toYaml . | nindent 4 }} {{- end }} http: + {{- if .Values.loopRuntime.enabled }} + {{- include "braintrust.loopRuntimeVirtualServiceRoutes" . | nindent 4 }} + {{- end }} {{- if $virtualServiceWorkloadIsolation.enabled }} {{- include "braintrust.apiWorkloadIsolationVirtualServiceRoutes" . | nindent 4 }} {{- end }} diff --git a/braintrust/tests/__fixtures__/loop-runtime-values.yaml b/braintrust/tests/__fixtures__/loop-runtime-values.yaml new file mode 100644 index 0000000..6144ba2 --- /dev/null +++ b/braintrust/tests/__fixtures__/loop-runtime-values.yaml @@ -0,0 +1,21 @@ +cloud: "aws" + +objectStorage: + aws: + brainstoreBucket: "test-brainstore-bucket" + responseBucket: "test-response-bucket" + codeBundleBucket: "test-code-bundle-bucket" + +loopRuntime: + enabled: true + image: + repository: "test/loop-runtime" + tag: "v1.0.0" + serviceAccount: + name: "braintrust-loop-runtime" + awsRoleArn: "arn:aws:iam::123456789012:role/test-loop-runtime-role" + sandbox: + imageIdentifier: "arn:aws:lambda:us-east-1:123456789012:microvm-image/test" + region: "us-east-1" + ingressNetworkConnectorArns: "arn:aws:lambda:us-east-1:123456789012:network-connector/in" + egressNetworkConnectorArns: "arn:aws:lambda:us-east-1:123456789012:network-connector/out" diff --git a/braintrust/tests/api-configmap_test.yaml b/braintrust/tests/api-configmap_test.yaml index 27de0d0..38f80a0 100644 --- a/braintrust/tests/api-configmap_test.yaml +++ b/braintrust/tests/api-configmap_test.yaml @@ -585,3 +585,14 @@ tests: asserts: - failedTemplate: errorMessage: "btqlAuditLogsBestEffortOrgIds: \"not-a-uuid\" is not a valid UUID" + + - it: should publish the Loop Runtime URL when Loop Runtime is enabled + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + release: + namespace: "braintrust" + asserts: + - equal: + path: data.LOOP_RUNTIME_URL + value: http://braintrust-loop-runtime.braintrust:4001 diff --git a/braintrust/tests/labels-merge-configmaps_test.yaml b/braintrust/tests/labels-merge-configmaps_test.yaml index b4c61eb..84ef5fd 100644 --- a/braintrust/tests/labels-merge-configmaps_test.yaml +++ b/braintrust/tests/labels-merge-configmaps_test.yaml @@ -5,6 +5,7 @@ templates: - brainstore-writer-configmap.yaml - brainstore-fastreader-configmap.yaml - brainstore-automationwriter-configmap.yaml + - loop-runtime-configmap.yaml tests: # These tests verify the deepCopy fix for configmaps. # Without deepCopy, labels would bleed between components due to merge mutation. @@ -153,3 +154,33 @@ tests: path: metadata.labels["writer-specific"] - isNull: path: metadata.labels["fastreader-specific"] + + - it: should give Loop Runtime configmap only global and runtime-specific labels + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + global.labels.shared: "global-cm" + loopRuntime.labels.loop-specific: "loop-cm" + api.labels.api-specific: "api-cm" + brainstore.reader.labels.reader-specific: "reader-cm" + brainstore.writer.labels.writer-specific: "writer-cm" + brainstore.fastreader.labels.fastreader-specific: "fastreader-cm" + release: + namespace: "braintrust" + template: loop-runtime-configmap.yaml + asserts: + - equal: + path: metadata.labels.shared + value: global-cm + - equal: + path: metadata.labels["loop-specific"] + value: loop-cm + - isNull: + path: metadata.labels["api-specific"] + - isNull: + path: metadata.labels["reader-specific"] + - isNull: + path: metadata.labels["writer-specific"] + - isNull: + path: metadata.labels["fastreader-specific"] diff --git a/braintrust/tests/labels-merge-isolation_test.yaml b/braintrust/tests/labels-merge-isolation_test.yaml index d545551..1833afc 100644 --- a/braintrust/tests/labels-merge-isolation_test.yaml +++ b/braintrust/tests/labels-merge-isolation_test.yaml @@ -10,6 +10,8 @@ templates: - brainstore-fastreader-configmap.yaml - brainstore-automationwriter-deployment.yaml - brainstore-automationwriter-configmap.yaml + - loop-runtime-deployment.yaml + - loop-runtime-configmap.yaml tests: # This test verifies that labels are isolated between components. # The bug that was fixed: Helm's merge function mutates the leftmost dict, @@ -504,3 +506,59 @@ tests: - equal: path: metadata.labels["automationwriter-only"] value: automationwriter-value + + - it: should isolate Loop Runtime deployment and pod labels + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + global.labels.shared: "global-loop" + global.labels.environment: "production" + loopRuntime.labels.component: "loop-runtime" + loopRuntime.labels.loop-specific: "loop-only" + loopRuntime.podLabels.pod-specific: "pod-only" + api.labels.api-specific: "api-only" + brainstore.reader.labels.reader-specific: "reader-only" + brainstore.writer.labels.writer-specific: "writer-only" + brainstore.fastreader.labels.fastreader-specific: "fastreader-only" + release: + namespace: "braintrust" + template: loop-runtime-deployment.yaml + asserts: + - equal: + path: metadata.labels.shared + value: global-loop + - equal: + path: metadata.labels.environment + value: production + - equal: + path: metadata.labels.component + value: loop-runtime + - equal: + path: metadata.labels["loop-specific"] + value: loop-only + - isNull: + path: metadata.labels["api-specific"] + - isNull: + path: metadata.labels["reader-specific"] + - isNull: + path: metadata.labels["writer-specific"] + - isNull: + path: metadata.labels["fastreader-specific"] + - equal: + path: spec.template.metadata.labels.shared + value: global-loop + - equal: + path: spec.template.metadata.labels["loop-specific"] + value: loop-only + - equal: + path: spec.template.metadata.labels["pod-specific"] + value: pod-only + - isNull: + path: spec.template.metadata.labels["api-specific"] + - isNull: + path: spec.template.metadata.labels["reader-specific"] + - isNull: + path: spec.template.metadata.labels["writer-specific"] + - isNull: + path: spec.template.metadata.labels["fastreader-specific"] diff --git a/braintrust/tests/labels-merge-serviceaccounts_test.yaml b/braintrust/tests/labels-merge-serviceaccounts_test.yaml index f26efed..6b98530 100644 --- a/braintrust/tests/labels-merge-serviceaccounts_test.yaml +++ b/braintrust/tests/labels-merge-serviceaccounts_test.yaml @@ -1,6 +1,7 @@ suite: test label merge isolation for service accounts (deepCopy fix) templates: - api-serviceaccount.yaml + - loop-runtime-serviceaccount.yaml - brainstore-serviceaccount.yaml tests: # These tests verify the deepCopy fix for service accounts. @@ -45,3 +46,27 @@ tests: value: brainstore-sa - isNull: path: metadata.labels["api-specific"] + + - it: should give Loop Runtime service account only global and runtime-specific labels + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + global.labels.shared: "global-sa" + loopRuntime.labels.loop-specific: "loop-sa" + api.labels.api-specific: "api-sa" + brainstore.labels.brainstore-specific: "brainstore-sa" + release: + namespace: "braintrust" + template: loop-runtime-serviceaccount.yaml + asserts: + - equal: + path: metadata.labels.shared + value: global-sa + - equal: + path: metadata.labels["loop-specific"] + value: loop-sa + - isNull: + path: metadata.labels["api-specific"] + - isNull: + path: metadata.labels["brainstore-specific"] diff --git a/braintrust/tests/labels-merge-services_test.yaml b/braintrust/tests/labels-merge-services_test.yaml index 5bb668c..02c3c4a 100644 --- a/braintrust/tests/labels-merge-services_test.yaml +++ b/braintrust/tests/labels-merge-services_test.yaml @@ -5,6 +5,7 @@ templates: - brainstore-writer-service.yaml - brainstore-fastreader-service.yaml - brainstore-automationwriter-service.yaml + - loop-runtime-service.yaml tests: # These tests verify the deepCopy fix for services. # Without deepCopy, labels would bleed between components due to merge mutation. @@ -164,3 +165,33 @@ tests: path: metadata.labels["writer-specific"] - isNull: path: metadata.labels["fastreader-specific"] + + - it: should give Loop Runtime service only global and runtime-specific labels + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + global.labels.shared: "global-service" + loopRuntime.labels.loop-specific: "loop-service" + api.labels.api-specific: "api-service" + brainstore.reader.labels.reader-specific: "reader-service" + brainstore.writer.labels.writer-specific: "writer-service" + brainstore.fastreader.labels.fastreader-specific: "fastreader-service" + release: + namespace: "braintrust" + template: loop-runtime-service.yaml + asserts: + - equal: + path: metadata.labels.shared + value: global-service + - equal: + path: metadata.labels["loop-specific"] + value: loop-service + - isNull: + path: metadata.labels["api-specific"] + - isNull: + path: metadata.labels["reader-specific"] + - isNull: + path: metadata.labels["writer-specific"] + - isNull: + path: metadata.labels["fastreader-specific"] diff --git a/braintrust/tests/loop-runtime-configmap_test.yaml b/braintrust/tests/loop-runtime-configmap_test.yaml new file mode 100644 index 0000000..b7fecef --- /dev/null +++ b/braintrust/tests/loop-runtime-configmap_test.yaml @@ -0,0 +1,67 @@ +suite: test Loop Runtime ConfigMap template +templates: + - loop-runtime-configmap.yaml +tests: + - it: should not render when Loop Runtime is disabled + values: + - __fixtures__/base-values.yaml + asserts: + - hasDocuments: + count: 0 + + - it: should render the AWS runtime and sandbox configuration + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + release: + name: test-release + namespace: braintrust + asserts: + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: braintrust-loop-runtime + - equal: + path: metadata.namespace + value: braintrust + - equal: + path: data.LOOP_RUNTIME_REDIS_NAMESPACE + value: loop-runtime:test-release + - equal: + path: data.BRAINTRUST_API_URL + value: http://braintrust-api.braintrust:8000 + - equal: + path: data.LOOP_RUNTIME_BRAINSTORE_READER_URL + value: http://brainstore-fastreader.braintrust:4000 + - equal: + path: data.BRAINSTORE_CODE_BUNDLE_URI + value: s3://test-code-bundle-bucket + - equal: + path: data.AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER + value: arn:aws:lambda:us-east-1:123456789012:microvm-image/test + - equal: + path: data.AWS_LAMBDA_MICROVM_EGRESS_NETWORK_CONNECTOR_ARNS + value: arn:aws:lambda:us-east-1:123456789012:network-connector/out + - isNull: + path: data.BRAINSTORE_LOCKS_URI + + - it: should use object storage locks when configured + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + brainstore.locksBackend: objectStorage + asserts: + - equal: + path: data.BRAINSTORE_LOCKS_URI + value: s3://test-brainstore-bucket/brainstore/locks + + - it: should reject Loop Runtime on non-AWS clouds + values: + - __fixtures__/base-values.yaml + set: + loopRuntime.enabled: true + asserts: + - failedTemplate: + errorMessage: "loopRuntime.enabled requires cloud: aws because the current sandbox backend uses AWS Lambda MicroVMs." diff --git a/braintrust/tests/loop-runtime-deployment_test.yaml b/braintrust/tests/loop-runtime-deployment_test.yaml new file mode 100644 index 0000000..4c9d114 --- /dev/null +++ b/braintrust/tests/loop-runtime-deployment_test.yaml @@ -0,0 +1,91 @@ +suite: test Loop Runtime Deployment template +templates: + - loop-runtime-deployment.yaml + - loop-runtime-configmap.yaml +tests: + - it: should not render when Loop Runtime is disabled + values: + - __fixtures__/base-values.yaml + asserts: + - hasDocuments: + count: 0 + + - it: should render a hardened runtime pod with data-plane secrets + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + asserts: + - isKind: + of: Deployment + - equal: + path: metadata.name + value: braintrust-loop-runtime + - equal: + path: spec.template.spec.serviceAccountName + value: braintrust-loop-runtime + - equal: + path: spec.template.spec.containers[0].image + value: test/loop-runtime:v1.0.0 + - equal: + path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation + value: false + - contains: + path: spec.template.spec.containers[0].securityContext.capabilities.drop + content: ALL + - contains: + path: spec.template.spec.containers[0].env + content: + name: BRAINSTORE_METADATA_URI + valueFrom: + secretKeyRef: + name: braintrust-secrets + key: PG_URL + - contains: + path: spec.template.spec.containers[0].env + content: + name: SERVICE_TOKEN_SECRET_KEY + valueFrom: + secretKeyRef: + name: braintrust-secrets + key: FUNCTION_SECRET_KEY + - contains: + path: spec.template.spec.containers[0].env + content: + name: BRAINSTORE_LOCKS_URI + valueFrom: + secretKeyRef: + name: braintrust-secrets + key: REDIS_URL + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: state-volume + mountPath: /mnt/tmp/loop-runtime + - contains: + path: spec.template.spec.volumes + content: + name: cache-volume + emptyDir: {} + + - it: should keep probes on a custom named service port + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.service.port: 5000 + loopRuntime.service.portName: metrics + asserts: + - equal: + path: spec.template.spec.containers[0].ports[0].containerPort + value: 5000 + - equal: + path: spec.template.spec.containers[0].ports[0].name + value: metrics + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.port + value: metrics + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.port + value: metrics diff --git a/braintrust/tests/loop-runtime-service_test.yaml b/braintrust/tests/loop-runtime-service_test.yaml new file mode 100644 index 0000000..f75368f --- /dev/null +++ b/braintrust/tests/loop-runtime-service_test.yaml @@ -0,0 +1,33 @@ +suite: test Loop Runtime Service template +templates: + - loop-runtime-service.yaml +tests: + - it: should not render when Loop Runtime is disabled + values: + - __fixtures__/base-values.yaml + asserts: + - hasDocuments: + count: 0 + + - it: should expose the runtime port through a ClusterIP service + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + asserts: + - isKind: + of: Service + - equal: + path: metadata.name + value: braintrust-loop-runtime + - equal: + path: spec.selector.app + value: braintrust-loop-runtime + - equal: + path: spec.ports[0].port + value: 4001 + - equal: + path: spec.ports[0].targetPort + value: 4001 + - equal: + path: spec.type + value: ClusterIP diff --git a/braintrust/tests/loop-runtime-serviceaccount_test.yaml b/braintrust/tests/loop-runtime-serviceaccount_test.yaml new file mode 100644 index 0000000..4f7498d --- /dev/null +++ b/braintrust/tests/loop-runtime-serviceaccount_test.yaml @@ -0,0 +1,45 @@ +suite: test Loop Runtime ServiceAccount template +templates: + - loop-runtime-serviceaccount.yaml +tests: + - it: should not render when Loop Runtime is disabled + values: + - __fixtures__/base-values.yaml + asserts: + - hasDocuments: + count: 0 + + - it: should attach the configured AWS role for IRSA + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + asserts: + - isKind: + of: ServiceAccount + - equal: + path: metadata.name + value: braintrust-loop-runtime + - equal: + path: metadata.annotations["eks.amazonaws.com/role-arn"] + value: arn:aws:iam::123456789012:role/test-loop-runtime-role + + - it: should support Pod Identity without adding an IRSA annotation + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.serviceAccount.awsRoleArn: "" + asserts: + - isNull: + path: metadata.annotations["eks.amazonaws.com/role-arn"] + + - it: should include ServiceAccount annotations from the nested configuration + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.serviceAccount.annotations.custom-annotation: value + asserts: + - equal: + path: metadata.annotations["custom-annotation"] + value: value diff --git a/braintrust/tests/virtualservice_test.yaml b/braintrust/tests/virtualservice_test.yaml index 40fb1ee..5853664 100644 --- a/braintrust/tests/virtualservice_test.yaml +++ b/braintrust/tests/virtualservice_test.yaml @@ -336,3 +336,26 @@ tests: asserts: - failedTemplate: errorMessage: "virtualService.workloadIsolation.enabled requires virtualService.enabled." + + - it: should route Loop Runtime before the API fallback + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + virtualService.enabled: true + asserts: + - equal: + path: spec.http[0].match[0].uri.exact + value: /loop/runtime + - equal: + path: spec.http[0].match[1].uri.prefix + value: /loop/runtime/ + - equal: + path: spec.http[0].route[0].destination.host + value: braintrust-loop-runtime + - equal: + path: spec.http[0].route[0].destination.port.number + value: 4001 + - equal: + path: spec.http[1].match[0].uri.prefix + value: / diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 02be806..f372c47 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -309,6 +309,107 @@ api: extraContainers: [] extraVolumes: [] +# Loop Runtime configuration (AWS EKS only) +# Terraform creates the AWS MicroVM image, network connectors, and IAM role; +# this chart runs the Loop Runtime process in Kubernetes. +loopRuntime: + enabled: false + name: "braintrust-loop-runtime" + labels: {} + podLabels: {} + annotations: + configmap: {} + deployment: {} + service: {} + pod: {} + serviceaccount: {} + replicas: 1 + image: + repository: public.ecr.aws/braintrust/loop-runtime + tag: latest-2.x + pullPolicy: IfNotPresent + service: + # Optional name for the Service. If empty, loopRuntime.name is used. + name: "" + type: ClusterIP + port: 4001 + portName: http + serviceAccount: + name: "braintrust-loop-runtime" + # Set this to Terraform's loop_runtime_eks_role_arn when using IRSA. Leave + # it empty when the role is attached through an EKS Pod Identity association. + awsRoleArn: "" + annotations: {} + # These URLs default to in-cluster services. Set them when the API, AI proxy, + # or Brainstore is exposed through a different service or namespace. + braintrustApiUrl: "" + aiProxyUrl: "" + brainstoreReaderUrl: "" + # Defaults to loop-runtime:; set explicitly when retaining + # an existing Loop Runtime Redis namespace across release renames. + redisNamespace: "" + lifecycleLog: "0" + brainstoreDisableStatusUpdates: "false" + stateDir: "/mnt/tmp/loop-runtime" + cacheDir: "/mnt/tmp/brainstore" + resources: + requests: + cpu: "2" + memory: "8Gi" + limits: + cpu: "2" + memory: "8Gi" + podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + livenessProbe: + httpGet: + path: /health/liveness + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 3 + successThreshold: 1 + readinessProbe: + httpGet: + path: /health/liveness + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + successThreshold: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + sandbox: + provider: "aws-lambda-microvm" + image: "lambda-microvm" + # These values come from Terraform outputs. They are required when enabled. + imageIdentifier: "" + region: "" + ingressNetworkConnectorArns: "" + egressNetworkConnectorArns: "" + maxIdleDurationSeconds: 900 + suspendedDurationSeconds: 28800 + autoResumeEnabled: true + maximumDurationSeconds: 28800 + authTokenExpirationMinutes: 30 + runtimePort: 8080 + executionRoleArn: "" + allowExecutionRole: false + extraEnvVars: [] + extraContainers: [] + extraVolumes: [] + # AI Gateway configuration (optional) # Deploys the Braintrust AI gateway service for LLM proxying and completions caching. # Recommended rollout: enabled=true first, wait for gateway pods to be ready, then useGateway=true. From 2171be45485f16b3eb610e585bd076cda09275d1 Mon Sep 17 00:00:00 2001 From: Alex R Date: Tue, 20 Oct 2026 19:35:25 +0000 Subject: [PATCH 02/25] Genericize Loop Runtime for GKE Run Loop Runtime on cloud: google as well as cloud: aws. Storage URIs follow the Brainstore ConfigMaps (s3:// or gs://), and the ServiceAccount and pod use IRSA or GKE Workload Identity. Add loopRuntime.sandbox.backend. awsLambdaMicrovm keeps the AWS Lambda MicroVM settings under sandbox.awsLambdaMicrovm. sandboxService points the runtime at Braintrust Sandbox Service workers over mTLS, which is the GKE default and also works on AWS. An optional sandbox egress gateway opens the runtime's forward proxy behind its own Service. Treat a missing loopRuntime map as disabled so helm upgrade --reuse-values from earlier chart releases keeps rendering. Co-Authored-By: Claude Opus 5.5 --- braintrust/README.md | 101 ++++- braintrust/templates/_helpers.tpl | 113 ++++- braintrust/templates/api-configmap.yaml | 2 +- .../templates/loop-runtime-configmap.yaml | 76 +++- .../templates/loop-runtime-deployment.yaml | 35 +- .../loop-runtime-egress-service.yaml | 31 ++ .../templates/loop-runtime-service.yaml | 2 +- .../loop-runtime-serviceaccount.yaml | 5 +- braintrust/templates/virtualservice.yaml | 2 +- .../loop-runtime-google-values.yaml | 15 + .../__fixtures__/loop-runtime-values.yaml | 9 +- .../__fixtures__/pre-loop-runtime-values.yaml | 4 + .../reuse-loop-runtime-enabled-values.yaml | 33 ++ braintrust/tests/api-configmap_test.yaml | 21 + .../tests/loop-runtime-configmap_test.yaml | 402 +++++++++++++++++- .../tests/loop-runtime-deployment_test.yaml | 137 ++++++ .../loop-runtime-egress-service_test.yaml | 94 ++++ .../tests/loop-runtime-service_test.yaml | 19 + .../loop-runtime-serviceaccount_test.yaml | 26 ++ braintrust/tests/virtualservice_test.yaml | 24 ++ braintrust/values.yaml | 82 +++- 21 files changed, 1152 insertions(+), 81 deletions(-) create mode 100644 braintrust/templates/loop-runtime-egress-service.yaml create mode 100644 braintrust/tests/__fixtures__/loop-runtime-google-values.yaml create mode 100644 braintrust/tests/__fixtures__/pre-loop-runtime-values.yaml create mode 100644 braintrust/tests/__fixtures__/reuse-loop-runtime-enabled-values.yaml create mode 100644 braintrust/tests/loop-runtime-egress-service_test.yaml diff --git a/braintrust/README.md b/braintrust/README.md index 2b8f2b7..6cdf220 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -18,12 +18,26 @@ The `braintrust-secrets` secret must contain the following keys: | `GCS_ACCESS_KEY_ID` | Google HMAC Access ID string | Valid S3 API Key Id (only required if `cloud` is `google` and if `enableGcsAuth` is `false`) | | `GCS_SECRET_ACCESS_KEY` | Google HMAC Secret string | Valid S3 Secret string (only required if `cloud` is `google` and if `enableGcsAuth` is `false`) | -## Optional: Loop Runtime on AWS EKS +## Optional: Loop Runtime -Set `loopRuntime.enabled: true` with `cloud: aws` to run the Loop Runtime -service in EKS. The AWS MicroVM image, network connectors, and IAM role are -created by the Terraform data-plane module. Configure the chart with the -Terraform outputs: +Set `loopRuntime.enabled: true` to run the Loop Runtime service in the cluster. +It supports `cloud: aws` (EKS) and `cloud: google` (GKE). The cloud's +Terraform data-plane module creates the runtime identity and the sandbox +infrastructure; the chart runs the runtime process. + +The chart reuses the existing `braintrust-secrets` values, reads the same +Brainstore buckets as the Brainstore pods, adds `LOOP_RUNTIME_URL` to the API, +and adds `/loop/runtime` routes to the chart-managed Istio VirtualService when +`virtualService.enabled` is true. + +`loopRuntime.sandbox.backend` selects where agent sandboxes run: + +| Backend | Clouds | Sandboxes run in | +| --- | --- | --- | +| `awsLambdaMicrovm` (default on AWS) | `aws` | AWS Lambda MicroVMs | +| `sandboxService` (default on Google) | `aws`, `google` | Braintrust Sandbox Service workers, reached over mTLS | + +### AWS EKS with Lambda MicroVMs ```yaml cloud: aws @@ -35,24 +49,77 @@ objectStorage: loopRuntime: enabled: true - image: - tag: "" serviceAccount: name: braintrust-loop-runtime # Set for IRSA; leave empty when using an EKS Pod Identity association. - awsRoleArn: "" + awsRoleArn: "" sandbox: - imageIdentifier: "" - region: "" - ingressNetworkConnectorArns: "" - egressNetworkConnectorArns: "" + awsLambdaMicrovm: + imageIdentifier: "" + region: "" + ingressNetworkConnectorArns: "" + egressNetworkConnectorArns: "" ``` -The connector values are in the Terraform -`loop_runtime_sandbox_env_vars` output. The chart reuses the existing -`braintrust-secrets` values, adds the Loop Runtime URL to the API, and adds -`/loop/runtime` routes to the chart-managed Istio VirtualService when -`virtualService.enabled` is true. +### Google GKE with the Sandbox Service + +The GKE Terraform module creates a Google service account for Loop Runtime and +binds it to the `braintrust/braintrust-loop-runtime` Kubernetes service account +through Workload Identity. Its optional isolated worker cluster hosts the +Sandbox Service. Install the Sandbox Service release in that cluster, then +create a Secret in this namespace with the Sandbox Service client certificate, +its key, and the server CA. + +```yaml +cloud: google + +objectStorage: + google: + brainstoreBucket: "" + apiBucket: "" + +loopRuntime: + enabled: true + serviceAccount: + name: braintrust-loop-runtime + googleServiceAccount: "" + sandbox: + sandboxService: + dnsUrl: "" + tls: + secretName: sandbox-client-tls + # Keys in that Secret. Defaults match the Sandbox Service client Secret. + certKey: client.pem + keyKey: client.key + caKey: server-ca.pem +``` + +`sandboxService.vcpuCount` and `sandboxService.memoryMib` must match the +per-sandbox sizing of the Sandbox Service release. + +### Optional sandbox egress gateway + +Sandboxes use a forward proxy in Loop Runtime for brokered Git access. Enable +it with `loopRuntime.sandbox.egressGateway`. The chart opens the proxy port on +the pod, renders a `-egress` Service, and tells sandboxes to use +`egressGateway.url`. Sandboxes run outside this cluster, so expose the Service +on a private address the sandboxes can reach. On GKE, use an internal load +balancer and allow the isolated worker node range in the worker cluster's +egress rules: + +```yaml +loopRuntime: + sandbox: + egressGateway: + enabled: true + url: "http://:4002" + service: + type: LoadBalancer + annotations: + networking.gke.io/load-balancer-type: "Internal" + loadBalancerSourceRanges: + - "" +``` ## Azure Key Vault Driver Integration diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index c22274f..3da68d3 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -147,23 +147,109 @@ http://{{ .Values.aiGateway.service.name | default .Values.aiGateway.name }}.{{ {{- end -}} {{/* -Validate the AWS-only Loop Runtime configuration. Terraform supplies the -MicroVM image and connector values; Helm supplies the Kubernetes workload. +Whether Loop Runtime is enabled. Values reused from a chart release that +predates Loop Runtime have no loopRuntime map, which means disabled. +*/}} +{{- define "braintrust.loopRuntime.enabled" -}} +{{- if (.Values.loopRuntime | default dict).enabled -}}true{{- end -}} +{{- end -}} + +{{/* +Loop Runtime sandbox backend. Empty selects the cloud default: +awsLambdaMicrovm on AWS and sandboxService elsewhere. +*/}} +{{- define "braintrust.loopRuntime.sandboxBackend" -}} +{{- .Values.loopRuntime.sandbox.backend | default (eq .Values.cloud "aws" | ternary "awsLambdaMicrovm" "sandboxService") -}} +{{- end -}} + +{{/* +Object-store scheme and Brainstore bucket for Loop Runtime. Matches the +Brainstore ConfigMaps so both read the same index, WAL, and locks. +*/}} +{{- define "braintrust.loopRuntime.brainstoreBucketUri" -}} +{{- if eq .Values.cloud "google" -}} +gs://{{ .Values.objectStorage.google.brainstoreBucket }} +{{- else -}} +s3://{{ .Values.objectStorage.aws.brainstoreBucket }} +{{- end -}} +{{- end -}} + +{{- define "braintrust.loopRuntime.codeBundleUri" -}} +{{- if eq .Values.cloud "google" -}} +gs://{{ .Values.objectStorage.google.apiBucket }} +{{- else -}} +s3://{{ .Values.objectStorage.aws.codeBundleBucket }} +{{- end -}} +{{- end -}} + +{{/* +Validate the Loop Runtime configuration. Terraform supplies the identity and +sandbox infrastructure; Helm supplies the Kubernetes workload. */}} {{- define "braintrust.loopRuntime.validate" -}} -{{- if .Values.loopRuntime.enabled -}} -{{- if ne .Values.cloud "aws" -}} -{{- fail "loopRuntime.enabled requires cloud: aws because the current sandbox backend uses AWS Lambda MicroVMs." -}} +{{- if include "braintrust.loopRuntime.enabled" . -}} +{{- range (list "image" "service" "serviceAccount" "sandbox") -}} +{{- if not (hasKey $.Values.loopRuntime .) -}} +{{- fail (printf "loopRuntime.%s is missing. Values reused from a chart release that predates Loop Runtime omit the loopRuntime defaults; upgrade with --reset-then-reuse-values or supply the full loopRuntime map." .) -}} {{- end -}} +{{- end -}} +{{- $sandbox := .Values.loopRuntime.sandbox -}} +{{- $backend := include "braintrust.loopRuntime.sandboxBackend" . -}} +{{- if not (has .Values.cloud (list "aws" "google")) -}} +{{- fail "loopRuntime.enabled requires cloud: aws or cloud: google." -}} +{{- end -}} +{{- if eq .Values.cloud "aws" -}} {{- $_ := required "objectStorage.aws.brainstoreBucket is required when loopRuntime.enabled is true" .Values.objectStorage.aws.brainstoreBucket -}} {{- $_ := required "objectStorage.aws.codeBundleBucket is required when loopRuntime.enabled is true" .Values.objectStorage.aws.codeBundleBucket -}} -{{- $_ := required "loopRuntime.sandbox.imageIdentifier is required when loopRuntime.enabled is true (use Terraform output loop_runtime_microvm_image_arn)" .Values.loopRuntime.sandbox.imageIdentifier -}} -{{- $_ := required "loopRuntime.sandbox.region is required when loopRuntime.enabled is true" .Values.loopRuntime.sandbox.region -}} -{{- $_ := required "loopRuntime.sandbox.ingressNetworkConnectorArns is required when loopRuntime.enabled is true (use Terraform sandbox output)" .Values.loopRuntime.sandbox.ingressNetworkConnectorArns -}} -{{- $_ := required "loopRuntime.sandbox.egressNetworkConnectorArns is required when loopRuntime.enabled is true (use Terraform sandbox output)" .Values.loopRuntime.sandbox.egressNetworkConnectorArns -}} +{{- else -}} +{{- $_ := required "objectStorage.google.brainstoreBucket is required when loopRuntime.enabled is true" .Values.objectStorage.google.brainstoreBucket -}} +{{- $_ := required "objectStorage.google.apiBucket is required when loopRuntime.enabled is true" .Values.objectStorage.google.apiBucket -}} +{{- $_ := required "loopRuntime.serviceAccount.googleServiceAccount is required when cloud is google (use Terraform output loop_runtime_service_account)" .Values.loopRuntime.serviceAccount.googleServiceAccount -}} +{{- end -}} {{- $_ := required "loopRuntime.serviceAccount.name is required when loopRuntime.enabled is true" .Values.loopRuntime.serviceAccount.name -}} -{{- if .Values.loopRuntime.sandbox.allowExecutionRole -}} -{{- $_ := required "loopRuntime.sandbox.executionRoleArn is required when loopRuntime.sandbox.allowExecutionRole is true" .Values.loopRuntime.sandbox.executionRoleArn -}} +{{- if eq $backend "awsLambdaMicrovm" -}} +{{- if ne .Values.cloud "aws" -}} +{{- fail "loopRuntime.sandbox.backend awsLambdaMicrovm requires cloud: aws. Use sandboxService on other clouds." -}} +{{- end -}} +{{- $microvm := $sandbox.awsLambdaMicrovm -}} +{{- $_ := required "loopRuntime.sandbox.awsLambdaMicrovm.imageIdentifier is required for the awsLambdaMicrovm backend (use Terraform output loop_runtime_microvm_image_arn)" $microvm.imageIdentifier -}} +{{- $_ := required "loopRuntime.sandbox.awsLambdaMicrovm.region is required for the awsLambdaMicrovm backend" $microvm.region -}} +{{- $_ := required "loopRuntime.sandbox.awsLambdaMicrovm.ingressNetworkConnectorArns is required for the awsLambdaMicrovm backend (use Terraform sandbox output)" $microvm.ingressNetworkConnectorArns -}} +{{- $_ := required "loopRuntime.sandbox.awsLambdaMicrovm.egressNetworkConnectorArns is required for the awsLambdaMicrovm backend (use Terraform sandbox output)" $microvm.egressNetworkConnectorArns -}} +{{- if $microvm.allowExecutionRole -}} +{{- $_ := required "loopRuntime.sandbox.awsLambdaMicrovm.executionRoleArn is required when loopRuntime.sandbox.awsLambdaMicrovm.allowExecutionRole is true" $microvm.executionRoleArn -}} +{{- end -}} +{{- else if eq $backend "sandboxService" -}} +{{- $service := $sandbox.sandboxService -}} +{{- $urls := $service.urls | default list -}} +{{- if not (kindIs "slice" $urls) -}} +{{- fail "loopRuntime.sandbox.sandboxService.urls must be a list of URLs" -}} +{{- end -}} +{{- if and $service.dnsUrl $urls -}} +{{- fail "Set only one of loopRuntime.sandbox.sandboxService.dnsUrl and loopRuntime.sandbox.sandboxService.urls." -}} +{{- end -}} +{{- if not (or $service.dnsUrl $urls) -}} +{{- fail "loopRuntime.sandbox.sandboxService.dnsUrl or loopRuntime.sandbox.sandboxService.urls is required for the sandboxService backend (GKE: use Terraform output gke_isolated_workers.discovery.endpoint)" -}} +{{- end -}} +{{- $_ := required "loopRuntime.sandbox.sandboxService.image is required for the sandboxService backend" $service.image -}} +{{- range (prepend $urls ($service.dnsUrl | default "")) -}} +{{- if and . (hasPrefix "https://" .) (not $service.tls.secretName) -}} +{{- fail "loopRuntime.sandbox.sandboxService.tls.secretName is required for https:// Sandbox Service endpoints" -}} +{{- end -}} +{{- end -}} +{{- if $service.tls.secretName -}} +{{- $_ := required "loopRuntime.sandbox.sandboxService.tls.certKey is required when tls.secretName is set" $service.tls.certKey -}} +{{- $_ := required "loopRuntime.sandbox.sandboxService.tls.keyKey is required when tls.secretName is set" $service.tls.keyKey -}} +{{- $_ := required "loopRuntime.sandbox.sandboxService.tls.caKey is required when tls.secretName is set" $service.tls.caKey -}} +{{- end -}} +{{- else -}} +{{- fail (printf "loopRuntime.sandbox.backend must be awsLambdaMicrovm or sandboxService, got %q" $backend) -}} +{{- end -}} +{{- if $sandbox.egressGateway.enabled -}} +{{- $_ := required "loopRuntime.sandbox.egressGateway.url is required when loopRuntime.sandbox.egressGateway.enabled is true" $sandbox.egressGateway.url -}} +{{- if eq (int $sandbox.egressGateway.port) (int .Values.loopRuntime.service.port) -}} +{{- fail "loopRuntime.sandbox.egressGateway.port must differ from loopRuntime.service.port" -}} +{{- end -}} {{- end -}} {{- end -}} {{- end -}} @@ -173,6 +259,11 @@ MicroVM image and connector values; Helm supplies the Kubernetes workload. http://{{ .Values.loopRuntime.service.name | default .Values.loopRuntime.name }}.{{ include "braintrust.namespace" . }}:{{ .Values.loopRuntime.service.port }} {{- end -}} +{{/* Service that exposes the Loop Runtime sandbox egress gateway. */}} +{{- define "braintrust.loopRuntimeEgressServiceName" -}} +{{- printf "%s-egress" (.Values.loopRuntime.service.name | default .Values.loopRuntime.name) -}} +{{- end -}} + {{/* Stable Redis key namespace for a Helm-managed Loop Runtime. */}} {{- define "braintrust.loopRuntimeRedisNamespace" -}} {{- .Values.loopRuntime.redisNamespace | default (printf "loop-runtime:%s" .Release.Name) -}} diff --git a/braintrust/templates/api-configmap.yaml b/braintrust/templates/api-configmap.yaml index 6f8a18b..72d9217 100644 --- a/braintrust/templates/api-configmap.yaml +++ b/braintrust/templates/api-configmap.yaml @@ -114,7 +114,7 @@ data: {{- if .Values.aiGateway.useGateway }} GATEWAY_URL: {{ include "braintrust.aiGatewayInternalUrl" . | quote }} {{- end }} - {{- if .Values.loopRuntime.enabled }} + {{- if include "braintrust.loopRuntime.enabled" . }} LOOP_RUNTIME_URL: {{ include "braintrust.loopRuntimeInternalUrl" . | quote }} {{- end }} {{- if $btqlStrictOrgIds }} diff --git a/braintrust/templates/loop-runtime-configmap.yaml b/braintrust/templates/loop-runtime-configmap.yaml index dfebfe8..d1b08a9 100644 --- a/braintrust/templates/loop-runtime-configmap.yaml +++ b/braintrust/templates/loop-runtime-configmap.yaml @@ -1,8 +1,11 @@ {{- include "braintrust.loopRuntime.validate" . -}} -{{- if .Values.loopRuntime.enabled -}} +{{- if include "braintrust.loopRuntime.enabled" . -}} {{- $apiUrl := default (include "braintrust.apiInternalUrl" .) .Values.loopRuntime.braintrustApiUrl -}} {{- $aiProxyUrl := default (include "braintrust.apiAiProxyInternalUrl" .) .Values.loopRuntime.aiProxyUrl -}} {{- $brainstoreReaderUrl := default (include "braintrust.brainstoreFastReaderInternalUrl" .) .Values.loopRuntime.brainstoreReaderUrl -}} +{{- $brainstoreBucketUri := include "braintrust.loopRuntime.brainstoreBucketUri" . -}} +{{- $backend := include "braintrust.loopRuntime.sandboxBackend" . -}} +{{- $egressGateway := .Values.loopRuntime.sandbox.egressGateway -}} apiVersion: v1 kind: ConfigMap metadata: @@ -28,36 +31,67 @@ data: LOOP_RUNTIME_AI_PROXY_URL: {{ $aiProxyUrl | quote }} LOOP_RUNTIME_BRAINSTORE_READER_URL: {{ $brainstoreReaderUrl | quote }} BRAINSTORE_VERBOSE: "1" - BRAINSTORE_INDEX_URI: "s3://{{ .Values.objectStorage.aws.brainstoreBucket }}/brainstore/index" - BRAINSTORE_REALTIME_WAL_URI: "s3://{{ .Values.objectStorage.aws.brainstoreBucket }}/brainstore/wal" - BRAINSTORE_CODE_BUNDLE_URI: "s3://{{ .Values.objectStorage.aws.codeBundleBucket }}" + BRAINSTORE_INDEX_URI: "{{ $brainstoreBucketUri }}/brainstore/index" + BRAINSTORE_REALTIME_WAL_URI: "{{ $brainstoreBucketUri }}/brainstore/wal" + BRAINSTORE_CODE_BUNDLE_URI: {{ include "braintrust.loopRuntime.codeBundleUri" . | quote }} BRAINSTORE_CACHE_DIR: {{ .Values.loopRuntime.cacheDir | quote }} BRAINSTORE_CONTROL_PLANE_TELEMETRY: {{ .Values.global.controlPlaneTelemetry | quote }} BRAINSTORE_DISABLE_STATUS_UPDATES: {{ .Values.loopRuntime.brainstoreDisableStatusUpdates | toString | quote }} NO_COLOR: "1" - AWS_DEFAULT_REGION: {{ .Values.loopRuntime.sandbox.region | quote }} - AWS_REGION: {{ .Values.loopRuntime.sandbox.region | quote }} - EXO_SANDBOX_PROVIDER: {{ .Values.loopRuntime.sandbox.provider | quote }} - EXO_SANDBOX_IMAGE: {{ .Values.loopRuntime.sandbox.image | quote }} - AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER: {{ .Values.loopRuntime.sandbox.imageIdentifier | quote }} - AWS_LAMBDA_MICROVM_REGION: {{ .Values.loopRuntime.sandbox.region | quote }} - AWS_LAMBDA_MICROVM_INGRESS_NETWORK_CONNECTOR_ARNS: {{ .Values.loopRuntime.sandbox.ingressNetworkConnectorArns | quote }} - AWS_LAMBDA_MICROVM_EGRESS_NETWORK_CONNECTOR_ARNS: {{ .Values.loopRuntime.sandbox.egressNetworkConnectorArns | quote }} - AWS_LAMBDA_MICROVM_MAX_IDLE_DURATION_SECONDS: {{ .Values.loopRuntime.sandbox.maxIdleDurationSeconds | quote }} - AWS_LAMBDA_MICROVM_SUSPENDED_DURATION_SECONDS: {{ .Values.loopRuntime.sandbox.suspendedDurationSeconds | quote }} - AWS_LAMBDA_MICROVM_AUTO_RESUME_ENABLED: {{ .Values.loopRuntime.sandbox.autoResumeEnabled | toString | quote }} - AWS_LAMBDA_MICROVM_MAXIMUM_DURATION_SECONDS: {{ .Values.loopRuntime.sandbox.maximumDurationSeconds | quote }} - AWS_LAMBDA_MICROVM_AUTH_TOKEN_EXPIRATION_MINUTES: {{ .Values.loopRuntime.sandbox.authTokenExpirationMinutes | quote }} - AWS_LAMBDA_MICROVM_RUNTIME_PORT: {{ .Values.loopRuntime.sandbox.runtimePort | quote }} - {{- if .Values.loopRuntime.sandbox.allowExecutionRole }} - AWS_LAMBDA_MICROVM_EXECUTION_ROLE_ARN: {{ .Values.loopRuntime.sandbox.executionRoleArn | quote }} + {{- if eq $backend "awsLambdaMicrovm" }} + {{- $microvm := .Values.loopRuntime.sandbox.awsLambdaMicrovm }} + AWS_DEFAULT_REGION: {{ $microvm.region | quote }} + AWS_REGION: {{ $microvm.region | quote }} + EXO_SANDBOX_PROVIDER: "aws-lambda-microvm" + EXO_SANDBOX_IMAGE: "lambda-microvm" + AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER: {{ $microvm.imageIdentifier | quote }} + {{- with $microvm.imageVersion }} + AWS_LAMBDA_MICROVM_IMAGE_VERSION: {{ . | toString | quote }} + {{- end }} + AWS_LAMBDA_MICROVM_REGION: {{ $microvm.region | quote }} + AWS_LAMBDA_MICROVM_INGRESS_NETWORK_CONNECTOR_ARNS: {{ $microvm.ingressNetworkConnectorArns | quote }} + AWS_LAMBDA_MICROVM_EGRESS_NETWORK_CONNECTOR_ARNS: {{ $microvm.egressNetworkConnectorArns | quote }} + AWS_LAMBDA_MICROVM_MAX_IDLE_DURATION_SECONDS: {{ $microvm.maxIdleDurationSeconds | quote }} + AWS_LAMBDA_MICROVM_SUSPENDED_DURATION_SECONDS: {{ $microvm.suspendedDurationSeconds | quote }} + AWS_LAMBDA_MICROVM_AUTO_RESUME_ENABLED: {{ $microvm.autoResumeEnabled | toString | quote }} + AWS_LAMBDA_MICROVM_MAXIMUM_DURATION_SECONDS: {{ $microvm.maximumDurationSeconds | quote }} + AWS_LAMBDA_MICROVM_AUTH_TOKEN_EXPIRATION_MINUTES: {{ $microvm.authTokenExpirationMinutes | quote }} + AWS_LAMBDA_MICROVM_RUNTIME_PORT: {{ $microvm.runtimePort | quote }} + {{- if $microvm.allowExecutionRole }} + AWS_LAMBDA_MICROVM_EXECUTION_ROLE_ARN: {{ $microvm.executionRoleArn | quote }} AWS_LAMBDA_MICROVM_ALLOW_EXECUTION_ROLE: "true" {{- end }} + {{- if $egressGateway.enabled }} + LOOP_RUNTIME_SANDBOX_EGRESS_GATEWAY_URL: {{ $egressGateway.url | quote }} + {{- end }} + {{- else }} + {{- $service := .Values.loopRuntime.sandbox.sandboxService }} + {{- /* Any Sandbox Service endpoint selects the remote backend; the provider stays unset. */}} + EXO_SANDBOX_IMAGE: {{ $service.image | quote }} + EXO_SANDBOX_VCPU_COUNT: {{ $service.vcpuCount | quote }} + EXO_SANDBOX_MEMORY_MIB: {{ $service.memoryMib | quote }} + {{- if $service.dnsUrl }} + EXO_SANDBOX_SERVICE_DNS_URL: {{ $service.dnsUrl | quote }} + {{- else }} + EXO_SANDBOX_SERVICE_URL: {{ $service.urls | join "," | quote }} + {{- end }} + {{- if $service.tls.secretName }} + SANDBOX_SERVICE_TLS_CERT: "/etc/sandbox-tls/client.pem" + SANDBOX_SERVICE_TLS_KEY: "/etc/sandbox-tls/client.key" + SANDBOX_SERVICE_TLS_CA: "/etc/sandbox-tls/server-ca.pem" + {{- end }} + {{- if $egressGateway.enabled }} + LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL: {{ $egressGateway.url | quote }} + {{- end }} + {{- end }} + {{- if $egressGateway.enabled }} + LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN: "0.0.0.0:{{ $egressGateway.port }}" + {{- end }} {{- with (.Values.global.allowedOrgIds | default "" | toString | trim) }} ALLOWED_ORG_IDS: {{ . | quote }} {{- end }} {{- if eq .Values.brainstore.locksBackend "objectStorage" }} - BRAINSTORE_LOCKS_URI: "s3://{{ .Values.objectStorage.aws.brainstoreBucket }}/brainstore/locks" + BRAINSTORE_LOCKS_URI: "{{ $brainstoreBucketUri }}/brainstore/locks" {{- end }} {{- if .Values.brainstoreWalFooterVersion }} BRAINSTORE_WAL_FOOTER_VERSION: {{ .Values.brainstoreWalFooterVersion | quote }} diff --git a/braintrust/templates/loop-runtime-deployment.yaml b/braintrust/templates/loop-runtime-deployment.yaml index a705649..b4b83d6 100644 --- a/braintrust/templates/loop-runtime-deployment.yaml +++ b/braintrust/templates/loop-runtime-deployment.yaml @@ -1,7 +1,15 @@ {{- include "braintrust.loopRuntime.validate" . -}} -{{- if .Values.loopRuntime.enabled -}} +{{- if include "braintrust.loopRuntime.enabled" . -}} {{- $resourceLabels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) -}} {{- $podLabels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) (deepCopy .Values.loopRuntime.podLabels) (dict "app" .Values.loopRuntime.name) -}} +{{- if eq .Values.cloud "google" -}} +{{- $_ := set $podLabels "gke-workload-identity/use" "true" -}} +{{- end -}} +{{- $sandboxServiceTls := "" -}} +{{- if eq (include "braintrust.loopRuntime.sandboxBackend" .) "sandboxService" -}} +{{- $sandboxServiceTls = .Values.loopRuntime.sandbox.sandboxService.tls -}} +{{- end -}} +{{- $egressGateway := .Values.loopRuntime.sandbox.egressGateway -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -31,6 +39,9 @@ spec: {{- toYaml $podLabels | nindent 8 }} annotations: checksum/config: {{ include (print $.Template.BasePath "/loop-runtime-configmap.yaml") . | sha256sum }} + {{- if eq .Values.cloud "google" }} + iam.gke.io/gcp-service-account: {{ .Values.loopRuntime.serviceAccount.googleServiceAccount }} + {{- end }} {{- with .Values.loopRuntime.annotations.pod }} {{- toYaml . | nindent 8 }} {{- end }} @@ -64,6 +75,11 @@ spec: - name: {{ .Values.loopRuntime.service.portName }} containerPort: {{ .Values.loopRuntime.service.port }} protocol: TCP + {{- if $egressGateway.enabled }} + - name: egress + containerPort: {{ $egressGateway.port }} + protocol: TCP + {{- end }} resources: {{- toYaml .Values.loopRuntime.resources | nindent 12 }} {{- with .Values.loopRuntime.livenessProbe }} @@ -131,6 +147,11 @@ spec: mountPath: {{ .Values.loopRuntime.stateDir }} - name: cache-volume mountPath: {{ .Values.loopRuntime.cacheDir }} + {{- if and $sandboxServiceTls $sandboxServiceTls.secretName }} + - name: sandbox-service-tls + mountPath: /etc/sandbox-tls + readOnly: true + {{- end }} {{- with .Values.loopRuntime.extraContainers }} {{- toYaml . | nindent 8 }} {{- end }} @@ -139,6 +160,18 @@ spec: emptyDir: {} - name: cache-volume emptyDir: {} + {{- if and $sandboxServiceTls $sandboxServiceTls.secretName }} + - name: sandbox-service-tls + secret: + secretName: {{ $sandboxServiceTls.secretName }} + items: + - key: {{ $sandboxServiceTls.certKey }} + path: client.pem + - key: {{ $sandboxServiceTls.keyKey }} + path: client.key + - key: {{ $sandboxServiceTls.caKey }} + path: server-ca.pem + {{- end }} {{- with .Values.loopRuntime.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/braintrust/templates/loop-runtime-egress-service.yaml b/braintrust/templates/loop-runtime-egress-service.yaml new file mode 100644 index 0000000..3796cfb --- /dev/null +++ b/braintrust/templates/loop-runtime-egress-service.yaml @@ -0,0 +1,31 @@ +{{- include "braintrust.loopRuntime.validate" . -}} +{{- if and (include "braintrust.loopRuntime.enabled" .) .Values.loopRuntime.sandbox.egressGateway.enabled -}} +{{- $resourceLabels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) -}} +{{- $egressGateway := .Values.loopRuntime.sandbox.egressGateway -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "braintrust.loopRuntimeEgressServiceName" . }} + namespace: {{ include "braintrust.namespace" . }} + {{- with $resourceLabels }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $egressGateway.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + app: {{ .Values.loopRuntime.name }} + ports: + - name: egress + protocol: TCP + port: {{ $egressGateway.port }} + targetPort: egress + type: {{ $egressGateway.service.type }} + {{- with $egressGateway.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/braintrust/templates/loop-runtime-service.yaml b/braintrust/templates/loop-runtime-service.yaml index 9243798..1174707 100644 --- a/braintrust/templates/loop-runtime-service.yaml +++ b/braintrust/templates/loop-runtime-service.yaml @@ -1,5 +1,5 @@ {{- include "braintrust.loopRuntime.validate" . -}} -{{- if .Values.loopRuntime.enabled -}} +{{- if include "braintrust.loopRuntime.enabled" . -}} {{- $resourceLabels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) -}} apiVersion: v1 kind: Service diff --git a/braintrust/templates/loop-runtime-serviceaccount.yaml b/braintrust/templates/loop-runtime-serviceaccount.yaml index 57ebd09..ac4269f 100644 --- a/braintrust/templates/loop-runtime-serviceaccount.yaml +++ b/braintrust/templates/loop-runtime-serviceaccount.yaml @@ -1,10 +1,13 @@ {{- include "braintrust.loopRuntime.validate" . -}} -{{- if .Values.loopRuntime.enabled -}} +{{- if include "braintrust.loopRuntime.enabled" . -}} {{- $labels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) -}} {{- $annotations := mergeOverwrite (deepCopy .Values.loopRuntime.annotations.serviceaccount) (deepCopy .Values.loopRuntime.serviceAccount.annotations) -}} {{- if and (eq .Values.cloud "aws") .Values.loopRuntime.serviceAccount.awsRoleArn -}} {{- $_ := set $annotations "eks.amazonaws.com/role-arn" .Values.loopRuntime.serviceAccount.awsRoleArn -}} {{- end -}} +{{- if eq .Values.cloud "google" -}} +{{- $_ := set $annotations "iam.gke.io/gcp-service-account" .Values.loopRuntime.serviceAccount.googleServiceAccount -}} +{{- end -}} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/braintrust/templates/virtualservice.yaml b/braintrust/templates/virtualservice.yaml index a979192..9026db8 100644 --- a/braintrust/templates/virtualservice.yaml +++ b/braintrust/templates/virtualservice.yaml @@ -30,7 +30,7 @@ spec: {{- toYaml . | nindent 4 }} {{- end }} http: - {{- if .Values.loopRuntime.enabled }} + {{- if include "braintrust.loopRuntime.enabled" . }} {{- include "braintrust.loopRuntimeVirtualServiceRoutes" . | nindent 4 }} {{- end }} {{- if $virtualServiceWorkloadIsolation.enabled }} diff --git a/braintrust/tests/__fixtures__/loop-runtime-google-values.yaml b/braintrust/tests/__fixtures__/loop-runtime-google-values.yaml new file mode 100644 index 0000000..4a0687b --- /dev/null +++ b/braintrust/tests/__fixtures__/loop-runtime-google-values.yaml @@ -0,0 +1,15 @@ +# Loop Runtime on GKE with the Sandbox Service backend. Layer on top of +# base-values.yaml and google-values.yaml. +loopRuntime: + enabled: true + image: + repository: "test/loop-runtime" + tag: "v1.0.0" + serviceAccount: + name: "braintrust-loop-runtime" + googleServiceAccount: "braintrust-loop@test-project.iam.gserviceaccount.com" + sandbox: + sandboxService: + dnsUrl: "https://workers.sandbox.internal:9400" + tls: + secretName: "sandbox-client-tls" diff --git a/braintrust/tests/__fixtures__/loop-runtime-values.yaml b/braintrust/tests/__fixtures__/loop-runtime-values.yaml index 6144ba2..feca0a2 100644 --- a/braintrust/tests/__fixtures__/loop-runtime-values.yaml +++ b/braintrust/tests/__fixtures__/loop-runtime-values.yaml @@ -15,7 +15,8 @@ loopRuntime: name: "braintrust-loop-runtime" awsRoleArn: "arn:aws:iam::123456789012:role/test-loop-runtime-role" sandbox: - imageIdentifier: "arn:aws:lambda:us-east-1:123456789012:microvm-image/test" - region: "us-east-1" - ingressNetworkConnectorArns: "arn:aws:lambda:us-east-1:123456789012:network-connector/in" - egressNetworkConnectorArns: "arn:aws:lambda:us-east-1:123456789012:network-connector/out" + awsLambdaMicrovm: + imageIdentifier: "arn:aws:lambda:us-east-1:123456789012:microvm-image/test" + region: "us-east-1" + ingressNetworkConnectorArns: "arn:aws:lambda:us-east-1:123456789012:network-connector/in" + egressNetworkConnectorArns: "arn:aws:lambda:us-east-1:123456789012:network-connector/out" diff --git a/braintrust/tests/__fixtures__/pre-loop-runtime-values.yaml b/braintrust/tests/__fixtures__/pre-loop-runtime-values.yaml new file mode 100644 index 0000000..23b1551 --- /dev/null +++ b/braintrust/tests/__fixtures__/pre-loop-runtime-values.yaml @@ -0,0 +1,4 @@ +# Simulate values retained by `helm upgrade --reuse-values` from a chart release +# that predates loopRuntime. An explicit null removes the new chart default +# during Helm value coalescing. +loopRuntime: null diff --git a/braintrust/tests/__fixtures__/reuse-loop-runtime-enabled-values.yaml b/braintrust/tests/__fixtures__/reuse-loop-runtime-enabled-values.yaml new file mode 100644 index 0000000..b43c8e9 --- /dev/null +++ b/braintrust/tests/__fixtures__/reuse-loop-runtime-enabled-values.yaml @@ -0,0 +1,33 @@ +# Simulate `helm upgrade --reuse-values --set loopRuntime.enabled=true` from a +# chart that predates Loop Runtime. Explicit nulls remove the new chart +# defaults during Helm value coalescing, leaving enabled as the only field. +loopRuntime: + enabled: true + name: null + labels: null + podLabels: null + annotations: null + replicas: null + image: null + service: null + serviceAccount: null + braintrustApiUrl: null + aiProxyUrl: null + brainstoreReaderUrl: null + redisNamespace: null + lifecycleLog: null + brainstoreDisableStatusUpdates: null + stateDir: null + cacheDir: null + resources: null + podSecurityContext: null + securityContext: null + livenessProbe: null + readinessProbe: null + nodeSelector: null + tolerations: null + affinity: null + sandbox: null + extraEnvVars: null + extraContainers: null + extraVolumes: null diff --git a/braintrust/tests/api-configmap_test.yaml b/braintrust/tests/api-configmap_test.yaml index 38f80a0..414a579 100644 --- a/braintrust/tests/api-configmap_test.yaml +++ b/braintrust/tests/api-configmap_test.yaml @@ -596,3 +596,24 @@ tests: - equal: path: data.LOOP_RUNTIME_URL value: http://braintrust-loop-runtime.braintrust:4001 + + - it: should publish the Loop Runtime URL on GKE + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + release: + namespace: "braintrust" + asserts: + - equal: + path: data.LOOP_RUNTIME_URL + value: http://braintrust-loop-runtime.braintrust:4001 + + - it: should omit the Loop Runtime URL when reused values have no loopRuntime map + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/pre-loop-runtime-values.yaml + asserts: + - isNull: + path: data.LOOP_RUNTIME_URL diff --git a/braintrust/tests/loop-runtime-configmap_test.yaml b/braintrust/tests/loop-runtime-configmap_test.yaml index b7fecef..ea0c08e 100644 --- a/braintrust/tests/loop-runtime-configmap_test.yaml +++ b/braintrust/tests/loop-runtime-configmap_test.yaml @@ -9,7 +9,25 @@ tests: - hasDocuments: count: 0 - - it: should render the AWS runtime and sandbox configuration + - it: should not render when reused values have no loopRuntime map + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/pre-loop-runtime-values.yaml + asserts: + - hasDocuments: + count: 0 + + - it: should explain how to upgrade when reused values enable Loop Runtime without defaults + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/reuse-loop-runtime-enabled-values.yaml + asserts: + - failedTemplate: + errorMessage: "loopRuntime.image is missing. Values reused from a chart release that predates Loop Runtime omit the loopRuntime defaults; upgrade with --reset-then-reuse-values or supply the full loopRuntime map." + + - it: should render the AWS runtime and Lambda MicroVM sandbox configuration values: - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml @@ -34,19 +52,90 @@ tests: - equal: path: data.LOOP_RUNTIME_BRAINSTORE_READER_URL value: http://brainstore-fastreader.braintrust:4000 + - equal: + path: data.BRAINSTORE_INDEX_URI + value: s3://test-brainstore-bucket/brainstore/index + - equal: + path: data.BRAINSTORE_REALTIME_WAL_URI + value: s3://test-brainstore-bucket/brainstore/wal - equal: path: data.BRAINSTORE_CODE_BUNDLE_URI value: s3://test-code-bundle-bucket + - equal: + path: data.AWS_REGION + value: us-east-1 + - equal: + path: data.EXO_SANDBOX_PROVIDER + value: aws-lambda-microvm + - equal: + path: data.EXO_SANDBOX_IMAGE + value: lambda-microvm - equal: path: data.AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER value: arn:aws:lambda:us-east-1:123456789012:microvm-image/test - equal: path: data.AWS_LAMBDA_MICROVM_EGRESS_NETWORK_CONNECTOR_ARNS value: arn:aws:lambda:us-east-1:123456789012:network-connector/out + - isNull: + path: data.AWS_LAMBDA_MICROVM_IMAGE_VERSION + - isNull: + path: data.EXO_SANDBOX_SERVICE_DNS_URL + - isNull: + path: data.EXO_SANDBOX_SERVICE_URL + - isNull: + path: data.SANDBOX_SERVICE_TLS_CERT + - isNull: + path: data.LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN - isNull: path: data.BRAINSTORE_LOCKS_URI - - it: should use object storage locks when configured + - it: should pin the MicroVM image version when configured + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.awsLambdaMicrovm.imageVersion: 7 + asserts: + - equal: + path: data.AWS_LAMBDA_MICROVM_IMAGE_VERSION + value: "7" + + - it: should pass the MicroVM execution role when allowed + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.awsLambdaMicrovm.allowExecutionRole: true + loopRuntime.sandbox.awsLambdaMicrovm.executionRoleArn: arn:aws:iam::123456789012:role/microvm + asserts: + - equal: + path: data.AWS_LAMBDA_MICROVM_EXECUTION_ROLE_ARN + value: arn:aws:iam::123456789012:role/microvm + - equal: + path: data.AWS_LAMBDA_MICROVM_ALLOW_EXECUTION_ROLE + value: "true" + + - it: should require the execution role ARN when the execution role is allowed + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.awsLambdaMicrovm.allowExecutionRole: true + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.awsLambdaMicrovm.executionRoleArn is required when loopRuntime.sandbox.awsLambdaMicrovm.allowExecutionRole is true" + + - it: should require MicroVM settings for the AWS default backend + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.awsLambdaMicrovm.imageIdentifier: "" + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.awsLambdaMicrovm.imageIdentifier is required for the awsLambdaMicrovm backend (use Terraform output loop_runtime_microvm_image_arn)" + + - it: should use AWS object storage locks when configured values: - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml @@ -57,11 +146,316 @@ tests: path: data.BRAINSTORE_LOCKS_URI value: s3://test-brainstore-bucket/brainstore/locks - - it: should reject Loop Runtime on non-AWS clouds + - it: should run the Sandbox Service backend on AWS without MicroVM settings + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.backend: sandboxService + loopRuntime.sandbox.awsLambdaMicrovm.imageIdentifier: "" + loopRuntime.sandbox.sandboxService.dnsUrl: https://nodes.sandbox.internal:9400 + loopRuntime.sandbox.sandboxService.tls.secretName: sandbox-client-tls + asserts: + - equal: + path: data.BRAINSTORE_INDEX_URI + value: s3://test-brainstore-bucket/brainstore/index + - equal: + path: data.EXO_SANDBOX_SERVICE_DNS_URL + value: https://nodes.sandbox.internal:9400 + - isNull: + path: data.EXO_SANDBOX_PROVIDER + - isNull: + path: data.AWS_REGION + - isNull: + path: data.AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER + + - it: should render GKE storage and the Sandbox Service backend by default on Google + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + release: + namespace: braintrust + asserts: + - equal: + path: data.BRAINSTORE_INDEX_URI + value: gs://test-brainstore-bucket/brainstore/index + - equal: + path: data.BRAINSTORE_REALTIME_WAL_URI + value: gs://test-brainstore-bucket/brainstore/wal + - equal: + path: data.BRAINSTORE_CODE_BUNDLE_URI + value: gs://test-api-bucket + - equal: + path: data.EXO_SANDBOX_IMAGE + value: public.ecr.aws/braintrust/codex-runtime:latest + - equal: + path: data.EXO_SANDBOX_VCPU_COUNT + value: "1" + - equal: + path: data.EXO_SANDBOX_MEMORY_MIB + value: "2048" + - equal: + path: data.EXO_SANDBOX_SERVICE_DNS_URL + value: https://workers.sandbox.internal:9400 + - equal: + path: data.SANDBOX_SERVICE_TLS_CERT + value: /etc/sandbox-tls/client.pem + - equal: + path: data.SANDBOX_SERVICE_TLS_KEY + value: /etc/sandbox-tls/client.key + - equal: + path: data.SANDBOX_SERVICE_TLS_CA + value: /etc/sandbox-tls/server-ca.pem + - isNull: + path: data.EXO_SANDBOX_PROVIDER + - isNull: + path: data.EXO_SANDBOX_SERVICE_URL + - isNull: + path: data.AWS_REGION + - isNull: + path: data.AWS_DEFAULT_REGION + - isNull: + path: data.AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER + - isNull: + path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL + - isNull: + path: data.BRAINSTORE_LOCKS_URI + + - it: should use GCS object storage locks when configured + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + brainstore.locksBackend: objectStorage + asserts: + - equal: + path: data.BRAINSTORE_LOCKS_URI + value: gs://test-brainstore-bucket/brainstore/locks + + - it: should join fixed Sandbox Service endpoints values: - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.sandboxService.dnsUrl: "" + loopRuntime.sandbox.sandboxService.urls: + - https://worker-0.sandbox.internal:9400 + - https://worker-1.sandbox.internal:9400 + asserts: + - equal: + path: data.EXO_SANDBOX_SERVICE_URL + value: https://worker-0.sandbox.internal:9400,https://worker-1.sandbox.internal:9400 + - isNull: + path: data.EXO_SANDBOX_SERVICE_DNS_URL + + - it: should allow a plain HTTP Sandbox Service endpoint without TLS + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.sandboxService.dnsUrl: http://sandbox-service.sandbox:9400 + loopRuntime.sandbox.sandboxService.tls.secretName: "" + asserts: + - equal: + path: data.EXO_SANDBOX_SERVICE_DNS_URL + value: http://sandbox-service.sandbox:9400 + - isNull: + path: data.SANDBOX_SERVICE_TLS_CERT + + - it: should require TLS for HTTPS Sandbox Service endpoints + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.sandboxService.tls.secretName: "" + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.sandboxService.tls.secretName is required for https:// Sandbox Service endpoints" + + - it: should require TLS when any fixed endpoint uses HTTPS + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.sandboxService.dnsUrl: "" + loopRuntime.sandbox.sandboxService.tls.secretName: "" + loopRuntime.sandbox.sandboxService.urls: + - http://worker-0.sandbox.internal:9400 + - https://worker-1.sandbox.internal:9400 + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.sandboxService.tls.secretName is required for https:// Sandbox Service endpoints" + + - it: should require a Sandbox Service endpoint + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.sandboxService.dnsUrl: "" + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.sandboxService.dnsUrl or loopRuntime.sandbox.sandboxService.urls is required for the sandboxService backend (GKE: use Terraform output gke_isolated_workers.discovery.endpoint)" + + - it: should reject both Sandbox Service discovery modes + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.sandboxService.urls: + - https://worker-0.sandbox.internal:9400 + asserts: + - failedTemplate: + errorMessage: "Set only one of loopRuntime.sandbox.sandboxService.dnsUrl and loopRuntime.sandbox.sandboxService.urls." + + - it: should reject a string of Sandbox Service endpoints + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.sandboxService.dnsUrl: "" + loopRuntime.sandbox.sandboxService.urls: https://worker-0.sandbox.internal:9400 + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.sandboxService.urls must be a list of URLs" + + - it: should require the Sandbox Service guest image + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.sandboxService.image: "" + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.sandboxService.image is required for the sandboxService backend" + + - it: should require the Loop Runtime Google service account on GKE + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.serviceAccount.googleServiceAccount: "" + asserts: + - failedTemplate: + errorMessage: "loopRuntime.serviceAccount.googleServiceAccount is required when cloud is google (use Terraform output loop_runtime_service_account)" + + - it: should require the GCS API bucket on GKE + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + objectStorage.google.apiBucket: "" + asserts: + - failedTemplate: + errorMessage: "objectStorage.google.apiBucket is required when loopRuntime.enabled is true" + + - it: should reject the Lambda MicroVM backend on Google + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.backend: awsLambdaMicrovm + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.backend awsLambdaMicrovm requires cloud: aws. Use sandboxService on other clouds." + + - it: should reject an unknown sandbox backend + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.backend: docker + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.backend must be awsLambdaMicrovm or sandboxService, got \"docker\"" + + - it: should reject Loop Runtime on Azure + values: + - __fixtures__/base-values.yaml + - __fixtures__/azure-values.yaml set: loopRuntime.enabled: true asserts: - failedTemplate: - errorMessage: "loopRuntime.enabled requires cloud: aws because the current sandbox backend uses AWS Lambda MicroVMs." + errorMessage: "loopRuntime.enabled requires cloud: aws or cloud: google." + + - it: should reject Loop Runtime without a cloud + values: + - __fixtures__/base-values.yaml + set: + loopRuntime.enabled: true + asserts: + - failedTemplate: + errorMessage: "loopRuntime.enabled requires cloud: aws or cloud: google." + + - it: should advertise the egress gateway to MicroVM sandboxes + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.egressGateway.enabled: true + loopRuntime.sandbox.egressGateway.url: http://egress.vpce.internal:4002 + asserts: + - equal: + path: data.LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN + value: 0.0.0.0:4002 + - equal: + path: data.LOOP_RUNTIME_SANDBOX_EGRESS_GATEWAY_URL + value: http://egress.vpce.internal:4002 + - isNull: + path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL + + - it: should advertise the egress gateway to Sandbox Service workers + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.egressGateway.enabled: true + loopRuntime.sandbox.egressGateway.port: 4100 + loopRuntime.sandbox.egressGateway.url: http://10.10.0.20:4100 + asserts: + - equal: + path: data.LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN + value: 0.0.0.0:4100 + - equal: + path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL + value: http://10.10.0.20:4100 + - isNull: + path: data.LOOP_RUNTIME_SANDBOX_EGRESS_GATEWAY_URL + + - it: should require the egress gateway URL when the gateway is enabled + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.egressGateway.enabled: true + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.url is required when loopRuntime.sandbox.egressGateway.enabled is true" + + - it: should keep the egress gateway off the runtime port + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.egressGateway.enabled: true + loopRuntime.sandbox.egressGateway.url: http://10.10.0.20:4001 + loopRuntime.sandbox.egressGateway.port: 4001 + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.port must differ from loopRuntime.service.port" diff --git a/braintrust/tests/loop-runtime-deployment_test.yaml b/braintrust/tests/loop-runtime-deployment_test.yaml index 4c9d114..e7929ad 100644 --- a/braintrust/tests/loop-runtime-deployment_test.yaml +++ b/braintrust/tests/loop-runtime-deployment_test.yaml @@ -89,3 +89,140 @@ tests: - equal: path: spec.template.spec.containers[0].readinessProbe.httpGet.port value: metrics + + - it: should not use GKE workload identity on AWS + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + asserts: + - isNull: + path: spec.template.metadata.labels["gke-workload-identity/use"] + - isNull: + path: spec.template.metadata.annotations["iam.gke.io/gcp-service-account"] + - lengthEqual: + path: spec.template.spec.volumes + count: 2 + - lengthEqual: + path: spec.template.spec.containers[0].ports + count: 1 + + - it: should use GKE workload identity and mount Sandbox Service client TLS on Google + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + asserts: + - equal: + path: spec.template.metadata.labels["gke-workload-identity/use"] + value: "true" + - equal: + path: spec.template.metadata.annotations["iam.gke.io/gcp-service-account"] + value: braintrust-loop@test-project.iam.gserviceaccount.com + - equal: + path: spec.template.spec.serviceAccountName + value: braintrust-loop-runtime + - isNull: + path: spec.template.spec.nodeSelector + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: sandbox-service-tls + mountPath: /etc/sandbox-tls + readOnly: true + - contains: + path: spec.template.spec.volumes + content: + name: sandbox-service-tls + secret: + secretName: sandbox-client-tls + items: + - key: client.pem + path: client.pem + - key: client.key + path: client.key + - key: server-ca.pem + path: server-ca.pem + + - it: should map custom TLS secret keys to the paths Loop Runtime reads + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.sandboxService.tls.secretName: cert-manager-client + loopRuntime.sandbox.sandboxService.tls.certKey: tls.crt + loopRuntime.sandbox.sandboxService.tls.keyKey: tls.key + loopRuntime.sandbox.sandboxService.tls.caKey: ca.crt + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: sandbox-service-tls + secret: + secretName: cert-manager-client + items: + - key: tls.crt + path: client.pem + - key: tls.key + path: client.key + - key: ca.crt + path: server-ca.pem + + - it: should omit the TLS volume for a plain HTTP Sandbox Service endpoint + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.sandboxService.dnsUrl: http://sandbox-service.sandbox:9400 + loopRuntime.sandbox.sandboxService.tls.secretName: "" + asserts: + - lengthEqual: + path: spec.template.spec.volumes + count: 2 + - lengthEqual: + path: spec.template.spec.containers[0].volumeMounts + count: 2 + + - it: should ignore Sandbox Service TLS settings for the MicroVM backend + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.sandboxService.tls.secretName: sandbox-client-tls + asserts: + - lengthEqual: + path: spec.template.spec.volumes + count: 2 + + - it: should expose the egress gateway container port when enabled + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.egressGateway.enabled: true + loopRuntime.sandbox.egressGateway.url: http://10.10.0.20:4002 + asserts: + - contains: + path: spec.template.spec.containers[0].ports + content: + name: egress + containerPort: 4002 + protocol: TCP + + - it: should not render when reused values have no loopRuntime map + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/pre-loop-runtime-values.yaml + asserts: + - hasDocuments: + count: 0 diff --git a/braintrust/tests/loop-runtime-egress-service_test.yaml b/braintrust/tests/loop-runtime-egress-service_test.yaml new file mode 100644 index 0000000..1ba6f35 --- /dev/null +++ b/braintrust/tests/loop-runtime-egress-service_test.yaml @@ -0,0 +1,94 @@ +suite: test Loop Runtime egress gateway Service template +templates: + - loop-runtime-egress-service.yaml +tests: + - it: should not render when Loop Runtime is disabled + values: + - __fixtures__/base-values.yaml + asserts: + - hasDocuments: + count: 0 + + - it: should not render when the egress gateway is disabled + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + asserts: + - hasDocuments: + count: 0 + + - it: should not render when reused values have no loopRuntime map + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/pre-loop-runtime-values.yaml + asserts: + - hasDocuments: + count: 0 + + - it: should expose the egress gateway through a ClusterIP Service by default + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.egressGateway.enabled: true + loopRuntime.sandbox.egressGateway.url: http://egress.vpce.internal:4002 + release: + namespace: braintrust + asserts: + - isKind: + of: Service + - equal: + path: metadata.name + value: braintrust-loop-runtime-egress + - equal: + path: spec.selector.app + value: braintrust-loop-runtime + - equal: + path: spec.ports[0] + value: + name: egress + protocol: TCP + port: 4002 + targetPort: egress + - equal: + path: spec.type + value: ClusterIP + - isNull: + path: spec.loadBalancerSourceRanges + - isNull: + path: metadata.annotations + + - it: should expose the egress gateway through a GKE internal load balancer + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.service.name: loop + loopRuntime.labels.loop-specific: loop-egress + loopRuntime.sandbox.egressGateway.enabled: true + loopRuntime.sandbox.egressGateway.url: http://10.10.0.20:4002 + loopRuntime.sandbox.egressGateway.service.type: LoadBalancer + loopRuntime.sandbox.egressGateway.service.annotations: + networking.gke.io/load-balancer-type: Internal + loopRuntime.sandbox.egressGateway.service.loadBalancerSourceRanges: + - 10.64.64.0/24 + asserts: + - equal: + path: metadata.name + value: loop-egress + - equal: + path: metadata.labels["loop-specific"] + value: loop-egress + - equal: + path: spec.type + value: LoadBalancer + - equal: + path: metadata.annotations["networking.gke.io/load-balancer-type"] + value: Internal + - equal: + path: spec.loadBalancerSourceRanges + value: + - 10.64.64.0/24 diff --git a/braintrust/tests/loop-runtime-service_test.yaml b/braintrust/tests/loop-runtime-service_test.yaml index f75368f..a5da702 100644 --- a/braintrust/tests/loop-runtime-service_test.yaml +++ b/braintrust/tests/loop-runtime-service_test.yaml @@ -31,3 +31,22 @@ tests: - equal: path: spec.type value: ClusterIP + + - it: should expose the runtime port on GKE + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + asserts: + - equal: + path: spec.ports[0].port + value: 4001 + + - it: should not render when reused values have no loopRuntime map + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/pre-loop-runtime-values.yaml + asserts: + - hasDocuments: + count: 0 diff --git a/braintrust/tests/loop-runtime-serviceaccount_test.yaml b/braintrust/tests/loop-runtime-serviceaccount_test.yaml index 4f7498d..5e472bb 100644 --- a/braintrust/tests/loop-runtime-serviceaccount_test.yaml +++ b/braintrust/tests/loop-runtime-serviceaccount_test.yaml @@ -43,3 +43,29 @@ tests: - equal: path: metadata.annotations["custom-annotation"] value: value + + - it: should bind the Loop Runtime Google service account on GKE + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + asserts: + - equal: + path: metadata.name + value: braintrust-loop-runtime + - equal: + path: metadata.annotations["iam.gke.io/gcp-service-account"] + value: braintrust-loop@test-project.iam.gserviceaccount.com + - isNull: + path: metadata.annotations["eks.amazonaws.com/role-arn"] + + - it: should not add the IRSA annotation on GKE + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.serviceAccount.awsRoleArn: arn:aws:iam::123456789012:role/ignored + asserts: + - isNull: + path: metadata.annotations["eks.amazonaws.com/role-arn"] diff --git a/braintrust/tests/virtualservice_test.yaml b/braintrust/tests/virtualservice_test.yaml index 5853664..d659b15 100644 --- a/braintrust/tests/virtualservice_test.yaml +++ b/braintrust/tests/virtualservice_test.yaml @@ -359,3 +359,27 @@ tests: - equal: path: spec.http[1].match[0].uri.prefix value: / + + - it: should route Loop Runtime on GKE + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + virtualService.enabled: true + asserts: + - equal: + path: spec.http[0].route[0].destination.host + value: braintrust-loop-runtime + + - it: should render without Loop Runtime routes when reused values have no loopRuntime map + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/pre-loop-runtime-values.yaml + set: + virtualService.enabled: true + asserts: + - equal: + path: spec.http[0].match[0].uri.prefix + value: / diff --git a/braintrust/values.yaml b/braintrust/values.yaml index f372c47..b4d2625 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -309,9 +309,9 @@ api: extraContainers: [] extraVolumes: [] -# Loop Runtime configuration (AWS EKS only) -# Terraform creates the AWS MicroVM image, network connectors, and IAM role; -# this chart runs the Loop Runtime process in Kubernetes. +# Loop Runtime configuration (AWS EKS and Google GKE) +# The cloud's Terraform module creates the runtime identity and the sandbox +# infrastructure; this chart runs the Loop Runtime process in Kubernetes. loopRuntime: enabled: false name: "braintrust-loop-runtime" @@ -336,9 +336,11 @@ loopRuntime: portName: http serviceAccount: name: "braintrust-loop-runtime" - # Set this to Terraform's loop_runtime_eks_role_arn when using IRSA. Leave + # AWS: set this to Terraform's Loop Runtime role ARN when using IRSA. Leave # it empty when the role is attached through an EKS Pod Identity association. awsRoleArn: "" + # Google: required. Use the Terraform output loop_runtime_service_account. + googleServiceAccount: "" annotations: {} # These URLs default to in-cluster services. Set them when the API, AI proxy, # or Brainstore is exposed through a different service or namespace. @@ -391,21 +393,63 @@ loopRuntime: tolerations: [] affinity: {} sandbox: - provider: "aws-lambda-microvm" - image: "lambda-microvm" - # These values come from Terraform outputs. They are required when enabled. - imageIdentifier: "" - region: "" - ingressNetworkConnectorArns: "" - egressNetworkConnectorArns: "" - maxIdleDurationSeconds: 900 - suspendedDurationSeconds: 28800 - autoResumeEnabled: true - maximumDurationSeconds: 28800 - authTokenExpirationMinutes: 30 - runtimePort: 8080 - executionRoleArn: "" - allowExecutionRole: false + # Where Loop Runtime runs agent sandboxes: + # awsLambdaMicrovm - AWS Lambda MicroVMs. Requires cloud: aws. + # sandboxService - Braintrust Sandbox Service workers reached over mTLS. + # Works on any cloud; on GKE the workers run in the + # Terraform-managed isolated worker cluster. + # Empty selects awsLambdaMicrovm on AWS and sandboxService on Google. + backend: "" + awsLambdaMicrovm: + # These values come from Terraform outputs. They are required for this backend. + imageIdentifier: "" + # Optional. Pins sandboxes to one published MicroVM image version. + imageVersion: "" + region: "" + ingressNetworkConnectorArns: "" + egressNetworkConnectorArns: "" + maxIdleDurationSeconds: 900 + suspendedDurationSeconds: 28800 + autoResumeEnabled: true + maximumDurationSeconds: 28800 + authTokenExpirationMinutes: 30 + runtimePort: 8080 + executionRoleArn: "" + allowExecutionRole: false + sandboxService: + # Set exactly one. dnsUrl discovers every worker behind one headless DNS + # name (GKE: Terraform output gke_isolated_workers.discovery.endpoint). + # urls lists fixed worker endpoints instead. + dnsUrl: "" + urls: [] + # Sandbox guest image the workers start. + image: "public.ecr.aws/braintrust/codex-runtime:latest" + # Must match the Sandbox Service release's per-sandbox sizing. + vcpuCount: 1 + memoryMib: 2048 + tls: + # Kubernetes Secret with the Sandbox Service client certificate, its + # key, and the server CA. Required for https:// endpoints. + secretName: "" + certKey: client.pem + keyKey: client.key + caKey: server-ca.pem + # Optional forward proxy that sandboxes use for brokered Git access. When + # enabled, Loop Runtime listens on port and advertises url to sandboxes. + egressGateway: + enabled: false + port: 4002 + # Address sandboxes use to reach this proxy, e.g. http://:4002. + url: "" + # Dedicated Service for the proxy port. Sandboxes usually run outside this + # cluster, so expose it through an internal load balancer. For GKE, set + # type LoadBalancer with the networking.gke.io/load-balancer-type: + # "Internal" annotation and restrict loadBalancerSourceRanges to the + # isolated worker node range. + service: + type: ClusterIP + annotations: {} + loadBalancerSourceRanges: [] extraEnvVars: [] extraContainers: [] extraVolumes: [] From a52428556e9f1d73f1994b1bc529efc653628efe Mon Sep 17 00:00:00 2001 From: Alex R Date: Tue, 20 Oct 2026 19:44:04 +0000 Subject: [PATCH 03/25] Run Loop Runtime sandboxes only on the Sandbox Service Drop the AWS Lambda MicroVM backend. Loop Runtime on EKS and GKE now sends every sandbox to Braintrust Sandbox Service workers. With a Sandbox Service endpoint and no EXO_SANDBOX_PROVIDER, the runtime uses the Sandbox Service for all sandboxes instead of the MicroVM default plus a LaunchDarkly rollout, as the CloudFormation mixed mode does. Flatten loopRuntime.sandbox.sandboxService.* into loopRuntime.sandbox.* and expose the sandbox egress mode. Co-Authored-By: Claude Opus 5.5 --- braintrust/README.md | 79 +++---- braintrust/templates/_helpers.tpl | 57 ++--- .../templates/loop-runtime-configmap.yaml | 54 +---- .../templates/loop-runtime-deployment.yaml | 9 +- .../loop-runtime-google-values.yaml | 10 +- .../__fixtures__/loop-runtime-values.yaml | 8 +- .../tests/loop-runtime-configmap_test.yaml | 212 ++++++------------ .../tests/loop-runtime-deployment_test.yaml | 27 +-- braintrust/values.yaml | 74 +++--- 9 files changed, 184 insertions(+), 346 deletions(-) diff --git a/braintrust/README.md b/braintrust/README.md index 6cdf220..3cc24d6 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -21,82 +21,73 @@ The `braintrust-secrets` secret must contain the following keys: ## Optional: Loop Runtime Set `loopRuntime.enabled: true` to run the Loop Runtime service in the cluster. -It supports `cloud: aws` (EKS) and `cloud: google` (GKE). The cloud's -Terraform data-plane module creates the runtime identity and the sandbox -infrastructure; the chart runs the runtime process. +It supports `cloud: aws` (EKS) and `cloud: google` (GKE). Loop Runtime runs +agent sandboxes on Braintrust Sandbox Service workers, which it reaches over +mTLS. The cloud's Terraform data-plane module creates the runtime identity and +the Sandbox Service infrastructure; the chart runs the runtime process. The chart reuses the existing `braintrust-secrets` values, reads the same Brainstore buckets as the Brainstore pods, adds `LOOP_RUNTIME_URL` to the API, and adds `/loop/runtime` routes to the chart-managed Istio VirtualService when `virtualService.enabled` is true. -`loopRuntime.sandbox.backend` selects where agent sandboxes run: +Create a Secret in this namespace with the Sandbox Service client certificate, +its key, and the server CA, and point `loopRuntime.sandbox.tls.secretName` at +it. `sandbox.vcpuCount` and `sandbox.memoryMib` must match the per-sandbox +sizing of the Sandbox Service release. -| Backend | Clouds | Sandboxes run in | -| --- | --- | --- | -| `awsLambdaMicrovm` (default on AWS) | `aws` | AWS Lambda MicroVMs | -| `sandboxService` (default on Google) | `aws`, `google` | Braintrust Sandbox Service workers, reached over mTLS | +### Google GKE -### AWS EKS with Lambda MicroVMs +The GKE Terraform module creates a Google service account for Loop Runtime and +binds it to the `braintrust/braintrust-loop-runtime` Kubernetes service account +through Workload Identity. Its isolated worker cluster hosts the Sandbox +Service. ```yaml -cloud: aws +cloud: google objectStorage: - aws: - brainstoreBucket: "" - codeBundleBucket: "" + google: + brainstoreBucket: "" + apiBucket: "" loopRuntime: enabled: true serviceAccount: name: braintrust-loop-runtime - # Set for IRSA; leave empty when using an EKS Pod Identity association. - awsRoleArn: "" + googleServiceAccount: "" sandbox: - awsLambdaMicrovm: - imageIdentifier: "" - region: "" - ingressNetworkConnectorArns: "" - egressNetworkConnectorArns: "" + dnsUrl: "" + tls: + secretName: sandbox-client-tls + # Keys in that Secret. Defaults match the Sandbox Service client Secret. + certKey: client.pem + keyKey: client.key + caKey: server-ca.pem ``` -### Google GKE with the Sandbox Service - -The GKE Terraform module creates a Google service account for Loop Runtime and -binds it to the `braintrust/braintrust-loop-runtime` Kubernetes service account -through Workload Identity. Its optional isolated worker cluster hosts the -Sandbox Service. Install the Sandbox Service release in that cluster, then -create a Secret in this namespace with the Sandbox Service client certificate, -its key, and the server CA. +### AWS EKS ```yaml -cloud: google +cloud: aws objectStorage: - google: - brainstoreBucket: "" - apiBucket: "" + aws: + brainstoreBucket: "" + codeBundleBucket: "" loopRuntime: enabled: true serviceAccount: name: braintrust-loop-runtime - googleServiceAccount: "" + # Set for IRSA; leave empty when using an EKS Pod Identity association. + awsRoleArn: "" sandbox: - sandboxService: - dnsUrl: "" - tls: - secretName: sandbox-client-tls - # Keys in that Secret. Defaults match the Sandbox Service client Secret. - certKey: client.pem - keyKey: client.key - caKey: server-ca.pem + dnsUrl: "https://:9400" + tls: + secretName: sandbox-client-tls ``` -`sandboxService.vcpuCount` and `sandboxService.memoryMib` must match the -per-sandbox sizing of the Sandbox Service release. - ### Optional sandbox egress gateway Sandboxes use a forward proxy in Loop Runtime for brokered Git access. Enable diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 3da68d3..14bb29a 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -154,14 +154,6 @@ predates Loop Runtime have no loopRuntime map, which means disabled. {{- if (.Values.loopRuntime | default dict).enabled -}}true{{- end -}} {{- end -}} -{{/* -Loop Runtime sandbox backend. Empty selects the cloud default: -awsLambdaMicrovm on AWS and sandboxService elsewhere. -*/}} -{{- define "braintrust.loopRuntime.sandboxBackend" -}} -{{- .Values.loopRuntime.sandbox.backend | default (eq .Values.cloud "aws" | ternary "awsLambdaMicrovm" "sandboxService") -}} -{{- end -}} - {{/* Object-store scheme and Brainstore bucket for Loop Runtime. Matches the Brainstore ConfigMaps so both read the same index, WAL, and locks. @@ -184,7 +176,7 @@ s3://{{ .Values.objectStorage.aws.codeBundleBucket }} {{/* Validate the Loop Runtime configuration. Terraform supplies the identity and -sandbox infrastructure; Helm supplies the Kubernetes workload. +Sandbox Service infrastructure; Helm supplies the Kubernetes workload. */}} {{- define "braintrust.loopRuntime.validate" -}} {{- if include "braintrust.loopRuntime.enabled" . -}} @@ -194,7 +186,6 @@ sandbox infrastructure; Helm supplies the Kubernetes workload. {{- end -}} {{- end -}} {{- $sandbox := .Values.loopRuntime.sandbox -}} -{{- $backend := include "braintrust.loopRuntime.sandboxBackend" . -}} {{- if not (has .Values.cloud (list "aws" "google")) -}} {{- fail "loopRuntime.enabled requires cloud: aws or cloud: google." -}} {{- end -}} @@ -207,43 +198,29 @@ sandbox infrastructure; Helm supplies the Kubernetes workload. {{- $_ := required "loopRuntime.serviceAccount.googleServiceAccount is required when cloud is google (use Terraform output loop_runtime_service_account)" .Values.loopRuntime.serviceAccount.googleServiceAccount -}} {{- end -}} {{- $_ := required "loopRuntime.serviceAccount.name is required when loopRuntime.enabled is true" .Values.loopRuntime.serviceAccount.name -}} -{{- if eq $backend "awsLambdaMicrovm" -}} -{{- if ne .Values.cloud "aws" -}} -{{- fail "loopRuntime.sandbox.backend awsLambdaMicrovm requires cloud: aws. Use sandboxService on other clouds." -}} -{{- end -}} -{{- $microvm := $sandbox.awsLambdaMicrovm -}} -{{- $_ := required "loopRuntime.sandbox.awsLambdaMicrovm.imageIdentifier is required for the awsLambdaMicrovm backend (use Terraform output loop_runtime_microvm_image_arn)" $microvm.imageIdentifier -}} -{{- $_ := required "loopRuntime.sandbox.awsLambdaMicrovm.region is required for the awsLambdaMicrovm backend" $microvm.region -}} -{{- $_ := required "loopRuntime.sandbox.awsLambdaMicrovm.ingressNetworkConnectorArns is required for the awsLambdaMicrovm backend (use Terraform sandbox output)" $microvm.ingressNetworkConnectorArns -}} -{{- $_ := required "loopRuntime.sandbox.awsLambdaMicrovm.egressNetworkConnectorArns is required for the awsLambdaMicrovm backend (use Terraform sandbox output)" $microvm.egressNetworkConnectorArns -}} -{{- if $microvm.allowExecutionRole -}} -{{- $_ := required "loopRuntime.sandbox.awsLambdaMicrovm.executionRoleArn is required when loopRuntime.sandbox.awsLambdaMicrovm.allowExecutionRole is true" $microvm.executionRoleArn -}} -{{- end -}} -{{- else if eq $backend "sandboxService" -}} -{{- $service := $sandbox.sandboxService -}} -{{- $urls := $service.urls | default list -}} +{{- $urls := $sandbox.urls | default list -}} {{- if not (kindIs "slice" $urls) -}} -{{- fail "loopRuntime.sandbox.sandboxService.urls must be a list of URLs" -}} +{{- fail "loopRuntime.sandbox.urls must be a list of URLs" -}} {{- end -}} -{{- if and $service.dnsUrl $urls -}} -{{- fail "Set only one of loopRuntime.sandbox.sandboxService.dnsUrl and loopRuntime.sandbox.sandboxService.urls." -}} +{{- if and $sandbox.dnsUrl $urls -}} +{{- fail "Set only one of loopRuntime.sandbox.dnsUrl and loopRuntime.sandbox.urls." -}} {{- end -}} -{{- if not (or $service.dnsUrl $urls) -}} -{{- fail "loopRuntime.sandbox.sandboxService.dnsUrl or loopRuntime.sandbox.sandboxService.urls is required for the sandboxService backend (GKE: use Terraform output gke_isolated_workers.discovery.endpoint)" -}} +{{- if not (or $sandbox.dnsUrl $urls) -}} +{{- fail "loopRuntime.sandbox.dnsUrl or loopRuntime.sandbox.urls is required when loopRuntime.enabled is true (GKE: use Terraform output gke_isolated_workers.discovery.endpoint)" -}} {{- end -}} -{{- $_ := required "loopRuntime.sandbox.sandboxService.image is required for the sandboxService backend" $service.image -}} -{{- range (prepend $urls ($service.dnsUrl | default "")) -}} -{{- if and . (hasPrefix "https://" .) (not $service.tls.secretName) -}} -{{- fail "loopRuntime.sandbox.sandboxService.tls.secretName is required for https:// Sandbox Service endpoints" -}} +{{- $_ := required "loopRuntime.sandbox.image is required when loopRuntime.enabled is true" $sandbox.image -}} +{{- if not (has $sandbox.egressMode (list "restricted" "internet")) -}} +{{- fail (printf "loopRuntime.sandbox.egressMode must be restricted or internet, got %q" (toString $sandbox.egressMode)) -}} {{- end -}} +{{- range (prepend $urls ($sandbox.dnsUrl | default "")) -}} +{{- if and . (hasPrefix "https://" .) (not $sandbox.tls.secretName) -}} +{{- fail "loopRuntime.sandbox.tls.secretName is required for https:// Sandbox Service endpoints" -}} {{- end -}} -{{- if $service.tls.secretName -}} -{{- $_ := required "loopRuntime.sandbox.sandboxService.tls.certKey is required when tls.secretName is set" $service.tls.certKey -}} -{{- $_ := required "loopRuntime.sandbox.sandboxService.tls.keyKey is required when tls.secretName is set" $service.tls.keyKey -}} -{{- $_ := required "loopRuntime.sandbox.sandboxService.tls.caKey is required when tls.secretName is set" $service.tls.caKey -}} {{- end -}} -{{- else -}} -{{- fail (printf "loopRuntime.sandbox.backend must be awsLambdaMicrovm or sandboxService, got %q" $backend) -}} +{{- if $sandbox.tls.secretName -}} +{{- $_ := required "loopRuntime.sandbox.tls.certKey is required when tls.secretName is set" $sandbox.tls.certKey -}} +{{- $_ := required "loopRuntime.sandbox.tls.keyKey is required when tls.secretName is set" $sandbox.tls.keyKey -}} +{{- $_ := required "loopRuntime.sandbox.tls.caKey is required when tls.secretName is set" $sandbox.tls.caKey -}} {{- end -}} {{- if $sandbox.egressGateway.enabled -}} {{- $_ := required "loopRuntime.sandbox.egressGateway.url is required when loopRuntime.sandbox.egressGateway.enabled is true" $sandbox.egressGateway.url -}} diff --git a/braintrust/templates/loop-runtime-configmap.yaml b/braintrust/templates/loop-runtime-configmap.yaml index d1b08a9..502e61e 100644 --- a/braintrust/templates/loop-runtime-configmap.yaml +++ b/braintrust/templates/loop-runtime-configmap.yaml @@ -4,8 +4,8 @@ {{- $aiProxyUrl := default (include "braintrust.apiAiProxyInternalUrl" .) .Values.loopRuntime.aiProxyUrl -}} {{- $brainstoreReaderUrl := default (include "braintrust.brainstoreFastReaderInternalUrl" .) .Values.loopRuntime.brainstoreReaderUrl -}} {{- $brainstoreBucketUri := include "braintrust.loopRuntime.brainstoreBucketUri" . -}} -{{- $backend := include "braintrust.loopRuntime.sandboxBackend" . -}} -{{- $egressGateway := .Values.loopRuntime.sandbox.egressGateway -}} +{{- $sandbox := .Values.loopRuntime.sandbox -}} +{{- $egressGateway := $sandbox.egressGateway -}} apiVersion: v1 kind: ConfigMap metadata: @@ -38,54 +38,24 @@ data: BRAINSTORE_CONTROL_PLANE_TELEMETRY: {{ .Values.global.controlPlaneTelemetry | quote }} BRAINSTORE_DISABLE_STATUS_UPDATES: {{ .Values.loopRuntime.brainstoreDisableStatusUpdates | toString | quote }} NO_COLOR: "1" - {{- if eq $backend "awsLambdaMicrovm" }} - {{- $microvm := .Values.loopRuntime.sandbox.awsLambdaMicrovm }} - AWS_DEFAULT_REGION: {{ $microvm.region | quote }} - AWS_REGION: {{ $microvm.region | quote }} - EXO_SANDBOX_PROVIDER: "aws-lambda-microvm" - EXO_SANDBOX_IMAGE: "lambda-microvm" - AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER: {{ $microvm.imageIdentifier | quote }} - {{- with $microvm.imageVersion }} - AWS_LAMBDA_MICROVM_IMAGE_VERSION: {{ . | toString | quote }} - {{- end }} - AWS_LAMBDA_MICROVM_REGION: {{ $microvm.region | quote }} - AWS_LAMBDA_MICROVM_INGRESS_NETWORK_CONNECTOR_ARNS: {{ $microvm.ingressNetworkConnectorArns | quote }} - AWS_LAMBDA_MICROVM_EGRESS_NETWORK_CONNECTOR_ARNS: {{ $microvm.egressNetworkConnectorArns | quote }} - AWS_LAMBDA_MICROVM_MAX_IDLE_DURATION_SECONDS: {{ $microvm.maxIdleDurationSeconds | quote }} - AWS_LAMBDA_MICROVM_SUSPENDED_DURATION_SECONDS: {{ $microvm.suspendedDurationSeconds | quote }} - AWS_LAMBDA_MICROVM_AUTO_RESUME_ENABLED: {{ $microvm.autoResumeEnabled | toString | quote }} - AWS_LAMBDA_MICROVM_MAXIMUM_DURATION_SECONDS: {{ $microvm.maximumDurationSeconds | quote }} - AWS_LAMBDA_MICROVM_AUTH_TOKEN_EXPIRATION_MINUTES: {{ $microvm.authTokenExpirationMinutes | quote }} - AWS_LAMBDA_MICROVM_RUNTIME_PORT: {{ $microvm.runtimePort | quote }} - {{- if $microvm.allowExecutionRole }} - AWS_LAMBDA_MICROVM_EXECUTION_ROLE_ARN: {{ $microvm.executionRoleArn | quote }} - AWS_LAMBDA_MICROVM_ALLOW_EXECUTION_ROLE: "true" - {{- end }} - {{- if $egressGateway.enabled }} - LOOP_RUNTIME_SANDBOX_EGRESS_GATEWAY_URL: {{ $egressGateway.url | quote }} - {{- end }} - {{- else }} - {{- $service := .Values.loopRuntime.sandbox.sandboxService }} - {{- /* Any Sandbox Service endpoint selects the remote backend; the provider stays unset. */}} - EXO_SANDBOX_IMAGE: {{ $service.image | quote }} - EXO_SANDBOX_VCPU_COUNT: {{ $service.vcpuCount | quote }} - EXO_SANDBOX_MEMORY_MIB: {{ $service.memoryMib | quote }} - {{- if $service.dnsUrl }} - EXO_SANDBOX_SERVICE_DNS_URL: {{ $service.dnsUrl | quote }} + {{- /* A Sandbox Service endpoint with no sandbox provider selects the Sandbox Service for every sandbox. */}} + EXO_SANDBOX_IMAGE: {{ $sandbox.image | quote }} + EXO_SANDBOX_VCPU_COUNT: {{ $sandbox.vcpuCount | quote }} + EXO_SANDBOX_MEMORY_MIB: {{ $sandbox.memoryMib | quote }} + LOOP_RUNTIME_SANDBOX_EGRESS_MODE: {{ $sandbox.egressMode | quote }} + {{- if $sandbox.dnsUrl }} + EXO_SANDBOX_SERVICE_DNS_URL: {{ $sandbox.dnsUrl | quote }} {{- else }} - EXO_SANDBOX_SERVICE_URL: {{ $service.urls | join "," | quote }} + EXO_SANDBOX_SERVICE_URL: {{ $sandbox.urls | join "," | quote }} {{- end }} - {{- if $service.tls.secretName }} + {{- if $sandbox.tls.secretName }} SANDBOX_SERVICE_TLS_CERT: "/etc/sandbox-tls/client.pem" SANDBOX_SERVICE_TLS_KEY: "/etc/sandbox-tls/client.key" SANDBOX_SERVICE_TLS_CA: "/etc/sandbox-tls/server-ca.pem" {{- end }} {{- if $egressGateway.enabled }} - LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL: {{ $egressGateway.url | quote }} - {{- end }} - {{- end }} - {{- if $egressGateway.enabled }} LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN: "0.0.0.0:{{ $egressGateway.port }}" + LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL: {{ $egressGateway.url | quote }} {{- end }} {{- with (.Values.global.allowedOrgIds | default "" | toString | trim) }} ALLOWED_ORG_IDS: {{ . | quote }} diff --git a/braintrust/templates/loop-runtime-deployment.yaml b/braintrust/templates/loop-runtime-deployment.yaml index b4b83d6..1ed96e9 100644 --- a/braintrust/templates/loop-runtime-deployment.yaml +++ b/braintrust/templates/loop-runtime-deployment.yaml @@ -5,10 +5,7 @@ {{- if eq .Values.cloud "google" -}} {{- $_ := set $podLabels "gke-workload-identity/use" "true" -}} {{- end -}} -{{- $sandboxServiceTls := "" -}} -{{- if eq (include "braintrust.loopRuntime.sandboxBackend" .) "sandboxService" -}} -{{- $sandboxServiceTls = .Values.loopRuntime.sandbox.sandboxService.tls -}} -{{- end -}} +{{- $sandboxServiceTls := .Values.loopRuntime.sandbox.tls -}} {{- $egressGateway := .Values.loopRuntime.sandbox.egressGateway -}} apiVersion: apps/v1 kind: Deployment @@ -147,7 +144,7 @@ spec: mountPath: {{ .Values.loopRuntime.stateDir }} - name: cache-volume mountPath: {{ .Values.loopRuntime.cacheDir }} - {{- if and $sandboxServiceTls $sandboxServiceTls.secretName }} + {{- if $sandboxServiceTls.secretName }} - name: sandbox-service-tls mountPath: /etc/sandbox-tls readOnly: true @@ -160,7 +157,7 @@ spec: emptyDir: {} - name: cache-volume emptyDir: {} - {{- if and $sandboxServiceTls $sandboxServiceTls.secretName }} + {{- if $sandboxServiceTls.secretName }} - name: sandbox-service-tls secret: secretName: {{ $sandboxServiceTls.secretName }} diff --git a/braintrust/tests/__fixtures__/loop-runtime-google-values.yaml b/braintrust/tests/__fixtures__/loop-runtime-google-values.yaml index 4a0687b..e729e86 100644 --- a/braintrust/tests/__fixtures__/loop-runtime-google-values.yaml +++ b/braintrust/tests/__fixtures__/loop-runtime-google-values.yaml @@ -1,5 +1,4 @@ -# Loop Runtime on GKE with the Sandbox Service backend. Layer on top of -# base-values.yaml and google-values.yaml. +# Loop Runtime on GKE. Layer on top of base-values.yaml and google-values.yaml. loopRuntime: enabled: true image: @@ -9,7 +8,6 @@ loopRuntime: name: "braintrust-loop-runtime" googleServiceAccount: "braintrust-loop@test-project.iam.gserviceaccount.com" sandbox: - sandboxService: - dnsUrl: "https://workers.sandbox.internal:9400" - tls: - secretName: "sandbox-client-tls" + dnsUrl: "https://workers.sandbox.internal:9400" + tls: + secretName: "sandbox-client-tls" diff --git a/braintrust/tests/__fixtures__/loop-runtime-values.yaml b/braintrust/tests/__fixtures__/loop-runtime-values.yaml index feca0a2..949b154 100644 --- a/braintrust/tests/__fixtures__/loop-runtime-values.yaml +++ b/braintrust/tests/__fixtures__/loop-runtime-values.yaml @@ -15,8 +15,6 @@ loopRuntime: name: "braintrust-loop-runtime" awsRoleArn: "arn:aws:iam::123456789012:role/test-loop-runtime-role" sandbox: - awsLambdaMicrovm: - imageIdentifier: "arn:aws:lambda:us-east-1:123456789012:microvm-image/test" - region: "us-east-1" - ingressNetworkConnectorArns: "arn:aws:lambda:us-east-1:123456789012:network-connector/in" - egressNetworkConnectorArns: "arn:aws:lambda:us-east-1:123456789012:network-connector/out" + dnsUrl: "https://nodes.sandbox.internal:9400" + tls: + secretName: "sandbox-client-tls" diff --git a/braintrust/tests/loop-runtime-configmap_test.yaml b/braintrust/tests/loop-runtime-configmap_test.yaml index ea0c08e..64a60f8 100644 --- a/braintrust/tests/loop-runtime-configmap_test.yaml +++ b/braintrust/tests/loop-runtime-configmap_test.yaml @@ -27,7 +27,7 @@ tests: - failedTemplate: errorMessage: "loopRuntime.image is missing. Values reused from a chart release that predates Loop Runtime omit the loopRuntime defaults; upgrade with --reset-then-reuse-values or supply the full loopRuntime map." - - it: should render the AWS runtime and Lambda MicroVM sandbox configuration + - it: should render AWS storage and the Sandbox Service configuration values: - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml @@ -62,79 +62,25 @@ tests: path: data.BRAINSTORE_CODE_BUNDLE_URI value: s3://test-code-bundle-bucket - equal: - path: data.AWS_REGION - value: us-east-1 - - equal: - path: data.EXO_SANDBOX_PROVIDER - value: aws-lambda-microvm + path: data.EXO_SANDBOX_SERVICE_DNS_URL + value: https://nodes.sandbox.internal:9400 - equal: path: data.EXO_SANDBOX_IMAGE - value: lambda-microvm - - equal: - path: data.AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER - value: arn:aws:lambda:us-east-1:123456789012:microvm-image/test + value: public.ecr.aws/braintrust/codex-runtime:latest - equal: - path: data.AWS_LAMBDA_MICROVM_EGRESS_NETWORK_CONNECTOR_ARNS - value: arn:aws:lambda:us-east-1:123456789012:network-connector/out - - isNull: - path: data.AWS_LAMBDA_MICROVM_IMAGE_VERSION + path: data.SANDBOX_SERVICE_TLS_CERT + value: /etc/sandbox-tls/client.pem - isNull: - path: data.EXO_SANDBOX_SERVICE_DNS_URL + path: data.EXO_SANDBOX_PROVIDER - isNull: - path: data.EXO_SANDBOX_SERVICE_URL + path: data.AWS_REGION - isNull: - path: data.SANDBOX_SERVICE_TLS_CERT + path: data.AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER - isNull: path: data.LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN - isNull: path: data.BRAINSTORE_LOCKS_URI - - it: should pin the MicroVM image version when configured - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - set: - loopRuntime.sandbox.awsLambdaMicrovm.imageVersion: 7 - asserts: - - equal: - path: data.AWS_LAMBDA_MICROVM_IMAGE_VERSION - value: "7" - - - it: should pass the MicroVM execution role when allowed - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - set: - loopRuntime.sandbox.awsLambdaMicrovm.allowExecutionRole: true - loopRuntime.sandbox.awsLambdaMicrovm.executionRoleArn: arn:aws:iam::123456789012:role/microvm - asserts: - - equal: - path: data.AWS_LAMBDA_MICROVM_EXECUTION_ROLE_ARN - value: arn:aws:iam::123456789012:role/microvm - - equal: - path: data.AWS_LAMBDA_MICROVM_ALLOW_EXECUTION_ROLE - value: "true" - - - it: should require the execution role ARN when the execution role is allowed - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - set: - loopRuntime.sandbox.awsLambdaMicrovm.allowExecutionRole: true - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.awsLambdaMicrovm.executionRoleArn is required when loopRuntime.sandbox.awsLambdaMicrovm.allowExecutionRole is true" - - - it: should require MicroVM settings for the AWS default backend - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - set: - loopRuntime.sandbox.awsLambdaMicrovm.imageIdentifier: "" - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.awsLambdaMicrovm.imageIdentifier is required for the awsLambdaMicrovm backend (use Terraform output loop_runtime_microvm_image_arn)" - - it: should use AWS object storage locks when configured values: - __fixtures__/base-values.yaml @@ -146,30 +92,7 @@ tests: path: data.BRAINSTORE_LOCKS_URI value: s3://test-brainstore-bucket/brainstore/locks - - it: should run the Sandbox Service backend on AWS without MicroVM settings - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - set: - loopRuntime.sandbox.backend: sandboxService - loopRuntime.sandbox.awsLambdaMicrovm.imageIdentifier: "" - loopRuntime.sandbox.sandboxService.dnsUrl: https://nodes.sandbox.internal:9400 - loopRuntime.sandbox.sandboxService.tls.secretName: sandbox-client-tls - asserts: - - equal: - path: data.BRAINSTORE_INDEX_URI - value: s3://test-brainstore-bucket/brainstore/index - - equal: - path: data.EXO_SANDBOX_SERVICE_DNS_URL - value: https://nodes.sandbox.internal:9400 - - isNull: - path: data.EXO_SANDBOX_PROVIDER - - isNull: - path: data.AWS_REGION - - isNull: - path: data.AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER - - - it: should render GKE storage and the Sandbox Service backend by default on Google + - it: should render GKE storage and the Sandbox Service configuration values: - __fixtures__/base-values.yaml - __fixtures__/google-values.yaml @@ -195,6 +118,9 @@ tests: - equal: path: data.EXO_SANDBOX_MEMORY_MIB value: "2048" + - equal: + path: data.LOOP_RUNTIME_SANDBOX_EGRESS_MODE + value: restricted - equal: path: data.EXO_SANDBOX_SERVICE_DNS_URL value: https://workers.sandbox.internal:9400 @@ -213,10 +139,6 @@ tests: path: data.EXO_SANDBOX_SERVICE_URL - isNull: path: data.AWS_REGION - - isNull: - path: data.AWS_DEFAULT_REGION - - isNull: - path: data.AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER - isNull: path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL - isNull: @@ -234,14 +156,45 @@ tests: path: data.BRAINSTORE_LOCKS_URI value: gs://test-brainstore-bucket/brainstore/locks + - it: should pass sandbox sizing and internet egress mode + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.vcpuCount: 2 + loopRuntime.sandbox.memoryMib: 4096 + loopRuntime.sandbox.egressMode: internet + asserts: + - equal: + path: data.EXO_SANDBOX_VCPU_COUNT + value: "2" + - equal: + path: data.EXO_SANDBOX_MEMORY_MIB + value: "4096" + - equal: + path: data.LOOP_RUNTIME_SANDBOX_EGRESS_MODE + value: internet + + - it: should reject an unknown sandbox egress mode + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/loop-runtime-google-values.yaml + set: + loopRuntime.sandbox.egressMode: open + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressMode must be restricted or internet, got \"open\"" + - it: should join fixed Sandbox Service endpoints values: - __fixtures__/base-values.yaml - __fixtures__/google-values.yaml - __fixtures__/loop-runtime-google-values.yaml set: - loopRuntime.sandbox.sandboxService.dnsUrl: "" - loopRuntime.sandbox.sandboxService.urls: + loopRuntime.sandbox.dnsUrl: "" + loopRuntime.sandbox.urls: - https://worker-0.sandbox.internal:9400 - https://worker-1.sandbox.internal:9400 asserts: @@ -257,8 +210,8 @@ tests: - __fixtures__/google-values.yaml - __fixtures__/loop-runtime-google-values.yaml set: - loopRuntime.sandbox.sandboxService.dnsUrl: http://sandbox-service.sandbox:9400 - loopRuntime.sandbox.sandboxService.tls.secretName: "" + loopRuntime.sandbox.dnsUrl: http://sandbox-service.sandbox:9400 + loopRuntime.sandbox.tls.secretName: "" asserts: - equal: path: data.EXO_SANDBOX_SERVICE_DNS_URL @@ -272,10 +225,10 @@ tests: - __fixtures__/google-values.yaml - __fixtures__/loop-runtime-google-values.yaml set: - loopRuntime.sandbox.sandboxService.tls.secretName: "" + loopRuntime.sandbox.tls.secretName: "" asserts: - failedTemplate: - errorMessage: "loopRuntime.sandbox.sandboxService.tls.secretName is required for https:// Sandbox Service endpoints" + errorMessage: "loopRuntime.sandbox.tls.secretName is required for https:// Sandbox Service endpoints" - it: should require TLS when any fixed endpoint uses HTTPS values: @@ -283,25 +236,24 @@ tests: - __fixtures__/google-values.yaml - __fixtures__/loop-runtime-google-values.yaml set: - loopRuntime.sandbox.sandboxService.dnsUrl: "" - loopRuntime.sandbox.sandboxService.tls.secretName: "" - loopRuntime.sandbox.sandboxService.urls: + loopRuntime.sandbox.dnsUrl: "" + loopRuntime.sandbox.tls.secretName: "" + loopRuntime.sandbox.urls: - http://worker-0.sandbox.internal:9400 - https://worker-1.sandbox.internal:9400 asserts: - failedTemplate: - errorMessage: "loopRuntime.sandbox.sandboxService.tls.secretName is required for https:// Sandbox Service endpoints" + errorMessage: "loopRuntime.sandbox.tls.secretName is required for https:// Sandbox Service endpoints" - it: should require a Sandbox Service endpoint values: - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + - __fixtures__/loop-runtime-values.yaml set: - loopRuntime.sandbox.sandboxService.dnsUrl: "" + loopRuntime.sandbox.dnsUrl: "" asserts: - failedTemplate: - errorMessage: "loopRuntime.sandbox.sandboxService.dnsUrl or loopRuntime.sandbox.sandboxService.urls is required for the sandboxService backend (GKE: use Terraform output gke_isolated_workers.discovery.endpoint)" + errorMessage: "loopRuntime.sandbox.dnsUrl or loopRuntime.sandbox.urls is required when loopRuntime.enabled is true (GKE: use Terraform output gke_isolated_workers.discovery.endpoint)" - it: should reject both Sandbox Service discovery modes values: @@ -309,11 +261,11 @@ tests: - __fixtures__/google-values.yaml - __fixtures__/loop-runtime-google-values.yaml set: - loopRuntime.sandbox.sandboxService.urls: + loopRuntime.sandbox.urls: - https://worker-0.sandbox.internal:9400 asserts: - failedTemplate: - errorMessage: "Set only one of loopRuntime.sandbox.sandboxService.dnsUrl and loopRuntime.sandbox.sandboxService.urls." + errorMessage: "Set only one of loopRuntime.sandbox.dnsUrl and loopRuntime.sandbox.urls." - it: should reject a string of Sandbox Service endpoints values: @@ -321,22 +273,22 @@ tests: - __fixtures__/google-values.yaml - __fixtures__/loop-runtime-google-values.yaml set: - loopRuntime.sandbox.sandboxService.dnsUrl: "" - loopRuntime.sandbox.sandboxService.urls: https://worker-0.sandbox.internal:9400 + loopRuntime.sandbox.dnsUrl: "" + loopRuntime.sandbox.urls: https://worker-0.sandbox.internal:9400 asserts: - failedTemplate: - errorMessage: "loopRuntime.sandbox.sandboxService.urls must be a list of URLs" + errorMessage: "loopRuntime.sandbox.urls must be a list of URLs" - - it: should require the Sandbox Service guest image + - it: should require the sandbox guest image values: - __fixtures__/base-values.yaml - __fixtures__/google-values.yaml - __fixtures__/loop-runtime-google-values.yaml set: - loopRuntime.sandbox.sandboxService.image: "" + loopRuntime.sandbox.image: "" asserts: - failedTemplate: - errorMessage: "loopRuntime.sandbox.sandboxService.image is required for the sandboxService backend" + errorMessage: "loopRuntime.sandbox.image is required when loopRuntime.enabled is true" - it: should require the Loop Runtime Google service account on GKE values: @@ -360,26 +312,15 @@ tests: - failedTemplate: errorMessage: "objectStorage.google.apiBucket is required when loopRuntime.enabled is true" - - it: should reject the Lambda MicroVM backend on Google - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml - set: - loopRuntime.sandbox.backend: awsLambdaMicrovm - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.backend awsLambdaMicrovm requires cloud: aws. Use sandboxService on other clouds." - - - it: should reject an unknown sandbox backend + - it: should require the code bundle bucket on AWS values: - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml set: - loopRuntime.sandbox.backend: docker + objectStorage.aws.codeBundleBucket: "" asserts: - failedTemplate: - errorMessage: "loopRuntime.sandbox.backend must be awsLambdaMicrovm or sandboxService, got \"docker\"" + errorMessage: "objectStorage.aws.codeBundleBucket is required when loopRuntime.enabled is true" - it: should reject Loop Runtime on Azure values: @@ -400,23 +341,6 @@ tests: - failedTemplate: errorMessage: "loopRuntime.enabled requires cloud: aws or cloud: google." - - it: should advertise the egress gateway to MicroVM sandboxes - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - set: - loopRuntime.sandbox.egressGateway.enabled: true - loopRuntime.sandbox.egressGateway.url: http://egress.vpce.internal:4002 - asserts: - - equal: - path: data.LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN - value: 0.0.0.0:4002 - - equal: - path: data.LOOP_RUNTIME_SANDBOX_EGRESS_GATEWAY_URL - value: http://egress.vpce.internal:4002 - - isNull: - path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL - - it: should advertise the egress gateway to Sandbox Service workers values: - __fixtures__/base-values.yaml diff --git a/braintrust/tests/loop-runtime-deployment_test.yaml b/braintrust/tests/loop-runtime-deployment_test.yaml index e7929ad..72afbca 100644 --- a/braintrust/tests/loop-runtime-deployment_test.yaml +++ b/braintrust/tests/loop-runtime-deployment_test.yaml @@ -100,9 +100,6 @@ tests: path: spec.template.metadata.labels["gke-workload-identity/use"] - isNull: path: spec.template.metadata.annotations["iam.gke.io/gcp-service-account"] - - lengthEqual: - path: spec.template.spec.volumes - count: 2 - lengthEqual: path: spec.template.spec.containers[0].ports count: 1 @@ -152,10 +149,10 @@ tests: - __fixtures__/google-values.yaml - __fixtures__/loop-runtime-google-values.yaml set: - loopRuntime.sandbox.sandboxService.tls.secretName: cert-manager-client - loopRuntime.sandbox.sandboxService.tls.certKey: tls.crt - loopRuntime.sandbox.sandboxService.tls.keyKey: tls.key - loopRuntime.sandbox.sandboxService.tls.caKey: ca.crt + loopRuntime.sandbox.tls.secretName: cert-manager-client + loopRuntime.sandbox.tls.certKey: tls.crt + loopRuntime.sandbox.tls.keyKey: tls.key + loopRuntime.sandbox.tls.caKey: ca.crt asserts: - contains: path: spec.template.spec.volumes @@ -178,8 +175,8 @@ tests: - __fixtures__/google-values.yaml - __fixtures__/loop-runtime-google-values.yaml set: - loopRuntime.sandbox.sandboxService.dnsUrl: http://sandbox-service.sandbox:9400 - loopRuntime.sandbox.sandboxService.tls.secretName: "" + loopRuntime.sandbox.dnsUrl: http://sandbox-service.sandbox:9400 + loopRuntime.sandbox.tls.secretName: "" asserts: - lengthEqual: path: spec.template.spec.volumes @@ -188,17 +185,21 @@ tests: path: spec.template.spec.containers[0].volumeMounts count: 2 - - it: should ignore Sandbox Service TLS settings for the MicroVM backend + - it: should mount Sandbox Service client TLS on AWS template: loop-runtime-deployment.yaml values: - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml - set: - loopRuntime.sandbox.sandboxService.tls.secretName: sandbox-client-tls asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: sandbox-service-tls + mountPath: /etc/sandbox-tls + readOnly: true - lengthEqual: path: spec.template.spec.volumes - count: 2 + count: 3 - it: should expose the egress gateway container port when enabled template: loop-runtime-deployment.yaml diff --git a/braintrust/values.yaml b/braintrust/values.yaml index b4d2625..1d46e6c 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -310,8 +310,8 @@ api: extraVolumes: [] # Loop Runtime configuration (AWS EKS and Google GKE) -# The cloud's Terraform module creates the runtime identity and the sandbox -# infrastructure; this chart runs the Loop Runtime process in Kubernetes. +# The cloud's Terraform module creates the runtime identity and the Sandbox +# Service infrastructure; this chart runs the Loop Runtime process in Kubernetes. loopRuntime: enabled: false name: "braintrust-loop-runtime" @@ -392,56 +392,38 @@ loopRuntime: nodeSelector: {} tolerations: [] affinity: {} + # Loop Runtime runs agent sandboxes on Braintrust Sandbox Service workers, + # reached over mTLS. On GKE the workers run in the Terraform-managed isolated + # worker cluster; on AWS they run on the Sandbox Service EKS cluster. sandbox: - # Where Loop Runtime runs agent sandboxes: - # awsLambdaMicrovm - AWS Lambda MicroVMs. Requires cloud: aws. - # sandboxService - Braintrust Sandbox Service workers reached over mTLS. - # Works on any cloud; on GKE the workers run in the - # Terraform-managed isolated worker cluster. - # Empty selects awsLambdaMicrovm on AWS and sandboxService on Google. - backend: "" - awsLambdaMicrovm: - # These values come from Terraform outputs. They are required for this backend. - imageIdentifier: "" - # Optional. Pins sandboxes to one published MicroVM image version. - imageVersion: "" - region: "" - ingressNetworkConnectorArns: "" - egressNetworkConnectorArns: "" - maxIdleDurationSeconds: 900 - suspendedDurationSeconds: 28800 - autoResumeEnabled: true - maximumDurationSeconds: 28800 - authTokenExpirationMinutes: 30 - runtimePort: 8080 - executionRoleArn: "" - allowExecutionRole: false - sandboxService: - # Set exactly one. dnsUrl discovers every worker behind one headless DNS - # name (GKE: Terraform output gke_isolated_workers.discovery.endpoint). - # urls lists fixed worker endpoints instead. - dnsUrl: "" - urls: [] - # Sandbox guest image the workers start. - image: "public.ecr.aws/braintrust/codex-runtime:latest" - # Must match the Sandbox Service release's per-sandbox sizing. - vcpuCount: 1 - memoryMib: 2048 - tls: - # Kubernetes Secret with the Sandbox Service client certificate, its - # key, and the server CA. Required for https:// endpoints. - secretName: "" - certKey: client.pem - keyKey: client.key - caKey: server-ca.pem + # Set exactly one. dnsUrl discovers every worker behind one headless DNS + # name (GKE: Terraform output gke_isolated_workers.discovery.endpoint). + # urls lists fixed worker endpoints instead. + dnsUrl: "" + urls: [] + # Sandbox guest image the workers start. + image: "public.ecr.aws/braintrust/codex-runtime:latest" + # Must match the Sandbox Service release's per-sandbox sizing. + vcpuCount: 1 + memoryMib: 2048 + # "restricted" permits only configured integrations; "internet" permits + # public destinations. + egressMode: "restricted" + tls: + # Kubernetes Secret with the Sandbox Service client certificate, its key, + # and the server CA. Required for https:// endpoints. + secretName: "" + certKey: client.pem + keyKey: client.key + caKey: server-ca.pem # Optional forward proxy that sandboxes use for brokered Git access. When - # enabled, Loop Runtime listens on port and advertises url to sandboxes. + # enabled, Loop Runtime listens on port and advertises url to the workers. egressGateway: enabled: false port: 4002 - # Address sandboxes use to reach this proxy, e.g. http://:4002. + # Address the workers use to reach this proxy, e.g. http://:4002. url: "" - # Dedicated Service for the proxy port. Sandboxes usually run outside this + # Dedicated Service for the proxy port. The workers run outside this # cluster, so expose it through an internal load balancer. For GKE, set # type LoadBalancer with the networking.gke.io/load-balancer-type: # "Internal" annotation and restrict loadBalancerSourceRanges to the From 729bce4939056a4394b186fc31c256504a8d5fb1 Mon Sep 17 00:00:00 2001 From: Alex R Date: Tue, 20 Oct 2026 19:54:44 +0000 Subject: [PATCH 04/25] Trim Loop Runtime validation and docs Drop checks the runtime already enforces (urls type, egress mode, egress port collision, TLS key names), inline the one-use egress Service name helper, and shorten the values comments and README. Co-Authored-By: Claude Opus 5.5 --- braintrust/README.md | 54 ++++++------------- braintrust/templates/_helpers.tpl | 29 +--------- .../templates/loop-runtime-configmap.yaml | 2 +- .../templates/loop-runtime-deployment.yaml | 14 ++--- .../loop-runtime-egress-service.yaml | 2 +- .../tests/loop-runtime-configmap_test.yaml | 36 ------------- braintrust/values.yaml | 36 ++++--------- 7 files changed, 39 insertions(+), 134 deletions(-) diff --git a/braintrust/README.md b/braintrust/README.md index 3cc24d6..891ada8 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -20,28 +20,20 @@ The `braintrust-secrets` secret must contain the following keys: ## Optional: Loop Runtime -Set `loopRuntime.enabled: true` to run the Loop Runtime service in the cluster. -It supports `cloud: aws` (EKS) and `cloud: google` (GKE). Loop Runtime runs -agent sandboxes on Braintrust Sandbox Service workers, which it reaches over -mTLS. The cloud's Terraform data-plane module creates the runtime identity and -the Sandbox Service infrastructure; the chart runs the runtime process. +Set `loopRuntime.enabled: true` to run Loop Runtime on EKS (`cloud: aws`) or +GKE (`cloud: google`). Loop Runtime runs sandboxes on Braintrust Sandbox +Service workers over mTLS. Terraform creates the runtime identity and the +Sandbox Service infrastructure. -The chart reuses the existing `braintrust-secrets` values, reads the same -Brainstore buckets as the Brainstore pods, adds `LOOP_RUNTIME_URL` to the API, -and adds `/loop/runtime` routes to the chart-managed Istio VirtualService when -`virtualService.enabled` is true. +The runtime reads `braintrust-secrets` and the Brainstore buckets. The chart +adds `LOOP_RUNTIME_URL` to the API and, when `virtualService.enabled` is true, +routes `/loop/runtime` to Loop Runtime. -Create a Secret in this namespace with the Sandbox Service client certificate, -its key, and the server CA, and point `loopRuntime.sandbox.tls.secretName` at -it. `sandbox.vcpuCount` and `sandbox.memoryMib` must match the per-sandbox -sizing of the Sandbox Service release. +Put the Sandbox Service client certificate, key, and server CA in a Secret and +set `loopRuntime.sandbox.tls.secretName`. `sandbox.vcpuCount` and +`sandbox.memoryMib` must match the Sandbox Service release. -### Google GKE - -The GKE Terraform module creates a Google service account for Loop Runtime and -binds it to the `braintrust/braintrust-loop-runtime` Kubernetes service account -through Workload Identity. Its isolated worker cluster hosts the Sandbox -Service. +GKE: ```yaml cloud: google @@ -54,19 +46,14 @@ objectStorage: loopRuntime: enabled: true serviceAccount: - name: braintrust-loop-runtime googleServiceAccount: "" sandbox: dnsUrl: "" tls: secretName: sandbox-client-tls - # Keys in that Secret. Defaults match the Sandbox Service client Secret. - certKey: client.pem - keyKey: client.key - caKey: server-ca.pem ``` -### AWS EKS +EKS: ```yaml cloud: aws @@ -79,24 +66,17 @@ objectStorage: loopRuntime: enabled: true serviceAccount: - name: braintrust-loop-runtime - # Set for IRSA; leave empty when using an EKS Pod Identity association. + # Leave empty when using EKS Pod Identity. awsRoleArn: "" sandbox: - dnsUrl: "https://:9400" + dnsUrl: "https://:9400" tls: secretName: sandbox-client-tls ``` -### Optional sandbox egress gateway - -Sandboxes use a forward proxy in Loop Runtime for brokered Git access. Enable -it with `loopRuntime.sandbox.egressGateway`. The chart opens the proxy port on -the pod, renders a `-egress` Service, and tells sandboxes to use -`egressGateway.url`. Sandboxes run outside this cluster, so expose the Service -on a private address the sandboxes can reach. On GKE, use an internal load -balancer and allow the isolated worker node range in the worker cluster's -egress rules: +Sandboxes reach Git through a forward proxy in Loop Runtime. To enable it, set +`loopRuntime.sandbox.egressGateway`. The workers run outside this cluster, so +expose the `-egress` Service on an internal load balancer. On GKE: ```yaml loopRuntime: diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 14bb29a..d5e30fd 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -154,10 +154,7 @@ predates Loop Runtime have no loopRuntime map, which means disabled. {{- if (.Values.loopRuntime | default dict).enabled -}}true{{- end -}} {{- end -}} -{{/* -Object-store scheme and Brainstore bucket for Loop Runtime. Matches the -Brainstore ConfigMaps so both read the same index, WAL, and locks. -*/}} +{{/* Loop Runtime object-store URIs. They match the Brainstore ConfigMaps. */}} {{- define "braintrust.loopRuntime.brainstoreBucketUri" -}} {{- if eq .Values.cloud "google" -}} gs://{{ .Values.objectStorage.google.brainstoreBucket }} @@ -174,10 +171,7 @@ s3://{{ .Values.objectStorage.aws.codeBundleBucket }} {{- end -}} {{- end -}} -{{/* -Validate the Loop Runtime configuration. Terraform supplies the identity and -Sandbox Service infrastructure; Helm supplies the Kubernetes workload. -*/}} +{{/* Validate the Loop Runtime configuration. */}} {{- define "braintrust.loopRuntime.validate" -}} {{- if include "braintrust.loopRuntime.enabled" . -}} {{- range (list "image" "service" "serviceAccount" "sandbox") -}} @@ -199,9 +193,6 @@ Sandbox Service infrastructure; Helm supplies the Kubernetes workload. {{- end -}} {{- $_ := required "loopRuntime.serviceAccount.name is required when loopRuntime.enabled is true" .Values.loopRuntime.serviceAccount.name -}} {{- $urls := $sandbox.urls | default list -}} -{{- if not (kindIs "slice" $urls) -}} -{{- fail "loopRuntime.sandbox.urls must be a list of URLs" -}} -{{- end -}} {{- if and $sandbox.dnsUrl $urls -}} {{- fail "Set only one of loopRuntime.sandbox.dnsUrl and loopRuntime.sandbox.urls." -}} {{- end -}} @@ -209,24 +200,13 @@ Sandbox Service infrastructure; Helm supplies the Kubernetes workload. {{- fail "loopRuntime.sandbox.dnsUrl or loopRuntime.sandbox.urls is required when loopRuntime.enabled is true (GKE: use Terraform output gke_isolated_workers.discovery.endpoint)" -}} {{- end -}} {{- $_ := required "loopRuntime.sandbox.image is required when loopRuntime.enabled is true" $sandbox.image -}} -{{- if not (has $sandbox.egressMode (list "restricted" "internet")) -}} -{{- fail (printf "loopRuntime.sandbox.egressMode must be restricted or internet, got %q" (toString $sandbox.egressMode)) -}} -{{- end -}} {{- range (prepend $urls ($sandbox.dnsUrl | default "")) -}} {{- if and . (hasPrefix "https://" .) (not $sandbox.tls.secretName) -}} {{- fail "loopRuntime.sandbox.tls.secretName is required for https:// Sandbox Service endpoints" -}} {{- end -}} {{- end -}} -{{- if $sandbox.tls.secretName -}} -{{- $_ := required "loopRuntime.sandbox.tls.certKey is required when tls.secretName is set" $sandbox.tls.certKey -}} -{{- $_ := required "loopRuntime.sandbox.tls.keyKey is required when tls.secretName is set" $sandbox.tls.keyKey -}} -{{- $_ := required "loopRuntime.sandbox.tls.caKey is required when tls.secretName is set" $sandbox.tls.caKey -}} -{{- end -}} {{- if $sandbox.egressGateway.enabled -}} {{- $_ := required "loopRuntime.sandbox.egressGateway.url is required when loopRuntime.sandbox.egressGateway.enabled is true" $sandbox.egressGateway.url -}} -{{- if eq (int $sandbox.egressGateway.port) (int .Values.loopRuntime.service.port) -}} -{{- fail "loopRuntime.sandbox.egressGateway.port must differ from loopRuntime.service.port" -}} -{{- end -}} {{- end -}} {{- end -}} {{- end -}} @@ -236,11 +216,6 @@ Sandbox Service infrastructure; Helm supplies the Kubernetes workload. http://{{ .Values.loopRuntime.service.name | default .Values.loopRuntime.name }}.{{ include "braintrust.namespace" . }}:{{ .Values.loopRuntime.service.port }} {{- end -}} -{{/* Service that exposes the Loop Runtime sandbox egress gateway. */}} -{{- define "braintrust.loopRuntimeEgressServiceName" -}} -{{- printf "%s-egress" (.Values.loopRuntime.service.name | default .Values.loopRuntime.name) -}} -{{- end -}} - {{/* Stable Redis key namespace for a Helm-managed Loop Runtime. */}} {{- define "braintrust.loopRuntimeRedisNamespace" -}} {{- .Values.loopRuntime.redisNamespace | default (printf "loop-runtime:%s" .Release.Name) -}} diff --git a/braintrust/templates/loop-runtime-configmap.yaml b/braintrust/templates/loop-runtime-configmap.yaml index 502e61e..5671f96 100644 --- a/braintrust/templates/loop-runtime-configmap.yaml +++ b/braintrust/templates/loop-runtime-configmap.yaml @@ -38,7 +38,7 @@ data: BRAINSTORE_CONTROL_PLANE_TELEMETRY: {{ .Values.global.controlPlaneTelemetry | quote }} BRAINSTORE_DISABLE_STATUS_UPDATES: {{ .Values.loopRuntime.brainstoreDisableStatusUpdates | toString | quote }} NO_COLOR: "1" - {{- /* A Sandbox Service endpoint with no sandbox provider selects the Sandbox Service for every sandbox. */}} + {{- /* Without EXO_SANDBOX_PROVIDER, a Sandbox Service endpoint sends every sandbox to the Sandbox Service. */}} EXO_SANDBOX_IMAGE: {{ $sandbox.image | quote }} EXO_SANDBOX_VCPU_COUNT: {{ $sandbox.vcpuCount | quote }} EXO_SANDBOX_MEMORY_MIB: {{ $sandbox.memoryMib | quote }} diff --git a/braintrust/templates/loop-runtime-deployment.yaml b/braintrust/templates/loop-runtime-deployment.yaml index 1ed96e9..42fb3d8 100644 --- a/braintrust/templates/loop-runtime-deployment.yaml +++ b/braintrust/templates/loop-runtime-deployment.yaml @@ -5,7 +5,7 @@ {{- if eq .Values.cloud "google" -}} {{- $_ := set $podLabels "gke-workload-identity/use" "true" -}} {{- end -}} -{{- $sandboxServiceTls := .Values.loopRuntime.sandbox.tls -}} +{{- $tls := .Values.loopRuntime.sandbox.tls -}} {{- $egressGateway := .Values.loopRuntime.sandbox.egressGateway -}} apiVersion: apps/v1 kind: Deployment @@ -144,7 +144,7 @@ spec: mountPath: {{ .Values.loopRuntime.stateDir }} - name: cache-volume mountPath: {{ .Values.loopRuntime.cacheDir }} - {{- if $sandboxServiceTls.secretName }} + {{- if $tls.secretName }} - name: sandbox-service-tls mountPath: /etc/sandbox-tls readOnly: true @@ -157,16 +157,16 @@ spec: emptyDir: {} - name: cache-volume emptyDir: {} - {{- if $sandboxServiceTls.secretName }} + {{- if $tls.secretName }} - name: sandbox-service-tls secret: - secretName: {{ $sandboxServiceTls.secretName }} + secretName: {{ $tls.secretName }} items: - - key: {{ $sandboxServiceTls.certKey }} + - key: {{ $tls.certKey }} path: client.pem - - key: {{ $sandboxServiceTls.keyKey }} + - key: {{ $tls.keyKey }} path: client.key - - key: {{ $sandboxServiceTls.caKey }} + - key: {{ $tls.caKey }} path: server-ca.pem {{- end }} {{- with .Values.loopRuntime.extraVolumes }} diff --git a/braintrust/templates/loop-runtime-egress-service.yaml b/braintrust/templates/loop-runtime-egress-service.yaml index 3796cfb..f3eaaa3 100644 --- a/braintrust/templates/loop-runtime-egress-service.yaml +++ b/braintrust/templates/loop-runtime-egress-service.yaml @@ -5,7 +5,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "braintrust.loopRuntimeEgressServiceName" . }} + name: {{ .Values.loopRuntime.service.name | default .Values.loopRuntime.name }}-egress namespace: {{ include "braintrust.namespace" . }} {{- with $resourceLabels }} labels: diff --git a/braintrust/tests/loop-runtime-configmap_test.yaml b/braintrust/tests/loop-runtime-configmap_test.yaml index 64a60f8..fee4df1 100644 --- a/braintrust/tests/loop-runtime-configmap_test.yaml +++ b/braintrust/tests/loop-runtime-configmap_test.yaml @@ -176,17 +176,6 @@ tests: path: data.LOOP_RUNTIME_SANDBOX_EGRESS_MODE value: internet - - it: should reject an unknown sandbox egress mode - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml - set: - loopRuntime.sandbox.egressMode: open - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressMode must be restricted or internet, got \"open\"" - - it: should join fixed Sandbox Service endpoints values: - __fixtures__/base-values.yaml @@ -267,18 +256,6 @@ tests: - failedTemplate: errorMessage: "Set only one of loopRuntime.sandbox.dnsUrl and loopRuntime.sandbox.urls." - - it: should reject a string of Sandbox Service endpoints - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml - set: - loopRuntime.sandbox.dnsUrl: "" - loopRuntime.sandbox.urls: https://worker-0.sandbox.internal:9400 - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.urls must be a list of URLs" - - it: should require the sandbox guest image values: - __fixtures__/base-values.yaml @@ -370,16 +347,3 @@ tests: asserts: - failedTemplate: errorMessage: "loopRuntime.sandbox.egressGateway.url is required when loopRuntime.sandbox.egressGateway.enabled is true" - - - it: should keep the egress gateway off the runtime port - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml - set: - loopRuntime.sandbox.egressGateway.enabled: true - loopRuntime.sandbox.egressGateway.url: http://10.10.0.20:4001 - loopRuntime.sandbox.egressGateway.port: 4001 - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.port must differ from loopRuntime.service.port" diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 1d46e6c..fc03867 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -309,9 +309,7 @@ api: extraContainers: [] extraVolumes: [] -# Loop Runtime configuration (AWS EKS and Google GKE) -# The cloud's Terraform module creates the runtime identity and the Sandbox -# Service infrastructure; this chart runs the Loop Runtime process in Kubernetes. +# Loop Runtime configuration (optional; cloud: aws or google) loopRuntime: enabled: false name: "braintrust-loop-runtime" @@ -336,10 +334,9 @@ loopRuntime: portName: http serviceAccount: name: "braintrust-loop-runtime" - # AWS: set this to Terraform's Loop Runtime role ARN when using IRSA. Leave - # it empty when the role is attached through an EKS Pod Identity association. + # AWS IRSA role. Leave empty when using EKS Pod Identity. awsRoleArn: "" - # Google: required. Use the Terraform output loop_runtime_service_account. + # Required on Google (Terraform output loop_runtime_service_account). googleServiceAccount: "" annotations: {} # These URLs default to in-cluster services. Set them when the API, AI proxy, @@ -392,42 +389,31 @@ loopRuntime: nodeSelector: {} tolerations: [] affinity: {} - # Loop Runtime runs agent sandboxes on Braintrust Sandbox Service workers, - # reached over mTLS. On GKE the workers run in the Terraform-managed isolated - # worker cluster; on AWS they run on the Sandbox Service EKS cluster. + # Sandboxes run on Braintrust Sandbox Service workers, reached over mTLS. sandbox: - # Set exactly one. dnsUrl discovers every worker behind one headless DNS - # name (GKE: Terraform output gke_isolated_workers.discovery.endpoint). - # urls lists fixed worker endpoints instead. + # Set one. dnsUrl resolves every worker behind a headless DNS name + # (GKE: Terraform output gke_isolated_workers.discovery.endpoint). dnsUrl: "" urls: [] - # Sandbox guest image the workers start. image: "public.ecr.aws/braintrust/codex-runtime:latest" # Must match the Sandbox Service release's per-sandbox sizing. vcpuCount: 1 memoryMib: 2048 - # "restricted" permits only configured integrations; "internet" permits - # public destinations. + # restricted or internet egressMode: "restricted" tls: - # Kubernetes Secret with the Sandbox Service client certificate, its key, - # and the server CA. Required for https:// endpoints. + # Secret with the client certificate, key, and server CA. + # Required for https:// endpoints. secretName: "" certKey: client.pem keyKey: client.key caKey: server-ca.pem - # Optional forward proxy that sandboxes use for brokered Git access. When - # enabled, Loop Runtime listens on port and advertises url to the workers. + # Forward proxy that sandboxes use for Git access. url is the address the + # workers use to reach the -egress Service. egressGateway: enabled: false port: 4002 - # Address the workers use to reach this proxy, e.g. http://:4002. url: "" - # Dedicated Service for the proxy port. The workers run outside this - # cluster, so expose it through an internal load balancer. For GKE, set - # type LoadBalancer with the networking.gke.io/load-balancer-type: - # "Internal" annotation and restrict loadBalancerSourceRanges to the - # isolated worker node range. service: type: ClusterIP annotations: {} From 6345ce6019a1595b63d0b6f96055342c7bba5e46 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Tue, 20 Oct 2026 14:42:10 -0700 Subject: [PATCH 05/25] Fix Loop Runtime egress port collision and label coverage --- .../templates/loop-runtime-deployment.yaml | 3 +- .../loop-runtime-egress-service.yaml | 2 +- .../tests/labels-merge-services_test.yaml | 51 +++++++++++++++++++ .../tests/loop-runtime-deployment_test.yaml | 48 ++++++++++++++++- .../loop-runtime-egress-service_test.yaml | 20 +++++++- 5 files changed, 119 insertions(+), 5 deletions(-) diff --git a/braintrust/templates/loop-runtime-deployment.yaml b/braintrust/templates/loop-runtime-deployment.yaml index 42fb3d8..9a7276f 100644 --- a/braintrust/templates/loop-runtime-deployment.yaml +++ b/braintrust/templates/loop-runtime-deployment.yaml @@ -73,8 +73,7 @@ spec: containerPort: {{ .Values.loopRuntime.service.port }} protocol: TCP {{- if $egressGateway.enabled }} - - name: egress - containerPort: {{ $egressGateway.port }} + - containerPort: {{ $egressGateway.port }} protocol: TCP {{- end }} resources: diff --git a/braintrust/templates/loop-runtime-egress-service.yaml b/braintrust/templates/loop-runtime-egress-service.yaml index f3eaaa3..36a42f5 100644 --- a/braintrust/templates/loop-runtime-egress-service.yaml +++ b/braintrust/templates/loop-runtime-egress-service.yaml @@ -22,7 +22,7 @@ spec: - name: egress protocol: TCP port: {{ $egressGateway.port }} - targetPort: egress + targetPort: {{ $egressGateway.port }} type: {{ $egressGateway.service.type }} {{- with $egressGateway.service.loadBalancerSourceRanges }} loadBalancerSourceRanges: diff --git a/braintrust/tests/labels-merge-services_test.yaml b/braintrust/tests/labels-merge-services_test.yaml index 02c3c4a..1f5c875 100644 --- a/braintrust/tests/labels-merge-services_test.yaml +++ b/braintrust/tests/labels-merge-services_test.yaml @@ -6,6 +6,7 @@ templates: - brainstore-fastreader-service.yaml - brainstore-automationwriter-service.yaml - loop-runtime-service.yaml + - loop-runtime-egress-service.yaml tests: # These tests verify the deepCopy fix for services. # Without deepCopy, labels would bleed between components due to merge mutation. @@ -195,3 +196,53 @@ tests: path: metadata.labels["writer-specific"] - isNull: path: metadata.labels["fastreader-specific"] + + - it: should give Loop Runtime egress service only global and runtime-specific labels + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.egressGateway.enabled: true + loopRuntime.sandbox.egressGateway.url: http://egress.example:4002 + global.labels.shared: global-service + global.labels.component: global + loopRuntime.labels.component: loop-runtime + loopRuntime.labels.loop-specific: loop-egress + api.labels.component: api + api.labels.api-specific: api-service + brainstore.reader.labels.reader-specific: reader-service + brainstore.writer.labels.writer-specific: writer-service + brainstore.fastreader.labels.fastreader-specific: fastreader-service + brainstore.image.tag: v2.17.0 + brainstore.automationwriter.replicas: 1 + brainstore.automationwriter.labels.automationwriter-specific: automationwriter-service + template: loop-runtime-egress-service.yaml + asserts: + - equal: + path: metadata.labels + value: + shared: global-service + component: loop-runtime + loop-specific: loop-egress + + - it: should keep Loop Runtime egress labels out of the API service + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.egressGateway.enabled: true + loopRuntime.sandbox.egressGateway.url: http://egress.example:4002 + global.labels.shared: global-service + global.labels.component: global + loopRuntime.labels.component: loop-runtime + loopRuntime.labels.loop-specific: loop-egress + api.labels.component: api + api.labels.api-specific: api-service + template: api-service.yaml + asserts: + - equal: + path: metadata.labels + value: + shared: global-service + component: api + api-specific: api-service diff --git a/braintrust/tests/loop-runtime-deployment_test.yaml b/braintrust/tests/loop-runtime-deployment_test.yaml index 72afbca..f848ab1 100644 --- a/braintrust/tests/loop-runtime-deployment_test.yaml +++ b/braintrust/tests/loop-runtime-deployment_test.yaml @@ -214,10 +214,56 @@ tests: - contains: path: spec.template.spec.containers[0].ports content: - name: egress containerPort: 4002 protocol: TCP + - it: should allow the main port to be named egress when the gateway is disabled + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.service.portName: egress + asserts: + - equal: + path: spec.template.spec.containers[0].ports + value: + - name: egress + containerPort: 4001 + protocol: TCP + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.port + value: egress + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.port + value: egress + + - it: should keep port names unique with the gateway enabled and a custom port + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.service.portName: egress + loopRuntime.sandbox.egressGateway.enabled: true + loopRuntime.sandbox.egressGateway.url: http://egress.example:5002 + loopRuntime.sandbox.egressGateway.port: 5002 + asserts: + - equal: + path: spec.template.spec.containers[0].ports + value: + - name: egress + containerPort: 4001 + protocol: TCP + - containerPort: 5002 + protocol: TCP + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.port + value: egress + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.port + value: egress + - it: should not render when reused values have no loopRuntime map template: loop-runtime-deployment.yaml values: diff --git a/braintrust/tests/loop-runtime-egress-service_test.yaml b/braintrust/tests/loop-runtime-egress-service_test.yaml index 1ba6f35..11a5e1d 100644 --- a/braintrust/tests/loop-runtime-egress-service_test.yaml +++ b/braintrust/tests/loop-runtime-egress-service_test.yaml @@ -51,7 +51,7 @@ tests: name: egress protocol: TCP port: 4002 - targetPort: egress + targetPort: 4002 - equal: path: spec.type value: ClusterIP @@ -60,6 +60,24 @@ tests: - isNull: path: metadata.annotations + - it: should target the numeric gateway port when the main port is named egress + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.service.portName: egress + loopRuntime.sandbox.egressGateway.enabled: true + loopRuntime.sandbox.egressGateway.url: http://egress.example:5002 + loopRuntime.sandbox.egressGateway.port: 5002 + asserts: + - equal: + path: spec.ports[0] + value: + name: egress + protocol: TCP + port: 5002 + targetPort: 5002 + - it: should expose the egress gateway through a GKE internal load balancer values: - __fixtures__/base-values.yaml From 6d76bdb05402b4194116f22612fe51bcc7243576 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Wed, 21 Oct 2026 00:36:42 -0700 Subject: [PATCH 06/25] Add an EKS installation using the shared Sandbox Service chart --- .github/workflows/test-helm-chart.yml | 1 + braintrust/README.md | 16 +- .../examples/eks-loop-runtime/README.md | 126 +++++++++++++ .../eks-loop-runtime/sandbox-service.yaml | 14 ++ .../examples/eks-loop-runtime/values.yaml | 17 ++ braintrust/templates/_helpers.tpl | 11 +- .../templates/loop-runtime-configmap.yaml | 2 +- braintrust/tests/eks-loop-runtime_test.yaml | 104 +++++++++++ .../tests/loop-runtime-configmap_test.yaml | 96 +++++++++- .../tests/loop-runtime-images_test.yaml | 151 ++++++++++++++++ braintrust/values.yaml | 14 +- scripts/test-eks-loop-runtime.sh | 30 ++++ test.sh | 4 + tests/sandbox-service_test.yaml | 165 ++++++++++++++++++ 14 files changed, 732 insertions(+), 19 deletions(-) create mode 100644 braintrust/examples/eks-loop-runtime/README.md create mode 100644 braintrust/examples/eks-loop-runtime/sandbox-service.yaml create mode 100644 braintrust/examples/eks-loop-runtime/values.yaml create mode 100644 braintrust/tests/eks-loop-runtime_test.yaml create mode 100644 braintrust/tests/loop-runtime-images_test.yaml create mode 100644 scripts/test-eks-loop-runtime.sh create mode 100644 tests/sandbox-service_test.yaml diff --git a/.github/workflows/test-helm-chart.yml b/.github/workflows/test-helm-chart.yml index 3bf0943..0c35884 100644 --- a/.github/workflows/test-helm-chart.yml +++ b/.github/workflows/test-helm-chart.yml @@ -27,6 +27,7 @@ jobs: run: | python3 -m unittest discover -s tests -p 'test_*.py' helm unittest braintrust + bash scripts/test-eks-loop-runtime.sh lint-test: name: Lint and Test diff --git a/braintrust/README.md b/braintrust/README.md index 891ada8..a49eeac 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -22,8 +22,10 @@ The `braintrust-secrets` secret must contain the following keys: Set `loopRuntime.enabled: true` to run Loop Runtime on EKS (`cloud: aws`) or GKE (`cloud: google`). Loop Runtime runs sandboxes on Braintrust Sandbox -Service workers over mTLS. Terraform creates the runtime identity and the -Sandbox Service infrastructure. +Service workers over mTLS. For a complete installation on EKS, follow the +[Loop Runtime and Sandbox Service example](examples/eks-loop-runtime/README.md). +It installs the shared worker chart as a separate Helm release and reuses the +existing AWS infrastructure and workload identities. The runtime reads `braintrust-secrets` and the Brainstore buckets. The chart adds `LOOP_RUNTIME_URL` to the API and, when `virtualService.enabled` is true, @@ -31,7 +33,7 @@ routes `/loop/runtime` to Loop Runtime. Put the Sandbox Service client certificate, key, and server CA in a Secret and set `loopRuntime.sandbox.tls.secretName`. `sandbox.vcpuCount` and -`sandbox.memoryMib` must match the Sandbox Service release. +`sandbox.memoryMib` specify the resources requested for each sandbox. GKE: @@ -74,9 +76,11 @@ loopRuntime: secretName: sandbox-client-tls ``` -Sandboxes reach Git through a forward proxy in Loop Runtime. To enable it, set -`loopRuntime.sandbox.egressGateway`. The workers run outside this cluster, so -expose the `-egress` Service on an internal load balancer. On GKE: +Sandboxes reach Git through a forward proxy in Loop Runtime. Enable +`loopRuntime.sandbox.egressGateway` to create the `-egress` Service. +Workers in the same cluster use its internal URL by default. For workers outside +the cluster, expose it on an internal load balancer and set a reachable `url`. +For example, on GKE: ```yaml loopRuntime: diff --git a/braintrust/examples/eks-loop-runtime/README.md b/braintrust/examples/eks-loop-runtime/README.md new file mode 100644 index 0000000..dc00aef --- /dev/null +++ b/braintrust/examples/eks-loop-runtime/README.md @@ -0,0 +1,126 @@ +# Loop Runtime and Sandbox Service on EKS + +This example installs Loop Runtime and Sandbox Service on an existing EKS +cluster. It uses two Helm releases: the Braintrust chart owns Loop Runtime and +the shared Sandbox Service chart owns workers and their drain controller. +The worker chart is the released artifact, rather than another copy of its +templates in this repository. + +The VPC, EKS cluster, node pools, IAM, database, Redis, and buckets remain owned +by your existing infrastructure configuration. Reuse its outputs as chart +inputs. An existing Terraform installer can install both releases with +`helm_release`; it does not need another implementation of those resources. + +## Infrastructure inputs + +| Input | Requirement | +| --- | --- | +| Cluster | Kubernetes 1.30 or newer, with permission to install the worker chart's admission policy and controller RBAC | +| Worker release | One per cluster; the shared chart uses fixed names for its cluster-scoped RBAC and admission policy | +| System nodes | Label `braintrust.io/workload=sandbox-system`; run Loop Runtime and the worker controller here | +| Worker nodes | Dedicated Linux nodes labeled `braintrust.io/workload=sandbox-service` and tainted `braintrust.io/sandbox=true:NoSchedule`, with working `/dev/kvm`, cgroup v2, host networking, and mount propagation | +| Worker disk | Prepare `/var/lib/braintrust-sandboxes/1` on node-local storage before scheduling workers; preserve it across worker restarts | +| Workload identities | Bind API, Brainstore, and Loop Runtime ServiceAccounts to their respective AWS permissions using IRSA or EKS Pod Identity | +| Data services | Set the existing chart's bucket names and `braintrust-secrets`; Loop uses the same database, Redis, and Brainstore storage | +| Network | Allow the runtime and controller to reach worker node addresses on TCP 9400; allow workers to reach the runtime egress Service on TCP 4002 | + +Worker nodes need privileged containers, host PID/network namespaces, and KVM; +Fargate cannot run them. Keep workers on their dedicated nodes and keep +application credentials off those nodes. Apply network isolation at the node +and VPC layers: the worker pods use host networking, so a pod NetworkPolicy +alone does not establish the sandbox boundary. Nodes also need the image-pull +and DNS connectivity required by your infrastructure configuration. + +The headless worker Service resolves directly to worker node addresses. With +both releases in this cluster, cluster DNS provides discovery and the runtime +egress Service provides Git proxy connectivity. No external DNS controller or +additional egress load balancer is required for this example. + +## TLS and namespaces + +Create the namespaces before installing either release: + +```sh +kubectl create namespace braintrust +kubectl create namespace braintrust-sandbox +``` + +Use your PKI to issue a server certificate with the DNS SAN +`sandbox-service.braintrust-sandbox.svc.cluster.local` and server authentication +usage, and a client certificate with client authentication usage. The server +must trust the client CA, and the runtime/controller must trust the server CA. +Keep the server private key exclusively in the worker namespace. + +The following commands use local PEM files, not certificate values in Helm or +Terraform configuration: + +```sh +kubectl -n braintrust-sandbox create secret generic sandbox-tls \ + --from-file=cert.pem=server.pem \ + --from-file=key.pem=server.key \ + --from-file=ca_cert.pem=client-ca.pem + +kubectl -n braintrust-sandbox create secret generic sandbox-client-tls \ + --from-file=client.pem=client.pem \ + --from-file=client.key=client.key \ + --from-file=server-ca.pem=server-ca.pem + +kubectl -n braintrust create secret generic sandbox-client-tls \ + --from-file=client.pem=client.pem \ + --from-file=client.key=client.key \ + --from-file=server-ca.pem=server-ca.pem +``` + +## Install + +These values use the immutable worker chart distributed with Braintrust +v2.18.0. Pin chart and image versions together when adopting a newer release. +From the root of this repository: + +```sh +SANDBOX_CHART_URL=https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/cd27a3983e304932a7e1617adf3105c2b7320a340f26bfea577df781827142f9/sandbox-service-0.1.0.tgz + +helm upgrade --install sandbox-service "$SANDBOX_CHART_URL" \ + --namespace braintrust-sandbox \ + --values braintrust/examples/eks-loop-runtime/sandbox-service.yaml \ + --wait + +helm upgrade --install braintrust ./braintrust \ + --namespace braintrust \ + --values infrastructure-values.yaml \ + --values braintrust/examples/eks-loop-runtime/values.yaml \ + --wait +``` + +`infrastructure-values.yaml` is your existing chart configuration for buckets, +API/Brainstore identities, and storage. Add the Loop Runtime role to +`loopRuntime.serviceAccount.awsRoleArn` when using IRSA, or associate its +ServiceAccount with EKS Pod Identity. Set `AWS_REGION` through +`loopRuntime.extraEnvVars` if your identity integration does not inject it. + +The example fixes worker capacity at one worker. Increase `replicas` in the +worker values according to available node capacity and the runtime's requested +sandbox sizes. Leave the worker chart's ECS-specific `loopRuntime.cluster` and +`loopRuntime.service` empty. Node scaling is handled by the existing cluster +node provisioner; allow capacity for an extra worker during rollouts. + +## Upgrade and disable + +Upgrade each release with the same values and namespace. The worker controller +owns replica changes and drains workers before replacing them; the StatefulSet +uses `OnDelete`, and deletion admission and a disruption budget prevent ordinary +pod deletion or node eviction from bypassing draining. Do not force-delete +worker pods or remove their nodes while sandboxes remain on them. + +To shut down, stop new Loop work and finish or terminate existing sessions while +the runtime is still available to release their sandboxes. Keep the worker +release and TLS Secrets installed. Upgrade the worker release with `replicas: 0` and inspect +`kubectl -n braintrust-sandbox get configmap sandbox-rollout -o yaml` while it +drains. Once draining completes, disable `loopRuntime.enabled` in the Braintrust +release and uninstall the worker release. Its pre-delete +hook refuses uninstall while workers are not drained. Keep node pools and TLS +Secrets until that uninstall succeeds. + +Workers hosted outside this cluster remain supported: supply their discovery +URL and TLS Secret, then expose the runtime egress Service through a reachable +internal endpoint as described in the chart README. diff --git a/braintrust/examples/eks-loop-runtime/sandbox-service.yaml b/braintrust/examples/eks-loop-runtime/sandbox-service.yaml new file mode 100644 index 0000000..a423ed6 --- /dev/null +++ b/braintrust/examples/eks-loop-runtime/sandbox-service.yaml @@ -0,0 +1,14 @@ +# Values for the separately released Sandbox Service chart, not the Braintrust chart. +hostname: sandbox-service.braintrust-sandbox.svc.cluster.local +image: public.ecr.aws/braintrust/sandbox-service:v2.18.0 +replicas: 1 +gitResources: true +controller: + clientTlsSecret: sandbox-client-tls +# Fixed worker capacity is reconciled by the worker controller. Do not set the +# ECS-only loopRuntime.cluster/service fields for a Kubernetes Loop Runtime. +# This installation uses cluster DNS and the cluster's existing node provisioner. +externalDns: + enabled: false +nodeAutoscaler: + enabled: false diff --git a/braintrust/examples/eks-loop-runtime/values.yaml b/braintrust/examples/eks-loop-runtime/values.yaml new file mode 100644 index 0000000..13c774d --- /dev/null +++ b/braintrust/examples/eks-loop-runtime/values.yaml @@ -0,0 +1,17 @@ +# Layer on top of the existing AWS infrastructure values for this chart. +cloud: aws + +loopRuntime: + enabled: true + image: + tag: v2.18.0 + nodeSelector: + braintrust.io/workload: sandbox-system + sandbox: + dnsUrl: https://sandbox-service.braintrust-sandbox.svc.cluster.local:9400 + tls: + secretName: sandbox-client-tls + egressGateway: + enabled: true + # Uses the in-cluster egress Service; no additional load balancer is needed. + url: "" diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index d5e30fd..771e38f 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -199,15 +199,20 @@ s3://{{ .Values.objectStorage.aws.codeBundleBucket }} {{- if not (or $sandbox.dnsUrl $urls) -}} {{- fail "loopRuntime.sandbox.dnsUrl or loopRuntime.sandbox.urls is required when loopRuntime.enabled is true (GKE: use Terraform output gke_isolated_workers.discovery.endpoint)" -}} {{- end -}} +{{- range $urls -}} +{{- if not (. | default "" | trim) -}} +{{- fail "loopRuntime.sandbox.urls entries must not be blank" -}} +{{- end -}} +{{- end -}} {{- $_ := required "loopRuntime.sandbox.image is required when loopRuntime.enabled is true" $sandbox.image -}} {{- range (prepend $urls ($sandbox.dnsUrl | default "")) -}} +{{- if ne . (trim .) -}} +{{- fail "Sandbox Service endpoints must not contain leading or trailing whitespace" -}} +{{- end -}} {{- if and . (hasPrefix "https://" .) (not $sandbox.tls.secretName) -}} {{- fail "loopRuntime.sandbox.tls.secretName is required for https:// Sandbox Service endpoints" -}} {{- end -}} {{- end -}} -{{- if $sandbox.egressGateway.enabled -}} -{{- $_ := required "loopRuntime.sandbox.egressGateway.url is required when loopRuntime.sandbox.egressGateway.enabled is true" $sandbox.egressGateway.url -}} -{{- end -}} {{- end -}} {{- end -}} diff --git a/braintrust/templates/loop-runtime-configmap.yaml b/braintrust/templates/loop-runtime-configmap.yaml index 5671f96..c24cc66 100644 --- a/braintrust/templates/loop-runtime-configmap.yaml +++ b/braintrust/templates/loop-runtime-configmap.yaml @@ -55,7 +55,7 @@ data: {{- end }} {{- if $egressGateway.enabled }} LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN: "0.0.0.0:{{ $egressGateway.port }}" - LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL: {{ $egressGateway.url | quote }} + LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL: {{ $egressGateway.url | default (printf "http://%s-egress.%s:%v" (.Values.loopRuntime.service.name | default .Values.loopRuntime.name) (include "braintrust.namespace" .) $egressGateway.port) | quote }} {{- end }} {{- with (.Values.global.allowedOrgIds | default "" | toString | trim) }} ALLOWED_ORG_IDS: {{ . | quote }} diff --git a/braintrust/tests/eks-loop-runtime_test.yaml b/braintrust/tests/eks-loop-runtime_test.yaml new file mode 100644 index 0000000..b409479 --- /dev/null +++ b/braintrust/tests/eks-loop-runtime_test.yaml @@ -0,0 +1,104 @@ +suite: EKS Loop Runtime installation with the shared Sandbox Service chart +templates: + - loop-runtime-configmap.yaml + - loop-runtime-deployment.yaml + - loop-runtime-egress-service.yaml + - api-configmap.yaml +tests: + - it: should discover the companion workers through cluster DNS and use the internal proxy + release: + namespace: braintrust + template: loop-runtime-configmap.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + - ../examples/eks-loop-runtime/values.yaml + asserts: + - equal: + path: data.EXO_SANDBOX_SERVICE_DNS_URL + value: https://sandbox-service.braintrust-sandbox.svc.cluster.local:9400 + - equal: + path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL + value: http://braintrust-loop-runtime-egress.braintrust:4002 + - equal: + path: data.SANDBOX_SERVICE_TLS_CERT + value: /etc/sandbox-tls/client.pem + - isNull: + path: data.EXO_SANDBOX_PROVIDER + - isNull: + path: data.AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER + + - it: should expose an internal egress Service for the companion workers + release: + namespace: braintrust + template: loop-runtime-egress-service.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + - ../examples/eks-loop-runtime/values.yaml + asserts: + - equal: + path: spec.type + value: ClusterIP + - equal: + path: spec.ports[0].targetPort + value: 4002 + + - it: should install the runtime on system nodes with the client TLS Secret + release: + namespace: braintrust + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + - ../examples/eks-loop-runtime/values.yaml + asserts: + - equal: + path: spec.template.spec.nodeSelector["braintrust.io/workload"] + value: sandbox-system + - equal: + path: spec.template.spec.containers[0].image + value: test/loop-runtime:v2.18.0 + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + - contains: + path: spec.template.spec.volumes + content: + name: sandbox-service-tls + secret: + secretName: sandbox-client-tls + items: + - key: client.pem + path: client.pem + - key: client.key + path: client.key + - key: server-ca.pem + path: server-ca.pem + + - it: should publish the runtime URL to the API + release: + namespace: braintrust + template: api-configmap.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + - ../examples/eks-loop-runtime/values.yaml + asserts: + - equal: + path: data.LOOP_RUNTIME_URL + value: http://braintrust-loop-runtime.braintrust:4001 + + - it: should remove the runtime resources when disabled with the worker release retained + release: + namespace: braintrust + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + - ../examples/eks-loop-runtime/values.yaml + set: + loopRuntime.enabled: false + asserts: + - hasDocuments: + count: 0 diff --git a/braintrust/tests/loop-runtime-configmap_test.yaml b/braintrust/tests/loop-runtime-configmap_test.yaml index fee4df1..4ea9614 100644 --- a/braintrust/tests/loop-runtime-configmap_test.yaml +++ b/braintrust/tests/loop-runtime-configmap_test.yaml @@ -337,13 +337,105 @@ tests: - isNull: path: data.LOOP_RUNTIME_SANDBOX_EGRESS_GATEWAY_URL - - it: should require the egress gateway URL when the gateway is enabled + - it: should use the in-cluster egress Service when no external URL is set + release: + namespace: braintrust values: - __fixtures__/base-values.yaml - __fixtures__/google-values.yaml - __fixtures__/loop-runtime-google-values.yaml set: loopRuntime.sandbox.egressGateway.enabled: true + loopRuntime.service.name: loop + loopRuntime.sandbox.egressGateway.port: 4100 + asserts: + - equal: + path: data.LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN + value: 0.0.0.0:4100 + - equal: + path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL + value: http://loop-egress.braintrust:4100 + + - it: should default the egress URL when reused values omit it + release: + namespace: braintrust + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.egressGateway.enabled: true + loopRuntime.sandbox.egressGateway.url: null + asserts: + - equal: + path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL + value: http://braintrust-loop-runtime-egress.braintrust:4002 + + - it: should treat a zero egress URL as empty + release: + namespace: braintrust + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.egressGateway.enabled: true + loopRuntime.sandbox.egressGateway.url: 0 + asserts: + - equal: + path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL + value: http://braintrust-loop-runtime-egress.braintrust:4002 + + + - it: should reject empty endpoint + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.dnsUrl: "" + loopRuntime.sandbox.urls: [""] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.urls entries must not be blank" + + - it: should reject whitespace endpoint + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.dnsUrl: "" + loopRuntime.sandbox.urls: [" "] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.urls entries must not be blank" + + - it: should reject null endpoint + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.dnsUrl: "" + loopRuntime.sandbox.urls: [null] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.urls entries must not be blank" + + - it: should reject blank entry alongside a usable endpoint + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.dnsUrl: "" + loopRuntime.sandbox.urls: ["https://worker.example:9400", ""] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.urls entries must not be blank" + + - it: should reject padded endpoint + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.sandbox.dnsUrl: "" + loopRuntime.sandbox.urls: [" https://worker.example:9400 "] asserts: - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.url is required when loopRuntime.sandbox.egressGateway.enabled is true" + errorMessage: "Sandbox Service endpoints must not contain leading or trailing whitespace" diff --git a/braintrust/tests/loop-runtime-images_test.yaml b/braintrust/tests/loop-runtime-images_test.yaml new file mode 100644 index 0000000..1db029e --- /dev/null +++ b/braintrust/tests/loop-runtime-images_test.yaml @@ -0,0 +1,151 @@ +suite: Loop Runtime image tag compatibility +templates: + - loop-runtime-deployment.yaml + - loop-runtime-configmap.yaml +tests: + - it: should render image tag v2.18.0 + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.image.tag: "v2.18.0" + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: "test/loop-runtime:v2.18.0" + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: should render image tag 2.18.0 + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.image.tag: "2.18.0" + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: "test/loop-runtime:2.18.0" + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: should render image tag v2.18.0-rc.1 + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.image.tag: "v2.18.0-rc.1" + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: "test/loop-runtime:v2.18.0-rc.1" + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: should render image tag 2.18.0-rc.1 + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.image.tag: "2.18.0-rc.1" + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: "test/loop-runtime:2.18.0-rc.1" + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: should render image tag a1b2c3d + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.image.tag: "a1b2c3d" + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: "test/loop-runtime:a1b2c3d" + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: should render image tag 1234567 + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.image.tag: "1234567" + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: "test/loop-runtime:1234567" + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: should render image tag 53af3580126bc915b67166a41d5c78b3985d1e94 + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.image.tag: "53af3580126bc915b67166a41d5c78b3985d1e94" + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: "test/loop-runtime:53af3580126bc915b67166a41d5c78b3985d1e94" + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: should render image tag custom-build + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.image.tag: "custom-build" + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: "test/loop-runtime:custom-build" + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: should render image tag latest-2.x + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.image.tag: "latest-2.x" + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: "test/loop-runtime:latest-2.x" + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + + - it: should retain an explicit pull policy for a pinned image + template: loop-runtime-deployment.yaml + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + set: + loopRuntime.image.pullPolicy: IfNotPresent + asserts: + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent diff --git a/braintrust/values.yaml b/braintrust/values.yaml index fc03867..4b06515 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -325,7 +325,7 @@ loopRuntime: image: repository: public.ecr.aws/braintrust/loop-runtime tag: latest-2.x - pullPolicy: IfNotPresent + pullPolicy: Always service: # Optional name for the Service. If empty, loopRuntime.name is used. name: "" @@ -336,7 +336,7 @@ loopRuntime: name: "braintrust-loop-runtime" # AWS IRSA role. Leave empty when using EKS Pod Identity. awsRoleArn: "" - # Required on Google (Terraform output loop_runtime_service_account). + # Required on Google: the Google service account bound to this workload. googleServiceAccount: "" annotations: {} # These URLs default to in-cluster services. Set them when the API, AI proxy, @@ -391,12 +391,12 @@ loopRuntime: affinity: {} # Sandboxes run on Braintrust Sandbox Service workers, reached over mTLS. sandbox: - # Set one. dnsUrl resolves every worker behind a headless DNS name - # (GKE: Terraform output gke_isolated_workers.discovery.endpoint). + # Set one. dnsUrl resolves every worker behind a headless DNS name. + # Workers may run in this cluster or on a separately managed cluster. dnsUrl: "" urls: [] image: "public.ecr.aws/braintrust/codex-runtime:latest" - # Must match the Sandbox Service release's per-sandbox sizing. + # Resources requested for each sandbox; size worker capacity accordingly. vcpuCount: 1 memoryMib: 2048 # restricted or internet @@ -408,8 +408,8 @@ loopRuntime: certKey: client.pem keyKey: client.key caKey: server-ca.pem - # Forward proxy that sandboxes use for Git access. url is the address the - # workers use to reach the -egress Service. + # Forward proxy that sandboxes use for Git access. An empty url defaults to + # the in-cluster -egress Service. Override it for external workers. egressGateway: enabled: false port: 4002 diff --git a/scripts/test-eks-loop-runtime.sh b/scripts/test-eks-loop-runtime.sh new file mode 100644 index 0000000..beb55b2 --- /dev/null +++ b/scripts/test-eks-loop-runtime.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TEST_DIR="$(mktemp -d)" +trap 'rm -rf "$TEST_DIR"' EXIT + +# Content-addressed worker chart distributed with Braintrust v2.18.0. +CHART_SHA256=cd27a3983e304932a7e1617adf3105c2b7320a340f26bfea577df781827142f9 +CHART_URL="https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/$CHART_SHA256/sandbox-service-0.1.0.tgz" +CHART_ARCHIVE="${SANDBOX_SERVICE_CHART:-$TEST_DIR/sandbox-service.tgz}" +if [[ -z "${SANDBOX_SERVICE_CHART:-}" ]]; then + curl --fail --location --silent --show-error --max-time 60 "$CHART_URL" --output "$CHART_ARCHIVE" +fi +python3 - "$CHART_ARCHIVE" "$CHART_SHA256" <<'PY' +import hashlib +import pathlib +import sys + +if hashlib.sha256(pathlib.Path(sys.argv[1]).read_bytes()).hexdigest() != sys.argv[2]: + raise SystemExit("Sandbox Service chart checksum mismatch") +PY +tar -xzf "$CHART_ARCHIVE" -C "$TEST_DIR" + +helm unittest \ + --values "$ROOT_DIR/braintrust/examples/eks-loop-runtime/sandbox-service.yaml" \ + --file "$ROOT_DIR/tests/sandbox-service_test.yaml" \ + "$TEST_DIR/sandbox-service" +helm lint "$TEST_DIR/sandbox-service" --strict \ + --values "$ROOT_DIR/braintrust/examples/eks-loop-runtime/sandbox-service.yaml" diff --git a/test.sh b/test.sh index 75159a6..d86d141 100755 --- a/test.sh +++ b/test.sh @@ -94,6 +94,10 @@ fi echo "" echo "✅ Chart rendering tests passed" +echo "" +echo "Testing the EKS integration with the released Sandbox Service chart..." +bash scripts/test-eks-loop-runtime.sh + echo "" echo "==========================================" echo "3. Running Helm Lint" diff --git a/tests/sandbox-service_test.yaml b/tests/sandbox-service_test.yaml new file mode 100644 index 0000000..6ee8b7a --- /dev/null +++ b/tests/sandbox-service_test.yaml @@ -0,0 +1,165 @@ +suite: released Sandbox Service chart with EKS Loop Runtime values +templates: + - configmap.yaml + - service.yaml + - worker-statefulset.yaml + - worker-controller.yaml + - worker-disruption-budget.yaml + - worker-admission-policy.yaml + - check-drained.yaml +tests: + - it: should advertise the same TLS hostname that the runtime discovers + release: + namespace: braintrust-sandbox + revision: 1 + template: configmap.yaml + asserts: + - matchRegex: + path: data["config.json"] + pattern: '"endpoint": "https://sandbox-service.braintrust-sandbox.svc.cluster.local:9400"' + + - it: should expose individual worker addresses through a headless Service + release: + namespace: braintrust-sandbox + revision: 1 + template: service.yaml + asserts: + - equal: + path: metadata.namespace + value: braintrust-sandbox + - equal: + path: spec.clusterIP + value: None + - equal: + path: spec.ports[0].port + value: 9400 + + - it: should keep Firecracker workers isolated on dedicated nodes and preserve host state + release: + namespace: braintrust-sandbox + revision: 1 + template: worker-statefulset.yaml + asserts: + - equal: + path: spec.updateStrategy.type + value: OnDelete + - isNull: + path: spec.replicas + - equal: + path: spec.template.spec.nodeSelector["braintrust.io/workload"] + value: sandbox-service + - equal: + path: spec.template.spec.hostNetwork + value: true + - equal: + path: spec.template.spec.hostPID + value: true + - equal: + path: spec.template.spec.automountServiceAccountToken + value: false + - equal: + path: spec.template.spec.containers[0].securityContext.privileged + value: true + - equal: + path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey + value: kubernetes.io/hostname + - contains: + path: spec.template.spec.volumes + content: + name: state + hostPath: + path: /var/lib/braintrust-sandboxes/1 + type: Directory + - contains: + path: spec.template.spec.volumes + content: + name: tls + secret: + secretName: sandbox-tls + defaultMode: 256 + + - it: should reconcile fixed worker capacity without depending on an ECS service + release: + namespace: braintrust-sandbox + revision: 1 + template: worker-controller.yaml + documentIndex: 6 + asserts: + - equal: + path: spec.template.spec.containers[0].command + value: [sandbox-service, reconcile-workers] + - equal: + path: spec.template.spec.containers[0].args + value: + - --endpoint=https://sandbox-service.braintrust-sandbox.svc.cluster.local:9400 + - --replicas=1 + - --configuration-revision=1 + - contains: + path: spec.template.spec.volumes + content: + name: tls + secret: + secretName: sandbox-client-tls + defaultMode: 288 + + - it: should request draining rather than directly deleting workers when capacity is zero + release: + namespace: braintrust-sandbox + revision: 1 + template: worker-controller.yaml + documentIndex: 6 + set: + replicas: 0 + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --replicas=0 + + - it: should reconcile a new Helm revision through the worker controller + template: worker-controller.yaml + documentIndex: 6 + release: + revision: 2 + namespace: braintrust-sandbox + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --configuration-revision=2 + + - it: should block ordinary eviction of workers holding local sandbox state + release: + namespace: braintrust-sandbox + revision: 1 + template: worker-disruption-budget.yaml + asserts: + - equal: + path: spec.maxUnavailable + value: 0 + + - it: should protect worker deletion in the sandbox namespace + release: + namespace: braintrust-sandbox + revision: 1 + template: worker-admission-policy.yaml + documentIndex: 0 + asserts: + - matchRegex: + path: spec.matchConditions[0].expression + pattern: "request.namespace == 'braintrust-sandbox'" + - equal: + path: spec.failurePolicy + value: Fail + + - it: should refuse uninstall until the worker release has drained + release: + namespace: braintrust-sandbox + revision: 1 + template: check-drained.yaml + documentIndex: 3 + asserts: + - equal: + path: metadata.annotations["helm.sh/hook"] + value: pre-delete + - equal: + path: spec.template.spec.containers[0].command + value: [sandbox-service, check-workers-drained] From 494f50d8c56119baf9b59e446770f7d885d1f69a Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Wed, 21 Oct 2026 00:57:51 -0700 Subject: [PATCH 07/25] Trim Loop Runtime examples and test scaffolding --- .github/workflows/test-helm-chart.yml | 1 - braintrust/README.md | 21 --- .../examples/eks-loop-runtime/README.md | 161 ++++++----------- .../eks-loop-runtime/sandbox-service.yaml | 11 +- .../templates/loop-runtime-deployment.yaml | 2 +- braintrust/tests/eks-loop-runtime_test.yaml | 50 +----- .../tests/loop-runtime-configmap_test.yaml | 149 ++++------------ .../tests/loop-runtime-deployment_test.yaml | 83 +++++---- .../loop-runtime-egress-service_test.yaml | 23 +-- .../tests/loop-runtime-images_test.yaml | 62 ++----- .../loop-runtime-serviceaccount_test.yaml | 23 +-- braintrust/values.yaml | 1 + scripts/test-eks-loop-runtime.sh | 30 ---- test.sh | 4 - tests/sandbox-service_test.yaml | 165 ------------------ 15 files changed, 172 insertions(+), 614 deletions(-) delete mode 100644 scripts/test-eks-loop-runtime.sh delete mode 100644 tests/sandbox-service_test.yaml diff --git a/.github/workflows/test-helm-chart.yml b/.github/workflows/test-helm-chart.yml index 0c35884..3bf0943 100644 --- a/.github/workflows/test-helm-chart.yml +++ b/.github/workflows/test-helm-chart.yml @@ -27,7 +27,6 @@ jobs: run: | python3 -m unittest discover -s tests -p 'test_*.py' helm unittest braintrust - bash scripts/test-eks-loop-runtime.sh lint-test: name: Lint and Test diff --git a/braintrust/README.md b/braintrust/README.md index a49eeac..3bbaf94 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -55,27 +55,6 @@ loopRuntime: secretName: sandbox-client-tls ``` -EKS: - -```yaml -cloud: aws - -objectStorage: - aws: - brainstoreBucket: "" - codeBundleBucket: "" - -loopRuntime: - enabled: true - serviceAccount: - # Leave empty when using EKS Pod Identity. - awsRoleArn: "" - sandbox: - dnsUrl: "https://:9400" - tls: - secretName: sandbox-client-tls -``` - Sandboxes reach Git through a forward proxy in Loop Runtime. Enable `loopRuntime.sandbox.egressGateway` to create the `-egress` Service. Workers in the same cluster use its internal URL by default. For workers outside diff --git a/braintrust/examples/eks-loop-runtime/README.md b/braintrust/examples/eks-loop-runtime/README.md index dc00aef..2a3c85b 100644 --- a/braintrust/examples/eks-loop-runtime/README.md +++ b/braintrust/examples/eks-loop-runtime/README.md @@ -1,126 +1,73 @@ # Loop Runtime and Sandbox Service on EKS -This example installs Loop Runtime and Sandbox Service on an existing EKS -cluster. It uses two Helm releases: the Braintrust chart owns Loop Runtime and -the shared Sandbox Service chart owns workers and their drain controller. -The worker chart is the released artifact, rather than another copy of its -templates in this repository. - -The VPC, EKS cluster, node pools, IAM, database, Redis, and buckets remain owned -by your existing infrastructure configuration. Reuse its outputs as chart -inputs. An existing Terraform installer can install both releases with -`helm_release`; it does not need another implementation of those resources. - -## Infrastructure inputs - -| Input | Requirement | +Install two Helm releases: this chart for Loop Runtime and the published +Sandbox Service chart for workers and their drain controller. Reuse the +existing AWS networking, database, Redis, buckets, and IAM configuration. + +## Prerequisites + +- Kubernetes 1.30+; one worker release per cluster because its cluster-scoped + RBAC and admission policy names are fixed. +- System nodes labeled `braintrust.io/workload=sandbox-system`. +- Dedicated Linux worker nodes labeled `braintrust.io/workload=sandbox-service` + and tainted `braintrust.io/sandbox=true:NoSchedule`, with `/dev/kvm`, + cgroup v2, mount propagation, and node-local storage prepared at + `/var/lib/braintrust-sandboxes/1`. Workers require privileged containers + and host PID/network namespaces; Fargate cannot run them. +- Node/VPC isolation for workers; pod NetworkPolicy alone does not isolate + host-networked pods. Keep application credentials off worker nodes. +- Runtime/controller connectivity to worker node addresses on TCP 9400, + worker connectivity to the runtime proxy Service on TCP 4002, and image-pull/DNS access. +- Your existing chart values and `braintrust-secrets`, plus Loop Runtime's + workload identity. Set `loopRuntime.serviceAccount.awsRoleArn` for IRSA or + associate its ServiceAccount with EKS Pod Identity. Supply `AWS_REGION` + through `loopRuntime.extraEnvVars` if identity integration does not inject it. + +Create namespaces `braintrust` and `braintrust-sandbox`, then these Secrets +from your PKI. The server certificate must cover +`sandbox-service.braintrust-sandbox.svc.cluster.local` and have server +authentication usage; client certificates need client authentication usage. + +| Namespace / Secret | Keys | | --- | --- | -| Cluster | Kubernetes 1.30 or newer, with permission to install the worker chart's admission policy and controller RBAC | -| Worker release | One per cluster; the shared chart uses fixed names for its cluster-scoped RBAC and admission policy | -| System nodes | Label `braintrust.io/workload=sandbox-system`; run Loop Runtime and the worker controller here | -| Worker nodes | Dedicated Linux nodes labeled `braintrust.io/workload=sandbox-service` and tainted `braintrust.io/sandbox=true:NoSchedule`, with working `/dev/kvm`, cgroup v2, host networking, and mount propagation | -| Worker disk | Prepare `/var/lib/braintrust-sandboxes/1` on node-local storage before scheduling workers; preserve it across worker restarts | -| Workload identities | Bind API, Brainstore, and Loop Runtime ServiceAccounts to their respective AWS permissions using IRSA or EKS Pod Identity | -| Data services | Set the existing chart's bucket names and `braintrust-secrets`; Loop uses the same database, Redis, and Brainstore storage | -| Network | Allow the runtime and controller to reach worker node addresses on TCP 9400; allow workers to reach the runtime egress Service on TCP 4002 | - -Worker nodes need privileged containers, host PID/network namespaces, and KVM; -Fargate cannot run them. Keep workers on their dedicated nodes and keep -application credentials off those nodes. Apply network isolation at the node -and VPC layers: the worker pods use host networking, so a pod NetworkPolicy -alone does not establish the sandbox boundary. Nodes also need the image-pull -and DNS connectivity required by your infrastructure configuration. - -The headless worker Service resolves directly to worker node addresses. With -both releases in this cluster, cluster DNS provides discovery and the runtime -egress Service provides Git proxy connectivity. No external DNS controller or -additional egress load balancer is required for this example. - -## TLS and namespaces - -Create the namespaces before installing either release: - -```sh -kubectl create namespace braintrust -kubectl create namespace braintrust-sandbox -``` - -Use your PKI to issue a server certificate with the DNS SAN -`sandbox-service.braintrust-sandbox.svc.cluster.local` and server authentication -usage, and a client certificate with client authentication usage. The server -must trust the client CA, and the runtime/controller must trust the server CA. -Keep the server private key exclusively in the worker namespace. +| `braintrust-sandbox/sandbox-tls` | `cert.pem`, `key.pem`, `ca_cert.pem` (client CA) | +| `braintrust-sandbox/sandbox-client-tls` | `client.pem`, `client.key`, `server-ca.pem` | +| `braintrust/sandbox-client-tls` | `client.pem`, `client.key`, `server-ca.pem` | -The following commands use local PEM files, not certificate values in Helm or -Terraform configuration: - -```sh -kubectl -n braintrust-sandbox create secret generic sandbox-tls \ - --from-file=cert.pem=server.pem \ - --from-file=key.pem=server.key \ - --from-file=ca_cert.pem=client-ca.pem - -kubectl -n braintrust-sandbox create secret generic sandbox-client-tls \ - --from-file=client.pem=client.pem \ - --from-file=client.key=client.key \ - --from-file=server-ca.pem=server-ca.pem - -kubectl -n braintrust create secret generic sandbox-client-tls \ - --from-file=client.pem=client.pem \ - --from-file=client.key=client.key \ - --from-file=server-ca.pem=server-ca.pem -``` +Keep the server private key in the worker namespace. The server trusts the +client CA; the runtime and controller trust the server CA. ## Install -These values use the immutable worker chart distributed with Braintrust -v2.18.0. Pin chart and image versions together when adopting a newer release. -From the root of this repository: +From the repository root, using the worker chart distributed with v2.18.0: ```sh SANDBOX_CHART_URL=https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/cd27a3983e304932a7e1617adf3105c2b7320a340f26bfea577df781827142f9/sandbox-service-0.1.0.tgz helm upgrade --install sandbox-service "$SANDBOX_CHART_URL" \ --namespace braintrust-sandbox \ - --values braintrust/examples/eks-loop-runtime/sandbox-service.yaml \ - --wait + --values braintrust/examples/eks-loop-runtime/sandbox-service.yaml --wait helm upgrade --install braintrust ./braintrust \ --namespace braintrust \ --values infrastructure-values.yaml \ - --values braintrust/examples/eks-loop-runtime/values.yaml \ - --wait + --values braintrust/examples/eks-loop-runtime/values.yaml --wait ``` -`infrastructure-values.yaml` is your existing chart configuration for buckets, -API/Brainstore identities, and storage. Add the Loop Runtime role to -`loopRuntime.serviceAccount.awsRoleArn` when using IRSA, or associate its -ServiceAccount with EKS Pod Identity. Set `AWS_REGION` through -`loopRuntime.extraEnvVars` if your identity integration does not inject it. - -The example fixes worker capacity at one worker. Increase `replicas` in the -worker values according to available node capacity and the runtime's requested -sandbox sizes. Leave the worker chart's ECS-specific `loopRuntime.cluster` and -`loopRuntime.service` empty. Node scaling is handled by the existing cluster -node provisioner; allow capacity for an extra worker during rollouts. - -## Upgrade and disable - -Upgrade each release with the same values and namespace. The worker controller -owns replica changes and drains workers before replacing them; the StatefulSet -uses `OnDelete`, and deletion admission and a disruption budget prevent ordinary -pod deletion or node eviction from bypassing draining. Do not force-delete -worker pods or remove their nodes while sandboxes remain on them. - -To shut down, stop new Loop work and finish or terminate existing sessions while -the runtime is still available to release their sandboxes. Keep the worker -release and TLS Secrets installed. Upgrade the worker release with `replicas: 0` and inspect -`kubectl -n braintrust-sandbox get configmap sandbox-rollout -o yaml` while it -drains. Once draining completes, disable `loopRuntime.enabled` in the Braintrust -release and uninstall the worker release. Its pre-delete -hook refuses uninstall while workers are not drained. Keep node pools and TLS -Secrets until that uninstall succeeds. - -Workers hosted outside this cluster remain supported: supply their discovery -URL and TLS Secret, then expose the runtime egress Service through a reachable -internal endpoint as described in the chart README. +`infrastructure-values.yaml` supplies your existing bucket, identity, and +storage settings. Cluster DNS discovers workers; an internal Service exposes +the Git proxy. External workers can instead use an explicit discovery/proxy URL. +The worker example uses fixed capacity (`replicas: 1`); leave its ECS-specific +`loopRuntime.cluster/service` unset. Use the cluster's existing node +provisioner and allow an extra worker during rollouts. Pin chart/image versions together. + +## Upgrade and shutdown + +Upgrade with the same values and namespaces. The worker controller owns +scale-down and replacement; do not force-delete workers or remove their nodes. +Before shutdown, stop new work and finish/terminate sessions while the runtime +can release their sandboxes. Upgrade the worker release with `replicas: 0` +and inspect `kubectl -n braintrust-sandbox get configmap sandbox-rollout -o yaml`. +After draining, disable `loopRuntime.enabled` and uninstall the worker release. +Its pre-delete hook refuses uninstall until drained; retain nodes and TLS +Secrets until it succeeds. diff --git a/braintrust/examples/eks-loop-runtime/sandbox-service.yaml b/braintrust/examples/eks-loop-runtime/sandbox-service.yaml index a423ed6..f251e2e 100644 --- a/braintrust/examples/eks-loop-runtime/sandbox-service.yaml +++ b/braintrust/examples/eks-loop-runtime/sandbox-service.yaml @@ -1,14 +1,5 @@ -# Values for the separately released Sandbox Service chart, not the Braintrust chart. +# Values for the separately released Sandbox Service chart. hostname: sandbox-service.braintrust-sandbox.svc.cluster.local image: public.ecr.aws/braintrust/sandbox-service:v2.18.0 replicas: 1 gitResources: true -controller: - clientTlsSecret: sandbox-client-tls -# Fixed worker capacity is reconciled by the worker controller. Do not set the -# ECS-only loopRuntime.cluster/service fields for a Kubernetes Loop Runtime. -# This installation uses cluster DNS and the cluster's existing node provisioner. -externalDns: - enabled: false -nodeAutoscaler: - enabled: false diff --git a/braintrust/templates/loop-runtime-deployment.yaml b/braintrust/templates/loop-runtime-deployment.yaml index 9a7276f..43d14bc 100644 --- a/braintrust/templates/loop-runtime-deployment.yaml +++ b/braintrust/templates/loop-runtime-deployment.yaml @@ -21,7 +21,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - replicas: {{ .Values.loopRuntime.replicas }} + replicas: {{ ternary 1 .Values.loopRuntime.replicas (has .Values.loopRuntime.replicas (list nil "")) }} strategy: type: RollingUpdate rollingUpdate: diff --git a/braintrust/tests/eks-loop-runtime_test.yaml b/braintrust/tests/eks-loop-runtime_test.yaml index b409479..4ad4281 100644 --- a/braintrust/tests/eks-loop-runtime_test.yaml +++ b/braintrust/tests/eks-loop-runtime_test.yaml @@ -9,7 +9,7 @@ tests: release: namespace: braintrust template: loop-runtime-configmap.yaml - values: + values: &eks-values - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml - ../examples/eks-loop-runtime/values.yaml @@ -28,30 +28,11 @@ tests: - isNull: path: data.AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER - - it: should expose an internal egress Service for the companion workers - release: - namespace: braintrust - template: loop-runtime-egress-service.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - - ../examples/eks-loop-runtime/values.yaml - asserts: - - equal: - path: spec.type - value: ClusterIP - - equal: - path: spec.ports[0].targetPort - value: 4002 - - it: should install the runtime on system nodes with the client TLS Secret release: namespace: braintrust template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - - ../examples/eks-loop-runtime/values.yaml + values: *eks-values asserts: - equal: path: spec.template.spec.nodeSelector["braintrust.io/workload"] @@ -75,30 +56,3 @@ tests: path: client.key - key: server-ca.pem path: server-ca.pem - - - it: should publish the runtime URL to the API - release: - namespace: braintrust - template: api-configmap.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - - ../examples/eks-loop-runtime/values.yaml - asserts: - - equal: - path: data.LOOP_RUNTIME_URL - value: http://braintrust-loop-runtime.braintrust:4001 - - - it: should remove the runtime resources when disabled with the worker release retained - release: - namespace: braintrust - template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - - ../examples/eks-loop-runtime/values.yaml - set: - loopRuntime.enabled: false - asserts: - - hasDocuments: - count: 0 diff --git a/braintrust/tests/loop-runtime-configmap_test.yaml b/braintrust/tests/loop-runtime-configmap_test.yaml index 4ea9614..b273703 100644 --- a/braintrust/tests/loop-runtime-configmap_test.yaml +++ b/braintrust/tests/loop-runtime-configmap_test.yaml @@ -3,9 +3,9 @@ templates: - loop-runtime-configmap.yaml tests: - it: should not render when Loop Runtime is disabled - values: + values: &base-values - __fixtures__/base-values.yaml - asserts: + asserts: &absent - hasDocuments: count: 0 @@ -14,9 +14,7 @@ tests: - __fixtures__/base-values.yaml - __fixtures__/google-values.yaml - __fixtures__/pre-loop-runtime-values.yaml - asserts: - - hasDocuments: - count: 0 + asserts: *absent - it: should explain how to upgrade when reused values enable Loop Runtime without defaults values: @@ -28,7 +26,7 @@ tests: errorMessage: "loopRuntime.image is missing. Values reused from a chart release that predates Loop Runtime omit the loopRuntime defaults; upgrade with --reset-then-reuse-values or supply the full loopRuntime map." - it: should render AWS storage and the Sandbox Service configuration - values: + values: &aws-values - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml release: @@ -82,9 +80,7 @@ tests: path: data.BRAINSTORE_LOCKS_URI - it: should use AWS object storage locks when configured - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: brainstore.locksBackend: objectStorage asserts: @@ -93,7 +89,7 @@ tests: value: s3://test-brainstore-bucket/brainstore/locks - it: should render GKE storage and the Sandbox Service configuration - values: + values: &gke-values - __fixtures__/base-values.yaml - __fixtures__/google-values.yaml - __fixtures__/loop-runtime-google-values.yaml @@ -145,10 +141,7 @@ tests: path: data.BRAINSTORE_LOCKS_URI - it: should use GCS object storage locks when configured - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: brainstore.locksBackend: objectStorage asserts: @@ -157,10 +150,7 @@ tests: value: gs://test-brainstore-bucket/brainstore/locks - it: should pass sandbox sizing and internet egress mode - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: loopRuntime.sandbox.vcpuCount: 2 loopRuntime.sandbox.memoryMib: 4096 @@ -177,10 +167,7 @@ tests: value: internet - it: should join fixed Sandbox Service endpoints - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: loopRuntime.sandbox.dnsUrl: "" loopRuntime.sandbox.urls: @@ -194,10 +181,7 @@ tests: path: data.EXO_SANDBOX_SERVICE_DNS_URL - it: should allow a plain HTTP Sandbox Service endpoint without TLS - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: loopRuntime.sandbox.dnsUrl: http://sandbox-service.sandbox:9400 loopRuntime.sandbox.tls.secretName: "" @@ -209,35 +193,25 @@ tests: path: data.SANDBOX_SERVICE_TLS_CERT - it: should require TLS for HTTPS Sandbox Service endpoints - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: loopRuntime.sandbox.tls.secretName: "" - asserts: + asserts: &tls-required - failedTemplate: errorMessage: "loopRuntime.sandbox.tls.secretName is required for https:// Sandbox Service endpoints" - it: should require TLS when any fixed endpoint uses HTTPS - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: loopRuntime.sandbox.dnsUrl: "" loopRuntime.sandbox.tls.secretName: "" loopRuntime.sandbox.urls: - http://worker-0.sandbox.internal:9400 - https://worker-1.sandbox.internal:9400 - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.tls.secretName is required for https:// Sandbox Service endpoints" + asserts: *tls-required - it: should require a Sandbox Service endpoint - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: loopRuntime.sandbox.dnsUrl: "" asserts: @@ -245,10 +219,7 @@ tests: errorMessage: "loopRuntime.sandbox.dnsUrl or loopRuntime.sandbox.urls is required when loopRuntime.enabled is true (GKE: use Terraform output gke_isolated_workers.discovery.endpoint)" - it: should reject both Sandbox Service discovery modes - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: loopRuntime.sandbox.urls: - https://worker-0.sandbox.internal:9400 @@ -257,10 +228,7 @@ tests: errorMessage: "Set only one of loopRuntime.sandbox.dnsUrl and loopRuntime.sandbox.urls." - it: should require the sandbox guest image - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: loopRuntime.sandbox.image: "" asserts: @@ -268,10 +236,7 @@ tests: errorMessage: "loopRuntime.sandbox.image is required when loopRuntime.enabled is true" - it: should require the Loop Runtime Google service account on GKE - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: loopRuntime.serviceAccount.googleServiceAccount: "" asserts: @@ -279,10 +244,7 @@ tests: errorMessage: "loopRuntime.serviceAccount.googleServiceAccount is required when cloud is google (use Terraform output loop_runtime_service_account)" - it: should require the GCS API bucket on GKE - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: objectStorage.google.apiBucket: "" asserts: @@ -290,9 +252,7 @@ tests: errorMessage: "objectStorage.google.apiBucket is required when loopRuntime.enabled is true" - it: should require the code bundle bucket on AWS - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: objectStorage.aws.codeBundleBucket: "" asserts: @@ -305,24 +265,18 @@ tests: - __fixtures__/azure-values.yaml set: loopRuntime.enabled: true - asserts: + asserts: &assertions-1 - failedTemplate: errorMessage: "loopRuntime.enabled requires cloud: aws or cloud: google." - it: should reject Loop Runtime without a cloud - values: - - __fixtures__/base-values.yaml + values: *base-values set: loopRuntime.enabled: true - asserts: - - failedTemplate: - errorMessage: "loopRuntime.enabled requires cloud: aws or cloud: google." + asserts: *assertions-1 - it: should advertise the egress gateway to Sandbox Service workers - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: loopRuntime.sandbox.egressGateway.enabled: true loopRuntime.sandbox.egressGateway.port: 4100 @@ -340,10 +294,7 @@ tests: - it: should use the in-cluster egress Service when no external URL is set release: namespace: braintrust - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: loopRuntime.sandbox.egressGateway.enabled: true loopRuntime.service.name: loop @@ -359,13 +310,11 @@ tests: - it: should default the egress URL when reused values omit it release: namespace: braintrust - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: loopRuntime.sandbox.egressGateway.enabled: true loopRuntime.sandbox.egressGateway.url: null - asserts: + asserts: &assertions-2 - equal: path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL value: http://braintrust-loop-runtime-egress.braintrust:4002 @@ -373,66 +322,44 @@ tests: - it: should treat a zero egress URL as empty release: namespace: braintrust - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: loopRuntime.sandbox.egressGateway.enabled: true loopRuntime.sandbox.egressGateway.url: 0 - asserts: - - equal: - path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL - value: http://braintrust-loop-runtime-egress.braintrust:4002 - + asserts: *assertions-2 - it: should reject empty endpoint - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: loopRuntime.sandbox.dnsUrl: "" loopRuntime.sandbox.urls: [""] - asserts: + asserts: &assertions-3 - failedTemplate: errorMessage: "loopRuntime.sandbox.urls entries must not be blank" - it: should reject whitespace endpoint - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: loopRuntime.sandbox.dnsUrl: "" loopRuntime.sandbox.urls: [" "] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.urls entries must not be blank" + asserts: *assertions-3 - it: should reject null endpoint - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: loopRuntime.sandbox.dnsUrl: "" loopRuntime.sandbox.urls: [null] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.urls entries must not be blank" + asserts: *assertions-3 - it: should reject blank entry alongside a usable endpoint - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: loopRuntime.sandbox.dnsUrl: "" loopRuntime.sandbox.urls: ["https://worker.example:9400", ""] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.urls entries must not be blank" + asserts: *assertions-3 - it: should reject padded endpoint - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: loopRuntime.sandbox.dnsUrl: "" loopRuntime.sandbox.urls: [" https://worker.example:9400 "] diff --git a/braintrust/tests/loop-runtime-deployment_test.yaml b/braintrust/tests/loop-runtime-deployment_test.yaml index f848ab1..91be698 100644 --- a/braintrust/tests/loop-runtime-deployment_test.yaml +++ b/braintrust/tests/loop-runtime-deployment_test.yaml @@ -6,13 +6,13 @@ tests: - it: should not render when Loop Runtime is disabled values: - __fixtures__/base-values.yaml - asserts: + asserts: &absent - hasDocuments: count: 0 - it: should render a hardened runtime pod with data-plane secrets template: loop-runtime-deployment.yaml - values: + values: &aws-values - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml asserts: @@ -21,6 +21,9 @@ tests: - equal: path: metadata.name value: braintrust-loop-runtime + - equal: + path: spec.replicas + value: 1 - equal: path: spec.template.spec.serviceAccountName value: braintrust-loop-runtime @@ -70,9 +73,7 @@ tests: - it: should keep probes on a custom named service port template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: loopRuntime.service.port: 5000 loopRuntime.service.portName: metrics @@ -92,9 +93,7 @@ tests: - it: should not use GKE workload identity on AWS template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values asserts: - isNull: path: spec.template.metadata.labels["gke-workload-identity/use"] @@ -106,7 +105,7 @@ tests: - it: should use GKE workload identity and mount Sandbox Service client TLS on Google template: loop-runtime-deployment.yaml - values: + values: &gke-values - __fixtures__/base-values.yaml - __fixtures__/google-values.yaml - __fixtures__/loop-runtime-google-values.yaml @@ -144,10 +143,7 @@ tests: - it: should map custom TLS secret keys to the paths Loop Runtime reads template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: loopRuntime.sandbox.tls.secretName: cert-manager-client loopRuntime.sandbox.tls.certKey: tls.crt @@ -170,10 +166,7 @@ tests: - it: should omit the TLS volume for a plain HTTP Sandbox Service endpoint template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: loopRuntime.sandbox.dnsUrl: http://sandbox-service.sandbox:9400 loopRuntime.sandbox.tls.secretName: "" @@ -187,9 +180,7 @@ tests: - it: should mount Sandbox Service client TLS on AWS template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values asserts: - contains: path: spec.template.spec.containers[0].volumeMounts @@ -203,10 +194,7 @@ tests: - it: should expose the egress gateway container port when enabled template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: loopRuntime.sandbox.egressGateway.enabled: true loopRuntime.sandbox.egressGateway.url: http://10.10.0.20:4002 @@ -219,9 +207,7 @@ tests: - it: should allow the main port to be named egress when the gateway is disabled template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: loopRuntime.service.portName: egress asserts: @@ -240,9 +226,7 @@ tests: - it: should keep port names unique with the gateway enabled and a custom port template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: loopRuntime.service.portName: egress loopRuntime.sandbox.egressGateway.enabled: true @@ -270,6 +254,41 @@ tests: - __fixtures__/base-values.yaml - __fixtures__/google-values.yaml - __fixtures__/pre-loop-runtime-values.yaml + asserts: *absent + + - it: should scale to zero when explicitly requested + template: loop-runtime-deployment.yaml + values: *aws-values + set: + loopRuntime.replicas: 0 asserts: - - hasDocuments: - count: 0 + - equal: + path: spec.replicas + value: 0 + + - it: should use an explicit positive replica count + template: loop-runtime-deployment.yaml + values: *aws-values + set: + loopRuntime.replicas: 2 + asserts: + - equal: + path: spec.replicas + value: 2 + + - it: should default null or missing replicas to one + template: loop-runtime-deployment.yaml + values: *aws-values + set: + loopRuntime.replicas: null + asserts: &default-replicas + - equal: + path: spec.replicas + value: 1 + + - it: should default empty replicas to one + template: loop-runtime-deployment.yaml + values: *aws-values + set: + loopRuntime.replicas: "" + asserts: *default-replicas diff --git a/braintrust/tests/loop-runtime-egress-service_test.yaml b/braintrust/tests/loop-runtime-egress-service_test.yaml index 11a5e1d..69146e5 100644 --- a/braintrust/tests/loop-runtime-egress-service_test.yaml +++ b/braintrust/tests/loop-runtime-egress-service_test.yaml @@ -5,30 +5,26 @@ tests: - it: should not render when Loop Runtime is disabled values: - __fixtures__/base-values.yaml - asserts: + asserts: &absent - hasDocuments: count: 0 - it: should not render when the egress gateway is disabled - values: + values: &gke-values - __fixtures__/base-values.yaml - __fixtures__/google-values.yaml - __fixtures__/loop-runtime-google-values.yaml - asserts: - - hasDocuments: - count: 0 + asserts: *absent - it: should not render when reused values have no loopRuntime map values: - __fixtures__/base-values.yaml - __fixtures__/google-values.yaml - __fixtures__/pre-loop-runtime-values.yaml - asserts: - - hasDocuments: - count: 0 + asserts: *absent - it: should expose the egress gateway through a ClusterIP Service by default - values: + values: &aws-values - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml set: @@ -61,9 +57,7 @@ tests: path: metadata.annotations - it: should target the numeric gateway port when the main port is named egress - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: loopRuntime.service.portName: egress loopRuntime.sandbox.egressGateway.enabled: true @@ -79,10 +73,7 @@ tests: targetPort: 5002 - it: should expose the egress gateway through a GKE internal load balancer - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: loopRuntime.service.name: loop loopRuntime.labels.loop-specific: loop-egress diff --git a/braintrust/tests/loop-runtime-images_test.yaml b/braintrust/tests/loop-runtime-images_test.yaml index 1db029e..96e7d91 100644 --- a/braintrust/tests/loop-runtime-images_test.yaml +++ b/braintrust/tests/loop-runtime-images_test.yaml @@ -5,7 +5,7 @@ templates: tests: - it: should render image tag v2.18.0 template: loop-runtime-deployment.yaml - values: + values: &runtime-values - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml set: @@ -20,129 +20,87 @@ tests: - it: should render image tag 2.18.0 template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *runtime-values set: loopRuntime.image.tag: "2.18.0" asserts: - equal: path: spec.template.spec.containers[0].image value: "test/loop-runtime:2.18.0" - - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: Always - it: should render image tag v2.18.0-rc.1 template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *runtime-values set: loopRuntime.image.tag: "v2.18.0-rc.1" asserts: - equal: path: spec.template.spec.containers[0].image value: "test/loop-runtime:v2.18.0-rc.1" - - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: Always - it: should render image tag 2.18.0-rc.1 template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *runtime-values set: loopRuntime.image.tag: "2.18.0-rc.1" asserts: - equal: path: spec.template.spec.containers[0].image value: "test/loop-runtime:2.18.0-rc.1" - - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: Always - it: should render image tag a1b2c3d template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *runtime-values set: loopRuntime.image.tag: "a1b2c3d" asserts: - equal: path: spec.template.spec.containers[0].image value: "test/loop-runtime:a1b2c3d" - - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: Always - it: should render image tag 1234567 template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *runtime-values set: loopRuntime.image.tag: "1234567" asserts: - equal: path: spec.template.spec.containers[0].image value: "test/loop-runtime:1234567" - - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: Always - it: should render image tag 53af3580126bc915b67166a41d5c78b3985d1e94 template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *runtime-values set: loopRuntime.image.tag: "53af3580126bc915b67166a41d5c78b3985d1e94" asserts: - equal: path: spec.template.spec.containers[0].image value: "test/loop-runtime:53af3580126bc915b67166a41d5c78b3985d1e94" - - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: Always - it: should render image tag custom-build template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *runtime-values set: loopRuntime.image.tag: "custom-build" asserts: - equal: path: spec.template.spec.containers[0].image value: "test/loop-runtime:custom-build" - - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: Always - it: should render image tag latest-2.x template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *runtime-values set: loopRuntime.image.tag: "latest-2.x" asserts: - equal: path: spec.template.spec.containers[0].image value: "test/loop-runtime:latest-2.x" - - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: Always - it: should retain an explicit pull policy for a pinned image template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *runtime-values set: loopRuntime.image.pullPolicy: IfNotPresent asserts: diff --git a/braintrust/tests/loop-runtime-serviceaccount_test.yaml b/braintrust/tests/loop-runtime-serviceaccount_test.yaml index 5e472bb..0e13e11 100644 --- a/braintrust/tests/loop-runtime-serviceaccount_test.yaml +++ b/braintrust/tests/loop-runtime-serviceaccount_test.yaml @@ -10,7 +10,7 @@ tests: count: 0 - it: should attach the configured AWS role for IRSA - values: + values: &aws-values - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml asserts: @@ -24,19 +24,15 @@ tests: value: arn:aws:iam::123456789012:role/test-loop-runtime-role - it: should support Pod Identity without adding an IRSA annotation - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: loopRuntime.serviceAccount.awsRoleArn: "" - asserts: + asserts: &assertions-1 - isNull: path: metadata.annotations["eks.amazonaws.com/role-arn"] - it: should include ServiceAccount annotations from the nested configuration - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *aws-values set: loopRuntime.serviceAccount.annotations.custom-annotation: value asserts: @@ -45,7 +41,7 @@ tests: value: value - it: should bind the Loop Runtime Google service account on GKE - values: + values: &gke-values - __fixtures__/base-values.yaml - __fixtures__/google-values.yaml - __fixtures__/loop-runtime-google-values.yaml @@ -60,12 +56,7 @@ tests: path: metadata.annotations["eks.amazonaws.com/role-arn"] - it: should not add the IRSA annotation on GKE - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + values: *gke-values set: loopRuntime.serviceAccount.awsRoleArn: arn:aws:iam::123456789012:role/ignored - asserts: - - isNull: - path: metadata.annotations["eks.amazonaws.com/role-arn"] + asserts: *assertions-1 diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 4b06515..89ece2e 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -321,6 +321,7 @@ loopRuntime: service: {} pod: {} serviceaccount: {} + # Null, missing, or empty replicas default to 1; 0 scales to zero. replicas: 1 image: repository: public.ecr.aws/braintrust/loop-runtime diff --git a/scripts/test-eks-loop-runtime.sh b/scripts/test-eks-loop-runtime.sh deleted file mode 100644 index beb55b2..0000000 --- a/scripts/test-eks-loop-runtime.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -TEST_DIR="$(mktemp -d)" -trap 'rm -rf "$TEST_DIR"' EXIT - -# Content-addressed worker chart distributed with Braintrust v2.18.0. -CHART_SHA256=cd27a3983e304932a7e1617adf3105c2b7320a340f26bfea577df781827142f9 -CHART_URL="https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/$CHART_SHA256/sandbox-service-0.1.0.tgz" -CHART_ARCHIVE="${SANDBOX_SERVICE_CHART:-$TEST_DIR/sandbox-service.tgz}" -if [[ -z "${SANDBOX_SERVICE_CHART:-}" ]]; then - curl --fail --location --silent --show-error --max-time 60 "$CHART_URL" --output "$CHART_ARCHIVE" -fi -python3 - "$CHART_ARCHIVE" "$CHART_SHA256" <<'PY' -import hashlib -import pathlib -import sys - -if hashlib.sha256(pathlib.Path(sys.argv[1]).read_bytes()).hexdigest() != sys.argv[2]: - raise SystemExit("Sandbox Service chart checksum mismatch") -PY -tar -xzf "$CHART_ARCHIVE" -C "$TEST_DIR" - -helm unittest \ - --values "$ROOT_DIR/braintrust/examples/eks-loop-runtime/sandbox-service.yaml" \ - --file "$ROOT_DIR/tests/sandbox-service_test.yaml" \ - "$TEST_DIR/sandbox-service" -helm lint "$TEST_DIR/sandbox-service" --strict \ - --values "$ROOT_DIR/braintrust/examples/eks-loop-runtime/sandbox-service.yaml" diff --git a/test.sh b/test.sh index d86d141..75159a6 100755 --- a/test.sh +++ b/test.sh @@ -94,10 +94,6 @@ fi echo "" echo "✅ Chart rendering tests passed" -echo "" -echo "Testing the EKS integration with the released Sandbox Service chart..." -bash scripts/test-eks-loop-runtime.sh - echo "" echo "==========================================" echo "3. Running Helm Lint" diff --git a/tests/sandbox-service_test.yaml b/tests/sandbox-service_test.yaml deleted file mode 100644 index 6ee8b7a..0000000 --- a/tests/sandbox-service_test.yaml +++ /dev/null @@ -1,165 +0,0 @@ -suite: released Sandbox Service chart with EKS Loop Runtime values -templates: - - configmap.yaml - - service.yaml - - worker-statefulset.yaml - - worker-controller.yaml - - worker-disruption-budget.yaml - - worker-admission-policy.yaml - - check-drained.yaml -tests: - - it: should advertise the same TLS hostname that the runtime discovers - release: - namespace: braintrust-sandbox - revision: 1 - template: configmap.yaml - asserts: - - matchRegex: - path: data["config.json"] - pattern: '"endpoint": "https://sandbox-service.braintrust-sandbox.svc.cluster.local:9400"' - - - it: should expose individual worker addresses through a headless Service - release: - namespace: braintrust-sandbox - revision: 1 - template: service.yaml - asserts: - - equal: - path: metadata.namespace - value: braintrust-sandbox - - equal: - path: spec.clusterIP - value: None - - equal: - path: spec.ports[0].port - value: 9400 - - - it: should keep Firecracker workers isolated on dedicated nodes and preserve host state - release: - namespace: braintrust-sandbox - revision: 1 - template: worker-statefulset.yaml - asserts: - - equal: - path: spec.updateStrategy.type - value: OnDelete - - isNull: - path: spec.replicas - - equal: - path: spec.template.spec.nodeSelector["braintrust.io/workload"] - value: sandbox-service - - equal: - path: spec.template.spec.hostNetwork - value: true - - equal: - path: spec.template.spec.hostPID - value: true - - equal: - path: spec.template.spec.automountServiceAccountToken - value: false - - equal: - path: spec.template.spec.containers[0].securityContext.privileged - value: true - - equal: - path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].topologyKey - value: kubernetes.io/hostname - - contains: - path: spec.template.spec.volumes - content: - name: state - hostPath: - path: /var/lib/braintrust-sandboxes/1 - type: Directory - - contains: - path: spec.template.spec.volumes - content: - name: tls - secret: - secretName: sandbox-tls - defaultMode: 256 - - - it: should reconcile fixed worker capacity without depending on an ECS service - release: - namespace: braintrust-sandbox - revision: 1 - template: worker-controller.yaml - documentIndex: 6 - asserts: - - equal: - path: spec.template.spec.containers[0].command - value: [sandbox-service, reconcile-workers] - - equal: - path: spec.template.spec.containers[0].args - value: - - --endpoint=https://sandbox-service.braintrust-sandbox.svc.cluster.local:9400 - - --replicas=1 - - --configuration-revision=1 - - contains: - path: spec.template.spec.volumes - content: - name: tls - secret: - secretName: sandbox-client-tls - defaultMode: 288 - - - it: should request draining rather than directly deleting workers when capacity is zero - release: - namespace: braintrust-sandbox - revision: 1 - template: worker-controller.yaml - documentIndex: 6 - set: - replicas: 0 - asserts: - - contains: - path: spec.template.spec.containers[0].args - content: --replicas=0 - - - it: should reconcile a new Helm revision through the worker controller - template: worker-controller.yaml - documentIndex: 6 - release: - revision: 2 - namespace: braintrust-sandbox - asserts: - - contains: - path: spec.template.spec.containers[0].args - content: --configuration-revision=2 - - - it: should block ordinary eviction of workers holding local sandbox state - release: - namespace: braintrust-sandbox - revision: 1 - template: worker-disruption-budget.yaml - asserts: - - equal: - path: spec.maxUnavailable - value: 0 - - - it: should protect worker deletion in the sandbox namespace - release: - namespace: braintrust-sandbox - revision: 1 - template: worker-admission-policy.yaml - documentIndex: 0 - asserts: - - matchRegex: - path: spec.matchConditions[0].expression - pattern: "request.namespace == 'braintrust-sandbox'" - - equal: - path: spec.failurePolicy - value: Fail - - - it: should refuse uninstall until the worker release has drained - release: - namespace: braintrust-sandbox - revision: 1 - template: check-drained.yaml - documentIndex: 3 - asserts: - - equal: - path: metadata.annotations["helm.sh/hook"] - value: pre-delete - - equal: - path: spec.template.spec.containers[0].command - value: [sandbox-service, check-workers-drained] From 04312d15c9eae2708e6cd31586ccceb5506693bb Mon Sep 17 00:00:00 2001 From: Alex R Date: Mon, 26 Oct 2026 04:51:32 +0000 Subject: [PATCH 08/25] Make Loop Runtime GKE-only and wire it to isolated Sandbox Service workers Loop Runtime on AWS stays on ECS through the Terraform module, so the chart now rejects every cloud except google. Workers run in the isolated GKE cluster from the GCP Terraform module; the runtime reaches them through the discovery endpoint and exposes its Git forward proxy on an internal load balancer bound to an address Terraform reserves. - Drop AWS (IRSA, s3:// URIs), sandbox.urls, and the EKS example - Always mount the client TLS Secret and run the egress gateway - Match the ECS Sandbox Service environment and pinned images - Default runtime pods to 1 vCPU / 2 GiB with two replicas - Add a GKE example with a TLS Secret script and worker chart values Co-Authored-By: Claude Opus 5.5 --- braintrust/README.md | 60 +--- .../examples/eks-loop-runtime/README.md | 73 ---- .../eks-loop-runtime/sandbox-service.yaml | 5 - .../examples/eks-loop-runtime/values.yaml | 17 - .../examples/gke-loop-runtime/README.md | 66 ++++ .../gke-loop-runtime/create-sandbox-tls.sh | 70 ++++ .../gke-loop-runtime/sandbox-service.yaml | 25 ++ .../examples/gke-loop-runtime/values.yaml | 14 + braintrust/templates/_helpers.tpl | 55 +-- .../templates/loop-runtime-configmap.yaml | 26 +- .../templates/loop-runtime-deployment.yaml | 24 +- .../loop-runtime-egress-service.yaml | 17 +- .../loop-runtime-serviceaccount.yaml | 5 - .../loop-runtime-google-values.yaml | 13 - .../__fixtures__/loop-runtime-values.yaml | 24 +- braintrust/tests/api-configmap_test.yaml | 12 - braintrust/tests/eks-loop-runtime_test.yaml | 58 ---- .../tests/labels-merge-services_test.yaml | 4 - .../tests/loop-runtime-configmap_test.yaml | 327 +++++------------- .../tests/loop-runtime-deployment_test.yaml | 300 +++++++--------- .../loop-runtime-egress-service_test.yaml | 96 ++--- .../tests/loop-runtime-images_test.yaml | 109 ------ .../tests/loop-runtime-service_test.yaml | 10 - .../loop-runtime-serviceaccount_test.yaml | 49 +-- braintrust/tests/virtualservice_test.yaml | 12 - braintrust/values.yaml | 56 ++- 26 files changed, 496 insertions(+), 1031 deletions(-) delete mode 100644 braintrust/examples/eks-loop-runtime/README.md delete mode 100644 braintrust/examples/eks-loop-runtime/sandbox-service.yaml delete mode 100644 braintrust/examples/eks-loop-runtime/values.yaml create mode 100644 braintrust/examples/gke-loop-runtime/README.md create mode 100755 braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh create mode 100644 braintrust/examples/gke-loop-runtime/sandbox-service.yaml create mode 100644 braintrust/examples/gke-loop-runtime/values.yaml delete mode 100644 braintrust/tests/__fixtures__/loop-runtime-google-values.yaml delete mode 100644 braintrust/tests/eks-loop-runtime_test.yaml delete mode 100644 braintrust/tests/loop-runtime-images_test.yaml diff --git a/braintrust/README.md b/braintrust/README.md index 3bbaf94..86d2239 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -20,60 +20,18 @@ The `braintrust-secrets` secret must contain the following keys: ## Optional: Loop Runtime -Set `loopRuntime.enabled: true` to run Loop Runtime on EKS (`cloud: aws`) or -GKE (`cloud: google`). Loop Runtime runs sandboxes on Braintrust Sandbox -Service workers over mTLS. For a complete installation on EKS, follow the -[Loop Runtime and Sandbox Service example](examples/eks-loop-runtime/README.md). -It installs the shared worker chart as a separate Helm release and reuses the -existing AWS infrastructure and workload identities. +Set `loopRuntime.enabled: true` to run Loop Runtime on GKE (`cloud: google`). +On AWS, Loop Runtime runs on ECS through the AWS Terraform module, so this +chart rejects other clouds. Loop Runtime runs sandboxes on Sandbox Service +workers in a separate isolated GKE cluster. Follow the +[Loop Runtime and Sandbox Service example](examples/gke-loop-runtime/README.md) +to create the TLS Secrets and install the worker release. The runtime reads `braintrust-secrets` and the Brainstore buckets. The chart adds `LOOP_RUNTIME_URL` to the API and, when `virtualService.enabled` is true, -routes `/loop/runtime` to Loop Runtime. - -Put the Sandbox Service client certificate, key, and server CA in a Secret and -set `loopRuntime.sandbox.tls.secretName`. `sandbox.vcpuCount` and -`sandbox.memoryMib` specify the resources requested for each sandbox. - -GKE: - -```yaml -cloud: google - -objectStorage: - google: - brainstoreBucket: "" - apiBucket: "" - -loopRuntime: - enabled: true - serviceAccount: - googleServiceAccount: "" - sandbox: - dnsUrl: "" - tls: - secretName: sandbox-client-tls -``` - -Sandboxes reach Git through a forward proxy in Loop Runtime. Enable -`loopRuntime.sandbox.egressGateway` to create the `-egress` Service. -Workers in the same cluster use its internal URL by default. For workers outside -the cluster, expose it on an internal load balancer and set a reachable `url`. -For example, on GKE: - -```yaml -loopRuntime: - sandbox: - egressGateway: - enabled: true - url: "http://:4002" - service: - type: LoadBalancer - annotations: - networking.gke.io/load-balancer-type: "Internal" - loadBalancerSourceRanges: - - "" -``` +routes `/loop/runtime` to Loop Runtime. It also creates the `-egress` +internal load balancer on the address that Terraform reserves, so sandboxes +can reach Git through Loop Runtime's forward proxy. ## Azure Key Vault Driver Integration diff --git a/braintrust/examples/eks-loop-runtime/README.md b/braintrust/examples/eks-loop-runtime/README.md deleted file mode 100644 index 2a3c85b..0000000 --- a/braintrust/examples/eks-loop-runtime/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# Loop Runtime and Sandbox Service on EKS - -Install two Helm releases: this chart for Loop Runtime and the published -Sandbox Service chart for workers and their drain controller. Reuse the -existing AWS networking, database, Redis, buckets, and IAM configuration. - -## Prerequisites - -- Kubernetes 1.30+; one worker release per cluster because its cluster-scoped - RBAC and admission policy names are fixed. -- System nodes labeled `braintrust.io/workload=sandbox-system`. -- Dedicated Linux worker nodes labeled `braintrust.io/workload=sandbox-service` - and tainted `braintrust.io/sandbox=true:NoSchedule`, with `/dev/kvm`, - cgroup v2, mount propagation, and node-local storage prepared at - `/var/lib/braintrust-sandboxes/1`. Workers require privileged containers - and host PID/network namespaces; Fargate cannot run them. -- Node/VPC isolation for workers; pod NetworkPolicy alone does not isolate - host-networked pods. Keep application credentials off worker nodes. -- Runtime/controller connectivity to worker node addresses on TCP 9400, - worker connectivity to the runtime proxy Service on TCP 4002, and image-pull/DNS access. -- Your existing chart values and `braintrust-secrets`, plus Loop Runtime's - workload identity. Set `loopRuntime.serviceAccount.awsRoleArn` for IRSA or - associate its ServiceAccount with EKS Pod Identity. Supply `AWS_REGION` - through `loopRuntime.extraEnvVars` if identity integration does not inject it. - -Create namespaces `braintrust` and `braintrust-sandbox`, then these Secrets -from your PKI. The server certificate must cover -`sandbox-service.braintrust-sandbox.svc.cluster.local` and have server -authentication usage; client certificates need client authentication usage. - -| Namespace / Secret | Keys | -| --- | --- | -| `braintrust-sandbox/sandbox-tls` | `cert.pem`, `key.pem`, `ca_cert.pem` (client CA) | -| `braintrust-sandbox/sandbox-client-tls` | `client.pem`, `client.key`, `server-ca.pem` | -| `braintrust/sandbox-client-tls` | `client.pem`, `client.key`, `server-ca.pem` | - -Keep the server private key in the worker namespace. The server trusts the -client CA; the runtime and controller trust the server CA. - -## Install - -From the repository root, using the worker chart distributed with v2.18.0: - -```sh -SANDBOX_CHART_URL=https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/cd27a3983e304932a7e1617adf3105c2b7320a340f26bfea577df781827142f9/sandbox-service-0.1.0.tgz - -helm upgrade --install sandbox-service "$SANDBOX_CHART_URL" \ - --namespace braintrust-sandbox \ - --values braintrust/examples/eks-loop-runtime/sandbox-service.yaml --wait - -helm upgrade --install braintrust ./braintrust \ - --namespace braintrust \ - --values infrastructure-values.yaml \ - --values braintrust/examples/eks-loop-runtime/values.yaml --wait -``` - -`infrastructure-values.yaml` supplies your existing bucket, identity, and -storage settings. Cluster DNS discovers workers; an internal Service exposes -the Git proxy. External workers can instead use an explicit discovery/proxy URL. -The worker example uses fixed capacity (`replicas: 1`); leave its ECS-specific -`loopRuntime.cluster/service` unset. Use the cluster's existing node -provisioner and allow an extra worker during rollouts. Pin chart/image versions together. - -## Upgrade and shutdown - -Upgrade with the same values and namespaces. The worker controller owns -scale-down and replacement; do not force-delete workers or remove their nodes. -Before shutdown, stop new work and finish/terminate sessions while the runtime -can release their sandboxes. Upgrade the worker release with `replicas: 0` -and inspect `kubectl -n braintrust-sandbox get configmap sandbox-rollout -o yaml`. -After draining, disable `loopRuntime.enabled` and uninstall the worker release. -Its pre-delete hook refuses uninstall until drained; retain nodes and TLS -Secrets until it succeeds. diff --git a/braintrust/examples/eks-loop-runtime/sandbox-service.yaml b/braintrust/examples/eks-loop-runtime/sandbox-service.yaml deleted file mode 100644 index f251e2e..0000000 --- a/braintrust/examples/eks-loop-runtime/sandbox-service.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# Values for the separately released Sandbox Service chart. -hostname: sandbox-service.braintrust-sandbox.svc.cluster.local -image: public.ecr.aws/braintrust/sandbox-service:v2.18.0 -replicas: 1 -gitResources: true diff --git a/braintrust/examples/eks-loop-runtime/values.yaml b/braintrust/examples/eks-loop-runtime/values.yaml deleted file mode 100644 index 13c774d..0000000 --- a/braintrust/examples/eks-loop-runtime/values.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Layer on top of the existing AWS infrastructure values for this chart. -cloud: aws - -loopRuntime: - enabled: true - image: - tag: v2.18.0 - nodeSelector: - braintrust.io/workload: sandbox-system - sandbox: - dnsUrl: https://sandbox-service.braintrust-sandbox.svc.cluster.local:9400 - tls: - secretName: sandbox-client-tls - egressGateway: - enabled: true - # Uses the in-cluster egress Service; no additional load balancer is needed. - url: "" diff --git a/braintrust/examples/gke-loop-runtime/README.md b/braintrust/examples/gke-loop-runtime/README.md new file mode 100644 index 0000000..e4a1560 --- /dev/null +++ b/braintrust/examples/gke-loop-runtime/README.md @@ -0,0 +1,66 @@ +# Loop Runtime and Sandbox Service on GKE + +Loop Runtime runs in the primary GKE cluster with this chart. Its sandboxes run +on Sandbox Service workers in a separate isolated GKE cluster, installed from +the published worker chart. The GCP Terraform module creates both clusters, the +worker node pool, private DNS, firewall rules, and the Loop Runtime identity +when `gke_isolated_workers` is set with `discovery`. + +## Connections + +| From | To | Port | Address | +| --- | --- | --- | --- | +| Loop Runtime, worker controller | Workers | TCP 9400, mTLS | `discovery.endpoint` | +| Sandboxes (Git) | Loop Runtime forward proxy | TCP 4002 | `loop_egress.url`, an internal load balancer on a reserved address | + +## Install + +All `<...>` values come from `terraform output gke_isolated_workers` and +`terraform output loop_runtime_service_account`. + +1. Create the mutual TLS Secrets in both clusters. The script refuses to + replace existing Secrets. + + ```sh + braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh \ + "" + ``` + +2. Install the workers in the isolated cluster, using the worker chart that + matches the pinned images. + + ```sh + helm --kube-context upgrade --install sandbox-service \ + https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/cd27a3983e304932a7e1617adf3105c2b7320a340f26bfea577df781827142f9/sandbox-service-0.1.0.tgz \ + --namespace braintrust-sandbox \ + --values braintrust/examples/gke-loop-runtime/sandbox-service.yaml --wait + ``` + +3. Install or upgrade this chart in the primary cluster with your existing GKE + values plus `values.yaml` from this directory. + + ```sh + helm --kube-context upgrade --install braintrust ./braintrust \ + --namespace braintrust \ + --values \ + --values braintrust/examples/gke-loop-runtime/values.yaml --wait + ``` + +## Capacity + +Workers run at a fixed count (`replicas: 1`). One 8-vCPU worker node holds about +seven 1 vCPU / 2 GiB sandboxes. When every slot is taken, new sandboxes fail +with `CapacityExhausted`. Raise `replicas` in `sandbox-service.yaml` to add +workers. The worker node pool autoscales up to its Terraform maximum and keeps +room for one extra worker during rollouts. + +## Upgrade and shutdown + +Upgrade both releases with the same values, and keep the pinned image tags in +`values.yaml` (chart defaults) and `sandbox-service.yaml` together. The worker +controller owns scale-down and replacement; do not force-delete workers or +their nodes. To shut down, stop new Loop work, upgrade the worker release with +`replicas: 0`, wait for +`kubectl -n braintrust-sandbox get configmap sandbox-rollout -o yaml` to show +zero workers, then disable `loopRuntime.enabled` and uninstall the worker +release. Its pre-delete hook refuses to uninstall until workers drain. diff --git a/braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh b/braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh new file mode 100755 index 0000000..f8fb22c --- /dev/null +++ b/braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Creates the mutual TLS Secrets shared by Loop Runtime and Sandbox Service. +# Run once before installing either release. Private keys stay in Kubernetes +# Secrets and never enter Terraform state. The certificates match those the +# AWS Terraform module creates: separate server and client CAs, P-256 keys, +# ten-year validity. +# +# Usage: create-sandbox-tls.sh +# worker hostname: Terraform output gke_isolated_workers.discovery.hostname +set -euo pipefail + +if [ "$#" -ne 3 ]; then + echo "usage: $0 " >&2 + exit 2 +fi +HOSTNAME_SAN="$1" +WORKER_CONTEXT="$2" +RUNTIME_CONTEXT="$3" +WORKER_NAMESPACE="${WORKER_NAMESPACE:-braintrust-sandbox}" +RUNTIME_NAMESPACE="${RUNTIME_NAMESPACE:-braintrust}" +DAYS=3650 + +if kubectl --context "$WORKER_CONTEXT" -n "$WORKER_NAMESPACE" get secret sandbox-tls >/dev/null 2>&1; then + echo "sandbox-tls already exists in $WORKER_CONTEXT/$WORKER_NAMESPACE; refusing to replace it." >&2 + exit 1 +fi + +dir="$(mktemp -d)" +trap 'rm -rf "$dir"' EXIT +cd "$dir" + +for purpose in server client; do + openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$purpose-ca.key" 2>/dev/null + openssl req -x509 -new -sha256 -key "$purpose-ca.key" -days "$DAYS" \ + -subj "/CN=sandbox-$purpose-ca" \ + -addext "basicConstraints=critical,CA:TRUE,pathlen:0" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" \ + -out "$purpose-ca.pem" + openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$purpose.key" 2>/dev/null + { + echo "basicConstraints=critical,CA:FALSE" + echo "keyUsage=critical,digitalSignature" + echo "authorityKeyIdentifier=keyid" + if [ "$purpose" = server ]; then + echo "extendedKeyUsage=serverAuth" + echo "subjectAltName=DNS:$HOSTNAME_SAN" + else + echo "extendedKeyUsage=clientAuth" + fi + } >"$purpose.ext" + openssl req -new -key "$purpose.key" -subj "/CN=sandbox-$purpose" | + openssl x509 -req -sha256 -days "$DAYS" -CA "$purpose-ca.pem" -CAkey "$purpose-ca.key" \ + -set_serial "0x$(openssl rand -hex 16)" -extfile "$purpose.ext" -out "$purpose.pem" 2>/dev/null +done + +apply_secret() { + kubectl --context "$1" create namespace "$2" --dry-run=client -o yaml | kubectl --context "$1" apply -f - >/dev/null + kubectl --context "$1" -n "$2" create secret generic "$3" "${@:4}" --dry-run=client -o yaml | + kubectl --context "$1" apply -f - +} + +# Workers present the server certificate and trust the client CA. +apply_secret "$WORKER_CONTEXT" "$WORKER_NAMESPACE" sandbox-tls \ + --from-file=cert.pem=server.pem --from-file=key.pem=server.key --from-file=ca_cert.pem=client-ca.pem +# The worker controller and Loop Runtime present the client certificate and trust the server CA. +for target in "$WORKER_CONTEXT $WORKER_NAMESPACE" "$RUNTIME_CONTEXT $RUNTIME_NAMESPACE"; do + # shellcheck disable=SC2086 + apply_secret $target sandbox-client-tls \ + --from-file=client.pem=client.pem --from-file=client.key=client.key --from-file=server-ca.pem=server-ca.pem +done diff --git a/braintrust/examples/gke-loop-runtime/sandbox-service.yaml b/braintrust/examples/gke-loop-runtime/sandbox-service.yaml new file mode 100644 index 0000000..bf8fd05 --- /dev/null +++ b/braintrust/examples/gke-loop-runtime/sandbox-service.yaml @@ -0,0 +1,25 @@ +# Values for the Sandbox Service worker chart, installed in the isolated worker cluster. +# Every <...> value comes from the GCP Terraform output gke_isolated_workers. +hostname: "" +image: public.ecr.aws/braintrust/sandbox-service:b96d95eaa269031cdd6f286a17b9b3e4795355d1 +# Fixed worker capacity. The worker node pool autoscales, so raising this is a values change. +replicas: 1 +gitResources: true +prewarmTemplates: + - image: public.ecr.aws/braintrust/codex-runtime@sha256:35db9c98d4a74e20758a29b82f8a3a6552be0002eb439c8c1a0f434ac573ee21 + resources: + vcpu_count: 1 + memory_mib: 2048 + mounts: [] + durable_file_systems: [] + policy: + networking: + type: disabled + credentials: [] + default_workdir: /home/exo/workspace +externalDns: + enabled: true + provider: google + # One entry per argument in discovery.external_dns_extra_args. + extraArgs: + - "" diff --git a/braintrust/examples/gke-loop-runtime/values.yaml b/braintrust/examples/gke-loop-runtime/values.yaml new file mode 100644 index 0000000..1c1aeec --- /dev/null +++ b/braintrust/examples/gke-loop-runtime/values.yaml @@ -0,0 +1,14 @@ +# Layer on top of your existing GKE values (for example, ../google-autopilot/values.yaml). +# Every value below comes from the GCP Terraform module outputs. +loopRuntime: + enabled: true + serviceAccount: + googleServiceAccount: "" + sandbox: + dnsUrl: "" + egressGateway: + address: "" + # One entry per range in gke_isolated_workers.loop_egress.source_ranges. + sourceRanges: + - "" + url: "" diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 771e38f..6c3ce84 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -154,23 +154,6 @@ predates Loop Runtime have no loopRuntime map, which means disabled. {{- if (.Values.loopRuntime | default dict).enabled -}}true{{- end -}} {{- end -}} -{{/* Loop Runtime object-store URIs. They match the Brainstore ConfigMaps. */}} -{{- define "braintrust.loopRuntime.brainstoreBucketUri" -}} -{{- if eq .Values.cloud "google" -}} -gs://{{ .Values.objectStorage.google.brainstoreBucket }} -{{- else -}} -s3://{{ .Values.objectStorage.aws.brainstoreBucket }} -{{- end -}} -{{- end -}} - -{{- define "braintrust.loopRuntime.codeBundleUri" -}} -{{- if eq .Values.cloud "google" -}} -gs://{{ .Values.objectStorage.google.apiBucket }} -{{- else -}} -s3://{{ .Values.objectStorage.aws.codeBundleBucket }} -{{- end -}} -{{- end -}} - {{/* Validate the Loop Runtime configuration. */}} {{- define "braintrust.loopRuntime.validate" -}} {{- if include "braintrust.loopRuntime.enabled" . -}} @@ -179,39 +162,21 @@ s3://{{ .Values.objectStorage.aws.codeBundleBucket }} {{- fail (printf "loopRuntime.%s is missing. Values reused from a chart release that predates Loop Runtime omit the loopRuntime defaults; upgrade with --reset-then-reuse-values or supply the full loopRuntime map." .) -}} {{- end -}} {{- end -}} -{{- $sandbox := .Values.loopRuntime.sandbox -}} -{{- if not (has .Values.cloud (list "aws" "google")) -}} -{{- fail "loopRuntime.enabled requires cloud: aws or cloud: google." -}} +{{- if ne .Values.cloud "google" -}} +{{- fail "loopRuntime.enabled requires cloud: google. On AWS, Loop Runtime runs on ECS through the Terraform module." -}} {{- end -}} -{{- if eq .Values.cloud "aws" -}} -{{- $_ := required "objectStorage.aws.brainstoreBucket is required when loopRuntime.enabled is true" .Values.objectStorage.aws.brainstoreBucket -}} -{{- $_ := required "objectStorage.aws.codeBundleBucket is required when loopRuntime.enabled is true" .Values.objectStorage.aws.codeBundleBucket -}} -{{- else -}} +{{- $sandbox := .Values.loopRuntime.sandbox -}} {{- $_ := required "objectStorage.google.brainstoreBucket is required when loopRuntime.enabled is true" .Values.objectStorage.google.brainstoreBucket -}} {{- $_ := required "objectStorage.google.apiBucket is required when loopRuntime.enabled is true" .Values.objectStorage.google.apiBucket -}} -{{- $_ := required "loopRuntime.serviceAccount.googleServiceAccount is required when cloud is google (use Terraform output loop_runtime_service_account)" .Values.loopRuntime.serviceAccount.googleServiceAccount -}} -{{- end -}} {{- $_ := required "loopRuntime.serviceAccount.name is required when loopRuntime.enabled is true" .Values.loopRuntime.serviceAccount.name -}} -{{- $urls := $sandbox.urls | default list -}} -{{- if and $sandbox.dnsUrl $urls -}} -{{- fail "Set only one of loopRuntime.sandbox.dnsUrl and loopRuntime.sandbox.urls." -}} -{{- end -}} -{{- if not (or $sandbox.dnsUrl $urls) -}} -{{- fail "loopRuntime.sandbox.dnsUrl or loopRuntime.sandbox.urls is required when loopRuntime.enabled is true (GKE: use Terraform output gke_isolated_workers.discovery.endpoint)" -}} -{{- end -}} -{{- range $urls -}} -{{- if not (. | default "" | trim) -}} -{{- fail "loopRuntime.sandbox.urls entries must not be blank" -}} -{{- end -}} -{{- end -}} +{{- $_ := required "loopRuntime.serviceAccount.googleServiceAccount is required (Terraform output loop_runtime_service_account)" .Values.loopRuntime.serviceAccount.googleServiceAccount -}} +{{- $_ := required "loopRuntime.sandbox.dnsUrl is required (Terraform output gke_isolated_workers.discovery.endpoint)" $sandbox.dnsUrl -}} {{- $_ := required "loopRuntime.sandbox.image is required when loopRuntime.enabled is true" $sandbox.image -}} -{{- range (prepend $urls ($sandbox.dnsUrl | default "")) -}} -{{- if ne . (trim .) -}} -{{- fail "Sandbox Service endpoints must not contain leading or trailing whitespace" -}} -{{- end -}} -{{- if and . (hasPrefix "https://" .) (not $sandbox.tls.secretName) -}} -{{- fail "loopRuntime.sandbox.tls.secretName is required for https:// Sandbox Service endpoints" -}} -{{- end -}} +{{- $_ := required "loopRuntime.sandbox.tlsSecretName is required when loopRuntime.enabled is true" $sandbox.tlsSecretName -}} +{{- $_ := required "loopRuntime.sandbox.egressGateway.address is required (Terraform output gke_isolated_workers.loop_egress.address)" $sandbox.egressGateway.address -}} +{{- $_ := required "loopRuntime.sandbox.egressGateway.url is required (Terraform output gke_isolated_workers.loop_egress.url)" $sandbox.egressGateway.url -}} +{{- if not $sandbox.egressGateway.sourceRanges -}} +{{- fail "loopRuntime.sandbox.egressGateway.sourceRanges is required (Terraform output gke_isolated_workers.loop_egress.source_ranges)" -}} {{- end -}} {{- end -}} {{- end -}} diff --git a/braintrust/templates/loop-runtime-configmap.yaml b/braintrust/templates/loop-runtime-configmap.yaml index c24cc66..6d3a9bd 100644 --- a/braintrust/templates/loop-runtime-configmap.yaml +++ b/braintrust/templates/loop-runtime-configmap.yaml @@ -3,9 +3,8 @@ {{- $apiUrl := default (include "braintrust.apiInternalUrl" .) .Values.loopRuntime.braintrustApiUrl -}} {{- $aiProxyUrl := default (include "braintrust.apiAiProxyInternalUrl" .) .Values.loopRuntime.aiProxyUrl -}} {{- $brainstoreReaderUrl := default (include "braintrust.brainstoreFastReaderInternalUrl" .) .Values.loopRuntime.brainstoreReaderUrl -}} -{{- $brainstoreBucketUri := include "braintrust.loopRuntime.brainstoreBucketUri" . -}} +{{- $brainstoreBucketUri := printf "gs://%s" .Values.objectStorage.google.brainstoreBucket -}} {{- $sandbox := .Values.loopRuntime.sandbox -}} -{{- $egressGateway := $sandbox.egressGateway -}} apiVersion: v1 kind: ConfigMap metadata: @@ -33,30 +32,23 @@ data: BRAINSTORE_VERBOSE: "1" BRAINSTORE_INDEX_URI: "{{ $brainstoreBucketUri }}/brainstore/index" BRAINSTORE_REALTIME_WAL_URI: "{{ $brainstoreBucketUri }}/brainstore/wal" - BRAINSTORE_CODE_BUNDLE_URI: {{ include "braintrust.loopRuntime.codeBundleUri" . | quote }} + BRAINSTORE_CODE_BUNDLE_URI: "gs://{{ .Values.objectStorage.google.apiBucket }}" BRAINSTORE_CACHE_DIR: {{ .Values.loopRuntime.cacheDir | quote }} BRAINSTORE_CONTROL_PLANE_TELEMETRY: {{ .Values.global.controlPlaneTelemetry | quote }} BRAINSTORE_DISABLE_STATUS_UPDATES: {{ .Values.loopRuntime.brainstoreDisableStatusUpdates | toString | quote }} NO_COLOR: "1" - {{- /* Without EXO_SANDBOX_PROVIDER, a Sandbox Service endpoint sends every sandbox to the Sandbox Service. */}} - EXO_SANDBOX_IMAGE: {{ $sandbox.image | quote }} + {{- /* Matches the ECS Loop Runtime settings for the Sandbox Service in the AWS Terraform module. */}} + EXO_SANDBOX_SERVICE_DNS_URL: {{ $sandbox.dnsUrl | quote }} + EXO_SANDBOX_SERVICE_IMAGE: {{ $sandbox.image | quote }} EXO_SANDBOX_VCPU_COUNT: {{ $sandbox.vcpuCount | quote }} EXO_SANDBOX_MEMORY_MIB: {{ $sandbox.memoryMib | quote }} - LOOP_RUNTIME_SANDBOX_EGRESS_MODE: {{ $sandbox.egressMode | quote }} - {{- if $sandbox.dnsUrl }} - EXO_SANDBOX_SERVICE_DNS_URL: {{ $sandbox.dnsUrl | quote }} - {{- else }} - EXO_SANDBOX_SERVICE_URL: {{ $sandbox.urls | join "," | quote }} - {{- end }} - {{- if $sandbox.tls.secretName }} SANDBOX_SERVICE_TLS_CERT: "/etc/sandbox-tls/client.pem" SANDBOX_SERVICE_TLS_KEY: "/etc/sandbox-tls/client.key" SANDBOX_SERVICE_TLS_CA: "/etc/sandbox-tls/server-ca.pem" - {{- end }} - {{- if $egressGateway.enabled }} - LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN: "0.0.0.0:{{ $egressGateway.port }}" - LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL: {{ $egressGateway.url | default (printf "http://%s-egress.%s:%v" (.Values.loopRuntime.service.name | default .Values.loopRuntime.name) (include "braintrust.namespace" .) $egressGateway.port) | quote }} - {{- end }} + LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN: "0.0.0.0:{{ $sandbox.egressGateway.port }}" + LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL: {{ $sandbox.egressGateway.url | quote }} + LOOP_RUNTIME_SANDBOX_EGRESS_MODE: "restricted" + LOOP_RUNTIME_ENABLE_SANDBOX_ADMIN: "true" {{- with (.Values.global.allowedOrgIds | default "" | toString | trim) }} ALLOWED_ORG_IDS: {{ . | quote }} {{- end }} diff --git a/braintrust/templates/loop-runtime-deployment.yaml b/braintrust/templates/loop-runtime-deployment.yaml index 43d14bc..6716afd 100644 --- a/braintrust/templates/loop-runtime-deployment.yaml +++ b/braintrust/templates/loop-runtime-deployment.yaml @@ -2,11 +2,8 @@ {{- if include "braintrust.loopRuntime.enabled" . -}} {{- $resourceLabels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) -}} {{- $podLabels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) (deepCopy .Values.loopRuntime.podLabels) (dict "app" .Values.loopRuntime.name) -}} -{{- if eq .Values.cloud "google" -}} {{- $_ := set $podLabels "gke-workload-identity/use" "true" -}} -{{- end -}} -{{- $tls := .Values.loopRuntime.sandbox.tls -}} -{{- $egressGateway := .Values.loopRuntime.sandbox.egressGateway -}} +{{- $sandbox := .Values.loopRuntime.sandbox -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -36,9 +33,7 @@ spec: {{- toYaml $podLabels | nindent 8 }} annotations: checksum/config: {{ include (print $.Template.BasePath "/loop-runtime-configmap.yaml") . | sha256sum }} - {{- if eq .Values.cloud "google" }} iam.gke.io/gcp-service-account: {{ .Values.loopRuntime.serviceAccount.googleServiceAccount }} - {{- end }} {{- with .Values.loopRuntime.annotations.pod }} {{- toYaml . | nindent 8 }} {{- end }} @@ -72,10 +67,8 @@ spec: - name: {{ .Values.loopRuntime.service.portName }} containerPort: {{ .Values.loopRuntime.service.port }} protocol: TCP - {{- if $egressGateway.enabled }} - - containerPort: {{ $egressGateway.port }} + - containerPort: {{ $sandbox.egressGateway.port }} protocol: TCP - {{- end }} resources: {{- toYaml .Values.loopRuntime.resources | nindent 12 }} {{- with .Values.loopRuntime.livenessProbe }} @@ -143,11 +136,9 @@ spec: mountPath: {{ .Values.loopRuntime.stateDir }} - name: cache-volume mountPath: {{ .Values.loopRuntime.cacheDir }} - {{- if $tls.secretName }} - name: sandbox-service-tls mountPath: /etc/sandbox-tls readOnly: true - {{- end }} {{- with .Values.loopRuntime.extraContainers }} {{- toYaml . | nindent 8 }} {{- end }} @@ -156,18 +147,9 @@ spec: emptyDir: {} - name: cache-volume emptyDir: {} - {{- if $tls.secretName }} - name: sandbox-service-tls secret: - secretName: {{ $tls.secretName }} - items: - - key: {{ $tls.certKey }} - path: client.pem - - key: {{ $tls.keyKey }} - path: client.key - - key: {{ $tls.caKey }} - path: server-ca.pem - {{- end }} + secretName: {{ $sandbox.tlsSecretName }} {{- with .Values.loopRuntime.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/braintrust/templates/loop-runtime-egress-service.yaml b/braintrust/templates/loop-runtime-egress-service.yaml index 36a42f5..f42ace4 100644 --- a/braintrust/templates/loop-runtime-egress-service.yaml +++ b/braintrust/templates/loop-runtime-egress-service.yaml @@ -1,7 +1,9 @@ {{- include "braintrust.loopRuntime.validate" . -}} -{{- if and (include "braintrust.loopRuntime.enabled" .) .Values.loopRuntime.sandbox.egressGateway.enabled -}} +{{- if include "braintrust.loopRuntime.enabled" . -}} {{- $resourceLabels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) -}} {{- $egressGateway := .Values.loopRuntime.sandbox.egressGateway -}} +{{- /* Workers in the isolated cluster reach the Git forward proxy through this +internal load balancer. Terraform reserves its address and DNS name. */}} apiVersion: v1 kind: Service metadata: @@ -11,11 +13,13 @@ metadata: labels: {{- toYaml . | nindent 4 }} {{- end }} - {{- with $egressGateway.service.annotations }} annotations: - {{- toYaml . | nindent 4 }} - {{- end }} + networking.gke.io/load-balancer-type: "Internal" spec: + type: LoadBalancer + loadBalancerIP: {{ $egressGateway.address | quote }} + loadBalancerSourceRanges: + {{- toYaml $egressGateway.sourceRanges | nindent 4 }} selector: app: {{ .Values.loopRuntime.name }} ports: @@ -23,9 +27,4 @@ spec: protocol: TCP port: {{ $egressGateway.port }} targetPort: {{ $egressGateway.port }} - type: {{ $egressGateway.service.type }} - {{- with $egressGateway.service.loadBalancerSourceRanges }} - loadBalancerSourceRanges: - {{- toYaml . | nindent 4 }} - {{- end }} {{- end }} diff --git a/braintrust/templates/loop-runtime-serviceaccount.yaml b/braintrust/templates/loop-runtime-serviceaccount.yaml index ac4269f..63b1c55 100644 --- a/braintrust/templates/loop-runtime-serviceaccount.yaml +++ b/braintrust/templates/loop-runtime-serviceaccount.yaml @@ -2,12 +2,7 @@ {{- if include "braintrust.loopRuntime.enabled" . -}} {{- $labels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) -}} {{- $annotations := mergeOverwrite (deepCopy .Values.loopRuntime.annotations.serviceaccount) (deepCopy .Values.loopRuntime.serviceAccount.annotations) -}} -{{- if and (eq .Values.cloud "aws") .Values.loopRuntime.serviceAccount.awsRoleArn -}} -{{- $_ := set $annotations "eks.amazonaws.com/role-arn" .Values.loopRuntime.serviceAccount.awsRoleArn -}} -{{- end -}} -{{- if eq .Values.cloud "google" -}} {{- $_ := set $annotations "iam.gke.io/gcp-service-account" .Values.loopRuntime.serviceAccount.googleServiceAccount -}} -{{- end -}} apiVersion: v1 kind: ServiceAccount metadata: diff --git a/braintrust/tests/__fixtures__/loop-runtime-google-values.yaml b/braintrust/tests/__fixtures__/loop-runtime-google-values.yaml deleted file mode 100644 index e729e86..0000000 --- a/braintrust/tests/__fixtures__/loop-runtime-google-values.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Loop Runtime on GKE. Layer on top of base-values.yaml and google-values.yaml. -loopRuntime: - enabled: true - image: - repository: "test/loop-runtime" - tag: "v1.0.0" - serviceAccount: - name: "braintrust-loop-runtime" - googleServiceAccount: "braintrust-loop@test-project.iam.gserviceaccount.com" - sandbox: - dnsUrl: "https://workers.sandbox.internal:9400" - tls: - secretName: "sandbox-client-tls" diff --git a/braintrust/tests/__fixtures__/loop-runtime-values.yaml b/braintrust/tests/__fixtures__/loop-runtime-values.yaml index 949b154..43af85a 100644 --- a/braintrust/tests/__fixtures__/loop-runtime-values.yaml +++ b/braintrust/tests/__fixtures__/loop-runtime-values.yaml @@ -1,10 +1,14 @@ -cloud: "aws" +# Loop Runtime on GKE. Layer on top of base-values.yaml. +cloud: "google" objectStorage: - aws: + google: brainstoreBucket: "test-brainstore-bucket" - responseBucket: "test-response-bucket" - codeBundleBucket: "test-code-bundle-bucket" + apiBucket: "test-api-bucket" + +brainstore: + serviceAccount: + googleServiceAccount: "test@project.iam.gserviceaccount.com" loopRuntime: enabled: true @@ -13,8 +17,12 @@ loopRuntime: tag: "v1.0.0" serviceAccount: name: "braintrust-loop-runtime" - awsRoleArn: "arn:aws:iam::123456789012:role/test-loop-runtime-role" + googleServiceAccount: "braintrust-loop@test-project.iam.gserviceaccount.com" sandbox: - dnsUrl: "https://nodes.sandbox.internal:9400" - tls: - secretName: "sandbox-client-tls" + dnsUrl: "https://workers.sandbox.internal:9400" + egressGateway: + address: "10.0.0.50" + sourceRanges: + - "10.64.64.0/24" + - "10.64.0.0/18" + url: "http://loop-egress.sandbox.internal:4002" diff --git a/braintrust/tests/api-configmap_test.yaml b/braintrust/tests/api-configmap_test.yaml index 414a579..1f5fde0 100644 --- a/braintrust/tests/api-configmap_test.yaml +++ b/braintrust/tests/api-configmap_test.yaml @@ -597,18 +597,6 @@ tests: path: data.LOOP_RUNTIME_URL value: http://braintrust-loop-runtime.braintrust:4001 - - it: should publish the Loop Runtime URL on GKE - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml - release: - namespace: "braintrust" - asserts: - - equal: - path: data.LOOP_RUNTIME_URL - value: http://braintrust-loop-runtime.braintrust:4001 - - it: should omit the Loop Runtime URL when reused values have no loopRuntime map values: - __fixtures__/base-values.yaml diff --git a/braintrust/tests/eks-loop-runtime_test.yaml b/braintrust/tests/eks-loop-runtime_test.yaml deleted file mode 100644 index 4ad4281..0000000 --- a/braintrust/tests/eks-loop-runtime_test.yaml +++ /dev/null @@ -1,58 +0,0 @@ -suite: EKS Loop Runtime installation with the shared Sandbox Service chart -templates: - - loop-runtime-configmap.yaml - - loop-runtime-deployment.yaml - - loop-runtime-egress-service.yaml - - api-configmap.yaml -tests: - - it: should discover the companion workers through cluster DNS and use the internal proxy - release: - namespace: braintrust - template: loop-runtime-configmap.yaml - values: &eks-values - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - - ../examples/eks-loop-runtime/values.yaml - asserts: - - equal: - path: data.EXO_SANDBOX_SERVICE_DNS_URL - value: https://sandbox-service.braintrust-sandbox.svc.cluster.local:9400 - - equal: - path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL - value: http://braintrust-loop-runtime-egress.braintrust:4002 - - equal: - path: data.SANDBOX_SERVICE_TLS_CERT - value: /etc/sandbox-tls/client.pem - - isNull: - path: data.EXO_SANDBOX_PROVIDER - - isNull: - path: data.AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER - - - it: should install the runtime on system nodes with the client TLS Secret - release: - namespace: braintrust - template: loop-runtime-deployment.yaml - values: *eks-values - asserts: - - equal: - path: spec.template.spec.nodeSelector["braintrust.io/workload"] - value: sandbox-system - - equal: - path: spec.template.spec.containers[0].image - value: test/loop-runtime:v2.18.0 - - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: Always - - contains: - path: spec.template.spec.volumes - content: - name: sandbox-service-tls - secret: - secretName: sandbox-client-tls - items: - - key: client.pem - path: client.pem - - key: client.key - path: client.key - - key: server-ca.pem - path: server-ca.pem diff --git a/braintrust/tests/labels-merge-services_test.yaml b/braintrust/tests/labels-merge-services_test.yaml index 1f5c875..cd3a717 100644 --- a/braintrust/tests/labels-merge-services_test.yaml +++ b/braintrust/tests/labels-merge-services_test.yaml @@ -202,8 +202,6 @@ tests: - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml set: - loopRuntime.sandbox.egressGateway.enabled: true - loopRuntime.sandbox.egressGateway.url: http://egress.example:4002 global.labels.shared: global-service global.labels.component: global loopRuntime.labels.component: loop-runtime @@ -230,8 +228,6 @@ tests: - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml set: - loopRuntime.sandbox.egressGateway.enabled: true - loopRuntime.sandbox.egressGateway.url: http://egress.example:4002 global.labels.shared: global-service global.labels.component: global loopRuntime.labels.component: loop-runtime diff --git a/braintrust/tests/loop-runtime-configmap_test.yaml b/braintrust/tests/loop-runtime-configmap_test.yaml index b273703..1085297 100644 --- a/braintrust/tests/loop-runtime-configmap_test.yaml +++ b/braintrust/tests/loop-runtime-configmap_test.yaml @@ -25,74 +25,10 @@ tests: - failedTemplate: errorMessage: "loopRuntime.image is missing. Values reused from a chart release that predates Loop Runtime omit the loopRuntime defaults; upgrade with --reset-then-reuse-values or supply the full loopRuntime map." - - it: should render AWS storage and the Sandbox Service configuration - values: &aws-values - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - release: - name: test-release - namespace: braintrust - asserts: - - isKind: - of: ConfigMap - - equal: - path: metadata.name - value: braintrust-loop-runtime - - equal: - path: metadata.namespace - value: braintrust - - equal: - path: data.LOOP_RUNTIME_REDIS_NAMESPACE - value: loop-runtime:test-release - - equal: - path: data.BRAINTRUST_API_URL - value: http://braintrust-api.braintrust:8000 - - equal: - path: data.LOOP_RUNTIME_BRAINSTORE_READER_URL - value: http://brainstore-fastreader.braintrust:4000 - - equal: - path: data.BRAINSTORE_INDEX_URI - value: s3://test-brainstore-bucket/brainstore/index - - equal: - path: data.BRAINSTORE_REALTIME_WAL_URI - value: s3://test-brainstore-bucket/brainstore/wal - - equal: - path: data.BRAINSTORE_CODE_BUNDLE_URI - value: s3://test-code-bundle-bucket - - equal: - path: data.EXO_SANDBOX_SERVICE_DNS_URL - value: https://nodes.sandbox.internal:9400 - - equal: - path: data.EXO_SANDBOX_IMAGE - value: public.ecr.aws/braintrust/codex-runtime:latest - - equal: - path: data.SANDBOX_SERVICE_TLS_CERT - value: /etc/sandbox-tls/client.pem - - isNull: - path: data.EXO_SANDBOX_PROVIDER - - isNull: - path: data.AWS_REGION - - isNull: - path: data.AWS_LAMBDA_MICROVM_IMAGE_IDENTIFIER - - isNull: - path: data.LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN - - isNull: - path: data.BRAINSTORE_LOCKS_URI - - - it: should use AWS object storage locks when configured - values: *aws-values - set: - brainstore.locksBackend: objectStorage - asserts: - - equal: - path: data.BRAINSTORE_LOCKS_URI - value: s3://test-brainstore-bucket/brainstore/locks - - it: should render GKE storage and the Sandbox Service configuration - values: &gke-values + values: &values - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + - __fixtures__/loop-runtime-values.yaml release: namespace: braintrust asserts: @@ -106,20 +42,17 @@ tests: path: data.BRAINSTORE_CODE_BUNDLE_URI value: gs://test-api-bucket - equal: - path: data.EXO_SANDBOX_IMAGE - value: public.ecr.aws/braintrust/codex-runtime:latest + path: data.EXO_SANDBOX_SERVICE_DNS_URL + value: https://workers.sandbox.internal:9400 + - equal: + path: data.EXO_SANDBOX_SERVICE_IMAGE + value: public.ecr.aws/braintrust/codex-runtime@sha256:35db9c98d4a74e20758a29b82f8a3a6552be0002eb439c8c1a0f434ac573ee21 - equal: path: data.EXO_SANDBOX_VCPU_COUNT value: "1" - equal: path: data.EXO_SANDBOX_MEMORY_MIB value: "2048" - - equal: - path: data.LOOP_RUNTIME_SANDBOX_EGRESS_MODE - value: restricted - - equal: - path: data.EXO_SANDBOX_SERVICE_DNS_URL - value: https://workers.sandbox.internal:9400 - equal: path: data.SANDBOX_SERVICE_TLS_CERT value: /etc/sandbox-tls/client.pem @@ -129,19 +62,28 @@ tests: - equal: path: data.SANDBOX_SERVICE_TLS_CA value: /etc/sandbox-tls/server-ca.pem + - equal: + path: data.LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN + value: 0.0.0.0:4002 + - equal: + path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL + value: http://loop-egress.sandbox.internal:4002 + - equal: + path: data.LOOP_RUNTIME_SANDBOX_EGRESS_MODE + value: restricted + - equal: + path: data.LOOP_RUNTIME_ENABLE_SANDBOX_ADMIN + value: "true" + - equal: + path: data.LOOP_RUNTIME_REDIS_NAMESPACE + value: loop-runtime:RELEASE-NAME - isNull: path: data.EXO_SANDBOX_PROVIDER - - isNull: - path: data.EXO_SANDBOX_SERVICE_URL - - isNull: - path: data.AWS_REGION - - isNull: - path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL - isNull: path: data.BRAINSTORE_LOCKS_URI - it: should use GCS object storage locks when configured - values: *gke-values + values: *values set: brainstore.locksBackend: objectStorage asserts: @@ -149,12 +91,13 @@ tests: path: data.BRAINSTORE_LOCKS_URI value: gs://test-brainstore-bucket/brainstore/locks - - it: should pass sandbox sizing and internet egress mode - values: *gke-values + - it: should pass custom sandbox sizing, Git proxy port, and Redis namespace + values: *values set: loopRuntime.sandbox.vcpuCount: 2 loopRuntime.sandbox.memoryMib: 4096 - loopRuntime.sandbox.egressMode: internet + loopRuntime.sandbox.egressGateway.port: 5002 + loopRuntime.redisNamespace: loop-runtime:existing asserts: - equal: path: data.EXO_SANDBOX_VCPU_COUNT @@ -163,206 +106,96 @@ tests: path: data.EXO_SANDBOX_MEMORY_MIB value: "4096" - equal: - path: data.LOOP_RUNTIME_SANDBOX_EGRESS_MODE - value: internet - - - it: should join fixed Sandbox Service endpoints - values: *gke-values - set: - loopRuntime.sandbox.dnsUrl: "" - loopRuntime.sandbox.urls: - - https://worker-0.sandbox.internal:9400 - - https://worker-1.sandbox.internal:9400 - asserts: - - equal: - path: data.EXO_SANDBOX_SERVICE_URL - value: https://worker-0.sandbox.internal:9400,https://worker-1.sandbox.internal:9400 - - isNull: - path: data.EXO_SANDBOX_SERVICE_DNS_URL - - - it: should allow a plain HTTP Sandbox Service endpoint without TLS - values: *gke-values - set: - loopRuntime.sandbox.dnsUrl: http://sandbox-service.sandbox:9400 - loopRuntime.sandbox.tls.secretName: "" - asserts: + path: data.LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN + value: 0.0.0.0:5002 - equal: - path: data.EXO_SANDBOX_SERVICE_DNS_URL - value: http://sandbox-service.sandbox:9400 - - isNull: - path: data.SANDBOX_SERVICE_TLS_CERT - - - it: should require TLS for HTTPS Sandbox Service endpoints - values: *gke-values - set: - loopRuntime.sandbox.tls.secretName: "" - asserts: &tls-required - - failedTemplate: - errorMessage: "loopRuntime.sandbox.tls.secretName is required for https:// Sandbox Service endpoints" - - - it: should require TLS when any fixed endpoint uses HTTPS - values: *gke-values - set: - loopRuntime.sandbox.dnsUrl: "" - loopRuntime.sandbox.tls.secretName: "" - loopRuntime.sandbox.urls: - - http://worker-0.sandbox.internal:9400 - - https://worker-1.sandbox.internal:9400 - asserts: *tls-required + path: data.LOOP_RUNTIME_REDIS_NAMESPACE + value: loop-runtime:existing - - it: should require a Sandbox Service endpoint - values: *aws-values + - it: should reject Loop Runtime on AWS + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml set: - loopRuntime.sandbox.dnsUrl: "" - asserts: + cloud: aws + asserts: ¬-google - failedTemplate: - errorMessage: "loopRuntime.sandbox.dnsUrl or loopRuntime.sandbox.urls is required when loopRuntime.enabled is true (GKE: use Terraform output gke_isolated_workers.discovery.endpoint)" + errorMessage: "loopRuntime.enabled requires cloud: google. On AWS, Loop Runtime runs on ECS through the Terraform module." - - it: should reject both Sandbox Service discovery modes - values: *gke-values + - it: should reject Loop Runtime on Azure + values: + - __fixtures__/base-values.yaml + - __fixtures__/azure-values.yaml set: - loopRuntime.sandbox.urls: - - https://worker-0.sandbox.internal:9400 - asserts: - - failedTemplate: - errorMessage: "Set only one of loopRuntime.sandbox.dnsUrl and loopRuntime.sandbox.urls." + loopRuntime.enabled: true + asserts: *not-google - - it: should require the sandbox guest image - values: *gke-values + - it: should reject Loop Runtime without a cloud + values: *base-values set: - loopRuntime.sandbox.image: "" - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.image is required when loopRuntime.enabled is true" + loopRuntime.enabled: true + asserts: *not-google - - it: should require the Loop Runtime Google service account on GKE - values: *gke-values + - it: should require the Loop Runtime Google service account + values: *values set: loopRuntime.serviceAccount.googleServiceAccount: "" asserts: - failedTemplate: - errorMessage: "loopRuntime.serviceAccount.googleServiceAccount is required when cloud is google (use Terraform output loop_runtime_service_account)" + errorMessage: "loopRuntime.serviceAccount.googleServiceAccount is required (Terraform output loop_runtime_service_account)" - - it: should require the GCS API bucket on GKE - values: *gke-values + - it: should require the GCS API bucket + values: *values set: objectStorage.google.apiBucket: "" asserts: - failedTemplate: errorMessage: "objectStorage.google.apiBucket is required when loopRuntime.enabled is true" - - it: should require the code bundle bucket on AWS - values: *aws-values + - it: should require the worker discovery endpoint + values: *values set: - objectStorage.aws.codeBundleBucket: "" + loopRuntime.sandbox.dnsUrl: null asserts: - failedTemplate: - errorMessage: "objectStorage.aws.codeBundleBucket is required when loopRuntime.enabled is true" + errorMessage: "loopRuntime.sandbox.dnsUrl is required (Terraform output gke_isolated_workers.discovery.endpoint)" - - it: should reject Loop Runtime on Azure - values: - - __fixtures__/base-values.yaml - - __fixtures__/azure-values.yaml + - it: should require the sandbox guest image + values: *values set: - loopRuntime.enabled: true - asserts: &assertions-1 + loopRuntime.sandbox.image: "" + asserts: - failedTemplate: - errorMessage: "loopRuntime.enabled requires cloud: aws or cloud: google." - - - it: should reject Loop Runtime without a cloud - values: *base-values - set: - loopRuntime.enabled: true - asserts: *assertions-1 + errorMessage: "loopRuntime.sandbox.image is required when loopRuntime.enabled is true" - - it: should advertise the egress gateway to Sandbox Service workers - values: *gke-values + - it: should require the TLS Secret name + values: *values set: - loopRuntime.sandbox.egressGateway.enabled: true - loopRuntime.sandbox.egressGateway.port: 4100 - loopRuntime.sandbox.egressGateway.url: http://10.10.0.20:4100 + loopRuntime.sandbox.tlsSecretName: "" asserts: - - equal: - path: data.LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN - value: 0.0.0.0:4100 - - equal: - path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL - value: http://10.10.0.20:4100 - - isNull: - path: data.LOOP_RUNTIME_SANDBOX_EGRESS_GATEWAY_URL + - failedTemplate: + errorMessage: "loopRuntime.sandbox.tlsSecretName is required when loopRuntime.enabled is true" - - it: should use the in-cluster egress Service when no external URL is set - release: - namespace: braintrust - values: *gke-values + - it: should require the reserved egress address + values: *values set: - loopRuntime.sandbox.egressGateway.enabled: true - loopRuntime.service.name: loop - loopRuntime.sandbox.egressGateway.port: 4100 + loopRuntime.sandbox.egressGateway.address: "" asserts: - - equal: - path: data.LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN - value: 0.0.0.0:4100 - - equal: - path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL - value: http://loop-egress.braintrust:4100 + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.address is required (Terraform output gke_isolated_workers.loop_egress.address)" - - it: should default the egress URL when reused values omit it - release: - namespace: braintrust - values: *aws-values + - it: should require the egress URL + values: *values set: - loopRuntime.sandbox.egressGateway.enabled: true loopRuntime.sandbox.egressGateway.url: null - asserts: &assertions-2 - - equal: - path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL - value: http://braintrust-loop-runtime-egress.braintrust:4002 - - - it: should treat a zero egress URL as empty - release: - namespace: braintrust - values: *aws-values - set: - loopRuntime.sandbox.egressGateway.enabled: true - loopRuntime.sandbox.egressGateway.url: 0 - asserts: *assertions-2 - - - it: should reject empty endpoint - values: *aws-values - set: - loopRuntime.sandbox.dnsUrl: "" - loopRuntime.sandbox.urls: [""] - asserts: &assertions-3 + asserts: - failedTemplate: - errorMessage: "loopRuntime.sandbox.urls entries must not be blank" - - - it: should reject whitespace endpoint - values: *aws-values - set: - loopRuntime.sandbox.dnsUrl: "" - loopRuntime.sandbox.urls: [" "] - asserts: *assertions-3 - - - it: should reject null endpoint - values: *aws-values - set: - loopRuntime.sandbox.dnsUrl: "" - loopRuntime.sandbox.urls: [null] - asserts: *assertions-3 - - - it: should reject blank entry alongside a usable endpoint - values: *aws-values - set: - loopRuntime.sandbox.dnsUrl: "" - loopRuntime.sandbox.urls: ["https://worker.example:9400", ""] - asserts: *assertions-3 + errorMessage: "loopRuntime.sandbox.egressGateway.url is required (Terraform output gke_isolated_workers.loop_egress.url)" - - it: should reject padded endpoint - values: *aws-values + - it: should require egress source ranges + values: *values set: - loopRuntime.sandbox.dnsUrl: "" - loopRuntime.sandbox.urls: [" https://worker.example:9400 "] + loopRuntime.sandbox.egressGateway.sourceRanges: [] asserts: - failedTemplate: - errorMessage: "Sandbox Service endpoints must not contain leading or trailing whitespace" + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges is required (Terraform output gke_isolated_workers.loop_egress.source_ranges)" diff --git a/braintrust/tests/loop-runtime-deployment_test.yaml b/braintrust/tests/loop-runtime-deployment_test.yaml index 91be698..c52c173 100644 --- a/braintrust/tests/loop-runtime-deployment_test.yaml +++ b/braintrust/tests/loop-runtime-deployment_test.yaml @@ -6,27 +6,39 @@ tests: - it: should not render when Loop Runtime is disabled values: - __fixtures__/base-values.yaml - asserts: &absent + asserts: + - hasDocuments: + count: 0 + + - it: should not render when reused values have no loopRuntime map + values: + - __fixtures__/base-values.yaml + - __fixtures__/google-values.yaml + - __fixtures__/pre-loop-runtime-values.yaml + asserts: - hasDocuments: count: 0 - - it: should render a hardened runtime pod with data-plane secrets + - it: should render a hardened runtime pod with GKE workload identity and data-plane secrets template: loop-runtime-deployment.yaml - values: &aws-values + values: &values - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml asserts: - isKind: of: Deployment - - equal: - path: metadata.name - value: braintrust-loop-runtime - equal: path: spec.replicas - value: 1 + value: 2 - equal: path: spec.template.spec.serviceAccountName value: braintrust-loop-runtime + - equal: + path: spec.template.metadata.labels["gke-workload-identity/use"] + value: "true" + - equal: + path: spec.template.metadata.annotations["iam.gke.io/gcp-service-account"] + value: braintrust-loop@test-project.iam.gserviceaccount.com - equal: path: spec.template.spec.containers[0].image value: test/loop-runtime:v1.0.0 @@ -36,6 +48,11 @@ tests: - contains: path: spec.template.spec.containers[0].securityContext.capabilities.drop content: ALL + - equal: + path: spec.template.spec.containers[0].resources.requests + value: + cpu: "1" + memory: 2Gi - contains: path: spec.template.spec.containers[0].env content: @@ -60,67 +77,11 @@ tests: secretKeyRef: name: braintrust-secrets key: REDIS_URL - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: state-volume - mountPath: /mnt/tmp/loop-runtime - - contains: - path: spec.template.spec.volumes - content: - name: cache-volume - emptyDir: {} - - - it: should keep probes on a custom named service port - template: loop-runtime-deployment.yaml - values: *aws-values - set: - loopRuntime.service.port: 5000 - loopRuntime.service.portName: metrics - asserts: - - equal: - path: spec.template.spec.containers[0].ports[0].containerPort - value: 5000 - - equal: - path: spec.template.spec.containers[0].ports[0].name - value: metrics - - equal: - path: spec.template.spec.containers[0].livenessProbe.httpGet.port - value: metrics - - equal: - path: spec.template.spec.containers[0].readinessProbe.httpGet.port - value: metrics - - it: should not use GKE workload identity on AWS + - it: should mount the Sandbox Service client TLS Secret and expose the Git proxy port template: loop-runtime-deployment.yaml - values: *aws-values + values: *values asserts: - - isNull: - path: spec.template.metadata.labels["gke-workload-identity/use"] - - isNull: - path: spec.template.metadata.annotations["iam.gke.io/gcp-service-account"] - - lengthEqual: - path: spec.template.spec.containers[0].ports - count: 1 - - - it: should use GKE workload identity and mount Sandbox Service client TLS on Google - template: loop-runtime-deployment.yaml - values: &gke-values - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml - asserts: - - equal: - path: spec.template.metadata.labels["gke-workload-identity/use"] - value: "true" - - equal: - path: spec.template.metadata.annotations["iam.gke.io/gcp-service-account"] - value: braintrust-loop@test-project.iam.gserviceaccount.com - - equal: - path: spec.template.spec.serviceAccountName - value: braintrust-loop-runtime - - isNull: - path: spec.template.spec.nodeSelector - contains: path: spec.template.spec.containers[0].volumeMounts content: @@ -133,90 +94,42 @@ tests: name: sandbox-service-tls secret: secretName: sandbox-client-tls - items: - - key: client.pem - path: client.pem - - key: client.key - path: client.key - - key: server-ca.pem - path: server-ca.pem - - - it: should map custom TLS secret keys to the paths Loop Runtime reads - template: loop-runtime-deployment.yaml - values: *gke-values - set: - loopRuntime.sandbox.tls.secretName: cert-manager-client - loopRuntime.sandbox.tls.certKey: tls.crt - loopRuntime.sandbox.tls.keyKey: tls.key - loopRuntime.sandbox.tls.caKey: ca.crt - asserts: - - contains: - path: spec.template.spec.volumes - content: - name: sandbox-service-tls - secret: - secretName: cert-manager-client - items: - - key: tls.crt - path: client.pem - - key: tls.key - path: client.key - - key: ca.crt - path: server-ca.pem - - - it: should omit the TLS volume for a plain HTTP Sandbox Service endpoint - template: loop-runtime-deployment.yaml - values: *gke-values - set: - loopRuntime.sandbox.dnsUrl: http://sandbox-service.sandbox:9400 - loopRuntime.sandbox.tls.secretName: "" - asserts: - - lengthEqual: - path: spec.template.spec.volumes - count: 2 - - lengthEqual: - path: spec.template.spec.containers[0].volumeMounts - count: 2 - - - it: should mount Sandbox Service client TLS on AWS - template: loop-runtime-deployment.yaml - values: *aws-values - asserts: - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: sandbox-service-tls - mountPath: /etc/sandbox-tls - readOnly: true - - lengthEqual: - path: spec.template.spec.volumes - count: 3 + - equal: + path: spec.template.spec.containers[0].ports + value: + - name: http + containerPort: 4001 + protocol: TCP + - containerPort: 4002 + protocol: TCP - - it: should expose the egress gateway container port when enabled + - it: should use a custom TLS Secret name and Git proxy port template: loop-runtime-deployment.yaml - values: *gke-values + values: *values set: - loopRuntime.sandbox.egressGateway.enabled: true - loopRuntime.sandbox.egressGateway.url: http://10.10.0.20:4002 + loopRuntime.sandbox.tlsSecretName: custom-tls + loopRuntime.sandbox.egressGateway.port: 5002 asserts: - - contains: - path: spec.template.spec.containers[0].ports - content: - containerPort: 4002 - protocol: TCP + - equal: + path: spec.template.spec.volumes[2].secret.secretName + value: custom-tls + - equal: + path: spec.template.spec.containers[0].ports[1].containerPort + value: 5002 - - it: should allow the main port to be named egress when the gateway is disabled + - it: should keep probes on a custom named service port template: loop-runtime-deployment.yaml - values: *aws-values + values: *values set: + loopRuntime.service.port: 5000 loopRuntime.service.portName: egress asserts: - equal: - path: spec.template.spec.containers[0].ports + path: spec.template.spec.containers[0].ports[0] value: - - name: egress - containerPort: 4001 - protocol: TCP + name: egress + containerPort: 5000 + protocol: TCP - equal: path: spec.template.spec.containers[0].livenessProbe.httpGet.port value: egress @@ -224,41 +137,29 @@ tests: path: spec.template.spec.containers[0].readinessProbe.httpGet.port value: egress - - it: should keep port names unique with the gateway enabled and a custom port + - it: should default null replicas to one template: loop-runtime-deployment.yaml - values: *aws-values + values: *values set: - loopRuntime.service.portName: egress - loopRuntime.sandbox.egressGateway.enabled: true - loopRuntime.sandbox.egressGateway.url: http://egress.example:5002 - loopRuntime.sandbox.egressGateway.port: 5002 + loopRuntime.replicas: null asserts: - equal: - path: spec.template.spec.containers[0].ports - value: - - name: egress - containerPort: 4001 - protocol: TCP - - containerPort: 5002 - protocol: TCP - - equal: - path: spec.template.spec.containers[0].livenessProbe.httpGet.port - value: egress - - equal: - path: spec.template.spec.containers[0].readinessProbe.httpGet.port - value: egress + path: spec.replicas + value: 1 - - it: should not render when reused values have no loopRuntime map + - it: should default empty replicas to one template: loop-runtime-deployment.yaml - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/pre-loop-runtime-values.yaml - asserts: *absent + values: *values + set: + loopRuntime.replicas: "" + asserts: + - equal: + path: spec.replicas + value: 1 - - it: should scale to zero when explicitly requested + - it: should scale to zero template: loop-runtime-deployment.yaml - values: *aws-values + values: *values set: loopRuntime.replicas: 0 asserts: @@ -266,29 +167,70 @@ tests: path: spec.replicas value: 0 - - it: should use an explicit positive replica count + - it: should keep an explicit replica count template: loop-runtime-deployment.yaml - values: *aws-values + values: *values set: - loopRuntime.replicas: 2 + loopRuntime.replicas: 3 asserts: - equal: path: spec.replicas - value: 2 + value: 3 - - it: should default null or missing replicas to one + - it: should default to the pinned image without a semver gate template: loop-runtime-deployment.yaml - values: *aws-values + values: *values set: - loopRuntime.replicas: null - asserts: &default-replicas + loopRuntime.image.repository: public.ecr.aws/braintrust/loop-runtime + loopRuntime.image.tag: b96d95eaa269031cdd6f286a17b9b3e4795355d1 + asserts: - equal: - path: spec.replicas - value: 1 + path: spec.template.spec.containers[0].image + value: public.ecr.aws/braintrust/loop-runtime:b96d95eaa269031cdd6f286a17b9b3e4795355d1 + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent - - it: should default empty replicas to one + - it: should render release tags without the v prefix template: loop-runtime-deployment.yaml - values: *aws-values + values: *values set: - loopRuntime.replicas: "" - asserts: *default-replicas + loopRuntime.image.tag: 2.20.0 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: test/loop-runtime:2.20.0 + + - it: should render prerelease tags + template: loop-runtime-deployment.yaml + values: *values + set: + loopRuntime.image.tag: v2.20.0-rc.1 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: test/loop-runtime:v2.20.0-rc.1 + + - it: should render all-digit short SHAs as strings + template: loop-runtime-deployment.yaml + values: *values + set: + loopRuntime.image.tag: "1234567" + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: test/loop-runtime:1234567 + + - it: should keep a mutable tag with an explicit pull policy + template: loop-runtime-deployment.yaml + values: *values + set: + loopRuntime.image.tag: latest-2.x + loopRuntime.image.pullPolicy: Always + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: test/loop-runtime:latest-2.x + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always diff --git a/braintrust/tests/loop-runtime-egress-service_test.yaml b/braintrust/tests/loop-runtime-egress-service_test.yaml index 69146e5..319aa86 100644 --- a/braintrust/tests/loop-runtime-egress-service_test.yaml +++ b/braintrust/tests/loop-runtime-egress-service_test.yaml @@ -5,31 +5,14 @@ tests: - it: should not render when Loop Runtime is disabled values: - __fixtures__/base-values.yaml - asserts: &absent + asserts: - hasDocuments: count: 0 - - it: should not render when the egress gateway is disabled - values: &gke-values - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml - asserts: *absent - - - it: should not render when reused values have no loopRuntime map + - it: should expose the Git proxy on an internal load balancer at the reserved address values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/pre-loop-runtime-values.yaml - asserts: *absent - - - it: should expose the egress gateway through a ClusterIP Service by default - values: &aws-values - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml - set: - loopRuntime.sandbox.egressGateway.enabled: true - loopRuntime.sandbox.egressGateway.url: http://egress.vpce.internal:4002 release: namespace: braintrust asserts: @@ -39,65 +22,44 @@ tests: path: metadata.name value: braintrust-loop-runtime-egress - equal: - path: spec.selector.app - value: braintrust-loop-runtime - - equal: - path: spec.ports[0] - value: - name: egress - protocol: TCP - port: 4002 - targetPort: 4002 + path: metadata.annotations["networking.gke.io/load-balancer-type"] + value: Internal - equal: path: spec.type - value: ClusterIP - - isNull: + value: LoadBalancer + - equal: + path: spec.loadBalancerIP + value: 10.0.0.50 + - equal: path: spec.loadBalancerSourceRanges - - isNull: - path: metadata.annotations - - - it: should target the numeric gateway port when the main port is named egress - values: *aws-values - set: - loopRuntime.service.portName: egress - loopRuntime.sandbox.egressGateway.enabled: true - loopRuntime.sandbox.egressGateway.url: http://egress.example:5002 - loopRuntime.sandbox.egressGateway.port: 5002 - asserts: + value: + - 10.64.64.0/24 + - 10.64.0.0/18 + - equal: + path: spec.selector.app + value: braintrust-loop-runtime - equal: - path: spec.ports[0] + path: spec.ports value: - name: egress - protocol: TCP - port: 5002 - targetPort: 5002 + - name: egress + protocol: TCP + port: 4002 + targetPort: 4002 - - it: should expose the egress gateway through a GKE internal load balancer - values: *gke-values + - it: should follow a custom service name and Git proxy port + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml set: loopRuntime.service.name: loop - loopRuntime.labels.loop-specific: loop-egress - loopRuntime.sandbox.egressGateway.enabled: true - loopRuntime.sandbox.egressGateway.url: http://10.10.0.20:4002 - loopRuntime.sandbox.egressGateway.service.type: LoadBalancer - loopRuntime.sandbox.egressGateway.service.annotations: - networking.gke.io/load-balancer-type: Internal - loopRuntime.sandbox.egressGateway.service.loadBalancerSourceRanges: - - 10.64.64.0/24 + loopRuntime.sandbox.egressGateway.port: 5002 asserts: - equal: path: metadata.name value: loop-egress - equal: - path: metadata.labels["loop-specific"] - value: loop-egress + path: spec.ports[0].port + value: 5002 - equal: - path: spec.type - value: LoadBalancer - - equal: - path: metadata.annotations["networking.gke.io/load-balancer-type"] - value: Internal - - equal: - path: spec.loadBalancerSourceRanges - value: - - 10.64.64.0/24 + path: spec.ports[0].targetPort + value: 5002 diff --git a/braintrust/tests/loop-runtime-images_test.yaml b/braintrust/tests/loop-runtime-images_test.yaml deleted file mode 100644 index 96e7d91..0000000 --- a/braintrust/tests/loop-runtime-images_test.yaml +++ /dev/null @@ -1,109 +0,0 @@ -suite: Loop Runtime image tag compatibility -templates: - - loop-runtime-deployment.yaml - - loop-runtime-configmap.yaml -tests: - - it: should render image tag v2.18.0 - template: loop-runtime-deployment.yaml - values: &runtime-values - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - set: - loopRuntime.image.tag: "v2.18.0" - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: "test/loop-runtime:v2.18.0" - - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: Always - - - it: should render image tag 2.18.0 - template: loop-runtime-deployment.yaml - values: *runtime-values - set: - loopRuntime.image.tag: "2.18.0" - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: "test/loop-runtime:2.18.0" - - - it: should render image tag v2.18.0-rc.1 - template: loop-runtime-deployment.yaml - values: *runtime-values - set: - loopRuntime.image.tag: "v2.18.0-rc.1" - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: "test/loop-runtime:v2.18.0-rc.1" - - - it: should render image tag 2.18.0-rc.1 - template: loop-runtime-deployment.yaml - values: *runtime-values - set: - loopRuntime.image.tag: "2.18.0-rc.1" - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: "test/loop-runtime:2.18.0-rc.1" - - - it: should render image tag a1b2c3d - template: loop-runtime-deployment.yaml - values: *runtime-values - set: - loopRuntime.image.tag: "a1b2c3d" - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: "test/loop-runtime:a1b2c3d" - - - it: should render image tag 1234567 - template: loop-runtime-deployment.yaml - values: *runtime-values - set: - loopRuntime.image.tag: "1234567" - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: "test/loop-runtime:1234567" - - - it: should render image tag 53af3580126bc915b67166a41d5c78b3985d1e94 - template: loop-runtime-deployment.yaml - values: *runtime-values - set: - loopRuntime.image.tag: "53af3580126bc915b67166a41d5c78b3985d1e94" - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: "test/loop-runtime:53af3580126bc915b67166a41d5c78b3985d1e94" - - - it: should render image tag custom-build - template: loop-runtime-deployment.yaml - values: *runtime-values - set: - loopRuntime.image.tag: "custom-build" - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: "test/loop-runtime:custom-build" - - - it: should render image tag latest-2.x - template: loop-runtime-deployment.yaml - values: *runtime-values - set: - loopRuntime.image.tag: "latest-2.x" - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: "test/loop-runtime:latest-2.x" - - - it: should retain an explicit pull policy for a pinned image - template: loop-runtime-deployment.yaml - values: *runtime-values - set: - loopRuntime.image.pullPolicy: IfNotPresent - asserts: - - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: IfNotPresent diff --git a/braintrust/tests/loop-runtime-service_test.yaml b/braintrust/tests/loop-runtime-service_test.yaml index a5da702..e4dc895 100644 --- a/braintrust/tests/loop-runtime-service_test.yaml +++ b/braintrust/tests/loop-runtime-service_test.yaml @@ -32,16 +32,6 @@ tests: path: spec.type value: ClusterIP - - it: should expose the runtime port on GKE - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml - asserts: - - equal: - path: spec.ports[0].port - value: 4001 - - it: should not render when reused values have no loopRuntime map values: - __fixtures__/base-values.yaml diff --git a/braintrust/tests/loop-runtime-serviceaccount_test.yaml b/braintrust/tests/loop-runtime-serviceaccount_test.yaml index 0e13e11..261132c 100644 --- a/braintrust/tests/loop-runtime-serviceaccount_test.yaml +++ b/braintrust/tests/loop-runtime-serviceaccount_test.yaml @@ -9,8 +9,8 @@ tests: - hasDocuments: count: 0 - - it: should attach the configured AWS role for IRSA - values: &aws-values + - it: should bind the Loop Runtime Google service account + values: &values - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml asserts: @@ -20,43 +20,18 @@ tests: path: metadata.name value: braintrust-loop-runtime - equal: - path: metadata.annotations["eks.amazonaws.com/role-arn"] - value: arn:aws:iam::123456789012:role/test-loop-runtime-role - - - it: should support Pod Identity without adding an IRSA annotation - values: *aws-values - set: - loopRuntime.serviceAccount.awsRoleArn: "" - asserts: &assertions-1 - - isNull: - path: metadata.annotations["eks.amazonaws.com/role-arn"] + path: metadata.annotations["iam.gke.io/gcp-service-account"] + value: braintrust-loop@test-project.iam.gserviceaccount.com - - it: should include ServiceAccount annotations from the nested configuration - values: *aws-values + - it: should merge annotations from both configuration paths + values: *values set: - loopRuntime.serviceAccount.annotations.custom-annotation: value - asserts: - - equal: - path: metadata.annotations["custom-annotation"] - value: value - - - it: should bind the Loop Runtime Google service account on GKE - values: &gke-values - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml + loopRuntime.annotations.serviceaccount.from-annotations: a + loopRuntime.serviceAccount.annotations.from-service-account: b asserts: - equal: - path: metadata.name - value: braintrust-loop-runtime + path: metadata.annotations["from-annotations"] + value: a - equal: - path: metadata.annotations["iam.gke.io/gcp-service-account"] - value: braintrust-loop@test-project.iam.gserviceaccount.com - - isNull: - path: metadata.annotations["eks.amazonaws.com/role-arn"] - - - it: should not add the IRSA annotation on GKE - values: *gke-values - set: - loopRuntime.serviceAccount.awsRoleArn: arn:aws:iam::123456789012:role/ignored - asserts: *assertions-1 + path: metadata.annotations["from-service-account"] + value: b diff --git a/braintrust/tests/virtualservice_test.yaml b/braintrust/tests/virtualservice_test.yaml index d659b15..b7c4037 100644 --- a/braintrust/tests/virtualservice_test.yaml +++ b/braintrust/tests/virtualservice_test.yaml @@ -360,18 +360,6 @@ tests: path: spec.http[1].match[0].uri.prefix value: / - - it: should route Loop Runtime on GKE - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/loop-runtime-google-values.yaml - set: - virtualService.enabled: true - asserts: - - equal: - path: spec.http[0].route[0].destination.host - value: braintrust-loop-runtime - - it: should render without Loop Runtime routes when reused values have no loopRuntime map values: - __fixtures__/base-values.yaml diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 89ece2e..b4fb6f3 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -309,7 +309,8 @@ api: extraContainers: [] extraVolumes: [] -# Loop Runtime configuration (optional; cloud: aws or google) +# Loop Runtime configuration (optional; cloud: google only) +# See examples/gke-loop-runtime for the Sandbox Service worker release it needs. loopRuntime: enabled: false name: "braintrust-loop-runtime" @@ -322,11 +323,12 @@ loopRuntime: pod: {} serviceaccount: {} # Null, missing, or empty replicas default to 1; 0 scales to zero. - replicas: 1 + replicas: 2 image: repository: public.ecr.aws/braintrust/loop-runtime - tag: latest-2.x - pullPolicy: Always + # Pinned to the release tested with the Sandbox Service worker chart. + tag: b96d95eaa269031cdd6f286a17b9b3e4795355d1 + pullPolicy: IfNotPresent service: # Optional name for the Service. If empty, loopRuntime.name is used. name: "" @@ -335,9 +337,7 @@ loopRuntime: portName: http serviceAccount: name: "braintrust-loop-runtime" - # AWS IRSA role. Leave empty when using EKS Pod Identity. - awsRoleArn: "" - # Required on Google: the Google service account bound to this workload. + # Required: Terraform output loop_runtime_service_account. googleServiceAccount: "" annotations: {} # These URLs default to in-cluster services. Set them when the API, AI proxy, @@ -354,11 +354,11 @@ loopRuntime: cacheDir: "/mnt/tmp/brainstore" resources: requests: - cpu: "2" - memory: "8Gi" + cpu: "1" + memory: "2Gi" limits: - cpu: "2" - memory: "8Gi" + cpu: "1" + memory: "2Gi" podSecurityContext: runAsNonRoot: true runAsUser: 1000 @@ -390,35 +390,27 @@ loopRuntime: nodeSelector: {} tolerations: [] affinity: {} - # Sandboxes run on Braintrust Sandbox Service workers, reached over mTLS. + # Sandboxes run on Sandbox Service workers in the isolated GKE cluster. sandbox: - # Set one. dnsUrl resolves every worker behind a headless DNS name. - # Workers may run in this cluster or on a separately managed cluster. + # Required: Terraform output gke_isolated_workers.discovery.endpoint. dnsUrl: "" - urls: [] - image: "public.ecr.aws/braintrust/codex-runtime:latest" + image: "public.ecr.aws/braintrust/codex-runtime@sha256:35db9c98d4a74e20758a29b82f8a3a6552be0002eb439c8c1a0f434ac573ee21" # Resources requested for each sandbox; size worker capacity accordingly. vcpuCount: 1 memoryMib: 2048 - # restricted or internet - egressMode: "restricted" - tls: - # Secret with the client certificate, key, and server CA. - # Required for https:// endpoints. - secretName: "" - certKey: client.pem - keyKey: client.key - caKey: server-ca.pem - # Forward proxy that sandboxes use for Git access. An empty url defaults to - # the in-cluster -egress Service. Override it for external workers. + # Secret with client.pem, client.key, and server-ca.pem. + # examples/gke-loop-runtime/create-sandbox-tls.sh creates it. + tlsSecretName: sandbox-client-tls + # Git forward proxy for sandboxes, on an internal load balancer that the + # workers reach. Both values come from Terraform output gke_isolated_workers. egressGateway: - enabled: false port: 4002 + # Required: reserved internal address (loop_egress.address). + address: "" + # Required: worker ranges allowed to connect (loop_egress.source_ranges). + sourceRanges: [] + # Required: loop_egress.url, http://:4002. url: "" - service: - type: ClusterIP - annotations: {} - loadBalancerSourceRanges: [] extraEnvVars: [] extraContainers: [] extraVolumes: [] From e002b6de37252e4cef8fba3ea2ab8bc22ee73d01 Mon Sep 17 00:00:00 2001 From: Alex R Date: Mon, 26 Oct 2026 05:07:11 +0000 Subject: [PATCH 09/25] Prepare GKE worker nodes for the Sandbox Service chart The worker chart expects node-local storage at /var/lib/braintrust-sandboxes/1, /run/netns, and /dev/kvm. EKS node bootstrap provides these; GKE node pools have no bootstrap hook, so the example adds a node-init DaemonSet that formats and mounts the raw local SSDs (striping them when there are several) in the host mount namespace. Co-Authored-By: Claude Opus 5.5 --- .../examples/gke-loop-runtime/README.md | 28 ++++- .../examples/gke-loop-runtime/node-init.yaml | 102 ++++++++++++++++++ 2 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 braintrust/examples/gke-loop-runtime/node-init.yaml diff --git a/braintrust/examples/gke-loop-runtime/README.md b/braintrust/examples/gke-loop-runtime/README.md index e4a1560..cb5dbc5 100644 --- a/braintrust/examples/gke-loop-runtime/README.md +++ b/braintrust/examples/gke-loop-runtime/README.md @@ -13,12 +13,34 @@ when `gke_isolated_workers` is set with `discovery`. | Loop Runtime, worker controller | Workers | TCP 9400, mTLS | `discovery.endpoint` | | Sandboxes (Git) | Loop Runtime forward proxy | TCP 4002 | `loop_egress.url`, an internal load balancer on a reserved address | +## Terraform + +Enable the isolated worker cluster and its discovery zone in the GCP module: + +```hcl +gke_isolated_workers = { + network_cidr = "10.64.0.0/16" # must not overlap any existing range + discovery = { + dns_name = "sandbox..internal" + } +} +``` + ## Install All `<...>` values come from `terraform output gke_isolated_workers` and `terraform output loop_runtime_service_account`. -1. Create the mutual TLS Secrets in both clusters. The script refuses to +1. Prepare the worker nodes. GKE node pools have no bootstrap hook, so this + DaemonSet mounts the local SSD at the path the worker chart expects, + creates `/run/netns`, and checks `/dev/kvm`. + + ```sh + kubectl --context create namespace braintrust-sandbox + kubectl --context apply -f braintrust/examples/gke-loop-runtime/node-init.yaml + ``` + +2. Create the mutual TLS Secrets in both clusters. The script refuses to replace existing Secrets. ```sh @@ -26,7 +48,7 @@ All `<...>` values come from `terraform output gke_isolated_workers` and "" ``` -2. Install the workers in the isolated cluster, using the worker chart that +3. Install the workers in the isolated cluster, using the worker chart that matches the pinned images. ```sh @@ -36,7 +58,7 @@ All `<...>` values come from `terraform output gke_isolated_workers` and --values braintrust/examples/gke-loop-runtime/sandbox-service.yaml --wait ``` -3. Install or upgrade this chart in the primary cluster with your existing GKE +4. Install or upgrade this chart in the primary cluster with your existing GKE values plus `values.yaml` from this directory. ```sh diff --git a/braintrust/examples/gke-loop-runtime/node-init.yaml b/braintrust/examples/gke-loop-runtime/node-init.yaml new file mode 100644 index 0000000..9d35fea --- /dev/null +++ b/braintrust/examples/gke-loop-runtime/node-init.yaml @@ -0,0 +1,102 @@ +# Prepares isolated worker nodes for the Sandbox Service worker chart, which +# expects node-local storage at /var/lib/braintrust-sandboxes/1, /run/netns, and +# /dev/kvm. On EKS the node bootstrap does this; GKE node pools have no +# bootstrap hook, so this DaemonSet runs the same steps in the host namespaces. +# Apply it to the isolated worker cluster before installing the worker release. +apiVersion: v1 +kind: ConfigMap +metadata: + name: sandbox-node-init + namespace: braintrust-sandbox +data: + init.sh: | + #!/bin/bash + # Formats and mounts the raw local SSDs once per boot. Local SSD contents + # do not survive node recreation, so there is no data to preserve. + set -euo pipefail + MOUNT="${MOUNT:-/var/lib/braintrust-sandboxes/1}" + DEVICES="${DEVICES:-/dev/disk/by-id/google-local-nvme-ssd-*}" + mkdir -p /run/netns "$MOUNT" + modprobe kvm_intel 2>/dev/null || modprobe kvm_amd 2>/dev/null || true + test -c /dev/kvm || { echo "/dev/kvm is missing; enable nested virtualization" >&2; exit 1; } + if mountpoint -q "$MOUNT"; then + echo "$MOUNT is already mounted" + exit 0 + fi + # shellcheck disable=SC2206 + disks=($DEVICES) + [ -e "${disks[0]}" ] || { echo "no local SSD matches $DEVICES" >&2; exit 1; } + if [ "${#disks[@]}" -eq 1 ]; then + device="$(readlink -f "${disks[0]}")" + else + # Reuse an array the kernel assembled at boot; otherwise stripe the SSDs. + first="$(basename "$(readlink -f "${disks[0]}")")" + device="" + for slaves in /sys/block/md*/slaves; do + [ -e "$slaves/$first" ] && device="/dev/$(basename "$(dirname "$slaves")")" + done + if [ -z "$device" ]; then + device=/dev/md0 + mdadm --create "$device" --level=0 --force --run \ + --raid-devices="${#disks[@]}" "${disks[@]}" + fi + fi + blkid "$device" >/dev/null 2>&1 || mkfs.ext4 -q -F "$device" + mount -o discard,defaults "$device" "$MOUNT" + echo "mounted $device at $MOUNT" +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: sandbox-node-init + namespace: braintrust-sandbox +spec: + selector: + matchLabels: + app: sandbox-node-init + template: + metadata: + labels: + app: sandbox-node-init + spec: + nodeSelector: + braintrust.io/workload: sandbox-service + tolerations: + - key: braintrust.io/sandbox + operator: Equal + value: "true" + effect: NoSchedule + hostPID: true + automountServiceAccountToken: false + initContainers: + - name: init + image: ubuntu:24.04 + # Run the script with the host's tools in the host mount namespace so + # kubelet and the worker Pod see the mount. + command: + - nsenter + - --target=1 + - --mount + - --uts + - --ipc + - --net + - --pid + - -- + - bash + - -c + - "$(SCRIPT)" + env: + - name: SCRIPT + valueFrom: + configMapKeyRef: + name: sandbox-node-init + key: init.sh + securityContext: + privileged: true + containers: + - name: pause + image: registry.k8s.io/pause:3.10 + resources: + requests: + cpu: 10m + memory: 16Mi From be9d997394d802cffb85069f47f96b025a8c25a1 Mon Sep 17 00:00:00 2001 From: Alex R Date: Mon, 26 Oct 2026 05:52:11 +0000 Subject: [PATCH 10/25] Address Loop Runtime review feedback - Give pods 120s to stop and set the runtime's 90s drain explicitly - Pass FUNCTION_SECRET_KEY so stored MCP credentials decrypt - Render ORG_NAME as * when orgName is empty, as the API does - Bound the Brainstore file cache, size-limit the state and cache emptyDirs, and request matching ephemeral storage - Reject blank, null, padded, and non-CIDR egress source ranges - Node init: mount the SSD at the parent and create the worker directory on it, so workers cannot start before the mount - TLS script: check both clusters first and write the worker server Secret last, so a failed run can be retried - Note the dependency on terraform-google-braintrust-data-plane#28 Co-Authored-By: Claude Opus 5.5 --- braintrust/README.md | 6 ++ .../gke-loop-runtime/create-sandbox-tls.sh | 14 ++-- .../examples/gke-loop-runtime/node-init.yaml | 52 +++++++-------- braintrust/templates/_helpers.tpl | 5 ++ .../templates/loop-runtime-configmap.yaml | 5 +- .../templates/loop-runtime-deployment.yaml | 13 +++- .../tests/loop-runtime-configmap_test.yaml | 64 +++++++++++++++++++ .../tests/loop-runtime-deployment_test.yaml | 32 ++++++++++ braintrust/values.yaml | 9 +++ 9 files changed, 168 insertions(+), 32 deletions(-) diff --git a/braintrust/README.md b/braintrust/README.md index 86d2239..c2593d7 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -27,6 +27,12 @@ workers in a separate isolated GKE cluster. Follow the [Loop Runtime and Sandbox Service example](examples/gke-loop-runtime/README.md) to create the TLS Secrets and install the worker release. +The Terraform outputs this needs (`loop_runtime_service_account` and +`gke_isolated_workers`) are not on `main` of +`terraform-google-braintrust-data-plane` yet. They come from +[terraform-google-braintrust-data-plane#28](https://github.com/braintrustdata/terraform-google-braintrust-data-plane/pull/28), +which is stacked on the GKE Standard work in that repository. + The runtime reads `braintrust-secrets` and the Brainstore buckets. The chart adds `LOOP_RUNTIME_URL` to the API and, when `virtualService.enabled` is true, routes `/loop/runtime` to Loop Runtime. It also creates the `-egress` diff --git a/braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh b/braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh index f8fb22c..2e1a528 100755 --- a/braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh +++ b/braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh @@ -20,6 +20,12 @@ WORKER_NAMESPACE="${WORKER_NAMESPACE:-braintrust-sandbox}" RUNTIME_NAMESPACE="${RUNTIME_NAMESPACE:-braintrust}" DAYS=3650 +# Fail before writing anything if either cluster is unreachable. +for context in "$WORKER_CONTEXT" "$RUNTIME_CONTEXT"; do + kubectl --context "$context" get namespaces >/dev/null +done +# sandbox-tls is written last, so it marks a completed run. A partial run +# leaves only client Secrets, which a rerun replaces. if kubectl --context "$WORKER_CONTEXT" -n "$WORKER_NAMESPACE" get secret sandbox-tls >/dev/null 2>&1; then echo "sandbox-tls already exists in $WORKER_CONTEXT/$WORKER_NAMESPACE; refusing to replace it." >&2 exit 1 @@ -59,12 +65,12 @@ apply_secret() { kubectl --context "$1" apply -f - } -# Workers present the server certificate and trust the client CA. -apply_secret "$WORKER_CONTEXT" "$WORKER_NAMESPACE" sandbox-tls \ - --from-file=cert.pem=server.pem --from-file=key.pem=server.key --from-file=ca_cert.pem=client-ca.pem # The worker controller and Loop Runtime present the client certificate and trust the server CA. -for target in "$WORKER_CONTEXT $WORKER_NAMESPACE" "$RUNTIME_CONTEXT $RUNTIME_NAMESPACE"; do +for target in "$RUNTIME_CONTEXT $RUNTIME_NAMESPACE" "$WORKER_CONTEXT $WORKER_NAMESPACE"; do # shellcheck disable=SC2086 apply_secret $target sandbox-client-tls \ --from-file=client.pem=client.pem --from-file=client.key=client.key --from-file=server-ca.pem=server-ca.pem done +# Workers present the server certificate and trust the client CA. +apply_secret "$WORKER_CONTEXT" "$WORKER_NAMESPACE" sandbox-tls \ + --from-file=cert.pem=server.pem --from-file=key.pem=server.key --from-file=ca_cert.pem=client-ca.pem diff --git a/braintrust/examples/gke-loop-runtime/node-init.yaml b/braintrust/examples/gke-loop-runtime/node-init.yaml index 9d35fea..af4d4f5 100644 --- a/braintrust/examples/gke-loop-runtime/node-init.yaml +++ b/braintrust/examples/gke-loop-runtime/node-init.yaml @@ -13,37 +13,39 @@ data: #!/bin/bash # Formats and mounts the raw local SSDs once per boot. Local SSD contents # do not survive node recreation, so there is no data to preserve. + # The worker's hostPath directory is created on the mounted SSD, so a + # worker cannot start until the mount exists (the same layout as EKS). set -euo pipefail - MOUNT="${MOUNT:-/var/lib/braintrust-sandboxes/1}" + ROOT="${ROOT:-/var/lib/braintrust-sandboxes}" DEVICES="${DEVICES:-/dev/disk/by-id/google-local-nvme-ssd-*}" - mkdir -p /run/netns "$MOUNT" + mkdir -p /run/netns modprobe kvm_intel 2>/dev/null || modprobe kvm_amd 2>/dev/null || true test -c /dev/kvm || { echo "/dev/kvm is missing; enable nested virtualization" >&2; exit 1; } - if mountpoint -q "$MOUNT"; then - echo "$MOUNT is already mounted" - exit 0 - fi - # shellcheck disable=SC2206 - disks=($DEVICES) - [ -e "${disks[0]}" ] || { echo "no local SSD matches $DEVICES" >&2; exit 1; } - if [ "${#disks[@]}" -eq 1 ]; then - device="$(readlink -f "${disks[0]}")" - else - # Reuse an array the kernel assembled at boot; otherwise stripe the SSDs. - first="$(basename "$(readlink -f "${disks[0]}")")" - device="" - for slaves in /sys/block/md*/slaves; do - [ -e "$slaves/$first" ] && device="/dev/$(basename "$(dirname "$slaves")")" - done - if [ -z "$device" ]; then - device=/dev/md0 - mdadm --create "$device" --level=0 --force --run \ - --raid-devices="${#disks[@]}" "${disks[@]}" + if ! mountpoint -q "$ROOT"; then + # shellcheck disable=SC2206 + disks=($DEVICES) + [ -e "${disks[0]}" ] || { echo "no local SSD matches $DEVICES" >&2; exit 1; } + if [ "${#disks[@]}" -eq 1 ]; then + device="$(readlink -f "${disks[0]}")" + else + # Reuse an array the kernel assembled at boot; otherwise stripe the SSDs. + first="$(basename "$(readlink -f "${disks[0]}")")" + device="" + for slaves in /sys/block/md*/slaves; do + if [ -e "$slaves/$first" ]; then device="/dev/$(basename "$(dirname "$slaves")")"; fi + done + if [ -z "$device" ]; then + device=/dev/md0 + mdadm --create "$device" --level=0 --force --run \ + --raid-devices="${#disks[@]}" "${disks[@]}" + fi fi + blkid "$device" >/dev/null 2>&1 || mkfs.ext4 -q -F "$device" + mkdir -p "$ROOT" + mount -o discard,defaults "$device" "$ROOT" + echo "mounted $device at $ROOT" fi - blkid "$device" >/dev/null 2>&1 || mkfs.ext4 -q -F "$device" - mount -o discard,defaults "$device" "$MOUNT" - echo "mounted $device at $MOUNT" + mkdir -p "$ROOT/1" --- apiVersion: apps/v1 kind: DaemonSet diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 6c3ce84..61f27ef 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -178,6 +178,11 @@ predates Loop Runtime have no loopRuntime map, which means disabled. {{- if not $sandbox.egressGateway.sourceRanges -}} {{- fail "loopRuntime.sandbox.egressGateway.sourceRanges is required (Terraform output gke_isolated_workers.loop_egress.source_ranges)" -}} {{- end -}} +{{- range $sandbox.egressGateway.sourceRanges -}} +{{- if not (regexMatch "^[0-9]{1,3}(\\.[0-9]{1,3}){3}/[0-9]{1,2}$" (toString .)) -}} +{{- fail (printf "loopRuntime.sandbox.egressGateway.sourceRanges entries must be IPv4 CIDR ranges, got %q" (toString .)) -}} +{{- end -}} +{{- end -}} {{- end -}} {{- end -}} diff --git a/braintrust/templates/loop-runtime-configmap.yaml b/braintrust/templates/loop-runtime-configmap.yaml index 6d3a9bd..605722f 100644 --- a/braintrust/templates/loop-runtime-configmap.yaml +++ b/braintrust/templates/loop-runtime-configmap.yaml @@ -25,7 +25,9 @@ data: LOOP_RUNTIME_LIFECYCLE_LOG: {{ .Values.loopRuntime.lifecycleLog | toString | quote }} LOOP_RUNTIME_REDIS_NAMESPACE: {{ include "braintrust.loopRuntimeRedisNamespace" . | quote }} LOOP_RUNTIME_STATE_DIR: {{ .Values.loopRuntime.stateDir | quote }} - ORG_NAME: {{ .Values.global.orgName | default "" | toString | trim | quote }} + {{- /* An empty orgName serves every org, matching the API. The runtime refuses an empty ORG_NAME. */}} + ORG_NAME: {{ .Values.global.orgName | default "" | toString | trim | default "*" | quote }} + LOOP_RUNTIME_DRAIN_TIMEOUT_SECONDS: "90" BRAINTRUST_API_URL: {{ $apiUrl | quote }} LOOP_RUNTIME_AI_PROXY_URL: {{ $aiProxyUrl | quote }} LOOP_RUNTIME_BRAINSTORE_READER_URL: {{ $brainstoreReaderUrl | quote }} @@ -34,6 +36,7 @@ data: BRAINSTORE_REALTIME_WAL_URI: "{{ $brainstoreBucketUri }}/brainstore/wal" BRAINSTORE_CODE_BUNDLE_URI: "gs://{{ .Values.objectStorage.google.apiBucket }}" BRAINSTORE_CACHE_DIR: {{ .Values.loopRuntime.cacheDir | quote }} + BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE: {{ .Values.loopRuntime.objectStoreCacheFileSize | quote }} BRAINSTORE_CONTROL_PLANE_TELEMETRY: {{ .Values.global.controlPlaneTelemetry | quote }} BRAINSTORE_DISABLE_STATUS_UPDATES: {{ .Values.loopRuntime.brainstoreDisableStatusUpdates | toString | quote }} NO_COLOR: "1" diff --git a/braintrust/templates/loop-runtime-deployment.yaml b/braintrust/templates/loop-runtime-deployment.yaml index 6716afd..8d91423 100644 --- a/braintrust/templates/loop-runtime-deployment.yaml +++ b/braintrust/templates/loop-runtime-deployment.yaml @@ -39,6 +39,7 @@ spec: {{- end }} spec: serviceAccountName: {{ .Values.loopRuntime.serviceAccount.name }} + terminationGracePeriodSeconds: {{ .Values.loopRuntime.terminationGracePeriodSeconds }} {{- with .Values.loopRuntime.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} @@ -116,6 +117,12 @@ spec: secretKeyRef: name: braintrust-secrets key: FUNCTION_SECRET_KEY + # Decrypts stored project MCP credentials. + - name: FUNCTION_SECRET_KEY + valueFrom: + secretKeyRef: + name: braintrust-secrets + key: FUNCTION_SECRET_KEY - name: BRAINSTORE_LICENSE_KEY valueFrom: secretKeyRef: @@ -144,9 +151,11 @@ spec: {{- end }} volumes: - name: state-volume - emptyDir: {} + emptyDir: + sizeLimit: {{ .Values.loopRuntime.stateSizeLimit | quote }} - name: cache-volume - emptyDir: {} + emptyDir: + sizeLimit: {{ .Values.loopRuntime.cacheSizeLimit | quote }} - name: sandbox-service-tls secret: secretName: {{ $sandbox.tlsSecretName }} diff --git a/braintrust/tests/loop-runtime-configmap_test.yaml b/braintrust/tests/loop-runtime-configmap_test.yaml index 1085297..0f51edb 100644 --- a/braintrust/tests/loop-runtime-configmap_test.yaml +++ b/braintrust/tests/loop-runtime-configmap_test.yaml @@ -199,3 +199,67 @@ tests: asserts: - failedTemplate: errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges is required (Terraform output gke_isolated_workers.loop_egress.source_ranges)" + + - it: should serve every org when orgName is empty + values: *values + set: + global.orgName: "" + asserts: + - equal: + path: data.ORG_NAME + value: "*" + + - it: should serve every org when orgName is whitespace + values: *values + set: + global.orgName: " " + asserts: + - equal: + path: data.ORG_NAME + value: "*" + + - it: should keep a configured orgName and set the drain and cache budgets + values: *values + set: + global.orgName: acme + asserts: + - equal: + path: data.ORG_NAME + value: acme + - equal: + path: data.LOOP_RUNTIME_DRAIN_TIMEOUT_SECONDS + value: "90" + - equal: + path: data.BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE + value: 5Gi + + - it: should reject a blank egress source range + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: [""] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges entries must be IPv4 CIDR ranges, got \"\"" + + - it: should reject a null egress source range + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: [null] + asserts: + - failedTemplate: {} + + - it: should reject a blank entry alongside a valid egress source range + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: ["10.64.64.0/24", " "] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges entries must be IPv4 CIDR ranges, got \" \"" + + - it: should reject a padded egress source range + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: [" 10.64.64.0/24"] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges entries must be IPv4 CIDR ranges, got \" 10.64.64.0/24\"" diff --git a/braintrust/tests/loop-runtime-deployment_test.yaml b/braintrust/tests/loop-runtime-deployment_test.yaml index c52c173..2f4332d 100644 --- a/braintrust/tests/loop-runtime-deployment_test.yaml +++ b/braintrust/tests/loop-runtime-deployment_test.yaml @@ -53,6 +53,7 @@ tests: value: cpu: "1" memory: 2Gi + ephemeral-storage: 9Gi - contains: path: spec.template.spec.containers[0].env content: @@ -234,3 +235,34 @@ tests: - equal: path: spec.template.spec.containers[0].imagePullPolicy value: Always + + - it: should outlast the runtime drain, pass the function secret, and bound local storage + template: loop-runtime-deployment.yaml + values: *values + asserts: + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 120 + - contains: + path: spec.template.spec.containers[0].env + content: + name: FUNCTION_SECRET_KEY + valueFrom: + secretKeyRef: + name: braintrust-secrets + key: FUNCTION_SECRET_KEY + - equal: + path: spec.template.spec.volumes[0] + value: + name: state-volume + emptyDir: + sizeLimit: 2Gi + - equal: + path: spec.template.spec.volumes[1] + value: + name: cache-volume + emptyDir: + sizeLimit: 6Gi + - equal: + path: spec.template.spec.containers[0].resources.requests["ephemeral-storage"] + value: 9Gi diff --git a/braintrust/values.yaml b/braintrust/values.yaml index b4fb6f3..86c3bdb 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -352,13 +352,22 @@ loopRuntime: brainstoreDisableStatusUpdates: "false" stateDir: "/mnt/tmp/loop-runtime" cacheDir: "/mnt/tmp/brainstore" + # Bounds the Brainstore file cache in cacheDir. Keep it below cacheSizeLimit. + objectStoreCacheFileSize: "5Gi" + stateSizeLimit: "2Gi" + cacheSizeLimit: "6Gi" + # On SIGTERM the runtime drains active turns for up to 90 seconds. + terminationGracePeriodSeconds: 120 resources: requests: cpu: "1" memory: "2Gi" + # State and cache emptyDirs plus container overhead. + ephemeral-storage: "9Gi" limits: cpu: "1" memory: "2Gi" + ephemeral-storage: "9Gi" podSecurityContext: runAsNonRoot: true runAsUser: 1000 From 5545930f30df9c1205bcd18558733fc54d132e98 Mon Sep 17 00:00:00 2001 From: Alex R Date: Mon, 26 Oct 2026 05:55:04 +0000 Subject: [PATCH 11/25] Require the Loop Runtime image repository and tag An empty, null, or missing repository or tag rendered an invalid image reference that only failed when pods started. Co-Authored-By: Claude Opus 5.5 --- braintrust/templates/_helpers.tpl | 2 + .../partial-loop-runtime-image-values.yaml | 5 +++ .../tests/loop-runtime-configmap_test.yaml | 41 +++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 braintrust/tests/__fixtures__/partial-loop-runtime-image-values.yaml diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index 61f27ef..a218e76 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -166,6 +166,8 @@ predates Loop Runtime have no loopRuntime map, which means disabled. {{- fail "loopRuntime.enabled requires cloud: google. On AWS, Loop Runtime runs on ECS through the Terraform module." -}} {{- end -}} {{- $sandbox := .Values.loopRuntime.sandbox -}} +{{- $_ := required "loopRuntime.image.repository is required when loopRuntime.enabled is true" .Values.loopRuntime.image.repository -}} +{{- $_ := required "loopRuntime.image.tag is required when loopRuntime.enabled is true" (.Values.loopRuntime.image.tag | default "" | toString | trim) -}} {{- $_ := required "objectStorage.google.brainstoreBucket is required when loopRuntime.enabled is true" .Values.objectStorage.google.brainstoreBucket -}} {{- $_ := required "objectStorage.google.apiBucket is required when loopRuntime.enabled is true" .Values.objectStorage.google.apiBucket -}} {{- $_ := required "loopRuntime.serviceAccount.name is required when loopRuntime.enabled is true" .Values.loopRuntime.serviceAccount.name -}} diff --git a/braintrust/tests/__fixtures__/partial-loop-runtime-image-values.yaml b/braintrust/tests/__fixtures__/partial-loop-runtime-image-values.yaml new file mode 100644 index 0000000..231fa7a --- /dev/null +++ b/braintrust/tests/__fixtures__/partial-loop-runtime-image-values.yaml @@ -0,0 +1,5 @@ +# Reused values whose image map lacks a tag. +loopRuntime: + image: + repository: "test/loop-runtime" + tag: null diff --git a/braintrust/tests/loop-runtime-configmap_test.yaml b/braintrust/tests/loop-runtime-configmap_test.yaml index 0f51edb..480831f 100644 --- a/braintrust/tests/loop-runtime-configmap_test.yaml +++ b/braintrust/tests/loop-runtime-configmap_test.yaml @@ -263,3 +263,44 @@ tests: asserts: - failedTemplate: errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges entries must be IPv4 CIDR ranges, got \" 10.64.64.0/24\"" + + - it: should require the runtime image repository when it is empty + values: *values + set: + loopRuntime.image.repository: "" + asserts: + - failedTemplate: + errorMessage: "loopRuntime.image.repository is required when loopRuntime.enabled is true" + + - it: should require the runtime image repository when it is null + values: *values + set: + loopRuntime.image.repository: null + asserts: + - failedTemplate: + errorMessage: "loopRuntime.image.repository is required when loopRuntime.enabled is true" + + - it: should require the runtime image tag when it is empty + values: *values + set: + loopRuntime.image.tag: "" + asserts: + - failedTemplate: + errorMessage: "loopRuntime.image.tag is required when loopRuntime.enabled is true" + + - it: should require the runtime image tag when it is null + values: *values + set: + loopRuntime.image.tag: null + asserts: + - failedTemplate: + errorMessage: "loopRuntime.image.tag is required when loopRuntime.enabled is true" + + - it: should require the runtime image tag when reused values omit it + values: + - __fixtures__/base-values.yaml + - __fixtures__/loop-runtime-values.yaml + - __fixtures__/partial-loop-runtime-image-values.yaml + asserts: + - failedTemplate: + errorMessage: "loopRuntime.image.tag is required when loopRuntime.enabled is true" From 1e15953c81da61a258397a43aa7c0a7cb87278d9 Mon Sep 17 00:00:00 2001 From: Alex R Date: Mon, 26 Oct 2026 06:02:46 +0000 Subject: [PATCH 12/25] Simplify Loop Runtime chart values, templates, and tests Model loopRuntime on the AI Gateway: the same value shapes, probes in values, and fixed defaults for settings with one correct value (state and cache paths and limits, security contexts, the 120s grace period, the TLS Secret name, the Git proxy port, in-cluster service URLs). Drop knobs no install needs and the tests that only covered them, and shorten the docs. Co-Authored-By: Claude Opus 5.5 --- braintrust/README.md | 21 +- .../examples/gke-loop-runtime/README.md | 85 ++----- .../gke-loop-runtime/create-sandbox-tls.sh | 11 +- .../examples/gke-loop-runtime/node-init.yaml | 17 +- braintrust/templates/_helpers.tpl | 41 +--- .../templates/loop-runtime-configmap.yaml | 43 ++-- .../templates/loop-runtime-deployment.yaml | 72 +++--- .../loop-runtime-egress-service.yaml | 12 +- .../loop-runtime-serviceaccount.yaml | 10 +- .../partial-loop-runtime-image-values.yaml | 5 - .../reuse-loop-runtime-enabled-values.yaml | 30 +-- .../tests/labels-merge-services_test.yaml | 51 +--- .../tests/loop-runtime-configmap_test.yaml | 232 +++--------------- .../tests/loop-runtime-deployment_test.yaml | 195 +-------------- .../loop-runtime-egress-service_test.yaml | 39 +-- .../tests/loop-runtime-service_test.yaml | 17 -- .../loop-runtime-serviceaccount_test.yaml | 20 +- braintrust/values.yaml | 72 ++---- 18 files changed, 176 insertions(+), 797 deletions(-) delete mode 100644 braintrust/tests/__fixtures__/partial-loop-runtime-image-values.yaml diff --git a/braintrust/README.md b/braintrust/README.md index c2593d7..2237d63 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -21,23 +21,12 @@ The `braintrust-secrets` secret must contain the following keys: ## Optional: Loop Runtime Set `loopRuntime.enabled: true` to run Loop Runtime on GKE (`cloud: google`). -On AWS, Loop Runtime runs on ECS through the AWS Terraform module, so this -chart rejects other clouds. Loop Runtime runs sandboxes on Sandbox Service -workers in a separate isolated GKE cluster. Follow the -[Loop Runtime and Sandbox Service example](examples/gke-loop-runtime/README.md) -to create the TLS Secrets and install the worker release. - -The Terraform outputs this needs (`loop_runtime_service_account` and -`gke_isolated_workers`) are not on `main` of -`terraform-google-braintrust-data-plane` yet. They come from +On AWS it runs on ECS through the Terraform module. Sandboxes run on Sandbox +Service workers in a separate GKE cluster; see +[examples/gke-loop-runtime](examples/gke-loop-runtime/README.md). The Terraform +outputs it needs come from [terraform-google-braintrust-data-plane#28](https://github.com/braintrustdata/terraform-google-braintrust-data-plane/pull/28), -which is stacked on the GKE Standard work in that repository. - -The runtime reads `braintrust-secrets` and the Brainstore buckets. The chart -adds `LOOP_RUNTIME_URL` to the API and, when `virtualService.enabled` is true, -routes `/loop/runtime` to Loop Runtime. It also creates the `-egress` -internal load balancer on the address that Terraform reserves, so sandboxes -can reach Git through Loop Runtime's forward proxy. +which is not merged yet. ## Azure Key Vault Driver Integration diff --git a/braintrust/examples/gke-loop-runtime/README.md b/braintrust/examples/gke-loop-runtime/README.md index cb5dbc5..c4775d7 100644 --- a/braintrust/examples/gke-loop-runtime/README.md +++ b/braintrust/examples/gke-loop-runtime/README.md @@ -1,88 +1,51 @@ -# Loop Runtime and Sandbox Service on GKE +# Loop Runtime on GKE -Loop Runtime runs in the primary GKE cluster with this chart. Its sandboxes run -on Sandbox Service workers in a separate isolated GKE cluster, installed from -the published worker chart. The GCP Terraform module creates both clusters, the -worker node pool, private DNS, firewall rules, and the Loop Runtime identity -when `gke_isolated_workers` is set with `discovery`. - -## Connections - -| From | To | Port | Address | -| --- | --- | --- | --- | -| Loop Runtime, worker controller | Workers | TCP 9400, mTLS | `discovery.endpoint` | -| Sandboxes (Git) | Loop Runtime forward proxy | TCP 4002 | `loop_egress.url`, an internal load balancer on a reserved address | - -## Terraform - -Enable the isolated worker cluster and its discovery zone in the GCP module: +Loop Runtime runs in the primary GKE cluster. Its sandboxes run on Sandbox +Service workers in a separate isolated cluster. Both clusters come from the +GCP Terraform module with: ```hcl gke_isolated_workers = { - network_cidr = "10.64.0.0/16" # must not overlap any existing range - discovery = { - dns_name = "sandbox..internal" - } + network_cidr = "10.64.0.0/16" # must not overlap existing ranges + discovery = { dns_name = "sandbox..internal" } } ``` -## Install - -All `<...>` values come from `terraform output gke_isolated_workers` and +`<...>` values below come from `terraform output gke_isolated_workers` and `terraform output loop_runtime_service_account`. -1. Prepare the worker nodes. GKE node pools have no bootstrap hook, so this - DaemonSet mounts the local SSD at the path the worker chart expects, - creates `/run/netns`, and checks `/dev/kvm`. +1. Prepare worker nodes (mounts the local SSD the worker chart expects): ```sh - kubectl --context create namespace braintrust-sandbox - kubectl --context apply -f braintrust/examples/gke-loop-runtime/node-init.yaml + kubectl --context create namespace braintrust-sandbox + kubectl --context apply -f braintrust/examples/gke-loop-runtime/node-init.yaml ``` -2. Create the mutual TLS Secrets in both clusters. The script refuses to - replace existing Secrets. +2. Create the mTLS Secrets in both clusters: ```sh - braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh \ - "" + braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh "" ``` -3. Install the workers in the isolated cluster, using the worker chart that - matches the pinned images. +3. Install the workers: ```sh - helm --kube-context upgrade --install sandbox-service \ + helm --kube-context upgrade --install sandbox-service \ https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/cd27a3983e304932a7e1617adf3105c2b7320a340f26bfea577df781827142f9/sandbox-service-0.1.0.tgz \ - --namespace braintrust-sandbox \ - --values braintrust/examples/gke-loop-runtime/sandbox-service.yaml --wait + --namespace braintrust-sandbox --values braintrust/examples/gke-loop-runtime/sandbox-service.yaml --wait ``` -4. Install or upgrade this chart in the primary cluster with your existing GKE - values plus `values.yaml` from this directory. +4. Install this chart with your GKE values plus `values.yaml` from this directory: ```sh - helm --kube-context upgrade --install braintrust ./braintrust \ - --namespace braintrust \ - --values \ - --values braintrust/examples/gke-loop-runtime/values.yaml --wait + helm --kube-context upgrade --install braintrust ./braintrust --namespace braintrust \ + --values --values braintrust/examples/gke-loop-runtime/values.yaml --wait ``` -## Capacity - -Workers run at a fixed count (`replicas: 1`). One 8-vCPU worker node holds about -seven 1 vCPU / 2 GiB sandboxes. When every slot is taken, new sandboxes fail -with `CapacityExhausted`. Raise `replicas` in `sandbox-service.yaml` to add -workers. The worker node pool autoscales up to its Terraform maximum and keeps -room for one extra worker during rollouts. - -## Upgrade and shutdown +Workers run at a fixed count. One `c4-standard-8-lssd` worker holds about seven +1 vCPU / 2 GiB sandboxes; beyond that, new sandboxes fail with +`CapacityExhausted`. Raise `replicas` in `sandbox-service.yaml` to add workers. -Upgrade both releases with the same values, and keep the pinned image tags in -`values.yaml` (chart defaults) and `sandbox-service.yaml` together. The worker -controller owns scale-down and replacement; do not force-delete workers or -their nodes. To shut down, stop new Loop work, upgrade the worker release with -`replicas: 0`, wait for -`kubectl -n braintrust-sandbox get configmap sandbox-rollout -o yaml` to show -zero workers, then disable `loopRuntime.enabled` and uninstall the worker -release. Its pre-delete hook refuses to uninstall until workers drain. +To shut down, set the worker release to `replicas: 0`, wait for the workers to +drain, then disable `loopRuntime.enabled` and uninstall the worker release. Its +pre-delete hook refuses to uninstall until workers drain. diff --git a/braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh b/braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh index 2e1a528..1366e60 100755 --- a/braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh +++ b/braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh @@ -1,9 +1,6 @@ #!/bin/bash -# Creates the mutual TLS Secrets shared by Loop Runtime and Sandbox Service. -# Run once before installing either release. Private keys stay in Kubernetes -# Secrets and never enter Terraform state. The certificates match those the -# AWS Terraform module creates: separate server and client CAs, P-256 keys, -# ten-year validity. +# Creates the mTLS Secrets for Loop Runtime and Sandbox Service, matching the +# AWS module: separate server and client CAs, P-256 keys, ten-year validity. # # Usage: create-sandbox-tls.sh # worker hostname: Terraform output gke_isolated_workers.discovery.hostname @@ -20,12 +17,10 @@ WORKER_NAMESPACE="${WORKER_NAMESPACE:-braintrust-sandbox}" RUNTIME_NAMESPACE="${RUNTIME_NAMESPACE:-braintrust}" DAYS=3650 -# Fail before writing anything if either cluster is unreachable. for context in "$WORKER_CONTEXT" "$RUNTIME_CONTEXT"; do kubectl --context "$context" get namespaces >/dev/null done -# sandbox-tls is written last, so it marks a completed run. A partial run -# leaves only client Secrets, which a rerun replaces. +# sandbox-tls is written last, so a failed run can be rerun. if kubectl --context "$WORKER_CONTEXT" -n "$WORKER_NAMESPACE" get secret sandbox-tls >/dev/null 2>&1; then echo "sandbox-tls already exists in $WORKER_CONTEXT/$WORKER_NAMESPACE; refusing to replace it." >&2 exit 1 diff --git a/braintrust/examples/gke-loop-runtime/node-init.yaml b/braintrust/examples/gke-loop-runtime/node-init.yaml index af4d4f5..5cc41b8 100644 --- a/braintrust/examples/gke-loop-runtime/node-init.yaml +++ b/braintrust/examples/gke-loop-runtime/node-init.yaml @@ -1,8 +1,6 @@ -# Prepares isolated worker nodes for the Sandbox Service worker chart, which -# expects node-local storage at /var/lib/braintrust-sandboxes/1, /run/netns, and -# /dev/kvm. On EKS the node bootstrap does this; GKE node pools have no -# bootstrap hook, so this DaemonSet runs the same steps in the host namespaces. -# Apply it to the isolated worker cluster before installing the worker release. +# GKE node pools have no bootstrap hook, so this DaemonSet does what EKS node +# bootstrap does for the Sandbox Service worker chart: mount the local SSD at +# /var/lib/braintrust-sandboxes, create /run/netns, and check /dev/kvm. apiVersion: v1 kind: ConfigMap metadata: @@ -11,10 +9,8 @@ metadata: data: init.sh: | #!/bin/bash - # Formats and mounts the raw local SSDs once per boot. Local SSD contents - # do not survive node recreation, so there is no data to preserve. - # The worker's hostPath directory is created on the mounted SSD, so a - # worker cannot start until the mount exists (the same layout as EKS). + # The worker's hostPath directory (1) is created on the mounted SSD, so a + # worker cannot start before the mount. set -euo pipefail ROOT="${ROOT:-/var/lib/braintrust-sandboxes}" DEVICES="${DEVICES:-/dev/disk/by-id/google-local-nvme-ssd-*}" @@ -73,8 +69,7 @@ spec: initContainers: - name: init image: ubuntu:24.04 - # Run the script with the host's tools in the host mount namespace so - # kubelet and the worker Pod see the mount. + # Run in the host mount namespace so kubelet sees the mount. command: - nsenter - --target=1 diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index a218e76..c7224e8 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -157,32 +157,27 @@ predates Loop Runtime have no loopRuntime map, which means disabled. {{/* Validate the Loop Runtime configuration. */}} {{- define "braintrust.loopRuntime.validate" -}} {{- if include "braintrust.loopRuntime.enabled" . -}} -{{- range (list "image" "service" "serviceAccount" "sandbox") -}} -{{- if not (hasKey $.Values.loopRuntime .) -}} -{{- fail (printf "loopRuntime.%s is missing. Values reused from a chart release that predates Loop Runtime omit the loopRuntime defaults; upgrade with --reset-then-reuse-values or supply the full loopRuntime map." .) -}} -{{- end -}} +{{- if not (hasKey .Values.loopRuntime "sandbox") -}} +{{- fail "loopRuntime defaults are missing. Values reused from a chart release that predates Loop Runtime omit them; upgrade with --reset-then-reuse-values." -}} {{- end -}} {{- if ne .Values.cloud "google" -}} {{- fail "loopRuntime.enabled requires cloud: google. On AWS, Loop Runtime runs on ECS through the Terraform module." -}} {{- end -}} {{- $sandbox := .Values.loopRuntime.sandbox -}} -{{- $_ := required "loopRuntime.image.repository is required when loopRuntime.enabled is true" .Values.loopRuntime.image.repository -}} -{{- $_ := required "loopRuntime.image.tag is required when loopRuntime.enabled is true" (.Values.loopRuntime.image.tag | default "" | toString | trim) -}} -{{- $_ := required "objectStorage.google.brainstoreBucket is required when loopRuntime.enabled is true" .Values.objectStorage.google.brainstoreBucket -}} -{{- $_ := required "objectStorage.google.apiBucket is required when loopRuntime.enabled is true" .Values.objectStorage.google.apiBucket -}} -{{- $_ := required "loopRuntime.serviceAccount.name is required when loopRuntime.enabled is true" .Values.loopRuntime.serviceAccount.name -}} -{{- $_ := required "loopRuntime.serviceAccount.googleServiceAccount is required (Terraform output loop_runtime_service_account)" .Values.loopRuntime.serviceAccount.googleServiceAccount -}} -{{- $_ := required "loopRuntime.sandbox.dnsUrl is required (Terraform output gke_isolated_workers.discovery.endpoint)" $sandbox.dnsUrl -}} -{{- $_ := required "loopRuntime.sandbox.image is required when loopRuntime.enabled is true" $sandbox.image -}} -{{- $_ := required "loopRuntime.sandbox.tlsSecretName is required when loopRuntime.enabled is true" $sandbox.tlsSecretName -}} -{{- $_ := required "loopRuntime.sandbox.egressGateway.address is required (Terraform output gke_isolated_workers.loop_egress.address)" $sandbox.egressGateway.address -}} -{{- $_ := required "loopRuntime.sandbox.egressGateway.url is required (Terraform output gke_isolated_workers.loop_egress.url)" $sandbox.egressGateway.url -}} +{{- $_ := required "objectStorage.google.brainstoreBucket is required" .Values.objectStorage.google.brainstoreBucket -}} +{{- $_ := required "objectStorage.google.apiBucket is required" .Values.objectStorage.google.apiBucket -}} +{{- $_ := required "loopRuntime.image.repository is required" .Values.loopRuntime.image.repository -}} +{{- $_ := required "loopRuntime.image.tag is required" (.Values.loopRuntime.image.tag | default "" | toString) -}} +{{- $_ := required "loopRuntime.serviceAccount.googleServiceAccount is required" .Values.loopRuntime.serviceAccount.googleServiceAccount -}} +{{- $_ := required "loopRuntime.sandbox.dnsUrl is required" $sandbox.dnsUrl -}} +{{- $_ := required "loopRuntime.sandbox.egressGateway.address is required" $sandbox.egressGateway.address -}} +{{- $_ := required "loopRuntime.sandbox.egressGateway.url is required" $sandbox.egressGateway.url -}} {{- if not $sandbox.egressGateway.sourceRanges -}} -{{- fail "loopRuntime.sandbox.egressGateway.sourceRanges is required (Terraform output gke_isolated_workers.loop_egress.source_ranges)" -}} +{{- fail "loopRuntime.sandbox.egressGateway.sourceRanges is required" -}} {{- end -}} {{- range $sandbox.egressGateway.sourceRanges -}} -{{- if not (regexMatch "^[0-9]{1,3}(\\.[0-9]{1,3}){3}/[0-9]{1,2}$" (toString .)) -}} -{{- fail (printf "loopRuntime.sandbox.egressGateway.sourceRanges entries must be IPv4 CIDR ranges, got %q" (toString .)) -}} +{{- if not (regexMatch "^[0-9.]+/[0-9]+$" (toString .)) -}} +{{- fail (printf "loopRuntime.sandbox.egressGateway.sourceRanges: %q is not a CIDR range" (toString .)) -}} {{- end -}} {{- end -}} {{- end -}} @@ -193,16 +188,6 @@ predates Loop Runtime have no loopRuntime map, which means disabled. http://{{ .Values.loopRuntime.service.name | default .Values.loopRuntime.name }}.{{ include "braintrust.namespace" . }}:{{ .Values.loopRuntime.service.port }} {{- end -}} -{{/* Stable Redis key namespace for a Helm-managed Loop Runtime. */}} -{{- define "braintrust.loopRuntimeRedisNamespace" -}} -{{- .Values.loopRuntime.redisNamespace | default (printf "loop-runtime:%s" .Release.Name) -}} -{{- end -}} - -{{/* Internal cluster URL for the Brainstore fast reader used by Loop Runtime. */}} -{{- define "braintrust.brainstoreFastReaderInternalUrl" -}} -http://{{ .Values.brainstore.fastreader.service.name | default .Values.brainstore.fastreader.name }}.{{ include "braintrust.namespace" . }}:{{ .Values.brainstore.fastreader.service.port }} -{{- end -}} - {{/* Product-owned Loop Runtime routes for an Istio VirtualService. */}} {{- define "braintrust.loopRuntimeVirtualServiceRoutes" -}} - match: diff --git a/braintrust/templates/loop-runtime-configmap.yaml b/braintrust/templates/loop-runtime-configmap.yaml index 605722f..8046148 100644 --- a/braintrust/templates/loop-runtime-configmap.yaml +++ b/braintrust/templates/loop-runtime-configmap.yaml @@ -1,9 +1,6 @@ {{- include "braintrust.loopRuntime.validate" . -}} {{- if include "braintrust.loopRuntime.enabled" . -}} -{{- $apiUrl := default (include "braintrust.apiInternalUrl" .) .Values.loopRuntime.braintrustApiUrl -}} -{{- $aiProxyUrl := default (include "braintrust.apiAiProxyInternalUrl" .) .Values.loopRuntime.aiProxyUrl -}} -{{- $brainstoreReaderUrl := default (include "braintrust.brainstoreFastReaderInternalUrl" .) .Values.loopRuntime.brainstoreReaderUrl -}} -{{- $brainstoreBucketUri := printf "gs://%s" .Values.objectStorage.google.brainstoreBucket -}} +{{- $bucket := printf "gs://%s" .Values.objectStorage.google.brainstoreBucket -}} {{- $sandbox := .Values.loopRuntime.sandbox -}} apiVersion: v1 kind: ConfigMap @@ -18,45 +15,43 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} +# Matches the ECS Loop Runtime environment in the AWS Terraform module. data: BRAINTRUST_LEGACY_IDS: "true" LOOP_RUNTIME_HOST: "0.0.0.0" LOOP_RUNTIME_PORT: {{ .Values.loopRuntime.service.port | quote }} - LOOP_RUNTIME_LIFECYCLE_LOG: {{ .Values.loopRuntime.lifecycleLog | toString | quote }} - LOOP_RUNTIME_REDIS_NAMESPACE: {{ include "braintrust.loopRuntimeRedisNamespace" . | quote }} - LOOP_RUNTIME_STATE_DIR: {{ .Values.loopRuntime.stateDir | quote }} - {{- /* An empty orgName serves every org, matching the API. The runtime refuses an empty ORG_NAME. */}} - ORG_NAME: {{ .Values.global.orgName | default "" | toString | trim | default "*" | quote }} LOOP_RUNTIME_DRAIN_TIMEOUT_SECONDS: "90" - BRAINTRUST_API_URL: {{ $apiUrl | quote }} - LOOP_RUNTIME_AI_PROXY_URL: {{ $aiProxyUrl | quote }} - LOOP_RUNTIME_BRAINSTORE_READER_URL: {{ $brainstoreReaderUrl | quote }} + LOOP_RUNTIME_REDIS_NAMESPACE: "loop-runtime:{{ .Release.Name }}" + LOOP_RUNTIME_STATE_DIR: /mnt/tmp/loop-runtime + {{- /* The runtime refuses an empty ORG_NAME; the API treats empty as every org. */}} + ORG_NAME: {{ .Values.global.orgName | default "" | toString | trim | default "*" | quote }} + BRAINTRUST_API_URL: {{ include "braintrust.apiInternalUrl" . | quote }} + LOOP_RUNTIME_AI_PROXY_URL: {{ include "braintrust.apiAiProxyInternalUrl" . | quote }} + LOOP_RUNTIME_BRAINSTORE_READER_URL: "http://{{ .Values.brainstore.fastreader.service.name | default .Values.brainstore.fastreader.name }}.{{ include "braintrust.namespace" . }}:{{ .Values.brainstore.fastreader.service.port }}" BRAINSTORE_VERBOSE: "1" - BRAINSTORE_INDEX_URI: "{{ $brainstoreBucketUri }}/brainstore/index" - BRAINSTORE_REALTIME_WAL_URI: "{{ $brainstoreBucketUri }}/brainstore/wal" + BRAINSTORE_INDEX_URI: "{{ $bucket }}/brainstore/index" + BRAINSTORE_REALTIME_WAL_URI: "{{ $bucket }}/brainstore/wal" BRAINSTORE_CODE_BUNDLE_URI: "gs://{{ .Values.objectStorage.google.apiBucket }}" - BRAINSTORE_CACHE_DIR: {{ .Values.loopRuntime.cacheDir | quote }} - BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE: {{ .Values.loopRuntime.objectStoreCacheFileSize | quote }} + BRAINSTORE_CACHE_DIR: /mnt/tmp/brainstore + BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE: 5Gi BRAINSTORE_CONTROL_PLANE_TELEMETRY: {{ .Values.global.controlPlaneTelemetry | quote }} - BRAINSTORE_DISABLE_STATUS_UPDATES: {{ .Values.loopRuntime.brainstoreDisableStatusUpdates | toString | quote }} NO_COLOR: "1" - {{- /* Matches the ECS Loop Runtime settings for the Sandbox Service in the AWS Terraform module. */}} EXO_SANDBOX_SERVICE_DNS_URL: {{ $sandbox.dnsUrl | quote }} EXO_SANDBOX_SERVICE_IMAGE: {{ $sandbox.image | quote }} EXO_SANDBOX_VCPU_COUNT: {{ $sandbox.vcpuCount | quote }} EXO_SANDBOX_MEMORY_MIB: {{ $sandbox.memoryMib | quote }} - SANDBOX_SERVICE_TLS_CERT: "/etc/sandbox-tls/client.pem" - SANDBOX_SERVICE_TLS_KEY: "/etc/sandbox-tls/client.key" - SANDBOX_SERVICE_TLS_CA: "/etc/sandbox-tls/server-ca.pem" - LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN: "0.0.0.0:{{ $sandbox.egressGateway.port }}" + SANDBOX_SERVICE_TLS_CERT: /etc/sandbox-tls/client.pem + SANDBOX_SERVICE_TLS_KEY: /etc/sandbox-tls/client.key + SANDBOX_SERVICE_TLS_CA: /etc/sandbox-tls/server-ca.pem + LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN: "0.0.0.0:4002" LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL: {{ $sandbox.egressGateway.url | quote }} - LOOP_RUNTIME_SANDBOX_EGRESS_MODE: "restricted" + LOOP_RUNTIME_SANDBOX_EGRESS_MODE: restricted LOOP_RUNTIME_ENABLE_SANDBOX_ADMIN: "true" {{- with (.Values.global.allowedOrgIds | default "" | toString | trim) }} ALLOWED_ORG_IDS: {{ . | quote }} {{- end }} {{- if eq .Values.brainstore.locksBackend "objectStorage" }} - BRAINSTORE_LOCKS_URI: "{{ $brainstoreBucketUri }}/brainstore/locks" + BRAINSTORE_LOCKS_URI: "{{ $bucket }}/brainstore/locks" {{- end }} {{- if .Values.brainstoreWalFooterVersion }} BRAINSTORE_WAL_FOOTER_VERSION: {{ .Values.brainstoreWalFooterVersion | quote }} diff --git a/braintrust/templates/loop-runtime-deployment.yaml b/braintrust/templates/loop-runtime-deployment.yaml index 8d91423..c0bdbd0 100644 --- a/braintrust/templates/loop-runtime-deployment.yaml +++ b/braintrust/templates/loop-runtime-deployment.yaml @@ -1,15 +1,12 @@ {{- include "braintrust.loopRuntime.validate" . -}} {{- if include "braintrust.loopRuntime.enabled" . -}} -{{- $resourceLabels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) -}} -{{- $podLabels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) (deepCopy .Values.loopRuntime.podLabels) (dict "app" .Values.loopRuntime.name) -}} -{{- $_ := set $podLabels "gke-workload-identity/use" "true" -}} -{{- $sandbox := .Values.loopRuntime.sandbox -}} +{{- $podLabels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) (deepCopy .Values.loopRuntime.podLabels) (dict "app" .Values.loopRuntime.name "gke-workload-identity/use" "true") -}} apiVersion: apps/v1 kind: Deployment metadata: name: {{ .Values.loopRuntime.name }} namespace: {{ include "braintrust.namespace" . }} - {{- with $resourceLabels }} + {{- with (mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels)) }} labels: {{- toYaml . | nindent 4 }} {{- end }} @@ -19,11 +16,6 @@ metadata: {{- end }} spec: replicas: {{ ternary 1 .Values.loopRuntime.replicas (has .Values.loopRuntime.replicas (list nil "")) }} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 100% - maxUnavailable: 0 selector: matchLabels: app: {{ .Values.loopRuntime.name }} @@ -39,11 +31,15 @@ spec: {{- end }} spec: serviceAccountName: {{ .Values.loopRuntime.serviceAccount.name }} - terminationGracePeriodSeconds: {{ .Values.loopRuntime.terminationGracePeriodSeconds }} - {{- with .Values.loopRuntime.podSecurityContext }} + # The runtime drains active turns for up to 90 seconds after SIGTERM. + terminationGracePeriodSeconds: 120 securityContext: - {{- toYaml . | nindent 8 }} - {{- end }} + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault {{- with .Values.loopRuntime.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -60,33 +56,25 @@ spec: - name: loop-runtime image: "{{ .Values.loopRuntime.image.repository }}:{{ .Values.loopRuntime.image.tag }}" imagePullPolicy: {{ .Values.loopRuntime.image.pullPolicy }} - {{- with .Values.loopRuntime.securityContext }} securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL ports: - name: {{ .Values.loopRuntime.service.portName }} containerPort: {{ .Values.loopRuntime.service.port }} - protocol: TCP - - containerPort: {{ $sandbox.egressGateway.port }} - protocol: TCP + # Git forward proxy for sandboxes. + - containerPort: 4002 resources: {{- toYaml .Values.loopRuntime.resources | nindent 12 }} {{- with .Values.loopRuntime.livenessProbe }} - {{- $livenessProbe := deepCopy . -}} - {{- if and (hasKey $livenessProbe "httpGet") (not (hasKey $livenessProbe.httpGet "port")) -}} - {{- $_ := set $livenessProbe.httpGet "port" $.Values.loopRuntime.service.portName -}} - {{- end }} livenessProbe: - {{- toYaml $livenessProbe | nindent 12 }} + {{- toYaml . | nindent 12 }} {{- end }} {{- with .Values.loopRuntime.readinessProbe }} - {{- $readinessProbe := deepCopy . -}} - {{- if and (hasKey $readinessProbe "httpGet") (not (hasKey $readinessProbe.httpGet "port")) -}} - {{- $_ := set $readinessProbe.httpGet "port" $.Values.loopRuntime.service.portName -}} - {{- end }} readinessProbe: - {{- toYaml $readinessProbe | nindent 12 }} + {{- toYaml . | nindent 12 }} {{- end }} envFrom: - configMapRef: @@ -117,7 +105,6 @@ spec: secretKeyRef: name: braintrust-secrets key: FUNCTION_SECRET_KEY - # Decrypts stored project MCP credentials. - name: FUNCTION_SECRET_KEY valueFrom: secretKeyRef: @@ -139,26 +126,27 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} volumeMounts: - - name: state-volume - mountPath: {{ .Values.loopRuntime.stateDir }} - - name: cache-volume - mountPath: {{ .Values.loopRuntime.cacheDir }} - - name: sandbox-service-tls + - name: state + mountPath: /mnt/tmp/loop-runtime + - name: cache + mountPath: /mnt/tmp/brainstore + - name: sandbox-tls mountPath: /etc/sandbox-tls readOnly: true {{- with .Values.loopRuntime.extraContainers }} {{- toYaml . | nindent 8 }} {{- end }} volumes: - - name: state-volume + - name: state emptyDir: - sizeLimit: {{ .Values.loopRuntime.stateSizeLimit | quote }} - - name: cache-volume + sizeLimit: 2Gi + - name: cache emptyDir: - sizeLimit: {{ .Values.loopRuntime.cacheSizeLimit | quote }} - - name: sandbox-service-tls + sizeLimit: 6Gi + # Created by examples/gke-loop-runtime/create-sandbox-tls.sh. + - name: sandbox-tls secret: - secretName: {{ $sandbox.tlsSecretName }} + secretName: sandbox-client-tls {{- with .Values.loopRuntime.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/braintrust/templates/loop-runtime-egress-service.yaml b/braintrust/templates/loop-runtime-egress-service.yaml index f42ace4..1808663 100644 --- a/braintrust/templates/loop-runtime-egress-service.yaml +++ b/braintrust/templates/loop-runtime-egress-service.yaml @@ -1,15 +1,14 @@ {{- include "braintrust.loopRuntime.validate" . -}} {{- if include "braintrust.loopRuntime.enabled" . -}} -{{- $resourceLabels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) -}} {{- $egressGateway := .Values.loopRuntime.sandbox.egressGateway -}} -{{- /* Workers in the isolated cluster reach the Git forward proxy through this -internal load balancer. Terraform reserves its address and DNS name. */}} +# Workers in the isolated cluster reach the Git forward proxy through this +# internal load balancer, on the address Terraform reserves. apiVersion: v1 kind: Service metadata: name: {{ .Values.loopRuntime.service.name | default .Values.loopRuntime.name }}-egress namespace: {{ include "braintrust.namespace" . }} - {{- with $resourceLabels }} + {{- with (mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels)) }} labels: {{- toYaml . | nindent 4 }} {{- end }} @@ -24,7 +23,6 @@ spec: app: {{ .Values.loopRuntime.name }} ports: - name: egress - protocol: TCP - port: {{ $egressGateway.port }} - targetPort: {{ $egressGateway.port }} + port: 4002 + targetPort: 4002 {{- end }} diff --git a/braintrust/templates/loop-runtime-serviceaccount.yaml b/braintrust/templates/loop-runtime-serviceaccount.yaml index 63b1c55..1e65de4 100644 --- a/braintrust/templates/loop-runtime-serviceaccount.yaml +++ b/braintrust/templates/loop-runtime-serviceaccount.yaml @@ -1,19 +1,17 @@ {{- include "braintrust.loopRuntime.validate" . -}} {{- if include "braintrust.loopRuntime.enabled" . -}} -{{- $labels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) -}} -{{- $annotations := mergeOverwrite (deepCopy .Values.loopRuntime.annotations.serviceaccount) (deepCopy .Values.loopRuntime.serviceAccount.annotations) -}} -{{- $_ := set $annotations "iam.gke.io/gcp-service-account" .Values.loopRuntime.serviceAccount.googleServiceAccount -}} apiVersion: v1 kind: ServiceAccount metadata: name: {{ .Values.loopRuntime.serviceAccount.name }} namespace: {{ include "braintrust.namespace" . }} - {{- with $labels }} + {{- with (mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels)) }} labels: {{- toYaml . | nindent 4 }} {{- end }} - {{- with $annotations }} annotations: + iam.gke.io/gcp-service-account: {{ .Values.loopRuntime.serviceAccount.googleServiceAccount }} + {{- with .Values.loopRuntime.annotations.serviceaccount }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} {{- end }} diff --git a/braintrust/tests/__fixtures__/partial-loop-runtime-image-values.yaml b/braintrust/tests/__fixtures__/partial-loop-runtime-image-values.yaml deleted file mode 100644 index 231fa7a..0000000 --- a/braintrust/tests/__fixtures__/partial-loop-runtime-image-values.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# Reused values whose image map lacks a tag. -loopRuntime: - image: - repository: "test/loop-runtime" - tag: null diff --git a/braintrust/tests/__fixtures__/reuse-loop-runtime-enabled-values.yaml b/braintrust/tests/__fixtures__/reuse-loop-runtime-enabled-values.yaml index b43c8e9..42e8585 100644 --- a/braintrust/tests/__fixtures__/reuse-loop-runtime-enabled-values.yaml +++ b/braintrust/tests/__fixtures__/reuse-loop-runtime-enabled-values.yaml @@ -1,33 +1,5 @@ # Simulate `helm upgrade --reuse-values --set loopRuntime.enabled=true` from a -# chart that predates Loop Runtime. Explicit nulls remove the new chart -# defaults during Helm value coalescing, leaving enabled as the only field. +# chart that predates Loop Runtime. An explicit null removes the new defaults. loopRuntime: enabled: true - name: null - labels: null - podLabels: null - annotations: null - replicas: null - image: null - service: null - serviceAccount: null - braintrustApiUrl: null - aiProxyUrl: null - brainstoreReaderUrl: null - redisNamespace: null - lifecycleLog: null - brainstoreDisableStatusUpdates: null - stateDir: null - cacheDir: null - resources: null - podSecurityContext: null - securityContext: null - livenessProbe: null - readinessProbe: null - nodeSelector: null - tolerations: null - affinity: null sandbox: null - extraEnvVars: null - extraContainers: null - extraVolumes: null diff --git a/braintrust/tests/labels-merge-services_test.yaml b/braintrust/tests/labels-merge-services_test.yaml index cd3a717..9608d7b 100644 --- a/braintrust/tests/labels-merge-services_test.yaml +++ b/braintrust/tests/labels-merge-services_test.yaml @@ -167,37 +167,7 @@ tests: - isNull: path: metadata.labels["fastreader-specific"] - - it: should give Loop Runtime service only global and runtime-specific labels - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - set: - global.labels.shared: "global-service" - loopRuntime.labels.loop-specific: "loop-service" - api.labels.api-specific: "api-service" - brainstore.reader.labels.reader-specific: "reader-service" - brainstore.writer.labels.writer-specific: "writer-service" - brainstore.fastreader.labels.fastreader-specific: "fastreader-service" - release: - namespace: "braintrust" - template: loop-runtime-service.yaml - asserts: - - equal: - path: metadata.labels.shared - value: global-service - - equal: - path: metadata.labels["loop-specific"] - value: loop-service - - isNull: - path: metadata.labels["api-specific"] - - isNull: - path: metadata.labels["reader-specific"] - - isNull: - path: metadata.labels["writer-specific"] - - isNull: - path: metadata.labels["fastreader-specific"] - - - it: should give Loop Runtime egress service only global and runtime-specific labels + - it: should give Loop Runtime services only global and runtime-specific labels values: - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml @@ -205,34 +175,24 @@ tests: global.labels.shared: global-service global.labels.component: global loopRuntime.labels.component: loop-runtime - loopRuntime.labels.loop-specific: loop-egress - api.labels.component: api api.labels.api-specific: api-service - brainstore.reader.labels.reader-specific: reader-service - brainstore.writer.labels.writer-specific: writer-service - brainstore.fastreader.labels.fastreader-specific: fastreader-service - brainstore.image.tag: v2.17.0 - brainstore.automationwriter.replicas: 1 - brainstore.automationwriter.labels.automationwriter-specific: automationwriter-service - template: loop-runtime-egress-service.yaml + templates: + - loop-runtime-service.yaml + - loop-runtime-egress-service.yaml asserts: - equal: path: metadata.labels value: shared: global-service component: loop-runtime - loop-specific: loop-egress - - it: should keep Loop Runtime egress labels out of the API service + - it: should keep Loop Runtime labels out of the API service values: - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml set: global.labels.shared: global-service - global.labels.component: global loopRuntime.labels.component: loop-runtime - loopRuntime.labels.loop-specific: loop-egress - api.labels.component: api api.labels.api-specific: api-service template: api-service.yaml asserts: @@ -240,5 +200,4 @@ tests: path: metadata.labels value: shared: global-service - component: api api-specific: api-service diff --git a/braintrust/tests/loop-runtime-configmap_test.yaml b/braintrust/tests/loop-runtime-configmap_test.yaml index 480831f..0f6edc3 100644 --- a/braintrust/tests/loop-runtime-configmap_test.yaml +++ b/braintrust/tests/loop-runtime-configmap_test.yaml @@ -3,9 +3,9 @@ templates: - loop-runtime-configmap.yaml tests: - it: should not render when Loop Runtime is disabled - values: &base-values + values: - __fixtures__/base-values.yaml - asserts: &absent + asserts: - hasDocuments: count: 0 @@ -14,7 +14,9 @@ tests: - __fixtures__/base-values.yaml - __fixtures__/google-values.yaml - __fixtures__/pre-loop-runtime-values.yaml - asserts: *absent + asserts: + - hasDocuments: + count: 0 - it: should explain how to upgrade when reused values enable Loop Runtime without defaults values: @@ -23,99 +25,54 @@ tests: - __fixtures__/reuse-loop-runtime-enabled-values.yaml asserts: - failedTemplate: - errorMessage: "loopRuntime.image is missing. Values reused from a chart release that predates Loop Runtime omit the loopRuntime defaults; upgrade with --reset-then-reuse-values or supply the full loopRuntime map." + errorMessage: "loopRuntime defaults are missing. Values reused from a chart release that predates Loop Runtime omit them; upgrade with --reset-then-reuse-values." - it: should render GKE storage and the Sandbox Service configuration values: &values - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml - release: - namespace: braintrust asserts: - equal: path: data.BRAINSTORE_INDEX_URI value: gs://test-brainstore-bucket/brainstore/index - - equal: - path: data.BRAINSTORE_REALTIME_WAL_URI - value: gs://test-brainstore-bucket/brainstore/wal - equal: path: data.BRAINSTORE_CODE_BUNDLE_URI value: gs://test-api-bucket - equal: path: data.EXO_SANDBOX_SERVICE_DNS_URL value: https://workers.sandbox.internal:9400 - - equal: - path: data.EXO_SANDBOX_SERVICE_IMAGE - value: public.ecr.aws/braintrust/codex-runtime@sha256:35db9c98d4a74e20758a29b82f8a3a6552be0002eb439c8c1a0f434ac573ee21 - - equal: - path: data.EXO_SANDBOX_VCPU_COUNT - value: "1" - - equal: - path: data.EXO_SANDBOX_MEMORY_MIB - value: "2048" - - equal: - path: data.SANDBOX_SERVICE_TLS_CERT - value: /etc/sandbox-tls/client.pem - - equal: - path: data.SANDBOX_SERVICE_TLS_KEY - value: /etc/sandbox-tls/client.key - - equal: - path: data.SANDBOX_SERVICE_TLS_CA - value: /etc/sandbox-tls/server-ca.pem - - equal: - path: data.LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN - value: 0.0.0.0:4002 - equal: path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL value: http://loop-egress.sandbox.internal:4002 - - equal: - path: data.LOOP_RUNTIME_SANDBOX_EGRESS_MODE - value: restricted - - equal: - path: data.LOOP_RUNTIME_ENABLE_SANDBOX_ADMIN - value: "true" - equal: path: data.LOOP_RUNTIME_REDIS_NAMESPACE value: loop-runtime:RELEASE-NAME - - isNull: - path: data.EXO_SANDBOX_PROVIDER + - equal: + path: data.ORG_NAME + value: test-org - isNull: path: data.BRAINSTORE_LOCKS_URI - - it: should use GCS object storage locks when configured + - it: should serve every org when orgName is empty values: *values set: - brainstore.locksBackend: objectStorage + global.orgName: "" asserts: - equal: - path: data.BRAINSTORE_LOCKS_URI - value: gs://test-brainstore-bucket/brainstore/locks + path: data.ORG_NAME + value: "*" - - it: should pass custom sandbox sizing, Git proxy port, and Redis namespace + - it: should use GCS object storage locks when configured values: *values set: - loopRuntime.sandbox.vcpuCount: 2 - loopRuntime.sandbox.memoryMib: 4096 - loopRuntime.sandbox.egressGateway.port: 5002 - loopRuntime.redisNamespace: loop-runtime:existing + brainstore.locksBackend: objectStorage asserts: - equal: - path: data.EXO_SANDBOX_VCPU_COUNT - value: "2" - - equal: - path: data.EXO_SANDBOX_MEMORY_MIB - value: "4096" - - equal: - path: data.LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN - value: 0.0.0.0:5002 - - equal: - path: data.LOOP_RUNTIME_REDIS_NAMESPACE - value: loop-runtime:existing + path: data.BRAINSTORE_LOCKS_URI + value: gs://test-brainstore-bucket/brainstore/locks - it: should reject Loop Runtime on AWS - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml + values: *values set: cloud: aws asserts: ¬-google @@ -123,34 +80,24 @@ tests: errorMessage: "loopRuntime.enabled requires cloud: google. On AWS, Loop Runtime runs on ECS through the Terraform module." - it: should reject Loop Runtime on Azure - values: - - __fixtures__/base-values.yaml - - __fixtures__/azure-values.yaml + values: *values set: - loopRuntime.enabled: true + cloud: azure asserts: *not-google - it: should reject Loop Runtime without a cloud - values: *base-values + values: *values set: - loopRuntime.enabled: true + cloud: "" asserts: *not-google - - it: should require the Loop Runtime Google service account + - it: should require the Google service account values: *values set: loopRuntime.serviceAccount.googleServiceAccount: "" asserts: - failedTemplate: - errorMessage: "loopRuntime.serviceAccount.googleServiceAccount is required (Terraform output loop_runtime_service_account)" - - - it: should require the GCS API bucket - values: *values - set: - objectStorage.google.apiBucket: "" - asserts: - - failedTemplate: - errorMessage: "objectStorage.google.apiBucket is required when loopRuntime.enabled is true" + errorMessage: "loopRuntime.serviceAccount.googleServiceAccount is required" - it: should require the worker discovery endpoint values: *values @@ -158,39 +105,15 @@ tests: loopRuntime.sandbox.dnsUrl: null asserts: - failedTemplate: - errorMessage: "loopRuntime.sandbox.dnsUrl is required (Terraform output gke_isolated_workers.discovery.endpoint)" - - - it: should require the sandbox guest image - values: *values - set: - loopRuntime.sandbox.image: "" - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.image is required when loopRuntime.enabled is true" + errorMessage: "loopRuntime.sandbox.dnsUrl is required" - - it: should require the TLS Secret name - values: *values - set: - loopRuntime.sandbox.tlsSecretName: "" - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.tlsSecretName is required when loopRuntime.enabled is true" - - - it: should require the reserved egress address + - it: should require the egress address values: *values set: loopRuntime.sandbox.egressGateway.address: "" asserts: - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.address is required (Terraform output gke_isolated_workers.loop_egress.address)" - - - it: should require the egress URL - values: *values - set: - loopRuntime.sandbox.egressGateway.url: null - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.url is required (Terraform output gke_isolated_workers.loop_egress.url)" + errorMessage: "loopRuntime.sandbox.egressGateway.address is required" - it: should require egress source ranges values: *values @@ -198,109 +121,20 @@ tests: loopRuntime.sandbox.egressGateway.sourceRanges: [] asserts: - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges is required (Terraform output gke_isolated_workers.loop_egress.source_ranges)" - - - it: should serve every org when orgName is empty - values: *values - set: - global.orgName: "" - asserts: - - equal: - path: data.ORG_NAME - value: "*" - - - it: should serve every org when orgName is whitespace - values: *values - set: - global.orgName: " " - asserts: - - equal: - path: data.ORG_NAME - value: "*" - - - it: should keep a configured orgName and set the drain and cache budgets - values: *values - set: - global.orgName: acme - asserts: - - equal: - path: data.ORG_NAME - value: acme - - equal: - path: data.LOOP_RUNTIME_DRAIN_TIMEOUT_SECONDS - value: "90" - - equal: - path: data.BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE - value: 5Gi - - - it: should reject a blank egress source range - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: [""] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges entries must be IPv4 CIDR ranges, got \"\"" + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges is required" - - it: should reject a null egress source range + - it: should reject a blank egress source range next to a valid one values: *values set: - loopRuntime.sandbox.egressGateway.sourceRanges: [null] - asserts: - - failedTemplate: {} - - - it: should reject a blank entry alongside a valid egress source range - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: ["10.64.64.0/24", " "] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges entries must be IPv4 CIDR ranges, got \" \"" - - - it: should reject a padded egress source range - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: [" 10.64.64.0/24"] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges entries must be IPv4 CIDR ranges, got \" 10.64.64.0/24\"" - - - it: should require the runtime image repository when it is empty - values: *values - set: - loopRuntime.image.repository: "" - asserts: - - failedTemplate: - errorMessage: "loopRuntime.image.repository is required when loopRuntime.enabled is true" - - - it: should require the runtime image repository when it is null - values: *values - set: - loopRuntime.image.repository: null + loopRuntime.sandbox.egressGateway.sourceRanges: ["10.64.64.0/24", ""] asserts: - failedTemplate: - errorMessage: "loopRuntime.image.repository is required when loopRuntime.enabled is true" + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"\" is not a CIDR range" - - it: should require the runtime image tag when it is empty - values: *values - set: - loopRuntime.image.tag: "" - asserts: - - failedTemplate: - errorMessage: "loopRuntime.image.tag is required when loopRuntime.enabled is true" - - - it: should require the runtime image tag when it is null + - it: should reject a null image tag values: *values set: loopRuntime.image.tag: null asserts: - failedTemplate: - errorMessage: "loopRuntime.image.tag is required when loopRuntime.enabled is true" - - - it: should require the runtime image tag when reused values omit it - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - - __fixtures__/partial-loop-runtime-image-values.yaml - asserts: - - failedTemplate: - errorMessage: "loopRuntime.image.tag is required when loopRuntime.enabled is true" + errorMessage: "loopRuntime.image.tag is required" diff --git a/braintrust/tests/loop-runtime-deployment_test.yaml b/braintrust/tests/loop-runtime-deployment_test.yaml index 2f4332d..166c25a 100644 --- a/braintrust/tests/loop-runtime-deployment_test.yaml +++ b/braintrust/tests/loop-runtime-deployment_test.yaml @@ -10,29 +10,15 @@ tests: - hasDocuments: count: 0 - - it: should not render when reused values have no loopRuntime map - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/pre-loop-runtime-values.yaml - asserts: - - hasDocuments: - count: 0 - - - it: should render a hardened runtime pod with GKE workload identity and data-plane secrets + - it: should render the runtime pod with workload identity, secrets, and the client TLS Secret template: loop-runtime-deployment.yaml values: &values - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml asserts: - - isKind: - of: Deployment - equal: path: spec.replicas value: 2 - - equal: - path: spec.template.spec.serviceAccountName - value: braintrust-loop-runtime - equal: path: spec.template.metadata.labels["gke-workload-identity/use"] value: "true" @@ -43,100 +29,26 @@ tests: path: spec.template.spec.containers[0].image value: test/loop-runtime:v1.0.0 - equal: - path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation - value: false - - contains: - path: spec.template.spec.containers[0].securityContext.capabilities.drop - content: ALL - - equal: - path: spec.template.spec.containers[0].resources.requests - value: - cpu: "1" - memory: 2Gi - ephemeral-storage: 9Gi - - contains: - path: spec.template.spec.containers[0].env - content: - name: BRAINSTORE_METADATA_URI - valueFrom: - secretKeyRef: - name: braintrust-secrets - key: PG_URL + path: spec.template.spec.terminationGracePeriodSeconds + value: 120 - contains: path: spec.template.spec.containers[0].env content: - name: SERVICE_TOKEN_SECRET_KEY + name: FUNCTION_SECRET_KEY valueFrom: secretKeyRef: name: braintrust-secrets key: FUNCTION_SECRET_KEY - - contains: - path: spec.template.spec.containers[0].env - content: - name: BRAINSTORE_LOCKS_URI - valueFrom: - secretKeyRef: - name: braintrust-secrets - key: REDIS_URL - - - it: should mount the Sandbox Service client TLS Secret and expose the Git proxy port - template: loop-runtime-deployment.yaml - values: *values - asserts: - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: sandbox-service-tls - mountPath: /etc/sandbox-tls - readOnly: true - contains: path: spec.template.spec.volumes content: - name: sandbox-service-tls + name: sandbox-tls secret: secretName: sandbox-client-tls - - equal: + - contains: path: spec.template.spec.containers[0].ports - value: - - name: http - containerPort: 4001 - protocol: TCP - - containerPort: 4002 - protocol: TCP - - - it: should use a custom TLS Secret name and Git proxy port - template: loop-runtime-deployment.yaml - values: *values - set: - loopRuntime.sandbox.tlsSecretName: custom-tls - loopRuntime.sandbox.egressGateway.port: 5002 - asserts: - - equal: - path: spec.template.spec.volumes[2].secret.secretName - value: custom-tls - - equal: - path: spec.template.spec.containers[0].ports[1].containerPort - value: 5002 - - - it: should keep probes on a custom named service port - template: loop-runtime-deployment.yaml - values: *values - set: - loopRuntime.service.port: 5000 - loopRuntime.service.portName: egress - asserts: - - equal: - path: spec.template.spec.containers[0].ports[0] - value: - name: egress - containerPort: 5000 - protocol: TCP - - equal: - path: spec.template.spec.containers[0].livenessProbe.httpGet.port - value: egress - - equal: - path: spec.template.spec.containers[0].readinessProbe.httpGet.port - value: egress + content: + containerPort: 4002 - it: should default null replicas to one template: loop-runtime-deployment.yaml @@ -168,51 +80,7 @@ tests: path: spec.replicas value: 0 - - it: should keep an explicit replica count - template: loop-runtime-deployment.yaml - values: *values - set: - loopRuntime.replicas: 3 - asserts: - - equal: - path: spec.replicas - value: 3 - - - it: should default to the pinned image without a semver gate - template: loop-runtime-deployment.yaml - values: *values - set: - loopRuntime.image.repository: public.ecr.aws/braintrust/loop-runtime - loopRuntime.image.tag: b96d95eaa269031cdd6f286a17b9b3e4795355d1 - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: public.ecr.aws/braintrust/loop-runtime:b96d95eaa269031cdd6f286a17b9b3e4795355d1 - - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: IfNotPresent - - - it: should render release tags without the v prefix - template: loop-runtime-deployment.yaml - values: *values - set: - loopRuntime.image.tag: 2.20.0 - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: test/loop-runtime:2.20.0 - - - it: should render prerelease tags - template: loop-runtime-deployment.yaml - values: *values - set: - loopRuntime.image.tag: v2.20.0-rc.1 - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: test/loop-runtime:v2.20.0-rc.1 - - - it: should render all-digit short SHAs as strings + - it: should render an all-digit short SHA tag template: loop-runtime-deployment.yaml values: *values set: @@ -221,48 +89,3 @@ tests: - equal: path: spec.template.spec.containers[0].image value: test/loop-runtime:1234567 - - - it: should keep a mutable tag with an explicit pull policy - template: loop-runtime-deployment.yaml - values: *values - set: - loopRuntime.image.tag: latest-2.x - loopRuntime.image.pullPolicy: Always - asserts: - - equal: - path: spec.template.spec.containers[0].image - value: test/loop-runtime:latest-2.x - - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: Always - - - it: should outlast the runtime drain, pass the function secret, and bound local storage - template: loop-runtime-deployment.yaml - values: *values - asserts: - - equal: - path: spec.template.spec.terminationGracePeriodSeconds - value: 120 - - contains: - path: spec.template.spec.containers[0].env - content: - name: FUNCTION_SECRET_KEY - valueFrom: - secretKeyRef: - name: braintrust-secrets - key: FUNCTION_SECRET_KEY - - equal: - path: spec.template.spec.volumes[0] - value: - name: state-volume - emptyDir: - sizeLimit: 2Gi - - equal: - path: spec.template.spec.volumes[1] - value: - name: cache-volume - emptyDir: - sizeLimit: 6Gi - - equal: - path: spec.template.spec.containers[0].resources.requests["ephemeral-storage"] - value: 9Gi diff --git a/braintrust/tests/loop-runtime-egress-service_test.yaml b/braintrust/tests/loop-runtime-egress-service_test.yaml index 319aa86..55cc8f5 100644 --- a/braintrust/tests/loop-runtime-egress-service_test.yaml +++ b/braintrust/tests/loop-runtime-egress-service_test.yaml @@ -1,4 +1,4 @@ -suite: test Loop Runtime egress gateway Service template +suite: test Loop Runtime egress Service template templates: - loop-runtime-egress-service.yaml tests: @@ -13,20 +13,10 @@ tests: values: - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml - release: - namespace: braintrust asserts: - - isKind: - of: Service - - equal: - path: metadata.name - value: braintrust-loop-runtime-egress - equal: path: metadata.annotations["networking.gke.io/load-balancer-type"] value: Internal - - equal: - path: spec.type - value: LoadBalancer - equal: path: spec.loadBalancerIP value: 10.0.0.50 @@ -35,31 +25,6 @@ tests: value: - 10.64.64.0/24 - 10.64.0.0/18 - - equal: - path: spec.selector.app - value: braintrust-loop-runtime - - equal: - path: spec.ports - value: - - name: egress - protocol: TCP - port: 4002 - targetPort: 4002 - - - it: should follow a custom service name and Git proxy port - values: - - __fixtures__/base-values.yaml - - __fixtures__/loop-runtime-values.yaml - set: - loopRuntime.service.name: loop - loopRuntime.sandbox.egressGateway.port: 5002 - asserts: - - equal: - path: metadata.name - value: loop-egress - equal: path: spec.ports[0].port - value: 5002 - - equal: - path: spec.ports[0].targetPort - value: 5002 + value: 4002 diff --git a/braintrust/tests/loop-runtime-service_test.yaml b/braintrust/tests/loop-runtime-service_test.yaml index e4dc895..4bc92b0 100644 --- a/braintrust/tests/loop-runtime-service_test.yaml +++ b/braintrust/tests/loop-runtime-service_test.yaml @@ -14,29 +14,12 @@ tests: - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml asserts: - - isKind: - of: Service - equal: path: metadata.name value: braintrust-loop-runtime - - equal: - path: spec.selector.app - value: braintrust-loop-runtime - equal: path: spec.ports[0].port value: 4001 - - equal: - path: spec.ports[0].targetPort - value: 4001 - equal: path: spec.type value: ClusterIP - - - it: should not render when reused values have no loopRuntime map - values: - - __fixtures__/base-values.yaml - - __fixtures__/google-values.yaml - - __fixtures__/pre-loop-runtime-values.yaml - asserts: - - hasDocuments: - count: 0 diff --git a/braintrust/tests/loop-runtime-serviceaccount_test.yaml b/braintrust/tests/loop-runtime-serviceaccount_test.yaml index 261132c..f84d085 100644 --- a/braintrust/tests/loop-runtime-serviceaccount_test.yaml +++ b/braintrust/tests/loop-runtime-serviceaccount_test.yaml @@ -10,28 +10,10 @@ tests: count: 0 - it: should bind the Loop Runtime Google service account - values: &values + values: - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml asserts: - - isKind: - of: ServiceAccount - - equal: - path: metadata.name - value: braintrust-loop-runtime - equal: path: metadata.annotations["iam.gke.io/gcp-service-account"] value: braintrust-loop@test-project.iam.gserviceaccount.com - - - it: should merge annotations from both configuration paths - values: *values - set: - loopRuntime.annotations.serviceaccount.from-annotations: a - loopRuntime.serviceAccount.annotations.from-service-account: b - asserts: - - equal: - path: metadata.annotations["from-annotations"] - value: a - - equal: - path: metadata.annotations["from-service-account"] - value: b diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 86c3bdb..20dd873 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -309,8 +309,8 @@ api: extraContainers: [] extraVolumes: [] -# Loop Runtime configuration (optional; cloud: google only) -# See examples/gke-loop-runtime for the Sandbox Service worker release it needs. +# Loop Runtime (optional, cloud: google only). Sandboxes run on Sandbox Service +# workers in a separate GKE cluster; see examples/gke-loop-runtime. loopRuntime: enabled: false name: "braintrust-loop-runtime" @@ -322,67 +322,35 @@ loopRuntime: service: {} pod: {} serviceaccount: {} - # Null, missing, or empty replicas default to 1; 0 scales to zero. replicas: 2 image: repository: public.ecr.aws/braintrust/loop-runtime - # Pinned to the release tested with the Sandbox Service worker chart. + # Tested with the Sandbox Service worker release in the GKE example. tag: b96d95eaa269031cdd6f286a17b9b3e4795355d1 pullPolicy: IfNotPresent service: - # Optional name for the Service. If empty, loopRuntime.name is used. + # Optional name for service object. If not specified (empty), loopRuntime.name is used. name: "" type: ClusterIP port: 4001 portName: http serviceAccount: name: "braintrust-loop-runtime" - # Required: Terraform output loop_runtime_service_account. + # Terraform output loop_runtime_service_account. googleServiceAccount: "" - annotations: {} - # These URLs default to in-cluster services. Set them when the API, AI proxy, - # or Brainstore is exposed through a different service or namespace. - braintrustApiUrl: "" - aiProxyUrl: "" - brainstoreReaderUrl: "" - # Defaults to loop-runtime:; set explicitly when retaining - # an existing Loop Runtime Redis namespace across release renames. - redisNamespace: "" - lifecycleLog: "0" - brainstoreDisableStatusUpdates: "false" - stateDir: "/mnt/tmp/loop-runtime" - cacheDir: "/mnt/tmp/brainstore" - # Bounds the Brainstore file cache in cacheDir. Keep it below cacheSizeLimit. - objectStoreCacheFileSize: "5Gi" - stateSizeLimit: "2Gi" - cacheSizeLimit: "6Gi" - # On SIGTERM the runtime drains active turns for up to 90 seconds. - terminationGracePeriodSeconds: 120 resources: requests: cpu: "1" memory: "2Gi" - # State and cache emptyDirs plus container overhead. ephemeral-storage: "9Gi" limits: cpu: "1" memory: "2Gi" ephemeral-storage: "9Gi" - podSecurityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL livenessProbe: httpGet: path: /health/liveness + port: 4001 initialDelaySeconds: 10 periodSeconds: 30 timeoutSeconds: 5 @@ -391,36 +359,28 @@ loopRuntime: readinessProbe: httpGet: path: /health/liveness + port: 4001 initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 5 failureThreshold: 3 successThreshold: 1 - nodeSelector: {} - tolerations: [] - affinity: {} - # Sandboxes run on Sandbox Service workers in the isolated GKE cluster. + # Values come from Terraform output gke_isolated_workers. sandbox: - # Required: Terraform output gke_isolated_workers.discovery.endpoint. + # discovery.endpoint dnsUrl: "" - image: "public.ecr.aws/braintrust/codex-runtime@sha256:35db9c98d4a74e20758a29b82f8a3a6552be0002eb439c8c1a0f434ac573ee21" - # Resources requested for each sandbox; size worker capacity accordingly. - vcpuCount: 1 - memoryMib: 2048 - # Secret with client.pem, client.key, and server-ca.pem. - # examples/gke-loop-runtime/create-sandbox-tls.sh creates it. - tlsSecretName: sandbox-client-tls - # Git forward proxy for sandboxes, on an internal load balancer that the - # workers reach. Both values come from Terraform output gke_isolated_workers. + # loop_egress.address, loop_egress.source_ranges, and loop_egress.url egressGateway: - port: 4002 - # Required: reserved internal address (loop_egress.address). address: "" - # Required: worker ranges allowed to connect (loop_egress.source_ranges). sourceRanges: [] - # Required: loop_egress.url, http://:4002. url: "" + image: "public.ecr.aws/braintrust/codex-runtime@sha256:35db9c98d4a74e20758a29b82f8a3a6552be0002eb439c8c1a0f434ac573ee21" + vcpuCount: 1 + memoryMib: 2048 extraEnvVars: [] + nodeSelector: {} + tolerations: [] + affinity: {} extraContainers: [] extraVolumes: [] From 028a2a228acd3a31237aabcb3833c8d683132a21 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Mon, 26 Oct 2026 10:29:39 -0700 Subject: [PATCH 13/25] Fix GKE worker storage and Loop Runtime validation --- .../examples/gke-loop-runtime/README.md | 6 +- .../examples/gke-loop-runtime/node-init.yaml | 24 +++-- braintrust/templates/_helpers.tpl | 11 +- .../templates/loop-runtime-deployment.yaml | 12 ++- .../tests/loop-runtime-deployment_test.yaml | 66 ++++++++++++ .../loop-runtime-egress-service_test.yaml | 99 ++++++++++++++++- braintrust/values.yaml | 2 - tests/test_sandbox_node_init.py | 101 ++++++++++++++++++ 8 files changed, 307 insertions(+), 14 deletions(-) create mode 100644 tests/test_sandbox_node_init.py diff --git a/braintrust/examples/gke-loop-runtime/README.md b/braintrust/examples/gke-loop-runtime/README.md index c4775d7..b253910 100644 --- a/braintrust/examples/gke-loop-runtime/README.md +++ b/braintrust/examples/gke-loop-runtime/README.md @@ -14,13 +14,17 @@ gke_isolated_workers = { `<...>` values below come from `terraform output gke_isolated_workers` and `terraform output loop_runtime_service_account`. -1. Prepare worker nodes (mounts the local SSD the worker chart expects): +1. Prepare worker nodes (mounts XFS directly at the worker's state path): ```sh kubectl --context create namespace braintrust-sandbox kubectl --context apply -f braintrust/examples/gke-loop-runtime/node-init.yaml ``` + The worker image refuses to start until its state path is an XFS filesystem + root, including after node reboots. Node initialization preserves existing + XFS data and refuses to reformat disks containing another filesystem. + 2. Create the mTLS Secrets in both clusters: ```sh diff --git a/braintrust/examples/gke-loop-runtime/node-init.yaml b/braintrust/examples/gke-loop-runtime/node-init.yaml index 5cc41b8..03bca41 100644 --- a/braintrust/examples/gke-loop-runtime/node-init.yaml +++ b/braintrust/examples/gke-loop-runtime/node-init.yaml @@ -1,6 +1,6 @@ # GKE node pools have no bootstrap hook, so this DaemonSet does what EKS node -# bootstrap does for the Sandbox Service worker chart: mount the local SSD at -# /var/lib/braintrust-sandboxes, create /run/netns, and check /dev/kvm. +# bootstrap does for the Sandbox Service worker chart: mount XFS at the worker's +# state path, create /run/netns, and check /dev/kvm. apiVersion: v1 kind: ConfigMap metadata: @@ -9,10 +9,10 @@ metadata: data: init.sh: | #!/bin/bash - # The worker's hostPath directory (1) is created on the mounted SSD, so a - # worker cannot start before the mount. + # The worker entrypoint refuses to start unless /state is an XFS filesystem + # root. This also keeps it off the boot disk while initialization runs. set -euo pipefail - ROOT="${ROOT:-/var/lib/braintrust-sandboxes}" + ROOT="${ROOT:-/var/lib/braintrust-sandboxes/1}" DEVICES="${DEVICES:-/dev/disk/by-id/google-local-nvme-ssd-*}" mkdir -p /run/netns modprobe kvm_intel 2>/dev/null || modprobe kvm_amd 2>/dev/null || true @@ -36,12 +36,22 @@ data: --raid-devices="${#disks[@]}" "${disks[@]}" fi fi - blkid "$device" >/dev/null 2>&1 || mkfs.ext4 -q -F "$device" + fstype="$(blkid -s TYPE -o value "$device" || true)" + if [ -z "$fstype" ]; then + mkfs.xfs -q "$device" + elif [ "$fstype" != xfs ]; then + echo "$device contains $fstype; refusing to replace it with XFS" >&2 + exit 1 + fi mkdir -p "$ROOT" mount -o discard,defaults "$device" "$ROOT" echo "mounted $device at $ROOT" fi - mkdir -p "$ROOT/1" + if [ "$(findmnt -n -o FSTYPE -T "$ROOT")" != xfs ] || + [ "$(findmnt -n -o FSROOT -T "$ROOT")" != / ]; then + echo "$ROOT must be the root of an XFS filesystem" >&2 + exit 1 + fi --- apiVersion: apps/v1 kind: DaemonSet diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index c7224e8..b3c232d 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -176,7 +176,16 @@ predates Loop Runtime have no loopRuntime map, which means disabled. {{- fail "loopRuntime.sandbox.egressGateway.sourceRanges is required" -}} {{- end -}} {{- range $sandbox.egressGateway.sourceRanges -}} -{{- if not (regexMatch "^[0-9.]+/[0-9]+$" (toString .)) -}} +{{- $cidr := toString . -}} +{{- $valid := regexMatch "^([0-9]{1,3}\\.){3}[0-9]{1,3}/[0-9]{1,2}$" $cidr -}} +{{- if $valid -}} +{{- $parts := splitList "/" $cidr -}} +{{- $valid = le (int (index $parts 1)) 32 -}} +{{- range splitList "." (index $parts 0) -}} +{{- $valid = and $valid (le (int .) 255) -}} +{{- end -}} +{{- end -}} +{{- if not $valid -}} {{- fail (printf "loopRuntime.sandbox.egressGateway.sourceRanges: %q is not a CIDR range" (toString .)) -}} {{- end -}} {{- end -}} diff --git a/braintrust/templates/loop-runtime-deployment.yaml b/braintrust/templates/loop-runtime-deployment.yaml index c0bdbd0..c10f99f 100644 --- a/braintrust/templates/loop-runtime-deployment.yaml +++ b/braintrust/templates/loop-runtime-deployment.yaml @@ -69,12 +69,20 @@ spec: resources: {{- toYaml .Values.loopRuntime.resources | nindent 12 }} {{- with .Values.loopRuntime.livenessProbe }} + {{- $probe := deepCopy . -}} + {{- if and (kindIs "map" $probe.httpGet) (not (hasKey $probe.httpGet "port")) -}} + {{- $_ := set $probe.httpGet "port" $.Values.loopRuntime.service.port -}} + {{- end }} livenessProbe: - {{- toYaml . | nindent 12 }} + {{- toYaml $probe | nindent 12 }} {{- end }} {{- with .Values.loopRuntime.readinessProbe }} + {{- $probe := deepCopy . -}} + {{- if and (kindIs "map" $probe.httpGet) (not (hasKey $probe.httpGet "port")) -}} + {{- $_ := set $probe.httpGet "port" $.Values.loopRuntime.service.port -}} + {{- end }} readinessProbe: - {{- toYaml . | nindent 12 }} + {{- toYaml $probe | nindent 12 }} {{- end }} envFrom: - configMapRef: diff --git a/braintrust/tests/loop-runtime-deployment_test.yaml b/braintrust/tests/loop-runtime-deployment_test.yaml index 166c25a..84c0a6a 100644 --- a/braintrust/tests/loop-runtime-deployment_test.yaml +++ b/braintrust/tests/loop-runtime-deployment_test.yaml @@ -31,6 +31,12 @@ tests: - equal: path: spec.template.spec.terminationGracePeriodSeconds value: 120 + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.port + value: 4001 + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.port + value: 4001 - contains: path: spec.template.spec.containers[0].env content: @@ -50,6 +56,66 @@ tests: content: containerPort: 4002 + - it: should follow the runtime port with a custom port name + template: loop-runtime-deployment.yaml + values: *values + set: + loopRuntime.service.port: 4888 + loopRuntime.service.portName: runtime + asserts: + - contains: + path: spec.template.spec.containers[0].ports + content: + name: runtime + containerPort: 4888 + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.port + value: 4888 + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.port + value: 4888 + + - it: should preserve explicit numeric and named HTTP probe ports + template: loop-runtime-deployment.yaml + values: *values + set: + loopRuntime.service.port: 4888 + loopRuntime.service.portName: runtime + loopRuntime.livenessProbe.httpGet.port: 4999 + loopRuntime.readinessProbe.httpGet.port: runtime + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet.port + value: 4999 + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.port + value: runtime + + - it: should retain disabled probes + template: loop-runtime-deployment.yaml + values: *values + set: + loopRuntime.livenessProbe: null + loopRuntime.readinessProbe: null + asserts: + - notExists: + path: spec.template.spec.containers[0].livenessProbe + - notExists: + path: spec.template.spec.containers[0].readinessProbe + + - it: should not add an HTTP port to an exec probe + template: loop-runtime-deployment.yaml + values: *values + set: + loopRuntime.livenessProbe.httpGet: null + loopRuntime.livenessProbe.exec.command: ["true"] + asserts: + - isNull: + path: spec.template.spec.containers[0].livenessProbe.httpGet + - equal: + path: spec.template.spec.containers[0].livenessProbe.exec.command + value: ["true"] + - it: should default null replicas to one template: loop-runtime-deployment.yaml values: *values diff --git a/braintrust/tests/loop-runtime-egress-service_test.yaml b/braintrust/tests/loop-runtime-egress-service_test.yaml index 55cc8f5..026c9fc 100644 --- a/braintrust/tests/loop-runtime-egress-service_test.yaml +++ b/braintrust/tests/loop-runtime-egress-service_test.yaml @@ -10,7 +10,7 @@ tests: count: 0 - it: should expose the Git proxy on an internal load balancer at the reserved address - values: + values: &values - __fixtures__/base-values.yaml - __fixtures__/loop-runtime-values.yaml asserts: @@ -28,3 +28,100 @@ tests: - equal: path: spec.ports[0].port value: 4002 + + - it: should retain IPv4 boundary ranges, host bits, and leading zeroes + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: ["0.0.0.0/0","255.255.255.255/32","10.64.64.1/24","010.064.000.001/08"] + asserts: + - equal: + path: spec.loadBalancerSourceRanges + value: ["0.0.0.0/0","255.255.255.255/32","10.64.64.1/24","010.064.000.001/08"] + + - it: should reject out of range octets + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: ["256.0.0.1/24"] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"256.0.0.1/24\" is not a CIDR range" + + - it: should reject out of range prefixes + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: ["10.0.0.0/33"] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"10.0.0.0/33\" is not a CIDR range" + + - it: should reject oversized octets and prefixes + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: ["999.999.999.999/999"] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"999.999.999.999/999\" is not a CIDR range" + + - it: should reject an oversized prefix + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: ["10.0.0.0/999"] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"10.0.0.0/999\" is not a CIDR range" + + - it: should reject missing octets + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: ["10.0.0/24"] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"10.0.0/24\" is not a CIDR range" + + - it: should reject IPv6 + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: ["::1/128"] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"::1/128\" is not a CIDR range" + + - it: should reject a negative prefix + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: ["10.0.0.0/-1"] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"10.0.0.0/-1\" is not a CIDR range" + + - it: should reject a padded range + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: [" 10.0.0.0/24 "] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \" 10.0.0.0/24 \" is not a CIDR range" + + - it: should reject a whitespace range + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: [" "] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \" \" is not a CIDR range" + + - it: should reject a blank range + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: [""] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"\" is not a CIDR range" + + - it: should reject a null range + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: [null] + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"\" is not a CIDR range" diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 20dd873..b97ee17 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -350,7 +350,6 @@ loopRuntime: livenessProbe: httpGet: path: /health/liveness - port: 4001 initialDelaySeconds: 10 periodSeconds: 30 timeoutSeconds: 5 @@ -359,7 +358,6 @@ loopRuntime: readinessProbe: httpGet: path: /health/liveness - port: 4001 initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 5 diff --git a/tests/test_sandbox_node_init.py b/tests/test_sandbox_node_init.py new file mode 100644 index 0000000..4e6ce88 --- /dev/null +++ b/tests/test_sandbox_node_init.py @@ -0,0 +1,101 @@ +import os +from pathlib import Path +import subprocess +import tempfile +import textwrap +import unittest + + +NODE_INIT = Path(__file__).resolve().parents[1] / "braintrust/examples/gke-loop-runtime/node-init.yaml" +SCRIPT = textwrap.dedent(NODE_INIT.read_text().split(" init.sh: |\n", 1)[1].split("\n---", 1)[0]) + +# Execute the deployed script, substituting only host/device operations. +HOST_COMMANDS = r""" +modprobe() { return 0; } +test() { + if [[ "$1" == -c && "$2" == /dev/kvm ]]; then + [[ "$KVM" == yes ]] + else + builtin test "$@" + fi +} +mkdir() { return 0; } +readlink() { printf '%s\n' "$2"; } +mountpoint() { [[ "$MOUNTED" == yes ]]; } +blkid() { [[ -n "$FSTYPE" ]] && printf '%s\n' "$FSTYPE"; } +mkfs.xfs() { echo format-xfs >>"$LOG"; FSTYPE=xfs; } +mkfs.ext4() { echo format-ext4 >>"$LOG"; FSTYPE=ext4; } +mdadm() { echo raid >>"$LOG"; } +mount() { echo "mount ${@: -1}" >>"$LOG"; MOUNTED=yes; FSROOT=/; } +findmnt() { + case "$3" in + FSTYPE) printf '%s\n' "$FSTYPE" ;; + FSROOT) printf '%s\n' "$FSROOT" ;; + esac +} +""" + + +class SandboxNodeInitTests(unittest.TestCase): + def run_init(self, filesystem="", mounted=False, fsroot="/", disks=1, kvm=True): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for index in range(disks): + (root / f"disk-{index}").touch() + log = root / "commands" + env = { + **os.environ, + "DEVICES": str(root / "disk-*"), + "FSTYPE": filesystem, + "MOUNTED": "yes" if mounted else "no", + "FSROOT": fsroot, + "KVM": "yes" if kvm else "no", + "LOG": str(log), + } + env.pop("ROOT", None) + result = subprocess.run( + ["/bin/bash", "-c", HOST_COMMANDS + SCRIPT], + env=env, text=True, capture_output=True, + ) + return result, log.read_text().splitlines() if log.exists() else [] + + def test_fresh_ssd_mounts_xfs_directly_at_worker_state_path(self): + result, commands = self.run_init() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(commands, ["format-xfs", "mount /var/lib/braintrust-sandboxes/1"]) + + def test_restart_and_reboot_preserve_existing_xfs(self): + for mounted in (True, False): + with self.subTest(mounted=mounted): + result, commands = self.run_init("xfs", mounted=mounted) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(commands, [] if mounted else ["mount /var/lib/braintrust-sandboxes/1"]) + + def test_existing_non_xfs_filesystem_is_never_reformatted(self): + for mounted in (True, False): + with self.subTest(mounted=mounted): + result, commands = self.run_init("ext4", mounted=mounted) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(commands, []) + + def test_xfs_subdirectory_is_not_accepted_as_a_filesystem_root(self): + result, commands = self.run_init("xfs", mounted=True, fsroot="/1") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(commands, []) + + def test_multiple_ssds_are_striped_before_formatting(self): + result, commands = self.run_init(disks=2) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(commands, ["raid", "format-xfs", "mount /var/lib/braintrust-sandboxes/1"]) + + def test_missing_ssd_or_kvm_prevents_disk_initialization(self): + for options in ({"disks": 0}, {"kvm": False}): + with self.subTest(options=options): + result, commands = self.run_init(**options) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(commands, []) + + +if __name__ == "__main__": + unittest.main() + From 3b5396f39d1bd5e3a7aaecea013eb1043316b408 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Mon, 26 Oct 2026 10:43:04 -0700 Subject: [PATCH 14/25] Trim Loop Runtime tests and fix worker init dependencies --- .../examples/gke-loop-runtime/README.md | 4 - .../examples/gke-loop-runtime/node-init.yaml | 42 +++++--- .../tests/loop-runtime-deployment_test.yaml | 24 +---- .../loop-runtime-egress-service_test.yaml | 60 ++--------- tests/test_sandbox_node_init.py | 101 ------------------ 5 files changed, 40 insertions(+), 191 deletions(-) delete mode 100644 tests/test_sandbox_node_init.py diff --git a/braintrust/examples/gke-loop-runtime/README.md b/braintrust/examples/gke-loop-runtime/README.md index b253910..e1a725b 100644 --- a/braintrust/examples/gke-loop-runtime/README.md +++ b/braintrust/examples/gke-loop-runtime/README.md @@ -21,10 +21,6 @@ gke_isolated_workers = { kubectl --context apply -f braintrust/examples/gke-loop-runtime/node-init.yaml ``` - The worker image refuses to start until its state path is an XFS filesystem - root, including after node reboots. Node initialization preserves existing - XFS data and refuses to reformat disks containing another filesystem. - 2. Create the mTLS Secrets in both clusters: ```sh diff --git a/braintrust/examples/gke-loop-runtime/node-init.yaml b/braintrust/examples/gke-loop-runtime/node-init.yaml index 03bca41..9988152 100644 --- a/braintrust/examples/gke-loop-runtime/node-init.yaml +++ b/braintrust/examples/gke-loop-runtime/node-init.yaml @@ -74,24 +74,15 @@ spec: operator: Equal value: "true" effect: NoSchedule - hostPID: true automountServiceAccountToken: false initContainers: - name: init - image: ubuntu:24.04 - # Run in the host mount namespace so kubelet sees the mount. - command: - - nsenter - - --target=1 - - --mount - - --uts - - --ipc - - --net - - --pid - - -- - - bash - - -c - - "$(SCRIPT)" + image: alpine:3.22 + command: [sh, -ec] + args: + - | + apk add --no-cache bash util-linux xfsprogs mdadm + exec bash -c "$SCRIPT" env: - name: SCRIPT valueFrom: @@ -100,6 +91,14 @@ spec: key: init.sh securityContext: privileged: true + volumeMounts: + - name: state + mountPath: /var/lib/braintrust-sandboxes + mountPropagation: Bidirectional + - name: devices + mountPath: /dev + - name: netns + mountPath: /run/netns containers: - name: pause image: registry.k8s.io/pause:3.10 @@ -107,3 +106,16 @@ spec: requests: cpu: 10m memory: 16Mi + volumes: + - name: state + hostPath: + path: /var/lib/braintrust-sandboxes + type: DirectoryOrCreate + - name: devices + hostPath: + path: /dev + type: Directory + - name: netns + hostPath: + path: /run/netns + type: DirectoryOrCreate diff --git a/braintrust/tests/loop-runtime-deployment_test.yaml b/braintrust/tests/loop-runtime-deployment_test.yaml index 84c0a6a..a101aeb 100644 --- a/braintrust/tests/loop-runtime-deployment_test.yaml +++ b/braintrust/tests/loop-runtime-deployment_test.yaml @@ -63,11 +63,6 @@ tests: loopRuntime.service.port: 4888 loopRuntime.service.portName: runtime asserts: - - contains: - path: spec.template.spec.containers[0].ports - content: - name: runtime - containerPort: 4888 - equal: path: spec.template.spec.containers[0].livenessProbe.httpGet.port value: 4888 @@ -91,29 +86,20 @@ tests: path: spec.template.spec.containers[0].readinessProbe.httpGet.port value: runtime - - it: should retain disabled probes + - it: should retain disabled and non-HTTP probes template: loop-runtime-deployment.yaml values: *values set: loopRuntime.livenessProbe: null - loopRuntime.readinessProbe: null + loopRuntime.readinessProbe.httpGet: null + loopRuntime.readinessProbe.exec.command: ["true"] asserts: - notExists: path: spec.template.spec.containers[0].livenessProbe - - notExists: - path: spec.template.spec.containers[0].readinessProbe - - - it: should not add an HTTP port to an exec probe - template: loop-runtime-deployment.yaml - values: *values - set: - loopRuntime.livenessProbe.httpGet: null - loopRuntime.livenessProbe.exec.command: ["true"] - asserts: - isNull: - path: spec.template.spec.containers[0].livenessProbe.httpGet + path: spec.template.spec.containers[0].readinessProbe.httpGet - equal: - path: spec.template.spec.containers[0].livenessProbe.exec.command + path: spec.template.spec.containers[0].readinessProbe.exec.command value: ["true"] - it: should default null replicas to one diff --git a/braintrust/tests/loop-runtime-egress-service_test.yaml b/braintrust/tests/loop-runtime-egress-service_test.yaml index 026c9fc..5610bec 100644 --- a/braintrust/tests/loop-runtime-egress-service_test.yaml +++ b/braintrust/tests/loop-runtime-egress-service_test.yaml @@ -42,86 +42,42 @@ tests: values: *values set: loopRuntime.sandbox.egressGateway.sourceRanges: ["256.0.0.1/24"] - asserts: + asserts: &invalid-range - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"256.0.0.1/24\" is not a CIDR range" + errorPattern: "is not a CIDR range$" - it: should reject out of range prefixes values: *values set: loopRuntime.sandbox.egressGateway.sourceRanges: ["10.0.0.0/33"] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"10.0.0.0/33\" is not a CIDR range" + asserts: *invalid-range - it: should reject oversized octets and prefixes values: *values set: loopRuntime.sandbox.egressGateway.sourceRanges: ["999.999.999.999/999"] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"999.999.999.999/999\" is not a CIDR range" + asserts: *invalid-range - it: should reject an oversized prefix values: *values set: loopRuntime.sandbox.egressGateway.sourceRanges: ["10.0.0.0/999"] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"10.0.0.0/999\" is not a CIDR range" + asserts: *invalid-range - it: should reject missing octets values: *values set: loopRuntime.sandbox.egressGateway.sourceRanges: ["10.0.0/24"] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"10.0.0/24\" is not a CIDR range" - - - it: should reject IPv6 - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: ["::1/128"] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"::1/128\" is not a CIDR range" - - - it: should reject a negative prefix - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: ["10.0.0.0/-1"] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"10.0.0.0/-1\" is not a CIDR range" + asserts: *invalid-range - it: should reject a padded range values: *values set: loopRuntime.sandbox.egressGateway.sourceRanges: [" 10.0.0.0/24 "] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \" 10.0.0.0/24 \" is not a CIDR range" - - - it: should reject a whitespace range - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: [" "] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \" \" is not a CIDR range" - - - it: should reject a blank range - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: [""] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"\" is not a CIDR range" + asserts: *invalid-range - it: should reject a null range values: *values set: loopRuntime.sandbox.egressGateway.sourceRanges: [null] - asserts: - - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"\" is not a CIDR range" + asserts: *invalid-range diff --git a/tests/test_sandbox_node_init.py b/tests/test_sandbox_node_init.py deleted file mode 100644 index 4e6ce88..0000000 --- a/tests/test_sandbox_node_init.py +++ /dev/null @@ -1,101 +0,0 @@ -import os -from pathlib import Path -import subprocess -import tempfile -import textwrap -import unittest - - -NODE_INIT = Path(__file__).resolve().parents[1] / "braintrust/examples/gke-loop-runtime/node-init.yaml" -SCRIPT = textwrap.dedent(NODE_INIT.read_text().split(" init.sh: |\n", 1)[1].split("\n---", 1)[0]) - -# Execute the deployed script, substituting only host/device operations. -HOST_COMMANDS = r""" -modprobe() { return 0; } -test() { - if [[ "$1" == -c && "$2" == /dev/kvm ]]; then - [[ "$KVM" == yes ]] - else - builtin test "$@" - fi -} -mkdir() { return 0; } -readlink() { printf '%s\n' "$2"; } -mountpoint() { [[ "$MOUNTED" == yes ]]; } -blkid() { [[ -n "$FSTYPE" ]] && printf '%s\n' "$FSTYPE"; } -mkfs.xfs() { echo format-xfs >>"$LOG"; FSTYPE=xfs; } -mkfs.ext4() { echo format-ext4 >>"$LOG"; FSTYPE=ext4; } -mdadm() { echo raid >>"$LOG"; } -mount() { echo "mount ${@: -1}" >>"$LOG"; MOUNTED=yes; FSROOT=/; } -findmnt() { - case "$3" in - FSTYPE) printf '%s\n' "$FSTYPE" ;; - FSROOT) printf '%s\n' "$FSROOT" ;; - esac -} -""" - - -class SandboxNodeInitTests(unittest.TestCase): - def run_init(self, filesystem="", mounted=False, fsroot="/", disks=1, kvm=True): - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - for index in range(disks): - (root / f"disk-{index}").touch() - log = root / "commands" - env = { - **os.environ, - "DEVICES": str(root / "disk-*"), - "FSTYPE": filesystem, - "MOUNTED": "yes" if mounted else "no", - "FSROOT": fsroot, - "KVM": "yes" if kvm else "no", - "LOG": str(log), - } - env.pop("ROOT", None) - result = subprocess.run( - ["/bin/bash", "-c", HOST_COMMANDS + SCRIPT], - env=env, text=True, capture_output=True, - ) - return result, log.read_text().splitlines() if log.exists() else [] - - def test_fresh_ssd_mounts_xfs_directly_at_worker_state_path(self): - result, commands = self.run_init() - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(commands, ["format-xfs", "mount /var/lib/braintrust-sandboxes/1"]) - - def test_restart_and_reboot_preserve_existing_xfs(self): - for mounted in (True, False): - with self.subTest(mounted=mounted): - result, commands = self.run_init("xfs", mounted=mounted) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(commands, [] if mounted else ["mount /var/lib/braintrust-sandboxes/1"]) - - def test_existing_non_xfs_filesystem_is_never_reformatted(self): - for mounted in (True, False): - with self.subTest(mounted=mounted): - result, commands = self.run_init("ext4", mounted=mounted) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(commands, []) - - def test_xfs_subdirectory_is_not_accepted_as_a_filesystem_root(self): - result, commands = self.run_init("xfs", mounted=True, fsroot="/1") - self.assertNotEqual(result.returncode, 0) - self.assertEqual(commands, []) - - def test_multiple_ssds_are_striped_before_formatting(self): - result, commands = self.run_init(disks=2) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(commands, ["raid", "format-xfs", "mount /var/lib/braintrust-sandboxes/1"]) - - def test_missing_ssd_or_kvm_prevents_disk_initialization(self): - for options in ({"disks": 0}, {"kvm": False}): - with self.subTest(options=options): - result, commands = self.run_init(**options) - self.assertNotEqual(result.returncode, 0) - self.assertEqual(commands, []) - - -if __name__ == "__main__": - unittest.main() - From b017ecc6fcb8d4f3bf164113dd6b934c6fdbe187 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Mon, 26 Oct 2026 12:09:51 -0700 Subject: [PATCH 15/25] Validate egress CIDRs for strict Kubernetes admission --- braintrust/templates/_helpers.tpl | 14 +++++++++-- .../loop-runtime-egress-service_test.yaml | 24 ++++++++++++++++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index b3c232d..c16bcb1 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -177,12 +177,22 @@ predates Loop Runtime have no loopRuntime map, which means disabled. {{- end -}} {{- range $sandbox.egressGateway.sourceRanges -}} {{- $cidr := toString . -}} -{{- $valid := regexMatch "^([0-9]{1,3}\\.){3}[0-9]{1,3}/[0-9]{1,2}$" $cidr -}} +{{- $valid := regexMatch "^((0|[1-9][0-9]{0,2})\\.){3}(0|[1-9][0-9]{0,2})/(0|[1-9][0-9]?)$" $cidr -}} {{- if $valid -}} {{- $parts := splitList "/" $cidr -}} -{{- $valid = le (int (index $parts 1)) 32 -}} +{{- $prefix := int (index $parts 1) -}} +{{- $valid = le $prefix 32 -}} +{{- $address := 0 -}} {{- range splitList "." (index $parts 0) -}} {{- $valid = and $valid (le (int .) 255) -}} +{{- $address = add (mul $address 256) (int .) -}} +{{- end -}} +{{- if $valid -}} +{{- $blockSize := 1 -}} +{{- range until (int (sub 32 $prefix)) -}} +{{- $blockSize = mul $blockSize 2 -}} +{{- end -}} +{{- $valid = eq (mod $address $blockSize) 0 -}} {{- end -}} {{- end -}} {{- if not $valid -}} diff --git a/braintrust/tests/loop-runtime-egress-service_test.yaml b/braintrust/tests/loop-runtime-egress-service_test.yaml index 5610bec..d419254 100644 --- a/braintrust/tests/loop-runtime-egress-service_test.yaml +++ b/braintrust/tests/loop-runtime-egress-service_test.yaml @@ -29,14 +29,14 @@ tests: path: spec.ports[0].port value: 4002 - - it: should retain IPv4 boundary ranges, host bits, and leading zeroes + - it: should retain IPv4 boundary ranges values: *values set: - loopRuntime.sandbox.egressGateway.sourceRanges: ["0.0.0.0/0","255.255.255.255/32","10.64.64.1/24","010.064.000.001/08"] + loopRuntime.sandbox.egressGateway.sourceRanges: ["0.0.0.0/0","255.255.255.255/32","10.64.64.0/24"] asserts: - equal: path: spec.loadBalancerSourceRanges - value: ["0.0.0.0/0","255.255.255.255/32","10.64.64.1/24","010.064.000.001/08"] + value: ["0.0.0.0/0","255.255.255.255/32","10.64.64.0/24"] - it: should reject out of range octets values: *values @@ -81,3 +81,21 @@ tests: set: loopRuntime.sandbox.egressGateway.sourceRanges: [null] asserts: *invalid-range + + - it: should reject zero-padded octets + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: ["010.064.000.000/8"] + asserts: *invalid-range + + - it: should reject zero-padded prefixes + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: ["10.0.0.0/08"] + asserts: *invalid-range + + - it: should reject host bits beyond the prefix + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: ["10.64.64.1/24"] + asserts: *invalid-range From a4903f4f0f60c87f1aac43f18ecd2f43df31a24d Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Mon, 26 Oct 2026 15:24:01 -0700 Subject: [PATCH 16/25] Wire Git authorization to the sandbox egress URL --- braintrust/templates/loop-runtime-configmap.yaml | 1 + braintrust/tests/loop-runtime-configmap_test.yaml | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/braintrust/templates/loop-runtime-configmap.yaml b/braintrust/templates/loop-runtime-configmap.yaml index 8046148..518f798 100644 --- a/braintrust/templates/loop-runtime-configmap.yaml +++ b/braintrust/templates/loop-runtime-configmap.yaml @@ -44,6 +44,7 @@ data: SANDBOX_SERVICE_TLS_KEY: /etc/sandbox-tls/client.key SANDBOX_SERVICE_TLS_CA: /etc/sandbox-tls/server-ca.pem LOOP_RUNTIME_SANDBOX_FORWARD_PROXY_LISTEN: "0.0.0.0:4002" + LOOP_RUNTIME_SANDBOX_EGRESS_GATEWAY_URL: {{ $sandbox.egressGateway.url | quote }} LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL: {{ $sandbox.egressGateway.url | quote }} LOOP_RUNTIME_SANDBOX_EGRESS_MODE: restricted LOOP_RUNTIME_ENABLE_SANDBOX_ADMIN: "true" diff --git a/braintrust/tests/loop-runtime-configmap_test.yaml b/braintrust/tests/loop-runtime-configmap_test.yaml index 0f6edc3..6fa69ff 100644 --- a/braintrust/tests/loop-runtime-configmap_test.yaml +++ b/braintrust/tests/loop-runtime-configmap_test.yaml @@ -44,6 +44,9 @@ tests: - equal: path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL value: http://loop-egress.sandbox.internal:4002 + - equal: + path: data.LOOP_RUNTIME_SANDBOX_EGRESS_GATEWAY_URL + value: http://loop-egress.sandbox.internal:4002 - equal: path: data.LOOP_RUNTIME_REDIS_NAMESPACE value: loop-runtime:RELEASE-NAME @@ -53,6 +56,18 @@ tests: - isNull: path: data.BRAINSTORE_LOCKS_URI + - it: should use the configured egress URL for Git authorization and sandbox traffic + values: *values + set: + loopRuntime.sandbox.egressGateway.url: http://custom-egress.internal:4002 + asserts: + - equal: + path: data.LOOP_RUNTIME_SANDBOX_EGRESS_GATEWAY_URL + value: http://custom-egress.internal:4002 + - equal: + path: data.LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL + value: http://custom-egress.internal:4002 + - it: should serve every org when orgName is empty values: *values set: From bbfd6d8fca1cdb6eb7647bba4bb56e67d2e44879 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Mon, 26 Oct 2026 18:13:49 -0700 Subject: [PATCH 17/25] Address GKE Loop Runtime deployment review --- .../examples/gke-loop-runtime/README.md | 20 ++- .../gke-loop-runtime/create-sandbox-tls.sh | 5 +- .../examples/gke-loop-runtime/node-init.yaml | 121 ------------------ .../gke-loop-runtime/sandbox-service.yaml | 2 + braintrust/templates/_helpers.tpl | 13 +- .../templates/loop-runtime-configmap.yaml | 4 +- .../tests/loop-runtime-configmap_test.yaml | 67 +++++++++- braintrust/values.yaml | 2 + 8 files changed, 93 insertions(+), 141 deletions(-) delete mode 100644 braintrust/examples/gke-loop-runtime/node-init.yaml diff --git a/braintrust/examples/gke-loop-runtime/README.md b/braintrust/examples/gke-loop-runtime/README.md index e1a725b..399daf0 100644 --- a/braintrust/examples/gke-loop-runtime/README.md +++ b/braintrust/examples/gke-loop-runtime/README.md @@ -14,28 +14,21 @@ gke_isolated_workers = { `<...>` values below come from `terraform output gke_isolated_workers` and `terraform output loop_runtime_service_account`. -1. Prepare worker nodes (mounts XFS directly at the worker's state path): - - ```sh - kubectl --context create namespace braintrust-sandbox - kubectl --context apply -f braintrust/examples/gke-loop-runtime/node-init.yaml - ``` - -2. Create the mTLS Secrets in both clusters: +1. Create the mTLS Secrets in both clusters: ```sh braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh "" ``` -3. Install the workers: +2. Install the workers. The worker release prepares local SSDs before starting Sandbox Service: ```sh helm --kube-context upgrade --install sandbox-service \ - https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/cd27a3983e304932a7e1617adf3105c2b7320a340f26bfea577df781827142f9/sandbox-service-0.1.0.tgz \ + https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/0140c182aaa9dd0e262bd638a4e868f7f6a7fe9700ac80ceb44b618b3108bd4b/sandbox-service-0.1.0.tgz \ --namespace braintrust-sandbox --values braintrust/examples/gke-loop-runtime/sandbox-service.yaml --wait ``` -4. Install this chart with your GKE values plus `values.yaml` from this directory: +3. Install this chart with your GKE values plus `values.yaml` from this directory: ```sh helm --kube-context upgrade --install braintrust ./braintrust --namespace braintrust \ @@ -46,6 +39,11 @@ Workers run at a fixed count. One `c4-standard-8-lssd` worker holds about seven 1 vCPU / 2 GiB sandboxes; beyond that, new sandboxes fail with `CapacityExhausted`. Raise `replicas` in `sandbox-service.yaml` to add workers. +Each runtime pod claims at most two conversations and has a 256Mi in-memory cache budget. +A two-turn GKE smoke run using Git, SQL, and shell tools peaked at 329MiB; the pod retains a 2Gi limit. +Sandbox administration is disabled by default. Set `loopRuntime.enableSandboxAdmin: true` +to allow authorized sysadmins to inspect and terminate sandboxes. + To shut down, set the worker release to `replicas: 0`, wait for the workers to drain, then disable `loopRuntime.enabled` and uninstall the worker release. Its pre-delete hook refuses to uninstall until workers drain. diff --git a/braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh b/braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh index 1366e60..9b7aa47 100755 --- a/braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh +++ b/braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh @@ -55,9 +55,10 @@ for purpose in server client; do done apply_secret() { - kubectl --context "$1" create namespace "$2" --dry-run=client -o yaml | kubectl --context "$1" apply -f - >/dev/null + kubectl --context "$1" create namespace "$2" --dry-run=client -o yaml | kubectl --context "$1" apply --server-side -f - >/dev/null kubectl --context "$1" -n "$2" create secret generic "$3" "${@:4}" --dry-run=client -o yaml | - kubectl --context "$1" apply -f - + kubectl --context "$1" apply --server-side -f - + kubectl --context "$1" -n "$2" annotate secret "$3" kubectl.kubernetes.io/last-applied-configuration- } # The worker controller and Loop Runtime present the client certificate and trust the server CA. diff --git a/braintrust/examples/gke-loop-runtime/node-init.yaml b/braintrust/examples/gke-loop-runtime/node-init.yaml deleted file mode 100644 index 9988152..0000000 --- a/braintrust/examples/gke-loop-runtime/node-init.yaml +++ /dev/null @@ -1,121 +0,0 @@ -# GKE node pools have no bootstrap hook, so this DaemonSet does what EKS node -# bootstrap does for the Sandbox Service worker chart: mount XFS at the worker's -# state path, create /run/netns, and check /dev/kvm. -apiVersion: v1 -kind: ConfigMap -metadata: - name: sandbox-node-init - namespace: braintrust-sandbox -data: - init.sh: | - #!/bin/bash - # The worker entrypoint refuses to start unless /state is an XFS filesystem - # root. This also keeps it off the boot disk while initialization runs. - set -euo pipefail - ROOT="${ROOT:-/var/lib/braintrust-sandboxes/1}" - DEVICES="${DEVICES:-/dev/disk/by-id/google-local-nvme-ssd-*}" - mkdir -p /run/netns - modprobe kvm_intel 2>/dev/null || modprobe kvm_amd 2>/dev/null || true - test -c /dev/kvm || { echo "/dev/kvm is missing; enable nested virtualization" >&2; exit 1; } - if ! mountpoint -q "$ROOT"; then - # shellcheck disable=SC2206 - disks=($DEVICES) - [ -e "${disks[0]}" ] || { echo "no local SSD matches $DEVICES" >&2; exit 1; } - if [ "${#disks[@]}" -eq 1 ]; then - device="$(readlink -f "${disks[0]}")" - else - # Reuse an array the kernel assembled at boot; otherwise stripe the SSDs. - first="$(basename "$(readlink -f "${disks[0]}")")" - device="" - for slaves in /sys/block/md*/slaves; do - if [ -e "$slaves/$first" ]; then device="/dev/$(basename "$(dirname "$slaves")")"; fi - done - if [ -z "$device" ]; then - device=/dev/md0 - mdadm --create "$device" --level=0 --force --run \ - --raid-devices="${#disks[@]}" "${disks[@]}" - fi - fi - fstype="$(blkid -s TYPE -o value "$device" || true)" - if [ -z "$fstype" ]; then - mkfs.xfs -q "$device" - elif [ "$fstype" != xfs ]; then - echo "$device contains $fstype; refusing to replace it with XFS" >&2 - exit 1 - fi - mkdir -p "$ROOT" - mount -o discard,defaults "$device" "$ROOT" - echo "mounted $device at $ROOT" - fi - if [ "$(findmnt -n -o FSTYPE -T "$ROOT")" != xfs ] || - [ "$(findmnt -n -o FSROOT -T "$ROOT")" != / ]; then - echo "$ROOT must be the root of an XFS filesystem" >&2 - exit 1 - fi ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: sandbox-node-init - namespace: braintrust-sandbox -spec: - selector: - matchLabels: - app: sandbox-node-init - template: - metadata: - labels: - app: sandbox-node-init - spec: - nodeSelector: - braintrust.io/workload: sandbox-service - tolerations: - - key: braintrust.io/sandbox - operator: Equal - value: "true" - effect: NoSchedule - automountServiceAccountToken: false - initContainers: - - name: init - image: alpine:3.22 - command: [sh, -ec] - args: - - | - apk add --no-cache bash util-linux xfsprogs mdadm - exec bash -c "$SCRIPT" - env: - - name: SCRIPT - valueFrom: - configMapKeyRef: - name: sandbox-node-init - key: init.sh - securityContext: - privileged: true - volumeMounts: - - name: state - mountPath: /var/lib/braintrust-sandboxes - mountPropagation: Bidirectional - - name: devices - mountPath: /dev - - name: netns - mountPath: /run/netns - containers: - - name: pause - image: registry.k8s.io/pause:3.10 - resources: - requests: - cpu: 10m - memory: 16Mi - volumes: - - name: state - hostPath: - path: /var/lib/braintrust-sandboxes - type: DirectoryOrCreate - - name: devices - hostPath: - path: /dev - type: Directory - - name: netns - hostPath: - path: /run/netns - type: DirectoryOrCreate diff --git a/braintrust/examples/gke-loop-runtime/sandbox-service.yaml b/braintrust/examples/gke-loop-runtime/sandbox-service.yaml index bf8fd05..d1bc21f 100644 --- a/braintrust/examples/gke-loop-runtime/sandbox-service.yaml +++ b/braintrust/examples/gke-loop-runtime/sandbox-service.yaml @@ -5,6 +5,8 @@ image: public.ecr.aws/braintrust/sandbox-service:b96d95eaa269031cdd6f286a17b9b3e # Fixed worker capacity. The worker node pool autoscales, so raising this is a values change. replicas: 1 gitResources: true +nodePrep: + enabled: true prewarmTemplates: - image: public.ecr.aws/braintrust/codex-runtime@sha256:35db9c98d4a74e20758a29b82f8a3a6552be0002eb439c8c1a0f434ac573ee21 resources: diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index c16bcb1..d8fa40a 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -157,25 +157,28 @@ predates Loop Runtime have no loopRuntime map, which means disabled. {{/* Validate the Loop Runtime configuration. */}} {{- define "braintrust.loopRuntime.validate" -}} {{- if include "braintrust.loopRuntime.enabled" . -}} -{{- if not (hasKey .Values.loopRuntime "sandbox") -}} +{{- range $key := list "sandbox" "image" "service" "serviceAccount" "resources" "annotations" -}} +{{- if not (kindIs "map" (get $.Values.loopRuntime $key)) -}} {{- fail "loopRuntime defaults are missing. Values reused from a chart release that predates Loop Runtime omit them; upgrade with --reset-then-reuse-values." -}} {{- end -}} +{{- end -}} {{- if ne .Values.cloud "google" -}} {{- fail "loopRuntime.enabled requires cloud: google. On AWS, Loop Runtime runs on ECS through the Terraform module." -}} {{- end -}} {{- $sandbox := .Values.loopRuntime.sandbox -}} +{{- $egress := $sandbox.egressGateway | default dict -}} {{- $_ := required "objectStorage.google.brainstoreBucket is required" .Values.objectStorage.google.brainstoreBucket -}} {{- $_ := required "objectStorage.google.apiBucket is required" .Values.objectStorage.google.apiBucket -}} {{- $_ := required "loopRuntime.image.repository is required" .Values.loopRuntime.image.repository -}} {{- $_ := required "loopRuntime.image.tag is required" (.Values.loopRuntime.image.tag | default "" | toString) -}} {{- $_ := required "loopRuntime.serviceAccount.googleServiceAccount is required" .Values.loopRuntime.serviceAccount.googleServiceAccount -}} {{- $_ := required "loopRuntime.sandbox.dnsUrl is required" $sandbox.dnsUrl -}} -{{- $_ := required "loopRuntime.sandbox.egressGateway.address is required" $sandbox.egressGateway.address -}} -{{- $_ := required "loopRuntime.sandbox.egressGateway.url is required" $sandbox.egressGateway.url -}} -{{- if not $sandbox.egressGateway.sourceRanges -}} +{{- $_ := required "loopRuntime.sandbox.egressGateway.address is required" $egress.address -}} +{{- $_ := required "loopRuntime.sandbox.egressGateway.url is required" $egress.url -}} +{{- if not $egress.sourceRanges -}} {{- fail "loopRuntime.sandbox.egressGateway.sourceRanges is required" -}} {{- end -}} -{{- range $sandbox.egressGateway.sourceRanges -}} +{{- range $egress.sourceRanges -}} {{- $cidr := toString . -}} {{- $valid := regexMatch "^((0|[1-9][0-9]{0,2})\\.){3}(0|[1-9][0-9]{0,2})/(0|[1-9][0-9]?)$" $cidr -}} {{- if $valid -}} diff --git a/braintrust/templates/loop-runtime-configmap.yaml b/braintrust/templates/loop-runtime-configmap.yaml index 518f798..8fed6bb 100644 --- a/braintrust/templates/loop-runtime-configmap.yaml +++ b/braintrust/templates/loop-runtime-configmap.yaml @@ -34,6 +34,8 @@ data: BRAINSTORE_CODE_BUNDLE_URI: "gs://{{ .Values.objectStorage.google.apiBucket }}" BRAINSTORE_CACHE_DIR: /mnt/tmp/brainstore BRAINSTORE_OBJECT_STORE_CACHE_FILE_SIZE: 5Gi + BRAINSTORE_OBJECT_STORE_CACHE_MEMORY_LIMIT: 256Mi + LOOP_RUNTIME_MAX_CLAIMED_CONVERSATIONS: "2" BRAINSTORE_CONTROL_PLANE_TELEMETRY: {{ .Values.global.controlPlaneTelemetry | quote }} NO_COLOR: "1" EXO_SANDBOX_SERVICE_DNS_URL: {{ $sandbox.dnsUrl | quote }} @@ -47,7 +49,7 @@ data: LOOP_RUNTIME_SANDBOX_EGRESS_GATEWAY_URL: {{ $sandbox.egressGateway.url | quote }} LOOP_RUNTIME_SANDBOX_SERVICE_EGRESS_GATEWAY_URL: {{ $sandbox.egressGateway.url | quote }} LOOP_RUNTIME_SANDBOX_EGRESS_MODE: restricted - LOOP_RUNTIME_ENABLE_SANDBOX_ADMIN: "true" + LOOP_RUNTIME_ENABLE_SANDBOX_ADMIN: {{ .Values.loopRuntime.enableSandboxAdmin | default false | quote }} {{- with (.Values.global.allowedOrgIds | default "" | toString | trim) }} ALLOWED_ORG_IDS: {{ . | quote }} {{- end }} diff --git a/braintrust/tests/loop-runtime-configmap_test.yaml b/braintrust/tests/loop-runtime-configmap_test.yaml index 6fa69ff..54b8c3f 100644 --- a/braintrust/tests/loop-runtime-configmap_test.yaml +++ b/braintrust/tests/loop-runtime-configmap_test.yaml @@ -23,7 +23,7 @@ tests: - __fixtures__/base-values.yaml - __fixtures__/google-values.yaml - __fixtures__/reuse-loop-runtime-enabled-values.yaml - asserts: + asserts: &missing-defaults - failedTemplate: errorMessage: "loopRuntime defaults are missing. Values reused from a chart release that predates Loop Runtime omit them; upgrade with --reset-then-reuse-values." @@ -53,9 +53,66 @@ tests: - equal: path: data.ORG_NAME value: test-org + - equal: + path: data.LOOP_RUNTIME_ENABLE_SANDBOX_ADMIN + value: "false" + - equal: + path: data.BRAINSTORE_OBJECT_STORE_CACHE_MEMORY_LIMIT + value: 256Mi + - equal: + path: data.LOOP_RUNTIME_MAX_CLAIMED_CONVERSATIONS + value: "2" - isNull: path: data.BRAINSTORE_LOCKS_URI + - it: should allow sandbox administration only when opted in + values: *values + set: + loopRuntime.enableSandboxAdmin: true + asserts: + - equal: + path: data.LOOP_RUNTIME_ENABLE_SANDBOX_ADMIN + value: "true" + + - it: should keep sandbox administration disabled for retained null values + values: *values + set: + loopRuntime.enableSandboxAdmin: null + asserts: + - equal: + path: data.LOOP_RUNTIME_ENABLE_SANDBOX_ADMIN + value: "false" + + - it: should explain reused values with sandbox overrides but no image defaults + values: *values + set: + loopRuntime.image: null + asserts: *missing-defaults + + - it: should explain reused values without service defaults + values: *values + set: + loopRuntime.service: null + asserts: *missing-defaults + + - it: should explain reused values without service account defaults + values: *values + set: + loopRuntime.serviceAccount: null + asserts: *missing-defaults + + - it: should explain reused values without resource defaults + values: *values + set: + loopRuntime.resources: null + asserts: *missing-defaults + + - it: should explain reused values without annotation defaults + values: *values + set: + loopRuntime.annotations: null + asserts: *missing-defaults + - it: should use the configured egress URL for Git authorization and sandbox traffic values: *values set: @@ -130,6 +187,14 @@ tests: - failedTemplate: errorMessage: "loopRuntime.sandbox.egressGateway.address is required" + - it: should explain a missing egress map + values: *values + set: + loopRuntime.sandbox.egressGateway: null + asserts: + - failedTemplate: + errorMessage: "loopRuntime.sandbox.egressGateway.address is required" + - it: should require egress source ranges values: *values set: diff --git a/braintrust/values.yaml b/braintrust/values.yaml index b97ee17..460551b 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -313,6 +313,8 @@ api: # workers in a separate GKE cluster; see examples/gke-loop-runtime. loopRuntime: enabled: false + # Allow Braintrust staff sysadmins to inspect and terminate sandboxes. + enableSandboxAdmin: false name: "braintrust-loop-runtime" labels: {} podLabels: {} From 7d83d409b1f7e7aa3419a9cf2df607859f9357c6 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Mon, 26 Oct 2026 18:38:56 -0700 Subject: [PATCH 18/25] Use worker chart with multiarch node preparation --- braintrust/examples/gke-loop-runtime/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/braintrust/examples/gke-loop-runtime/README.md b/braintrust/examples/gke-loop-runtime/README.md index 399daf0..62d7bd5 100644 --- a/braintrust/examples/gke-loop-runtime/README.md +++ b/braintrust/examples/gke-loop-runtime/README.md @@ -24,7 +24,7 @@ gke_isolated_workers = { ```sh helm --kube-context upgrade --install sandbox-service \ - https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/0140c182aaa9dd0e262bd638a4e868f7f6a7fe9700ac80ceb44b618b3108bd4b/sandbox-service-0.1.0.tgz \ + https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/fdd9e1be00e421df4b33ddcc68ac5c775c4b082d914c21aed4dbd056500638e8/sandbox-service-0.1.0.tgz \ --namespace braintrust-sandbox --values braintrust/examples/gke-loop-runtime/sandbox-service.yaml --wait ``` From a447ce7b452512b8c772dcdb520bed154137d8c2 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Mon, 26 Oct 2026 19:33:58 -0700 Subject: [PATCH 19/25] Use worker chart with GKE disk discovery fix --- braintrust/examples/gke-loop-runtime/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/braintrust/examples/gke-loop-runtime/README.md b/braintrust/examples/gke-loop-runtime/README.md index 62d7bd5..ede6800 100644 --- a/braintrust/examples/gke-loop-runtime/README.md +++ b/braintrust/examples/gke-loop-runtime/README.md @@ -24,7 +24,7 @@ gke_isolated_workers = { ```sh helm --kube-context upgrade --install sandbox-service \ - https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/fdd9e1be00e421df4b33ddcc68ac5c775c4b082d914c21aed4dbd056500638e8/sandbox-service-0.1.0.tgz \ + https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/30a6a5e0cb7cd27783cdf18f3adaaa446dc47e5b8c94eb566af221a05ae2abb2/sandbox-service-0.1.0.tgz \ --namespace braintrust-sandbox --values braintrust/examples/gke-loop-runtime/sandbox-service.yaml --wait ``` From d6bfc1939b4ebdb163eca7f20adcdb02019010c5 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Mon, 26 Oct 2026 19:57:56 -0700 Subject: [PATCH 20/25] Use shared Sandbox Service image for node prep --- braintrust/examples/gke-loop-runtime/README.md | 4 ++-- braintrust/examples/gke-loop-runtime/sandbox-service.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/braintrust/examples/gke-loop-runtime/README.md b/braintrust/examples/gke-loop-runtime/README.md index ede6800..b48f3c8 100644 --- a/braintrust/examples/gke-loop-runtime/README.md +++ b/braintrust/examples/gke-loop-runtime/README.md @@ -20,11 +20,11 @@ gke_isolated_workers = { braintrust/examples/gke-loop-runtime/create-sandbox-tls.sh "" ``` -2. Install the workers. The worker release prepares local SSDs before starting Sandbox Service: +2. Install the workers. The configured Sandbox Service image prepares local SSDs before starting the worker: ```sh helm --kube-context upgrade --install sandbox-service \ - https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/30a6a5e0cb7cd27783cdf18f3adaaa446dc47e5b8c94eb566af221a05ae2abb2/sandbox-service-0.1.0.tgz \ + https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/8c933b9aae1eb3985bb64fad22e7c69d7a1a477d5db603114408dba00f053a02/sandbox-service-0.1.0.tgz \ --namespace braintrust-sandbox --values braintrust/examples/gke-loop-runtime/sandbox-service.yaml --wait ``` diff --git a/braintrust/examples/gke-loop-runtime/sandbox-service.yaml b/braintrust/examples/gke-loop-runtime/sandbox-service.yaml index d1bc21f..3afc1a6 100644 --- a/braintrust/examples/gke-loop-runtime/sandbox-service.yaml +++ b/braintrust/examples/gke-loop-runtime/sandbox-service.yaml @@ -1,7 +1,7 @@ # Values for the Sandbox Service worker chart, installed in the isolated worker cluster. # Every <...> value comes from the GCP Terraform output gke_isolated_workers. hostname: "" -image: public.ecr.aws/braintrust/sandbox-service:b96d95eaa269031cdd6f286a17b9b3e4795355d1 +image: public.ecr.aws/braintrust/sandbox-service:gke-node-prep-2fe13797d8ce # Fixed worker capacity. The worker node pool autoscales, so raising this is a values change. replicas: 1 gitResources: true From 7196a6f3f9cafd0e5da05995d22da77b1b716780 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Tue, 27 Oct 2026 15:41:45 -0700 Subject: [PATCH 21/25] Use published Sandbox Service release for GKE --- braintrust/examples/gke-loop-runtime/README.md | 2 +- braintrust/examples/gke-loop-runtime/sandbox-service.yaml | 4 ++-- braintrust/values.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/braintrust/examples/gke-loop-runtime/README.md b/braintrust/examples/gke-loop-runtime/README.md index b48f3c8..6187165 100644 --- a/braintrust/examples/gke-loop-runtime/README.md +++ b/braintrust/examples/gke-loop-runtime/README.md @@ -24,7 +24,7 @@ gke_isolated_workers = { ```sh helm --kube-context upgrade --install sandbox-service \ - https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/8c933b9aae1eb3985bb64fad22e7c69d7a1a477d5db603114408dba00f053a02/sandbox-service-0.1.0.tgz \ + https://braintrust-cf-us-east-1.s3.amazonaws.com/sandbox-service/1f4b6826c61064575b3db956b67625133257bd8a6e49e4ea86cdb55012dd53a7/sandbox-service-0.1.0.tgz \ --namespace braintrust-sandbox --values braintrust/examples/gke-loop-runtime/sandbox-service.yaml --wait ``` diff --git a/braintrust/examples/gke-loop-runtime/sandbox-service.yaml b/braintrust/examples/gke-loop-runtime/sandbox-service.yaml index 3afc1a6..5396dec 100644 --- a/braintrust/examples/gke-loop-runtime/sandbox-service.yaml +++ b/braintrust/examples/gke-loop-runtime/sandbox-service.yaml @@ -1,14 +1,14 @@ # Values for the Sandbox Service worker chart, installed in the isolated worker cluster. # Every <...> value comes from the GCP Terraform output gke_isolated_workers. hostname: "" -image: public.ecr.aws/braintrust/sandbox-service:gke-node-prep-2fe13797d8ce +image: public.ecr.aws/braintrust/sandbox-service:38b05599cdc5e4c3a213bbf0718211b397183a7a # Fixed worker capacity. The worker node pool autoscales, so raising this is a values change. replicas: 1 gitResources: true nodePrep: enabled: true prewarmTemplates: - - image: public.ecr.aws/braintrust/codex-runtime@sha256:35db9c98d4a74e20758a29b82f8a3a6552be0002eb439c8c1a0f434ac573ee21 + - image: public.ecr.aws/braintrust/codex-runtime@sha256:490ab364fd6a787f73957d3d46bc5e173fb418c47d53bcf7a2d3afba8b419049 resources: vcpu_count: 1 memory_mib: 2048 diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 460551b..5690e74 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -328,7 +328,7 @@ loopRuntime: image: repository: public.ecr.aws/braintrust/loop-runtime # Tested with the Sandbox Service worker release in the GKE example. - tag: b96d95eaa269031cdd6f286a17b9b3e4795355d1 + tag: 38b05599cdc5e4c3a213bbf0718211b397183a7a pullPolicy: IfNotPresent service: # Optional name for service object. If not specified (empty), loopRuntime.name is used. @@ -374,7 +374,7 @@ loopRuntime: address: "" sourceRanges: [] url: "" - image: "public.ecr.aws/braintrust/codex-runtime@sha256:35db9c98d4a74e20758a29b82f8a3a6552be0002eb439c8c1a0f434ac573ee21" + image: "public.ecr.aws/braintrust/codex-runtime@sha256:490ab364fd6a787f73957d3d46bc5e173fb418c47d53bcf7a2d3afba8b419049" vcpuCount: 1 memoryMib: 2048 extraEnvVars: [] From 5c194cc127b7317b8e990139ea67ad0de57a91f5 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Tue, 27 Oct 2026 17:50:18 -0700 Subject: [PATCH 22/25] Document Loop API and gateway requirements --- braintrust/examples/gke-loop-runtime/README.md | 7 +++++++ braintrust/examples/gke-loop-runtime/values.yaml | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/braintrust/examples/gke-loop-runtime/README.md b/braintrust/examples/gke-loop-runtime/README.md index 6187165..b24af86 100644 --- a/braintrust/examples/gke-loop-runtime/README.md +++ b/braintrust/examples/gke-loop-runtime/README.md @@ -14,6 +14,13 @@ gke_isolated_workers = { `<...>` values below come from `terraform output gke_isolated_workers` and `terraform output loop_runtime_service_account`. +Use compatible API, Brainstore, and AI Gateway images for the runtime build. +The API must report `ff_version` 82 or newer, and the Rust gateway must support +`x-bt-provider-model-map`; the chart's default API image `v2.18.0` is too old. +Deploy the gateway with `aiGateway.enabled=true` first and wait for it to become +ready before applying this example, which turns on `aiGateway.useGateway`. +The TypeScript API proxy alone cannot restore the provider model IDs sent by Codex. + 1. Create the mTLS Secrets in both clusters: ```sh diff --git a/braintrust/examples/gke-loop-runtime/values.yaml b/braintrust/examples/gke-loop-runtime/values.yaml index 1c1aeec..b355e15 100644 --- a/braintrust/examples/gke-loop-runtime/values.yaml +++ b/braintrust/examples/gke-loop-runtime/values.yaml @@ -1,5 +1,9 @@ # Layer on top of your existing GKE values (for example, ../google-autopilot/values.yaml). -# Every value below comes from the GCP Terraform module outputs. +# The sandbox and service-account values come from the GCP Terraform module outputs. +aiGateway: + enabled: true + useGateway: true + loopRuntime: enabled: true serviceAccount: From 6337c84f240c01561d793b8120c6b54b5e189e46 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Thu, 29 Oct 2026 15:53:43 -0700 Subject: [PATCH 23/25] Document Loop placement on Standard nodes --- braintrust/examples/gke-loop-runtime/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/braintrust/examples/gke-loop-runtime/README.md b/braintrust/examples/gke-loop-runtime/README.md index b24af86..a352633 100644 --- a/braintrust/examples/gke-loop-runtime/README.md +++ b/braintrust/examples/gke-loop-runtime/README.md @@ -35,7 +35,9 @@ The TypeScript API proxy alone cannot restore the provider model IDs sent by Cod --namespace braintrust-sandbox --values braintrust/examples/gke-loop-runtime/sandbox-service.yaml --wait ``` -3. Install this chart with your GKE values plus `values.yaml` from this directory: +3. Install this chart with your GKE values plus `values.yaml` from this directory. + For a Standard primary cluster, set `loopRuntime.nodeSelector` to + `{"braintrust/workload": "services"}` to use the services pool: ```sh helm --kube-context upgrade --install braintrust ./braintrust --namespace braintrust \ From 0939ef0a61925651daecfb92d5ba24bd336ef33e Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Thu, 29 Oct 2026 16:56:45 -0700 Subject: [PATCH 24/25] Simplify Loop Runtime rendering and label assertions --- .../examples/gke-loop-runtime/README.md | 1 - .../templates/loop-runtime-deployment.yaml | 12 ++--- .../templates/loop-runtime-service.yaml | 3 +- .../tests/labels-merge-configmaps_test.yaml | 17 ++----- .../tests/labels-merge-isolation_test.yaml | 49 +++++-------------- .../labels-merge-serviceaccounts_test.yaml | 13 ++--- 6 files changed, 25 insertions(+), 70 deletions(-) diff --git a/braintrust/examples/gke-loop-runtime/README.md b/braintrust/examples/gke-loop-runtime/README.md index a352633..3db6923 100644 --- a/braintrust/examples/gke-loop-runtime/README.md +++ b/braintrust/examples/gke-loop-runtime/README.md @@ -49,7 +49,6 @@ Workers run at a fixed count. One `c4-standard-8-lssd` worker holds about seven `CapacityExhausted`. Raise `replicas` in `sandbox-service.yaml` to add workers. Each runtime pod claims at most two conversations and has a 256Mi in-memory cache budget. -A two-turn GKE smoke run using Git, SQL, and shell tools peaked at 329MiB; the pod retains a 2Gi limit. Sandbox administration is disabled by default. Set `loopRuntime.enableSandboxAdmin: true` to allow authorized sysadmins to inspect and terminate sandboxes. diff --git a/braintrust/templates/loop-runtime-deployment.yaml b/braintrust/templates/loop-runtime-deployment.yaml index c10f99f..f72c75d 100644 --- a/braintrust/templates/loop-runtime-deployment.yaml +++ b/braintrust/templates/loop-runtime-deployment.yaml @@ -68,21 +68,15 @@ spec: - containerPort: 4002 resources: {{- toYaml .Values.loopRuntime.resources | nindent 12 }} - {{- with .Values.loopRuntime.livenessProbe }} + {{- range $name := list "livenessProbe" "readinessProbe" }} + {{- with index $.Values.loopRuntime $name }} {{- $probe := deepCopy . -}} {{- if and (kindIs "map" $probe.httpGet) (not (hasKey $probe.httpGet "port")) -}} {{- $_ := set $probe.httpGet "port" $.Values.loopRuntime.service.port -}} {{- end }} - livenessProbe: + {{ $name }}: {{- toYaml $probe | nindent 12 }} {{- end }} - {{- with .Values.loopRuntime.readinessProbe }} - {{- $probe := deepCopy . -}} - {{- if and (kindIs "map" $probe.httpGet) (not (hasKey $probe.httpGet "port")) -}} - {{- $_ := set $probe.httpGet "port" $.Values.loopRuntime.service.port -}} - {{- end }} - readinessProbe: - {{- toYaml $probe | nindent 12 }} {{- end }} envFrom: - configMapRef: diff --git a/braintrust/templates/loop-runtime-service.yaml b/braintrust/templates/loop-runtime-service.yaml index 1174707..0bf20ac 100644 --- a/braintrust/templates/loop-runtime-service.yaml +++ b/braintrust/templates/loop-runtime-service.yaml @@ -1,12 +1,11 @@ {{- include "braintrust.loopRuntime.validate" . -}} {{- if include "braintrust.loopRuntime.enabled" . -}} -{{- $resourceLabels := mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels) -}} apiVersion: v1 kind: Service metadata: name: {{ .Values.loopRuntime.service.name | default .Values.loopRuntime.name }} namespace: {{ include "braintrust.namespace" . }} - {{- with $resourceLabels }} + {{- with (mergeOverwrite (deepCopy .Values.global.labels) (deepCopy .Values.loopRuntime.labels)) }} labels: {{- toYaml . | nindent 4 }} {{- end }} diff --git a/braintrust/tests/labels-merge-configmaps_test.yaml b/braintrust/tests/labels-merge-configmaps_test.yaml index 84ef5fd..ed0b910 100644 --- a/braintrust/tests/labels-merge-configmaps_test.yaml +++ b/braintrust/tests/labels-merge-configmaps_test.yaml @@ -171,16 +171,7 @@ tests: template: loop-runtime-configmap.yaml asserts: - equal: - path: metadata.labels.shared - value: global-cm - - equal: - path: metadata.labels["loop-specific"] - value: loop-cm - - isNull: - path: metadata.labels["api-specific"] - - isNull: - path: metadata.labels["reader-specific"] - - isNull: - path: metadata.labels["writer-specific"] - - isNull: - path: metadata.labels["fastreader-specific"] + path: metadata.labels + value: + shared: global-cm + loop-specific: loop-cm diff --git a/braintrust/tests/labels-merge-isolation_test.yaml b/braintrust/tests/labels-merge-isolation_test.yaml index 1833afc..4e7940a 100644 --- a/braintrust/tests/labels-merge-isolation_test.yaml +++ b/braintrust/tests/labels-merge-isolation_test.yaml @@ -526,39 +526,16 @@ tests: template: loop-runtime-deployment.yaml asserts: - equal: - path: metadata.labels.shared - value: global-loop - - equal: - path: metadata.labels.environment - value: production - - equal: - path: metadata.labels.component - value: loop-runtime - - equal: - path: metadata.labels["loop-specific"] - value: loop-only - - isNull: - path: metadata.labels["api-specific"] - - isNull: - path: metadata.labels["reader-specific"] - - isNull: - path: metadata.labels["writer-specific"] - - isNull: - path: metadata.labels["fastreader-specific"] - - equal: - path: spec.template.metadata.labels.shared - value: global-loop - - equal: - path: spec.template.metadata.labels["loop-specific"] - value: loop-only - - equal: - path: spec.template.metadata.labels["pod-specific"] - value: pod-only - - isNull: - path: spec.template.metadata.labels["api-specific"] - - isNull: - path: spec.template.metadata.labels["reader-specific"] - - isNull: - path: spec.template.metadata.labels["writer-specific"] - - isNull: - path: spec.template.metadata.labels["fastreader-specific"] + path: metadata.labels + value: &loopLabels + shared: global-loop + environment: production + component: loop-runtime + loop-specific: loop-only + - equal: + path: spec.template.metadata.labels + value: + <<: *loopLabels + pod-specific: pod-only + app: braintrust-loop-runtime + gke-workload-identity/use: "true" diff --git a/braintrust/tests/labels-merge-serviceaccounts_test.yaml b/braintrust/tests/labels-merge-serviceaccounts_test.yaml index 6b98530..8e661f9 100644 --- a/braintrust/tests/labels-merge-serviceaccounts_test.yaml +++ b/braintrust/tests/labels-merge-serviceaccounts_test.yaml @@ -61,12 +61,7 @@ tests: template: loop-runtime-serviceaccount.yaml asserts: - equal: - path: metadata.labels.shared - value: global-sa - - equal: - path: metadata.labels["loop-specific"] - value: loop-sa - - isNull: - path: metadata.labels["api-specific"] - - isNull: - path: metadata.labels["brainstore-specific"] + path: metadata.labels + value: + shared: global-sa + loop-specific: loop-sa From 5472a436c3770704cd19476fd7f85d3bac0537e7 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Thu, 29 Oct 2026 17:46:39 -0700 Subject: [PATCH 25/25] Simplify Loop CIDR validation and require a sandbox image --- .../examples/gke-loop-runtime/README.md | 7 +- braintrust/templates/_helpers.tpl | 25 +------ .../tests/loop-runtime-configmap_test.yaml | 35 ++++++++-- .../loop-runtime-egress-service_test.yaml | 65 ++----------------- 4 files changed, 40 insertions(+), 92 deletions(-) diff --git a/braintrust/examples/gke-loop-runtime/README.md b/braintrust/examples/gke-loop-runtime/README.md index 3db6923..6cd6776 100644 --- a/braintrust/examples/gke-loop-runtime/README.md +++ b/braintrust/examples/gke-loop-runtime/README.md @@ -6,8 +6,9 @@ GCP Terraform module with: ```hcl gke_isolated_workers = { - network_cidr = "10.64.0.0/16" # must not overlap existing ranges - discovery = { dns_name = "sandbox..internal" } + network_cidr = "10.64.0.0/16" # must not overlap existing ranges + authorized_cidrs = ["192.0.2.10/32"] # replace with operator and CI egress ranges + discovery = { dns_name = "sandbox..internal" } } ``` @@ -16,7 +17,7 @@ gke_isolated_workers = { Use compatible API, Brainstore, and AI Gateway images for the runtime build. The API must report `ff_version` 82 or newer, and the Rust gateway must support -`x-bt-provider-model-map`; the chart's default API image `v2.18.0` is too old. +`x-bt-provider-model-map`. Deploy the gateway with `aiGateway.enabled=true` first and wait for it to become ready before applying this example, which turns on `aiGateway.useGateway`. The TypeScript API proxy alone cannot restore the provider model IDs sent by Codex. diff --git a/braintrust/templates/_helpers.tpl b/braintrust/templates/_helpers.tpl index d8fa40a..116c108 100644 --- a/braintrust/templates/_helpers.tpl +++ b/braintrust/templates/_helpers.tpl @@ -173,35 +173,12 @@ predates Loop Runtime have no loopRuntime map, which means disabled. {{- $_ := required "loopRuntime.image.tag is required" (.Values.loopRuntime.image.tag | default "" | toString) -}} {{- $_ := required "loopRuntime.serviceAccount.googleServiceAccount is required" .Values.loopRuntime.serviceAccount.googleServiceAccount -}} {{- $_ := required "loopRuntime.sandbox.dnsUrl is required" $sandbox.dnsUrl -}} +{{- $_ := required "loopRuntime.sandbox.image is required" $sandbox.image -}} {{- $_ := required "loopRuntime.sandbox.egressGateway.address is required" $egress.address -}} {{- $_ := required "loopRuntime.sandbox.egressGateway.url is required" $egress.url -}} {{- if not $egress.sourceRanges -}} {{- fail "loopRuntime.sandbox.egressGateway.sourceRanges is required" -}} {{- end -}} -{{- range $egress.sourceRanges -}} -{{- $cidr := toString . -}} -{{- $valid := regexMatch "^((0|[1-9][0-9]{0,2})\\.){3}(0|[1-9][0-9]{0,2})/(0|[1-9][0-9]?)$" $cidr -}} -{{- if $valid -}} -{{- $parts := splitList "/" $cidr -}} -{{- $prefix := int (index $parts 1) -}} -{{- $valid = le $prefix 32 -}} -{{- $address := 0 -}} -{{- range splitList "." (index $parts 0) -}} -{{- $valid = and $valid (le (int .) 255) -}} -{{- $address = add (mul $address 256) (int .) -}} -{{- end -}} -{{- if $valid -}} -{{- $blockSize := 1 -}} -{{- range until (int (sub 32 $prefix)) -}} -{{- $blockSize = mul $blockSize 2 -}} -{{- end -}} -{{- $valid = eq (mod $address $blockSize) 0 -}} -{{- end -}} -{{- end -}} -{{- if not $valid -}} -{{- fail (printf "loopRuntime.sandbox.egressGateway.sourceRanges: %q is not a CIDR range" (toString .)) -}} -{{- end -}} -{{- end -}} {{- end -}} {{- end -}} diff --git a/braintrust/tests/loop-runtime-configmap_test.yaml b/braintrust/tests/loop-runtime-configmap_test.yaml index 54b8c3f..5a26ee1 100644 --- a/braintrust/tests/loop-runtime-configmap_test.yaml +++ b/braintrust/tests/loop-runtime-configmap_test.yaml @@ -203,18 +203,41 @@ tests: - failedTemplate: errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges is required" - - it: should reject a blank egress source range next to a valid one + - it: should reject a null image tag values: *values set: - loopRuntime.sandbox.egressGateway.sourceRanges: ["10.64.64.0/24", ""] + loopRuntime.image.tag: null asserts: - failedTemplate: - errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges: \"\" is not a CIDR range" + errorMessage: "loopRuntime.image.tag is required" - - it: should reject a null image tag + - it: should require a sandbox image when enabled values: *values set: - loopRuntime.image.tag: null + loopRuntime.sandbox.image: "" + asserts: &missing-sandbox-image + - failedTemplate: + errorMessage: "loopRuntime.sandbox.image is required" + + - it: should require a retained null sandbox image when enabled + values: *values + set: + loopRuntime.sandbox.image: null + asserts: *missing-sandbox-image + + - it: should ignore missing sandbox configuration when disabled + values: *values + set: + loopRuntime.enabled: false + loopRuntime.sandbox: null + asserts: + - hasDocuments: + count: 0 + + - it: should require retained null source ranges + values: *values + set: + loopRuntime.sandbox.egressGateway.sourceRanges: null asserts: - failedTemplate: - errorMessage: "loopRuntime.image.tag is required" + errorMessage: "loopRuntime.sandbox.egressGateway.sourceRanges is required" diff --git a/braintrust/tests/loop-runtime-egress-service_test.yaml b/braintrust/tests/loop-runtime-egress-service_test.yaml index d419254..48f207d 100644 --- a/braintrust/tests/loop-runtime-egress-service_test.yaml +++ b/braintrust/tests/loop-runtime-egress-service_test.yaml @@ -38,64 +38,11 @@ tests: path: spec.loadBalancerSourceRanges value: ["0.0.0.0/0","255.255.255.255/32","10.64.64.0/24"] - - it: should reject out of range octets + - it: should pass supplied ranges through for Kubernetes validation values: *values set: - loopRuntime.sandbox.egressGateway.sourceRanges: ["256.0.0.1/24"] - asserts: &invalid-range - - failedTemplate: - errorPattern: "is not a CIDR range$" - - - it: should reject out of range prefixes - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: ["10.0.0.0/33"] - asserts: *invalid-range - - - it: should reject oversized octets and prefixes - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: ["999.999.999.999/999"] - asserts: *invalid-range - - - it: should reject an oversized prefix - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: ["10.0.0.0/999"] - asserts: *invalid-range - - - it: should reject missing octets - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: ["10.0.0/24"] - asserts: *invalid-range - - - it: should reject a padded range - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: [" 10.0.0.0/24 "] - asserts: *invalid-range - - - it: should reject a null range - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: [null] - asserts: *invalid-range - - - it: should reject zero-padded octets - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: ["010.064.000.000/8"] - asserts: *invalid-range - - - it: should reject zero-padded prefixes - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: ["10.0.0.0/08"] - asserts: *invalid-range - - - it: should reject host bits beyond the prefix - values: *values - set: - loopRuntime.sandbox.egressGateway.sourceRanges: ["10.64.64.1/24"] - asserts: *invalid-range + loopRuntime.sandbox.egressGateway.sourceRanges: ["not-a-cidr", "2001:db8::/64"] + asserts: + - equal: + path: spec.loadBalancerSourceRanges + value: ["not-a-cidr", "2001:db8::/64"]