From 5e444a02f06cd80c52cfbff4cf97f8298444767e Mon Sep 17 00:00:00 2001 From: Jeff McCollum Date: Wed, 1 Jul 2026 21:26:06 -0500 Subject: [PATCH] add ca bundle support --- braintrust/README.md | 45 +++++ braintrust/templates/api-deployment.yaml | 32 +++- .../tests/__fixtures__/base-values.yaml | 6 + braintrust/tests/api-deployment_test.yaml | 178 ++++++++++++++++++ braintrust/values.yaml | 8 + 5 files changed, 267 insertions(+), 2 deletions(-) diff --git a/braintrust/README.md b/braintrust/README.md index 5b66cd1..b14b845 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -50,6 +50,51 @@ The CSI driver will: 2. Automatically sync them to the `braintrust-secrets` Kubernetes secret 3. Keep the secrets in sync as they change in Key Vault +## Optional: Enterprise CA bundle for user-code runtimes + +If your organization uses private or enterprise CAs, you can provide a dedicated Secret for a CA bundle and configure the API container, plus user-code runtimes that inherit environment from it, to trust that bundle. + +This feature: + +1. Uses a separate Secret for the CA bundle. +2. Mounts only the configured Secret key via `items` (not the full Secret). +3. Does not modify files under the system certificate store; clients that honor the configured env vars will trust the mounted bundle instead of (or in addition to) system defaults. +4. Does not require root. +5. Sets standard CA bundle environment variables on the API container (`NODE_EXTRA_CA_CERTS`, `REQUESTS_CA_BUNDLE`, `SSL_CERT_FILE`, `CURL_CA_BUNDLE`, `AWS_CA_BUNDLE`, and `PIP_CERT`). + +Example values: + +```yaml +api: + customCA: + enabled: true + secretName: braintrust-runtime-ca + secretKey: ca-bundle.pem + mountPath: /etc/braintrust/runtime-ca + filename: ca-bundle.pem +``` + +Create the Secret before installing or upgrading the chart: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: braintrust-runtime-ca +type: Opaque +stringData: + ca-bundle.pem: | + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- +``` + +The configured key should contain a full CA bundle, not only your private CA chain. Several of the environment variables set by this feature cause clients to use the configured file as their CA bundle, so include the public root certificates your workloads still need along with your enterprise root and intermediate certificates. + +`NODE_EXTRA_CA_CERTS` appends to the default trust store in Node.js, while `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`, and `CURL_CA_BUNDLE` typically replace the default bundle for OpenSSL, Python, and curl clients. Because all of these variables point at the same mounted file, behavior can differ by runtime: a bundle with only your enterprise CA may work for Node.js user code but break Python or curl outbound HTTPS unless public roots are included. + +Kubernetes limits each Secret to 1 MiB. A normal combined CA bundle is typically well below that limit, but very large enterprise bundles should be checked before rollout. + ## GKE with Local SSDs Braintrust requires local SSDs for maximum disk performance. Configuration varies depending on whether you're using GKE Autopilot or Standard mode. diff --git a/braintrust/templates/api-deployment.yaml b/braintrust/templates/api-deployment.yaml index fbbde8b..24d7e73 100644 --- a/braintrust/templates/api-deployment.yaml +++ b/braintrust/templates/api-deployment.yaml @@ -127,10 +127,25 @@ spec: value: {{ .Values.api.healthServer.host | quote }} - name: TS_API_HEALTHSERVER_PORT value: {{ .Values.api.healthServer.port | quote }} + {{- if .Values.api.customCA.enabled }} + {{- $customCAPath := printf "%s/%s" (required "api.customCA.mountPath is required when api.customCA.enabled is true" .Values.api.customCA.mountPath) (required "api.customCA.filename is required when api.customCA.enabled is true" .Values.api.customCA.filename) }} + - name: NODE_EXTRA_CA_CERTS + value: {{ $customCAPath | quote }} + - name: REQUESTS_CA_BUNDLE + value: {{ $customCAPath | quote }} + - name: SSL_CERT_FILE + value: {{ $customCAPath | quote }} + - name: CURL_CA_BUNDLE + value: {{ $customCAPath | quote }} + - name: AWS_CA_BUNDLE + value: {{ $customCAPath | quote }} + - name: PIP_CERT + value: {{ $customCAPath | quote }} + {{- end }} {{- if .Values.api.extraEnvVars }} {{- toYaml .Values.api.extraEnvVars | nindent 12 }} {{- end }} - {{- if or .Values.api.tmpVolume.enabled (and (eq .Values.cloud "azure") .Values.azure.enableAzureKeyVaultDriver) }} + {{- if or .Values.api.tmpVolume.enabled (and (eq .Values.cloud "azure") .Values.azure.enableAzureKeyVaultDriver) .Values.api.customCA.enabled }} volumeMounts: {{- if .Values.api.tmpVolume.enabled }} - name: tmp-volume @@ -141,12 +156,17 @@ spec: mountPath: "/mnt/secrets-store" readOnly: true {{- end }} + {{- if .Values.api.customCA.enabled }} + - name: custom-ca-bundle + mountPath: {{ required "api.customCA.mountPath is required when api.customCA.enabled is true" .Values.api.customCA.mountPath | quote }} + readOnly: true + {{- end }} {{- end }} {{- with .Values.api.extraContainers }} {{- toYaml . | nindent 8 }} {{- end }} volumes: - {{- if or .Values.api.tmpVolume.enabled (and (eq .Values.cloud "azure") .Values.azure.enableAzureKeyVaultDriver) .Values.api.extraVolumes }} + {{- if or .Values.api.tmpVolume.enabled (and (eq .Values.cloud "azure") .Values.azure.enableAzureKeyVaultDriver) .Values.api.customCA.enabled .Values.api.extraVolumes }} {{- if .Values.api.tmpVolume.enabled }} - name: tmp-volume emptyDir: @@ -164,6 +184,14 @@ spec: volumeAttributes: secretProviderClass: {{ .Values.azure.keyVaultName }} {{- end }} + {{- if .Values.api.customCA.enabled }} + - name: custom-ca-bundle + secret: + secretName: {{ required "api.customCA.secretName is required when api.customCA.enabled is true" .Values.api.customCA.secretName | quote }} + items: + - key: {{ required "api.customCA.secretKey is required when api.customCA.enabled is true" .Values.api.customCA.secretKey | quote }} + path: {{ required "api.customCA.filename is required when api.customCA.enabled is true" .Values.api.customCA.filename | quote }} + {{- end }} {{- with .Values.api.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/braintrust/tests/__fixtures__/base-values.yaml b/braintrust/tests/__fixtures__/base-values.yaml index d603711..f06cf53 100644 --- a/braintrust/tests/__fixtures__/base-values.yaml +++ b/braintrust/tests/__fixtures__/base-values.yaml @@ -39,6 +39,12 @@ api: cpu: "4" memory: "8Gi" allowCodeFunctionExecution: false + customCA: + enabled: false + secretName: "braintrust-runtime-ca" + secretKey: "ca-bundle.pem" + mountPath: "/etc/braintrust/runtime-ca" + filename: "ca-bundle.pem" backfillDisableHistorical: false backfillDisableNonhistorical: false allowInvalidBase64: false diff --git a/braintrust/tests/api-deployment_test.yaml b/braintrust/tests/api-deployment_test.yaml index b862d1f..cd833bd 100644 --- a/braintrust/tests/api-deployment_test.yaml +++ b/braintrust/tests/api-deployment_test.yaml @@ -169,6 +169,184 @@ tests: name: ANOTHER_VAR value: "another-value" + - it: should mount only the dedicated custom CA secret key and set standard CA env vars when enabled + template: api-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + api.customCA.enabled: true + api.customCA.secretName: braintrust-runtime-ca + api.customCA.secretKey: ca-bundle.pem + api.customCA.mountPath: /etc/braintrust/runtime-ca + api.customCA.filename: ca-bundle.pem + release: + namespace: "braintrust" + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: custom-ca-bundle + mountPath: /etc/braintrust/runtime-ca + readOnly: true + - contains: + path: spec.template.spec.volumes + content: + name: custom-ca-bundle + secret: + secretName: braintrust-runtime-ca + items: + - key: ca-bundle.pem + path: ca-bundle.pem + - contains: + path: spec.template.spec.containers[0].env + content: + name: NODE_EXTRA_CA_CERTS + value: /etc/braintrust/runtime-ca/ca-bundle.pem + - contains: + path: spec.template.spec.containers[0].env + content: + name: REQUESTS_CA_BUNDLE + value: /etc/braintrust/runtime-ca/ca-bundle.pem + - contains: + path: spec.template.spec.containers[0].env + content: + name: SSL_CERT_FILE + value: /etc/braintrust/runtime-ca/ca-bundle.pem + - contains: + path: spec.template.spec.containers[0].env + content: + name: CURL_CA_BUNDLE + value: /etc/braintrust/runtime-ca/ca-bundle.pem + - contains: + path: spec.template.spec.containers[0].env + content: + name: AWS_CA_BUNDLE + value: /etc/braintrust/runtime-ca/ca-bundle.pem + - contains: + path: spec.template.spec.containers[0].env + content: + name: PIP_CERT + value: /etc/braintrust/runtime-ca/ca-bundle.pem + - notContains: + path: spec.template.spec.containers[0].env + content: + name: SSL_CERT_DIR + - notContains: + path: spec.template.spec.containers[0].env + content: + name: BRAINTRUST_RUNTIME_CA_BUNDLE_PATH + + - it: should not include custom CA volume mounts or CA env vars when disabled + template: api-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + api.customCA.enabled: false + api.customCA.secretKey: ca-bundle.pem + release: + namespace: "braintrust" + asserts: + - equal: + path: spec.template.spec.volumes + value: [] + - isNull: + path: spec.template.spec.containers[0].volumeMounts + - notContains: + path: spec.template.spec.containers[0].env + content: + name: NODE_EXTRA_CA_CERTS + - notContains: + path: spec.template.spec.containers[0].env + content: + name: REQUESTS_CA_BUNDLE + - notContains: + path: spec.template.spec.containers[0].env + content: + name: SSL_CERT_FILE + - notContains: + path: spec.template.spec.containers[0].env + content: + name: CURL_CA_BUNDLE + - notContains: + path: spec.template.spec.containers[0].env + content: + name: AWS_CA_BUNDLE + - notContains: + path: spec.template.spec.containers[0].env + content: + name: PIP_CERT + + - it: should include custom CA and extraVolumes when both are configured + template: api-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + api.customCA.enabled: true + api.extraVolumes: + - name: otel-config + configMap: + name: otel-collector-config + release: + namespace: "braintrust" + asserts: + - contains: + path: spec.template.spec.volumes + content: + name: custom-ca-bundle + secret: + secretName: braintrust-runtime-ca + items: + - key: ca-bundle.pem + path: ca-bundle.pem + - contains: + path: spec.template.spec.volumes + content: + name: otel-config + configMap: + name: otel-collector-config + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: custom-ca-bundle + mountPath: /etc/braintrust/runtime-ca + readOnly: true + + - it: should include custom CA and tmpVolume mounts when both are configured + template: api-deployment.yaml + values: + - __fixtures__/base-values.yaml + set: + api.customCA.enabled: true + api.tmpVolume.enabled: true + release: + namespace: "braintrust" + asserts: + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: custom-ca-bundle + mountPath: /etc/braintrust/runtime-ca + readOnly: true + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: tmp-volume + mountPath: /tmp + - contains: + path: spec.template.spec.volumes + content: + name: custom-ca-bundle + secret: + secretName: braintrust-runtime-ca + items: + - key: ca-bundle.pem + path: ca-bundle.pem + - contains: + path: spec.template.spec.volumes + content: + name: tmp-volume + emptyDir: {} + - it: should merge global and api labels template: api-deployment.yaml values: diff --git a/braintrust/values.yaml b/braintrust/values.yaml index d2f71cb..9234680 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -128,6 +128,14 @@ api: sizeLimit: "" # Allow running user generated code functions (e.g. scorers/tools) allowCodeFunctionExecution: false + # Optional custom CA bundle support for user-code runtimes. + # References a dedicated secret and mounts only one key via `items`. + customCA: + enabled: false + secretName: "braintrust-runtime-ca" + secretKey: "ca-bundle.pem" + mountPath: "/etc/braintrust/runtime-ca" + filename: "ca-bundle.pem" # Brainstore backfill configuration. These defaults are fine for most cases. backfillDisableHistorical: false backfillDisableNonhistorical: false