From e33d12bb6bdef91d35aeba71cd99758276886843 Mon Sep 17 00:00:00 2001 From: Guzman Date: Thu, 17 Sep 2026 14:05:28 +0000 Subject: [PATCH 1/7] CMP-4504: Run tls-scanner PQC/ML-KEM readiness check in FIO e2e-aws job Appends the tls-scanner-run step registry ref to the existing e2e-aws test for file-integrity-operator, scoped to the openshift-file-integrity namespace where the operator and its aide daemonset run, with PQC_CHECK=true to assert TLS 1.3 + ML-KEM (X25519MLKEM768) readiness. Follows the pattern already merged for RHOAI (openshift/release#82555): append '- ref: tls-scanner-run' to the existing e2e test's step list and add the required tls-scanner-tool base image, rather than provisioning a separate cluster/job as compliance-operator's manual approach did. Verified manually on a live OCP 4.22.13 cluster: deployed FIO built from the current Go 1.25 source and ran the actual upstream tls-scanner tool against the openshift-file-integrity namespace with --pqc-check. The operator's metrics endpoint (port 8585) reported: TLS 1.3: SUPPORTED, ML-KEM: SUPPORTED, ML-KEM KEMs: X25519MLKEM768 PQC COMPLIANCE CHECK: PASSED No code changes were required in file-integrity-operator itself: the metrics TLS listener (pkg/controller/metrics/metrics.go) sets no CurvePreferences/MaxVersion, so Go 1.25's default TLS 1.3 group preference (which includes X25519MLKEM768) already applies. Ref: https://redhat.atlassian.net/browse/CMP-4504 --- .../openshift-file-integrity-operator-master.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml b/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml index f302c496ed0a9..1ad64184d51f3 100644 --- a/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml +++ b/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml @@ -11,6 +11,10 @@ base_images: name: rosa-aws-cli namespace: ci tag: release + tls-scanner-tool: + name: "4.22" + namespace: ocp + tag: tls-scanner-tool build_root: image_stream_tag: name: builder @@ -69,6 +73,9 @@ tests: pipeline_skip_if_only_changed: ^(?:\.github|\.tekton)|\.md$|bundle-hack|^(?:\.gitignore|OWNERS|LICENSE)$ steps: cluster_profile: openshift-org-aws + env: + PQC_CHECK: "true" + SCAN_NAMESPACE: openshift-file-integrity test: - as: test cli: latest @@ -80,6 +87,7 @@ tests: resources: requests: cpu: 100m + - ref: tls-scanner-run workflow: ipi-aws - always_run: false as: e2e-rosa From a41303e493af0466e0974a2ff0a96990f071c8d7 Mon Sep 17 00:00:00 2001 From: Guzman Date: Thu, 17 Sep 2026 18:22:37 +0000 Subject: [PATCH 2/7] Address review: run tls-scanner in restricted pod mode Setting SCANNER_NAMESPACE to the existing openshift-file-integrity namespace (already created by the e2e-aws test's 'make e2e' step) makes tls-scanner-run take the OWNS_NAMESPACE=false code path: it deploys the scanner pod into that namespace instead of creating its own, with hostNetwork: false, hostPID: false, a restricted (non-privileged, non-root, runAsUser: 65532) securityContext, and no 'privileged' SCC grant -- instead of the default owned-namespace path which runs hostNetwork/hostPID/privileged as root. This mode uses the kube API for pod discovery and exec rather than host networking, which matches how we already validated ML-KEM support manually on a live cluster (via pods/exec into the target pod), so it does not change scan coverage for FIO's single-namespace target. Verified with 'make ci-operator-config' (idempotent after this change). --- .../openshift-file-integrity-operator-master.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml b/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml index 1ad64184d51f3..f096a34046325 100644 --- a/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml +++ b/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml @@ -76,6 +76,7 @@ tests: env: PQC_CHECK: "true" SCAN_NAMESPACE: openshift-file-integrity + SCANNER_NAMESPACE: openshift-file-integrity test: - as: test cli: latest From 0fb408bd873d7473bf945207c57fbf92df66c843 Mon Sep 17 00:00:00 2001 From: Guzman Date: Fri, 18 Sep 2026 12:56:42 +0000 Subject: [PATCH 3/7] Fix: run tls-scanner against a persistent deploy, not the self-cleaning e2e test The e2e-aws job's 'make e2e' step tears down its own namespace on exit (every test in tests/e2e/e2e_test.go runs 'defer testctx.Cleanup()'), regardless of pass/fail. Appending tls-scanner-run after it fails with 'namespaces "openshift-file-integrity" not found', confirmed by the first real rehearsal run: https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/openshift_release/85402/rehearse-85402-pull-ci-openshift-file-integrity-operator-master-e2e-aws/2100886946011156480 Revert the e2e-aws change entirely, and instead add a new, separate optional test (e2e-aws-tls-scanner) that uses 'make deploy' (just kustomize apply, no test framework, no cleanup) to stand up a persistent operator installation, then scans it. This matches what compliance-operator's own tls-scanner PRs settled on for the same reason -- see commit message 99893f0 in openshift/release#84296: 'Following yuumasato's approach in PR #84972... Use make deploy directly (skip e2e-deployment test)'. --- ...nshift-file-integrity-operator-master.yaml | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml b/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml index f096a34046325..bcd3d521cae62 100644 --- a/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml +++ b/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml @@ -73,10 +73,6 @@ tests: pipeline_skip_if_only_changed: ^(?:\.github|\.tekton)|\.md$|bundle-hack|^(?:\.gitignore|OWNERS|LICENSE)$ steps: cluster_profile: openshift-org-aws - env: - PQC_CHECK: "true" - SCAN_NAMESPACE: openshift-file-integrity - SCANNER_NAMESPACE: openshift-file-integrity test: - as: test cli: latest @@ -88,7 +84,6 @@ tests: resources: requests: cpu: 100m - - ref: tls-scanner-run workflow: ipi-aws - always_run: false as: e2e-rosa @@ -189,6 +184,30 @@ tests: requests: cpu: 100m workflow: ipi-aws +- always_run: false + as: e2e-aws-tls-scanner + optional: true + steps: + cluster_profile: openshift-org-aws + env: + PQC_CHECK: "true" + SCAN_NAMESPACE: openshift-file-integrity + SCANNER_NAMESPACE: openshift-file-integrity + test: + - as: deploy + cli: latest + commands: | + make e2e-set-image deploy + oc wait --timeout=5m --for=condition=Available -n openshift-file-integrity deployment/file-integrity-operator + dependencies: + - env: IMAGE_FROM_CI + name: file-integrity-operator + from: src + resources: + requests: + cpu: 100m + - ref: tls-scanner-run + workflow: ipi-aws zz_generated_metadata: branch: master org: openshift From 0d29d69a1b488390be4112d927d17b3459a620bc Mon Sep 17 00:00:00 2001 From: Guzman Date: Fri, 18 Sep 2026 13:13:34 +0000 Subject: [PATCH 4/7] Add generated Prow job for the new e2e-aws-tls-scanner test Unlike the previous revision (which only added a step to the existing e2e-aws test, leaving the generated job spec untouched since it merely invokes 'ci-operator --target=e2e-aws'), this revision introduces a brand new test name, which requires its own generated Prow job entry in ci-operator/jobs/. pj-rehearse confirmed this is required: 'job(s): pull-ci-openshift-file-integrity-operator-master-e2e-aws-tls-scanner either don't exist or were not found to be affected, and cannot be rehearsed' Hand-crafted following the exact boilerplate of the sibling e2e-bundle-aws job (same cluster_profile: openshift-org-aws, same optional:true/always_run:false shape), since 'make jobs' (ci-operator-prowgen) could not be run standalone here -- it requires the full step-registry tree to resolve every ref/chain/workflow used anywhere in the config, which isn't practical in this sparse checkout. Validated instead with 'make sanitize-prow-jobs', which is repo-local (only needs ci-operator/jobs + core-services/sanitize-prow-jobs) and produced zero additional changes on top of this hand-written entry, confirming it already matches the tool's expected normalized form. The real CI 'generated-config' presubmit will independently verify this is byte-for-byte what the full generator would produce. --- ...-integrity-operator-master-presubmits.yaml | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/ci-operator/jobs/openshift/file-integrity-operator/openshift-file-integrity-operator-master-presubmits.yaml b/ci-operator/jobs/openshift/file-integrity-operator/openshift-file-integrity-operator-master-presubmits.yaml index 7653fc116f4dc..1a627b7ccdbd2 100644 --- a/ci-operator/jobs/openshift/file-integrity-operator/openshift-file-integrity-operator-master-presubmits.yaml +++ b/ci-operator/jobs/openshift/file-integrity-operator/openshift-file-integrity-operator-master-presubmits.yaml @@ -160,6 +160,109 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-aws,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^master$ + - ^master- + cluster: build05 + context: ci/prow/e2e-aws-tls-scanner + decorate: true + decoration_config: + sparse_checkout_files: + - Dockerfile.ci + labels: + ci-operator.openshift.io/cloud: aws + ci-operator.openshift.io/cloud-cluster-profile: openshift-org-aws + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-file-integrity-operator-master-e2e-aws-tls-scanner + optional: true + rerun_command: /test e2e-aws-tls-scanner + spec: + containers: + - args: + - --enable-secrets-store-csi-driver=true + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --gsm-config=/etc/gsm-config/gsm-config.yaml + - --gsm-credentials-file=/etc/gsm-credentials/key.json + - --gsm-project-config=/etc/gsm-config/gsm-project-config.yaml + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=e2e-aws-tls-scanner + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /etc/gsm-config + name: gsm-config + readOnly: true + - mountPath: /etc/gsm-credentials + name: gsm-sa-key + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - configMap: + name: gsm-config + name: gsm-config + - csi: + driver: secrets-store.csi.k8s.io + readOnly: true + volumeAttributes: + secretProviderClass: ci-operator-sa-key-spc + name: gsm-sa-key + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-aws-tls-scanner,?($|\s.*) - agent: kubernetes always_run: false annotations: From ea6402e64f0807eb23cd0b4351bfe25a6dc71a0f Mon Sep 17 00:00:00 2001 From: Guzman Date: Fri, 18 Sep 2026 14:42:14 +0000 Subject: [PATCH 5/7] Fix: shim kubectl->oc for make deploy in the CI cli image The rehearsal run failed inside 'make deploy' with 'kubectl: command not found': https://storage.googleapis.com/test-platform-results-public/pr-logs/pull/openshift_release/85402/rehearse-85402-pull-ci-openshift-file-integrity-operator-master-e2e-aws-tls-scanner/2100939653510074368/artifacts/e2e-aws-tls-scanner/deploy/build-log.txt FIO's Makefile 'install'/'deploy' targets are kubebuilder-scaffolded boilerplate that shell out to a literal 'kubectl' binary, unlike 'make e2e' (which applies manifests via the Go client library inside the test binary itself, never invoking a CLI tool). The CI cli image only provides 'oc'. Symlink oc as kubectl on PATH before invoking make, entirely within this CI step -- no change to file-integrity-operator's Makefile needed, keeping this PR CI-only as intended. --- .../openshift-file-integrity-operator-master.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml b/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml index bcd3d521cae62..999e47ee7580b 100644 --- a/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml +++ b/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml @@ -197,6 +197,11 @@ tests: - as: deploy cli: latest commands: | + # The Makefile's install/deploy targets shell out to "kubectl" directly + # (kubebuilder-scaffolded boilerplate); the CI cli image only has "oc". + mkdir -p /tmp/bin + ln -sf "$(command -v oc)" /tmp/bin/kubectl + export PATH="/tmp/bin:$PATH" make e2e-set-image deploy oc wait --timeout=5m --for=condition=Available -n openshift-file-integrity deployment/file-integrity-operator dependencies: From 8005e948d1760818c2644ea2a51f7e390c293b76 Mon Sep 17 00:00:00 2001 From: Guzman Date: Mon, 21 Sep 2026 07:49:41 +0000 Subject: [PATCH 6/7] Fix: deploy the CI-built image, not the public :latest release The rehearsal's 'deploy' step logged two 'kustomize edit set image' calls: first correctly to IMAGE_FROM_CI (from 'e2e-set-image'), then immediately clobbered back to the public quay.io :latest image by 'deploy's own recipe line (it always does its own 'kustomize edit set image ...=${IMG}', independent of e2e-set-image, and IMG defaults to the public release image). Confirmed from the pod events: the deployed operator pulled 'quay.io/file-integrity-operator/file-integrity-operator:latest' instead of the freshly built PR image. https://storage.googleapis.com/test-platform-results-public/pr-logs/pull/openshift_release/85402/rehearse-85402-pull-ci-openshift-file-integrity-operator-master-e2e-aws-tls-scanner/2100961169303932928/artifacts/e2e-aws-tls-scanner/deploy/build-log.txt Fix: skip e2e-set-image entirely and pass IMG directly to 'make deploy', which is the variable deploy's own recipe actually reads. Also fix a real scheduling failure from the same run: the tls-scanner pod's default 4-CPU request didn't fit the test cluster's nodes ('Insufficient cpu'). Set SCANNER_CPU_REQUEST/SCANNER_MEM_REQUEST to the same lower values (500m/1Gi) already used by tls-scanner's own constrained-capacity job (baremetalds-e2e in ci-operator/config/openshift/tls-scanner/openshift-tls-scanner-release-5.1.yaml), keeping the higher default limit for burst headroom. --- .../openshift-file-integrity-operator-master.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml b/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml index 999e47ee7580b..3832b28adfd48 100644 --- a/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml +++ b/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml @@ -192,6 +192,8 @@ tests: env: PQC_CHECK: "true" SCAN_NAMESPACE: openshift-file-integrity + SCANNER_CPU_REQUEST: 500m + SCANNER_MEM_REQUEST: 1Gi SCANNER_NAMESPACE: openshift-file-integrity test: - as: deploy @@ -202,7 +204,11 @@ tests: mkdir -p /tmp/bin ln -sf "$(command -v oc)" /tmp/bin/kubectl export PATH="/tmp/bin:$PATH" - make e2e-set-image deploy + # NOTE: do not chain "e2e-set-image deploy" -- deploy's own recipe + # unconditionally re-runs "kustomize edit set image ...=${IMG}", + # clobbering whatever e2e-set-image just set back to IMG's default + # (the public quay.io release image). Pass IMG directly instead. + make deploy IMG="${IMAGE_FROM_CI}" oc wait --timeout=5m --for=condition=Available -n openshift-file-integrity deployment/file-integrity-operator dependencies: - env: IMAGE_FROM_CI From d514f563af14f361b5567e83a580f195dd0cdc0c Mon Sep 17 00:00:00 2001 From: Guzman Date: Mon, 21 Sep 2026 11:22:05 +0000 Subject: [PATCH 7/7] Fix: wait for the metrics TLS listener before scanning, not just pod Available The previous 'passing' rehearsal was a false positive. The scan log showed only 2 discovered ports on the operator pod, 8383 and 8081 -- both plain HTTP -- and NOT 8585, the actual HTTPS metrics endpoint this whole PR exists to check. tls-scanner's PQC check 'passed' only because it found zero TLS endpoints to check, not because the real one was validated: https://gcs.ci.openshift.org/gcs/test-platform-results-public/pr-logs/pull/openshift_release/85402/rehearse-85402-pull-ci-openshift-file-integrity-operator-master-e2e-aws-tls-scanner/2101943554979926016/artifacts/e2e-aws-tls-scanner/tls-scanner-run/build-log.txt Root cause: config/manager/deployment.yaml has no readiness/liveness probe, so 'oc wait --for=condition=Available' only confirms the container process started -- nothing about pkg/controller/metrics/ metrics.go's Start(), which deliberately blocks (up to 5m, via waitForServingCert()) for the service-ca-issued serving cert to be mounted before binding :8585. tls-scanner discovers ports via /proc/net/tcp introspection (the Deployment declares no containerPorts), so it only sees whatever is already listening at scan time -- which structurally, on every run, will be the two ports that come up immediately (8081 health, 8383 controller-runtime metrics) and never 8585, unless something else in the test happens to delay the scan long enough by chance. Fix (CI-side only, no file-integrity-operator code change): after confirming the Deployment is Available, additionally poll for the same serving-cert file inside the running container that the operator's own code waits on, before invoking tls-scanner-run. --- ...nshift-file-integrity-operator-master.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml b/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml index 3832b28adfd48..71d2088bfc218 100644 --- a/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml +++ b/ci-operator/config/openshift/file-integrity-operator/openshift-file-integrity-operator-master.yaml @@ -210,6 +210,25 @@ tests: # (the public quay.io release image). Pass IMG directly instead. make deploy IMG="${IMAGE_FROM_CI}" oc wait --timeout=5m --for=condition=Available -n openshift-file-integrity deployment/file-integrity-operator + # deployment.yaml has no readiness/liveness probe, so "Available" above + # only means the container process started -- it says nothing about the + # metrics HTTPS listener on :8585, which pkg/controller/metrics/metrics.go + # deliberately brings up later, after blocking (up to 5m) for the + # service-ca-issued serving cert to be mounted. Wait for that same file + # to actually appear in the running container before scanning, otherwise + # tls-scanner's /proc/net/tcp-based port discovery (this Deployment + # declares no containerPorts) races it and only ever sees the two plain + # HTTP ports (8081 health, 8383 controller-runtime metrics) that are up + # immediately -- silently skipping the one HTTPS endpoint under test. + for i in $(seq 1 36); do + if oc exec -n openshift-file-integrity deploy/file-integrity-operator -- test -f /var/run/secrets/serving-cert/tls.crt 2>/dev/null; then + echo "metrics serving cert is mounted, endpoint should be listening" + sleep 5 + break + fi + echo "waiting for metrics serving cert to be mounted (attempt $i/36)..." + sleep 5 + done dependencies: - env: IMAGE_FROM_CI name: file-integrity-operator