From fdedafc0d502405ca5be3f3f840048906f03f217 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 3 Sep 2026 14:16:15 +0200 Subject: [PATCH 1/5] fix(test): adapt fixtures and asserts in graceful-shutdown and product-config-compat tests --- .../kuttl/graceful-shutdown/10-assert.yaml | 49 ++++++++++++++---- .../kuttl/graceful-shutdown/12-assert.yaml | 51 +++++++++++++++---- .../13-check-driver-shutdown-propagation.yaml | 16 ++++-- .../pyspark-pi-driver-pod-template-data.json | 2 +- ...pyspark-pi-executor-pod-template-data.json | 2 +- .../pyspark-pi-job-template-spec.json | 3 +- .../fixtures/spark-connect-server-data.json | 2 +- 7 files changed, 99 insertions(+), 26 deletions(-) diff --git a/tests/templates/kuttl/graceful-shutdown/10-assert.yaml b/tests/templates/kuttl/graceful-shutdown/10-assert.yaml index f30e7b3f..5b0c6f88 100644 --- a/tests/templates/kuttl/graceful-shutdown/10-assert.yaml +++ b/tests/templates/kuttl/graceful-shutdown/10-assert.yaml @@ -5,15 +5,46 @@ timeout: 900 commands: # Both executors have to be registered and working: an executor that is still registering is # not in the driver's list yet and would never be asked to shut down. - - script: | + # + # Pods that are terminating are still listed by `kubectl get pods` and keep the log lines below + # for as long as they linger, so they would satisfy this check without being able to do any + # work. Only executors that are alive are counted. + # + # The polling happens here and not by letting kuttl retry the assert, because kuttl reprints the + # whole script on every attempt and would bury the rest of the test log. + - timeout: 900 + script: | set -eu - registered=0 - for pod in $(kubectl -n "$NAMESPACE" get pods -o name \ - -l spark-role=executor,app.kubernetes.io/instance=graceful-shutdown | cut -d/ -f2); do - log=$(kubectl -n "$NAMESPACE" logs --tail=-1 "$pod" -c spark) - echo "$log" | grep -q 'Successfully registered with driver' - echo "$log" | grep -q 'Running task' - registered=$(( registered + 1 )) + selector="spark-role=executor,app.kubernetes.io/instance=graceful-shutdown" + budget_seconds=870 + poll_seconds=5 + + start=$(date +%s) + while :; do + registered=0 + for pod in $(kubectl -n "$NAMESPACE" get pods -l "$selector" -o json \ + | jq -r '.items[] + | select(.metadata.deletionTimestamp == null and .status.phase == "Running") + | .metadata.name'); do + # The Pod can disappear between the listing and the read, which is not a failure. + log=$(kubectl -n "$NAMESPACE" logs --tail=-1 "$pod" -c spark 2>/dev/null || true) + echo "$log" | grep -q 'Successfully registered with driver' || continue + echo "$log" | grep -q 'Running task' || continue + registered=$(( registered + 1 )) + done + + if [ "$registered" -eq 2 ]; then + echo "OK: 2 executors are registered with the driver and running a task" + exit 0 + fi + + if [ $(( $(date +%s) - start )) -ge "$budget_seconds" ]; then + echo "FAIL: ${budget_seconds}s elapsed with $registered of 2 executors registered and" + echo "running a task, so the driver's list is not what the rest of the test assumes" + kubectl -n "$NAMESPACE" get pods -l "$selector" + exit 1 + fi + + sleep "$poll_seconds" done - test "$registered" -eq 2 diff --git a/tests/templates/kuttl/graceful-shutdown/12-assert.yaml b/tests/templates/kuttl/graceful-shutdown/12-assert.yaml index bf511321..6099d73d 100644 --- a/tests/templates/kuttl/graceful-shutdown/12-assert.yaml +++ b/tests/templates/kuttl/graceful-shutdown/12-assert.yaml @@ -5,15 +5,48 @@ timeout: 600 commands: # Same check as in step 10: the replacement for the stopped executor has to be registered and # working before the driver is stopped, otherwise it is not in the driver's list yet. - - script: | + # + # The liveness filter is what makes this wait for the replacement at all. The executor deleted + # in step 11 stays listed while it drains -- its JVM can outlive the shutdown hooks for the + # whole 30s grace period -- and its log still contains both lines below, so counting every + # listed Pod lets {draining executor, survivor} pass instantly and hands step 13 an executor + # that can no longer answer. + # + # The polling happens here and not by letting kuttl retry the assert, because kuttl reprints the + # whole script on every attempt and would bury the rest of the test log. + - timeout: 600 + script: | set -eu - registered=0 - for pod in $(kubectl -n "$NAMESPACE" get pods -o name \ - -l spark-role=executor,app.kubernetes.io/instance=graceful-shutdown | cut -d/ -f2); do - log=$(kubectl -n "$NAMESPACE" logs --tail=-1 "$pod" -c spark) - echo "$log" | grep -q 'Successfully registered with driver' - echo "$log" | grep -q 'Running task' - registered=$(( registered + 1 )) + selector="spark-role=executor,app.kubernetes.io/instance=graceful-shutdown" + budget_seconds=570 + poll_seconds=5 + + start=$(date +%s) + while :; do + registered=0 + for pod in $(kubectl -n "$NAMESPACE" get pods -l "$selector" -o json \ + | jq -r '.items[] + | select(.metadata.deletionTimestamp == null and .status.phase == "Running") + | .metadata.name'); do + # The Pod can disappear between the listing and the read, which is not a failure. + log=$(kubectl -n "$NAMESPACE" logs --tail=-1 "$pod" -c spark 2>/dev/null || true) + echo "$log" | grep -q 'Successfully registered with driver' || continue + echo "$log" | grep -q 'Running task' || continue + registered=$(( registered + 1 )) + done + + if [ "$registered" -eq 2 ]; then + echo "OK: the replacement executor is registered with the driver and running a task" + exit 0 + fi + + if [ $(( $(date +%s) - start )) -ge "$budget_seconds" ]; then + echo "FAIL: ${budget_seconds}s after the executor was stopped, $registered of 2" + echo "executors are registered and running a task, so no replacement came back" + kubectl -n "$NAMESPACE" get pods -l "$selector" + exit 1 + fi + + sleep "$poll_seconds" done - test "$registered" -eq 2 diff --git a/tests/templates/kuttl/graceful-shutdown/13-check-driver-shutdown-propagation.yaml b/tests/templates/kuttl/graceful-shutdown/13-check-driver-shutdown-propagation.yaml index 683d94e5..0e2a1047 100644 --- a/tests/templates/kuttl/graceful-shutdown/13-check-driver-shutdown-propagation.yaml +++ b/tests/templates/kuttl/graceful-shutdown/13-check-driver-shutdown-propagation.yaml @@ -17,8 +17,13 @@ commands: capture=$(mktemp -d) driver=$(kubectl -n "$NAMESPACE" get pods -o name \ -l "spark-role=driver,app.kubernetes.io/instance=$app" | head -1 | cut -d/ -f2) - executors=$(kubectl -n "$NAMESPACE" get pods -o name \ - -l "spark-role=executor,app.kubernetes.io/instance=$app" | cut -d/ -f2) + # Terminating Pods are still listed by `kubectl get pods`, so an executor left over from + # step 11 would be waited on for a shutdown it has already performed and can never log + # again. Only executors that are alive can still be asked to stop. + executors=$(kubectl -n "$NAMESPACE" get pods -l "spark-role=executor,app.kubernetes.io/instance=$app" -o json \ + | jq -r '.items[] + | select(.metadata.deletionTimestamp == null and .status.phase == "Running") + | .metadata.name') # Without these the loop below has nothing to wait for and would report success while having # checked nothing at all. @@ -26,8 +31,11 @@ commands: echo "FAIL: no driver Pod found for $app" exit 1 fi - if [ -z "$executors" ]; then - echo "FAIL: no executor Pods found for $app, so no shutdown can be propagated to any" + running=$(printf '%s\n' "$executors" | grep -c . || true) + if [ "$running" -ne 2 ]; then + echo "FAIL: expected 2 running executor Pods for $app, found $running, so no shutdown can" + echo "be propagated to the replicas the application asked for" + kubectl -n "$NAMESPACE" get pods -l "spark-role=executor,app.kubernetes.io/instance=$app" exit 1 fi diff --git a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json index 5fb909b5..9e0c192e 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json @@ -2,5 +2,5 @@ "log4j2.properties": "appenders = FILE, CONSOLE\n\nappender.CONSOLE.type = Console\nappender.CONSOLE.name = CONSOLE\nappender.CONSOLE.target = SYSTEM_ERR\nappender.CONSOLE.layout.type = PatternLayout\nappender.CONSOLE.layout.pattern = %d{ISO8601} %p [%t] %c - %m%n\nappender.CONSOLE.filter.threshold.type = ThresholdFilter\nappender.CONSOLE.filter.threshold.level = INFO\n\nappender.FILE.type = RollingFile\nappender.FILE.name = FILE\nappender.FILE.fileName = /stackable/log/spark/spark.log4j2.xml\nappender.FILE.filePattern = /stackable/log/spark/spark.log4j2.xml.%i\nappender.FILE.layout.type = XMLLayout\nappender.FILE.policies.type = Policies\nappender.FILE.policies.size.type = SizeBasedTriggeringPolicy\nappender.FILE.policies.size.size = 5MB\nappender.FILE.strategy.type = DefaultRolloverStrategy\nappender.FILE.strategy.max = 1\nappender.FILE.filter.threshold.type = ThresholdFilter\nappender.FILE.filter.threshold.level = INFO\n\n\nrootLogger.level=INFO\nrootLogger.appenderRefs = CONSOLE, FILE\nrootLogger.appenderRef.CONSOLE.ref = CONSOLE\nrootLogger.appenderRef.FILE.ref = FILE", "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n", "spark-env.sh": "", - "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n prometheus.io/scrape: 'true'\n stackable.tech/vendor: Stackable\n name: spark\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: config\n" + "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n prometheus.io/scrape: 'true'\n stackable.tech/vendor: Stackable\n name: spark\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n runAsNonRoot: true\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: config\n" } diff --git a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json index c82835cd..8867f65d 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json @@ -2,5 +2,5 @@ "log4j2.properties": "appenders = FILE, CONSOLE\n\nappender.CONSOLE.type = Console\nappender.CONSOLE.name = CONSOLE\nappender.CONSOLE.target = SYSTEM_ERR\nappender.CONSOLE.layout.type = PatternLayout\nappender.CONSOLE.layout.pattern = %d{ISO8601} %p [%t] %c - %m%n\nappender.CONSOLE.filter.threshold.type = ThresholdFilter\nappender.CONSOLE.filter.threshold.level = INFO\n\nappender.FILE.type = RollingFile\nappender.FILE.name = FILE\nappender.FILE.fileName = /stackable/log/spark/spark.log4j2.xml\nappender.FILE.filePattern = /stackable/log/spark/spark.log4j2.xml.%i\nappender.FILE.layout.type = XMLLayout\nappender.FILE.policies.type = Policies\nappender.FILE.policies.size.type = SizeBasedTriggeringPolicy\nappender.FILE.policies.size.size = 5MB\nappender.FILE.strategy.type = DefaultRolloverStrategy\nappender.FILE.strategy.max = 1\nappender.FILE.filter.threshold.type = ThresholdFilter\nappender.FILE.filter.threshold.level = INFO\n\n\nrootLogger.level=INFO\nrootLogger.appenderRefs = CONSOLE, FILE\nrootLogger.appenderRef.CONSOLE.ref = CONSOLE\nrootLogger.appenderRef.FILE.ref = FILE", "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n", "spark-env.sh": "", - "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\n name: spark\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: config\n" + "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\n name: spark\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n runAsNonRoot: true\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: config\n" } diff --git a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json index 976109ef..2a23c138 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json @@ -65,7 +65,8 @@ "restartPolicy": "Never", "schedulerName": "default-scheduler", "securityContext": { - "fsGroup": 1000 + "fsGroup": 1000, + "runAsNonRoot": true }, "serviceAccount": "pyspark-pi", "serviceAccountName": "pyspark-pi", diff --git a/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json b/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json index ac0b2aef..8a4df2d3 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json @@ -2,5 +2,5 @@ "metrics.properties": "*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet\n*.sink.prometheusServlet.path=/metrics/prometheus\n", "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n", "spark-defaults.conf": "spark.driver.cores=3\nspark.driver.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\\ -Dmy.custom.jvm.arg\\=customValue\nspark.driver.extraClassPath=/stackable/spark/extra-jars/*\\:/stackable/spark/connect/spark-connect-3.5.8.jar\nspark.driver.host=spark-connect-server-headless\nspark.executor.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\nspark.executor.instances=3\nspark.executor.memory=1024M\nspark.executor.memoryOverhead=1m\nspark.kubernetes.authenticate.driver.serviceAccountName=spark-connect-serviceaccount\nspark.kubernetes.driver.container.image=oci.stackable.tech/sdp/spark-k8s\\:3.5.8-stackable0.0.0-dev\nspark.kubernetes.driver.pod.name=${env\\:HOSTNAME}\nspark.kubernetes.executor.container.image=oci.stackable.tech/sdp/spark-k8s\\:3.5.8-stackable0.0.0-dev\nspark.kubernetes.executor.limit.cores=1\nspark.kubernetes.executor.podTemplateContainerName=spark\nspark.kubernetes.executor.podTemplateFile=/stackable/spark/conf/template.yaml\nspark.kubernetes.executor.request.cores=1\nspark.kubernetes.namespace=__NAMESPACE__\nspark.metrics.conf=/stackable/spark/conf/metrics.properties\nspark.ui.prometheus.enabled=true\n", - "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/managed-by: spark.stackable.tech_connect\n app.kubernetes.io/name: spark-connect\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\nspec:\n affinity:\n podAntiAffinity:\n preferredDuringSchedulingIgnoredDuringExecution:\n - podAffinityTerm:\n labelSelector:\n matchLabels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/name: spark-connect\n topologyKey: kubernetes.io/hostname\n weight: 70\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n name: spark\n volumeMounts:\n - mountPath: /stackable/spark/conf\n name: config\n - mountPath: /stackable/log\n name: log\n - mountPath: /stackable/truststore\n name: stackable-truststore\n - mountPath: /stackable/log_config\n name: log-config\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n volumes:\n - emptyDir:\n sizeLimit: 30Mi\n name: log\n - configMap:\n name: spark-connect-executor\n name: config\n - emptyDir: {}\n name: stackable-truststore\n - configMap:\n name: spark-connect-log-config\n name: log-config\n" + "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/managed-by: spark.stackable.tech_connect\n app.kubernetes.io/name: spark-connect\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\nspec:\n affinity:\n podAntiAffinity:\n preferredDuringSchedulingIgnoredDuringExecution:\n - podAffinityTerm:\n labelSelector:\n matchLabels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/name: spark-connect\n topologyKey: kubernetes.io/hostname\n weight: 70\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n name: spark\n volumeMounts:\n - mountPath: /stackable/spark/conf\n name: config\n - mountPath: /stackable/log\n name: log\n - mountPath: /stackable/truststore\n name: stackable-truststore\n - mountPath: /stackable/log_config\n name: log-config\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n runAsNonRoot: true\n volumes:\n - emptyDir:\n sizeLimit: 30Mi\n name: log\n - configMap:\n name: spark-connect-executor\n name: config\n - emptyDir: {}\n name: stackable-truststore\n - configMap:\n name: spark-connect-log-config\n name: log-config\n" } From 58568ea7f891469bce21d6df9ca6f08a71d7a1f8 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 3 Sep 2026 14:22:39 +0200 Subject: [PATCH 2/5] fix: improve test comment --- tests/templates/kuttl/graceful-shutdown/12-assert.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/templates/kuttl/graceful-shutdown/12-assert.yaml b/tests/templates/kuttl/graceful-shutdown/12-assert.yaml index 6099d73d..c9c16cca 100644 --- a/tests/templates/kuttl/graceful-shutdown/12-assert.yaml +++ b/tests/templates/kuttl/graceful-shutdown/12-assert.yaml @@ -3,15 +3,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 commands: - # Same check as in step 10: the replacement for the stopped executor has to be registered and - # working before the driver is stopped, otherwise it is not in the driver's list yet. - # - # The liveness filter is what makes this wait for the replacement at all. The executor deleted - # in step 11 stays listed while it drains -- its JVM can outlive the shutdown hooks for the - # whole 30s grace period -- and its log still contains both lines below, so counting every - # listed Pod lets {draining executor, survivor} pass instantly and hands step 13 an executor - # that can no longer answer. - # # The polling happens here and not by letting kuttl retry the assert, because kuttl reprints the # whole script on every attempt and would bury the rest of the test log. - timeout: 600 From acee5ec9ce1cb6048f16ab8cda9b75e104b695f8 Mon Sep 17 00:00:00 2001 From: maltesander Date: Fri, 4 Sep 2026 07:36:25 +0200 Subject: [PATCH 3/5] Update tests/templates/kuttl/graceful-shutdown/12-assert.yaml Co-authored-by: Techassi --- tests/templates/kuttl/graceful-shutdown/12-assert.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/templates/kuttl/graceful-shutdown/12-assert.yaml b/tests/templates/kuttl/graceful-shutdown/12-assert.yaml index c9c16cca..05073d79 100644 --- a/tests/templates/kuttl/graceful-shutdown/12-assert.yaml +++ b/tests/templates/kuttl/graceful-shutdown/12-assert.yaml @@ -21,9 +21,9 @@ commands: | select(.metadata.deletionTimestamp == null and .status.phase == "Running") | .metadata.name'); do # The Pod can disappear between the listing and the read, which is not a failure. - log=$(kubectl -n "$NAMESPACE" logs --tail=-1 "$pod" -c spark 2>/dev/null || true) - echo "$log" | grep -q 'Successfully registered with driver' || continue - echo "$log" | grep -q 'Running task' || continue + logs=$(kubectl -n "$NAMESPACE" logs --tail=-1 "$pod" -c spark 2>/dev/null || true) + echo "$logs" | grep -q 'Successfully registered with driver' || continue + echo "$logs" | grep -q 'Running task' || continue registered=$(( registered + 1 )) done From 6a8d8baf064e4a16e046cc189eaac3b5bb192d71 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 4 Sep 2026 07:41:49 +0200 Subject: [PATCH 4/5] fix(test): count running executors with jq length --- .../13-check-driver-shutdown-propagation.yaml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/templates/kuttl/graceful-shutdown/13-check-driver-shutdown-propagation.yaml b/tests/templates/kuttl/graceful-shutdown/13-check-driver-shutdown-propagation.yaml index 0e2a1047..4de4738e 100644 --- a/tests/templates/kuttl/graceful-shutdown/13-check-driver-shutdown-propagation.yaml +++ b/tests/templates/kuttl/graceful-shutdown/13-check-driver-shutdown-propagation.yaml @@ -20,10 +20,16 @@ commands: # Terminating Pods are still listed by `kubectl get pods`, so an executor left over from # step 11 would be waited on for a shutdown it has already performed and can never log # again. Only executors that are alive can still be asked to stop. - executors=$(kubectl -n "$NAMESPACE" get pods -l "spark-role=executor,app.kubernetes.io/instance=$app" -o json \ - | jq -r '.items[] - | select(.metadata.deletionTimestamp == null and .status.phase == "Running") - | .metadata.name') + live=$( + kubectl -n "$NAMESPACE" get pods \ + -l "spark-role=executor,app.kubernetes.io/instance=$app" \ + -o json | jq ' + [ .items[] + | select(.metadata.deletionTimestamp == null and .status.phase == "Running") + | .metadata.name ]' + ) + executors=$(echo "$live" | jq -r '.[]') + running=$(echo "$live" | jq 'length') # Without these the loop below has nothing to wait for and would report success while having # checked nothing at all. @@ -31,7 +37,6 @@ commands: echo "FAIL: no driver Pod found for $app" exit 1 fi - running=$(printf '%s\n' "$executors" | grep -c . || true) if [ "$running" -ne 2 ]; then echo "FAIL: expected 2 running executor Pods for $app, found $running, so no shutdown can" echo "be propagated to the replicas the application asked for" From 678d450eb462dba9f545f589204a607f43bfd9b5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 4 Sep 2026 07:43:15 +0200 Subject: [PATCH 5/5] fix(test): pull the executor listing out of the for loop header --- .../kuttl/graceful-shutdown/10-assert.yaml | 18 +++++++++++------- .../kuttl/graceful-shutdown/12-assert.yaml | 12 ++++++++---- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/tests/templates/kuttl/graceful-shutdown/10-assert.yaml b/tests/templates/kuttl/graceful-shutdown/10-assert.yaml index 5b0c6f88..abaa736c 100644 --- a/tests/templates/kuttl/graceful-shutdown/10-assert.yaml +++ b/tests/templates/kuttl/graceful-shutdown/10-assert.yaml @@ -22,15 +22,19 @@ commands: start=$(date +%s) while :; do + executors=$( + kubectl -n "$NAMESPACE" get pods -l "$selector" -o json | jq -r ' + .items[] + | select(.metadata.deletionTimestamp == null and .status.phase == "Running") + | .metadata.name' + ) + registered=0 - for pod in $(kubectl -n "$NAMESPACE" get pods -l "$selector" -o json \ - | jq -r '.items[] - | select(.metadata.deletionTimestamp == null and .status.phase == "Running") - | .metadata.name'); do + for pod in $executors; do # The Pod can disappear between the listing and the read, which is not a failure. - log=$(kubectl -n "$NAMESPACE" logs --tail=-1 "$pod" -c spark 2>/dev/null || true) - echo "$log" | grep -q 'Successfully registered with driver' || continue - echo "$log" | grep -q 'Running task' || continue + logs=$(kubectl -n "$NAMESPACE" logs --tail=-1 "$pod" -c spark 2>/dev/null || true) + echo "$logs" | grep -q 'Successfully registered with driver' || continue + echo "$logs" | grep -q 'Running task' || continue registered=$(( registered + 1 )) done diff --git a/tests/templates/kuttl/graceful-shutdown/12-assert.yaml b/tests/templates/kuttl/graceful-shutdown/12-assert.yaml index 05073d79..7533aa2d 100644 --- a/tests/templates/kuttl/graceful-shutdown/12-assert.yaml +++ b/tests/templates/kuttl/graceful-shutdown/12-assert.yaml @@ -15,11 +15,15 @@ commands: start=$(date +%s) while :; do + executors=$( + kubectl -n "$NAMESPACE" get pods -l "$selector" -o json | jq -r ' + .items[] + | select(.metadata.deletionTimestamp == null and .status.phase == "Running") + | .metadata.name' + ) + registered=0 - for pod in $(kubectl -n "$NAMESPACE" get pods -l "$selector" -o json \ - | jq -r '.items[] - | select(.metadata.deletionTimestamp == null and .status.phase == "Running") - | .metadata.name'); do + for pod in $executors; do # The Pod can disappear between the listing and the read, which is not a failure. logs=$(kubectl -n "$NAMESPACE" logs --tail=-1 "$pod" -c spark 2>/dev/null || true) echo "$logs" | grep -q 'Successfully registered with driver' || continue