Adapt fixtures and asserts in graceful-shutdown and product-config-compat tests

tests/templates/kuttl/graceful-shutdown/10-assert.yaml

@@ -5,15 +5,50 @@ timeout: 900
 commands:
   # Both executors have to be registered and working: an executor that is still registering is
   # not in the driver's list yet and would never be asked to shut down.
-  - script: |
+  #
+  # Pods that are terminating are still listed by `kubectl get pods` and keep the log lines below
+  # for as long as they linger, so they would satisfy this check without being able to do any
+  # work. Only executors that are alive are counted.
+  #
+  # The polling happens here and not by letting kuttl retry the assert, because kuttl reprints the
+  # whole script on every attempt and would bury the rest of the test log.
+  - timeout: 900
+    script: |
       set -eu
 
-      registered=0
-      for pod in $(kubectl -n "$NAMESPACE" get pods -o name \
-        -l spark-role=executor,app.kubernetes.io/instance=graceful-shutdown | cut -d/ -f2); do
-        log=$(kubectl -n "$NAMESPACE" logs --tail=-1 "$pod" -c spark)
-        echo "$log" | grep -q 'Successfully registered with driver'
-        echo "$log" | grep -q 'Running task'
-        registered=$(( registered + 1 ))
+      selector="spark-role=executor,app.kubernetes.io/instance=graceful-shutdown"
+      budget_seconds=870
+      poll_seconds=5
+
+      start=$(date +%s)
+      while :; do
+        executors=$(
+          kubectl -n "$NAMESPACE" get pods -l "$selector" -o json | jq -r '
+            .items[]
+            | select(.metadata.deletionTimestamp == null and .status.phase == "Running")
+            | .metadata.name'
+        )
+
+        registered=0
+        for pod in $executors; do
+          # The Pod can disappear between the listing and the read, which is not a failure.
+          logs=$(kubectl -n "$NAMESPACE" logs --tail=-1 "$pod" -c spark 2>/dev/null || true)
+          echo "$logs" | grep -q 'Successfully registered with driver' || continue
+          echo "$logs" | grep -q 'Running task' || continue
+          registered=$(( registered + 1 ))
+        done
+
+        if [ "$registered" -eq 2 ]; then
+          echo "OK: 2 executors are registered with the driver and running a task"
+          exit 0
+        fi
+
+        if [ $(( $(date +%s) - start )) -ge "$budget_seconds" ]; then
+          echo "FAIL: ${budget_seconds}s elapsed with $registered of 2 executors registered and"
+          echo "running a task, so the driver's list is not what the rest of the test assumes"
+          kubectl -n "$NAMESPACE" get pods -l "$selector"
+          exit 1
+        fi
+
+        sleep "$poll_seconds"
       done
-      test "$registered" -eq 2

tests/templates/kuttl/graceful-shutdown/12-assert.yaml

@@ -3,17 +3,45 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 timeout: 600
 commands:
-  # Same check as in step 10: the replacement for the stopped executor has to be registered and
-  # working before the driver is stopped, otherwise it is not in the driver's list yet.
-  - script: |
+  # The polling happens here and not by letting kuttl retry the assert, because kuttl reprints the
+  # whole script on every attempt and would bury the rest of the test log.
+  - timeout: 600
+    script: |
       set -eu
 
-      registered=0
-      for pod in $(kubectl -n "$NAMESPACE" get pods -o name \
-        -l spark-role=executor,app.kubernetes.io/instance=graceful-shutdown | cut -d/ -f2); do
-        log=$(kubectl -n "$NAMESPACE" logs --tail=-1 "$pod" -c spark)
-        echo "$log" | grep -q 'Successfully registered with driver'
-        echo "$log" | grep -q 'Running task'
-        registered=$(( registered + 1 ))
+      selector="spark-role=executor,app.kubernetes.io/instance=graceful-shutdown"
+      budget_seconds=570
+      poll_seconds=5
+
+      start=$(date +%s)
+      while :; do
+        executors=$(
+          kubectl -n "$NAMESPACE" get pods -l "$selector" -o json | jq -r '
+            .items[]
+            | select(.metadata.deletionTimestamp == null and .status.phase == "Running")
+            | .metadata.name'
+        )
+
+        registered=0
+        for pod in $executors; do
+          # The Pod can disappear between the listing and the read, which is not a failure.
+          logs=$(kubectl -n "$NAMESPACE" logs --tail=-1 "$pod" -c spark 2>/dev/null || true)
+          echo "$logs" | grep -q 'Successfully registered with driver' || continue
+          echo "$logs" | grep -q 'Running task' || continue
+          registered=$(( registered + 1 ))
+        done
+
+        if [ "$registered" -eq 2 ]; then
+          echo "OK: the replacement executor is registered with the driver and running a task"
+          exit 0
+        fi
+
+        if [ $(( $(date +%s) - start )) -ge "$budget_seconds" ]; then
+          echo "FAIL: ${budget_seconds}s after the executor was stopped, $registered of 2"
+          echo "executors are registered and running a task, so no replacement came back"
+          kubectl -n "$NAMESPACE" get pods -l "$selector"
+          exit 1
+        fi
+
+        sleep "$poll_seconds"
       done
-      test "$registered" -eq 2

tests/templates/kuttl/graceful-shutdown/13-check-driver-shutdown-propagation.yaml

@@ -17,17 +17,30 @@ commands:
       capture=$(mktemp -d)
       driver=$(kubectl -n "$NAMESPACE" get pods -o name \
         -l "spark-role=driver,app.kubernetes.io/instance=$app" | head -1 | cut -d/ -f2)
-      executors=$(kubectl -n "$NAMESPACE" get pods -o name \
-        -l "spark-role=executor,app.kubernetes.io/instance=$app" | cut -d/ -f2)
+      # Terminating Pods are still listed by `kubectl get pods`, so an executor left over from
+      # step 11 would be waited on for a shutdown it has already performed and can never log
+      # again. Only executors that are alive can still be asked to stop.
+      live=$(
+        kubectl -n "$NAMESPACE" get pods \
+          -l "spark-role=executor,app.kubernetes.io/instance=$app" \
+          -o json | jq '
+            [ .items[]
+              | select(.metadata.deletionTimestamp == null and .status.phase == "Running")
+              | .metadata.name ]'
+      )
+      executors=$(echo "$live" | jq -r '.[]')
+      running=$(echo "$live" | jq 'length')
 
       # Without these the loop below has nothing to wait for and would report success while having
       # checked nothing at all.
       if [ -z "$driver" ]; then
         echo "FAIL: no driver Pod found for $app"
         exit 1
       fi
-      if [ -z "$executors" ]; then
-        echo "FAIL: no executor Pods found for $app, so no shutdown can be propagated to any"
+      if [ "$running" -ne 2 ]; then
+        echo "FAIL: expected 2 running executor Pods for $app, found $running, so no shutdown can"
+        echo "be propagated to the replicas the application asked for"
+        kubectl -n "$NAMESPACE" get pods -l "spark-role=executor,app.kubernetes.io/instance=$app"
         exit 1
       fi
 

tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json

@@ -2,5 +2,5 @@
   "log4j2.properties": "appenders = FILE, CONSOLE\n\nappender.CONSOLE.type = Console\nappender.CONSOLE.name = CONSOLE\nappender.CONSOLE.target = SYSTEM_ERR\nappender.CONSOLE.layout.type = PatternLayout\nappender.CONSOLE.layout.pattern = %d{ISO8601} %p [%t] %c - %m%n\nappender.CONSOLE.filter.threshold.type = ThresholdFilter\nappender.CONSOLE.filter.threshold.level = INFO\n\nappender.FILE.type = RollingFile\nappender.FILE.name = FILE\nappender.FILE.fileName = /stackable/log/spark/spark.log4j2.xml\nappender.FILE.filePattern = /stackable/log/spark/spark.log4j2.xml.%i\nappender.FILE.layout.type = XMLLayout\nappender.FILE.policies.type = Policies\nappender.FILE.policies.size.type = SizeBasedTriggeringPolicy\nappender.FILE.policies.size.size = 5MB\nappender.FILE.strategy.type = DefaultRolloverStrategy\nappender.FILE.strategy.max = 1\nappender.FILE.filter.threshold.type = ThresholdFilter\nappender.FILE.filter.threshold.level = INFO\n\n\nrootLogger.level=INFO\nrootLogger.appenderRefs = CONSOLE, FILE\nrootLogger.appenderRef.CONSOLE.ref = CONSOLE\nrootLogger.appenderRef.FILE.ref = FILE",
   "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n",
   "spark-env.sh": "",
-  "template.yaml": "metadata:\n  labels:\n    app.kubernetes.io/component: spark\n    app.kubernetes.io/instance: pyspark-pi\n    app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n    app.kubernetes.io/name: spark-k8s\n    app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n    prometheus.io/scrape: 'true'\n    stackable.tech/vendor: Stackable\n  name: spark\nspec:\n  affinity: {}\n  containers:\n  - env:\n    - name: CONTAINERDEBUG_LOG_DIRECTORY\n      value: /stackable/log/containerdebug\n    - name: _STACKABLE_PRE_HOOK\n      value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n    image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n    imagePullPolicy: IfNotPresent\n    name: spark\n    resources:\n      limits:\n        cpu: '2'\n        memory: 1Gi\n      requests:\n        cpu: '1'\n        memory: 1Gi\n    volumeMounts:\n    - mountPath: /stackable/log_config\n      name: log-config\n    - mountPath: /stackable/log\n      name: log\n  enableServiceLinks: false\n  securityContext:\n    fsGroup: 1000\n  serviceAccountName: pyspark-pi\n  volumes:\n  - emptyDir:\n      sizeLimit: 39Mi\n    name: log\n  - configMap:\n      name: pyspark-pi-driver-pod-template\n    name: log-config\n  - configMap:\n      name: pyspark-pi-driver-pod-template\n    name: config\n"
+  "template.yaml": "metadata:\n  labels:\n    app.kubernetes.io/component: spark\n    app.kubernetes.io/instance: pyspark-pi\n    app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n    app.kubernetes.io/name: spark-k8s\n    app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n    prometheus.io/scrape: 'true'\n    stackable.tech/vendor: Stackable\n  name: spark\nspec:\n  affinity: {}\n  containers:\n  - env:\n    - name: CONTAINERDEBUG_LOG_DIRECTORY\n      value: /stackable/log/containerdebug\n    - name: _STACKABLE_PRE_HOOK\n      value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n    image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n    imagePullPolicy: IfNotPresent\n    name: spark\n    resources:\n      limits:\n        cpu: '2'\n        memory: 1Gi\n      requests:\n        cpu: '1'\n        memory: 1Gi\n    volumeMounts:\n    - mountPath: /stackable/log_config\n      name: log-config\n    - mountPath: /stackable/log\n      name: log\n  enableServiceLinks: false\n  securityContext:\n    fsGroup: 1000\n    runAsNonRoot: true\n  serviceAccountName: pyspark-pi\n  volumes:\n  - emptyDir:\n      sizeLimit: 39Mi\n    name: log\n  - configMap:\n      name: pyspark-pi-driver-pod-template\n    name: log-config\n  - configMap:\n      name: pyspark-pi-driver-pod-template\n    name: config\n"
 }

tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json

@@ -2,5 +2,5 @@
   "log4j2.properties": "appenders = FILE, CONSOLE\n\nappender.CONSOLE.type = Console\nappender.CONSOLE.name = CONSOLE\nappender.CONSOLE.target = SYSTEM_ERR\nappender.CONSOLE.layout.type = PatternLayout\nappender.CONSOLE.layout.pattern = %d{ISO8601} %p [%t] %c - %m%n\nappender.CONSOLE.filter.threshold.type = ThresholdFilter\nappender.CONSOLE.filter.threshold.level = INFO\n\nappender.FILE.type = RollingFile\nappender.FILE.name = FILE\nappender.FILE.fileName = /stackable/log/spark/spark.log4j2.xml\nappender.FILE.filePattern = /stackable/log/spark/spark.log4j2.xml.%i\nappender.FILE.layout.type = XMLLayout\nappender.FILE.policies.type = Policies\nappender.FILE.policies.size.type = SizeBasedTriggeringPolicy\nappender.FILE.policies.size.size = 5MB\nappender.FILE.strategy.type = DefaultRolloverStrategy\nappender.FILE.strategy.max = 1\nappender.FILE.filter.threshold.type = ThresholdFilter\nappender.FILE.filter.threshold.level = INFO\n\n\nrootLogger.level=INFO\nrootLogger.appenderRefs = CONSOLE, FILE\nrootLogger.appenderRef.CONSOLE.ref = CONSOLE\nrootLogger.appenderRef.FILE.ref = FILE",
   "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n",
   "spark-env.sh": "",
-  "template.yaml": "metadata:\n  labels:\n    app.kubernetes.io/component: spark\n    app.kubernetes.io/instance: pyspark-pi\n    app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n    app.kubernetes.io/name: spark-k8s\n    app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n    stackable.tech/vendor: Stackable\n  name: spark\nspec:\n  affinity: {}\n  containers:\n  - env:\n    - name: CONTAINERDEBUG_LOG_DIRECTORY\n      value: /stackable/log/containerdebug\n    - name: _STACKABLE_PRE_HOOK\n      value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n    image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n    imagePullPolicy: IfNotPresent\n    name: spark\n    resources:\n      limits:\n        cpu: '2'\n        memory: 1Gi\n      requests:\n        cpu: '1'\n        memory: 1Gi\n    volumeMounts:\n    - mountPath: /stackable/log_config\n      name: log-config\n    - mountPath: /stackable/log\n      name: log\n  enableServiceLinks: false\n  securityContext:\n    fsGroup: 1000\n  serviceAccountName: pyspark-pi\n  volumes:\n  - emptyDir:\n      sizeLimit: 39Mi\n    name: log\n  - configMap:\n      name: pyspark-pi-executor-pod-template\n    name: log-config\n  - configMap:\n      name: pyspark-pi-executor-pod-template\n    name: config\n"
+  "template.yaml": "metadata:\n  labels:\n    app.kubernetes.io/component: spark\n    app.kubernetes.io/instance: pyspark-pi\n    app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n    app.kubernetes.io/name: spark-k8s\n    app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n    stackable.tech/vendor: Stackable\n  name: spark\nspec:\n  affinity: {}\n  containers:\n  - env:\n    - name: CONTAINERDEBUG_LOG_DIRECTORY\n      value: /stackable/log/containerdebug\n    - name: _STACKABLE_PRE_HOOK\n      value: containerdebug --output=/stackable/log/containerdebug-state.json --loop &\n    image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n    imagePullPolicy: IfNotPresent\n    name: spark\n    resources:\n      limits:\n        cpu: '2'\n        memory: 1Gi\n      requests:\n        cpu: '1'\n        memory: 1Gi\n    volumeMounts:\n    - mountPath: /stackable/log_config\n      name: log-config\n    - mountPath: /stackable/log\n      name: log\n  enableServiceLinks: false\n  securityContext:\n    fsGroup: 1000\n    runAsNonRoot: true\n  serviceAccountName: pyspark-pi\n  volumes:\n  - emptyDir:\n      sizeLimit: 39Mi\n    name: log\n  - configMap:\n      name: pyspark-pi-executor-pod-template\n    name: log-config\n  - configMap:\n      name: pyspark-pi-executor-pod-template\n    name: config\n"
 }

tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json

@@ -65,7 +65,8 @@
   "restartPolicy": "Never",
   "schedulerName": "default-scheduler",
   "securityContext": {
-    "fsGroup": 1000
+    "fsGroup": 1000,
+    "runAsNonRoot": true
   },
   "serviceAccount": "pyspark-pi",
   "serviceAccountName": "pyspark-pi",

tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json

@@ -2,5 +2,5 @@
   "metrics.properties": "*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet\n*.sink.prometheusServlet.path=/metrics/prometheus\n",
   "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n",
   "spark-defaults.conf": "spark.driver.cores=3\nspark.driver.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\\ -Dmy.custom.jvm.arg\\=customValue\nspark.driver.extraClassPath=/stackable/spark/extra-jars/*\\:/stackable/spark/connect/spark-connect-3.5.8.jar\nspark.driver.host=spark-connect-server-headless\nspark.executor.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\nspark.executor.instances=3\nspark.executor.memory=1024M\nspark.executor.memoryOverhead=1m\nspark.kubernetes.authenticate.driver.serviceAccountName=spark-connect-serviceaccount\nspark.kubernetes.driver.container.image=oci.stackable.tech/sdp/spark-k8s\\:3.5.8-stackable0.0.0-dev\nspark.kubernetes.driver.pod.name=${env\\:HOSTNAME}\nspark.kubernetes.executor.container.image=oci.stackable.tech/sdp/spark-k8s\\:3.5.8-stackable0.0.0-dev\nspark.kubernetes.executor.limit.cores=1\nspark.kubernetes.executor.podTemplateContainerName=spark\nspark.kubernetes.executor.podTemplateFile=/stackable/spark/conf/template.yaml\nspark.kubernetes.executor.request.cores=1\nspark.kubernetes.namespace=__NAMESPACE__\nspark.metrics.conf=/stackable/spark/conf/metrics.properties\nspark.ui.prometheus.enabled=true\n",
-  "template.yaml": "metadata:\n  labels:\n    app.kubernetes.io/component: executor\n    app.kubernetes.io/instance: spark-connect\n    app.kubernetes.io/managed-by: spark.stackable.tech_connect\n    app.kubernetes.io/name: spark-connect\n    app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n    stackable.tech/vendor: Stackable\nspec:\n  affinity:\n    podAntiAffinity:\n      preferredDuringSchedulingIgnoredDuringExecution:\n      - podAffinityTerm:\n          labelSelector:\n            matchLabels:\n              app.kubernetes.io/component: executor\n              app.kubernetes.io/instance: spark-connect\n              app.kubernetes.io/name: spark-connect\n          topologyKey: kubernetes.io/hostname\n        weight: 70\n  containers:\n  - env:\n    - name: CONTAINERDEBUG_LOG_DIRECTORY\n      value: /stackable/log/containerdebug\n    name: spark\n    volumeMounts:\n    - mountPath: /stackable/spark/conf\n      name: config\n    - mountPath: /stackable/log\n      name: log\n    - mountPath: /stackable/truststore\n      name: stackable-truststore\n    - mountPath: /stackable/log_config\n      name: log-config\n  enableServiceLinks: false\n  securityContext:\n    fsGroup: 1000\n  volumes:\n  - emptyDir:\n      sizeLimit: 30Mi\n    name: log\n  - configMap:\n      name: spark-connect-executor\n    name: config\n  - emptyDir: {}\n    name: stackable-truststore\n  - configMap:\n      name: spark-connect-log-config\n    name: log-config\n"
+  "template.yaml": "metadata:\n  labels:\n    app.kubernetes.io/component: executor\n    app.kubernetes.io/instance: spark-connect\n    app.kubernetes.io/managed-by: spark.stackable.tech_connect\n    app.kubernetes.io/name: spark-connect\n    app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n    stackable.tech/vendor: Stackable\nspec:\n  affinity:\n    podAntiAffinity:\n      preferredDuringSchedulingIgnoredDuringExecution:\n      - podAffinityTerm:\n          labelSelector:\n            matchLabels:\n              app.kubernetes.io/component: executor\n              app.kubernetes.io/instance: spark-connect\n              app.kubernetes.io/name: spark-connect\n          topologyKey: kubernetes.io/hostname\n        weight: 70\n  containers:\n  - env:\n    - name: CONTAINERDEBUG_LOG_DIRECTORY\n      value: /stackable/log/containerdebug\n    name: spark\n    volumeMounts:\n    - mountPath: /stackable/spark/conf\n      name: config\n    - mountPath: /stackable/log\n      name: log\n    - mountPath: /stackable/truststore\n      name: stackable-truststore\n    - mountPath: /stackable/log_config\n      name: log-config\n  enableServiceLinks: false\n  securityContext:\n    fsGroup: 1000\n    runAsNonRoot: true\n  volumes:\n  - emptyDir:\n      sizeLimit: 30Mi\n    name: log\n  - configMap:\n      name: spark-connect-executor\n    name: config\n  - emptyDir: {}\n    name: stackable-truststore\n  - configMap:\n      name: spark-connect-log-config\n    name: log-config\n"
 }