LCORE-3059: Lightspeed Stack 0.5.3 e2e test fixes [release/0.5]

.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml

@@ -80,6 +80,8 @@ spec:
                 value: "$(steps.pick-version.results.version)"
               - name: instanceType
                 value: "m5.large"
+              - name: timeout
+                value: "4h0m0s"
     - name: get-stack-images
       description: Extract lightspeed-stack image and commit from SNAPSHOT (Llama Stack runs from source in-pod)
       runAfter:

scripts/llama-stack-entrypoint.sh

@@ -5,19 +5,20 @@
 set -e
 
 INPUT_CONFIG="${LLAMA_STACK_CONFIG:-/opt/app-root/run.yaml}"
-ENRICHED_CONFIG="/opt/app-root/run.yaml"
+ENRICHED_CONFIG="/tmp/enriched-run.yaml"
 LIGHTSPEED_CONFIG="${LIGHTSPEED_CONFIG:-/opt/app-root/lightspeed-stack.yaml}"
 ENV_FILE="/opt/app-root/.env"
 
 # Enrich config if lightspeed config exists
 if [ -f "$LIGHTSPEED_CONFIG" ]; then
     echo "Enriching llama-stack config..."
     ENRICHMENT_FAILED=0
+    ENRICHMENT_LOG="/tmp/enrichment.log"
     python3 /opt/app-root/llama_stack_configuration.py \
         -c "$LIGHTSPEED_CONFIG" \
         -i "$INPUT_CONFIG" \
         -o "$ENRICHED_CONFIG" \
-        -e "$ENV_FILE" 2>&1 || ENRICHMENT_FAILED=1
+        -e "$ENV_FILE" 2>&1 | tee "$ENRICHMENT_LOG" || ENRICHMENT_FAILED=1
 
     # Source .env if generated (contains AZURE_API_KEY)
     if [ -f "$ENV_FILE" ]; then
@@ -28,6 +29,10 @@ if [ -f "$LIGHTSPEED_CONFIG" ]; then
     if [ -f "$ENRICHED_CONFIG" ] && [ "$ENRICHMENT_FAILED" -eq 0 ]; then
         echo "Using enriched config: $ENRICHED_CONFIG"
         exec llama stack run "$ENRICHED_CONFIG"
+    else
+        echo "WARNING: enrichment failed (exit=$ENRICHMENT_FAILED), log:"
+        cat "$ENRICHMENT_LOG" 2>/dev/null || true
+        echo "WARNING: falling through to unenriched config"
     fi
 fi
 

src/version.py

@@ -9,4 +9,4 @@
 # [tool.pdm.version]
 # source = "file"
 # path = "src/version.py"
-__version__ = "0.5.1"
+__version__ = "0.5.3"

tests/e2e-prow/rhoai/configs/lightspeed-stack-auth-noop-token.yaml

@@ -29,3 +29,16 @@ conversation_cache:
 
 authentication:
   module: "noop-with-token"
+
+byok_rag:
+  - rag_id: e2e-test-docs
+    rag_type: inline::faiss
+    embedding_model: sentence-transformers/all-mpnet-base-v2
+    embedding_dimension: 768
+    vector_db_id: ${env.FAISS_VECTOR_STORE_ID}
+    db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db}
+    score_multiplier: 1.0
+
+rag:
+  tool:
+    - e2e-test-docs

tests/e2e-prow/rhoai/configs/lightspeed-stack-rbac.yaml

@@ -75,6 +75,7 @@ authorization:
         - "get_tools"
         - "info"
         - "model_override"
+        - "rlsapi_v1_infer"
     # Viewer role can only read (no mutations)
     - role: "viewer"
       actions:

tests/e2e-prow/rhoai/configs/lightspeed-stack.yaml

@@ -24,6 +24,19 @@ user_data_collection:
 authentication:
   module: "noop"
 
+byok_rag:
+  - rag_id: e2e-test-docs
+    rag_type: inline::faiss
+    embedding_model: sentence-transformers/all-mpnet-base-v2
+    embedding_dimension: 768
+    vector_db_id: ${env.FAISS_VECTOR_STORE_ID}
+    db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db}
+    score_multiplier: 1.0
+
+rag:
+  tool:
+    - e2e-test-docs
+
 mcp_servers:
   # Mock server with client-provided auth - should appear in mcp-auth/client-options response
   - name: "github-api"

tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml

@@ -22,6 +22,11 @@ spec:
             secretKeyRef:
               name: llama-stack-ip-secret
               key: key
+        - name: FAISS_VECTOR_STORE_ID
+          valueFrom:
+            secretKeyRef:
+              name: faiss-vector-store-secret
+              key: id
       image: ${LIGHTSPEED_STACK_IMAGE}
       ports:
         - containerPort: 8080

tests/e2e-prow/rhoai/manifests/lightspeed/llama-stack-openai.yaml

@@ -21,13 +21,21 @@ spec:
   securityContext:
     seccompProfile:
       type: RuntimeDefault
+    fsGroup: 0
   initContainers:
     - name: setup-from-source
       image: registry.access.redhat.com/ubi9/python-312
-      # runAsUser: 0 required for dnf install; avoids capabilities.drop so dnf/chroot works (PodSecurity restricted will warn)
       securityContext:
-        runAsUser: 0
+        allowPrivilegeEscalation: false
+        capabilities:
+          drop: ["ALL"]
+        runAsNonRoot: true
+        runAsUser: 1001
+        seccompProfile:
+          type: RuntimeDefault
       env:
+        - name: HOME
+          value: "/opt/app-root/src"
         - name: REPO_URL
           valueFrom:
             configMapKeyRef:
@@ -44,28 +52,39 @@ spec:
         - /bin/bash
         - -c
         - |
-          set -e
-          # Repo URL/revision: from ConfigMap or defaults; convert SSH to HTTPS for clone
+          set -ex
+          echo "[init] starting setup-from-source"
+          echo "[init] emptyDir contents:" && ls -la /opt/app-root/ 2>&1 || true
+          # WORKDIR is /opt/app-root/src (from image); move out before removing it
+          cd /opt/app-root
+          # emptyDir src/ is root-owned; recreate it so user 1001 can write
+          rm -rf /opt/app-root/src && mkdir -p /opt/app-root/src
           REPO_URL="${REPO_URL:-https://github.com/lightspeed-core/lightspeed-stack.git}"
           REPO_REVISION="${REPO_REVISION:-main}"
           case "$REPO_URL" in git@github.com:*) REPO_URL="https://github.com/${REPO_URL#git@github.com:}"; esac
-          # Install system deps and uv
-          dnf -y install git tar gcc gcc-c++ make
-          curl -LsSf https://astral.sh/uv/install.sh | sh
-          export PATH="/root/.local/bin:$PATH"
-          ln -sf /usr/bin/python3.12 /usr/bin/python3 2>/dev/null || true
-          # Clone and install Python deps
+          # Install uv to /tmp (emptyDir $HOME/.local not writable)
+          echo "[init] installing uv"
+          curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/tmp/uv-bin INSTALLER_NO_MODIFY_PATH=1 XDG_CONFIG_HOME=/tmp sh
+          export PATH="/tmp/uv-bin:$PATH"
+          export UV_CACHE_DIR=/tmp/uv-cache
+          uv --version
+          # Clone and install Python deps (rm -rf handles restarts where emptyDir persists)
+          echo "[init] cloning $REPO_URL @ $REPO_REVISION"
+          rm -rf /opt/app-root/repo
           git clone -q "${REPO_URL}" /opt/app-root/repo
           cd /opt/app-root/repo && git fetch origin "${REPO_REVISION}" 2>/dev/null || true && git checkout -q "${REPO_REVISION}"
+          echo "[init] running uv sync"
           uv sync --locked --no-install-project --group llslibdev
+          echo "[init] uv sync complete, moving repo to /opt/app-root"
           # Move repo to /opt/app-root and fix venv shebangs (paths changed from repo/.venv to .venv)
-          (cd /opt/app-root/repo && tar cf - .) | (cd /opt/app-root && tar xf -)
+          (cd /opt/app-root/repo && tar cf - .) | (cd /opt/app-root && tar x --no-overwrite-dir -f -)
           rm -rf /opt/app-root/repo
           sed -i 's|/opt/app-root/repo/.venv|/opt/app-root/.venv|g' /opt/app-root/.venv/bin/* 2>/dev/null || true
-          # Layout and permissions for main container
+          # Layout for main container (no chown needed -- already running as 1001)
           mkdir -p /opt/app-root/src/.llama/storage/rag /opt/app-root/src/.llama/storage/files
           cp /opt/app-root/src/llama_stack_configuration.py /opt/app-root/llama_stack_configuration.py
-          chmod -R 775 /opt/app-root && chown -R 1001:0 /opt/app-root
+          echo "[init] setup-from-source complete"
+          echo "[init] setup-from-source complete"
       volumeMounts:
         - name: app-root
           mountPath: /opt/app-root
@@ -76,17 +95,17 @@ spec:
         capabilities:
           drop: ["ALL"]
         runAsNonRoot: true
-        runAsUser: 65534
+        runAsUser: 1001
         seccompProfile:
           type: RuntimeDefault
       command:
         - /bin/sh
         - -c
         - |
+          set -e
           mkdir -p /data/src/.llama/storage/rag /data/src/.llama/storage/files
-          chmod -R 777 /data
           gunzip -c /rag-data/kv_store.db.gz > /data/src/.llama/storage/rag/kv_store.db
-          chmod -R 777 /data
+          ls -la /data/src/.llama/storage/rag/kv_store.db
           echo "RAG data extracted successfully"
       volumeMounts:
         - name: app-root
@@ -110,6 +129,10 @@ spec:
           value: "/opt/app-root/.venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
         - name: HOME
           value: "/opt/app-root/src"
+        - name: PYTHONPATH
+          value: "/opt/app-root/src"
+        - name: E2E_LLAMA_HOSTNAME
+          value: "localhost"
         - name: KV_STORE_PATH
           value: "/opt/app-root/src/.llama/storage/rag/kv_store.db"
         - name: KV_RAG_PATH
@@ -157,12 +180,25 @@ spec:
         - name: config
           mountPath: /opt/app-root/run.yaml
           subPath: run.yaml
+        - name: lcs-config
+          mountPath: /opt/app-root/lightspeed-stack.yaml
+          subPath: lightspeed-stack.yaml
+        - name: entrypoint
+          mountPath: /opt/app-root/scripts/llama-stack-entrypoint.sh
+          subPath: llama-stack-entrypoint.sh
   volumes:
     - name: app-root
       emptyDir: {}
     - name: config
       configMap:
         name: llama-stack-config
+    - name: lcs-config
+      configMap:
+        name: lightspeed-stack-config
+    - name: entrypoint
+      configMap:
+        name: llama-stack-entrypoint
+        defaultMode: 0755
     - name: rag-data
       configMap:
         name: rag-data

tests/e2e-prow/rhoai/pipeline-konflux.sh

@@ -21,7 +21,9 @@ log() { [ "$QUIET" != "1" ] && echo "$@"; }
 progress() { echo "[e2e] $*"; }
 
 # Lightspeed-stack image (from Konflux SNAPSHOT or default). Llama Stack runs from source in-pod (no image).
-LIGHTSPEED_STACK_IMAGE="${LIGHTSPEED_STACK_IMAGE:-quay.io/lightspeed-core/lightspeed-stack:dev-latest}"
+if [[ -z "${LIGHTSPEED_STACK_IMAGE:-}" ]]; then
+  echo "LIGHTSPEED_STACK_IMAGE is not set"; exit 1
+fi
 log "Using lightspeed-stack image: $LIGHTSPEED_STACK_IMAGE"
 export LIGHTSPEED_STACK_IMAGE
 
@@ -195,19 +197,40 @@ oc create configmap llama-stack-source -n "$NAMESPACE" \
   --dry-run=client -o yaml | oc apply -f -
 log "llama-stack-source ConfigMap: repo @ ${REPO_REVISION}"
 
+# Entrypoint from THIS workspace (not from the cloned repo) so enrichment fixes are always applied
+oc create configmap llama-stack-entrypoint -n "$NAMESPACE" \
+  --from-file=llama-stack-entrypoint.sh="$REPO_ROOT/scripts/llama-stack-entrypoint.sh" \
+  --dry-run=client -o yaml | oc apply -f -
+
 "$PIPELINE_DIR/pipeline-services-konflux.sh"
 
 progress "Waiting for lightspeed-stack and llama-stack pods"
 if ! oc wait pod/lightspeed-stack-service pod/llama-stack-service \
     -n "$NAMESPACE" --for=condition=Ready --timeout=600s; then
   progress "❌ One or both service pods failed to become ready within timeout"
+  echo "[e2e] ========== pod status =========="
+  oc get pods -n "$NAMESPACE" -o wide 2>&1 || true
+  echo "[e2e] ========== llama-stack-service describe =========="
+  oc describe pod llama-stack-service -n "$NAMESPACE" 2>&1 | tail -50 || true
+  echo "[e2e] ========== llama-stack-service init: setup-from-source logs =========="
+  oc logs llama-stack-service -n "$NAMESPACE" -c setup-from-source 2>&1 | tail -80 || true
+  echo "[e2e] ========== llama-stack-service init: setup-rag-data logs =========="
+  oc logs llama-stack-service -n "$NAMESPACE" -c setup-rag-data 2>&1 | tail -40 || true
+  echo "[e2e] ========== llama-stack-service container logs =========="
+  oc logs llama-stack-service -n "$NAMESPACE" -c llama-stack-container 2>&1 | tail -80 || true
+  echo "[e2e] ========== lightspeed-stack-service describe =========="
+  oc describe pod lightspeed-stack-service -n "$NAMESPACE" 2>&1 | tail -50 || true
+  echo "[e2e] ========== lightspeed-stack-service logs =========="
+  oc logs lightspeed-stack-service -n "$NAMESPACE" 2>&1 | tail -80 || true
+  echo "[e2e] ========== recent events =========="
+  oc get events -n "$NAMESPACE" --sort-by='.lastTimestamp' 2>&1 | tail -30 || true
   exit 1
 fi
 log "✅ Both service pods are ready"
 
 # Print pod logs with echo so CI/Konflux log capture shows each line (especially when QUIET=1)
 e2e_echo_pod_logs() {
-  local n="${1:-120}"
+  local n="${1:-500}"
   echo "[e2e] ========== lightspeed-stack-service logs (tail $n) =========="
   while IFS= read -r line || [[ -n "$line" ]]; do
     echo "[e2e] $line"
@@ -222,7 +245,7 @@ if [ "$QUIET" = "1" ]; then
   e2e_echo_pod_logs 80
 else
   oc get pods -n "$NAMESPACE"
-  e2e_echo_pod_logs 200
+  e2e_echo_pod_logs 500
   echo "[e2e] ========== oc describe lightspeed-stack-service =========="
   oc describe pod lightspeed-stack-service -n "$NAMESPACE" 2>&1 | while IFS= read -r line || [[ -n "$line" ]]; do echo "[e2e] $line"; done || true
   echo "[e2e] ========== oc describe llama-stack-service =========="

tests/e2e-prow/rhoai/pipeline-services-konflux.sh

@@ -20,7 +20,19 @@ fi
 
 # 1. Llama Stack (run from source; manifest is static, no envsubst)
 oc apply -n "$NAMESPACE" -f "$BASE_DIR/manifests/lightspeed/llama-stack-openai.yaml"
-oc wait pod/llama-stack-service -n "$NAMESPACE" --for=condition=Ready --timeout=600s
+if ! oc wait pod/llama-stack-service -n "$NAMESPACE" --for=condition=Ready --timeout=600s; then
+  echo "FAILED: llama-stack-service pod did not become ready"
+  echo "========== pod status =========="
+  oc get pod llama-stack-service -n "$NAMESPACE" -o wide 2>&1 || true
+  echo "========== pod describe (events) =========="
+  oc describe pod llama-stack-service -n "$NAMESPACE" 2>&1 | tail -40 || true
+  echo "========== init container: setup-from-source logs =========="
+  oc logs llama-stack-service -n "$NAMESPACE" -c setup-from-source 2>&1 | tail -80 || true
+  echo "========== init container: setup-rag-data logs =========="
+  oc logs llama-stack-service -n "$NAMESPACE" -c setup-rag-data 2>&1 | tail -40 || true
+  echo "========== main container: llama-stack-container logs =========="
+  oc logs llama-stack-service -n "$NAMESPACE" -c llama-stack-container 2>&1 | tail -80 || true
+fi
 
 oc label pod llama-stack-service pod=llama-stack-service -n "$NAMESPACE"
 oc expose pod llama-stack-service --name=llama-stack-service-svc --port=8321 --type=ClusterIP -n "$NAMESPACE"
@@ -29,7 +41,9 @@ export E2E_LLAMA_HOSTNAME="llama-stack-service-svc.${NAMESPACE}.svc.cluster.loca
 oc create secret generic llama-stack-ip-secret --from-literal=key="$E2E_LLAMA_HOSTNAME" -n "$NAMESPACE" || true
 
 # 2. Lightspeed Stack (image from env; default if unset)
-LIGHTSPEED_STACK_IMAGE="${LIGHTSPEED_STACK_IMAGE:-quay.io/lightspeed-core/lightspeed-stack:dev-latest}"
+if [[ -z "${LIGHTSPEED_STACK_IMAGE:-}" ]]; then
+  echo "LIGHTSPEED_STACK_IMAGE is not set"; exit 1
+fi
 export LIGHTSPEED_STACK_IMAGE
 LIGHTSPEED_MANIFEST="$BASE_DIR/manifests/lightspeed/lightspeed-stack.yaml"
 if command -v envsubst >/dev/null 2>&1; then