diff --git a/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml b/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml index 33e967776..dfb106b18 100644 --- a/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml +++ b/.tekton/integration-tests/pipeline/lightspeed-stack-integration-test.yaml @@ -80,6 +80,8 @@ spec: value: "$(steps.pick-version.results.version)" - name: instanceType value: "m5.large" + - name: timeout + value: "4h0m0s" - name: get-stack-images description: Extract lightspeed-stack image and commit from SNAPSHOT (Llama Stack runs from source in-pod) runAfter: diff --git a/scripts/llama-stack-entrypoint.sh b/scripts/llama-stack-entrypoint.sh index a7eeb797b..b235b8e53 100755 --- a/scripts/llama-stack-entrypoint.sh +++ b/scripts/llama-stack-entrypoint.sh @@ -5,7 +5,7 @@ set -e INPUT_CONFIG="${LLAMA_STACK_CONFIG:-/opt/app-root/run.yaml}" -ENRICHED_CONFIG="/opt/app-root/run.yaml" +ENRICHED_CONFIG="/tmp/enriched-run.yaml" LIGHTSPEED_CONFIG="${LIGHTSPEED_CONFIG:-/opt/app-root/lightspeed-stack.yaml}" ENV_FILE="/opt/app-root/.env" @@ -13,11 +13,12 @@ ENV_FILE="/opt/app-root/.env" if [ -f "$LIGHTSPEED_CONFIG" ]; then echo "Enriching llama-stack config..." ENRICHMENT_FAILED=0 + ENRICHMENT_LOG="/tmp/enrichment.log" python3 /opt/app-root/llama_stack_configuration.py \ -c "$LIGHTSPEED_CONFIG" \ -i "$INPUT_CONFIG" \ -o "$ENRICHED_CONFIG" \ - -e "$ENV_FILE" 2>&1 || ENRICHMENT_FAILED=1 + -e "$ENV_FILE" 2>&1 | tee "$ENRICHMENT_LOG" || ENRICHMENT_FAILED=1 # Source .env if generated (contains AZURE_API_KEY) if [ -f "$ENV_FILE" ]; then @@ -28,6 +29,10 @@ if [ -f "$LIGHTSPEED_CONFIG" ]; then if [ -f "$ENRICHED_CONFIG" ] && [ "$ENRICHMENT_FAILED" -eq 0 ]; then echo "Using enriched config: $ENRICHED_CONFIG" exec llama stack run "$ENRICHED_CONFIG" + else + echo "WARNING: enrichment failed (exit=$ENRICHMENT_FAILED), log:" + cat "$ENRICHMENT_LOG" 2>/dev/null || true + echo "WARNING: falling through to unenriched config" fi fi diff --git a/src/version.py b/src/version.py index a96388aeb..5aefa72eb 100644 --- a/src/version.py +++ b/src/version.py @@ -9,4 +9,4 @@ # [tool.pdm.version] # source = "file" # path = "src/version.py" -__version__ = "0.5.1" +__version__ = "0.5.3" diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack-auth-noop-token.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack-auth-noop-token.yaml index 4dfd3ed4e..4cad37b0f 100644 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack-auth-noop-token.yaml +++ b/tests/e2e-prow/rhoai/configs/lightspeed-stack-auth-noop-token.yaml @@ -29,3 +29,16 @@ conversation_cache: authentication: module: "noop-with-token" + +byok_rag: + - rag_id: e2e-test-docs + rag_type: inline::faiss + embedding_model: sentence-transformers/all-mpnet-base-v2 + embedding_dimension: 768 + vector_db_id: ${env.FAISS_VECTOR_STORE_ID} + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} + score_multiplier: 1.0 + +rag: + tool: + - e2e-test-docs diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack-rbac.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack-rbac.yaml index e2e4bfa16..fd5e02b2d 100644 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack-rbac.yaml +++ b/tests/e2e-prow/rhoai/configs/lightspeed-stack-rbac.yaml @@ -75,6 +75,7 @@ authorization: - "get_tools" - "info" - "model_override" + - "rlsapi_v1_infer" # Viewer role can only read (no mutations) - role: "viewer" actions: diff --git a/tests/e2e-prow/rhoai/configs/lightspeed-stack.yaml b/tests/e2e-prow/rhoai/configs/lightspeed-stack.yaml index b1fecbdbb..36efd189f 100644 --- a/tests/e2e-prow/rhoai/configs/lightspeed-stack.yaml +++ b/tests/e2e-prow/rhoai/configs/lightspeed-stack.yaml @@ -24,6 +24,19 @@ user_data_collection: authentication: module: "noop" +byok_rag: + - rag_id: e2e-test-docs + rag_type: inline::faiss + embedding_model: sentence-transformers/all-mpnet-base-v2 + embedding_dimension: 768 + vector_db_id: ${env.FAISS_VECTOR_STORE_ID} + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} + score_multiplier: 1.0 + +rag: + tool: + - e2e-test-docs + mcp_servers: # Mock server with client-provided auth - should appear in mcp-auth/client-options response - name: "github-api" diff --git a/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml b/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml index bd93c4c44..2b0588228 100644 --- a/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml +++ b/tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml @@ -22,6 +22,11 @@ spec: secretKeyRef: name: llama-stack-ip-secret key: key + - name: FAISS_VECTOR_STORE_ID + valueFrom: + secretKeyRef: + name: faiss-vector-store-secret + key: id image: ${LIGHTSPEED_STACK_IMAGE} ports: - containerPort: 8080 diff --git a/tests/e2e-prow/rhoai/manifests/lightspeed/llama-stack-openai.yaml b/tests/e2e-prow/rhoai/manifests/lightspeed/llama-stack-openai.yaml index 58b5c29f1..2c196622d 100644 --- a/tests/e2e-prow/rhoai/manifests/lightspeed/llama-stack-openai.yaml +++ b/tests/e2e-prow/rhoai/manifests/lightspeed/llama-stack-openai.yaml @@ -21,13 +21,21 @@ spec: securityContext: seccompProfile: type: RuntimeDefault + fsGroup: 0 initContainers: - name: setup-from-source image: registry.access.redhat.com/ubi9/python-312 - # runAsUser: 0 required for dnf install; avoids capabilities.drop so dnf/chroot works (PodSecurity restricted will warn) securityContext: - runAsUser: 0 + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault env: + - name: HOME + value: "/opt/app-root/src" - name: REPO_URL valueFrom: configMapKeyRef: @@ -44,28 +52,39 @@ spec: - /bin/bash - -c - | - set -e - # Repo URL/revision: from ConfigMap or defaults; convert SSH to HTTPS for clone + set -ex + echo "[init] starting setup-from-source" + echo "[init] emptyDir contents:" && ls -la /opt/app-root/ 2>&1 || true + # WORKDIR is /opt/app-root/src (from image); move out before removing it + cd /opt/app-root + # emptyDir src/ is root-owned; recreate it so user 1001 can write + rm -rf /opt/app-root/src && mkdir -p /opt/app-root/src REPO_URL="${REPO_URL:-https://github.com/lightspeed-core/lightspeed-stack.git}" REPO_REVISION="${REPO_REVISION:-main}" case "$REPO_URL" in git@github.com:*) REPO_URL="https://github.com/${REPO_URL#git@github.com:}"; esac - # Install system deps and uv - dnf -y install git tar gcc gcc-c++ make - curl -LsSf https://astral.sh/uv/install.sh | sh - export PATH="/root/.local/bin:$PATH" - ln -sf /usr/bin/python3.12 /usr/bin/python3 2>/dev/null || true - # Clone and install Python deps + # Install uv to /tmp (emptyDir $HOME/.local not writable) + echo "[init] installing uv" + curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/tmp/uv-bin INSTALLER_NO_MODIFY_PATH=1 XDG_CONFIG_HOME=/tmp sh + export PATH="/tmp/uv-bin:$PATH" + export UV_CACHE_DIR=/tmp/uv-cache + uv --version + # Clone and install Python deps (rm -rf handles restarts where emptyDir persists) + echo "[init] cloning $REPO_URL @ $REPO_REVISION" + rm -rf /opt/app-root/repo git clone -q "${REPO_URL}" /opt/app-root/repo cd /opt/app-root/repo && git fetch origin "${REPO_REVISION}" 2>/dev/null || true && git checkout -q "${REPO_REVISION}" + echo "[init] running uv sync" uv sync --locked --no-install-project --group llslibdev + echo "[init] uv sync complete, moving repo to /opt/app-root" # Move repo to /opt/app-root and fix venv shebangs (paths changed from repo/.venv to .venv) - (cd /opt/app-root/repo && tar cf - .) | (cd /opt/app-root && tar xf -) + (cd /opt/app-root/repo && tar cf - .) | (cd /opt/app-root && tar x --no-overwrite-dir -f -) rm -rf /opt/app-root/repo sed -i 's|/opt/app-root/repo/.venv|/opt/app-root/.venv|g' /opt/app-root/.venv/bin/* 2>/dev/null || true - # Layout and permissions for main container + # Layout for main container (no chown needed -- already running as 1001) mkdir -p /opt/app-root/src/.llama/storage/rag /opt/app-root/src/.llama/storage/files cp /opt/app-root/src/llama_stack_configuration.py /opt/app-root/llama_stack_configuration.py - chmod -R 775 /opt/app-root && chown -R 1001:0 /opt/app-root + echo "[init] setup-from-source complete" + echo "[init] setup-from-source complete" volumeMounts: - name: app-root mountPath: /opt/app-root @@ -76,17 +95,17 @@ spec: capabilities: drop: ["ALL"] runAsNonRoot: true - runAsUser: 65534 + runAsUser: 1001 seccompProfile: type: RuntimeDefault command: - /bin/sh - -c - | + set -e mkdir -p /data/src/.llama/storage/rag /data/src/.llama/storage/files - chmod -R 777 /data gunzip -c /rag-data/kv_store.db.gz > /data/src/.llama/storage/rag/kv_store.db - chmod -R 777 /data + ls -la /data/src/.llama/storage/rag/kv_store.db echo "RAG data extracted successfully" volumeMounts: - name: app-root @@ -110,6 +129,10 @@ spec: value: "/opt/app-root/.venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - name: HOME value: "/opt/app-root/src" + - name: PYTHONPATH + value: "/opt/app-root/src" + - name: E2E_LLAMA_HOSTNAME + value: "localhost" - name: KV_STORE_PATH value: "/opt/app-root/src/.llama/storage/rag/kv_store.db" - name: KV_RAG_PATH @@ -157,12 +180,25 @@ spec: - name: config mountPath: /opt/app-root/run.yaml subPath: run.yaml + - name: lcs-config + mountPath: /opt/app-root/lightspeed-stack.yaml + subPath: lightspeed-stack.yaml + - name: entrypoint + mountPath: /opt/app-root/scripts/llama-stack-entrypoint.sh + subPath: llama-stack-entrypoint.sh volumes: - name: app-root emptyDir: {} - name: config configMap: name: llama-stack-config + - name: lcs-config + configMap: + name: lightspeed-stack-config + - name: entrypoint + configMap: + name: llama-stack-entrypoint + defaultMode: 0755 - name: rag-data configMap: name: rag-data diff --git a/tests/e2e-prow/rhoai/pipeline-konflux.sh b/tests/e2e-prow/rhoai/pipeline-konflux.sh index 1dce15714..921f2333d 100755 --- a/tests/e2e-prow/rhoai/pipeline-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-konflux.sh @@ -21,7 +21,9 @@ log() { [ "$QUIET" != "1" ] && echo "$@"; } progress() { echo "[e2e] $*"; } # Lightspeed-stack image (from Konflux SNAPSHOT or default). Llama Stack runs from source in-pod (no image). -LIGHTSPEED_STACK_IMAGE="${LIGHTSPEED_STACK_IMAGE:-quay.io/lightspeed-core/lightspeed-stack:dev-latest}" +if [[ -z "${LIGHTSPEED_STACK_IMAGE:-}" ]]; then + echo "LIGHTSPEED_STACK_IMAGE is not set"; exit 1 +fi log "Using lightspeed-stack image: $LIGHTSPEED_STACK_IMAGE" export LIGHTSPEED_STACK_IMAGE @@ -195,19 +197,40 @@ oc create configmap llama-stack-source -n "$NAMESPACE" \ --dry-run=client -o yaml | oc apply -f - log "llama-stack-source ConfigMap: repo @ ${REPO_REVISION}" +# Entrypoint from THIS workspace (not from the cloned repo) so enrichment fixes are always applied +oc create configmap llama-stack-entrypoint -n "$NAMESPACE" \ + --from-file=llama-stack-entrypoint.sh="$REPO_ROOT/scripts/llama-stack-entrypoint.sh" \ + --dry-run=client -o yaml | oc apply -f - + "$PIPELINE_DIR/pipeline-services-konflux.sh" progress "Waiting for lightspeed-stack and llama-stack pods" if ! oc wait pod/lightspeed-stack-service pod/llama-stack-service \ -n "$NAMESPACE" --for=condition=Ready --timeout=600s; then progress "❌ One or both service pods failed to become ready within timeout" + echo "[e2e] ========== pod status ==========" + oc get pods -n "$NAMESPACE" -o wide 2>&1 || true + echo "[e2e] ========== llama-stack-service describe ==========" + oc describe pod llama-stack-service -n "$NAMESPACE" 2>&1 | tail -50 || true + echo "[e2e] ========== llama-stack-service init: setup-from-source logs ==========" + oc logs llama-stack-service -n "$NAMESPACE" -c setup-from-source 2>&1 | tail -80 || true + echo "[e2e] ========== llama-stack-service init: setup-rag-data logs ==========" + oc logs llama-stack-service -n "$NAMESPACE" -c setup-rag-data 2>&1 | tail -40 || true + echo "[e2e] ========== llama-stack-service container logs ==========" + oc logs llama-stack-service -n "$NAMESPACE" -c llama-stack-container 2>&1 | tail -80 || true + echo "[e2e] ========== lightspeed-stack-service describe ==========" + oc describe pod lightspeed-stack-service -n "$NAMESPACE" 2>&1 | tail -50 || true + echo "[e2e] ========== lightspeed-stack-service logs ==========" + oc logs lightspeed-stack-service -n "$NAMESPACE" 2>&1 | tail -80 || true + echo "[e2e] ========== recent events ==========" + oc get events -n "$NAMESPACE" --sort-by='.lastTimestamp' 2>&1 | tail -30 || true exit 1 fi log "✅ Both service pods are ready" # Print pod logs with echo so CI/Konflux log capture shows each line (especially when QUIET=1) e2e_echo_pod_logs() { - local n="${1:-120}" + local n="${1:-500}" echo "[e2e] ========== lightspeed-stack-service logs (tail $n) ==========" while IFS= read -r line || [[ -n "$line" ]]; do echo "[e2e] $line" @@ -222,7 +245,7 @@ if [ "$QUIET" = "1" ]; then e2e_echo_pod_logs 80 else oc get pods -n "$NAMESPACE" - e2e_echo_pod_logs 200 + e2e_echo_pod_logs 500 echo "[e2e] ========== oc describe lightspeed-stack-service ==========" oc describe pod lightspeed-stack-service -n "$NAMESPACE" 2>&1 | while IFS= read -r line || [[ -n "$line" ]]; do echo "[e2e] $line"; done || true echo "[e2e] ========== oc describe llama-stack-service ==========" diff --git a/tests/e2e-prow/rhoai/pipeline-services-konflux.sh b/tests/e2e-prow/rhoai/pipeline-services-konflux.sh index 244de991f..f05aa0ecd 100755 --- a/tests/e2e-prow/rhoai/pipeline-services-konflux.sh +++ b/tests/e2e-prow/rhoai/pipeline-services-konflux.sh @@ -20,7 +20,19 @@ fi # 1. Llama Stack (run from source; manifest is static, no envsubst) oc apply -n "$NAMESPACE" -f "$BASE_DIR/manifests/lightspeed/llama-stack-openai.yaml" -oc wait pod/llama-stack-service -n "$NAMESPACE" --for=condition=Ready --timeout=600s +if ! oc wait pod/llama-stack-service -n "$NAMESPACE" --for=condition=Ready --timeout=600s; then + echo "FAILED: llama-stack-service pod did not become ready" + echo "========== pod status ==========" + oc get pod llama-stack-service -n "$NAMESPACE" -o wide 2>&1 || true + echo "========== pod describe (events) ==========" + oc describe pod llama-stack-service -n "$NAMESPACE" 2>&1 | tail -40 || true + echo "========== init container: setup-from-source logs ==========" + oc logs llama-stack-service -n "$NAMESPACE" -c setup-from-source 2>&1 | tail -80 || true + echo "========== init container: setup-rag-data logs ==========" + oc logs llama-stack-service -n "$NAMESPACE" -c setup-rag-data 2>&1 | tail -40 || true + echo "========== main container: llama-stack-container logs ==========" + oc logs llama-stack-service -n "$NAMESPACE" -c llama-stack-container 2>&1 | tail -80 || true +fi oc label pod llama-stack-service pod=llama-stack-service -n "$NAMESPACE" oc expose pod llama-stack-service --name=llama-stack-service-svc --port=8321 --type=ClusterIP -n "$NAMESPACE" @@ -29,7 +41,9 @@ export E2E_LLAMA_HOSTNAME="llama-stack-service-svc.${NAMESPACE}.svc.cluster.loca oc create secret generic llama-stack-ip-secret --from-literal=key="$E2E_LLAMA_HOSTNAME" -n "$NAMESPACE" || true # 2. Lightspeed Stack (image from env; default if unset) -LIGHTSPEED_STACK_IMAGE="${LIGHTSPEED_STACK_IMAGE:-quay.io/lightspeed-core/lightspeed-stack:dev-latest}" +if [[ -z "${LIGHTSPEED_STACK_IMAGE:-}" ]]; then + echo "LIGHTSPEED_STACK_IMAGE is not set"; exit 1 +fi export LIGHTSPEED_STACK_IMAGE LIGHTSPEED_MANIFEST="$BASE_DIR/manifests/lightspeed/lightspeed-stack.yaml" if command -v envsubst >/dev/null 2>&1; then diff --git a/tests/e2e-prow/rhoai/scripts/e2e-ops.sh b/tests/e2e-prow/rhoai/scripts/e2e-ops.sh index 3f354e43f..1fc1c9f73 100755 --- a/tests/e2e-prow/rhoai/scripts/e2e-ops.sh +++ b/tests/e2e-prow/rhoai/scripts/e2e-ops.sh @@ -243,7 +243,9 @@ cmd_restart_lightspeed() { } # Apply manifest (expand LIGHTSPEED_STACK_IMAGE) - LIGHTSPEED_STACK_IMAGE="${LIGHTSPEED_STACK_IMAGE:-quay.io/lightspeed-core/lightspeed-stack:dev-latest}" + if [[ -z "${LIGHTSPEED_STACK_IMAGE:-}" ]]; then + echo "ERROR: LIGHTSPEED_STACK_IMAGE is not set" >&2; return 1 + fi export LIGHTSPEED_STACK_IMAGE _ls_manifest="$MANIFEST_DIR/lightspeed-stack.yaml" if command -v envsubst >/dev/null 2>&1; then @@ -302,34 +304,45 @@ cmd_restart_llama_stack() { cmd_restart_port_forward() { local local_port="${LOCAL_PORT:-8080}" local remote_port="${REMOTE_PORT:-8080}" - local max_attempts=6 + local max_attempts=8 local pf_pid local pf_resource echo "Re-establishing port-forward on $local_port:$remote_port..." + # Wait for Service endpoints to be populated after pod recreate. + # Without this, the first 1–2 attempts always fail with "no endpoints available". + echo "Waiting for Service endpoints to populate..." + for ((ep_wait=1; ep_wait<=10; ep_wait++)); do + local ep_count + ep_count=$(oc get endpoints lightspeed-stack-service-svc -n "$NAMESPACE" \ + -o jsonpath='{.subsets[0].addresses}' 2>/dev/null | grep -c "ip" 2>/dev/null) || ep_count=0 + if [[ "$ep_count" -gt 0 ]]; then + echo "✓ Service endpoints ready (wait $ep_wait)" + break + fi + sleep 2 + done + for ((attempt=1; attempt<=max_attempts; attempt++)); do kill_stale_lightspeed_forward "$local_port" - # Let the kernel release LISTEN sockets after pkill (avoids immediate "address already in use") - sleep 3 + sleep 2 - # Service can lag endpoints after pod recreate; pod-direct forward is more reliable. - if [[ $attempt -le 2 ]]; then - pf_resource="svc/lightspeed-stack-service-svc" - else + # Pod-direct forward is more reliable right after recreate. + if [[ $attempt -le 3 ]]; then pf_resource="pod/lightspeed-stack-service" + else + pf_resource="svc/lightspeed-stack-service-svc" fi echo "Port-forward attempt $attempt/$max_attempts -> $pf_resource" : > /tmp/port-forward.log - # Redirect stdin from /dev/null so oc does not see a closed pipe when the parent is a short-lived subprocess. nohup oc port-forward "$pf_resource" "$local_port:$remote_port" -n "$NAMESPACE" \ /tmp/port-forward.log 2>&1 & pf_pid=$! disown "$pf_pid" 2>/dev/null || true sleep 3 - # Bind error or API error: process exits quickly — surface /tmp/port-forward.log every time if ! kill -0 "$pf_pid" 2>/dev/null; then echo "Port-forward process exited immediately:" e2e_ops_emit_port_forward_immediate_failure_diag @@ -337,7 +350,7 @@ cmd_restart_port_forward() { sleep 2 continue fi - sleep 6 + sleep 4 if verify_connectivity 15; then echo "$pf_pid" >"$E2E_LSC_PORT_FORWARD_PID_FILE" @@ -385,7 +398,7 @@ verify_llama_local_forward() { cmd_restart_llama_port_forward() { local local_port="${LOCAL_LLAMA_PORT:-8321}" local remote_port="${REMOTE_LLAMA_PORT:-8321}" - local max_attempts=6 + local max_attempts=8 local pf_pid local pf_resource local llama_pf_log="/tmp/port-forward-llama.log" @@ -394,12 +407,13 @@ cmd_restart_llama_port_forward() { for ((attempt=1; attempt<=max_attempts; attempt++)); do kill_stale_llama_forward "$local_port" - sleep 3 + sleep 2 - if [[ $attempt -le 2 ]]; then - pf_resource="svc/llama-stack-service-svc" - else + # Pod-direct is more reliable right after recreate. + if [[ $attempt -le 3 ]]; then pf_resource="pod/llama-stack-service" + else + pf_resource="svc/llama-stack-service-svc" fi echo "Llama port-forward attempt $attempt/$max_attempts -> $pf_resource" diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index fa89c45e6..1cbffeeac 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -22,6 +22,7 @@ unregister_shield, ) from tests.e2e.utils.prow_utils import ( + ensure_port_forwards_healthy, restart_lightspeed_stack_only, restore_llama_stack_pod, ) @@ -221,6 +222,15 @@ def before_scenario(context: Context, scenario: Scenario) -> None: scenario.skip("Skipped in library mode (no separate llama-stack container)") return + if is_prow_environment() and "skip-in-prow" in scenario.effective_tags: + scenario.skip( + "Skipped in Prow/Konflux (requires Docker Compose infrastructure)" + ) + return + + if is_prow_environment(): + ensure_port_forwards_healthy() + # @disable-shields: unregister shield via client.shields.delete("llama-guard"). # Only in server mode: in library mode there is no separate Llama Stack to call, # and unregistering in the test process would not affect the app's in-process instance. diff --git a/tests/e2e/features/info.feature b/tests/e2e/features/info.feature index 3d53fa2d4..beea137e8 100644 --- a/tests/e2e/features/info.feature +++ b/tests/e2e/features/info.feature @@ -15,8 +15,8 @@ Feature: Info tests Given The system is in default state When I access REST API endpoint "info" using HTTP GET method Then The status code of the response is 200 - And The body of the response has proper name Lightspeed Core Service (LCS) and version 0.5.1 - And The body of the response has llama-stack version 0.5.2 + And The body of the response has proper name Lightspeed Core Service (LCS) and version 0.5.3 + And The body of the response has llama-stack version 0.5.4 @skip-in-library-mode Scenario: Check if info endpoint reports error when llama-stack connection is not working diff --git a/tests/e2e/features/proxy.feature b/tests/e2e/features/proxy.feature index 6e1b064ec..50073a432 100644 --- a/tests/e2e/features/proxy.feature +++ b/tests/e2e/features/proxy.feature @@ -1,4 +1,5 @@ @skip-in-library-mode +@skip-in-prow Feature: Proxy and TLS networking tests for Llama Stack providers Verify that the Lightspeed Stack works correctly when Llama Stack's diff --git a/tests/e2e/features/tls.feature b/tests/e2e/features/tls.feature index a9c179aaa..e8d760ba9 100644 --- a/tests/e2e/features/tls.feature +++ b/tests/e2e/features/tls.feature @@ -1,4 +1,5 @@ @skip-in-library-mode +@skip-in-prow Feature: TLS configuration for remote inference providers Validate that Llama Stack's NetworkConfig.tls settings are applied correctly when connecting to a remote inference provider over HTTPS. diff --git a/tests/e2e/utils/prow_utils.py b/tests/e2e/utils/prow_utils.py index c5df82a1f..47487d010 100644 --- a/tests/e2e/utils/prow_utils.py +++ b/tests/e2e/utils/prow_utils.py @@ -4,9 +4,11 @@ and are only used when running tests in the Prow CI environment. """ +import http.client import os import subprocess import tempfile +import urllib.request from typing import Optional @@ -75,17 +77,57 @@ def wait_for_pod_health(pod_name: str, max_attempts: int = 12) -> None: raise +def _is_llama_stack_healthy() -> bool: + """Check whether llama-stack pod is Ready and reachable on localhost:8321. + + Returns True when both the pod is Running/Ready *and* the local port-forward + serves /v1/health. Used to skip the expensive ``restore_llama_stack_pod()`` + when the pod never went away. + """ + namespace = get_namespace() + try: + result = subprocess.run( + [ + "oc", + "get", + "pod", + "llama-stack-service", + "-n", + namespace, + "-o", + "jsonpath={.status.containerStatuses[0].ready}", + ], + capture_output=True, + text=True, + timeout=10, + check=False, + ) + if result.stdout.strip() != "true": + return False + except (FileNotFoundError, subprocess.TimeoutExpired): + return False + + try: + with urllib.request.urlopen( + "http://127.0.0.1:8321/v1/health", timeout=5 + ) as resp: + return resp.status == 200 + except Exception: # pylint: disable=broad-exception-caught + return False + + def restart_pod(container_name: str) -> None: """Restart lightspeed-stack pod in OpenShift/Prow environment. - Ensures Llama Stack is running first, since LSC fails to start if it cannot - connect to Llama Stack at startup. Llama pod logs may look unchanged (apply is a - no-op when healthy); that is expected. - - CI failures with healthy pod logs are often **localhost port-forward** contention - (pipeline forward vs hook restart), not application crashes—see e2e-ops.sh header. + Only restores Llama Stack when it is not already healthy, avoiding the + ~30 s overhead on every restart. LSC needs Llama Stack at startup, so + the check is still performed. """ - restore_llama_stack_pod() + if not _is_llama_stack_healthy(): + print("[restart_pod] Llama Stack not healthy — restoring first") + restore_llama_stack_pod() + else: + print("[restart_pod] Llama Stack already healthy — skipping restore") try: result = run_e2e_ops("restart-lightspeed", timeout=200) print(result.stdout, end="") @@ -115,6 +157,50 @@ def restore_llama_stack_pod() -> None: print("✓ Llama Stack pod restored successfully") +def ensure_port_forwards_healthy() -> None: + """Verify both LSC and Llama Stack port-forwards are alive; restart dead ones. + + Intended to be called from ``before_scenario`` so that a stale forward left + by a previous restart/cleanup does not cascade-fail unrelated scenarios. + """ + # --- Llama Stack (8321) --- + try: + with urllib.request.urlopen( + "http://127.0.0.1:8321/v1/health", timeout=5 + ) as resp: + if resp.status != 200: + raise ConnectionError("non-200") + except Exception: # pylint: disable=broad-exception-caught + print("[self-heal] Llama port-forward dead — re-establishing") + try: + result = run_e2e_ops("restart-llama-port-forward", timeout=120) + print(result.stdout, end="") + if result.returncode != 0: + print(result.stderr, end="") + print("[self-heal] WARNING: could not restore Llama port-forward") + except subprocess.TimeoutExpired: + print("[self-heal] WARNING: Llama port-forward restore timed out") + + # --- Lightspeed Stack (8080) --- + try: + conn = http.client.HTTPConnection("127.0.0.1", 8080, timeout=5) + conn.request("GET", "/readiness") + resp = conn.getresponse() + if resp.status not in (200, 401): + raise ConnectionError(f"HTTP {resp.status}") + conn.close() + except Exception: # pylint: disable=broad-exception-caught + print("[self-heal] LSC port-forward dead — re-establishing") + try: + result = run_e2e_ops("restart-port-forward", timeout=120) + print(result.stdout, end="") + if result.returncode != 0: + print(result.stderr, end="") + print("[self-heal] WARNING: could not restore LSC port-forward") + except subprocess.TimeoutExpired: + print("[self-heal] WARNING: LSC port-forward restore timed out") + + def restart_lightspeed_stack_only() -> None: """Restart lightspeed-stack pod and port-forward on Prow without restoring Llama first.