From c829fbb78d3df32701bf0cdf6febdbe8d445bd2b Mon Sep 17 00:00:00 2001 From: Wu Yi Date: Tue, 23 Jun 2026 00:56:05 +0000 Subject: [PATCH] test(feast): first-cycle e2e smoke for Feature Store GA (8.5.2) Adds a self-contained, egress-free Feast first-cycle smoke validated on g1-c1-x86 (CPU-only): create the first FeatureStore CR -> operator reconciles to Ready -> synthesize a tiny in-cluster data source -> feast apply -> materialize-incremental -> get_online_features() asserting the expected value. - e2e/cases/c13_feast_firstcycle.sh: smoke (SKIP=77 if operator/CRD/image missing), registered in e2e/run_all.sh as C13. - docs/en/feast/assets/first-cycle-e2e/: operator-install.yaml, featurestore.yaml, online-features-job.yaml, README.md (runnable solution). Uses only cluster-pullable images (feature-server:0.63.0 from build-harbor.alauda.cn, feast SDK 0.63.0). Online read returns conv_rate=0.42 for driver_id=1001. Documents that the operator must live in feast-operator-system or every FeatureStore fails with 'namespaces "feast-operator-system" not found'. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../en/feast/assets/first-cycle-e2e/README.md | 85 +++++++++++ .../assets/first-cycle-e2e/featurestore.yaml | 47 ++++++ .../first-cycle-e2e/online-features-job.yaml | 134 ++++++++++++++++++ .../first-cycle-e2e/operator-install.yaml | 35 +++++ e2e/cases/c13_feast_firstcycle.sh | 96 +++++++++++++ e2e/run_all.sh | 3 + 6 files changed, 400 insertions(+) create mode 100644 docs/en/feast/assets/first-cycle-e2e/README.md create mode 100644 docs/en/feast/assets/first-cycle-e2e/featurestore.yaml create mode 100644 docs/en/feast/assets/first-cycle-e2e/online-features-job.yaml create mode 100644 docs/en/feast/assets/first-cycle-e2e/operator-install.yaml create mode 100755 e2e/cases/c13_feast_firstcycle.sh diff --git a/docs/en/feast/assets/first-cycle-e2e/README.md b/docs/en/feast/assets/first-cycle-e2e/README.md new file mode 100644 index 00000000..35979821 --- /dev/null +++ b/docs/en/feast/assets/first-cycle-e2e/README.md @@ -0,0 +1,85 @@ +# Feast first-cycle e2e smoke (runnable solution) + +Roadmap 8.5.2 — *Feature Store GA*. This is a self-contained, egress-free smoke that +takes a brand-new cluster from "no FeatureStore exists" to a verified **online feature +read**, using only cluster-pullable images. It is the runbook behind +`e2e/cases/c13_feast_firstcycle.sh`. + +## What it proves + +1. The Feast Operator reconciles the **first** `FeatureStore` CR to `status.phase=Ready` + (it scaffolds a demo feature repo and runs `feast apply` on init). +2. The full feature lifecycle works end-to-end on the same `feature-server` image the + operator deploys: **synthesize data → `feast apply` → `feast materialize-incremental` + → `get_online_features()`**, returning the expected value. + +All data is generated in-cluster (pandas, bundled in the image). All stores are local +(file registry, file offline, sqlite online). Nothing is pulled from PyPI / docker.io / +HuggingFace. + +## Files + +| File | Purpose | +| ---- | ------- | +| `operator-install.yaml` | OLM install of the Feast Operator (Namespace + OperatorGroup + Subscription). | +| `featurestore.yaml` | Minimal all-local PVC-backed `FeatureStore` CR. | +| `online-features-job.yaml` | Single Job that runs the whole online-read lifecycle and asserts the value. | + +## Environment facts (g1-c1-x86, validated 2026-06-23) + +- CRD `featurestores.feast.dev` is present on `g1-c1-x86` and `g1-c2-arm`. +- Operator package: platform OLM catalog `cpaas-system/platform`, package `feast-operator`, + channel `alpha`, CSV `feast-operator.v0.63.0-build.20260529061714` (AllNamespaces). +- Operator image: `build-harbor.alauda.cn/mlops/feast/feast-operator:v0.63.0-build.20260529061714` +- Feature-server image: `build-harbor.alauda.cn/mlops/feast/feature-server:0.63.0` + (feast SDK 0.63.0, Python 3.12; the cluster transparently mirrors this to + `152-231-registry.alauda.cn:60070/...`). Runs as uid 1001 / gid 0, HOME `/opt/app-root/src`. + +> **Operator namespace matters.** The operator must be installed in +> **`feast-operator-system`**. When reconciling a FeatureStore it deploys a shared +> "namespace registry" into its *own* namespace, which it resolves to the upstream default +> `feast-operator-system`. Installed elsewhere, the FeatureStore is stuck `phase=Failed` +> with `namespaces "feast-operator-system" not found`, even though every sub-store reports +> Ready. + +## Run it + +```bash +CTX=g1-c1-x86-admin@g1-c1-x86 +kc() { kubectl --context "$CTX" "$@"; } + +# 0. Operator (one-time; skip if already installed in feast-operator-system) +kc apply -f operator-install.yaml +kc -n feast-operator-system rollout status deploy/feast-operator-controller-manager --timeout=300s + +# 1. First FeatureStore +kc create ns feast-e2e +kc apply -f featurestore.yaml +# wait until Ready +until [ "$(kc get featurestore feast-e2e -n feast-e2e -o jsonpath='{.status.phase}')" = Ready ]; do sleep 5; done + +# 2. Online-read lifecycle (apply -> materialize -> read) +kc apply -f online-features-job.yaml +kc -n feast-e2e wait --for=condition=complete job/feast-online-read --timeout=600s +kc -n feast-e2e logs job/feast-online-read | grep -E 'ONLINE_RESULT|SMOKE_OK' +``` + +Expected tail: + +```text +ONLINE_RESULT: {'driver_id': [1001], 'acc_rate': [0.55...], 'conv_rate': [0.42...], 'avg_daily_trips': [99]} +SMOKE_OK driver_id=1001 conv_rate=0.42... +``` + +## Clean up + +```bash +kc delete ns feast-e2e --wait=false # FeatureStore CR, PVCs, Jobs +# operator (feast-operator-system) is left installed as a platform component +``` + +## Or via the harness + +```bash +cd e2e && ./run_all.sh C13 # exits 77 (SKIP) if operator/CRD/image is missing +``` diff --git a/docs/en/feast/assets/first-cycle-e2e/featurestore.yaml b/docs/en/feast/assets/first-cycle-e2e/featurestore.yaml new file mode 100644 index 00000000..92e4543a --- /dev/null +++ b/docs/en/feast/assets/first-cycle-e2e/featurestore.yaml @@ -0,0 +1,47 @@ +# Minimal first-cycle FeatureStore — all-local, PVC-backed (file offline/online/registry). +# The operator runs `feast apply` on init (runFeastApplyOnInit) against the demo repo it +# scaffolds via `feast init`, and creates a (suspended) materialize CronJob. +# +# Reconciles to status.phase=Ready ONLY when the operator is installed in +# `feast-operator-system` (see operator-install.yaml). PVC `path` values must be a bare +# filename (no slashes) — the operator places them under the matching `mountPath`. +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: feast-e2e + namespace: feast-e2e +spec: + feastProject: feast_e2e + services: + offlineStore: + persistence: + file: + type: duckdb + pvc: + create: + resources: + requests: + storage: 2Gi + mountPath: /data/offline + onlineStore: + persistence: + file: + path: online_store.db + pvc: + create: + resources: + requests: + storage: 2Gi + mountPath: /data/online + registry: + local: + persistence: + file: + path: registry.db + pvc: + create: + resources: + requests: + storage: 2Gi + mountPath: /data/registry + server: {} diff --git a/docs/en/feast/assets/first-cycle-e2e/online-features-job.yaml b/docs/en/feast/assets/first-cycle-e2e/online-features-job.yaml new file mode 100644 index 00000000..7cedf561 --- /dev/null +++ b/docs/en/feast/assets/first-cycle-e2e/online-features-job.yaml @@ -0,0 +1,134 @@ +# Self-contained Feast online-read smoke, as a single Kubernetes Job. +# +# Runs the full first-cycle lifecycle in ONE pod using the same feature-server image +# the Feast Operator deploys: synthesize a tiny data source -> `feast apply` -> +# `feast materialize-incremental` -> `store.get_online_features()` and assert the value. +# +# No external egress: the data source is generated in-cluster with pandas (bundled in +# the image). Stores are all local (file registry, file offline, sqlite online) inside +# the pod's writable HOME, so the read does not depend on cross-pod networking. +# +# The synthetic row for driver_id=1001 has conv_rate=0.42; the Job asserts the online +# read returns exactly that, so the Job's success IS the verification. +apiVersion: batch/v1 +kind: Job +metadata: + name: feast-online-read + namespace: feast-e2e + labels: + e2e.alauda.io/feast-firstcycle: "true" +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 1800 + template: + metadata: + labels: + e2e.alauda.io/feast-firstcycle: "true" + spec: + restartPolicy: Never + securityContext: + runAsNonRoot: true + seccompProfile: { type: RuntimeDefault } + containers: + - name: feast + image: build-harbor.alauda.cn/mlops/feast/feature-server:0.63.0 + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: { drop: ["ALL"] } + command: ["bash", "-lc"] + args: + - | + set -euo pipefail + REPO="${HOME}/feast_e2e_repo" + rm -rf "${REPO}"; mkdir -p "${REPO}/data"; cd "${REPO}" + + echo "=== [1/5] synthesize a tiny in-cluster data source ===" + python - <<'PY' + import pandas as pd + from datetime import datetime, timedelta, timezone + now = datetime.now(timezone.utc) + ts = now - timedelta(hours=2) + df = pd.DataFrame({ + "driver_id": [1001, 1002], + "event_timestamp": [ts, ts], + "created": [ts, ts], + "conv_rate": [0.42, 0.13], + "acc_rate": [0.55, 0.20], + "avg_daily_trips": [99, 42], + }) + df.to_parquet("data/driver_stats.parquet") + print(df.to_string(index=False)) + PY + + echo "=== [2/5] write feature_store.yaml + feature definitions ===" + cat > feature_store.yaml <<'YAML' + project: feast_e2e + provider: local + registry: data/registry.db + online_store: + type: sqlite + path: data/online_store.db + offline_store: + type: file + entity_key_serialization_version: 3 + YAML + + cat > definitions.py <<'PY' + from datetime import timedelta + from feast import Entity, FeatureView, Field, FileSource, FeatureService + from feast.types import Float32, Int64 + from feast.value_type import ValueType + + driver = Entity(name="driver", join_keys=["driver_id"], value_type=ValueType.INT64) + + driver_stats_source = FileSource( + name="driver_stats_source", + path="data/driver_stats.parquet", + timestamp_field="event_timestamp", + created_timestamp_column="created", + ) + + driver_hourly_stats = FeatureView( + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=3650), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_source, + ) + + driver_activity_v1 = FeatureService( + name="driver_activity_v1", + features=[driver_hourly_stats], + ) + PY + + echo "=== [3/5] feast apply ===" + feast apply + + echo "=== [4/5] feast materialize-incremental ===" + feast materialize-incremental "$(date -u +%Y-%m-%dT%H:%M:%S)" + + echo "=== [5/5] online read + assert ===" + python - <<'PY' + from feast import FeatureStore + store = FeatureStore(repo_path=".") + resp = store.get_online_features( + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + ], + entity_rows=[{"driver_id": 1001}], + ).to_dict() + print("ONLINE_RESULT:", resp) + conv = resp["conv_rate"][0] + assert conv is not None, "online conv_rate is None (materialize did not populate online store)" + assert abs(conv - 0.42) < 1e-3, f"unexpected conv_rate {conv}, expected 0.42" + print(f"SMOKE_OK driver_id=1001 conv_rate={conv}") + PY diff --git a/docs/en/feast/assets/first-cycle-e2e/operator-install.yaml b/docs/en/feast/assets/first-cycle-e2e/operator-install.yaml new file mode 100644 index 00000000..1279836a --- /dev/null +++ b/docs/en/feast/assets/first-cycle-e2e/operator-install.yaml @@ -0,0 +1,35 @@ +# Feast Operator install (OLM) — AllNamespaces mode. +# +# IMPORTANT: the operator MUST be installed in the namespace `feast-operator-system`. +# When it reconciles a FeatureStore it deploys a shared "namespace registry" into its +# own namespace, which it resolves to `feast-operator-system` (the upstream default). +# Installing it anywhere else leaves the FeatureStore stuck in phase=Failed with +# `namespaces "feast-operator-system" not found`, even though every sub-store is Ready. +# +# Source catalog: the platform OLM catalog (cpaas-system/platform), package `feast-operator`, +# channel `alpha`. Operator image (cluster-pullable): build-harbor.alauda.cn/mlops/feast/feast-operator +# Feature-server image it deploys: build-harbor.alauda.cn/mlops/feast/feature-server:0.63.0 +--- +apiVersion: v1 +kind: Namespace +metadata: + name: feast-operator-system +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: feast-operator + namespace: feast-operator-system +spec: {} # empty spec == AllNamespaces watch (required by this operator) +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: feast-operator + namespace: feast-operator-system +spec: + channel: alpha + name: feast-operator + source: platform + sourceNamespace: cpaas-system + installPlanApproval: Automatic diff --git a/e2e/cases/c13_feast_firstcycle.sh b/e2e/cases/c13_feast_firstcycle.sh new file mode 100755 index 00000000..73f33d3c --- /dev/null +++ b/e2e/cases/c13_feast_firstcycle.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# C13 — Feast first-cycle e2e SMOKE (roadmap 8.5.2 Feature Store GA). +# +# Proves a brand-new Feast deployment can complete one full feature lifecycle on the +# GPU cluster (CPU-only path), with NO external egress: +# 1. Operator preflight: CRD + controller present (else SKIP 77). +# 2. Create the FIRST FeatureStore CR -> operator reconciles to status.phase=Ready. +# (operator runs `feast apply` on init against the scaffolded demo repo.) +# 3. Self-contained online-read Job using the SAME feature-server image the operator +# deploys: synthesize a tiny data source in-cluster -> feast apply -> +# feast materialize-incremental -> store.get_online_features() -> assert conv_rate. +# +# The data source is generated in-pod with pandas (bundled in the image); all stores are +# local (file registry, file offline, sqlite online), so the read needs no cross-pod +# networking and no PyPI/docker.io/HF access. The Job asserts driver_id=1001 reads back +# conv_rate=0.42, so a green Job == a verified online read. +# +# Exit codes: 0 pass / 1 fail / 77 skip (operator/image/CRD missing). +set -uo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +source "${HERE}/../lib.sh" + +CTX="${GPU_CONTEXT}" +NS="${FEAST_E2E_NS:-feast-e2e}" +OP_NS="${FEAST_OPERATOR_NS:-feast-operator-system}" +ASSETS="${E2E_ROOT}/../docs/en/feast/assets/first-cycle-e2e" +kc() { kubectl --context "${CTX}" "$@"; } + +# ---- preflight (skip, don't fail, when the platform piece is absent) ---------- +if ! kc get crd featurestores.feast.dev >/dev/null 2>&1; then + log "C13: CRD featurestores.feast.dev missing — operator not installed; SKIP" + exit 77 +fi +if ! kc -n "${OP_NS}" get deploy feast-operator-controller-manager >/dev/null 2>&1; then + log "C13: feast operator controller not found in ns/${OP_NS}; SKIP" + log "C13: install it with: kubectl apply -f ${ASSETS}/operator-install.yaml" + exit 77 +fi +OP_READY="$(kc -n "${OP_NS}" get deploy feast-operator-controller-manager -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true)" +if [ "${OP_READY:-0}" -lt 1 ] 2>/dev/null; then + log "C13: feast operator controller present but not Ready (readyReplicas=${OP_READY:-0}); SKIP" + exit 77 +fi +# Resolve the feature-server image the operator actually uses, so the smoke pins the +# exact same (cluster-pullable) image instead of a hard-coded one. +FEAST_IMAGE="$(kc -n "${OP_NS}" get deploy feast-operator-controller-manager \ + -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="RELATED_IMAGE_FEATURE_SERVER")]}{.value}{end}' 2>/dev/null || true)" +[ -z "${FEAST_IMAGE}" ] && FEAST_IMAGE="build-harbor.alauda.cn/mlops/feast/feature-server:0.63.0" +log "C13: operator ready; feature-server image=${FEAST_IMAGE}" + +cleanup() { kc delete ns "${NS}" --ignore-not-found --wait=false >/dev/null 2>&1 || true; } +trap cleanup EXIT + +# ---- 1. fresh namespace + first FeatureStore CR ------------------------------- +kc create ns "${NS}" >/dev/null 2>&1 || true +log "C13: applying FeatureStore CR to ns/${NS}" +kc apply -f "${ASSETS}/featurestore.yaml" >/dev/null + +log "C13: waiting for FeatureStore phase=Ready (<=10m)" +phase="$(wait_for_status kc featurestore feast-e2e "${NS}" '{.status.phase}' Ready Failed 600)" +if [ "${phase}" != "Ready" ]; then + log "C13: FeatureStore did not reach Ready (phase=${phase}); conditions:" + kc get featurestore feast-e2e -n "${NS}" -o jsonpath='{range .status.conditions[*]}{.type}={.status}|{.message}{"\n"}{end}' || true + exit 1 +fi +log "C13: FeatureStore Ready" + +# ---- 2. self-contained online-read Job --------------------------------------- +log "C13: applying online-read Job (image pinned to operator's feature-server)" +sed "s#image: build-harbor.alauda.cn/mlops/feast/feature-server:0.63.0#image: ${FEAST_IMAGE}#" \ + "${ASSETS}/online-features-job.yaml" | kc apply -f - >/dev/null + +log "C13: waiting for online-read Job to finish (<=10m)" +deadline=$((SECONDS + 600)); jobstate="" +while [ "${SECONDS}" -lt "${deadline}" ]; do + s="$(kc get job feast-online-read -n "${NS}" -o jsonpath='{.status.succeeded}' 2>/dev/null || true)" + f="$(kc get job feast-online-read -n "${NS}" -o jsonpath='{.status.failed}' 2>/dev/null || true)" + [ "${s:-0}" = "1" ] && { jobstate="succeeded"; break; } + [ "${f:-0}" = "1" ] && { jobstate="failed"; break; } + sleep 10 +done + +log "C13: ===== online-read Job logs =====" +kc logs job/feast-online-read -n "${NS}" --tail=80 || true +log "C13: ===== end logs =====" + +if [ "${jobstate}" != "succeeded" ]; then + log "C13: online-read Job did not succeed (state=${jobstate:-timeout})" + exit 1 +fi +if ! kc logs job/feast-online-read -n "${NS}" 2>/dev/null | grep -q "SMOKE_OK"; then + log "C13: Job succeeded but SMOKE_OK marker missing" + exit 1 +fi +log "C13: PASS — FeatureStore Ready and online read returned the expected conv_rate" +exit 0 diff --git a/e2e/run_all.sh b/e2e/run_all.sh index dadb8b49..89c2e671 100755 --- a/e2e/run_all.sh +++ b/e2e/run_all.sh @@ -29,6 +29,9 @@ CASES=( # C12 needs Kueue installed; it skips with rc=77 if the kueue.x-k8s.io API # group is missing. See preemptible-trainjobs-with-kueue.mdx. "C12:GPU:cases/c12_kueue_preemption.sh" + # C13 — Feast first-cycle (roadmap 8.5.2). Skips rc=77 if the feast operator / + # CRD / feature-server image is missing. See docs/en/feast/assets/first-cycle-e2e. + "C13:GPU:cases/c13_feast_firstcycle.sh" ) want=( "$@" )