feat: migrate to Crossplane v2 and upgrade Backstage to 1.54.4 with Crossplane/Kyverno/Argo CD plugins

.bootstrap/argocd/manifests/argocd-cm-patch.yaml

@@ -23,8 +23,13 @@ data:
           return false
         end
 
+        -- Crossplane v2 introduced ClusterProviderConfig (group aws.m.upbound.io)
+        -- as the cluster scoped config that namespaced managed resources share.
+        -- It reports only `status.users`, exactly like ProviderConfig, so it has
+        -- to be listed here too -- otherwise it never leaves "Progressing".
         local has_no_status = {
           "ProviderConfig",
+          "ClusterProviderConfig",
           "ProviderConfigUsage"
         }
 
@@ -35,7 +40,7 @@ data:
         end
 
         if obj.status == nil or next(obj.status) == nil or obj.status.conditions == nil then
-          if obj.kind == "ProviderConfig" and obj.status.users ~= nil then
+          if contains(has_no_status, obj.kind) and obj.status.users ~= nil then
             health_status.status = "Healthy"
             health_status.message = "Resource is in use."
             return health_status
@@ -47,3 +52,4 @@ data:
       - "*"
       kinds:
       - ProviderConfigUsage
+      - ClusterProviderConfigUsage

.bootstrap/argocd/up.sh

@@ -22,8 +22,14 @@ helm repo add argo https://argoproj.github.io/argo-helm 2>/dev/null || true
 helm repo update
 
 echo "Installing or upgrading Argo CD..."
+# --force-conflicts because this script patches argocd-cm with kubectl a few
+# lines below. Helm 4 applies server-side, so that patch's field manager owns
+# .data.resource.exclusions, and every later run fails with "conflict with
+# \"kubectl-patch\"" before anything else can happen. The patch is re-applied
+# immediately afterwards, so Helm reclaiming the field first is harmless.
 helm upgrade --install argocd \
     --namespace "$NS" \
+    --force-conflicts \
     --create-namespace argo/argo-cd
 
 echo "Waiting for Argo CD deployment to be ready..."
@@ -38,22 +44,37 @@ kubectl patch svc argocd-server -n $NS --patch-file $MANIFESTS_DIR/service.yaml
 kubectl rollout restart deployment argocd-server -n $NS
 kubectl rollout status deployment argocd-server -n $NS
 
-# Start port-forward only if not already active
-if ! lsof -i TCP:$PORT >/dev/null 2>&1; then
-    if kubectl get svc/argocd-server -n "$NS" >/dev/null 2>&1; then
-        echo "Starting port-forward for Argo CD on port $PORT..."
-        nohup kubectl --namespace "$NS" port-forward svc/argocd-server $PORT:443 >/dev/null 2>&1 &
-    else
+# Probe the port rather than just checking that something holds it. The
+# `rollout restart` above kills any existing port-forward along with its pod, but
+# the dying process still owns the socket for a moment -- so an `lsof` check sees
+# the port as busy, skips starting a replacement, and the login below then fails
+# against a forward that is already gone.
+if ! curl -sk --max-time 3 "https://localhost:$PORT/" >/dev/null 2>&1; then
+    if ! kubectl get svc/argocd-server -n "$NS" >/dev/null 2>&1; then
         echo "Argo CD service not found. Skipping port-forward."
         exit 1
     fi
+
+    # Only one process can listen on the port, and it just failed the probe.
+    pkill -f "port-forward svc/argocd-server" 2>/dev/null || true
+
+    echo "Starting port-forward for Argo CD on port $PORT..."
+    nohup kubectl --namespace "$NS" port-forward svc/argocd-server $PORT:443 >/dev/null 2>&1 &
+
+    # Wait for it to actually accept connections instead of guessing with sleep.
+    for _ in $(seq 1 30); do
+        curl -sk --max-time 2 "https://localhost:$PORT/" >/dev/null 2>&1 && break
+        sleep 1
+    done
+
+    if ! curl -sk --max-time 3 "https://localhost:$PORT/" >/dev/null 2>&1; then
+        echo "Port-forward on $PORT did not become ready."
+        exit 1
+    fi
 else
-    echo "Port $PORT is already in use. Assuming port-forward is running."
+    echo "Port-forward on $PORT is already serving."
 fi
 
-# Wait briefly for port-forward to establish
-sleep 5
-
 # Get the initial admin password from the secret
 INITIAL_PASSWORD=$(kubectl -n "$NS" get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d)
 

.bootstrap/backstage/backstage-secrets.template.yaml

@@ -0,0 +1,16 @@
+# Rendered by up.sh, which substitutes <placeholder> with the base64 of
+# GITHUB_TOKEN from .env, then pipes the result to kubectl.
+#
+# This file sits outside .bootstrap/backstage/manifests on purpose. That
+# directory is the source path of the `backstage-app` Argo CD Application, and
+# `<placeholder>` is not valid base64 -- so every sync failed with "illegal
+# base64 data at input byte 0", leaving the app permanently OutOfSync while
+# trying to overwrite the real token with a broken value.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: backstage-secrets
+  namespace: backstage-system
+type: Opaque
+data:
+  GITHUB_TOKEN: <placeholder>

.bootstrap/backstage/manifests/secrets.yaml

@@ -1,12 +1,7 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: backstage-secrets
-  namespace: backstage-system
-type: Opaque
-data:
-  GITHUB_TOKEN: <placeholder>
----
+# backstage-secrets is deliberately NOT here. This directory is the source path
+# for the `backstage-app` Argo CD Application, and that secret holds a real
+# GitHub token rendered from .env at bootstrap time -- it cannot live in Git.
+# Its shape is documented in ../backstage-secrets.template.yaml.
 apiVersion: v1
 kind: Secret
 metadata:

.bootstrap/backstage/up.sh

@@ -30,20 +30,47 @@ fi
 echo "Checking if Backstage image '$IMAGE' already exists..."
 if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then
     echo "🔨 Building Backstage image $IMAGE..."
-    cd ./backstage
-    yarn install
-    yarn build:all
-    yarn build-image --tag "$IMAGE" --no-cache
-    cd ..
+    # Multi-stage build: yarn install and the repo build run inside Docker, so
+    # the host needs no Node toolchain. The build compiles native modules
+    # (isolated-vm, better-sqlite3) that only work on the Node major versions
+    # listed under `engines` in backstage/package.json, which is easy to get
+    # wrong on a host with a newer Node on PATH.
+    DOCKER_BUILDKIT=1 docker build ./backstage \
+        -f ./backstage/packages/backend/Dockerfile \
+        --tag "$IMAGE"
 else
     echo "✅ Docker image $IMAGE already exists. Skipping build."
 fi
 
 kind load docker-image "$IMAGE" --name "$CLUSTER_NAME"
 
-export $(cat .env | xargs) &&
-    sed "s|<placeholder>|$(echo "$GITHUB_TOKEN" | base64)|" $BASE_DIR/manifests/secrets.yaml |
-    kubectl apply -n $NS -f -
+# `export $(cat .env | xargs)` splits a trailing `# comment` -- or any value
+# containing spaces -- into bare words that export rejects. Because the apply was
+# chained onto it with `&&`, that failure silently skipped creating the secret
+# rather than stopping the script. Sourcing under `set -a` applies normal shell
+# parsing, so comments and quoting behave.
+if [[ ! -f .env ]]; then
+    echo "❌ .env not found in the repo root. It must define GITHUB_TOKEN."
+    exit 1
+fi
+
+set -a
+# shellcheck source=/dev/null
+source ./.env
+set +a
+
+if [[ -z "${GITHUB_TOKEN:-}" ]]; then
+    echo "❌ GITHUB_TOKEN is not set in .env"
+    exit 1
+fi
+
+# printf rather than echo: echo appends a newline, which would be baked into the
+# base64 and give Backstage a token with a trailing \n. tr strips the line wrap
+# that GNU base64 adds for inputs over 76 chars (macOS base64 does not wrap).
+GITHUB_TOKEN_B64="$(printf '%s' "$GITHUB_TOKEN" | base64 | tr -d '\n')"
+
+sed "s|<placeholder>|$GITHUB_TOKEN_B64|" "$BASE_DIR/backstage-secrets.template.yaml" |
+    kubectl apply -n "$NS" -f -
 
 # Wait for postgres deployment to be ready
 echo "Waiting for postgres deployment to be ready..."

.bootstrap/crossplane/up.sh

@@ -13,7 +13,10 @@ fi
 NS=crossplane-system
 BASE_DIR=./crossplane
 MANIFESTS_DIR="$(dirname "$0")/manifests"
-REQUIRED_PROVIDERS=("provider-kubernetes" "provider-aws-sqs")
+# provider-kubernetes was dropped in the v2 migration. provider-family-aws is
+# pulled in as a dependency of provider-aws-sqs and supplies the
+# ClusterProviderConfig CRD the composition references, so wait for it too.
+REQUIRED_PROVIDERS=("provider-family-aws" "provider-aws-sqs")
 TIMEOUT=600
 INTERVAL=5
 ELAPSED=0

.bootstrap/kyverno/up.sh

@@ -24,7 +24,11 @@ kubectl wait deployment/kyverno-admission-controller \
 
 echo "Ensuring all Kyverno pods are ready..."
 while true; do
-    READY=$(kubectl get pods -n "$NS" -o jsonpath='{.items[*].status.containerStatuses[*].ready}' | tr " " "\n" | grep -c false || true)
+    # Restrict to Running pods. Kyverno ships a `kyverno-migrate-resources` Job
+    # whose pod stays Succeeded with its container reporting ready=false forever;
+    # counting every pod in the namespace therefore never reaches zero once that
+    # job has completed, and the wait times out even though Kyverno is healthy.
+    READY=$(kubectl get pods -n "$NS" --field-selector=status.phase=Running -o jsonpath='{.items[*].status.containerStatuses[*].ready}' | tr " " "\n" | grep -c false || true)
     if [[ "$READY" -eq 0 ]]; then
         break
     fi

.github/workflows/validate-claims.yaml → .github/workflows/validate-xrs.yaml

@@ -1,16 +1,16 @@
-name: Validate Crossplane Claims
+name: Validate Crossplane XRs
 
 on:
   pull_request:
     paths:
-      - "crossplane/claims/**/*.yaml"
-      - "crossplane/claims/**/*.yml"
+      - "crossplane/xrs/**/*.yaml"
+      - "crossplane/xrs/**/*.yml"
       - "kyverno/**/*.yaml"
       - "kyverno/**/*.yml"
 
 jobs:
-  validate-claims:
-    name: Validate Claims YAML with Kyverno
+  validate-xrs:
+    name: Validate XR YAML with Kyverno
     runs-on: ubuntu-latest
 
     steps:
@@ -25,14 +25,14 @@ jobs:
       - name: Check install
         run: kyverno version
 
-      - name: Run Kyverno policy checks on claims
+      - name: Run Kyverno policy checks on XRs
         run: |
           echo "## 🛡️ Kyverno Policy Validation Results" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
           echo '```' >> $GITHUB_STEP_SUMMARY
 
           set +e
-          kyverno apply ./kyverno --resource ./crossplane/claims 2>&1 | tee result.txt
+          kyverno apply ./kyverno --resource ./crossplane/xrs 2>&1 | tee result.txt
           KYVERNO_EXIT_CODE=${PIPESTATUS[0]}
           set -e
 

.gitignore

@@ -25,3 +25,6 @@ Thumbs.db
 
 .env*
 !.env.example
+
+# Personal Claude Code settings (permissions etc.) - not for the shared repo
+.claude/settings.local.json