From 9024cf4e2f6d4d38667a10e79801569c4c8a3e62 Mon Sep 17 00:00:00 2001 From: "S, Gurunath" Date: Thu, 30 Jul 2026 23:59:04 +0530 Subject: [PATCH] refactor: manage Keycloak setup credentials via Kubernetes Secrets Move the Keycloak realm setup Jobs off inline credential templating and onto standard Kubernetes Secret handling. Previously the admin user, password and client id were rendered directly into the Job container args, and the resulting OIDC client secret was passed back to Ansible by parsing it out of the Job's stdout. This couples the deployment to log formatting and stores configuration values in places that are awkward to rotate or manage. Changes: - Store admin user, password and client id in a Kubernetes Secret (keycloak-admin-credentials-env) and inject them into the Job with envFrom, so the Job spec no longer carries the values. The readiness check reads them from the environment and uses --data-urlencode for correct encoding. - keycloak-realmcreation.sh now writes the client secret to a Kubernetes Secret (keycloak-client-secret) through the API server using the pod ServiceAccount, instead of emitting it on stdout. Positional arguments are still accepted as a fallback so standalone and documented usage is unchanged. - Playbooks read the client secret with k8s_info. Retrieval uses failed_when: false to preserve the previous non-fatal behaviour when the secret is unavailable. - Add a keycloak-realm-setup ServiceAccount with a namespaced Role/RoleBinding scoped to get/create/update/patch on secrets, and set serviceAccountName on both Jobs. - Mark credential-handling tasks no_log: true to keep values out of Ansible output. No configuration changes are required: the same inference-config.cfg variables are consumed, and client_secret continues to be passed to the model Helm releases as before. Signed-off-by: S, Gurunath --- core/playbooks/deploy-inference-models.yml | 89 +++++++++++++--- core/playbooks/deploy-keycloak-tls-cert.yml | 111 +++++++++++++++----- core/scripts/keycloak-realmcreation.sh | 73 +++++++++++-- 3 files changed, 223 insertions(+), 50 deletions(-) diff --git a/core/playbooks/deploy-inference-models.yml b/core/playbooks/deploy-inference-models.yml index ee370268..b916afef 100644 --- a/core/playbooks/deploy-inference-models.yml +++ b/core/playbooks/deploy-inference-models.yml @@ -465,6 +465,56 @@ keycloak-realmcreation.sh: "{{ lookup('file', remote_home_dir + '/keycloak-realmcreation.sh') }}" run_once: true + - name: Create Secret with Keycloak admin credentials for the setup Job + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Secret + metadata: + name: keycloak-admin-credentials-env + namespace: default + type: Opaque + stringData: + KEYCLOAK_ADMIN_USER: "{{ keycloak_admin_user }}" + KEYCLOAK_ADMIN_PASSWORD: "{{ keycloak_admin_password }}" + KEYCLOAK_CLIENT_ID: "{{ keycloak_client_id }}" + run_once: true + no_log: true + + - name: Create ServiceAccount and RBAC for Keycloak realm setup Job + kubernetes.core.k8s: + state: present + definition: + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: keycloak-realm-setup + namespace: default + - apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: keycloak-realm-setup-secret-writer + namespace: default + rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update", "patch"] + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: keycloak-realm-setup-secret-writer + namespace: default + subjects: + - kind: ServiceAccount + name: keycloak-realm-setup + namespace: default + roleRef: + kind: Role + name: keycloak-realm-setup-secret-writer + apiGroup: rbac.authorization.k8s.io + run_once: true + - name: Create Keycloak realm setup Job kubernetes.core.k8s: state: present @@ -482,6 +532,7 @@ name: keycloak-realm-setup spec: restartPolicy: Never + serviceAccountName: keycloak-realm-setup containers: - name: keycloak-setup image: ubuntu:22.04 @@ -496,6 +547,9 @@ {'name': 'NO_PROXY', 'value': env_proxy.no_proxy | default('')} ] if (env_proxy is defined and env_proxy.http_proxy | default('') != '') else [] }} + envFrom: + - secretRef: + name: keycloak-admin-credentials-env command: ["/bin/bash", "-c"] args: - | @@ -522,8 +576,8 @@ HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \ -X POST \ -d "client_id=admin-cli" \ - -d "username={{ keycloak_admin_user }}" \ - -d "password={{ keycloak_admin_password }}" \ + --data-urlencode "username=${KEYCLOAK_ADMIN_USER}" \ + --data-urlencode "password=${KEYCLOAK_ADMIN_PASSWORD}" \ -d "grant_type=password" \ http://keycloak.default.svc.cluster.local:80/realms/master/protocol/openid-connect/token) @@ -550,11 +604,7 @@ # Run the realm creation script echo "Running Keycloak realm creation script..." - /tmp/keycloak-realmcreation.sh \ - keycloak.default.svc.cluster.local:80 \ - "{{ keycloak_admin_user }}" \ - "{{ keycloak_admin_password }}" \ - "{{ keycloak_client_id }}" + /tmp/keycloak-realmcreation.sh keycloak.default.svc.cluster.local:80 volumeMounts: - name: scripts mountPath: /scripts @@ -586,20 +636,33 @@ delay: 10 run_once: true - - name: Get Keycloak realm setup Job logs - command: kubectl logs job/keycloak-realm-setup-models -n default - register: script_output + - name: Read Keycloak client secret from Kubernetes Secret + kubernetes.core.k8s_info: + kind: Secret + namespace: default + name: keycloak-client-secret + register: client_secret_obj + until: + - client_secret_obj.resources | length > 0 + - client_secret_obj.resources[0].data['client-secret'] is defined + retries: 12 + delay: 5 + failed_when: false run_once: true + no_log: true - name: Set Keycloak client fact set_fact: - client_secret: "{{ script_output.stdout | regex_search('Client secret: (.*)') | join('') | regex_replace('^Client secret: ') }}" - when: script_output.stdout is search('Client secret:') + client_secret: "{{ client_secret_obj.resources[0].data['client-secret'] | b64decode }}" + when: + - client_secret_obj.resources | default([]) | length > 0 + - client_secret_obj.resources[0].data['client-secret'] is defined run_once: true + no_log: true - name: Warning when client secret not found debug: - msg: "WARNING: Client secret was not found in the Keycloak setup job output. Please check the job logs manually." + msg: "WARNING: Client secret was not found in the keycloak-client-secret Secret. Please check the job logs manually." when: client_secret is not defined run_once: true run_once: true diff --git a/core/playbooks/deploy-keycloak-tls-cert.yml b/core/playbooks/deploy-keycloak-tls-cert.yml index 65836378..37601bbf 100644 --- a/core/playbooks/deploy-keycloak-tls-cert.yml +++ b/core/playbooks/deploy-keycloak-tls-cert.yml @@ -492,6 +492,58 @@ run_once: true when: deploy_keycloak == "yes" + - name: Create Secret with Keycloak admin credentials for the setup Job + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Secret + metadata: + name: keycloak-admin-credentials-env + namespace: default + type: Opaque + stringData: + KEYCLOAK_ADMIN_USER: "{{ keycloak_admin_user }}" + KEYCLOAK_ADMIN_PASSWORD: "{{ keycloak_admin_password }}" + KEYCLOAK_CLIENT_ID: "{{ keycloak_client_id }}" + run_once: true + no_log: true + when: deploy_keycloak == "yes" + + - name: Create ServiceAccount and RBAC for Keycloak realm setup Job + kubernetes.core.k8s: + state: present + definition: + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: keycloak-realm-setup + namespace: default + - apiVersion: rbac.authorization.k8s.io/v1 + kind: Role + metadata: + name: keycloak-realm-setup-secret-writer + namespace: default + rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update", "patch"] + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: keycloak-realm-setup-secret-writer + namespace: default + subjects: + - kind: ServiceAccount + name: keycloak-realm-setup + namespace: default + roleRef: + kind: Role + name: keycloak-realm-setup-secret-writer + apiGroup: rbac.authorization.k8s.io + run_once: true + when: deploy_keycloak == "yes" + - name: Create Keycloak realm setup Job kubernetes.core.k8s: state: present @@ -509,6 +561,7 @@ name: keycloak-realm-setup spec: restartPolicy: Never + serviceAccountName: keycloak-realm-setup containers: - name: keycloak-setup image: ubuntu:22.04 @@ -523,39 +576,42 @@ {'name': 'NO_PROXY', 'value': env_proxy.no_proxy | default('')} ] if (env_proxy is defined and env_proxy.http_proxy | default('') != '') else [] }} + envFrom: + - secretRef: + name: keycloak-admin-credentials-env command: ["/bin/bash", "-c"] args: - | set -e - + # Install dependencies apt-get update -qq apt-get install -y -qq curl jq - + # Copy script to writable location cp /scripts/keycloak-realmcreation.sh /tmp/keycloak-realmcreation.sh chmod +x /tmp/keycloak-realmcreation.sh - + # Retry mechanism for Keycloak availability MAX_RETRIES=30 RETRY_DELAY=10 RETRY_COUNT=0 - + echo "Waiting for Keycloak to be ready..." while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do # Test token endpoint HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ -d "client_id=admin-cli" \ - -d "username={{ keycloak_admin_user }}" \ - -d "password={{ keycloak_admin_password }}" \ + --data-urlencode "username=${KEYCLOAK_ADMIN_USER}" \ + --data-urlencode "password=${KEYCLOAK_ADMIN_PASSWORD}" \ -d "grant_type=password" \ http://keycloak.default.svc.cluster.local:80/realms/master/protocol/openid-connect/token || echo "000") - + if [ "$HTTP_CODE" = "200" ]; then echo "Keycloak is ready (attempt $((RETRY_COUNT + 1))/$MAX_RETRIES)" break fi - + RETRY_COUNT=$((RETRY_COUNT + 1)) if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then echo "Keycloak not ready yet (attempt $RETRY_COUNT/$MAX_RETRIES, HTTP: $HTTP_CODE). Retrying in ${RETRY_DELAY}s..." @@ -565,18 +621,14 @@ exit 1 fi done - + # Additional stability wait echo "Waiting 15 seconds for Keycloak to stabilize..." sleep 15 - + # Run the script with internal service name echo "Running realm creation script..." - /tmp/keycloak-realmcreation.sh \ - keycloak.default.svc.cluster.local:80 \ - "{{ keycloak_admin_user }}" \ - "{{ keycloak_admin_password }}" \ - "{{ keycloak_client_id }}" + /tmp/keycloak-realmcreation.sh keycloak.default.svc.cluster.local:80 volumeMounts: - name: scripts mountPath: /scripts @@ -610,25 +662,32 @@ run_once: true when: deploy_keycloak == "yes" - - name: Get Keycloak realm setup Job logs - command: kubectl logs job/keycloak-realm-setup -n default - register: script_output + - name: Read Keycloak client secret from Kubernetes Secret + kubernetes.core.k8s_info: + kind: Secret + namespace: default + name: keycloak-client-secret + register: client_secret_obj + until: + - client_secret_obj.resources | length > 0 + - client_secret_obj.resources[0].data['client-secret'] is defined + retries: 12 + delay: 5 + failed_when: false run_once: true - environment: - http_proxy: "" - https_proxy: "" - no_proxy: "" + no_log: true when: deploy_keycloak == "yes" - name: Set client_secret fact set_fact: - client_secret: "{{ script_output.stdout | regex_search('Client secret: (.*)') | join('') | regex_replace('^Client secret: ') }}" + client_secret: "{{ client_secret_obj.resources[0].data['client-secret'] | b64decode }}" when: - - script_output is defined - - script_output.stdout is defined - - script_output.stdout is search('Client secret:') + - client_secret_obj is defined + - client_secret_obj.resources | default([]) | length > 0 + - client_secret_obj.resources[0].data['client-secret'] is defined - deploy_keycloak == "yes" run_once: true + no_log: true - name: Verify Keycloak ApisixRoute is synced shell: | diff --git a/core/scripts/keycloak-realmcreation.sh b/core/scripts/keycloak-realmcreation.sh index 87ed0b67..8cf677b7 100644 --- a/core/scripts/keycloak-realmcreation.sh +++ b/core/scripts/keycloak-realmcreation.sh @@ -11,15 +11,18 @@ # This script automates the creation and configuration of a Keycloak client. -# +# # Usage: -# ./keycloak-realmcreation.sh +# ./keycloak-realmcreation.sh # # Arguments: -# KEYCLOAK_URL - The base URL of the Keycloak server. -# USERNAME - The username for Keycloak admin login. -# PASSWORD - The password for Keycloak admin login. -# CLIENT_ID - The client ID to be created in Keycloak. +# KEYCLOAK_HOST - The host[:port] of the Keycloak server (no scheme). +# +# Credentials are read from the environment (positional args 2-4 are accepted +# as a fallback): +# KEYCLOAK_ADMIN_USER - The username for Keycloak admin login. +# KEYCLOAK_ADMIN_PASSWORD - The password for Keycloak admin login. +# KEYCLOAK_CLIENT_ID - The client ID to be created in Keycloak. # # Steps performed by the script: # 1. Logs in to Keycloak and retrieves an access token. @@ -28,7 +31,8 @@ # 3. Retrieves the UUID of the created client. # 4. Enables client authentication capability with service account roles checked. # 5. Updates the realm settings to set the access token lifespan to 15 minutes. -# 6. Retrieves and displays the client secret. +# 6. Retrieves the client secret and stores it in a Kubernetes Secret +# (CLIENT_SECRET_K8S_SECRET). When not running in-cluster, it is printed. # # Dependencies: # - curl: Command-line tool for making HTTP requests. @@ -40,9 +44,16 @@ KEYCLOAK_URL="http://$1" -USERNAME=$2 -PASSWORD=$3 -CLIENT_ID=$4 +USERNAME="${KEYCLOAK_ADMIN_USER:-$2}" +PASSWORD="${KEYCLOAK_ADMIN_PASSWORD:-$3}" +CLIENT_ID="${KEYCLOAK_CLIENT_ID:-$4}" + +CLIENT_SECRET_K8S_SECRET="${CLIENT_SECRET_K8S_SECRET:-keycloak-client-secret}" + +if [ -z "$USERNAME" ] || [ -z "$PASSWORD" ] || [ -z "$CLIENT_ID" ]; then + echo "ERROR: admin user, password and client id must be provided via environment (or positional args for standalone use)" + exit 1 +fi # Get access token TOKEN=$(curl -s -X POST "$KEYCLOAK_URL/realms/master/protocol/openid-connect/token" \ @@ -127,9 +138,49 @@ echo "Script executed successfully" CLIENT_SECRET=$(curl -s -X GET "$KEYCLOAK_URL/admin/realms/master/clients/$CLIENT_UUID/client-secret" \ -H "Authorization: Bearer $TOKEN" | jq -r '.value') -if [ -z "$CLIENT_SECRET" ]; then +if [ -z "$CLIENT_SECRET" ] || [ "$CLIENT_SECRET" = "null" ]; then echo "Failed to retrieve client secret" exit 1 +fi + +SA_DIR="/var/run/secrets/kubernetes.io/serviceaccount" +if [ -f "$SA_DIR/token" ] && [ -n "$KUBERNETES_SERVICE_HOST" ]; then + K8S_TOKEN=$(cat "$SA_DIR/token") + K8S_NAMESPACE=$(cat "$SA_DIR/namespace") + K8S_CACERT="$SA_DIR/ca.crt" + K8S_API="https://$KUBERNETES_SERVICE_HOST:${KUBERNETES_SERVICE_PORT_HTTPS:-443}" + SECRET_B64=$(printf '%s' "$CLIENT_SECRET" | base64 | tr -d '\n') + + SECRET_PAYLOAD=$(jq -nc \ + --arg name "$CLIENT_SECRET_K8S_SECRET" \ + --arg ns "$K8S_NAMESPACE" \ + --arg data "$SECRET_B64" \ + '{apiVersion:"v1",kind:"Secret",metadata:{name:$name,namespace:$ns},type:"Opaque",data:{"client-secret":$data}}') + + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --cacert "$K8S_CACERT" \ + -X GET "$K8S_API/api/v1/namespaces/$K8S_NAMESPACE/secrets/$CLIENT_SECRET_K8S_SECRET" \ + -H "Authorization: Bearer $K8S_TOKEN") + + if [ "$HTTP_CODE" = "200" ]; then + RESP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --cacert "$K8S_CACERT" \ + -X PUT "$K8S_API/api/v1/namespaces/$K8S_NAMESPACE/secrets/$CLIENT_SECRET_K8S_SECRET" \ + -H "Authorization: Bearer $K8S_TOKEN" \ + -H "Content-Type: application/json" \ + -d "$SECRET_PAYLOAD") + else + RESP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --cacert "$K8S_CACERT" \ + -X POST "$K8S_API/api/v1/namespaces/$K8S_NAMESPACE/secrets" \ + -H "Authorization: Bearer $K8S_TOKEN" \ + -H "Content-Type: application/json" \ + -d "$SECRET_PAYLOAD") + fi + + if [ "$RESP_CODE" = "200" ] || [ "$RESP_CODE" = "201" ]; then + echo "Client secret stored in Kubernetes Secret '$CLIENT_SECRET_K8S_SECRET' (namespace: $K8S_NAMESPACE)" + else + echo "ERROR: failed to store client secret in Kubernetes Secret (HTTP $RESP_CODE)" + exit 1 + fi else echo "Client secret: $CLIENT_SECRET" fi