feat: Show why a workload's instances could not be created

api/v1alpha/instance_types.go

@@ -1024,6 +1024,13 @@ const (
 	// Workload.Available when quota is blocking one or more instances.
 	WorkloadDeploymentReasonQuotaNotGranted = "QuotaNotGranted"
 
+	// WorkloadDeploymentReasonInstanceRejected is set on WorkloadDeployment.Available
+	// and Workload.Available when the API server refuses to create or update an
+	// Instance, for example because a generated name or label is invalid. Retrying
+	// the same request fails the same way, so the deployment stays blocked until
+	// the workload or platform changes. The message carries the API error.
+	WorkloadDeploymentReasonInstanceRejected = "InstanceRejected"
+
 	// WorkloadReasonNoAvailablePlacements is set on Workload.Available when all
 	// placements report no available deployments. Used as the last-resort default.
 	WorkloadReasonNoAvailablePlacements = "NoAvailablePlacements"

internal/agent/catalog.go

@@ -488,6 +488,18 @@ var catalog = []ReasonInfo{
 		Remediation: "Read the QuotaGranted status on the blocked instances.",
 		Skill:       SkillQuotaTriage,
 	},
+	{
+		Reason: computev1alpha.WorkloadDeploymentReasonInstanceRejected,
+		ConditionTypes: []string{
+			computev1alpha.WorkloadDeploymentAvailable,
+			computev1alpha.WorkloadDeploymentReplicasReady,
+			computev1alpha.WorkloadAvailable,
+		},
+		Actionability: ActionabilityPlatform,
+		Explanation: "Datum accepted the workload but could not start an instance for it, and retrying " +
+			"will not help. The status message quotes the error word for word.",
+		Remediation: "Raise this with Datum and include the status message.",
+	},
 	{
 		Reason:         computev1alpha.WorkloadReasonNetworkNotFound,
 		ConditionTypes: []string{computev1alpha.WorkloadAvailable},

internal/controller/workload_controller.go

@@ -752,8 +752,9 @@ func (r *WorkloadReconciler) SetupWithManager(mgr mcmanager.Manager) error {
 //	3 - QuotaNotGranted / PendingQuota (operator action may be needed)
 //	4 - ReferencedDataNotReady / AwaitingPropagation / Resolving (transient)
 //	5 - SourceNotFound / SourceTooLarge / SourceUnauthorized (hard spec error)
-//	6 - NetworkNotFound        (hard error; user action required)
-//	7 - NetworkFailedToCreate  (hard infra error)
+//	6 - NetworkNotFound / NoMatchingLocations / InstanceRejected (hard error;
+//	    retrying cannot clear it)
+//	7 - RuntimeClassNotServed  (no cell can accept the deployment)
 func workloadBlockingReasonPriority(reason string) int {
 	switch reason {
 	case computev1alpha.WorkloadReasonNoAvailablePlacements,
@@ -775,7 +776,8 @@ func workloadBlockingReasonPriority(reason string) int {
 		computev1alpha.ReferencedDataReasonSourceUnauthorized:
 		return 5
 	case computev1alpha.WorkloadReasonNetworkNotFound,
-		computev1alpha.WorkloadReasonNoMatchingLocations:
+		computev1alpha.WorkloadReasonNoMatchingLocations,
+		computev1alpha.WorkloadDeploymentReasonInstanceRejected:
 		return 6
 	// This reason outranks every other blocker. No cell can accept the
 	// deployment, so nothing else can make progress, and the user resolves the

internal/controller/workload_controller_test.go

@@ -199,6 +199,31 @@ func TestReconcileWorkloadStatus_MixedReasons(t *testing.T) {
 		"QuotaNotGranted (priority 3) must beat InstancesProvisioning (priority 1)")
 }
 
+// TestReconcileWorkloadStatus_InstanceRejectedOutranksTransient verifies that a
+// deployment whose instances the API server rejects names the workload's
+// blocker over deployments that are only slow to start or awaiting data.
+func TestReconcileWorkloadStatus_InstanceRejectedOutranksTransient(t *testing.T) {
+	workload := makeWorkload(1)
+	const rejectedMsg = `Create of instance "wd-b-0" was rejected: metadata.labels: must be no more than 63 bytes`
+	placements := map[string][]computev1alpha.WorkloadDeployment{
+		testPlacementA: {
+			makeWDWithAvailCond("wd-a", metav1.ConditionFalse,
+				computev1alpha.WorkloadDeploymentReasonInstancesProvisioning,
+				"Instances are being provisioned"),
+			makeWDWithAvailCond("wd-b", metav1.ConditionFalse,
+				computev1alpha.WorkloadDeploymentReasonInstanceRejected, rejectedMsg),
+			makeWDWithAvailCond("wd-c", metav1.ConditionFalse,
+				computev1alpha.ReferencedDataReasonSourceNotFound, testMsgConfigMapNotFound),
+		},
+	}
+
+	cond := runReconcileWorkloadStatus(t, workload, placements)
+	require.NotNil(t, cond)
+	assert.Equal(t, metav1.ConditionFalse, cond.Status)
+	assert.Equal(t, computev1alpha.WorkloadDeploymentReasonInstanceRejected, cond.Reason)
+	assert.Equal(t, rejectedMsg, cond.Message)
+}
+
 // TestReconcileWorkloadStatus_OneAvailableDeployment verifies that when at
 // least one deployment is Available=True, the Workload reports Available=True
 // regardless of other deployments' blocking reasons.
@@ -352,6 +377,10 @@ func TestWorkloadBlockingReasonPriority(t *testing.T) {
 		{computev1alpha.ReferencedDataReasonSourceUnauthorized, 5},
 		// Priority 6
 		{computev1alpha.WorkloadReasonNetworkNotFound, 6},
+		{computev1alpha.WorkloadReasonNoMatchingLocations, 6},
+		{computev1alpha.WorkloadDeploymentReasonInstanceRejected, 6},
+		// Priority 7
+		{computev1alpha.WorkloadDeploymentReasonRuntimeClassNotServed, 7},
 	}
 
 	for _, tt := range tests {

internal/controller/workloaddeployment_controller.go

@@ -4,6 +4,7 @@ package controller
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"slices"
 	"strings"
@@ -210,6 +211,7 @@ func (r *WorkloadDeploymentReconciler) Reconcile(ctx context.Context, req mcreco
 		logger.Info("instance control action", "instance", action.Object.GetName(), "action", action.ActionType())
 
 		if err := action.Execute(ctx, cl.GetClient()); err != nil {
+			reportRejectedInstance(ctx, cl.GetClient(), &deployment, existingStatus, action, instances.Items, err)
 			return ctrl.Result{}, fmt.Errorf("failed executing instance control action: %w", err)
 		}
 	}
@@ -598,6 +600,83 @@ func selectWDBlockingCondition(
 	}
 }
 
+// isInstanceRejected reports whether the API server refused an Instance write
+// outright. Such a request fails identically on every retry, unlike a conflict
+// or an unreachable server, so it is worth surfacing to the user.
+func isInstanceRejected(err error) bool {
+	return apierrors.IsInvalid(err) || apierrors.IsForbidden(err)
+}
+
+// reportRejectedInstance writes the rejection onto the deployment's status when
+// err is one retrying cannot clear. A failure to write is logged rather than
+// returned so the caller still requeues on the original error.
+func reportRejectedInstance(
+	ctx context.Context,
+	c client.Client,
+	deployment *computev1alpha.WorkloadDeployment,
+	existingStatus computev1alpha.WorkloadDeploymentStatus,
+	action instancecontrol.Action,
+	instances []computev1alpha.Instance,
+	err error,
+) {
+	if !isInstanceRejected(err) {
+		return
+	}
+	setInstanceRejectedConditions(deployment, action, instances, err)
+	if equality.Semantic.DeepEqual(existingStatus, deployment.Status) {
+		return
+	}
+	if statusErr := c.Status().Update(ctx, deployment); statusErr != nil {
+		log.FromContext(ctx).Error(statusErr, "failed reporting rejected instance on deployment status")
+	}
+}
+
+// setInstanceRejectedConditions records on the deployment that an Instance write
+// was rejected. ReplicasReady always reports it, since the deployment cannot
+// reach its desired replicas. Available reports it only while no instance is
+// ready; a deployment that is already serving stays available.
+func setInstanceRejectedConditions(
+	deployment *computev1alpha.WorkloadDeployment,
+	action instancecontrol.Action,
+	instances []computev1alpha.Instance,
+	err error,
+) {
+	message := fmt.Sprintf("%s of instance %q was rejected: %s",
+		action.ActionType(), action.Object.GetName(), apiErrorMessage(err))
+
+	apimeta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{
+		Type:               computev1alpha.WorkloadDeploymentReplicasReady,
+		Status:             metav1.ConditionFalse,
+		Reason:             computev1alpha.WorkloadDeploymentReasonInstanceRejected,
+		Message:            message,
+		ObservedGeneration: deployment.Generation,
+	})
+
+	for _, instance := range instances {
+		if apimeta.IsStatusConditionTrue(instance.Status.Conditions, computev1alpha.InstanceReady) {
+			return
+		}
+	}
+
+	apimeta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{
+		Type:               computev1alpha.WorkloadDeploymentAvailable,
+		Status:             metav1.ConditionFalse,
+		Reason:             computev1alpha.WorkloadDeploymentReasonInstanceRejected,
+		Message:            message,
+		ObservedGeneration: deployment.Generation,
+	})
+}
+
+// apiErrorMessage returns the API server's own message for err, without the
+// wrapping added on the way up, falling back to the full error text.
+func apiErrorMessage(err error) string {
+	var status apierrors.APIStatus
+	if errors.As(err, &status) && status.Status().Message != "" {
+		return status.Status().Message
+	}
+	return err.Error()
+}
+
 // wdBlockingReasonPriority returns the relative priority of a blocking reason on
 // WorkloadDeployment.Available. Higher numbers indicate causes that are more
 // actionable and should be surfaced over lower-priority transient states.

internal/controller/workloaddeployment_controller_test.go

@@ -4,15 +4,21 @@ package controller
 
 import (
 	"context"
+	"errors"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/validation/field"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
 	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/finalizer"
 	mcreconcile "sigs.k8s.io/multicluster-runtime/pkg/reconcile"
@@ -1184,3 +1190,142 @@ func TestWDAvailableCondition_AnnotationMalformedJSON(t *testing.T) {
 	assert.Equal(t, computev1alpha.WorkloadDeploymentReasonInstancesProvisioning, cond.Reason,
 		"malformed annotation must be silently ignored; fallback to InstancesProvisioning")
 }
+
+// ─── Instance rejection reporting ─────────────────────────────────────────────
+
+var wdTestInstanceGroupKind = schema.GroupKind{Group: computev1alpha.GroupVersion.Group, Kind: "Instance"}
+
+// newRejectingWDClient returns a project client that fails every Instance
+// create with createErr.
+func newRejectingWDClient(deployment *computev1alpha.WorkloadDeployment, createErr error) client.Client {
+	return fake.NewClientBuilder().
+		WithScheme(newProjectScheme()).
+		WithObjects(deployment).
+		WithStatusSubresource(deployment).
+		WithInterceptorFuncs(interceptor.Funcs{
+			Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
+				if _, ok := obj.(*computev1alpha.Instance); ok {
+					return createErr
+				}
+				return c.Create(ctx, obj, opts...)
+			},
+		}).
+		Build()
+}
+
+func wdControllerTestRequest() mcreconcile.Request {
+	return mcreconcile.Request{
+		ClusterName: testCluster,
+		Request: ctrl.Request{
+			NamespacedName: types.NamespacedName{Name: wdControllerTestName, Namespace: wdControllerTestNS},
+		},
+	}
+}
+
+// TestWorkloadDeploymentReconcile_ReportsRejectedInstance verifies that an
+// Instance the API server refuses to accept is reported on the deployment's
+// status, with the API server's message, and still requeues.
+func TestWorkloadDeploymentReconcile_ReportsRejectedInstance(t *testing.T) {
+	t.Parallel()
+
+	invalid := apierrors.NewInvalid(
+		wdTestInstanceGroupKind,
+		"test-wd-0",
+		field.ErrorList{field.Invalid(field.NewPath("metadata", "labels"), "x", "must be no more than 63 bytes")},
+	)
+
+	tests := []struct {
+		name string
+		err  error
+	}{
+		{name: "invalid", err: invalid},
+		{name: "forbidden", err: apierrors.NewForbidden(
+			schema.GroupResource{Group: computev1alpha.GroupVersion.Group, Resource: "instances"},
+			"test-wd-0", errors.New("denied by admission webhook"))},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			deployment := wdControllerTestDeployment(1)
+			deployment.Finalizers = []string{workloadControllerFinalizer}
+			deployment.Generation = 4
+			cl := newRejectingWDClient(deployment, tt.err)
+			r := newTestWDReconciler(cl)
+
+			_, err := r.Reconcile(context.Background(), wdControllerTestRequest())
+			require.Error(t, err, "a rejected instance must still requeue")
+
+			var updated computev1alpha.WorkloadDeployment
+			require.NoError(t, cl.Get(context.Background(), wdControllerTestRequest().NamespacedName, &updated))
+
+			wantMessage := tt.err.(apierrors.APIStatus).Status().Message
+			for _, condType := range []string{computev1alpha.WorkloadDeploymentAvailable, computev1alpha.WorkloadDeploymentReplicasReady} {
+				cond := apimeta.FindStatusCondition(updated.Status.Conditions, condType)
+				require.NotNil(t, cond, "%s must be set", condType)
+				assert.Equal(t, metav1.ConditionFalse, cond.Status, condType)
+				assert.Equal(t, computev1alpha.WorkloadDeploymentReasonInstanceRejected, cond.Reason, condType)
+				assert.Contains(t, cond.Message, wantMessage, condType)
+				assert.NotContains(t, cond.Message, "failed to create", "the message must carry the API error, not the controller's wrapping")
+				assert.Equal(t, int64(4), cond.ObservedGeneration, condType)
+			}
+		})
+	}
+}
+
+// TestWorkloadDeploymentReconcile_TransientInstanceErrorNotReported verifies
+// that an error expected to clear on retry does not replace the deployment's
+// status with a rejection.
+func TestWorkloadDeploymentReconcile_TransientInstanceErrorNotReported(t *testing.T) {
+	t.Parallel()
+
+	deployment := wdControllerTestDeployment(1)
+	deployment.Finalizers = []string{workloadControllerFinalizer}
+	cl := newRejectingWDClient(deployment, apierrors.NewServiceUnavailable("apiserver is shutting down"))
+	r := newTestWDReconciler(cl)
+
+	_, err := r.Reconcile(context.Background(), wdControllerTestRequest())
+	require.Error(t, err)
+
+	var updated computev1alpha.WorkloadDeployment
+	require.NoError(t, cl.Get(context.Background(), wdControllerTestRequest().NamespacedName, &updated))
+	for _, cond := range updated.Status.Conditions {
+		assert.NotEqual(t, computev1alpha.WorkloadDeploymentReasonInstanceRejected, cond.Reason,
+			"a transient error must not be reported as a rejection on %s", cond.Type)
+	}
+}
+
+// TestSetInstanceRejectedConditions_ServingDeploymentStaysAvailable verifies
+// that a rejection while another instance is ready leaves the deployment
+// Available and reports the shortfall on ReplicasReady only.
+func TestSetInstanceRejectedConditions_ServingDeploymentStaysAvailable(t *testing.T) {
+	t.Parallel()
+
+	deployment := wdControllerTestDeployment(2)
+	apimeta.SetStatusCondition(&deployment.Status.Conditions, metav1.Condition{
+		Type:   computev1alpha.WorkloadDeploymentAvailable,
+		Status: metav1.ConditionTrue,
+		Reason: computev1alpha.WorkloadDeploymentReasonStableInstanceFound,
+	})
+
+	ready := wdControllerTestInstance("test-wd-0")
+	apimeta.SetStatusCondition(&ready.Status.Conditions, metav1.Condition{
+		Type:   computev1alpha.InstanceReady,
+		Status: metav1.ConditionTrue,
+		Reason: wdTestReasonReady,
+	})
+	rejected := wdControllerTestInstance("test-wd-1")
+	action := instancecontrol.NewCreateAction(&rejected)
+
+	setInstanceRejectedConditions(deployment, action, []computev1alpha.Instance{ready},
+		apierrors.NewInvalid(wdTestInstanceGroupKind, "test-wd-1", nil))
+
+	assert.True(t, apimeta.IsStatusConditionTrue(deployment.Status.Conditions, computev1alpha.WorkloadDeploymentAvailable),
+		"a deployment with a ready instance must stay available")
+	replicasReady := apimeta.FindStatusCondition(deployment.Status.Conditions, computev1alpha.WorkloadDeploymentReplicasReady)
+	require.NotNil(t, replicasReady)
+	assert.Equal(t, metav1.ConditionFalse, replicasReady.Status)
+	assert.Equal(t, computev1alpha.WorkloadDeploymentReasonInstanceRejected, replicasReady.Reason)
+	assert.Contains(t, replicasReady.Message, `"test-wd-1"`)
+}