diff --git a/api/v1alpha1/vpcattachment_types.go b/api/v1alpha1/vpcattachment_types.go index 674ee0c..40f0968 100644 --- a/api/v1alpha1/vpcattachment_types.go +++ b/api/v1alpha1/vpcattachment_types.go @@ -28,6 +28,12 @@ const ( // NetworkAttachmentDefinition is written. ConditionTypeReady = "Ready" + // ConditionTypeInternetEgressReady reports whether this attachment's + // outbound traffic reaches the destinations its network declared. It is + // written by the controller that records the shard on the attachment's + // node, and read back onto the interface a consumer holds. + ConditionTypeInternetEgressReady = "InternetEgressReady" + // ConditionTypeProgrammed reports that the data plane realized the attachment. ConditionTypeProgrammed = "Programmed" ) @@ -128,6 +134,11 @@ type InternetEgressAddressFamily string // InternetEgressAddressFamilyIPv6 is an IPv6 egress source address. const InternetEgressAddressFamilyIPv6 InternetEgressAddressFamily = "IPv6" +// InternetEgressAddressFamilyIPv4 is an IPv4 egress source address. The +// family type does not admit it yet, so nothing can ask for it; it is defined +// so the binder's family check means today what it will mean then. +const InternetEgressAddressFamilyIPv4 InternetEgressAddressFamily = "IPv4" + // InternetEgressAddressStability is how far a consumer may rely on an egress // source address. It is the consumer-side projection of the serving class's // sharing, derived here so a consumer never reads a class. diff --git a/cmd/main.go b/cmd/main.go index f499299..9bbc1f3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -117,6 +117,18 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "NetworkContext") os.Exit(1) } + if err := (&controller.EgressShardClaimReconciler{ + Client: mgr.GetClient(), Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "EgressShardClaim") + os.Exit(1) + } + if err := (&controller.EgressShardBindingReconciler{ + Client: mgr.GetClient(), Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "EgressShardBinding") + os.Exit(1) + } if err := (&controller.NetworkInterfaceReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), APIReader: mgr.GetAPIReader(), AttachmentMode: attachmentMode, diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 01f8cca..b012a84 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -71,10 +71,39 @@ rules: resources: - bgpadvertisements - bgprouters + verbs: + - get + - list + - watch +- apiGroups: + - network.datumapis.com + resources: + - egressshardclaims + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - network.datumapis.com + resources: + - egressshardclaims/status + verbs: + - get + - patch + - update +- apiGroups: + - network.datumapis.com + resources: - egressshards verbs: - get - list + - patch + - update - watch - apiGroups: - networking.datumapis.com diff --git a/docs/api/vpc.md b/docs/api/vpc.md index 3481582..a15848e 100644 --- a/docs/api/vpc.md +++ b/docs/api/vpc.md @@ -50,6 +50,7 @@ _Appears in:_ | Field | Description | | --- | --- | | `IPv6` | | +| `IPv4` | | #### InternetEgressAddressStability diff --git a/go.mod b/go.mod index 258934f..cc5f779 100644 --- a/go.mod +++ b/go.mod @@ -79,6 +79,6 @@ require ( sigs.k8s.io/yaml v1.6.0 // indirect ) -replace go.datum.net/network => github.com/datum-cloud/network v0.1.1-0.20260923215140-1ed44c853eea +replace go.datum.net/network => github.com/datum-cloud/network v0.1.1-0.20260926162437-e92beadac630 replace go.datum.net/network-services-operator => github.com/datum-cloud/network-services-operator v0.27.2-0.20260924225138-f78ee4cb7f7f diff --git a/go.sum b/go.sum index 298a125..7cb46b9 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/datum-cloud/network v0.1.1-0.20260923215140-1ed44c853eea h1:/a67sX+DD7+tRpaQSDAuoFuOfoMz9hC22wi4AGakiuk= -github.com/datum-cloud/network v0.1.1-0.20260923215140-1ed44c853eea/go.mod h1:dqzM8WZczbiZ9bCvsxjkoI10GJqQ24NVWnc9boXgOkE= +github.com/datum-cloud/network v0.1.1-0.20260926162437-e92beadac630 h1:qy9/S+Y4C67n7zvwgN1rjfTVxJVRziGxCgMWIkBwKL8= +github.com/datum-cloud/network v0.1.1-0.20260926162437-e92beadac630/go.mod h1:dqzM8WZczbiZ9bCvsxjkoI10GJqQ24NVWnc9boXgOkE= github.com/datum-cloud/network-services-operator v0.27.2-0.20260924225138-f78ee4cb7f7f h1:O/pK+GHjZqAQIBWYmkkEJNmxTPp6tmrL60bXMkWzyC0= github.com/datum-cloud/network-services-operator v0.27.2-0.20260924225138-f78ee4cb7f7f/go.mod h1:IyFCDsNxfxvXHFfotZsD8eBBI9EyxpxAbQ7UTg6CNdw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/controller/egressshardbinding_controller.go b/internal/controller/egressshardbinding_controller.go new file mode 100644 index 0000000..5204c4f --- /dev/null +++ b/internal/controller/egressshardbinding_controller.go @@ -0,0 +1,175 @@ +/* +Copyright © 2026 Datum Technology, Inc. All rights reserved. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package controller + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// EgressShardBindingReconciler keeps a shard's finalizer in step with its +// consumer set. +// +// The finalizer is the only state a binder adds to a shard. It holds while any +// claim is bound, so decommissioning a shard is an act someone takes rather +// than an outcome networks discover: deleting one strands the return traffic of +// every flow it is translating, and nothing rebinds a claim, so a network whose +// shard vanished has no egress and no second answer coming. +// +// The consumer set is a list query over the claims, never a field on the shard. +// That is the whole point: a shard that recorded its own consumers would be +// holding the list of served networks the model refuses it, and the list would +// have to be kept in step by whoever binds. +type EgressShardBindingReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshards,verbs=get;list;watch;update;patch +// finalizerEgressShardBinding is held on a shard while any claim is bound to it, +// so decommissioning a shard is an act someone takes rather than an outcome +// instances discover. +const finalizerEgressShardBinding = "cloud.datumapis.com/egress-shard-binding" + +// +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshardclaims,verbs=get;list;watch + +func (r *EgressShardBindingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var shard bgpv1alpha1.EgressShard + if err := r.Get(ctx, req.NamespacedName, &shard); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + consumers, err := boundEgressShardClaims(ctx, r.Client, &shard) + if err != nil { + return ctrl.Result{}, err + } + if len(consumers) == 0 { + return ctrl.Result{}, releaseShard(ctx, r.Client, &shard) + } + if !shard.DeletionTimestamp.IsZero() { + // Said out loud, because the alternative to a stuck object here is a + // network losing the internet with nothing to point at. Deleting the + // claims is what lets the shard go. + logf.FromContext(ctx).Info("holding an egress shard open while networks are bound to it", + "shard", client.ObjectKeyFromObject(&shard), "consumers", len(consumers)) + } + return ctrl.Result{}, holdShard(ctx, r.Client, &shard) +} + +// holdShard adds the binder's finalizer, so the shard cannot go while a network +// is bound to it. +func holdShard(ctx context.Context, cl client.Client, shard *bgpv1alpha1.EgressShard) error { + if controllerutil.ContainsFinalizer(shard, finalizerEgressShardBinding) { + return nil + } + // Patched rather than updated. The controller holding the addressing-service + // credential writes this spec's addresses, and a whole-object update from a + // copy read before that write would put the old value back over a field that + // is write-once. + patch := client.MergeFrom(shard.DeepCopy()) + controllerutil.AddFinalizer(shard, finalizerEgressShardBinding) + if err := cl.Patch(ctx, shard, patch); err != nil { + return fmt.Errorf("hold egress shard %s open for the networks bound to it: %w", + client.ObjectKeyFromObject(shard), err) + } + return nil +} + +// releaseShard removes the binder's finalizer from a shard no network is bound +// to, which is what lets an operator decommission a drained node. +func releaseShard(ctx context.Context, cl client.Client, shard *bgpv1alpha1.EgressShard) error { + if !controllerutil.ContainsFinalizer(shard, finalizerEgressShardBinding) { + return nil + } + patch := client.MergeFrom(shard.DeepCopy()) + controllerutil.RemoveFinalizer(shard, finalizerEgressShardBinding) + if err := cl.Patch(ctx, shard, patch); err != nil { + return fmt.Errorf("release egress shard %s: %w", client.ObjectKeyFromObject(shard), err) + } + logf.FromContext(ctx).Info("released an egress shard no network is bound to", + "shard", client.ObjectKeyFromObject(shard)) + return nil +} + +// boundEgressShardClaims is a shard's consumer set: the networks bound to it. +// +// It is a list query over the label the binder stamps, which is what stands in +// for the list of served networks a shard does not hold. The label narrows the +// query and each claim's own status settles it, so a label left behind by a +// binding that never completed counts as nothing. +func boundEgressShardClaims( + ctx context.Context, reader client.Reader, shard *bgpv1alpha1.EgressShard, +) ([]bgpv1alpha1.EgressShardClaim, error) { + var claims bgpv1alpha1.EgressShardClaimList + if err := reader.List(ctx, &claims, client.MatchingLabels{ + bgpv1alpha1.LabelEgressShardClaimShard: shard.Name, + }); err != nil { + return nil, fmt.Errorf("list the claims bound to egress shard %s: %w", shard.Name, err) + } + + bound := make([]bgpv1alpha1.EgressShardClaim, 0, len(claims.Items)) + for i := range claims.Items { + claim := claims.Items[i] + held := claim.Status.ShardRef + if held == nil || held.Name != shard.Name || held.Namespace != shard.Namespace { + continue + } + if !claim.DeletionTimestamp.IsZero() { + continue + } + bound = append(bound, claim) + } + return bound, nil +} + +// SetupWithManager registers the reconciler with the manager. +func (r *EgressShardBindingReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&bgpv1alpha1.EgressShard{}). + Watches(&bgpv1alpha1.EgressShardClaim{}, + handler.EnqueueRequestsFromMapFunc(egressShardForClaim)). + Named("egressshardbinding"). + Complete(r) +} + +// egressShardForClaim maps a claim to the shard it holds, including the last +// state of one being deleted — which is the event that lets the final claim on +// a shard release it. +// +// A claim holding no binding maps to nothing, and needs to: it was never part +// of any shard's consumer set, which is counted from this same field. +func egressShardForClaim(_ context.Context, object client.Object) []reconcile.Request { + claim, ok := object.(*bgpv1alpha1.EgressShardClaim) + if !ok || claim.Status.ShardRef == nil { + return nil + } + return []reconcile.Request{{NamespacedName: client.ObjectKey{ + Namespace: claim.Status.ShardRef.Namespace, + Name: claim.Status.ShardRef.Name, + }}} +} diff --git a/internal/controller/egressshardbinding_controller_test.go b/internal/controller/egressshardbinding_controller_test.go new file mode 100644 index 0000000..683bbf4 --- /dev/null +++ b/internal/controller/egressshardbinding_controller_test.go @@ -0,0 +1,144 @@ +/* +Copyright © 2026 Datum Technology, Inc. All rights reserved. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package controller + +import ( + "testing" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// egressTestShard is the one shard these tests hold open or release. +const egressTestShard = "shard-a" + +func newShardBinder(t *testing.T, objects ...client.Object) (*EgressShardBindingReconciler, client.Client) { + t.Helper() + + scheme := runtime.NewScheme() + if err := cloudv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("build the cloud scheme: %v", err) + } + if err := bgpv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("build the fabric scheme: %v", err) + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + return &EgressShardBindingReconciler{Client: fakeClient, Scheme: scheme}, fakeClient +} + +func reconcileShard(t *testing.T, r *EgressShardBindingReconciler) { + t.Helper() + key := client.ObjectKey{Namespace: egressShardNamespace, Name: egressTestShard} + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: key}); err != nil { + t.Fatalf("reconcile the shard: %v", err) + } +} + +func heldOpen(t *testing.T, cl client.Client) bool { + t.Helper() + var shard bgpv1alpha1.EgressShard + key := client.ObjectKey{Namespace: egressShardNamespace, Name: egressTestShard} + if err := cl.Get(t.Context(), key, &shard); err != nil { + t.Fatalf("get the shard: %v", err) + } + return controllerutil.ContainsFinalizer(&shard, finalizerEgressShardBinding) +} + +// The finalizer is the only state a binder adds to a shard, and it holds while +// a network is bound: deleting a translating shard strands the return traffic of +// every flow on it, and nothing rebinds a claim. +func TestShardIsHeldOpenWhileANetworkIsBound(t *testing.T) { + r, cl := newShardBinder(t, newEgressClaim("shard-a"), + newEgressShard("shard-a", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) + + reconcileShard(t, r) + + if !heldOpen(t, cl) { + t.Error("a shard with a network bound to it is not held open") + } +} + +// A drained shard is released, which is what lets an operator decommission the +// node it runs on. +func TestShardIsReleasedWhenNoNetworkIsBound(t *testing.T) { + shard := newEgressShard("shard-a", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100") + shard.Finalizers = []string{finalizerEgressShardBinding} + r, cl := newShardBinder(t, shard) + + reconcileShard(t, r) + + if heldOpen(t, cl) { + t.Error("a shard no network is bound to is still held open") + } +} + +// A label with no binding behind it is no consumer. A claim that was labelled +// by a pass that then failed must not hold a shard open forever. +func TestShardIsReleasedWhenAClaimHoldsOnlyTheLabel(t *testing.T) { + claim := newEgressClaim("shard-a") + claim.Status.ShardRef = nil + shard := newEgressShard("shard-a", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100") + shard.Finalizers = []string{finalizerEgressShardBinding} + r, cl := newShardBinder(t, claim, shard) + + reconcileShard(t, r) + + if heldOpen(t, cl) { + t.Error("a label with no binding behind it held a shard open") + } +} + +// A claim naming a shard of the same name in another namespace is another +// cell's business, not a consumer of this one. +func TestShardIgnoresAClaimBoundElsewhere(t *testing.T) { + claim := newEgressClaim("shard-a") + claim.Status.ShardRef.Namespace = "some-other-namespace" + shard := newEgressShard("shard-a", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100") + shard.Finalizers = []string{finalizerEgressShardBinding} + r, cl := newShardBinder(t, claim, shard) + + reconcileShard(t, r) + + if heldOpen(t, cl) { + t.Error("a claim bound to another namespace's shard held this one open") + } +} + +// The mapping that makes the last claim's deletion release its shard. A claim +// with no binding maps to nothing, because it was in no consumer set. +func TestEgressShardForClaim(t *testing.T) { + bound := newEgressClaim("shard-a") + requests := egressShardForClaim(t.Context(), bound) + if len(requests) != 1 { + t.Fatalf("requests: got %d, want 1", len(requests)) + } + if requests[0].Name != "shard-a" || requests[0].Namespace != egressShardNamespace { + t.Errorf("got %v, want the bound shard's key", requests[0].NamespacedName) + } + + if requests := egressShardForClaim(t.Context(), newEgressClaim("")); len(requests) != 0 { + t.Errorf("an unbound claim mapped to %v", requests) + } +} diff --git a/internal/controller/egressshardclaim_controller.go b/internal/controller/egressshardclaim_controller.go new file mode 100644 index 0000000..180fcf0 --- /dev/null +++ b/internal/controller/egressshardclaim_controller.go @@ -0,0 +1,575 @@ +/* +Copyright © 2026 Datum Technology, Inc. All rights reserved. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package controller + +import ( + "context" + "errors" + "fmt" + "slices" + + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// egressUnavailableMessage is what a consumer reads when the node serving their +// instance provides no egress. It names no node and no shard: a consumer cannot +// act on either, and each says where the platform runs their workload. The +// cause lives on the claim, which is an operator's object. +const egressUnavailableMessage = "No component serving this instance provides internet egress" + +// bindingRefusedError is why one shard may not be recorded, carrying the named +// reason it is reported under. +type bindingRefusedError struct { + reason string + message string +} + +func (e *bindingRefusedError) Error() string { return e.message } + +// EgressShardClaimReconciler records which shard each attachment egresses +// through: the one on the node the attachment landed on. +// +// It decides nothing. The node installs its route from its own configuration +// the moment the attachment exists, and this runs after the attachment has +// reported its node. The record exists so the binding is readable, so a node +// without a usable shard produces a condition a consumer can see on the +// attachment, and so a later tier that does select among shards binds through +// the same object. +type EgressShardClaimReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=vpcattachments,verbs=get;list;watch +// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=vpcattachments/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkcontexts,verbs=get;list;watch +// +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshardclaims,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshardclaims/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshards,verbs=get;list;watch;update;patch + +func (r *EgressShardClaimReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // A claim carries its attachment's name, so one key reads both. + var attachment cloudv1alpha1.VPCAttachment + attachmentFound := true + if err := r.Get(ctx, req.NamespacedName, &attachment); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("get VPCAttachment %s: %w", req.NamespacedName, err) + } + attachmentFound = false + } + + var claim bgpv1alpha1.EgressShardClaim + claimFound := true + if err := r.Get(ctx, req.NamespacedName, &claim); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("get EgressShardClaim %s: %w", req.NamespacedName, err) + } + claimFound = false + } + if claimFound && !claim.DeletionTimestamp.IsZero() { + // The shard's finalizer is released by the reconciler watching shards, + // which sees this claim leave the consumer set. + return ctrl.Result{}, nil + } + + if !attachmentFound || !attachment.DeletionTimestamp.IsZero() { + return ctrl.Result{}, r.releaseClaim(ctx, &claim, claimFound, "the attachment is gone") + } + + terms, err := r.claimTerms(ctx, &attachment) + if err != nil { + var refused *bindingRefusedError + if errors.As(err, &refused) { + logf.FromContext(ctx).Info("nothing can be recorded for this attachment's egress", + "attachment", client.ObjectKeyFromObject(&attachment), "reason", refused.reason, + "cause", refused.message) + if err := r.releaseClaim(ctx, &claim, claimFound, refused.message); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, r.reportAttachment(ctx, &attachment, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, egressUnavailableMessage) + } + return ctrl.Result{}, err + } + if terms == nil { + // The network reaches nothing, or the attachment has not landed yet. + // Both are ordinary and neither is this controller's to report on. + return ctrl.Result{}, r.releaseClaim(ctx, &claim, claimFound, + "this attachment declares no internet egress, or has not reported its node") + } + + if !claimFound { + return ctrl.Result{}, r.createClaim(ctx, &attachment, terms) + } + + if !equality.Semantic.DeepEqual(claim.Spec, *terms) { + // The spec is immutable and the attachment moved, most often to another + // node. The old record describes a node this attachment is no longer on, + // so it is replaced on the next pass. + return ctrl.Result{}, r.releaseClaim(ctx, &claim, claimFound, + "the attachment no longer matches the record") + } + + if claim.Status.ShardRef != nil { + return ctrl.Result{}, r.reportExistingBinding(ctx, &attachment, &claim) + } + + return ctrl.Result{}, r.bind(ctx, &attachment, &claim) +} + +// claimTerms are the terms a claim would be written with for this attachment, +// or nil when there is nothing to record yet. +func (r *EgressShardClaimReconciler) claimTerms( + ctx context.Context, attachment *cloudv1alpha1.VPCAttachment, +) (*bgpv1alpha1.EgressShardClaimSpec, error) { + if attachment.Status.Node == "" { + return nil, nil + } + + var networkContext networkingv1alpha.NetworkContext + key := client.ObjectKey{Namespace: attachment.Namespace, Name: attachment.Spec.VPC.Name} + if err := r.Get(ctx, key, &networkContext); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("get NetworkContext %s: %w", key, err) + } + intent := internetEgressIntent(&networkContext) + if intent == nil || intent.Mode != networkingv1alpha.NetworkInternetEgressEnabled { + return nil, nil + } + + families, err := claimFamilies(intent.Reach) + if err != nil { + return nil, err + } + return &bgpv1alpha1.EgressShardClaimSpec{ + Attachment: bgpv1alpha1.EgressShardClaimAttachmentRef{Name: attachment.Name}, + VPC: bgpv1alpha1.EgressShardClaimVPCRef{Name: attachment.Spec.VPC.Name}, + NodeName: attachment.Status.Node, + Families: families, + }, nil +} + +// claimFamilies carries the families the network declared, copied from the +// network rather than stated per attachment, so two attachments of one network +// on one node can never ask for different shards. +// +// Only IPv6 is accepted, because it is the only family the declaration can +// carry. A family that arrives anyway is refused rather than dropped: silently +// recording a shard that translates nothing for it would report egress a +// consumer does not have. +func claimFamilies( + reach []networkingv1alpha.IPFamily, +) ([]bgpv1alpha1.EgressAddressFamily, error) { + if len(reach) == 0 { + return nil, &bindingRefusedError{ + reason: networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, + message: "The network declares no address family to reach, so no shard can serve it", + } + } + families := make([]bgpv1alpha1.EgressAddressFamily, 0, len(reach)) + for _, family := range reach { + if family != networkingv1alpha.IPv6Protocol { + return nil, &bindingRefusedError{ + reason: networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, + message: fmt.Sprintf("The network declares %s destinations, which no shard translates", + family), + } + } + families = append(families, bgpv1alpha1.EgressAddressFamilyIPv6) + } + return families, nil +} + +// createClaim writes the one claim recording this attachment's egress. It is +// owned by the attachment, so an attachment that goes takes its record with it +// and the shard it held is released. +func (r *EgressShardClaimReconciler) createClaim( + ctx context.Context, + attachment *cloudv1alpha1.VPCAttachment, + terms *bgpv1alpha1.EgressShardClaimSpec, +) error { + claim := &bgpv1alpha1.EgressShardClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: attachment.Namespace, + Name: attachment.Name, + Labels: map[string]string{bgpv1alpha1.LabelEgressShardClaimNode: terms.NodeName}, + }, + Spec: *terms, + } + if err := controllerutil.SetControllerReference(attachment, claim, r.Scheme); err != nil { + return fmt.Errorf("set the owner on EgressShardClaim %s: %w", claim.Name, err) + } + if err := r.Create(ctx, claim); err != nil { + if apierrors.IsAlreadyExists(err) { + return nil + } + return fmt.Errorf("create EgressShardClaim %s: %w", client.ObjectKeyFromObject(claim), err) + } + logf.FromContext(ctx).Info("recorded an attachment's egress", + "claim", client.ObjectKeyFromObject(claim), "node", terms.NodeName) + return nil +} + +// releaseClaim deletes the claim for an attachment that no longer has egress +// to record. Deleting it is what releases the shard: the consumer set is a +// list of claims. +func (r *EgressShardClaimReconciler) releaseClaim( + ctx context.Context, claim *bgpv1alpha1.EgressShardClaim, claimFound bool, why string, +) error { + if !claimFound { + return nil + } + if err := r.Delete(ctx, claim); err != nil { + return client.IgnoreNotFound(fmt.Errorf("delete EgressShardClaim %s: %w", + client.ObjectKeyFromObject(claim), err)) + } + logf.FromContext(ctx).Info("released an egress shard claim", + "claim", client.ObjectKeyFromObject(claim), "reason", why) + return nil +} + +// bind records the shard on the attachment's node, or why none can be. +func (r *EgressShardClaimReconciler) bind( + ctx context.Context, + attachment *cloudv1alpha1.VPCAttachment, + claim *bgpv1alpha1.EgressShardClaim, +) error { + shard, err := egressShardOnNode(ctx, r.Client, claim.Spec.NodeName) + if err != nil { + return err + } + if shard == nil { + return r.refuse(ctx, attachment, claim, &bindingRefusedError{ + reason: bgpv1alpha1.EgressShardClaimReasonNoShardOnNode, + message: fmt.Sprintf("No egress shard names node %q, so nothing on it translates this attachment's traffic", + claim.Spec.NodeName), + }) + } + if refusal := shardRefusal(shard, claim.Spec.Families); refusal != nil { + return r.refuse(ctx, attachment, claim, refusal) + } + return r.recordBinding(ctx, attachment, claim, shard) +} + +// shardRefusal is why a shard may not be recorded, or nil if it may be. +func shardRefusal( + shard *bgpv1alpha1.EgressShard, families []bgpv1alpha1.EgressAddressFamily, +) *bindingRefusedError { + if !shard.DeletionTimestamp.IsZero() { + return &bindingRefusedError{ + reason: bgpv1alpha1.EgressShardClaimReasonShardTerminating, + message: fmt.Sprintf("Egress shard %q is being deleted, so it takes no further attachment", + shard.Name), + } + } + if shard.Status.ShardSID == "" { + return &bindingRefusedError{ + reason: bgpv1alpha1.EgressShardClaimReasonShardNotReady, + message: fmt.Sprintf("Egress shard %q has reported no identifier a node can route toward", + shard.Name), + } + } + if (shard.Spec.ShardSID != "" && shard.Spec.ShardSID != shard.Status.ShardSID) || + (shard.Spec.ShardAddressIPv6 != "" && shard.Status.ShardAddressIPv6 != "" && + shard.Spec.ShardAddressIPv6 != shard.Status.ShardAddressIPv6) { + return &bindingRefusedError{ + reason: bgpv1alpha1.EgressShardClaimReasonShardMismatch, + message: fmt.Sprintf("Egress shard %q runs an identity other than the one its spec states, so which one serves this node is unknown", + shard.Name), + } + } + if slices.Contains(families, bgpv1alpha1.EgressAddressFamilyIPv4) && + shard.Status.ShardAddressIPv4 == "" { + return &bindingRefusedError{ + reason: bgpv1alpha1.EgressShardClaimReasonFamilyUnsupported, + message: fmt.Sprintf("Egress shard %q translates no IPv4 flow, which the network declares it reaches", + shard.Name), + } + } + return nil +} + +// recordBinding writes the record: the shard's finalizer first, then the label +// that makes this claim part of the shard's consumer set, then the record +// itself, so a crash between the writes leaves nothing that reads as bound +// without being held. +func (r *EgressShardClaimReconciler) recordBinding( + ctx context.Context, + attachment *cloudv1alpha1.VPCAttachment, + claim *bgpv1alpha1.EgressShardClaim, + shard *bgpv1alpha1.EgressShard, +) error { + if err := holdShard(ctx, r.Client, shard); err != nil { + return err + } + if err := r.labelClaim(ctx, claim, shard.Name); err != nil { + return err + } + + claim.Status.ShardRef = &bgpv1alpha1.EgressShardClaimShardRef{ + Namespace: shard.Namespace, + Name: shard.Name, + } + logf.FromContext(ctx).Info("recorded an attachment's egress shard", + "claim", client.ObjectKeyFromObject(claim), "node", claim.Spec.NodeName, + "shard", client.ObjectKeyFromObject(shard)) + + if err := r.publishClaimStatus(ctx, claim, metav1.ConditionTrue, + bgpv1alpha1.EgressShardClaimReasonBound, + fmt.Sprintf("Attachment %q egresses through egress shard %q", claim.Spec.Attachment.Name, shard.Name)); err != nil { + return err + } + return r.reportBinding(ctx, attachment, shard) +} + +// labelClaim stamps the shard a claim records, so the shard's consumer set is +// a list query. It is re-asserted on every pass over a bound claim, because a +// label lost to an edit would hide an attachment from the query that holds a +// shard open. +func (r *EgressShardClaimReconciler) labelClaim( + ctx context.Context, claim *bgpv1alpha1.EgressShardClaim, shardName string, +) error { + if claim.Labels[bgpv1alpha1.LabelEgressShardClaimShard] == shardName { + return nil + } + patch := client.MergeFrom(claim.DeepCopy()) + if claim.Labels == nil { + claim.Labels = map[string]string{} + } + claim.Labels[bgpv1alpha1.LabelEgressShardClaimShard] = shardName + if err := r.Patch(ctx, claim, patch); err != nil { + return fmt.Errorf("label EgressShardClaim %s with its shard: %w", + client.ObjectKeyFromObject(claim), err) + } + return nil +} + +// reportExistingBinding says what a record already made is delivering, on the +// claim and on the attachment, and repairs the label the consumer set is +// counted by. +func (r *EgressShardClaimReconciler) reportExistingBinding( + ctx context.Context, + attachment *cloudv1alpha1.VPCAttachment, + claim *bgpv1alpha1.EgressShardClaim, +) error { + if err := r.labelClaim(ctx, claim, claim.Status.ShardRef.Name); err != nil { + return err + } + + var shard bgpv1alpha1.EgressShard + key := client.ObjectKey{ + Namespace: claim.Status.ShardRef.Namespace, + Name: claim.Status.ShardRef.Name, + } + if err := r.Get(ctx, key, &shard); err != nil { + if !apierrors.IsNotFound(err) { + return fmt.Errorf("get the recorded EgressShard %s: %w", key, err) + } + message := fmt.Sprintf("Egress shard %q no longer exists", key.Name) + if err := r.publishClaimStatus(ctx, claim, metav1.ConditionFalse, + bgpv1alpha1.EgressShardClaimReasonShardMissing, message); err != nil { + return err + } + return r.reportAttachment(ctx, attachment, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, egressUnavailableMessage) + } + return r.reportBinding(ctx, attachment, &shard) +} + +// reportBinding projects a recorded shard onto the attachment condition a +// consumer reads. +// +// Degraded is deliberately never written. It means egress works for some +// declared families and not others, and only one family is accepted anywhere +// on this path, so no state can reach it. +func (r *EgressShardClaimReconciler) reportBinding( + ctx context.Context, + attachment *cloudv1alpha1.VPCAttachment, + shard *bgpv1alpha1.EgressShard, +) error { + if shard.Status.ShardAddressIPv6 == "" { + return r.reportAttachment(ctx, attachment, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonAddressUnavailable, + "No egress address has been allocated for the node serving this instance yet") + } + return r.reportAttachment(ctx, attachment, metav1.ConditionTrue, + networkingv1alpha.NetworkContextInternetEgressReasonReady, + fmt.Sprintf("This instance reaches the internet, and %s is the address it reaches it from", + shard.Status.ShardAddressIPv6)) +} + +// refuse records that nothing was recorded, and why. The claim stays, unbound, +// and takes the shard the moment one names its node and is usable. +func (r *EgressShardClaimReconciler) refuse( + ctx context.Context, + attachment *cloudv1alpha1.VPCAttachment, + claim *bgpv1alpha1.EgressShardClaim, + refusal *bindingRefusedError, +) error { + if err := r.publishClaimStatus(ctx, claim, metav1.ConditionFalse, + refusal.reason, refusal.message); err != nil { + return err + } + // The attachment carries the fact about the consumer's instance. Which + // shard refused it, and why, is on the claim, which is an operator's object. + return r.reportAttachment(ctx, attachment, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, egressUnavailableMessage) +} + +func (r *EgressShardClaimReconciler) publishClaimStatus( + ctx context.Context, + claim *bgpv1alpha1.EgressShardClaim, + status metav1.ConditionStatus, + reason, message string, +) error { + claim.Status.ObservedGeneration = claim.Generation + meta.SetStatusCondition(&claim.Status.Conditions, metav1.Condition{ + Type: cloudv1alpha1.ConditionTypeReady, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: claim.Generation, + }) + if err := r.Status().Update(ctx, claim); err != nil { + return fmt.Errorf("update EgressShardClaim %s status: %w", + client.ObjectKeyFromObject(claim), err) + } + return nil +} + +// reportAttachment writes the egress readiness a consumer reads for this +// instance, on the attachment the interface's status is read from. +// +// It is patched rather than updated. The attachment's status has another +// writer, the controller that renders it, and a whole-object update from a +// copy read before that write would put stale values back over its fields. +func (r *EgressShardClaimReconciler) reportAttachment( + ctx context.Context, + attachment *cloudv1alpha1.VPCAttachment, + status metav1.ConditionStatus, + reason, message string, +) error { + condition := metav1.Condition{ + Type: cloudv1alpha1.ConditionTypeInternetEgressReady, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: attachment.Generation, + } + patch := client.MergeFrom(attachment.DeepCopy()) + if !meta.SetStatusCondition(&attachment.Status.Conditions, condition) { + return nil + } + if err := r.Status().Patch(ctx, attachment, patch); err != nil { + return fmt.Errorf("report internet egress on VPCAttachment %s: %w", + client.ObjectKeyFromObject(attachment), err) + } + return nil +} + +// SetupWithManager registers the reconciler with the manager. +func (r *EgressShardClaimReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&bgpv1alpha1.EgressShardClaim{}, + builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { + // A bound claim is never reconsidered on its own events. It is + // re-read when its attachment or its shard changes. + claim, ok := object.(*bgpv1alpha1.EgressShardClaim) + return ok && claim.Status.ShardRef == nil + }))). + Watches(&cloudv1alpha1.VPCAttachment{}, + handler.EnqueueRequestsFromMapFunc(claimForAttachment)). + Watches(&networkingv1alpha.NetworkContext{}, + handler.EnqueueRequestsFromMapFunc(r.claimsForNetworkContext)). + Watches(&bgpv1alpha1.EgressShard{}, + handler.EnqueueRequestsFromMapFunc(r.claimsForEgressShard)). + Named("egressshardclaim"). + Complete(r) +} + +// claimForAttachment maps an attachment to its one claim, which carries the +// same name. +func claimForAttachment(_ context.Context, object client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: client.ObjectKeyFromObject(object)}} +} + +// claimsForNetworkContext wakes every attachment of a network when its +// declaration changes, so a network disabled releases its records. +func (r *EgressShardClaimReconciler) claimsForNetworkContext( + ctx context.Context, object client.Object, +) []reconcile.Request { + var attachments cloudv1alpha1.VPCAttachmentList + if err := r.List(ctx, &attachments, client.InNamespace(object.GetNamespace())); err != nil { + return nil + } + requests := make([]reconcile.Request, 0, len(attachments.Items)) + for i := range attachments.Items { + if attachments.Items[i].Spec.VPC.Name != object.GetName() { + continue + } + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&attachments.Items[i]), + }) + } + return requests +} + +// claimsForEgressShard wakes the claims on a shard's node when it arrives, +// reports its identity, or leaves, and the claims recorded against it so a +// missing or draining shard is reported. +func (r *EgressShardClaimReconciler) claimsForEgressShard( + ctx context.Context, object client.Object, +) []reconcile.Request { + shard, ok := object.(*bgpv1alpha1.EgressShard) + if !ok { + return nil + } + var claims bgpv1alpha1.EgressShardClaimList + if err := r.List(ctx, &claims); err != nil { + return nil + } + requests := make([]reconcile.Request, 0, len(claims.Items)) + for i := range claims.Items { + claim := &claims.Items[i] + onNode := claim.Spec.NodeName == shard.Spec.TargetRef.Name + recorded := claim.Status.ShardRef != nil && claim.Status.ShardRef.Name == shard.Name + if !onNode && !recorded { + continue + } + requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(claim)}) + } + return requests +} diff --git a/internal/controller/egressshardclaim_controller_test.go b/internal/controller/egressshardclaim_controller_test.go new file mode 100644 index 0000000..102466f --- /dev/null +++ b/internal/controller/egressshardclaim_controller_test.go @@ -0,0 +1,459 @@ +/* +Copyright © 2026 Datum Technology, Inc. All rights reserved. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package controller + +import ( + "testing" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// egressAttachmentName is the attachment every egress test records, and +// therefore the name of the one claim recording it. +const egressAttachmentName = "web-eth0" + +func newBinder(t *testing.T, objects ...client.Object) (*EgressShardClaimReconciler, client.Client) { + t.Helper() + + scheme := runtime.NewScheme() + if err := networkingv1alpha.AddToScheme(scheme); err != nil { + t.Fatalf("build the networking scheme: %v", err) + } + if err := cloudv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("build the cloud scheme: %v", err) + } + if err := bgpv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("build the fabric scheme: %v", err) + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...). + WithStatusSubresource(&bgpv1alpha1.EgressShardClaim{}, &cloudv1alpha1.VPCAttachment{}). + Build() + return &EgressShardClaimReconciler{Client: fakeClient, Scheme: scheme}, fakeClient +} + +// newLandedAttachment is an attachment of the egress test network that has +// reported the node it landed on. +func newLandedAttachment(node string) *cloudv1alpha1.VPCAttachment { + attachment := &cloudv1alpha1.VPCAttachment{} + attachment.Namespace = egressTestNamespace + attachment.Name = egressAttachmentName + attachment.Spec.VPC = cloudv1alpha1.VPCRef{Name: "default-us-central-1"} + attachment.Spec.Interface.Name = "eth0" + attachment.Status.Node = node + return attachment +} + +// newEgressClaim is a record already written for the test attachment, bound +// to shardName or, with an empty name, still unbound. +func newEgressClaim(shardName string) *bgpv1alpha1.EgressShardClaim { + claim := &bgpv1alpha1.EgressShardClaim{} + claim.Namespace = egressTestNamespace + claim.Name = egressAttachmentName + claim.Spec = bgpv1alpha1.EgressShardClaimSpec{ + Attachment: bgpv1alpha1.EgressShardClaimAttachmentRef{Name: egressAttachmentName}, + VPC: bgpv1alpha1.EgressShardClaimVPCRef{Name: "default-us-central-1"}, + NodeName: egressTestNode, + Families: []bgpv1alpha1.EgressAddressFamily{bgpv1alpha1.EgressAddressFamilyIPv6}, + } + if shardName != "" { + claim.Labels = map[string]string{bgpv1alpha1.LabelEgressShardClaimShard: shardName} + claim.Status.ShardRef = &bgpv1alpha1.EgressShardClaimShardRef{ + Namespace: egressShardNamespace, + Name: shardName, + } + } + return claim +} + +func reconcileBinding(t *testing.T, r *EgressShardClaimReconciler) { + t.Helper() + key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressAttachmentName} + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: key}); err != nil { + t.Fatalf("reconcile the claim: %v", err) + } +} + +func readClaim(t *testing.T, cl client.Client) *bgpv1alpha1.EgressShardClaim { + t.Helper() + var claim bgpv1alpha1.EgressShardClaim + key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressAttachmentName} + if err := cl.Get(t.Context(), key, &claim); err != nil { + t.Fatalf("get the claim: %v", err) + } + return &claim +} + +func claimExists(t *testing.T, cl client.Client) bool { + t.Helper() + var claim bgpv1alpha1.EgressShardClaim + key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressAttachmentName} + return cl.Get(t.Context(), key, &claim) == nil +} + +// One claim per attachment that has landed on a node, carrying the node and +// the families the network declared and nothing this controller invented. +func TestBinderRecordsOncePerAttachment(t *testing.T) { + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) + + reconcileBinding(t, r) + + claim := readClaim(t, cl) + if claim.Spec.Attachment.Name != egressAttachmentName { + t.Errorf("attachment: got %q, want %q", claim.Spec.Attachment.Name, egressAttachmentName) + } + if claim.Spec.NodeName != egressTestNode { + t.Errorf("node: got %q, want %q", claim.Spec.NodeName, egressTestNode) + } + if len(claim.Spec.Families) != 1 || + claim.Spec.Families[0] != bgpv1alpha1.EgressAddressFamilyIPv6 { + t.Errorf("families: got %v, want [IPv6]", claim.Spec.Families) + } + if claim.Status.ShardRef != nil { + t.Errorf("the claim recorded %v in the pass that wrote it", claim.Status.ShardRef) + } + if !metav1.IsControlledBy(claim, newLandedAttachment(egressTestNode)) && + len(claim.OwnerReferences) == 0 { + t.Error("the claim is not owned by its attachment") + } +} + +// Nothing is recorded before the attachment reports where it landed. The node +// is the whole content of the record. +func TestBinderWaitsForTheAttachmentsNode(t *testing.T) { + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(""), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) + + reconcileBinding(t, r) + + if claimExists(t, cl) { + t.Error("a claim was written for an attachment on no known node") + } +} + +// The shard recorded is the one on the attachment's node and no other. +func TestBinderRecordsTheShardOnTheNode(t *testing.T) { + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode), + newEgressShard("worker-2-egress", "worker-2", "2001:db8:ff02::", "2001:db8:f00d::200"), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) + + reconcileBinding(t, r) + reconcileBinding(t, r) + + claim := readClaim(t, cl) + if claim.Status.ShardRef == nil { + t.Fatal("a shard on the node recorded nothing") + } + if claim.Status.ShardRef.Name != "worker-3-egress" { + t.Errorf("shard: got %q, want worker-3-egress", claim.Status.ShardRef.Name) + } + if claim.Status.ShardRef.Namespace != egressShardNamespace { + t.Errorf("shard namespace: got %q, want %q", claim.Status.ShardRef.Namespace, egressShardNamespace) + } + if got := claim.Labels[bgpv1alpha1.LabelEgressShardClaimShard]; got != "worker-3-egress" { + t.Errorf("shard label: got %q, want worker-3-egress", got) + } + assertClaimCondition(t, cl, metav1.ConditionTrue, bgpv1alpha1.EgressShardClaimReasonBound) + assertAttachmentCondition(t, cl, metav1.ConditionTrue, + networkingv1alpha.NetworkContextInternetEgressReasonReady) + + // The finalizer is the only state a binder puts on a shard, written before + // the record so a recorded shard is never held by nothing. + var shard bgpv1alpha1.EgressShard + key := client.ObjectKey{Namespace: egressShardNamespace, Name: "worker-3-egress"} + if err := cl.Get(t.Context(), key, &shard); err != nil { + t.Fatalf("get the recorded shard: %v", err) + } + if !controllerutil.ContainsFinalizer(&shard, finalizerEgressShardBinding) { + t.Error("the recorded shard is not held open") + } + if shard.Spec.ShardAddressIPv6 != "2001:db8:f00d::100" { + t.Errorf("the binder rewrote the shard's address: %q", shard.Spec.ShardAddressIPv6) + } +} + +// A node whose shard cannot be recorded leaves the claim unbound with the +// reason, and the consumer reads a fact about their own instance. +func TestBinderWaitsForAShardItCanUse(t *testing.T) { + mismatched := newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100") + mismatched.Status.ShardSID = "2001:db8:ffff::" + + tests := []struct { + name string + objects []client.Object + reason string + }{ + { + name: "no shard names the node", + reason: bgpv1alpha1.EgressShardClaimReasonNoShardOnNode, + }, + { + name: "the shard reports no identifier", + objects: []client.Object{ + newEgressShard("worker-3-egress", egressTestNode, "", "2001:db8:f00d::100")}, + reason: bgpv1alpha1.EgressShardClaimReasonShardNotReady, + }, + { + name: "the shard runs an identity its spec does not state", + objects: []client.Object{mismatched}, + reason: bgpv1alpha1.EgressShardClaimReasonShardMismatch, + }, + { + name: "the shard is being deleted", + objects: []client.Object{terminatingShard( + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100"))}, + reason: bgpv1alpha1.EgressShardClaimReasonShardTerminating, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + objects := append([]client.Object{ + newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode)}, test.objects...) + r, cl := newBinder(t, objects...) + + reconcileBinding(t, r) + reconcileBinding(t, r) + + claim := readClaim(t, cl) + if claim.Status.ShardRef != nil { + t.Fatalf("recorded %v, want nothing", claim.Status.ShardRef) + } + assertClaimCondition(t, cl, metav1.ConditionFalse, test.reason) + assertAttachmentCondition(t, cl, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonUnavailable) + }) + } +} + +// Egress that works and an address that cannot yet be stated are different +// facts, and the condition says which. +func TestBinderReportsThatNoAddressIsAllocatedYet(t *testing.T) { + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "")) + + reconcileBinding(t, r) + reconcileBinding(t, r) + + if readClaim(t, cl).Status.ShardRef == nil { + t.Fatal("a shard with an identifier and no address recorded nothing") + } + assertClaimCondition(t, cl, metav1.ConditionTrue, bgpv1alpha1.EgressShardClaimReasonBound) + assertAttachmentCondition(t, cl, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonAddressUnavailable) +} + +// A record is not remade while the attachment stays where it is. A second +// shard naming the node is an operator error, and the one already recorded +// stands. +func TestBinderNeverRebinds(t *testing.T) { + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode), + newEgressShard("worker-3-egress-b", egressTestNode, "2001:db8:ff02::", "2001:db8:f00d::200")) + + reconcileBinding(t, r) + reconcileBinding(t, r) + if got := readClaim(t, cl).Status.ShardRef; got == nil || got.Name != "worker-3-egress-b" { + t.Fatalf("got %v, want worker-3-egress-b", got) + } + + earlier := newEgressShard("worker-3-egress-a", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100") + if err := cl.Create(t.Context(), earlier); err != nil { + t.Fatalf("commission a shard sorting earlier: %v", err) + } + reconcileBinding(t, r) + if got := readClaim(t, cl).Status.ShardRef; got == nil || got.Name != "worker-3-egress-b" { + t.Errorf("got %v, want the shard already recorded", got) + } +} + +// Egress withdrawn is a claim released, which lets the shard go. +func TestBinderReleasesTheClaimWhenEgressIsWithdrawn(t *testing.T) { + unprojected := newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled) + unprojected.Spec.Egress = nil + + tests := []struct { + name string + networkContext *networkingv1alpha.NetworkContext + }{ + {"egress disabled", newEgressContext(networkingv1alpha.NetworkInternetEgressDisabled)}, + {"intent never projected", unprojected}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r, cl := newBinder(t, test.networkContext, newLandedAttachment(egressTestNode), + newEgressClaim("worker-3-egress"), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) + + reconcileBinding(t, r) + + if claimExists(t, cl) { + t.Error("the claim survived the egress that justified it") + } + }) + } +} + +// An attachment that is gone takes its record with it, rather than holding a +// shard open for an instance that no longer exists. +func TestBinderReleasesTheClaimWhenTheAttachmentLeaves(t *testing.T) { + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newEgressClaim("worker-3-egress"), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) + + reconcileBinding(t, r) + + if claimExists(t, cl) { + t.Error("the claim survived its attachment") + } +} + +// An attachment that moved nodes is a different record. The spec is immutable, +// so the stale one is released and the next pass writes the new one. +func TestBinderReplacesTheRecordWhenTheAttachmentMovesNodes(t *testing.T) { + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment("worker-4"), + newEgressClaim("worker-3-egress"), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100"), + newEgressShard("worker-4-egress", "worker-4", "2001:db8:ff04::", "2001:db8:f00d::400")) + + reconcileBinding(t, r) + if claimExists(t, cl) { + t.Fatal("a record for the node the attachment left was kept") + } + + reconcileBinding(t, r) + reconcileBinding(t, r) + claim := readClaim(t, cl) + if claim.Spec.NodeName != "worker-4" { + t.Errorf("node: got %q, want worker-4", claim.Spec.NodeName) + } + if claim.Status.ShardRef == nil || claim.Status.ShardRef.Name != "worker-4-egress" { + t.Errorf("shard: got %v, want worker-4-egress", claim.Status.ShardRef) + } +} + +// A label lost to an edit would hide an attachment from the query a shard's +// consumer set is counted by, so it is re-asserted on every pass. +func TestBinderRepairsTheShardLabel(t *testing.T) { + claim := newEgressClaim("worker-3-egress") + claim.Labels = nil + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode), claim, + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) + + reconcileBinding(t, r) + + if got := readClaim(t, cl).Labels[bgpv1alpha1.LabelEgressShardClaimShard]; got != "worker-3-egress" { + t.Errorf("shard label: got %q, want worker-3-egress", got) + } +} + +// A recorded shard that vanished is said on both objects. Nothing rebinds. +func TestBinderReportsAMissingShard(t *testing.T) { + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode), + newEgressClaim("worker-3-egress")) + + reconcileBinding(t, r) + + assertClaimCondition(t, cl, metav1.ConditionFalse, bgpv1alpha1.EgressShardClaimReasonShardMissing) + assertAttachmentCondition(t, cl, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonUnavailable) +} + +// Degraded means egress works for some declared families and not others. Only +// one family is accepted anywhere on this path, so nothing may write it. +func TestBinderNeverReportsDegraded(t *testing.T) { + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) + + reconcileBinding(t, r) + reconcileBinding(t, r) + + condition := attachmentEgressCondition(t, cl) + if condition == nil { + t.Fatal("the attachment reports no egress readiness") + } + if condition.Reason == networkingv1alpha.NetworkContextInternetEgressReasonDegraded { + t.Error("Degraded was reported for a path that accepts one address family") + } +} + +func terminatingShard(shard *bgpv1alpha1.EgressShard) *bgpv1alpha1.EgressShard { + shard.Finalizers = []string{finalizerEgressShardBinding} + deletion := metav1.Now() + shard.DeletionTimestamp = &deletion + return shard +} + +func assertClaimCondition( + t *testing.T, cl client.Client, status metav1.ConditionStatus, reason string, +) { + t.Helper() + condition := meta.FindStatusCondition(readClaim(t, cl).Status.Conditions, + cloudv1alpha1.ConditionTypeReady) + if condition == nil { + t.Fatal("the claim reports no Ready condition") + } + if condition.Status != status || condition.Reason != reason { + t.Errorf("claim Ready: got %s/%s, want %s/%s", + condition.Status, condition.Reason, status, reason) + } +} + +func attachmentEgressCondition(t *testing.T, cl client.Client) *metav1.Condition { + t.Helper() + var attachment cloudv1alpha1.VPCAttachment + key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressAttachmentName} + if err := cl.Get(t.Context(), key, &attachment); err != nil { + t.Fatalf("get the attachment: %v", err) + } + return meta.FindStatusCondition(attachment.Status.Conditions, + cloudv1alpha1.ConditionTypeInternetEgressReady) +} + +func assertAttachmentCondition( + t *testing.T, cl client.Client, status metav1.ConditionStatus, reason string, +) { + t.Helper() + condition := attachmentEgressCondition(t, cl) + if condition == nil { + t.Fatal("the attachment reports no InternetEgressReady condition") + } + if condition.Status != status || condition.Reason != reason { + t.Errorf("attachment InternetEgressReady: got %s/%s, want %s/%s", + condition.Status, condition.Reason, status, reason) + } +} diff --git a/internal/controller/networkinterface_controller.go b/internal/controller/networkinterface_controller.go index c827681..b289375 100644 --- a/internal/controller/networkinterface_controller.go +++ b/internal/controller/networkinterface_controller.go @@ -383,7 +383,7 @@ func (r *NetworkInterfaceReconciler) resolveInternetEgress( if attachment.Status.Node == "" { return resolved, nil } - shard, err := r.egressShardOnNode(ctx, attachment.Status.Node) + shard, err := egressShardOnNode(ctx, r.Client, attachment.Status.Node) if err != nil { return nil, err } @@ -433,11 +433,11 @@ func internetEgressIntent( // operator's to choose, and the node reference is what ties one to a node. Two // shards naming one node is an operator error, and the first by name is taken // so that every attachment on that node computes the same answer. -func (r *NetworkInterfaceReconciler) egressShardOnNode( - ctx context.Context, node string, +func egressShardOnNode( + ctx context.Context, reader client.Reader, node string, ) (*bgpv1alpha1.EgressShard, error) { var shards bgpv1alpha1.EgressShardList - if err := r.List(ctx, &shards, client.InNamespace(galactic.SystemNamespace)); err != nil { + if err := reader.List(ctx, &shards, client.InNamespace(galactic.SystemNamespace)); err != nil { return nil, fmt.Errorf("list egress shards: %w", err) } var found *bgpv1alpha1.EgressShard