From 892016a36e2daf734c259da8da8e37121ecaddf5 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 17 Sep 2026 23:25:09 -0500 Subject: [PATCH 1/4] feat: Build a cell's egress shards from a pool Nothing created an egress shard object. Every shard in existence was hand-written next to a per-node DaemonSet overlay in a container lab, so a cell's egress tier existed only where someone had typed it out, and the pool label the shard type exports had nothing stamping it for a class's selector to read. An operator now declares the pool once per cell: the namespace its shards live in, the nodes commissioned into it, and the labels a class selects on. The builder in the cell expands that into one shard per selected node. Commissioning stays an operator's act and provisioning is not demand-driven. A shard is unusable until its SRv6 identifier exists, that identifier is still operator-supplied process configuration on the node with no allocator behind it and a documented collision hazard, so a shard created in response to a network's demand would attach, translate nothing, and then be skipped by the very selector meant to find it. Key changes: - Add EgressShardPool, cluster-scoped beside EgressShardParameters, whose schema refuses an empty node selector and refuses to stamp the family labels that restate an address assignment a pool does not make - Add a node opt-in label, because the label every translating node carries is compute's own role label and selecting on it would build a shard per compute node, making the egress address per-node - Build shards in name order with the pool label set from the pool's own name, so an operator's labels cannot point it elsewhere - Create and never rewrite: a shard's address is write-once and its labels are what moves traffic, so an existing shard is left as found and the builder's shard access gains create alone - Leave a shard alone when its pool is deleted or its node stops matching, rather than stranding the return traffic of every flow it translates - Report a pool that selected no node, which otherwise looks exactly like a working one until a network asks to egress through it - Repin the network module to its branch head Co-Authored-By: Claude Opus 5 (1M context) --- api/v1alpha1/egressshardpool_types.go | 162 ++++++++++ api/v1alpha1/zz_generated.deepcopy.go | 104 +++++++ cmd/main.go | 6 + .../cloud.datumapis.com_egressshardpools.yaml | 231 +++++++++++++++ config/crd/kustomization.yaml | 3 + config/rbac/role.yaml | 36 ++- docs/api/vpc.md | 77 +++++ go.mod | 2 +- go.sum | 4 +- .../controller/egressshardpool_controller.go | 243 +++++++++++++++ .../egressshardpool_controller_test.go | 280 ++++++++++++++++++ test/e2e/Taskfile.yaml | 1 + .../chainsaw-test.yaml | 121 ++++++++ 13 files changed, 1258 insertions(+), 12 deletions(-) create mode 100644 api/v1alpha1/egressshardpool_types.go create mode 100644 config/crd/cloud.datumapis.com_egressshardpools.yaml create mode 100644 internal/controller/egressshardpool_controller.go create mode 100644 internal/controller/egressshardpool_controller_test.go create mode 100644 test/e2e/tests/egress-shard-pool-crd-schema/chainsaw-test.yaml diff --git a/api/v1alpha1/egressshardpool_types.go b/api/v1alpha1/egressshardpool_types.go new file mode 100644 index 0000000..996be28 --- /dev/null +++ b/api/v1alpha1/egressshardpool_types.go @@ -0,0 +1,162 @@ +/* +Copyright © 2026 Datum Technology, Inc. All rights reserved. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// LabelNodeEgressPool is the label a node carries to be commissioned into an +// egress shard pool, set to the pool's own name. +// +// It exists because the label a translating node already carries is not usable +// for this. galactic-nat is compute's role-differentiating component: every +// compute node runs it unconditionally under +// galactic.datumapis.com/node=compute, and its DaemonSet says in as many words +// that no separate opt-in capability label exists. Selecting on that label +// would build one shard per compute node, which makes the egress address +// per-node and dissolves the shared address the whole model rests on. +// +// A pool's node selector is an ordinary selector and an operator may write any +// other requirement into it. This key is the one to reach for, and the reason +// it has to be a new one. +const LabelNodeEgressPool = "cloud.datumapis.com/egress-pool" + +// EgressShardPoolSpec declares the egress shards a cell has, by naming the +// nodes that translate for it. +type EgressShardPoolSpec struct { + // ShardNamespace is the namespace the shards this pool builds live in. + // + // It is required, for the same reason EgressShardParameters requires one: + // the namespace names the nodes that every network on a class leaves the + // platform through, which is worth an operator stating rather than + // defaulting to wherever the data plane happens to keep its objects. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + ShardNamespace string `json:"shardNamespace"` + + // NodeSelector selects the nodes this pool builds a shard for. Reach for + // LabelNodeEgressPool; its doc comment explains why the label a translating + // node already carries cannot be used here. + // + // An empty selector is refused rather than treated as "every node". A + // selector that matched every node would build a shard per node, which + // makes the egress address per-node and dissolves the shared address. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:XValidation:rule="(has(self.matchLabels) && size(self.matchLabels) > 0) || (has(self.matchExpressions) && size(self.matchExpressions) > 0)",message="nodeSelector must state at least one requirement; an empty selector would build a shard on every node" + NodeSelector metav1.LabelSelector `json:"nodeSelector"` + + // ShardLabels are stamped on each shard this pool builds, on top of the + // pool label the builder always writes. + // + // They are what a class's shard selector matches, so a pool that stamps no + // cell is selected together with another cell's shards. The family labels + // belong to whoever assigns the addresses and are deliberately not settable + // here: they restate an assignment this pool does not make, and a pool that + // claimed a family its shards hold no address for would be selected for + // traffic that then translates nothing. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MaxProperties=16 + // +kubebuilder:validation:XValidation:rule="!self.exists(k, k.startsWith('network.datumapis.com/egress-ipv'))",message="the egress family labels are set by whoever assigns a shard's addresses, not by a pool" + ShardLabels map[string]string `json:"shardLabels,omitempty"` +} + +// EgressShardPoolStatus reports what this pool built. +// +// It reports no shard names and no counts of the networks using them. Which +// shards a pool built is answered by listing the pool label in the shard +// namespace, and which networks a shard serves by listing the claims bound to +// it — neither is a number stored here to fall out of step. +type EgressShardPoolStatus struct { + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // +listType=map + // +listMapKey=type + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +const ( + // EgressShardPoolReasonShardsBuilt means every node this pool selects has a + // shard object. + EgressShardPoolReasonShardsBuilt = "ShardsBuilt" + + // EgressShardPoolReasonNoNodeSelected means the selector matched no node, + // so this pool builds nothing. It is reported rather than logged: a pool + // whose label an operator never set on a node looks exactly like a pool + // that is working until a network asks to egress through it. + EgressShardPoolReasonNoNodeSelected = "NoNodeSelected" +) + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:printcolumn:name="Shard Namespace",type="string",JSONPath=".spec.shardNamespace" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// EgressShardPool is an operator's declaration of the egress shards one cell +// has. The builder expands it into one EgressShard per selected node, which is +// the object a class's selector then matches. +// +// It replaces a hand-written shard object per node and nothing more. It does +// not provision on demand: a shard is unusable until its SRv6 identifier +// exists, that identifier is still operator-supplied process configuration on +// the node with no allocator behind it, and a shard created in response to a +// network's demand would therefore attach, translate nothing, and be skipped by +// the very selector meant to find it. Commissioning a node stays an operator's +// act; this only removes the YAML that act used to require. +// +// It is cluster-scoped like EgressShardParameters: the content is an operator's +// statement about the cell's own data plane rather than anything belonging to +// one tenant, and no consumer reads or writes one. +type EgressShardPool struct { + metav1.TypeMeta `json:",inline"` + + // metadata is a standard object metadata. The name is stamped on every + // shard this pool builds as the pool label's value, so it is what a class's + // selector names. + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // spec declares the shards this pool has + // +required + Spec EgressShardPoolSpec `json:"spec"` + + // status reports what this pool built + // +optional + Status EgressShardPoolStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// EgressShardPoolList contains a list of EgressShardPool. +type EgressShardPoolList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []EgressShardPool `json:"items"` +} + +func init() { + SchemeBuilder.Register(&EgressShardPool{}, &EgressShardPoolList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index f75c2fd..488a4bd 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -100,6 +100,110 @@ func (in *EgressShardParametersSpec) DeepCopy() *EgressShardParametersSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressShardPool) DeepCopyInto(out *EgressShardPool) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardPool. +func (in *EgressShardPool) DeepCopy() *EgressShardPool { + if in == nil { + return nil + } + out := new(EgressShardPool) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressShardPool) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressShardPoolList) DeepCopyInto(out *EgressShardPoolList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]EgressShardPool, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardPoolList. +func (in *EgressShardPoolList) DeepCopy() *EgressShardPoolList { + if in == nil { + return nil + } + out := new(EgressShardPoolList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressShardPoolList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressShardPoolSpec) DeepCopyInto(out *EgressShardPoolSpec) { + *out = *in + in.NodeSelector.DeepCopyInto(&out.NodeSelector) + if in.ShardLabels != nil { + in, out := &in.ShardLabels, &out.ShardLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardPoolSpec. +func (in *EgressShardPoolSpec) DeepCopy() *EgressShardPoolSpec { + if in == nil { + return nil + } + out := new(EgressShardPoolSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressShardPoolStatus) DeepCopyInto(out *EgressShardPoolStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardPoolStatus. +func (in *EgressShardPoolStatus) DeepCopy() *EgressShardPoolStatus { + if in == nil { + return nil + } + out := new(EgressShardPoolStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InternetEgressSourceAddress) DeepCopyInto(out *InternetEgressSourceAddress) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index f499299..2976616 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -117,6 +117,12 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "NetworkContext") os.Exit(1) } + if err := (&controller.EgressShardPoolReconciler{ + Client: mgr.GetClient(), Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "EgressShardPool") + os.Exit(1) + } if err := (&controller.NetworkInterfaceReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), APIReader: mgr.GetAPIReader(), AttachmentMode: attachmentMode, diff --git a/config/crd/cloud.datumapis.com_egressshardpools.yaml b/config/crd/cloud.datumapis.com_egressshardpools.yaml new file mode 100644 index 0000000..7e114c3 --- /dev/null +++ b/config/crd/cloud.datumapis.com_egressshardpools.yaml @@ -0,0 +1,231 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: egressshardpools.cloud.datumapis.com +spec: + group: cloud.datumapis.com + names: + kind: EgressShardPool + listKind: EgressShardPoolList + plural: egressshardpools + singular: egressshardpool + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.shardNamespace + name: Shard Namespace + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + EgressShardPool is an operator's declaration of the egress shards one cell + has. The builder expands it into one EgressShard per selected node, which is + the object a class's selector then matches. + + It replaces a hand-written shard object per node and nothing more. It does + not provision on demand: a shard is unusable until its SRv6 identifier + exists, that identifier is still operator-supplied process configuration on + the node with no allocator behind it, and a shard created in response to a + network's demand would therefore attach, translate nothing, and be skipped by + the very selector meant to find it. Commissioning a node stays an operator's + act; this only removes the YAML that act used to require. + + It is cluster-scoped like EgressShardParameters: the content is an operator's + statement about the cell's own data plane rather than anything belonging to + one tenant, and no consumer reads or writes one. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec declares the shards this pool has + properties: + nodeSelector: + description: |- + NodeSelector selects the nodes this pool builds a shard for. Reach for + LabelNodeEgressPool; its doc comment explains why the label a translating + node already carries cannot be used here. + + An empty selector is refused rather than treated as "every node". A + selector that matched every node would build a shard per node, which + makes the egress address per-node and dissolves the shared address. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: nodeSelector must state at least one requirement; an empty + selector would build a shard on every node + rule: (has(self.matchLabels) && size(self.matchLabels) > 0) || (has(self.matchExpressions) + && size(self.matchExpressions) > 0) + shardLabels: + additionalProperties: + type: string + description: |- + ShardLabels are stamped on each shard this pool builds, on top of the + pool label the builder always writes. + + They are what a class's shard selector matches, so a pool that stamps no + cell is selected together with another cell's shards. The family labels + belong to whoever assigns the addresses and are deliberately not settable + here: they restate an assignment this pool does not make, and a pool that + claimed a family its shards hold no address for would be selected for + traffic that then translates nothing. + maxProperties: 16 + type: object + x-kubernetes-validations: + - message: the egress family labels are set by whoever assigns a shard's + addresses, not by a pool + rule: '!self.exists(k, k.startsWith(''network.datumapis.com/egress-ipv''))' + shardNamespace: + description: |- + ShardNamespace is the namespace the shards this pool builds live in. + + It is required, for the same reason EgressShardParameters requires one: + the namespace names the nodes that every network on a class leaves the + platform through, which is worth an operator stating rather than + defaulting to wherever the data plane happens to keep its objects. + maxLength: 63 + minLength: 1 + type: string + required: + - nodeSelector + - shardNamespace + type: object + status: + description: status reports what this pool built + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + format: int64 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 390e02d..7c43702 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -9,3 +9,6 @@ resources: # Cell-local: an operator writes one per InternetEgressClass this cell serves, # and only the controller in the cell reads it. - cloud.datumapis.com_egressshardparameters.yaml + # Cell-local too: an operator declares the shards the cell has, and the + # builder in the cell expands it into the shard objects a class selects. + - cloud.datumapis.com_egressshardpools.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 23aaa1c..ab1f087 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -12,9 +12,9 @@ rules: - create - patch - apiGroups: - - cloud.datumapis.com + - "" resources: - - egressshardparameters + - nodes verbs: - get - list @@ -22,26 +22,36 @@ rules: - apiGroups: - cloud.datumapis.com resources: - - networkfabricidentities - - vpcattachments - - vpcs + - egressshardparameters + - egressshardpools verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - cloud.datumapis.com resources: + - egressshardpools/status - vpcattachments/status - vpcs/status verbs: - get - patch - update +- apiGroups: + - cloud.datumapis.com + resources: + - networkfabricidentities + - vpcattachments + - vpcs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - compute.datumapis.com resources: @@ -79,8 +89,16 @@ rules: resources: - bgpadvertisements - bgprouters + verbs: + - get + - list + - watch +- apiGroups: + - network.datumapis.com + resources: - egressshards verbs: + - create - get - list - watch diff --git a/docs/api/vpc.md b/docs/api/vpc.md index 191fdad..6a64e1d 100644 --- a/docs/api/vpc.md +++ b/docs/api/vpc.md @@ -10,6 +10,7 @@ Package v1alpha1 contains API Schema definitions for the cloud.datumapis.com/v1a ### Resource Types - [EgressShardParameters](#egressshardparameters) +- [EgressShardPool](#egressshardpool) - [NetworkFabricIdentity](#networkfabricidentity) - [VPC](#vpc) - [VPCAttachment](#vpcattachment) @@ -65,6 +66,82 @@ _Appears in:_ | `shardSelector` _[LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#labelselector-v1-meta)_ | ShardSelector selects the EgressShards a network on this class egresses
through, by the network.datumapis.com/egress-* labels an operator sets
on them.
An empty selector matches every shard in the namespace, which sends a
consumer's traffic out of an arbitrary cell. Egress is realized per
cell, so a selector is expected to pin a cell and a pool.
The selector runs one way, as the only binding between a class and the
shards serving it: a shard names nothing that selects it, which is what
keeps the data-plane API group independent of the consumer-facing one. | | Required: \{\}
| +#### EgressShardPool + + + +EgressShardPool is an operator's declaration of the egress shards one cell +has. The builder expands it into one EgressShard per selected node, which is +the object a class's selector then matches. + +It replaces a hand-written shard object per node and nothing more. It does +not provision on demand: a shard is unusable until its SRv6 identifier +exists, that identifier is still operator-supplied process configuration on +the node with no allocator behind it, and a shard created in response to a +network's demand would therefore attach, translate nothing, and be skipped by +the very selector meant to find it. Commissioning a node stays an operator's +act; this only removes the YAML that act used to require. + +It is cluster-scoped like EgressShardParameters: the content is an operator's +statement about the cell's own data plane rather than anything belonging to +one tenant, and no consumer reads or writes one. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `cloud.datumapis.com/v1alpha1` | | | +| `kind` _string_ | `EgressShardPool` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[EgressShardPoolSpec](#egressshardpoolspec)_ | spec declares the shards this pool has | | | +| `status` _[EgressShardPoolStatus](#egressshardpoolstatus)_ | status reports what this pool built | | | + + +#### EgressShardPoolSpec + + + +EgressShardPoolSpec declares the egress shards a cell has, by naming the +nodes that translate for it. + + + +_Appears in:_ +- [EgressShardPool](#egressshardpool) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `shardNamespace` _string_ | ShardNamespace is the namespace the shards this pool builds live in.
It is required, for the same reason EgressShardParameters requires one:
the namespace names the nodes that every network on a class leaves the
platform through, which is worth an operator stating rather than
defaulting to wherever the data plane happens to keep its objects. | | MaxLength: 63
MinLength: 1
Required: \{\}
| +| `nodeSelector` _[LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#labelselector-v1-meta)_ | NodeSelector selects the nodes this pool builds a shard for. Reach for
LabelNodeEgressPool; its doc comment explains why the label a translating
node already carries cannot be used here.
An empty selector is refused rather than treated as "every node". A
selector that matched every node would build a shard per node, which
makes the egress address per-node and dissolves the shared address. | | Required: \{\}
| +| `shardLabels` _object (keys:string, values:string)_ | ShardLabels are stamped on each shard this pool builds, on top of the
pool label the builder always writes.
They are what a class's shard selector matches, so a pool that stamps no
cell is selected together with another cell's shards. The family labels
belong to whoever assigns the addresses and are deliberately not settable
here: they restate an assignment this pool does not make, and a pool that
claimed a family its shards hold no address for would be selected for
traffic that then translates nothing. | | MaxProperties: 16
Optional: \{\}
| + + +#### EgressShardPoolStatus + + + +EgressShardPoolStatus reports what this pool built. + +It reports no shard names and no counts of the networks using them. Which +shards a pool built is answered by listing the pool label in the shard +namespace, and which networks a shard serves by listing the claims bound to +it — neither is a number stored here to fall out of step. + + + +_Appears in:_ +- [EgressShardPool](#egressshardpool) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `observedGeneration` _integer_ | | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | | | | + + #### IPAddress _Underlying type:_ _string_ diff --git a/go.mod b/go.mod index 52a4709..b1a1628 100644 --- a/go.mod +++ b/go.mod @@ -79,6 +79,6 @@ require ( sigs.k8s.io/yaml v1.6.0 // indirect ) -replace go.datum.net/network => github.com/datum-cloud/network v0.1.1-0.20260917203135-45eb71a1eb0c +replace go.datum.net/network => github.com/datum-cloud/network v0.1.1-0.20260918024131-d31482d46c4f replace go.datum.net/network-services-operator => github.com/datum-cloud/network-services-operator v0.27.2-0.20260917225730-eccf0e8922b2 diff --git a/go.sum b/go.sum index 32ea3ee..5d9899c 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/datum-cloud/network v0.1.1-0.20260917203135-45eb71a1eb0c h1:NT1bxxVHTzqathHdkh4zKmBqlLLXkX5N39+IjcWeitY= -github.com/datum-cloud/network v0.1.1-0.20260917203135-45eb71a1eb0c/go.mod h1:dqzM8WZczbiZ9bCvsxjkoI10GJqQ24NVWnc9boXgOkE= +github.com/datum-cloud/network v0.1.1-0.20260918024131-d31482d46c4f h1:eEuvQIsMzHc97FF5diQ9DFrBOFtZlqFRDLVChLoeH1E= +github.com/datum-cloud/network v0.1.1-0.20260918024131-d31482d46c4f/go.mod h1:dqzM8WZczbiZ9bCvsxjkoI10GJqQ24NVWnc9boXgOkE= github.com/datum-cloud/network-services-operator v0.27.2-0.20260917225730-eccf0e8922b2 h1:2yKJV4XRmoQMNM+VrOJdUgNkP1pM5z6l2P/2qF+K5yI= github.com/datum-cloud/network-services-operator v0.27.2-0.20260917225730-eccf0e8922b2/go.mod h1:9nuuBWdrkdnIBMaWJsWM3j4CcbKJIF1GDuCmAjdpIHo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/controller/egressshardpool_controller.go b/internal/controller/egressshardpool_controller.go new file mode 100644 index 0000000..3518481 --- /dev/null +++ b/internal/controller/egressshardpool_controller.go @@ -0,0 +1,243 @@ +/* +Copyright © 2026 Datum Technology, Inc. All rights reserved. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package controller + +import ( + "context" + "fmt" + "maps" + "slices" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// EgressShardPoolReconciler builds the egress shard objects a cell has, one per +// node an operator commissioned into a pool. +// +// It creates and never rewrites. A shard's addresses are write-once because the +// datapath claims a reply by exact match against the address it translates to, +// so a shard holding the wrong value is deleted and recreated at a moment +// someone chose rather than edited underneath live flows. The same reasoning +// covers its labels, which is what a class's selector matches: moving a shard +// between pools by relabelling it moves traffic silently. This reconciler +// therefore leaves an existing shard exactly as it found it, and its shard +// access is get, list, watch and create. +// +// Decommissioning is likewise not its job. A node that stops matching keeps its +// shard, because deleting one strands the return traffic of every flow it is +// translating and nothing here knows whether that is what an operator meant. +type EgressShardPoolReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=egressshardpools,verbs=get;list;watch +// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=egressshardpools/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch +// +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshards,verbs=get;list;watch;create + +func (r *EgressShardPoolReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var pool cloudv1alpha1.EgressShardPool + if err := r.Get(ctx, req.NamespacedName, &pool); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if !pool.DeletionTimestamp.IsZero() { + // The shards outlive the declaration that built them. A pool deleted by + // an operator who is reorganizing their cell must not take the + // translation of every network on it with it. + return ctrl.Result{}, nil + } + + nodes, err := r.selectedNodes(ctx, &pool) + if err != nil { + return ctrl.Result{}, err + } + if len(nodes) == 0 { + return ctrl.Result{}, r.publishPoolStatus(ctx, &pool, metav1.ConditionFalse, + cloudv1alpha1.EgressShardPoolReasonNoNodeSelected, + "No node carries the labels this pool selects, so it builds no shard") + } + + for _, node := range nodes { + name := shardNameForNode(&pool, node) + if problems := validation.IsDNS1123Subdomain(name); len(problems) > 0 { + return ctrl.Result{}, r.publishPoolStatus(ctx, &pool, metav1.ConditionFalse, + "ShardNameInvalid", + fmt.Sprintf("Pool %q and node %q derive shard name %q, which is not a valid object name: %s", + pool.Name, node, name, problems[0])) + } + if err := r.buildShard(ctx, &pool, node, name); err != nil { + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, r.publishPoolStatus(ctx, &pool, metav1.ConditionTrue, + cloudv1alpha1.EgressShardPoolReasonShardsBuilt, + fmt.Sprintf("Every one of the %d nodes this pool selects has a shard in namespace %s", + len(nodes), pool.Spec.ShardNamespace)) +} + +// selectedNodes are the nodes this pool commissions, in name order so the +// shards are built in a stable order and a partial pass resumes where it left +// off rather than somewhere else. +func (r *EgressShardPoolReconciler) selectedNodes( + ctx context.Context, pool *cloudv1alpha1.EgressShardPool, +) ([]string, error) { + selector, err := metav1.LabelSelectorAsSelector(&pool.Spec.NodeSelector) + if err != nil { + return nil, fmt.Errorf("parse the node selector on EgressShardPool %s: %w", pool.Name, err) + } + // An empty selector is refused by the schema, but a selector parsed from an + // object written before that rule reaches everything selects every node in + // the cell, which is a shard per compute node. Refuse it here too. + if selector.Empty() { + return nil, nil + } + + var nodes corev1.NodeList + if err := r.List(ctx, &nodes, client.MatchingLabelsSelector{Selector: selector}); err != nil { + return nil, fmt.Errorf("list the nodes EgressShardPool %s selects: %w", pool.Name, err) + } + + names := make([]string, 0, len(nodes.Items)) + for i := range nodes.Items { + names = append(names, nodes.Items[i].Name) + } + slices.Sort(names) + return names, nil +} + +// buildShard creates the shard for one node if it does not exist yet. +// +// Nothing here writes an address. The controller holding the addressing-service +// credential claims one and writes it into this spec afterwards, which is what +// keeps that credential off every translating node — so a shard is born with a +// target and nothing else, and says so as Programmed=False/AddressUnassigned +// until the claim lands. +func (r *EgressShardPoolReconciler) buildShard( + ctx context.Context, pool *cloudv1alpha1.EgressShardPool, node, name string, +) error { + key := client.ObjectKey{Namespace: pool.Spec.ShardNamespace, Name: name} + var existing bgpv1alpha1.EgressShard + switch err := r.Get(ctx, key, &existing); { + case err == nil: + return nil + case !apierrors.IsNotFound(err): + return fmt.Errorf("get EgressShard %s: %w", key, err) + } + + shard := &bgpv1alpha1.EgressShard{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: key.Namespace, + Name: key.Name, + Labels: shardLabels(pool), + }, + Spec: bgpv1alpha1.EgressShardSpec{ + TargetRef: bgpv1alpha1.TargetRef{Kind: "Node", Name: node}, + }, + } + // No owner reference: the pool is cluster-scoped and the shard is not, and + // a shard must outlive the declaration that built it in any case. + if err := r.Create(ctx, shard); err != nil { + if apierrors.IsAlreadyExists(err) { + return nil + } + return fmt.Errorf("create EgressShard %s for node %s: %w", key, node, err) + } + logf.FromContext(ctx).Info("built an egress shard for a commissioned node", + "pool", pool.Name, "shard", key, "node", node) + return nil +} + +// shardLabels are what a class's shard selector matches on the built shard. The +// pool label carries the pool's own name, so it cannot drift from the object +// that stamped it, and an operator's own labels cannot overwrite it. +func shardLabels(pool *cloudv1alpha1.EgressShardPool) map[string]string { + labels := make(map[string]string, len(pool.Spec.ShardLabels)+1) + maps.Copy(labels, pool.Spec.ShardLabels) + labels[bgpv1alpha1.LabelEgressShardPool] = pool.Name + return labels +} + +// shardNameForNode derives a shard's name from its pool and its node, so the +// same pass finds the shard it built last time without reading a reference +// anyone has to keep in step. +func shardNameForNode(pool *cloudv1alpha1.EgressShardPool, node string) string { + return pool.Name + "-" + node +} + +func (r *EgressShardPoolReconciler) publishPoolStatus( + ctx context.Context, + pool *cloudv1alpha1.EgressShardPool, + status metav1.ConditionStatus, + reason, message string, +) error { + pool.Status.ObservedGeneration = pool.Generation + meta.SetStatusCondition(&pool.Status.Conditions, metav1.Condition{ + Type: cloudv1alpha1.ConditionTypeReady, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: pool.Generation, + }) + if err := r.Status().Update(ctx, pool); err != nil { + return fmt.Errorf("update EgressShardPool %s status: %w", pool.Name, err) + } + return nil +} + +// SetupWithManager registers the reconciler with the manager. +func (r *EgressShardPoolReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&cloudv1alpha1.EgressShardPool{}). + Watches(&corev1.Node{}, handler.EnqueueRequestsFromMapFunc(r.poolsForNode)). + Named("egressshardpool"). + Complete(r) +} + +// poolsForNode wakes every pool when a node is labelled, so commissioning a +// node builds its shard immediately instead of waiting out a poll interval that +// does not exist. Which pool a node joined is a label the pool selects on +// rather than a field, so every pool is asked rather than one being looked up. +func (r *EgressShardPoolReconciler) poolsForNode(ctx context.Context, _ client.Object) []reconcile.Request { + var pools cloudv1alpha1.EgressShardPoolList + if err := r.List(ctx, &pools); err != nil { + return nil + } + + requests := make([]reconcile.Request, 0, len(pools.Items)) + for i := range pools.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&pools.Items[i]), + }) + } + return requests +} diff --git a/internal/controller/egressshardpool_controller_test.go b/internal/controller/egressshardpool_controller_test.go new file mode 100644 index 0000000..1bd3e95 --- /dev/null +++ b/internal/controller/egressshardpool_controller_test.go @@ -0,0 +1,280 @@ +/* +Copyright © 2026 Datum Technology, Inc. All rights reserved. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package controller + +import ( + "slices" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +const egressPoolName = "shared" + +// newPoolReconciler builds a builder over a cell holding the pools and nodes +// given, so a test states only what it is about. +func newPoolReconciler(t *testing.T, objects ...client.Object) (*EgressShardPoolReconciler, client.Client) { + t.Helper() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatalf("build the core scheme: %v", err) + } + if err := cloudv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("build the cloud scheme: %v", err) + } + if err := bgpv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("build the fabric scheme: %v", err) + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...). + WithStatusSubresource(&cloudv1alpha1.EgressShardPool{}).Build() + return &EgressShardPoolReconciler{Client: fakeClient, Scheme: scheme}, fakeClient +} + +// newEgressPool is a cell's pool declaration: the namespace its shards live in, +// the nodes an operator commissioned, and the labels a class selects on. +func newEgressPool() *cloudv1alpha1.EgressShardPool { + pool := &cloudv1alpha1.EgressShardPool{} + pool.Name = egressPoolName + pool.Spec.ShardNamespace = egressShardNamespace + pool.Spec.NodeSelector = metav1.LabelSelector{MatchLabels: map[string]string{ + cloudv1alpha1.LabelNodeEgressPool: egressPoolName, + }} + pool.Spec.ShardLabels = map[string]string{ + bgpv1alpha1.LabelEgressShardCell: "us-central-1", + } + return pool +} + +// newNode is a node carrying whatever labels a test gives it. +func newNode(name string, nodeLabels map[string]string) *corev1.Node { + node := &corev1.Node{} + node.Name = name + node.Labels = nodeLabels + return node +} + +// commissionedNode is a node an operator opted into the pool. +func commissionedNode(name string) *corev1.Node { + return newNode(name, map[string]string{ + cloudv1alpha1.LabelNodeEgressPool: egressPoolName, + }) +} + +func reconcilePool(t *testing.T, r *EgressShardPoolReconciler) { + t.Helper() + if _, err := r.Reconcile(t.Context(), + ctrl.Request{NamespacedName: client.ObjectKey{Name: egressPoolName}}); err != nil { + t.Fatalf("reconcile the pool: %v", err) + } +} + +func shardNames(t *testing.T, cl client.Client) []string { + t.Helper() + var shards bgpv1alpha1.EgressShardList + if err := cl.List(t.Context(), &shards); err != nil { + t.Fatalf("list shards: %v", err) + } + names := make([]string, 0, len(shards.Items)) + for i := range shards.Items { + names = append(names, shards.Items[i].Name) + } + slices.Sort(names) + return names +} + +func TestEgressShardPoolBuildsAShardPerCommissionedNode(t *testing.T) { + r, cl := newPoolReconciler(t, newEgressPool(), + commissionedNode("node-b"), commissionedNode("node-a")) + + reconcilePool(t, r) + + want := []string{"shared-node-a", "shared-node-b"} + if got := shardNames(t, cl); !slices.Equal(got, want) { + t.Fatalf("shards: got %v, want %v", got, want) + } + + var shard bgpv1alpha1.EgressShard + key := client.ObjectKey{Namespace: egressShardNamespace, Name: "shared-node-a"} + if err := cl.Get(t.Context(), key, &shard); err != nil { + t.Fatalf("get the built shard: %v", err) + } + if shard.Spec.TargetRef.Kind != "Node" || shard.Spec.TargetRef.Name != "node-a" { + t.Errorf("target: got %v, want Node/node-a", shard.Spec.TargetRef) + } + if got := shard.Labels[bgpv1alpha1.LabelEgressShardPool]; got != egressPoolName { + t.Errorf("pool label: got %q, want %q", got, egressPoolName) + } + if got := shard.Labels[bgpv1alpha1.LabelEgressShardCell]; got != "us-central-1" { + t.Errorf("cell label: got %q, want us-central-1", got) + } + // The address is claimed by the controller holding the addressing-service + // credential. A pool that wrote one would be a second writer for a + // write-once field. + if shard.Spec.ShardAddressIPv6 != "" { + t.Errorf("built shard carries address %q, want none", shard.Spec.ShardAddressIPv6) + } + // A family label restates an address assignment this pool does not make. + if _, set := shard.Labels[bgpv1alpha1.LabelEgressShardIPv6]; set { + t.Error("built shard claims to serve IPv6 before it holds an address") + } +} + +// The label every translating node already carries is compute's own role label, +// so selecting on it would build a shard per compute node and make the egress +// address per-node. A node that did not opt in gets nothing. +func TestEgressShardPoolIgnoresAnUncommissionedComputeNode(t *testing.T) { + r, cl := newPoolReconciler(t, newEgressPool(), + newNode("compute-1", map[string]string{"galactic.datumapis.com/node": "compute"}), + commissionedNode("shard-1"), + ) + + reconcilePool(t, r) + + if got := shardNames(t, cl); !slices.Equal(got, []string{"shared-shard-1"}) { + t.Errorf("shards: got %v, want only the commissioned node's", got) + } +} + +// An object written before the schema refused an empty selector still selects +// every node in the cell, which is the shard-per-node outcome the whole model +// rests on not happening. +func TestEgressShardPoolRefusesAnEmptySelector(t *testing.T) { + pool := newEgressPool() + pool.Spec.NodeSelector = metav1.LabelSelector{} + r, cl := newPoolReconciler(t, pool, commissionedNode("node-a")) + + reconcilePool(t, r) + + if got := shardNames(t, cl); len(got) != 0 { + t.Errorf("shards: got %v, want none", got) + } + assertPoolCondition(t, cl, metav1.ConditionFalse, cloudv1alpha1.EgressShardPoolReasonNoNodeSelected) +} + +// A pool whose label an operator never set on a node looks exactly like a +// working pool until a network asks to egress through it. +func TestEgressShardPoolReportsThatItSelectedNoNode(t *testing.T) { + r, cl := newPoolReconciler(t, newEgressPool()) + + reconcilePool(t, r) + + assertPoolCondition(t, cl, metav1.ConditionFalse, cloudv1alpha1.EgressShardPoolReasonNoNodeSelected) +} + +func TestEgressShardPoolReportsTheShardsItBuilt(t *testing.T) { + r, cl := newPoolReconciler(t, newEgressPool(), commissionedNode("node-a")) + + reconcilePool(t, r) + + assertPoolCondition(t, cl, metav1.ConditionTrue, cloudv1alpha1.EgressShardPoolReasonShardsBuilt) +} + +// An existing shard is left exactly as it was found. Its address is write-once +// because the datapath claims a reply by exact match against it, and its labels +// are what a class's selector matches, so relabelling one moves live traffic. +func TestEgressShardPoolNeverRewritesAnExistingShard(t *testing.T) { + existing := &bgpv1alpha1.EgressShard{} + existing.Namespace = egressShardNamespace + existing.Name = "shared-node-a" + existing.Labels = map[string]string{ + bgpv1alpha1.LabelEgressShardPool: "some-other-pool", + bgpv1alpha1.LabelEgressShardIPv6: bgpv1alpha1.LabelValueEgressFamilyServed, + } + existing.Spec.TargetRef = bgpv1alpha1.TargetRef{Kind: "Node", Name: "node-a"} + existing.Spec.ShardAddressIPv6 = "2001:db8:f00d::100" + + r, cl := newPoolReconciler(t, newEgressPool(), commissionedNode("node-a"), existing) + + reconcilePool(t, r) + + var shard bgpv1alpha1.EgressShard + key := client.ObjectKey{Namespace: egressShardNamespace, Name: "shared-node-a"} + if err := cl.Get(t.Context(), key, &shard); err != nil { + t.Fatalf("get the pre-existing shard: %v", err) + } + if shard.Spec.ShardAddressIPv6 != "2001:db8:f00d::100" { + t.Errorf("address: got %q, want the one it already held", shard.Spec.ShardAddressIPv6) + } + if got := shard.Labels[bgpv1alpha1.LabelEgressShardPool]; got != "some-other-pool" { + t.Errorf("pool label: got %q, want the one it already held", got) + } +} + +// The pool label's value is the pool's own name, so it cannot drift from the +// object that stamped it, and cannot be pointed elsewhere by hand. +func TestEgressShardPoolLabelIsNotOverridable(t *testing.T) { + pool := newEgressPool() + pool.Spec.ShardLabels[bgpv1alpha1.LabelEgressShardPool] = "somewhere-else" + r, cl := newPoolReconciler(t, pool, commissionedNode("node-a")) + + reconcilePool(t, r) + + var shard bgpv1alpha1.EgressShard + key := client.ObjectKey{Namespace: egressShardNamespace, Name: "shared-node-a"} + if err := cl.Get(t.Context(), key, &shard); err != nil { + t.Fatalf("get the built shard: %v", err) + } + if got := shard.Labels[bgpv1alpha1.LabelEgressShardPool]; got != egressPoolName { + t.Errorf("pool label: got %q, want %q", got, egressPoolName) + } +} + +// The shards outlive the declaration that built them: a pool deleted by an +// operator reorganizing their cell must not take the translation of every +// network on it away. +func TestEgressShardPoolBuildsNothingWhileTerminating(t *testing.T) { + pool := newEgressPool() + pool.Finalizers = []string{"test.datumapis.com/hold"} + deletion := metav1.Now() + pool.DeletionTimestamp = &deletion + r, cl := newPoolReconciler(t, pool, commissionedNode("node-a")) + + reconcilePool(t, r) + + if got := shardNames(t, cl); len(got) != 0 { + t.Errorf("shards: got %v, want none", got) + } +} + +func assertPoolCondition( + t *testing.T, cl client.Client, status metav1.ConditionStatus, reason string, +) { + t.Helper() + var pool cloudv1alpha1.EgressShardPool + if err := cl.Get(t.Context(), client.ObjectKey{Name: egressPoolName}, &pool); err != nil { + t.Fatalf("get the pool: %v", err) + } + condition := meta.FindStatusCondition(pool.Status.Conditions, cloudv1alpha1.ConditionTypeReady) + if condition == nil { + t.Fatal("the pool reports no Ready condition") + } + if condition.Status != status || condition.Reason != reason { + t.Errorf("Ready: got %s/%s, want %s/%s", condition.Status, condition.Reason, status, reason) + } +} diff --git a/test/e2e/Taskfile.yaml b/test/e2e/Taskfile.yaml index ca5048b..928c09f 100644 --- a/test/e2e/Taskfile.yaml +++ b/test/e2e/Taskfile.yaml @@ -57,6 +57,7 @@ tasks: --for=condition=Established crd/vpcs.cloud.datumapis.com crd/vpcattachments.cloud.datumapis.com + crd/egressshardpools.cloud.datumapis.com --timeout=60s test: diff --git a/test/e2e/tests/egress-shard-pool-crd-schema/chainsaw-test.yaml b/test/e2e/tests/egress-shard-pool-crd-schema/chainsaw-test.yaml new file mode 100644 index 0000000..8a64d22 --- /dev/null +++ b/test/e2e/tests/egress-shard-pool-crd-schema/chainsaw-test.yaml @@ -0,0 +1,121 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: egress-shard-pool-crd-schema +spec: + description: > + Verify schema validation for the EgressShardPool CRD. A pool declares the egress + shards a cell has, and the rule that matters most is the one refusing an empty node + selector: a pool matching every node would build a shard per node and make the egress + address per-node, which dissolves the shared address the model rests on. Does not + require any controller — CRD schema is enforced by the API server alone. + steps: + - name: create-valid-pool + try: + - apply: + resource: + apiVersion: cloud.datumapis.com/v1alpha1 + kind: EgressShardPool + metadata: + name: e2e-shared + spec: + shardNamespace: galactic-system + nodeSelector: + matchLabels: + cloud.datumapis.com/egress-pool: e2e-shared + shardLabels: + network.datumapis.com/egress-cell: e2e-cell + - assert: + resource: + apiVersion: cloud.datumapis.com/v1alpha1 + kind: EgressShardPool + metadata: + name: e2e-shared + spec: + shardNamespace: galactic-system + + - name: reject-empty-node-selector + try: + - script: + content: | + set +e + OUTPUT=$(kubectl apply -f - 2>&1 <<'EOF' + apiVersion: cloud.datumapis.com/v1alpha1 + kind: EgressShardPool + metadata: + name: e2e-every-node + spec: + shardNamespace: galactic-system + nodeSelector: {} + EOF + ) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: a pool selecting every node was accepted" + exit 1 + fi + echo "$OUTPUT" | grep -q "at least one requirement" || { + echo "ERROR: rejected for the wrong reason: $OUTPUT" + exit 1 + } + echo "OK: an empty node selector is refused" + + - name: reject-family-label + try: + - script: + content: | + set +e + OUTPUT=$(kubectl apply -f - 2>&1 <<'EOF' + apiVersion: cloud.datumapis.com/v1alpha1 + kind: EgressShardPool + metadata: + name: e2e-claims-ipv6 + spec: + shardNamespace: galactic-system + nodeSelector: + matchLabels: + cloud.datumapis.com/egress-pool: e2e-claims-ipv6 + shardLabels: + network.datumapis.com/egress-ipv6: "true" + EOF + ) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: a pool claiming an address family it does not assign was accepted" + exit 1 + fi + echo "OK: the egress family labels are not settable by a pool" + + - name: reject-missing-shard-namespace + try: + - script: + content: | + set +e + OUTPUT=$(kubectl apply -f - 2>&1 <<'EOF' + apiVersion: cloud.datumapis.com/v1alpha1 + kind: EgressShardPool + metadata: + name: e2e-no-namespace + spec: + nodeSelector: + matchLabels: + cloud.datumapis.com/egress-pool: e2e-no-namespace + EOF + ) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: a pool naming no shard namespace was accepted" + exit 1 + fi + echo "OK: a pool must name the namespace its shards live in" + + - name: delete-valid-pool + try: + - delete: + ref: + apiVersion: cloud.datumapis.com/v1alpha1 + kind: EgressShardPool + name: e2e-shared From a4337800110d4d2e0d9a45cc81bac1f1210d5889 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 17 Sep 2026 23:40:06 -0500 Subject: [PATCH 2/4] feat: Bind a network's egress to one shard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A location's egress was selected twice from the same inputs: once when the node was told which shards to route toward, and again when a consumer was told which address their traffic leaves on. The two disagreed. The reported address came from the first shard by name, while the node installs the first shard whose SID it can resolve a route toward, and a node that is itself a shard can never resolve a route to its own advertised SID — every compute node runs the translator. A workload on such a node read one source address and egressed from another, silently breaking any allow-list built on it. A claim now records the binding, made once. One claim per network context that declares egress, so the binding outlives the instances using it: an interface is replaced routinely, and a binding that followed one would move a network's source address every time that happened. The shard holds no list of the networks it serves. Which networks a shard serves is answered by listing the claims labelled with its name, the way the network and location labels already make counting a presence's consumers a list query, and the only state a binder adds to a shard is one finalizer while a claim is bound. Key changes: - Add EgressShardClaim, whose spec carries the resolved terms verbatim and is immutable, and whose status holds the binding — following SubnetClaim, which records its allocation the same way, rather than the interface claim's reference on the provisioned object, which only a one-to-one binding can use - Bind in the cell's single writer, refusing a candidate with a named reason per incompatibility so a claim that bound nothing says what stopped it - Write the readiness reasons that were defined with no writer: Ready, and AddressUnavailable for a bound shard whose address is unclaimed, and Unavailable for a location with no usable shard - Hold a bound shard open with a finalizer, and release it when its consumer set empties, so decommissioning is an act someone takes rather than an outcome networks discover - Read the binding where egress used to be selected a second time, keeping the parameters guard that lets another implementation serve a class, and keeping the node's candidate list an ordered array so recording a standby shard later needs no node change - Narrow the shard event handler from every interface in the cell to the interfaces of the networks bound to that shard - Carry the class's sharing onto the claim and branch on nothing: dedicated capacity is a hand-commissioned node no controller can grow while nothing allocates a shard's identifier Co-Authored-By: Claude Opus 5 (1M context) --- api/v1alpha1/egressshardclaim_types.go | 268 +++++++ api/v1alpha1/zz_generated.deepcopy.go | 153 ++++ cmd/main.go | 12 + ...cloud.datumapis.com_egressshardclaims.yaml | 242 ++++++ config/crd/kustomization.yaml | 3 + config/rbac/role.yaml | 23 +- docs/api/vpc.md | 165 +++++ .../egressshardbinding_controller.go | 171 +++++ .../egressshardbinding_controller_test.go | 144 ++++ .../controller/egressshardclaim_controller.go | 700 ++++++++++++++++++ .../egressshardclaim_controller_test.go | 539 ++++++++++++++ .../controller/networkinterface_controller.go | 224 +++--- .../networkinterface_controller_test.go | 191 ++--- test/e2e/Taskfile.yaml | 1 + .../chainsaw-test.yaml | 128 ++++ 15 files changed, 2757 insertions(+), 207 deletions(-) create mode 100644 api/v1alpha1/egressshardclaim_types.go create mode 100644 config/crd/cloud.datumapis.com_egressshardclaims.yaml create mode 100644 internal/controller/egressshardbinding_controller.go create mode 100644 internal/controller/egressshardbinding_controller_test.go create mode 100644 internal/controller/egressshardclaim_controller.go create mode 100644 internal/controller/egressshardclaim_controller_test.go create mode 100644 test/e2e/tests/egress-shard-claim-crd-schema/chainsaw-test.yaml diff --git a/api/v1alpha1/egressshardclaim_types.go b/api/v1alpha1/egressshardclaim_types.go new file mode 100644 index 0000000..fc6a19a --- /dev/null +++ b/api/v1alpha1/egressshardclaim_types.go @@ -0,0 +1,268 @@ +/* +Copyright © 2026 Datum Technology, Inc. All rights reserved. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// LabelEgressShardClaimShard names the shard a claim is bound to. +// +// It is what makes a shard's consumer set a list rather than a number anyone +// has to keep in step: a shard holds no list of the networks it serves, so +// "which networks does this shard serve" is answered by listing claims +// carrying this label, the same way the network and location labels make +// counting a presence's consumers a list query. +// +// The value is the shard's name. The binding itself lives on the claim's +// status, which is what a reader trusts; this label narrows the query that +// finds the claims to ask. +const LabelEgressShardClaimShard = "cloud.datumapis.com/egress-shard" + +// FinalizerEgressShardBinding is the one piece of state a binder adds to a +// shard, held while any claim is bound to it. +// +// It exists so that decommissioning a shard is an act someone takes rather +// than an outcome networks discover. Deleting a shard that is translating +// strands the return traffic of every flow on it, and nothing rebinds a claim: +// a binding is decided once, so a network whose shard vanished has no egress +// and no second answer coming. +const FinalizerEgressShardBinding = "cloud.datumapis.com/egress-shard-binding" + +// EgressSharing is how many networks may share one egress shard. It is the +// serving class's sharing, copied verbatim and recorded as the fact this +// binding was made under. +// +// Nothing branches on it. Every claim binds a shared shard, because dedicated +// capacity is a hand-commissioned shard node that no controller can grow while +// nothing allocates the identifier a shard is unusable without — a claim beyond +// that count would wait indefinitely. The value is carried because the +// projection writes it and a claim records what it was created from. +// +// +kubebuilder:validation:Enum=Shared;Dedicated +type EgressSharing string + +const ( + // EgressSharingShared lets many networks bind one shard and therefore + // leave the platform on one address. + EgressSharingShared EgressSharing = "Shared" + + // EgressSharingDedicated would let exactly one network bind a shard, which + // is what makes that shard's address the network's own. It is not offered + // yet and nothing here enforces it; the value is defined so that a claim + // written when it is offered means today what it will mean then. + EgressSharingDedicated EgressSharing = "Dedicated" +) + +// EgressShardClaimSpec is the network being bound to an egress shard, and the +// terms the binding has to satisfy. +// +// Every field is already resolved upstream and copied here verbatim. Nothing +// reading a claim selects a class, picks a default, or interprets a class's +// parameters. +// +// The whole spec is immutable. The binding is decided once from these facts +// and never recomputed, so a fact that moved underneath it would describe a +// binding that was never made under it. A consumer changing what they asked +// for is a claim deleted and a new one written, which is a crossing someone +// can see. +// +// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec is immutable; a binding is decided once, so delete the claim to ask for different terms" +type EgressShardClaimSpec struct { + // Network is the network reaching the internet. + // +required + Network NetworkRef `json:"network"` + + // NetworkContext is that network's presence in this cell, which is what + // makes the claim one per location. + // +required + NetworkContext NetworkContextRef `json:"networkContext"` + + // ClassName is the InternetEgressClass resolved for this network. It is + // recorded rather than read: the class is cluster-scoped upstream and no + // copy of it reaches this cell. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +required + ClassName string `json:"className"` + + // Sharing is how many networks that class allows on one shard. + // +required + Sharing EgressSharing `json:"sharing"` + + // Families are the destination address families this binding has to reach, + // so the shard it binds is one that translates them. + // +listType=set + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=2 + // +required + Families []InternetEgressAddressFamily `json:"families"` +} + +// NetworkRef references a networking.datumapis.com Network by name. +type NetworkRef struct { + // Name of the Network. + // +kubebuilder:validation:MinLength=1 + // +required + Name string `json:"name"` +} + +// NetworkContextRef references a networking.datumapis.com NetworkContext by +// name in the same namespace. +type NetworkContextRef struct { + // Name of the NetworkContext. + // +kubebuilder:validation:MinLength=1 + // +required + Name string `json:"name"` +} + +// EgressShardReference names one egress shard. +type EgressShardReference struct { + // Namespace holding the shard. It is stated rather than assumed: the + // shards are in the namespace an operator gave the serving class, which is + // not the namespace a claim lives in. + // +kubebuilder:validation:MinLength=1 + // +required + Namespace string `json:"namespace"` + + // Name of the shard. + // +kubebuilder:validation:MinLength=1 + // +required + Name string `json:"name"` +} + +// EgressShardClaimStatus is the binding. +// +// The binding is recorded here and nowhere else. The shard side carries no +// reference back, unlike the interface and subnet claims this follows in every +// other respect: both of those are strictly one-to-one and the reference on the +// provisioned object is what enforces it, whereas many networks share one +// shard, so a shard-side reference would have to be a list of the networks +// served — which is the state a shard deliberately does not hold. +type EgressShardClaimStatus struct { + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // +listType=map + // +listMapKey=type + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` + + // ShardRef is the shard this network egresses through. + // + // Absent means no shard is bound, which is what a location whose cell holds + // no usable shard reads. Nothing publishes an address or a route in that + // state: an address a consumer might allow-list is withheld until the + // platform can state which one their packets leave on. + // + // Present is permanent for this claim's life. It is written once, and + // nothing recomputes it: a rebinding would move a live VPC's egress to a + // different source address, which is the value a consumer allow-listed at + // their destination. + // +optional + ShardRef *EgressShardReference `json:"shardRef,omitempty"` +} + +// Reasons reported on an EgressShardClaim's Ready condition. +const ( + // EgressShardClaimReasonBound means this network egresses through the + // shard status names. + EgressShardClaimReasonBound = "Bound" + + // EgressShardClaimReasonParametersUnavailable means the parameters the + // serving class names do not exist in this cell, so which shards serve the + // class is unknown here. + EgressShardClaimReasonParametersUnavailable = "ParametersUnavailable" + + // EgressShardClaimReasonNoShardMatchesTheClass means no shard in the + // namespace the class names carries the labels its selector requires. + EgressShardClaimReasonNoShardMatchesTheClass = "NoShardMatchesTheClass" + + // EgressShardClaimReasonNoShardIdentifier means every shard the class + // selects is still without the SRv6 identifier a node routes toward, so + // there is nothing to bind that would carry a packet. + EgressShardClaimReasonNoShardIdentifier = "NoShardIdentifier" + + // EgressShardClaimReasonShardMissing means the bound shard no longer + // exists. Nothing rebinds a claim, so this network has no egress and no + // second answer coming; the finalizer is what makes the state reachable + // only by someone removing it. + EgressShardClaimReasonShardMissing = "ShardMissing" + + // EgressShardClaimReasonShardTerminating means the bound shard is being + // deleted. The binding stands, because nothing rebinds a claim, and the + // shard is held until the claim is gone. + EgressShardClaimReasonShardTerminating = "ShardTerminating" + + // EgressShardClaimReasonTermsChanged means the egress this location is + // instructed to provide no longer matches the terms this binding was made + // under. The binding stands and delivers what it always did; changing the + // terms means deleting the claim. + EgressShardClaimReasonTermsChanged = "TermsChanged" +) + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced +// +kubebuilder:printcolumn:name="Network",type="string",JSONPath=".spec.network.name" +// +kubebuilder:printcolumn:name="Sharing",type="string",JSONPath=".spec.sharing" +// +kubebuilder:printcolumn:name="Shard",type="string",JSONPath=".status.shardRef.name" +// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Reason",type="string",JSONPath=`.status.conditions[?(@.type=="Ready")].reason` +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// EgressShardClaim is one network being bound to one egress shard in this cell. +// +// There is one claim per network context that declares egress — not one per +// interface and not one per attachment. The binding has to outlive the +// workloads using it: an instance is replaced routinely, and a binding that +// followed an attachment would move a network's source address every time that +// happened, which is the address a consumer allow-listed at their destination. +// +// The claim names no shard, no selector, no address and no pool. It states what +// the network needs and the cell answers with which shard serves it, the same +// division a subnet claim makes. +type EgressShardClaim struct { + metav1.TypeMeta `json:",inline"` + + // metadata is a standard object metadata. The name is the network context's + // own, because there is exactly one claim per context. + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // spec is the network being bound and the terms the binding satisfies + // +required + Spec EgressShardClaimSpec `json:"spec"` + + // status is the binding + // +optional + Status EgressShardClaimStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// EgressShardClaimList contains a list of EgressShardClaim. +type EgressShardClaimList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []EgressShardClaim `json:"items"` +} + +func init() { + SchemeBuilder.Register(&EgressShardClaim{}, &EgressShardClaimList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 488a4bd..2965bd9 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -26,6 +26,114 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressShardClaim) DeepCopyInto(out *EgressShardClaim) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardClaim. +func (in *EgressShardClaim) DeepCopy() *EgressShardClaim { + if in == nil { + return nil + } + out := new(EgressShardClaim) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressShardClaim) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressShardClaimList) DeepCopyInto(out *EgressShardClaimList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]EgressShardClaim, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardClaimList. +func (in *EgressShardClaimList) DeepCopy() *EgressShardClaimList { + if in == nil { + return nil + } + out := new(EgressShardClaimList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressShardClaimList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressShardClaimSpec) DeepCopyInto(out *EgressShardClaimSpec) { + *out = *in + out.Network = in.Network + out.NetworkContext = in.NetworkContext + if in.Families != nil { + in, out := &in.Families, &out.Families + *out = make([]InternetEgressAddressFamily, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardClaimSpec. +func (in *EgressShardClaimSpec) DeepCopy() *EgressShardClaimSpec { + if in == nil { + return nil + } + out := new(EgressShardClaimSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressShardClaimStatus) DeepCopyInto(out *EgressShardClaimStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ShardRef != nil { + in, out := &in.ShardRef, &out.ShardRef + *out = new(EgressShardReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardClaimStatus. +func (in *EgressShardClaimStatus) DeepCopy() *EgressShardClaimStatus { + if in == nil { + return nil + } + out := new(EgressShardClaimStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EgressShardParameters) DeepCopyInto(out *EgressShardParameters) { *out = *in @@ -204,6 +312,21 @@ func (in *EgressShardPoolStatus) DeepCopy() *EgressShardPoolStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressShardReference) DeepCopyInto(out *EgressShardReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardReference. +func (in *EgressShardReference) DeepCopy() *EgressShardReference { + if in == nil { + return nil + } + out := new(EgressShardReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InternetEgressSourceAddress) DeepCopyInto(out *InternetEgressSourceAddress) { *out = *in @@ -219,6 +342,21 @@ func (in *InternetEgressSourceAddress) DeepCopy() *InternetEgressSourceAddress { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkContextRef) DeepCopyInto(out *NetworkContextRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkContextRef. +func (in *NetworkContextRef) DeepCopy() *NetworkContextRef { + if in == nil { + return nil + } + out := new(NetworkContextRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkFabricIdentity) DeepCopyInto(out *NetworkFabricIdentity) { *out = *in @@ -323,6 +461,21 @@ func (in *NetworkInterfaceRef) DeepCopy() *NetworkInterfaceRef { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkRef) DeepCopyInto(out *NetworkRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkRef. +func (in *NetworkRef) DeepCopy() *NetworkRef { + if in == nil { + return nil + } + out := new(NetworkRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VPC) DeepCopyInto(out *VPC) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index 2976616..8d72e02 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -123,6 +123,18 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "EgressShardPool") os.Exit(1) } + if err := (&controller.EgressShardClaimReconciler{ + Client: mgr.GetClient(), Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "EgressShardClaim") + os.Exit(1) + } + if err := (&controller.EgressShardBindingReconciler{ + Client: mgr.GetClient(), Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "EgressShardBinding") + os.Exit(1) + } if err := (&controller.NetworkInterfaceReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), APIReader: mgr.GetAPIReader(), AttachmentMode: attachmentMode, diff --git a/config/crd/cloud.datumapis.com_egressshardclaims.yaml b/config/crd/cloud.datumapis.com_egressshardclaims.yaml new file mode 100644 index 0000000..305e831 --- /dev/null +++ b/config/crd/cloud.datumapis.com_egressshardclaims.yaml @@ -0,0 +1,242 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: egressshardclaims.cloud.datumapis.com +spec: + group: cloud.datumapis.com + names: + kind: EgressShardClaim + listKind: EgressShardClaimList + plural: egressshardclaims + singular: egressshardclaim + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.network.name + name: Network + type: string + - jsonPath: .spec.sharing + name: Sharing + type: string + - jsonPath: .status.shardRef.name + name: Shard + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + EgressShardClaim is one network being bound to one egress shard in this cell. + + There is one claim per network context that declares egress — not one per + interface and not one per attachment. The binding has to outlive the + workloads using it: an instance is replaced routinely, and a binding that + followed an attachment would move a network's source address every time that + happened, which is the address a consumer allow-listed at their destination. + + The claim names no shard, no selector, no address and no pool. It states what + the network needs and the cell answers with which shard serves it, the same + division a subnet claim makes. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec is the network being bound and the terms the binding + satisfies + properties: + className: + description: |- + ClassName is the InternetEgressClass resolved for this network. It is + recorded rather than read: the class is cluster-scoped upstream and no + copy of it reaches this cell. + maxLength: 253 + minLength: 1 + type: string + families: + description: |- + Families are the destination address families this binding has to reach, + so the shard it binds is one that translates them. + items: + description: |- + InternetEgressAddressFamily is the address family of an egress source + address. + + Only IPv6 is reported. Reaching an IPv4 destination needs a resolver and a + translator sharing a prefix, which the platform pairs neither of, so the + value is withheld rather than reported and not delivered. An address written + today records IPv6, so accepting IPv4 later changes no attachment. + enum: + - IPv6 + type: string + maxItems: 2 + minItems: 1 + type: array + x-kubernetes-list-type: set + network: + description: Network is the network reaching the internet. + properties: + name: + description: Name of the Network. + minLength: 1 + type: string + required: + - name + type: object + networkContext: + description: |- + NetworkContext is that network's presence in this cell, which is what + makes the claim one per location. + properties: + name: + description: Name of the NetworkContext. + minLength: 1 + type: string + required: + - name + type: object + sharing: + description: Sharing is how many networks that class allows on one + shard. + enum: + - Shared + - Dedicated + type: string + required: + - className + - families + - network + - networkContext + - sharing + type: object + x-kubernetes-validations: + - message: spec is immutable; a binding is decided once, so delete the + claim to ask for different terms + rule: self == oldSelf + status: + description: status is the binding + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedGeneration: + format: int64 + type: integer + shardRef: + description: |- + ShardRef is the shard this network egresses through. + + Absent means no shard is bound, which is what a location whose cell holds + no usable shard reads. Nothing publishes an address or a route in that + state: an address a consumer might allow-list is withheld until the + platform can state which one their packets leave on. + + Present is permanent for this claim's life. It is written once, and + nothing recomputes it: a rebinding would move a live VPC's egress to a + different source address, which is the value a consumer allow-listed at + their destination. + properties: + name: + description: Name of the shard. + minLength: 1 + type: string + namespace: + description: |- + Namespace holding the shard. It is stated rather than assumed: the + shards are in the namespace an operator gave the serving class, which is + not the namespace a claim lives in. + minLength: 1 + type: string + required: + - name + - namespace + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 7c43702..a163845 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -12,3 +12,6 @@ resources: # Cell-local too: an operator declares the shards the cell has, and the # builder in the cell expands it into the shard objects a class selects. - cloud.datumapis.com_egressshardpools.yaml + # Cell-local as well: one per network context that declares egress, written + # and read only by the binder in the cell. + - cloud.datumapis.com_egressshardclaims.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ab1f087..9fa08b8 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -22,15 +22,22 @@ rules: - apiGroups: - cloud.datumapis.com resources: - - egressshardparameters - - egressshardpools + - egressshardclaims + - networkfabricidentities + - vpcattachments + - vpcs verbs: + - create + - delete - get - list + - patch + - update - watch - apiGroups: - cloud.datumapis.com resources: + - egressshardclaims/status - egressshardpools/status - vpcattachments/status - vpcs/status @@ -41,16 +48,11 @@ rules: - apiGroups: - cloud.datumapis.com resources: - - networkfabricidentities - - vpcattachments - - vpcs + - egressshardparameters + - egressshardpools verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - compute.datumapis.com @@ -101,6 +103,8 @@ rules: - create - get - list + - patch + - update - watch - apiGroups: - networking.datumapis.com @@ -117,6 +121,7 @@ rules: - apiGroups: - networking.datumapis.com resources: + - networkcontexts/status - networkinterfaceclaims/status - networkinterfaces/status verbs: diff --git a/docs/api/vpc.md b/docs/api/vpc.md index 6a64e1d..cabda11 100644 --- a/docs/api/vpc.md +++ b/docs/api/vpc.md @@ -9,6 +9,7 @@ Package v1alpha1 contains API Schema definitions for the cloud.datumapis.com/v1alpha1 API group. ### Resource Types +- [EgressShardClaim](#egressshardclaim) - [EgressShardParameters](#egressshardparameters) - [EgressShardPool](#egressshardpool) - [NetworkFabricIdentity](#networkfabricidentity) @@ -17,6 +18,93 @@ Package v1alpha1 contains API Schema definitions for the cloud.datumapis.com/v1a +#### EgressShardClaim + + + +EgressShardClaim is one network being bound to one egress shard in this cell. + +There is one claim per network context that declares egress — not one per +interface and not one per attachment. The binding has to outlive the +workloads using it: an instance is replaced routinely, and a binding that +followed an attachment would move a network's source address every time that +happened, which is the address a consumer allow-listed at their destination. + +The claim names no shard, no selector, no address and no pool. It states what +the network needs and the cell answers with which shard serves it, the same +division a subnet claim makes. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `cloud.datumapis.com/v1alpha1` | | | +| `kind` _string_ | `EgressShardClaim` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[EgressShardClaimSpec](#egressshardclaimspec)_ | spec is the network being bound and the terms the binding satisfies | | | +| `status` _[EgressShardClaimStatus](#egressshardclaimstatus)_ | status is the binding | | | + + +#### EgressShardClaimSpec + + + +EgressShardClaimSpec is the network being bound to an egress shard, and the +terms the binding has to satisfy. + +Every field is already resolved upstream and copied here verbatim. Nothing +reading a claim selects a class, picks a default, or interprets a class's +parameters. + +The whole spec is immutable. The binding is decided once from these facts +and never recomputed, so a fact that moved underneath it would describe a +binding that was never made under it. A consumer changing what they asked +for is a claim deleted and a new one written, which is a crossing someone +can see. + + + +_Appears in:_ +- [EgressShardClaim](#egressshardclaim) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `network` _[NetworkRef](#networkref)_ | Network is the network reaching the internet. | | | +| `networkContext` _[NetworkContextRef](#networkcontextref)_ | NetworkContext is that network's presence in this cell, which is what
makes the claim one per location. | | | +| `className` _string_ | ClassName is the InternetEgressClass resolved for this network. It is
recorded rather than read: the class is cluster-scoped upstream and no
copy of it reaches this cell. | | MaxLength: 253
MinLength: 1
| +| `sharing` _[EgressSharing](#egresssharing)_ | Sharing is how many networks that class allows on one shard. | | Enum: [Shared Dedicated]
| +| `families` _[InternetEgressAddressFamily](#internetegressaddressfamily) array_ | Families are the destination address families this binding has to reach,
so the shard it binds is one that translates them. | | Enum: [IPv6]
MaxItems: 2
MinItems: 1
| + + +#### EgressShardClaimStatus + + + +EgressShardClaimStatus is the binding. + +The binding is recorded here and nowhere else. The shard side carries no +reference back, unlike the interface and subnet claims this follows in every +other respect: both of those are strictly one-to-one and the reference on the +provisioned object is what enforces it, whereas many networks share one +shard, so a shard-side reference would have to be a list of the networks +served — which is the state a shard deliberately does not hold. + + + +_Appears in:_ +- [EgressShardClaim](#egressshardclaim) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `observedGeneration` _integer_ | | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | | | | +| `shardRef` _[EgressShardReference](#egressshardreference)_ | ShardRef is the shard this network egresses through.
Absent means no shard is bound, which is what a location whose cell holds
no usable shard reads. Nothing publishes an address or a route in that
state: an address a consumer might allow-list is withheld until the
platform can state which one their packets leave on.
Present is permanent for this claim's life. It is written once, and
nothing recomputes it: a rebinding would move a live VPC's egress to a
different source address, which is the value a consumer allow-listed at
their destination. | | | + + #### EgressShardParameters @@ -142,6 +230,49 @@ _Appears in:_ | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | | | | +#### EgressShardReference + + + +EgressShardReference names one egress shard. + + + +_Appears in:_ +- [EgressShardClaimStatus](#egressshardclaimstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `namespace` _string_ | Namespace holding the shard. It is stated rather than assumed: the
shards are in the namespace an operator gave the serving class, which is
not the namespace a claim lives in. | | MinLength: 1
| +| `name` _string_ | Name of the shard. | | MinLength: 1
| + + +#### EgressSharing + +_Underlying type:_ _string_ + +EgressSharing is how many networks may share one egress shard. It is the +serving class's sharing, copied verbatim and recorded as the fact this +binding was made under. + +Nothing branches on it. Every claim binds a shared shard, because dedicated +capacity is a hand-commissioned shard node that no controller can grow while +nothing allocates the identifier a shard is unusable without — a claim beyond +that count would wait indefinitely. The value is carried because the +projection writes it and a claim records what it was created from. + +_Validation:_ +- Enum: [Shared Dedicated] + +_Appears in:_ +- [EgressShardClaimSpec](#egressshardclaimspec) + +| Field | Description | +| --- | --- | +| `Shared` | EgressSharingShared lets many networks bind one shard and therefore
leave the platform on one address.
| +| `Dedicated` | EgressSharingDedicated would let exactly one network bind a shard, which
is what makes that shard's address the network's own. It is not offered
yet and nothing here enforces it; the value is defined so that a claim
written when it is offered means today what it will mean then.
| + + #### IPAddress _Underlying type:_ _string_ @@ -172,6 +303,7 @@ _Validation:_ - Enum: [IPv6] _Appears in:_ +- [EgressShardClaimSpec](#egressshardclaimspec) - [InternetEgressSourceAddress](#internetegresssourceaddress) | Field | Description | @@ -231,6 +363,23 @@ _Appears in:_ +#### NetworkContextRef + + + +NetworkContextRef references a networking.datumapis.com NetworkContext by +name in the same namespace. + + + +_Appears in:_ +- [EgressShardClaimSpec](#egressshardclaimspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | Name of the NetworkContext. | | MinLength: 1
| + + #### NetworkFabricIdentity @@ -317,6 +466,22 @@ _Appears in:_ | `name` _string_ | Name of the NetworkInterface. | | MinLength: 1
| +#### NetworkRef + + + +NetworkRef references a networking.datumapis.com Network by name. + + + +_Appears in:_ +- [EgressShardClaimSpec](#egressshardclaimspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | Name of the Network. | | MinLength: 1
| + + #### VPC diff --git a/internal/controller/egressshardbinding_controller.go b/internal/controller/egressshardbinding_controller.go new file mode 100644 index 0000000..c8dbd41 --- /dev/null +++ b/internal/controller/egressshardbinding_controller.go @@ -0,0 +1,171 @@ +/* +Copyright © 2026 Datum Technology, Inc. All rights reserved. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package controller + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// EgressShardBindingReconciler keeps a shard's finalizer in step with its +// consumer set. +// +// The finalizer is the only state a binder adds to a shard. It holds while any +// claim is bound, so decommissioning a shard is an act someone takes rather +// than an outcome networks discover: deleting one strands the return traffic of +// every flow it is translating, and nothing rebinds a claim, so a network whose +// shard vanished has no egress and no second answer coming. +// +// The consumer set is a list query over the claims, never a field on the shard. +// That is the whole point: a shard that recorded its own consumers would be +// holding the list of served networks the model refuses it, and the list would +// have to be kept in step by whoever binds. +type EgressShardBindingReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshards,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=egressshardclaims,verbs=get;list;watch + +func (r *EgressShardBindingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var shard bgpv1alpha1.EgressShard + if err := r.Get(ctx, req.NamespacedName, &shard); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + consumers, err := boundEgressShardClaims(ctx, r.Client, &shard) + if err != nil { + return ctrl.Result{}, err + } + if len(consumers) == 0 { + return ctrl.Result{}, releaseShard(ctx, r.Client, &shard) + } + if !shard.DeletionTimestamp.IsZero() { + // Said out loud, because the alternative to a stuck object here is a + // network losing the internet with nothing to point at. Deleting the + // claims is what lets the shard go. + logf.FromContext(ctx).Info("holding an egress shard open while networks are bound to it", + "shard", client.ObjectKeyFromObject(&shard), "consumers", len(consumers)) + } + return ctrl.Result{}, holdShard(ctx, r.Client, &shard) +} + +// holdShard adds the binder's finalizer, so the shard cannot go while a network +// is bound to it. +func holdShard(ctx context.Context, cl client.Client, shard *bgpv1alpha1.EgressShard) error { + if controllerutil.ContainsFinalizer(shard, cloudv1alpha1.FinalizerEgressShardBinding) { + return nil + } + // Patched rather than updated. The controller holding the addressing-service + // credential writes this spec's addresses, and a whole-object update from a + // copy read before that write would put the old value back over a field that + // is write-once. + patch := client.MergeFrom(shard.DeepCopy()) + controllerutil.AddFinalizer(shard, cloudv1alpha1.FinalizerEgressShardBinding) + if err := cl.Patch(ctx, shard, patch); err != nil { + return fmt.Errorf("hold egress shard %s open for the networks bound to it: %w", + client.ObjectKeyFromObject(shard), err) + } + return nil +} + +// releaseShard removes the binder's finalizer from a shard no network is bound +// to, which is what lets an operator decommission a drained node. +func releaseShard(ctx context.Context, cl client.Client, shard *bgpv1alpha1.EgressShard) error { + if !controllerutil.ContainsFinalizer(shard, cloudv1alpha1.FinalizerEgressShardBinding) { + return nil + } + patch := client.MergeFrom(shard.DeepCopy()) + controllerutil.RemoveFinalizer(shard, cloudv1alpha1.FinalizerEgressShardBinding) + if err := cl.Patch(ctx, shard, patch); err != nil { + return fmt.Errorf("release egress shard %s: %w", client.ObjectKeyFromObject(shard), err) + } + logf.FromContext(ctx).Info("released an egress shard no network is bound to", + "shard", client.ObjectKeyFromObject(shard)) + return nil +} + +// boundEgressShardClaims is a shard's consumer set: the networks bound to it. +// +// It is a list query over the label the binder stamps, which is what stands in +// for the list of served networks a shard does not hold. The label narrows the +// query and each claim's own status settles it, so a label left behind by a +// binding that never completed counts as nothing. +func boundEgressShardClaims( + ctx context.Context, reader client.Reader, shard *bgpv1alpha1.EgressShard, +) ([]cloudv1alpha1.EgressShardClaim, error) { + var claims cloudv1alpha1.EgressShardClaimList + if err := reader.List(ctx, &claims, client.MatchingLabels{ + cloudv1alpha1.LabelEgressShardClaimShard: shard.Name, + }); err != nil { + return nil, fmt.Errorf("list the claims bound to egress shard %s: %w", shard.Name, err) + } + + bound := make([]cloudv1alpha1.EgressShardClaim, 0, len(claims.Items)) + for i := range claims.Items { + claim := claims.Items[i] + held := claim.Status.ShardRef + if held == nil || held.Name != shard.Name || held.Namespace != shard.Namespace { + continue + } + if !claim.DeletionTimestamp.IsZero() { + continue + } + bound = append(bound, claim) + } + return bound, nil +} + +// SetupWithManager registers the reconciler with the manager. +func (r *EgressShardBindingReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&bgpv1alpha1.EgressShard{}). + Watches(&cloudv1alpha1.EgressShardClaim{}, + handler.EnqueueRequestsFromMapFunc(egressShardForClaim)). + Named("egressshardbinding"). + Complete(r) +} + +// egressShardForClaim maps a claim to the shard it holds, including the last +// state of one being deleted — which is the event that lets the final claim on +// a shard release it. +// +// A claim holding no binding maps to nothing, and needs to: it was never part +// of any shard's consumer set, which is counted from this same field. +func egressShardForClaim(_ context.Context, object client.Object) []reconcile.Request { + claim, ok := object.(*cloudv1alpha1.EgressShardClaim) + if !ok || claim.Status.ShardRef == nil { + return nil + } + return []reconcile.Request{{NamespacedName: client.ObjectKey{ + Namespace: claim.Status.ShardRef.Namespace, + Name: claim.Status.ShardRef.Name, + }}} +} diff --git a/internal/controller/egressshardbinding_controller_test.go b/internal/controller/egressshardbinding_controller_test.go new file mode 100644 index 0000000..4dbd42b --- /dev/null +++ b/internal/controller/egressshardbinding_controller_test.go @@ -0,0 +1,144 @@ +/* +Copyright © 2026 Datum Technology, Inc. All rights reserved. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package controller + +import ( + "testing" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// egressTestShard is the one shard these tests hold open or release. +const egressTestShard = "shard-a" + +func newShardBinder(t *testing.T, objects ...client.Object) (*EgressShardBindingReconciler, client.Client) { + t.Helper() + + scheme := runtime.NewScheme() + if err := cloudv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("build the cloud scheme: %v", err) + } + if err := bgpv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("build the fabric scheme: %v", err) + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + return &EgressShardBindingReconciler{Client: fakeClient, Scheme: scheme}, fakeClient +} + +func reconcileShard(t *testing.T, r *EgressShardBindingReconciler) { + t.Helper() + key := client.ObjectKey{Namespace: egressShardNamespace, Name: egressTestShard} + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: key}); err != nil { + t.Fatalf("reconcile the shard: %v", err) + } +} + +func heldOpen(t *testing.T, cl client.Client) bool { + t.Helper() + var shard bgpv1alpha1.EgressShard + key := client.ObjectKey{Namespace: egressShardNamespace, Name: egressTestShard} + if err := cl.Get(t.Context(), key, &shard); err != nil { + t.Fatalf("get the shard: %v", err) + } + return controllerutil.ContainsFinalizer(&shard, cloudv1alpha1.FinalizerEgressShardBinding) +} + +// The finalizer is the only state a binder adds to a shard, and it holds while +// a network is bound: deleting a translating shard strands the return traffic of +// every flow on it, and nothing rebinds a claim. +func TestShardIsHeldOpenWhileANetworkIsBound(t *testing.T) { + r, cl := newShardBinder(t, newEgressClaim("shard-a"), + newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) + + reconcileShard(t, r) + + if !heldOpen(t, cl) { + t.Error("a shard with a network bound to it is not held open") + } +} + +// A drained shard is released, which is what lets an operator decommission the +// node it runs on. +func TestShardIsReleasedWhenNoNetworkIsBound(t *testing.T) { + shard := newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels()) + shard.Finalizers = []string{cloudv1alpha1.FinalizerEgressShardBinding} + r, cl := newShardBinder(t, shard) + + reconcileShard(t, r) + + if heldOpen(t, cl) { + t.Error("a shard no network is bound to is still held open") + } +} + +// A label with no binding behind it is no consumer. A claim that was labelled +// by a pass that then failed must not hold a shard open forever. +func TestShardIsReleasedWhenAClaimHoldsOnlyTheLabel(t *testing.T) { + claim := newEgressClaim("shard-a") + claim.Status.ShardRef = nil + shard := newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels()) + shard.Finalizers = []string{cloudv1alpha1.FinalizerEgressShardBinding} + r, cl := newShardBinder(t, claim, shard) + + reconcileShard(t, r) + + if heldOpen(t, cl) { + t.Error("a label with no binding behind it held a shard open") + } +} + +// A claim naming a shard of the same name in another namespace is another +// cell's business, not a consumer of this one. +func TestShardIgnoresAClaimBoundElsewhere(t *testing.T) { + claim := newEgressClaim("shard-a") + claim.Status.ShardRef.Namespace = "some-other-namespace" + shard := newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels()) + shard.Finalizers = []string{cloudv1alpha1.FinalizerEgressShardBinding} + r, cl := newShardBinder(t, claim, shard) + + reconcileShard(t, r) + + if heldOpen(t, cl) { + t.Error("a claim bound to another namespace's shard held this one open") + } +} + +// The mapping that makes the last claim's deletion release its shard. A claim +// with no binding maps to nothing, because it was in no consumer set. +func TestEgressShardForClaim(t *testing.T) { + bound := newEgressClaim("shard-a") + requests := egressShardForClaim(t.Context(), bound) + if len(requests) != 1 { + t.Fatalf("requests: got %d, want 1", len(requests)) + } + if requests[0].Name != "shard-a" || requests[0].Namespace != egressShardNamespace { + t.Errorf("got %v, want the bound shard's key", requests[0].NamespacedName) + } + + if requests := egressShardForClaim(t.Context(), newEgressClaim("")); len(requests) != 0 { + t.Errorf("an unbound claim mapped to %v", requests) + } +} diff --git a/internal/controller/egressshardclaim_controller.go b/internal/controller/egressshardclaim_controller.go new file mode 100644 index 0000000..131de3e --- /dev/null +++ b/internal/controller/egressshardclaim_controller.go @@ -0,0 +1,700 @@ +/* +Copyright © 2026 Datum Technology, Inc. All rights reserved. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package controller + +import ( + "cmp" + "context" + "errors" + "fmt" + "slices" + + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/selection" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// egressUnavailableMessage is what a consumer reads when this location +// provides no egress for their network. It names no node, no shard and no +// other consumer: a consumer cannot act on any of those, and each says where +// the platform runs their workload. The cause is logged and lives on the +// claim, which is an operator's object. +const egressUnavailableMessage = "No component in this location provides internet egress for this network" + +// bindingRefusedError is why one binding may not be made, carrying the named +// reason it is reported under. A binding that cannot be made says which +// incompatibility stopped it; a generic failure would leave an operator to work +// that out from the objects. +type bindingRefusedError struct { + reason string + message string +} + +func (e *bindingRefusedError) Error() string { return e.message } + +// EgressShardClaimReconciler binds a network's presence in this cell to one +// egress shard, once. +// +// It is the cell's single decision point for egress. The intent a location was +// instructed with says what the network needs; this decides which shard answers +// it, records that on the claim, and reports the result on the network context +// a consumer already reads. Everything downstream — the route a node installs, +// the address a consumer allow-lists — reads the binding rather than selecting +// again, because two selections made from the same inputs at different moments +// are two answers, and the datapath can hold one. +// +// Every claim binds a shared shard. Dedicated capacity is a hand-commissioned +// shard node and no controller can grow it while nothing allocates the +// identifier a shard is unusable without, so the platform withholds the value +// rather than accepting a request that would wait indefinitely — the same way +// it withholds reaching IPv4 destinations until a resolver and a translator +// share a prefix. The claim records the sharing it was created under and +// nothing here branches on it. +type EgressShardClaimReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkcontexts,verbs=get;list;watch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkcontexts/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=egressshardparameters,verbs=get;list;watch +// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=egressshardclaims,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=egressshardclaims/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshards,verbs=get;list;watch;update;patch + +func (r *EgressShardClaimReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // A claim carries its network context's name, so one key reads both: the + // instruction this cell was given, and the binding made from it. + var networkContext networkingv1alpha.NetworkContext + contextFound := true + if err := r.Get(ctx, req.NamespacedName, &networkContext); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("get NetworkContext %s: %w", req.NamespacedName, err) + } + contextFound = false + } + + var claim cloudv1alpha1.EgressShardClaim + claimFound := true + if err := r.Get(ctx, req.NamespacedName, &claim); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("get EgressShardClaim %s: %w", req.NamespacedName, err) + } + claimFound = false + } + if claimFound && !claim.DeletionTimestamp.IsZero() { + // The shard's finalizer is released by the reconciler watching shards, + // which sees this claim leave the consumer set. + return ctrl.Result{}, nil + } + + if !contextFound || !networkContext.DeletionTimestamp.IsZero() { + return ctrl.Result{}, r.releaseClaim(ctx, &claim, claimFound, + "the network is no longer present in this cell") + } + + intent := internetEgressIntent(&networkContext) + terms, err := r.claimTerms(ctx, &networkContext) + if err != nil { + var refused *bindingRefusedError + if errors.As(err, &refused) { + // Nothing can be bound and nothing is: no claim, so no route and no + // address, which is what a consumer reads as no egress here. The + // cause is logged rather than published, for the same reason the + // condition's message never carries one. + logf.FromContext(ctx).Info("nothing can be bound for this location's egress", + "networkContext", networkContext.Name, "reason", refused.reason, + "cause", refused.message) + if err := r.releaseClaim(ctx, &claim, claimFound, refused.message); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, r.reportContext(ctx, &networkContext, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, + egressUnavailableMessage) + } + return ctrl.Result{}, err + } + if terms == nil { + // Either this location was told to reach nothing, or its class is + // served by another implementation. Both are ordinary answers and + // neither is this controller's to report on. + return ctrl.Result{}, r.releaseClaim(ctx, &claim, claimFound, + "this location provides no internet egress this controller serves") + } + + if !claimFound { + return ctrl.Result{}, r.createClaim(ctx, &networkContext, terms) + } + + if !equality.Semantic.DeepEqual(claim.Spec, *terms) { + // The spec is immutable, so the terms cannot be brought into line. An + // unbound claim is discarded and rewritten; a bound one keeps + // delivering what it was bound for and says that it no longer matches. + if claim.Status.ShardRef == nil { + return ctrl.Result{}, r.releaseClaim(ctx, &claim, claimFound, + "the terms this location is instructed with changed before a shard was bound") + } + logf.FromContext(ctx).Info("the egress terms changed after a shard was bound; keeping the binding", + "claim", client.ObjectKeyFromObject(&claim), "shard", claim.Status.ShardRef.Name) + if err := r.publishClaimStatus(ctx, &claim, metav1.ConditionFalse, + cloudv1alpha1.EgressShardClaimReasonTermsChanged, + fmt.Sprintf("Network %q is bound to egress shard %q under terms this location no longer states; delete this claim to bind under the new ones", + claim.Spec.Network.Name, claim.Status.ShardRef.Name)); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, r.reportBoundContext(ctx, &networkContext, &claim) + } + + if claim.Status.ShardRef != nil { + // Decided once. Nothing here recomputes a binding: a rebinding moves a + // live VPC's egress to a different source address, which is the value a + // consumer allow-listed at their destination. + return ctrl.Result{}, r.reportExistingBinding(ctx, &networkContext, &claim) + } + + return ctrl.Result{}, r.bind(ctx, &networkContext, &claim, intent.ParametersRef.Name) +} + +// claimTerms are the terms a claim would be written with for this location, or +// nil when this controller has nothing to bind. +// +// It reads the instruction and never the class: every field it copies was +// resolved upstream, so nothing here selects a class, picks a default, or +// interprets a parameters reference beyond recognizing whether it is this +// controller's to serve. +func (r *EgressShardClaimReconciler) claimTerms( + ctx context.Context, networkContext *networkingv1alpha.NetworkContext, +) (*cloudv1alpha1.EgressShardClaimSpec, error) { + intent := internetEgressIntent(networkContext) + if intent == nil || intent.Mode != networkingv1alpha.NetworkInternetEgressEnabled { + return nil, nil + } + ref := intent.ParametersRef + if ref == nil { + return nil, &bindingRefusedError{ + reason: networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, + message: fmt.Sprintf("Internet egress class %q names no parameters, so which shards serve it is unstated", + intent.ClassName), + } + } + if ref.Group != cloudv1alpha1.GroupVersion.Group || ref.Kind != cloudv1alpha1.KindEgressShardParameters { + return nil, nil + } + + sharing, err := claimSharing(intent.Sharing) + if err != nil { + return nil, err + } + families, err := claimFamilies(intent.Reach) + if err != nil { + return nil, err + } + if networkContext.Spec.Network.Name == "" { + return nil, &bindingRefusedError{ + reason: networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, + message: "This location names no network, so there is nothing to bind to a shard", + } + } + // Read only to establish that the class this cell was pointed at exists + // here. Which shards it selects is read at the moment of binding. + if err := r.Get(ctx, client.ObjectKey{Name: ref.Name}, + &cloudv1alpha1.EgressShardParameters{}); err != nil { + if apierrors.IsNotFound(err) { + return nil, &bindingRefusedError{ + reason: cloudv1alpha1.EgressShardClaimReasonParametersUnavailable, + message: fmt.Sprintf("Internet egress class %q is served by parameters %q, which do not exist in this location", + intent.ClassName, ref.Name), + } + } + return nil, fmt.Errorf("get EgressShardParameters %s: %w", ref.Name, err) + } + + return &cloudv1alpha1.EgressShardClaimSpec{ + Network: cloudv1alpha1.NetworkRef{Name: networkContext.Spec.Network.Name}, + NetworkContext: cloudv1alpha1.NetworkContextRef{Name: networkContext.Name}, + ClassName: intent.ClassName, + Sharing: sharing, + Families: families, + }, nil +} + +// claimSharing carries the class's sharing onto the claim. +// +// An unprojected value is refused rather than defaulted. Sharing decides how +// many networks a shard may take, and there is no safe guess: reading it as +// Shared would place a network promised its own address alongside others, and +// reading it as Dedicated would hold a shard open for a network that asked for +// no such thing. +func claimSharing(sharing networkingv1alpha.InternetEgressSharing) (cloudv1alpha1.EgressSharing, error) { + switch sharing { + case networkingv1alpha.InternetEgressSharingShared: + return cloudv1alpha1.EgressSharingShared, nil + case networkingv1alpha.InternetEgressSharingDedicated: + return cloudv1alpha1.EgressSharingDedicated, nil + default: + return "", &bindingRefusedError{ + reason: networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, + message: "The serving class's sharing did not reach this location, so how many networks may share a shard is unknown here", + } + } +} + +// claimFamilies carries the families this location was told to reach. +// +// Only IPv6 is accepted, because it is the only family the instruction can +// carry and the only one a shard is selected for. A family that arrives anyway +// is refused rather than dropped: silently binding a shard that translates +// nothing for it would report egress a consumer does not have. +func claimFamilies( + reach []networkingv1alpha.IPFamily, +) ([]cloudv1alpha1.InternetEgressAddressFamily, error) { + if len(reach) == 0 { + return nil, &bindingRefusedError{ + reason: networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, + message: "This location was told to reach no address family, so no shard can be selected for it", + } + } + families := make([]cloudv1alpha1.InternetEgressAddressFamily, 0, len(reach)) + for _, family := range reach { + if family != networkingv1alpha.IPv6Protocol { + return nil, &bindingRefusedError{ + reason: networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, + message: fmt.Sprintf("This location was told to reach %s destinations, which no shard in it translates", + family), + } + } + families = append(families, cloudv1alpha1.InternetEgressAddressFamilyIPv6) + } + return families, nil +} + +// createClaim writes the one claim this location's egress is bound through. It +// is owned by the network context, so a network withdrawn from the cell takes +// its claim with it and the shard it held is released. +func (r *EgressShardClaimReconciler) createClaim( + ctx context.Context, + networkContext *networkingv1alpha.NetworkContext, + terms *cloudv1alpha1.EgressShardClaimSpec, +) error { + claim := &cloudv1alpha1.EgressShardClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: networkContext.Namespace, + Name: networkContext.Name, + }, + Spec: *terms, + } + if err := controllerutil.SetControllerReference(networkContext, claim, r.Scheme); err != nil { + return fmt.Errorf("set the owner on EgressShardClaim %s: %w", claim.Name, err) + } + if err := r.Create(ctx, claim); err != nil { + if apierrors.IsAlreadyExists(err) { + return nil + } + return fmt.Errorf("create EgressShardClaim %s: %w", client.ObjectKeyFromObject(claim), err) + } + logf.FromContext(ctx).Info("claimed an egress shard for a location", + "claim", client.ObjectKeyFromObject(claim), "network", terms.Network.Name, + "class", terms.ClassName, "sharing", terms.Sharing) + return nil +} + +// releaseClaim deletes the claim for a location that no longer has egress this +// controller provides. Deleting it is what releases the shard: the consumer set +// is a list of claims, so leaving the shard, and takes the route and the +// address with it. +func (r *EgressShardClaimReconciler) releaseClaim( + ctx context.Context, claim *cloudv1alpha1.EgressShardClaim, claimFound bool, why string, +) error { + if !claimFound { + return nil + } + if err := r.Delete(ctx, claim); err != nil { + return client.IgnoreNotFound(fmt.Errorf("delete EgressShardClaim %s: %w", + client.ObjectKeyFromObject(claim), err)) + } + logf.FromContext(ctx).Info("released an egress shard claim", + "claim", client.ObjectKeyFromObject(claim), "reason", why) + return nil +} + +// bind chooses the shard this network egresses through and records it. +// +// The candidates are the shards the class selects, in name order, and the first +// usable one wins. A refused candidate carries the reason it was refused, so a +// claim that binds nothing says which incompatibility stopped it rather than +// reporting a generic failure. A claim that found no usable shard waits: it is +// not refused, and nothing provisions a shard for it, because a shard is +// unusable until an operator commissions its identifier. +func (r *EgressShardClaimReconciler) bind( + ctx context.Context, + networkContext *networkingv1alpha.NetworkContext, + claim *cloudv1alpha1.EgressShardClaim, + parametersName string, +) error { + shards, err := egressShards(ctx, r.Client, parametersName) + if err != nil { + return err + } + if len(shards) == 0 { + return r.refuse(ctx, networkContext, claim, &bindingRefusedError{ + reason: cloudv1alpha1.EgressShardClaimReasonNoShardMatchesTheClass, + message: fmt.Sprintf("No egress shard carries the labels internet egress class %q selects", + claim.Spec.ClassName), + }) + } + + var firstRefusal *bindingRefusedError + for i := range shards { + refusal := shardRefusal(&shards[i]) + if refusal == nil { + return r.recordBinding(ctx, networkContext, claim, &shards[i]) + } + if firstRefusal == nil { + firstRefusal = refusal + } + } + return r.refuse(ctx, networkContext, claim, firstRefusal) +} + +// shardRefusal is why no claim may bind one shard, or nil if it may be bound. +// +// It reads the shard alone. Nothing about the claim asking narrows the set, +// because every claim takes a shared shard; when dedicated capacity is offered, +// this is where a claim's own terms would start to matter. +func shardRefusal(shard *bgpv1alpha1.EgressShard) *bindingRefusedError { + if shard.Status.ShardSID == "" { + return &bindingRefusedError{ + reason: cloudv1alpha1.EgressShardClaimReasonNoShardIdentifier, + message: fmt.Sprintf("Egress shard %q has no identifier a node can route toward, so binding it would carry no packet", + shard.Name), + } + } + if !shard.DeletionTimestamp.IsZero() { + return &bindingRefusedError{ + reason: cloudv1alpha1.EgressShardClaimReasonShardTerminating, + message: fmt.Sprintf("Egress shard %q is being deleted, so it takes no further network", + shard.Name), + } + } + return nil +} + +// recordBinding writes the binding: the shard's finalizer first, then the label +// that makes this claim part of the shard's consumer set, then the binding +// itself. +// +// The order is what keeps a crash between the writes harmless. A finalizer with +// no binding behind it is removed by the reconciler that watches shards, and a +// label with no binding behind it counts as no consumer and is overwritten by +// the next pass. A binding recorded before either would be a network egressing +// through a shard nothing holds open and nothing counts. +func (r *EgressShardClaimReconciler) recordBinding( + ctx context.Context, + networkContext *networkingv1alpha.NetworkContext, + claim *cloudv1alpha1.EgressShardClaim, + shard *bgpv1alpha1.EgressShard, +) error { + if err := holdShard(ctx, r.Client, shard); err != nil { + return err + } + if err := r.labelClaim(ctx, claim, shard.Name); err != nil { + return err + } + + claim.Status.ShardRef = &cloudv1alpha1.EgressShardReference{ + Namespace: shard.Namespace, + Name: shard.Name, + } + logf.FromContext(ctx).Info("bound a network's egress to a shard", + "claim", client.ObjectKeyFromObject(claim), "network", claim.Spec.Network.Name, + "shard", client.ObjectKeyFromObject(shard), "sharing", claim.Spec.Sharing) + + if err := r.publishClaimStatus(ctx, claim, metav1.ConditionTrue, + cloudv1alpha1.EgressShardClaimReasonBound, + fmt.Sprintf("Network %q egresses through egress shard %q", claim.Spec.Network.Name, shard.Name)); err != nil { + return err + } + return r.reportBinding(ctx, networkContext, shard) +} + +// labelClaim stamps the shard a claim is bound to, so the shard's consumer set +// is a list query. It is re-asserted on every pass over a bound claim, because +// a label lost to an edit would hide a network from the count that keeps a +// dedicated shard exclusive. +func (r *EgressShardClaimReconciler) labelClaim( + ctx context.Context, claim *cloudv1alpha1.EgressShardClaim, shardName string, +) error { + if claim.Labels[cloudv1alpha1.LabelEgressShardClaimShard] == shardName { + return nil + } + patch := client.MergeFrom(claim.DeepCopy()) + if claim.Labels == nil { + claim.Labels = map[string]string{} + } + claim.Labels[cloudv1alpha1.LabelEgressShardClaimShard] = shardName + if err := r.Patch(ctx, claim, patch); err != nil { + return fmt.Errorf("label EgressShardClaim %s with its shard: %w", + client.ObjectKeyFromObject(claim), err) + } + return nil +} + +// reportExistingBinding says what a binding already made is delivering, on the +// claim and on the network context, and repairs the label the consumer set is +// counted by. +func (r *EgressShardClaimReconciler) reportExistingBinding( + ctx context.Context, + networkContext *networkingv1alpha.NetworkContext, + claim *cloudv1alpha1.EgressShardClaim, +) error { + if err := r.labelClaim(ctx, claim, claim.Status.ShardRef.Name); err != nil { + return err + } + return r.reportBoundContext(ctx, networkContext, claim) +} + +// reportBoundContext reports a binding whose shard has to be read back for it. +func (r *EgressShardClaimReconciler) reportBoundContext( + ctx context.Context, + networkContext *networkingv1alpha.NetworkContext, + claim *cloudv1alpha1.EgressShardClaim, +) error { + var shard bgpv1alpha1.EgressShard + key := client.ObjectKey{ + Namespace: claim.Status.ShardRef.Namespace, + Name: claim.Status.ShardRef.Name, + } + if err := r.Get(ctx, key, &shard); err != nil { + if !apierrors.IsNotFound(err) { + return fmt.Errorf("get the bound EgressShard %s: %w", key, err) + } + // Nothing rebinds, so this network has no egress and no second answer + // coming. The finalizer exists to make this reachable only by someone + // removing it. + message := fmt.Sprintf("Egress shard %q no longer exists, and a binding is not remade", key.Name) + if err := r.publishClaimStatus(ctx, claim, metav1.ConditionFalse, + cloudv1alpha1.EgressShardClaimReasonShardMissing, message); err != nil { + return err + } + return r.reportContext(ctx, networkContext, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, + egressUnavailableMessage) + } + return r.reportBinding(ctx, networkContext, &shard) +} + +// reportBinding projects a binding onto the network context condition a +// consumer reads. +// +// Degraded is deliberately never written. It means egress works for some +// declared families and not others, and only one family is accepted anywhere on +// this path, so no state can reach it. +func (r *EgressShardClaimReconciler) reportBinding( + ctx context.Context, + networkContext *networkingv1alpha.NetworkContext, + shard *bgpv1alpha1.EgressShard, +) error { + if shard.Status.ShardAddressIPv6 == "" { + return r.reportContext(ctx, networkContext, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonAddressUnavailable, + "No egress address has been allocated for this location yet") + } + return r.reportContext(ctx, networkContext, metav1.ConditionTrue, + networkingv1alpha.NetworkContextInternetEgressReasonReady, + fmt.Sprintf("Instances in this location reach the internet, and %s is the address they reach it from", + shard.Status.ShardAddressIPv6)) +} + +// refuse records that nothing was bound, and why. The claim stays, unbound, +// and takes the first usable shard that appears. +func (r *EgressShardClaimReconciler) refuse( + ctx context.Context, + networkContext *networkingv1alpha.NetworkContext, + claim *cloudv1alpha1.EgressShardClaim, + refusal *bindingRefusedError, +) error { + if err := r.publishClaimStatus(ctx, claim, metav1.ConditionFalse, + refusal.reason, refusal.message); err != nil { + return err + } + // The context carries the fact about the consumer's network. Which shard + // refused it, and why, is on the claim, which is an operator's object. + return r.reportContext(ctx, networkContext, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, + egressUnavailableMessage) +} + +func (r *EgressShardClaimReconciler) publishClaimStatus( + ctx context.Context, + claim *cloudv1alpha1.EgressShardClaim, + status metav1.ConditionStatus, + reason, message string, +) error { + claim.Status.ObservedGeneration = claim.Generation + meta.SetStatusCondition(&claim.Status.Conditions, metav1.Condition{ + Type: cloudv1alpha1.ConditionTypeReady, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: claim.Generation, + }) + if err := r.Status().Update(ctx, claim); err != nil { + return fmt.Errorf("update EgressShardClaim %s status: %w", + client.ObjectKeyFromObject(claim), err) + } + return nil +} + +// reportContext writes the readiness a consumer reads for this location. +// +// The reasons were defined with no writer; this is the writer of all of them +// but the class-resolution refusal, which is written upstream where the class +// is read. The message states a fact about the consumer's network and names no +// node, no shard and no other consumer: a consumer cannot act on those, and +// they describe where the platform runs their workload. +func (r *EgressShardClaimReconciler) reportContext( + ctx context.Context, + networkContext *networkingv1alpha.NetworkContext, + status metav1.ConditionStatus, + reason, message string, +) error { + condition := metav1.Condition{ + Type: networkingv1alpha.NetworkContextInternetEgressReady, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: networkContext.Generation, + } + if !meta.SetStatusCondition(&networkContext.Status.Conditions, condition) { + return nil + } + if err := r.Status().Update(ctx, networkContext); err != nil { + return fmt.Errorf("update NetworkContext %s status: %w", + client.ObjectKeyFromObject(networkContext), err) + } + return nil +} + +// egressShards lists the shards a class's parameters select, in name order. +// +// It moved here from the interface controller with binding. The order used to +// be what made a candidate list a function of the matched set alone, so two +// attachments of one VPC could not compute different lists; now it is what +// makes the binding itself deterministic over the set of shards it saw. +func egressShards( + ctx context.Context, reader client.Reader, parametersName string, +) ([]bgpv1alpha1.EgressShard, error) { + var parameters cloudv1alpha1.EgressShardParameters + if err := reader.Get(ctx, client.ObjectKey{Name: parametersName}, ¶meters); err != nil { + return nil, fmt.Errorf("get EgressShardParameters %s: %w", parametersName, err) + } + + selector, err := metav1.LabelSelectorAsSelector(¶meters.Spec.ShardSelector) + if err != nil { + return nil, fmt.Errorf("parse the shard selector on EgressShardParameters %s: %w", + parameters.Name, err) + } + // Only IPv6 is reached, so a shard that translates no IPv6 flow is no + // candidate however an operator wrote the selector. The family label is + // matched on presence: absence, not a false value, means the family is + // unserved, so a shard predating the label never reads as serving one. + servesIPv6, err := labels.NewRequirement(bgpv1alpha1.LabelEgressShardIPv6, selection.Exists, nil) + if err != nil { + return nil, fmt.Errorf("build the IPv6 shard requirement: %w", err) + } + + var shards bgpv1alpha1.EgressShardList + if err := reader.List(ctx, &shards, + client.InNamespace(parameters.Spec.ShardNamespace), + client.MatchingLabelsSelector{Selector: selector.Add(*servesIPv6)}, + ); err != nil { + return nil, fmt.Errorf("list egress shards for EgressShardParameters %s: %w", + parameters.Name, err) + } + slices.SortFunc(shards.Items, func(a, b bgpv1alpha1.EgressShard) int { + return cmp.Compare(a.Name, b.Name) + }) + return shards.Items, nil +} + +// SetupWithManager registers the reconciler with the manager. +func (r *EgressShardClaimReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&cloudv1alpha1.EgressShardClaim{}, + builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { + // A bound claim is never reconsidered on its own events, which + // is what makes "decided once" a property of the controller + // rather than a check inside it. + claim, ok := object.(*cloudv1alpha1.EgressShardClaim) + return ok && claim.Status.ShardRef == nil + }))). + Watches(&networkingv1alpha.NetworkContext{}, + handler.EnqueueRequestsFromMapFunc(claimForNetworkContext)). + Watches(&bgpv1alpha1.EgressShard{}, + handler.EnqueueRequestsFromMapFunc(r.unboundClaimsForEgressShard)). + Named("egressshardclaim"). + Complete(r) +} + +// claimForNetworkContext maps a location to its one claim, which carries the +// same name. +func claimForNetworkContext(_ context.Context, object client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: client.ObjectKeyFromObject(object)}} +} + +// unboundClaimsForEgressShard wakes the claims still waiting for a shard when +// one arrives, reports its identifier, or leaves. +// +// Only unbound claims are enqueued. A bound one has nothing to recompute, and a +// shard arriving is exactly the moment a claim that was told there was no free +// dedicated shard can stop waiting. +func (r *EgressShardClaimReconciler) unboundClaimsForEgressShard( + ctx context.Context, _ client.Object, +) []reconcile.Request { + var claims cloudv1alpha1.EgressShardClaimList + if err := r.List(ctx, &claims); err != nil { + return nil + } + + requests := make([]reconcile.Request, 0, len(claims.Items)) + for i := range claims.Items { + if claims.Items[i].Status.ShardRef != nil { + continue + } + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&claims.Items[i]), + }) + } + return requests +} diff --git a/internal/controller/egressshardclaim_controller_test.go b/internal/controller/egressshardclaim_controller_test.go new file mode 100644 index 0000000..94cae04 --- /dev/null +++ b/internal/controller/egressshardclaim_controller_test.go @@ -0,0 +1,539 @@ +/* +Copyright © 2026 Datum Technology, Inc. All rights reserved. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package controller + +import ( + "testing" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" + networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// egressContextName is the network context every egress test binds, and +// therefore the name of the one claim that binds it. +const egressContextName = "default-us-central-1" + +func newBinder(t *testing.T, objects ...client.Object) (*EgressShardClaimReconciler, client.Client) { + t.Helper() + + scheme := runtime.NewScheme() + if err := networkingv1alpha.AddToScheme(scheme); err != nil { + t.Fatalf("build the networking scheme: %v", err) + } + if err := cloudv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("build the cloud scheme: %v", err) + } + if err := bgpv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("build the fabric scheme: %v", err) + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...). + WithStatusSubresource(&cloudv1alpha1.EgressShardClaim{}, &networkingv1alpha.NetworkContext{}). + Build() + return &EgressShardClaimReconciler{Client: fakeClient, Scheme: scheme}, fakeClient +} + +// newBoundContext is the location the binder works from: the projected intent, +// with the network it belongs to named. +func newBoundContext(mode networkingv1alpha.NetworkInternetEgressMode) *networkingv1alpha.NetworkContext { + networkContext := newEgressContext(mode) + networkContext.Spec.Network = networkingv1alpha.LocalNetworkRef{Name: "default"} + return networkContext +} + +func reconcileBinding(t *testing.T, r *EgressShardClaimReconciler) { + t.Helper() + key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressContextName} + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: key}); err != nil { + t.Fatalf("reconcile the claim: %v", err) + } +} + +func readClaim(t *testing.T, cl client.Client) *cloudv1alpha1.EgressShardClaim { + t.Helper() + var claim cloudv1alpha1.EgressShardClaim + key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressContextName} + if err := cl.Get(t.Context(), key, &claim); err != nil { + t.Fatalf("get the claim: %v", err) + } + return &claim +} + +func claimExists(t *testing.T, cl client.Client) bool { + t.Helper() + var claim cloudv1alpha1.EgressShardClaim + key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressContextName} + err := cl.Get(t.Context(), key, &claim) + return err == nil +} + +// One claim per network context that declares egress, carrying the terms the +// projection resolved and nothing this controller invented. +func TestBinderClaimsOncePerNetworkContext(t *testing.T) { + r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled), + newEgressParameters(), + newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) + + reconcileBinding(t, r) + + claim := readClaim(t, cl) + if claim.Spec.Network.Name != "default" { + t.Errorf("network: got %q, want default", claim.Spec.Network.Name) + } + if claim.Spec.NetworkContext.Name != egressContextName { + t.Errorf("network context: got %q, want %q", claim.Spec.NetworkContext.Name, egressContextName) + } + if claim.Spec.ClassName != "shared" { + t.Errorf("class: got %q, want shared", claim.Spec.ClassName) + } + if claim.Spec.Sharing != cloudv1alpha1.EgressSharingShared { + t.Errorf("sharing: got %q, want Shared", claim.Spec.Sharing) + } + if len(claim.Spec.Families) != 1 || + claim.Spec.Families[0] != cloudv1alpha1.InternetEgressAddressFamilyIPv6 { + t.Errorf("families: got %v, want [IPv6]", claim.Spec.Families) + } + // The claim names no shard, no selector, no address and no pool: the cell + // answers with the shard, and it answers on status. + if claim.Status.ShardRef != nil { + t.Errorf("the claim bound %v in the pass that wrote it", claim.Status.ShardRef) + } +} + +func TestBinderBindsTheClaimToAShard(t *testing.T) { + r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled), + newEgressParameters(), + newEgressShard("shard-b", "2001:db8:ff02::", "2001:db8:f00d::200", poolLabels()), + newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) + + reconcileBinding(t, r) + reconcileBinding(t, r) + + claim := readClaim(t, cl) + if claim.Status.ShardRef == nil { + t.Fatal("two usable shards bound nothing") + } + // Name order, moved here with the selection it belongs to: the binding has + // to be deterministic over the set of shards it saw. + if claim.Status.ShardRef.Name != "shard-a" { + t.Errorf("shard: got %q, want shard-a", claim.Status.ShardRef.Name) + } + if claim.Status.ShardRef.Namespace != egressShardNamespace { + t.Errorf("shard namespace: got %q, want %q", claim.Status.ShardRef.Namespace, egressShardNamespace) + } + // The label is what makes the shard's consumer set a list query, which is + // what stands in for the list of networks a shard does not hold. + if got := claim.Labels[cloudv1alpha1.LabelEgressShardClaimShard]; got != "shard-a" { + t.Errorf("shard label: got %q, want shard-a", got) + } + assertClaimCondition(t, cl, metav1.ConditionTrue, cloudv1alpha1.EgressShardClaimReasonBound) + assertContextCondition(t, cl, metav1.ConditionTrue, + networkingv1alpha.NetworkContextInternetEgressReasonReady) + + // The finalizer is the only state a binder puts on a shard, written before + // the binding so a recorded binding is never held by nothing. + var shard bgpv1alpha1.EgressShard + key := client.ObjectKey{Namespace: egressShardNamespace, Name: "shard-a"} + if err := cl.Get(t.Context(), key, &shard); err != nil { + t.Fatalf("get the bound shard: %v", err) + } + if !controllerutil.ContainsFinalizer(&shard, cloudv1alpha1.FinalizerEgressShardBinding) { + t.Error("the bound shard is not held open") + } + // Nothing else is written to it. A shard holds no list of the networks it + // serves and no count of them. + if len(shard.Labels) != len(poolLabels()) { + t.Errorf("the binder wrote labels onto the shard: %v", shard.Labels) + } + if shard.Spec.ShardAddressIPv6 != "2001:db8:f00d::100" { + t.Errorf("the binder rewrote the shard's address: %q", shard.Spec.ShardAddressIPv6) + } +} + +// Many networks bind one shard. Nothing branches on the sharing a claim +// records, because dedicated capacity is not offered. +func TestBinderBindsManyNetworksToOneShard(t *testing.T) { + first := newEgressClaim("shard-a") + first.Name = "other-us-central-1" + first.Spec.Network.Name = "other" + first.Spec.NetworkContext.Name = "other-us-central-1" + + r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled), + newEgressParameters(), first, + newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) + + reconcileBinding(t, r) + reconcileBinding(t, r) + + claim := readClaim(t, cl) + if claim.Status.ShardRef == nil || claim.Status.ShardRef.Name != "shard-a" { + t.Fatalf("got %v, want the shard another network already holds", claim.Status.ShardRef) + } +} + +// A shard with no identifier has nothing a node can route toward, so binding it +// would report egress that carries no packet. The claim waits, and the address +// stays unpublished. +func TestBinderWaitsForAShardItCanUse(t *testing.T) { + tests := []struct { + name string + objects []client.Object + reason string + }{ + { + name: "no shard carries the class's labels", + objects: []client.Object{newEgressParameters()}, + reason: cloudv1alpha1.EgressShardClaimReasonNoShardMatchesTheClass, + }, + { + name: "the only shard reports no identifier", + objects: []client.Object{newEgressParameters(), + newEgressShard("unprogrammed", "", "2001:db8:f00d::100", poolLabels())}, + reason: cloudv1alpha1.EgressShardClaimReasonNoShardIdentifier, + }, + { + name: "the only shard translates no IPv6", + objects: []client.Object{newEgressParameters(), + newEgressShard("ipv4-only", "2001:db8:ff01::", "", map[string]string{ + bgpv1alpha1.LabelEgressShardPool: "shared", + bgpv1alpha1.LabelEgressShardCell: "us-central-1", + bgpv1alpha1.LabelEgressShardIPv4: bgpv1alpha1.LabelValueEgressFamilyServed, + })}, + reason: cloudv1alpha1.EgressShardClaimReasonNoShardMatchesTheClass, + }, + { + name: "the only shard is being deleted", + objects: []client.Object{newEgressParameters(), + terminatingShard(newEgressShard("draining", "2001:db8:ff01::", + "2001:db8:f00d::100", poolLabels()))}, + reason: cloudv1alpha1.EgressShardClaimReasonShardTerminating, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + objects := append([]client.Object{ + newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled)}, test.objects...) + r, cl := newBinder(t, objects...) + + reconcileBinding(t, r) + reconcileBinding(t, r) + + claim := readClaim(t, cl) + if claim.Status.ShardRef != nil { + t.Fatalf("bound %v, want nothing", claim.Status.ShardRef) + } + assertClaimCondition(t, cl, metav1.ConditionFalse, test.reason) + // The consumer reads a fact about their own network. Which shard + // refused it is on the claim, which is an operator's object. + assertContextCondition(t, cl, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonUnavailable) + }) + } +} + +// Egress that works and an address that cannot yet be stated are different +// facts, and the condition says which. +func TestBinderReportsThatNoAddressIsAllocatedYet(t *testing.T) { + r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled), + newEgressParameters(), + newEgressShard("shard-a", "2001:db8:ff01::", "", poolLabels())) + + reconcileBinding(t, r) + reconcileBinding(t, r) + + claim := readClaim(t, cl) + if claim.Status.ShardRef == nil { + t.Fatal("a shard with an identifier and no address bound nothing") + } + assertClaimCondition(t, cl, metav1.ConditionTrue, cloudv1alpha1.EgressShardClaimReasonBound) + assertContextCondition(t, cl, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonAddressUnavailable) +} + +// Decided once. A shard that would sort ahead of the bound one arriving later +// does not move a live network's egress, which is the address a consumer +// allow-listed at their destination. +func TestBinderNeverRebinds(t *testing.T) { + r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled), + newEgressParameters(), + newEgressShard("shard-b", "2001:db8:ff02::", "2001:db8:f00d::200", poolLabels())) + + reconcileBinding(t, r) + reconcileBinding(t, r) + if got := readClaim(t, cl).Status.ShardRef; got == nil || got.Name != "shard-b" { + t.Fatalf("got %v, want shard-b", got) + } + + earlier := newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels()) + if err := cl.Create(t.Context(), earlier); err != nil { + t.Fatalf("commission a shard sorting earlier: %v", err) + } + reconcileBinding(t, r) + + if got := readClaim(t, cl).Status.ShardRef; got == nil || got.Name != "shard-b" { + t.Errorf("got %v, want the shard it was already bound to", got) + } +} + +// Egress withdrawn is a claim released, which is what takes the route and the +// address away and lets the shard go. +func TestBinderReleasesTheClaimWhenEgressIsWithdrawn(t *testing.T) { + tests := []struct { + name string + networkContext func() *networkingv1alpha.NetworkContext + }{ + { + name: "egress disabled", + networkContext: func() *networkingv1alpha.NetworkContext { + return newBoundContext(networkingv1alpha.NetworkInternetEgressDisabled) + }, + }, + { + name: "class served by another implementation", + networkContext: func() *networkingv1alpha.NetworkContext { + networkContext := newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled) + networkContext.Spec.Egress.Internet.ParametersRef.Kind = "SomeOtherParameters" + return networkContext + }, + }, + { + name: "intent never projected", + networkContext: func() *networkingv1alpha.NetworkContext { + networkContext := newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled) + networkContext.Spec.Egress = nil + return networkContext + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r, cl := newBinder(t, test.networkContext(), newEgressParameters(), + newEgressClaim("shard-a"), + newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) + + reconcileBinding(t, r) + + if claimExists(t, cl) { + t.Error("the claim survived the egress that justified it") + } + }) + } +} + +// A network no longer present in the cell takes its claim with it, rather than +// holding a shard open for a location that does not exist. +func TestBinderReleasesTheClaimWhenTheNetworkLeaves(t *testing.T) { + r, cl := newBinder(t, newEgressParameters(), newEgressClaim("shard-a"), + newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) + + reconcileBinding(t, r) + + if claimExists(t, cl) { + t.Error("the claim survived its network context") + } +} + +// Parameters an operator has not written in this cell are an answer, not a +// silent nothing: the class this cell was pointed at does not exist here. +func TestBinderReportsAbsentParameters(t *testing.T) { + r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled)) + + reconcileBinding(t, r) + + if claimExists(t, cl) { + t.Error("a claim was written for a class this cell cannot serve") + } + assertContextCondition(t, cl, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonUnavailable) +} + +// Sharing decides nothing here, but an unprojected value still means the claim +// cannot record what it was created under, and a claim is refused rather than +// written with a guess. +func TestBinderRefusesTermsItCannotRecord(t *testing.T) { + tests := []struct { + name string + amend func(*networkingv1alpha.NetworkContext) + }{ + { + name: "sharing never projected", + amend: func(networkContext *networkingv1alpha.NetworkContext) { + networkContext.Spec.Egress.Internet.Sharing = "" + }, + }, + { + name: "no address family to reach", + amend: func(networkContext *networkingv1alpha.NetworkContext) { + networkContext.Spec.Egress.Internet.Reach = nil + }, + }, + { + name: "location names no network", + amend: func(networkContext *networkingv1alpha.NetworkContext) { + networkContext.Spec.Network = networkingv1alpha.LocalNetworkRef{} + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + networkContext := newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled) + test.amend(networkContext) + r, cl := newBinder(t, networkContext, newEgressParameters(), + newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) + + reconcileBinding(t, r) + + if claimExists(t, cl) { + t.Error("a claim was written from terms it could not record") + } + assertContextCondition(t, cl, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonUnavailable) + }) + } +} + +// The terms are immutable, so a class change reaching a bound location cannot +// be applied to the binding. The binding keeps delivering what it was made for +// and says it no longer matches. +func TestBinderKeepsABindingWhoseTermsChanged(t *testing.T) { + networkContext := newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled) + networkContext.Spec.Egress.Internet.ClassName = "some-other-class" + + r, cl := newBinder(t, networkContext, newEgressParameters(), newEgressClaim("shard-a"), + newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) + + reconcileBinding(t, r) + + claim := readClaim(t, cl) + if claim.Status.ShardRef == nil || claim.Status.ShardRef.Name != "shard-a" { + t.Fatalf("got %v, want the binding it already had", claim.Status.ShardRef) + } + assertClaimCondition(t, cl, metav1.ConditionFalse, + cloudv1alpha1.EgressShardClaimReasonTermsChanged) +} + +// An unbound claim whose terms changed is discarded rather than kept, because +// nothing is bound to protect and the next pass writes one that matches. +func TestBinderDiscardsAnUnboundClaimWhoseTermsChanged(t *testing.T) { + networkContext := newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled) + networkContext.Spec.Egress.Internet.ClassName = "some-other-class" + + r, cl := newBinder(t, networkContext, newEgressParameters(), newEgressClaim(""), + newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) + + reconcileBinding(t, r) + + if claimExists(t, cl) { + t.Error("an unbound claim with stale terms was kept") + } +} + +// A label lost to an edit would hide a network from the query a shard's +// consumer set is counted by, so it is re-asserted on every pass. +func TestBinderRepairsTheShardLabel(t *testing.T) { + claim := newEgressClaim("shard-a") + claim.Labels = nil + + r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled), + newEgressParameters(), claim, + newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) + + reconcileBinding(t, r) + + if got := readClaim(t, cl).Labels[cloudv1alpha1.LabelEgressShardClaimShard]; got != "shard-a" { + t.Errorf("shard label: got %q, want shard-a", got) + } +} + +// Degraded means egress works for some declared families and not others. Only +// one family is accepted anywhere on this path, so nothing may write it — the +// reason stays defined and unreachable rather than being given a fabricated +// path to reach it. +func TestBinderNeverReportsDegraded(t *testing.T) { + r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled), + newEgressParameters(), + newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) + + reconcileBinding(t, r) + reconcileBinding(t, r) + + var networkContext networkingv1alpha.NetworkContext + key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressContextName} + if err := cl.Get(t.Context(), key, &networkContext); err != nil { + t.Fatalf("get the network context: %v", err) + } + condition := meta.FindStatusCondition(networkContext.Status.Conditions, + networkingv1alpha.NetworkContextInternetEgressReady) + if condition == nil { + t.Fatal("the location reports no egress readiness") + } + if condition.Reason == networkingv1alpha.NetworkContextInternetEgressReasonDegraded { + t.Error("Degraded was reported for a path that accepts one address family") + } +} + +func terminatingShard(shard *bgpv1alpha1.EgressShard) *bgpv1alpha1.EgressShard { + shard.Finalizers = []string{cloudv1alpha1.FinalizerEgressShardBinding} + deletion := metav1.Now() + shard.DeletionTimestamp = &deletion + return shard +} + +func assertClaimCondition( + t *testing.T, cl client.Client, status metav1.ConditionStatus, reason string, +) { + t.Helper() + condition := meta.FindStatusCondition(readClaim(t, cl).Status.Conditions, + cloudv1alpha1.ConditionTypeReady) + if condition == nil { + t.Fatal("the claim reports no Ready condition") + } + if condition.Status != status || condition.Reason != reason { + t.Errorf("claim Ready: got %s/%s, want %s/%s", + condition.Status, condition.Reason, status, reason) + } +} + +func assertContextCondition( + t *testing.T, cl client.Client, status metav1.ConditionStatus, reason string, +) { + t.Helper() + var networkContext networkingv1alpha.NetworkContext + key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressContextName} + if err := cl.Get(t.Context(), key, &networkContext); err != nil { + t.Fatalf("get the network context: %v", err) + } + condition := meta.FindStatusCondition(networkContext.Status.Conditions, + networkingv1alpha.NetworkContextInternetEgressReady) + if condition == nil { + t.Fatal("the location reports no egress readiness") + } + if condition.Status != status || condition.Reason != reason { + t.Errorf("InternetEgressReady: got %s/%s, want %s/%s", + condition.Status, condition.Reason, status, reason) + } +} diff --git a/internal/controller/networkinterface_controller.go b/internal/controller/networkinterface_controller.go index 69eb33b..7766b9d 100644 --- a/internal/controller/networkinterface_controller.go +++ b/internal/controller/networkinterface_controller.go @@ -18,19 +18,15 @@ along with this program. If not, see . package controller import ( - "cmp" "context" "fmt" - "slices" "time" nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -102,7 +98,7 @@ type NetworkInterfaceReconciler struct { // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaceclaims/status,verbs=get;update;patch // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkcontexts,verbs=get;list;watch // +kubebuilder:rbac:groups=cloud.datumapis.com,resources=vpcs,verbs=get;list;watch -// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=egressshardparameters,verbs=get;list;watch +// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=egressshardclaims,verbs=get;list;watch // +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshards,verbs=get;list;watch // +kubebuilder:rbac:groups=cloud.datumapis.com,resources=vpcattachments,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=cloud.datumapis.com,resources=vpcattachments/status,verbs=get;update;patch @@ -323,6 +319,9 @@ func interfaceAddresses(networkInterface *networkingv1alpha.NetworkInterface) [] // It is not an empty one: absence is the instruction, in the conflist and on // the attachment alike. type internetEgress struct { + // shardSIDs is the ordered candidate list the node reads. Binding yields + // one entry, and the shape stays an ordered array anyway so that a binder + // recording a standby shard later needs no change on the node. shardSIDs []string // sourceAddress is what translation writes, resolved from the shard the @@ -355,22 +354,33 @@ func (e *internetEgress) status() *cloudv1alpha1.VPCAttachmentEgressStatus { } } -// resolveInternetEgress turns the egress intent projected onto a NetworkContext -// into the ordered shard candidates a node routes this VPC's VRF toward, and -// the source address those candidates translate to. +// resolveInternetEgress reads the shard this location's egress was bound to, +// and renders the route the node installs and the address a consumer reads +// back. // -// Egress intent is a function of (VPC, cell) and of nothing else — not of the -// attachment, the interface, or the claim. The kernel VRF is shared by every -// attachment of a VPC on a node and the datapath's route key has no -// per-attachment component, so two attachments of one VPC asking for different -// egress is undefined: the last ADD wins and silently redirects the traffic of -// every attachment already up. Nothing here can express that divergence, -// because the only input is the NetworkContext, the VPC is named after it, and -// every attachment of a VPC therefore resolves the same context and computes -// the same list. The install is idempotent by construction rather than by a -// check. Keep it that way: an input read off the interface, the claim or the -// attachment, or a selection that is not deterministic over the shards it -// matched, breaks the invariant without breaking a test. +// It selects nothing. A binder made the decision once, recorded it on this +// location's claim, and the claim names one shard; reading it is all that +// happens here. That removed the second selection point, and with it a real +// divergence: this used to report the address of the first shard by name while +// the node installed the first shard whose SID it could resolve a route toward, +// and a node that is itself a shard can never resolve a route to its own +// advertised SID — BGP does not reflect a self-originated path back to its +// originator — while every compute node runs the translator. A workload on such +// a node therefore read one source address and egressed from another, silently +// breaking any allow-list built on it. One bound shard cannot diverge from +// itself, and a node that cannot reach the shard its network is bound to fails +// the attachment loudly instead. +// +// Egress intent remains a function of (VPC, cell) and of nothing else — not of +// the attachment, the interface, or the claim's consumer. The kernel VRF is +// shared by every attachment of a VPC on a node and the datapath's route key +// has no per-attachment component, so two attachments of one VPC asking for +// different egress is undefined: the last ADD wins and silently redirects the +// traffic of every attachment already up. Nothing here can express that +// divergence, because the only input is the NetworkContext, the VPC is named +// after it, and the one claim is named after it too. Keep it that way: an input +// read off the interface, the claim's consumer or the attachment breaks the +// invariant without breaking a test. func (r *NetworkInterfaceReconciler) resolveInternetEgress( ctx context.Context, networkContext *networkingv1alpha.NetworkContext, ) (*internetEgress, error) { @@ -391,13 +401,13 @@ func (r *NetworkInterfaceReconciler) resolveInternetEgress( ref := intent.ParametersRef if ref == nil { - log.Info("internet egress is enabled but the serving class names no parameters", - "networkContext", networkContext.Name, "class", intent.ClassName) return nil, nil } - // The reference is opaque, so this controller recognizes only its own - // parameters and leaves another implementation's class alone rather than - // guessing at a type it does not own. + // The reference is opaque, and recognizing it is the one thing still read + // from it here: this renders egress only for a class this implementation + // serves, and leaves another implementation's class alone rather than + // guessing at a type it does not own. Which shards that class selects is + // the binder's business. if ref.Group != cloudv1alpha1.GroupVersion.Group || ref.Kind != cloudv1alpha1.KindEgressShardParameters { log.V(1).Info("internet egress class is served by another implementation", "networkContext", networkContext.Name, "class", intent.ClassName, @@ -405,63 +415,58 @@ func (r *NetworkInterfaceReconciler) resolveInternetEgress( return nil, nil } - var parameters cloudv1alpha1.EgressShardParameters - if err := r.Get(ctx, client.ObjectKey{Name: ref.Name}, ¶meters); err != nil { + // One claim per network context, carrying its name. No claim, or a claim + // with nothing bound, is a network with no egress here — and it is reported + // as that on the context by the binder, which is why nothing is logged for + // it now. + var claim cloudv1alpha1.EgressShardClaim + if err := r.Get(ctx, client.ObjectKeyFromObject(networkContext), &claim); err != nil { if apierrors.IsNotFound(err) { - log.Info("internet egress parameters do not exist in this cell", - "networkContext", networkContext.Name, "class", intent.ClassName, - "parameters", ref.Name) return nil, nil } - return nil, fmt.Errorf("get EgressShardParameters %s: %w", ref.Name, err) + return nil, fmt.Errorf("get EgressShardClaim %s: %w", + client.ObjectKeyFromObject(networkContext), err) } - - shards, err := r.egressShards(ctx, ¶meters) - if err != nil { - return nil, err + if claim.Status.ShardRef == nil { + return nil, nil } - resolved := &internetEgress{shardSIDs: make([]string, 0, len(shards))} - var preferred *bgpv1alpha1.EgressShard - for i := range shards { - // A shard whose SID is unreported has nothing a node can route toward. - // The SID stays in status because nothing allocates one yet. - if shards[i].Status.ShardSID == "" { - continue - } - if slices.Contains(resolved.shardSIDs, shards[i].Status.ShardSID) { - continue - } - resolved.shardSIDs = append(resolved.shardSIDs, shards[i].Status.ShardSID) - if preferred == nil { - preferred = &shards[i] + shardKey := client.ObjectKey{ + Namespace: claim.Status.ShardRef.Namespace, + Name: claim.Status.ShardRef.Name, + } + var shard bgpv1alpha1.EgressShard + if err := r.Get(ctx, shardKey, &shard); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil } + return nil, fmt.Errorf("get the bound EgressShard %s: %w", shardKey, err) } - if len(resolved.shardSIDs) == 0 { - log.Info("internet egress is enabled but no shard serves this network", - "networkContext", networkContext.Name, "class", intent.ClassName, - "parameters", parameters.Name) + if shard.Status.ShardSID == "" { + // A shard whose SID is unreported has nothing a node can route toward. + // The SID stays in status because nothing allocates one yet. return nil, nil } - resolved.sourceAddress = sourceAddress(preferred, intent.Sharing) + resolved := &internetEgress{shardSIDs: []string{shard.Status.ShardSID}} + resolved.sourceAddress = sourceAddress(&shard, intent.Sharing) if resolved.sourceAddress == nil { log.Info("internet egress is bound but no source address can be reported", - "networkContext", networkContext.Name, "shard", preferred.Name, + "networkContext", networkContext.Name, "shard", shard.Name, "sharing", intent.Sharing) } log.V(1).Info("internet egress bound", "networkContext", networkContext.Name, - "class", intent.ClassName, "shardSIDs", resolved.shardSIDs, - "sourceAddress", resolved.sourceAddress) + "class", intent.ClassName, "shard", shard.Name, + "shardSIDs", resolved.shardSIDs, "sourceAddress", resolved.sourceAddress) return resolved, nil } -// sourceAddress is what a consumer reads back for the shard the node prefers. +// sourceAddress is what a consumer reads back for the shard this network is +// bound to. // -// The candidate list is a preference the node resolves down to one entry, so -// the first candidate is the shard traffic is intended to leave through and its -// address is the one to report. Reporting every candidate's address would tell -// a consumer their traffic leaves on addresses it does not. +// There is one, and it is the one the node routes toward, which is what makes +// the value reportable at all: the address a consumer allow-lists has to be the +// address their packets actually leave on. // // The address itself is write-once and immutable upstream, so a reported value // that changes means the shard it came from was replaced, not that the platform @@ -521,43 +526,6 @@ func internetEgressIntent( return networkContext.Spec.Egress.Internet } -// egressShards lists the shards the parameters select, in name order. -// -// The order is what makes the candidate list a function of the matched set -// alone: an unordered list would differ between two attachments of one VPC -// reconciled moments apart, which is exactly the divergence the VRF cannot -// represent. -func (r *NetworkInterfaceReconciler) egressShards( - ctx context.Context, parameters *cloudv1alpha1.EgressShardParameters, -) ([]bgpv1alpha1.EgressShard, error) { - selector, err := metav1.LabelSelectorAsSelector(¶meters.Spec.ShardSelector) - if err != nil { - return nil, fmt.Errorf("parse the shard selector on EgressShardParameters %s: %w", - parameters.Name, err) - } - // Only IPv6 is reached, so a shard that translates no IPv6 flow is no - // candidate however an operator wrote the selector. The family label is - // matched on presence: absence, not a false value, means the family is - // unserved, so a shard predating the label never reads as serving one. - servesIPv6, err := labels.NewRequirement(bgpv1alpha1.LabelEgressShardIPv6, selection.Exists, nil) - if err != nil { - return nil, fmt.Errorf("build the IPv6 shard requirement: %w", err) - } - - var shards bgpv1alpha1.EgressShardList - if err := r.List(ctx, &shards, - client.InNamespace(parameters.Spec.ShardNamespace), - client.MatchingLabelsSelector{Selector: selector.Add(*servesIPv6)}, - ); err != nil { - return nil, fmt.Errorf("list egress shards for EgressShardParameters %s: %w", - parameters.Name, err) - } - slices.SortFunc(shards.Items, func(a, b bgpv1alpha1.EgressShard) int { - return cmp.Compare(a.Name, b.Name) - }) - return shards.Items, nil -} - // allocateAttachmentIdentifier draws a random identifier unused within the VPC. // Random rather than lowest-free, so a freed identifier is not immediately // reissued while its BGPAdvertisement is still being garbage collected. @@ -705,6 +673,8 @@ func (r *NetworkInterfaceReconciler) SetupWithManager(mgr ctrl.Manager) error { handler.EnqueueRequestsFromMapFunc(r.interfacesForNetworkContext)). Watches(&bgpv1alpha1.EgressShard{}, handler.EnqueueRequestsFromMapFunc(r.interfacesForEgressShard)). + Watches(&cloudv1alpha1.EgressShardClaim{}, + handler.EnqueueRequestsFromMapFunc(r.interfacesForEgressShardClaim)). Named("networkinterface"). Complete(r) } @@ -733,27 +703,53 @@ func (r *NetworkInterfaceReconciler) interfacesForNetworkContext( return requests } -// interfacesForEgressShard re-renders every attachment in the cell when a shard -// arrives, reports its SID, or leaves. +// interfacesForEgressShard re-renders the attachments of the networks bound to +// a shard when it arrives, reports its SID, or leaves. // -// It enqueues everything rather than working out which networks a shard serves: -// the binding runs the other way, from a class's selector to the shards, so a -// shard cannot name the networks on it. The sweep is affordable because a shard -// is an operator-written object in one namespace and there are a handful of -// them per cell, and re-rendering an unaffected attachment writes nothing. +// Which networks those are is a list query over the claims carrying the shard's +// label, which is what a shard holding no list of its own is answered by. The +// cell-wide sweep this replaces was only needed while nothing recorded the +// binding. func (r *NetworkInterfaceReconciler) interfacesForEgressShard( - ctx context.Context, _ client.Object, + ctx context.Context, object client.Object, ) []reconcile.Request { - var interfaces networkingv1alpha.NetworkInterfaceList - if err := r.List(ctx, &interfaces); err != nil { + var claims cloudv1alpha1.EgressShardClaimList + if err := r.List(ctx, &claims, client.MatchingLabels{ + cloudv1alpha1.LabelEgressShardClaimShard: object.GetName(), + }); err != nil { return nil } - requests := make([]reconcile.Request, 0, len(interfaces.Items)) - for i := range interfaces.Items { - requests = append(requests, reconcile.Request{ - NamespacedName: client.ObjectKeyFromObject(&interfaces.Items[i]), - }) + var requests []reconcile.Request + for i := range claims.Items { + held := claims.Items[i].Status.ShardRef + if held == nil || held.Name != object.GetName() || held.Namespace != object.GetNamespace() { + continue + } + requests = append(requests, r.interfacesForNetworkContext(ctx, &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: claims.Items[i].Namespace, + Name: claims.Items[i].Spec.NetworkContext.Name, + }, + })...) } return requests } + +// interfacesForEgressShardClaim re-renders a location's attachments the moment +// its egress is bound, so a network that was waiting for a shard does not wait +// out a poll interval it does not have. +func (r *NetworkInterfaceReconciler) interfacesForEgressShardClaim( + ctx context.Context, object client.Object, +) []reconcile.Request { + claim, ok := object.(*cloudv1alpha1.EgressShardClaim) + if !ok { + return nil + } + return r.interfacesForNetworkContext(ctx, &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: claim.Namespace, + Name: claim.Spec.NetworkContext.Name, + }, + }) +} diff --git a/internal/controller/networkinterface_controller_test.go b/internal/controller/networkinterface_controller_test.go index 544ab6d..7f79eca 100644 --- a/internal/controller/networkinterface_controller_test.go +++ b/internal/controller/networkinterface_controller_test.go @@ -229,11 +229,35 @@ func newEgressContext(mode networkingv1alpha.NetworkInternetEgressMode) *network return networkContext } -func TestResolveInternetEgressSelectsMatchingShards(t *testing.T) { +// newEgressClaim is the binding a location's egress was decided into: the one +// claim per network context, naming the one shard it egresses through. +func newEgressClaim(shardName string) *cloudv1alpha1.EgressShardClaim { + claim := &cloudv1alpha1.EgressShardClaim{} + claim.Namespace = egressTestNamespace + claim.Name = "default-us-central-1" + claim.Labels = map[string]string{cloudv1alpha1.LabelEgressShardClaimShard: shardName} + claim.Spec = cloudv1alpha1.EgressShardClaimSpec{ + Network: cloudv1alpha1.NetworkRef{Name: "default"}, + NetworkContext: cloudv1alpha1.NetworkContextRef{Name: "default-us-central-1"}, + ClassName: "shared", + Sharing: cloudv1alpha1.EgressSharingShared, + Families: []cloudv1alpha1.InternetEgressAddressFamily{cloudv1alpha1.InternetEgressAddressFamilyIPv6}, + } + if shardName != "" { + claim.Status.ShardRef = &cloudv1alpha1.EgressShardReference{ + Namespace: egressShardNamespace, + Name: shardName, + } + } + return claim +} + +// The route a node installs, read off the binding rather than selected here. +func TestResolveInternetEgressReadsTheBoundShard(t *testing.T) { r := newEgressReconciler(t, - newEgressParameters(), - newEgressShard("shard-b", "2001:db8:ff02::", "2001:db8:1::2", poolLabels()), - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:1::1", poolLabels()), + newEgressClaim("shard-b"), + newEgressShard("shard-b", "2001:db8:ff02::", "2001:db8:f00d::200", poolLabels()), + newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels()), ) egress, err := r.resolveInternetEgress(t.Context(), @@ -242,17 +266,46 @@ func TestResolveInternetEgressSelectsMatchingShards(t *testing.T) { t.Fatalf("resolveInternetEgress: %v", err) } if egress == nil { - t.Fatal("two matching shards resolved no egress") + t.Fatal("a bound claim resolved no egress") } - // Name order, not list order: two attachments of one VPC reconciled moments - // apart have to compute the same list, because the VRF they share cannot - // hold two answers. - want := []string{"2001:db8:ff01::", "2001:db8:ff02::"} + // One entry, the bound shard's, even though another shard sorts ahead of it + // by name and matches the same class. + want := []string{"2001:db8:ff02::"} if !slices.Equal(egress.shardSIDs, want) { t.Errorf("shard SIDs: got %v, want %v", egress.shardSIDs, want) } } +// The bug binding removes. The node installs the first shard whose SID it can +// resolve a route toward, which is not the first shard by name: a node that is +// itself a shard can never resolve a route to its own advertised SID, and every +// compute node runs the translator. Reporting the first shard by name therefore +// told a workload on such a node one source address while its packets left on +// another, breaking any allow-list built on the value. One bound shard is one +// answer on both sides. +func TestResolveInternetEgressReportsTheBoundShardsOwnAddress(t *testing.T) { + r := newEgressReconciler(t, + newEgressClaim("shard-b"), + newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels()), + newEgressShard("shard-b", "2001:db8:ff02::", "2001:db8:f00d::200", poolLabels()), + ) + + egress, err := r.resolveInternetEgress(t.Context(), + newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled)) + if err != nil { + t.Fatalf("resolveInternetEgress: %v", err) + } + addresses := egress.status().Internet.SourceAddresses + if len(addresses) != 1 || addresses[0].Address != "2001:db8:f00d::200" { + t.Fatalf("got %v, want only the bound shard's address", addresses) + } + // The address reported and the SID the node routes toward have to come from + // the same shard, which is the property the divergence broke. + if !slices.Equal(egress.shardSIDs, []string{"2001:db8:ff02::"}) { + t.Errorf("shard SIDs: got %v, want the bound shard's", egress.shardSIDs) + } +} + // Every reason a network reaches nothing renders the same absent block. A node // that receives no block installs no route. func TestResolveInternetEgressYieldsNothingWhenUnbound(t *testing.T) { @@ -262,15 +315,14 @@ func TestResolveInternetEgressYieldsNothingWhenUnbound(t *testing.T) { noParameters := newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled) noParameters.Spec.Egress.Internet.ParametersRef = nil - missingParameters := newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled) - missingParameters.Spec.Egress.Internet.ParametersRef.Name = "not-in-this-cell" - unprojected := newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled) unprojected.Spec.Egress = nil noInternet := newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled) noInternet.Spec.Egress.Internet = nil + boundShard := newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels()) + tests := []struct { name string networkContext *networkingv1alpha.NetworkContext @@ -279,65 +331,53 @@ func TestResolveInternetEgressYieldsNothingWhenUnbound(t *testing.T) { { name: "disabled", networkContext: newEgressContext(networkingv1alpha.NetworkInternetEgressDisabled), - objects: []client.Object{newEgressParameters(), - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:1::1", poolLabels())}, + objects: []client.Object{newEgressClaim("shard-a"), boundShard}, }, { name: "mode never projected", networkContext: newEgressContext(""), - objects: []client.Object{newEgressParameters(), - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:1::1", poolLabels())}, + objects: []client.Object{newEgressClaim("shard-a"), boundShard}, }, { name: "egress never projected", networkContext: unprojected, - objects: []client.Object{newEgressParameters()}, + objects: []client.Object{newEgressClaim("shard-a"), boundShard}, }, { name: "no internet egress projected", networkContext: noInternet, - objects: []client.Object{newEgressParameters()}, + objects: []client.Object{newEgressClaim("shard-a"), boundShard}, }, { name: "class names no parameters", networkContext: noParameters, - objects: []client.Object{newEgressParameters()}, + objects: []client.Object{newEgressClaim("shard-a"), boundShard}, }, { name: "parameters owned by another implementation", networkContext: otherImplementation, - objects: []client.Object{newEgressParameters()}, + objects: []client.Object{newEgressClaim("shard-a"), boundShard}, }, { - name: "parameters absent from this cell", - networkContext: missingParameters, - objects: []client.Object{newEgressParameters()}, + name: "nothing claimed a shard for this location", + networkContext: newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + objects: []client.Object{boundShard}, }, { - name: "no shard matches the selector", + name: "the claim is still waiting for a shard", networkContext: newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), - objects: []client.Object{newEgressParameters(), - newEgressShard("elsewhere", "2001:db8:ff01::", "2001:db8:1::1", map[string]string{ - bgpv1alpha1.LabelEgressShardPool: "shared", - bgpv1alpha1.LabelEgressShardCell: "us-east-1", - bgpv1alpha1.LabelEgressShardIPv6: bgpv1alpha1.LabelValueEgressFamilyServed, - })}, + objects: []client.Object{newEgressClaim(""), boundShard}, }, { - name: "matching shard translates no IPv6", + name: "the bound shard is gone", networkContext: newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), - objects: []client.Object{newEgressParameters(), - newEgressShard("ipv4-only", "2001:db8:ff01::", "", map[string]string{ - bgpv1alpha1.LabelEgressShardPool: "shared", - bgpv1alpha1.LabelEgressShardCell: "us-central-1", - bgpv1alpha1.LabelEgressShardIPv4: bgpv1alpha1.LabelValueEgressFamilyServed, - })}, + objects: []client.Object{newEgressClaim("shard-a")}, }, { - name: "matching shard reports no SID", + name: "the bound shard reports no identifier", networkContext: newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), - objects: []client.Object{newEgressParameters(), - newEgressShard("unprogrammed", "", "2001:db8:1::1", poolLabels())}, + objects: []client.Object{newEgressClaim("unprogrammed"), + newEgressShard("unprogrammed", "", "2001:db8:f00d::100", poolLabels())}, }, } for _, test := range tests { @@ -363,33 +403,15 @@ func TestResolveInternetEgressYieldsNothingWhenUnbound(t *testing.T) { } } -// A SID an operator typed onto two shards is one candidate, not two. -func TestResolveInternetEgressDeduplicatesShardSIDs(t *testing.T) { - r := newEgressReconciler(t, - newEgressParameters(), - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:1::1", poolLabels()), - newEgressShard("shard-b", "2001:db8:ff01::", "2001:db8:1::2", poolLabels()), - ) - - egress, err := r.resolveInternetEgress(t.Context(), - newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled)) - if err != nil { - t.Fatalf("resolveInternetEgress: %v", err) - } - if want := []string{"2001:db8:ff01::"}; egress == nil || !slices.Equal(egress.shardSIDs, want) { - t.Errorf("shard SIDs: got %v, want %v", egress, want) - } -} - // The invariant the datapath depends on: intent is a function of (VPC, cell) // and nothing else, so every attachment of a VPC computes the same value and // the install is idempotent. This asserts the property at the only seam where // it could be broken — the resolver takes the context and nothing else. func TestResolveInternetEgressIsAFunctionOfTheNetworkContextAlone(t *testing.T) { r := newEgressReconciler(t, - newEgressParameters(), - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:1::1", poolLabels()), - newEgressShard("shard-b", "2001:db8:ff02::", "2001:db8:1::2", poolLabels()), + newEgressClaim("shard-a"), + newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels()), + newEgressShard("shard-b", "2001:db8:ff02::", "2001:db8:f00d::200", poolLabels()), ) networkContext := newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled) @@ -406,6 +428,28 @@ func TestResolveInternetEgressIsAFunctionOfTheNetworkContextAlone(t *testing.T) } } +// The ordered array shape stays even though binding yields one entry, so a +// binder recording a standby shard later needs no change on the node. +func TestResolveInternetEgressRendersAnOrderedCandidateList(t *testing.T) { + r := newEgressReconciler(t, + newEgressClaim("shard-a"), + newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels()), + ) + + egress, err := r.resolveInternetEgress(t.Context(), + newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled)) + if err != nil { + t.Fatalf("resolveInternetEgress: %v", err) + } + block := egress.conflist() + if block == nil { + t.Fatal("a bound claim rendered no conflist block") + } + if !slices.Equal(block.ShardSIDs, []string{"2001:db8:ff01::"}) { + t.Errorf("shardSIDs: got %v, want a one-entry list", block.ShardSIDs) + } +} + // The address a consumer reads back, and the contract that qualifies it. The // stability is derived here rather than by the consumer, so this is the only // place the class's sharing is interpreted. @@ -423,7 +467,7 @@ func TestResolveInternetEgressPublishesTheSourceAddress(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { r := newEgressReconciler(t, - newEgressParameters(), + newEgressClaim("shard-a"), newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels()), ) networkContext := newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled) @@ -469,7 +513,7 @@ func TestResolveInternetEgressWithholdsAnAddressItCannotState(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { r := newEgressReconciler(t, - newEgressParameters(), + newEgressClaim("shard-a"), newEgressShard("shard-a", "2001:db8:ff01::", test.address, poolLabels()), ) networkContext := newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled) @@ -489,27 +533,6 @@ func TestResolveInternetEgressWithholdsAnAddressItCannotState(t *testing.T) { } } -// One address is reported, for the shard the node prefers. Reporting every -// candidate's address would tell a consumer their traffic leaves on addresses -// it does not. -func TestResolveInternetEgressReportsThePreferredShardsAddress(t *testing.T) { - r := newEgressReconciler(t, - newEgressParameters(), - newEgressShard("shard-b", "2001:db8:ff02::", "2001:db8:f00d::200", poolLabels()), - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels()), - ) - - egress, err := r.resolveInternetEgress(t.Context(), - newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled)) - if err != nil { - t.Fatalf("resolveInternetEgress: %v", err) - } - addresses := egress.status().Internet.SourceAddresses - if len(addresses) != 1 || addresses[0].Address != "2001:db8:f00d::100" { - t.Errorf("got %v, want only the first candidate's address", addresses) - } -} - // Egress withdrawn has to be egress unreported. An address left behind on the // attachment is one a consumer keeps allow-listing after the path is gone. func TestPublishAttachmentStatusWithdrawsAnUnboundAddress(t *testing.T) { diff --git a/test/e2e/Taskfile.yaml b/test/e2e/Taskfile.yaml index 928c09f..923797d 100644 --- a/test/e2e/Taskfile.yaml +++ b/test/e2e/Taskfile.yaml @@ -58,6 +58,7 @@ tasks: crd/vpcs.cloud.datumapis.com crd/vpcattachments.cloud.datumapis.com crd/egressshardpools.cloud.datumapis.com + crd/egressshardclaims.cloud.datumapis.com --timeout=60s test: diff --git a/test/e2e/tests/egress-shard-claim-crd-schema/chainsaw-test.yaml b/test/e2e/tests/egress-shard-claim-crd-schema/chainsaw-test.yaml new file mode 100644 index 0000000..ee811b3 --- /dev/null +++ b/test/e2e/tests/egress-shard-claim-crd-schema/chainsaw-test.yaml @@ -0,0 +1,128 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: egress-shard-claim-crd-schema +spec: + description: > + Verify schema validation and storage for the EgressShardClaim CRD. The rule that + matters most is the immutable spec: a binding is decided once from these terms and + never recomputed, so terms that moved underneath it would describe a binding never + made under them. Does not require any controller — CRD schema is enforced by the API + server alone, and the binding itself lives on status, which no consumer writes. + steps: + - name: create-valid-claim + try: + - apply: + resource: + apiVersion: cloud.datumapis.com/v1alpha1 + kind: EgressShardClaim + metadata: + name: e2e-default-us-central-1 + spec: + network: + name: default + networkContext: + name: e2e-default-us-central-1 + className: shared + sharing: Shared + families: + - IPv6 + - assert: + resource: + apiVersion: cloud.datumapis.com/v1alpha1 + kind: EgressShardClaim + metadata: + name: e2e-default-us-central-1 + spec: + className: shared + sharing: Shared + + - name: reject-spec-change + try: + - script: + content: | + set +e + OUTPUT=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF' + apiVersion: cloud.datumapis.com/v1alpha1 + kind: EgressShardClaim + metadata: + name: e2e-default-us-central-1 + spec: + network: + name: default + networkContext: + name: e2e-default-us-central-1 + className: some-other-class + sharing: Shared + families: + - IPv6 + EOF + ) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: the terms a binding was decided from were changed under it" + exit 1 + fi + echo "$OUTPUT" | grep -q "spec is immutable" || { + echo "ERROR: rejected for the wrong reason: $OUTPUT" + exit 1 + } + echo "OK: the terms are immutable once written" + + - name: reject-ipv4-family + try: + - script: + content: | + set +e + OUTPUT=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF' + apiVersion: cloud.datumapis.com/v1alpha1 + kind: EgressShardClaim + metadata: + name: e2e-reaches-ipv4 + spec: + network: + name: default + networkContext: + name: e2e-reaches-ipv4 + className: shared + sharing: Shared + families: + - IPv4 + EOF + ) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: a claim requiring a family no shard translates was accepted" + exit 1 + fi + echo "OK: only the family the platform delivers is accepted" + + - name: reject-empty-families + try: + - script: + content: | + set +e + OUTPUT=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF' + apiVersion: cloud.datumapis.com/v1alpha1 + kind: EgressShardClaim + metadata: + name: e2e-reaches-nothing + spec: + network: + name: default + networkContext: + name: e2e-reaches-nothing + className: shared + sharing: Shared + families: [] + EOF + ) + EXIT=$? + set -e + if [ "$EXIT" -eq 0 ]; then + echo "ERROR: a claim reaching no address family was accepted" + exit 1 + fi + echo "OK: a claim must state what it reaches" From 2acb51e8205d1c4dfb24edfd5feb173b742bfdda Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 24 Sep 2026 18:07:57 -0500 Subject: [PATCH 3/4] feat: Record each attachment's egress shard by node Translation runs on the node an instance attaches to, so nothing selects a shard. The claim becomes a record: one per attachment, owned by it, naming the node the attachment landed on and the families its network declared. The binder resolves the shard on that node and writes it to status, and reports egress readiness on the attachment. The record decides nothing. The node routes toward its own shard from the moment the attachment exists, and the record follows once the attachment has reported where it landed. Key changes: - Reshape EgressShardClaim to attachment, nodeName, and families - Bind by node reference, refusing a shard that is not ready, whose spec and status disagree, or that serves no declared family - Report InternetEgressReady on the attachment, patched so the controller rendering it is not overwritten - Replace a record when its attachment moves nodes - Remove EgressShardPool, its controller, CRD, and tests - Merge the attachment branch that carries the declaration to the node Co-Authored-By: Claude Fable 5.1 --- api/v1alpha1/egressshardclaim_types.go | 204 ++----- api/v1alpha1/vpcattachment_types.go | 11 + api/v1alpha1/zz_generated.deepcopy.go | 137 +++++ ...cloud.datumapis.com_egressshardclaims.yaml | 124 ++-- config/rbac/role.yaml | 11 + docs/api/vpc.md | 108 ++++ .../egressshardbinding_controller_test.go | 8 +- .../controller/egressshardclaim_controller.go | 547 +++++++----------- .../egressshardclaim_controller_test.go | 467 ++++++--------- .../controller/networkinterface_controller.go | 8 +- .../chainsaw-test.yaml | 73 +-- 11 files changed, 819 insertions(+), 879 deletions(-) diff --git a/api/v1alpha1/egressshardclaim_types.go b/api/v1alpha1/egressshardclaim_types.go index fc6a19a..749719e 100644 --- a/api/v1alpha1/egressshardclaim_types.go +++ b/api/v1alpha1/egressshardclaim_types.go @@ -24,12 +24,9 @@ import ( // LabelEgressShardClaimShard names the shard a claim is bound to. // // It is what makes a shard's consumer set a list rather than a number anyone -// has to keep in step: a shard holds no list of the networks it serves, so -// "which networks does this shard serve" is answered by listing claims -// carrying this label, the same way the network and location labels make -// counting a presence's consumers a list query. -// -// The value is the shard's name. The binding itself lives on the claim's +// has to keep in step: a shard holds no list of the attachments it serves, so +// "what does this shard serve" is answered by listing claims carrying this +// label. The value is the shard's name. The binding itself lives on the claim's // status, which is what a reader trusts; this label narrows the query that // finds the claims to ask. const LabelEgressShardClaimShard = "cloud.datumapis.com/egress-shard" @@ -38,75 +35,38 @@ const LabelEgressShardClaimShard = "cloud.datumapis.com/egress-shard" // shard, held while any claim is bound to it. // // It exists so that decommissioning a shard is an act someone takes rather -// than an outcome networks discover. Deleting a shard that is translating -// strands the return traffic of every flow on it, and nothing rebinds a claim: -// a binding is decided once, so a network whose shard vanished has no egress -// and no second answer coming. +// than an outcome instances discover. Deleting a shard that is translating +// strands the return traffic of every flow on it. const FinalizerEgressShardBinding = "cloud.datumapis.com/egress-shard-binding" -// EgressSharing is how many networks may share one egress shard. It is the -// serving class's sharing, copied verbatim and recorded as the fact this -// binding was made under. -// -// Nothing branches on it. Every claim binds a shared shard, because dedicated -// capacity is a hand-commissioned shard node that no controller can grow while -// nothing allocates the identifier a shard is unusable without — a claim beyond -// that count would wait indefinitely. The value is carried because the -// projection writes it and a claim records what it was created from. +// EgressShardClaimSpec is one attachment being recorded against the egress +// shard on its node. // -// +kubebuilder:validation:Enum=Shared;Dedicated -type EgressSharing string - -const ( - // EgressSharingShared lets many networks bind one shard and therefore - // leave the platform on one address. - EgressSharingShared EgressSharing = "Shared" - - // EgressSharingDedicated would let exactly one network bind a shard, which - // is what makes that shard's address the network's own. It is not offered - // yet and nothing here enforces it; the value is defined so that a claim - // written when it is offered means today what it will mean then. - EgressSharingDedicated EgressSharing = "Dedicated" -) - -// EgressShardClaimSpec is the network being bound to an egress shard, and the -// terms the binding has to satisfy. +// The claim decides nothing. The node already routes toward its own shard from +// the moment the attachment exists; the claim records which shard that is, so +// the binding is readable, so a node without a usable shard produces a +// condition a consumer can see, and so a later tier that does select among +// shards binds through the same object. // -// Every field is already resolved upstream and copied here verbatim. Nothing -// reading a claim selects a class, picks a default, or interprets a class's -// parameters. +// The whole spec is immutable. An attachment that lands on a different node is +// a different record, so the claim is replaced rather than edited. // -// The whole spec is immutable. The binding is decided once from these facts -// and never recomputed, so a fact that moved underneath it would describe a -// binding that was never made under it. A consumer changing what they asked -// for is a claim deleted and a new one written, which is a crossing someone -// can see. -// -// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec is immutable; a binding is decided once, so delete the claim to ask for different terms" +// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec is immutable; an attachment that moved nodes gets a new claim" type EgressShardClaimSpec struct { - // Network is the network reaching the internet. - // +required - Network NetworkRef `json:"network"` - - // NetworkContext is that network's presence in this cell, which is what - // makes the claim one per location. + // Attachment is the attachment this claim records egress for. The claim + // carries the attachment's name and namespace, so the two are read by one + // key. // +required - NetworkContext NetworkContextRef `json:"networkContext"` + Attachment AttachmentRef `json:"attachment"` - // ClassName is the InternetEgressClass resolved for this network. It is - // recorded rather than read: the class is cluster-scoped upstream and no - // copy of it reaches this cell. + // NodeName is the node the attachment landed on, and therefore the node + // whose shard serves it. // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:MaxLength=253 // +required - ClassName string `json:"className"` + NodeName string `json:"nodeName"` - // Sharing is how many networks that class allows on one shard. - // +required - Sharing EgressSharing `json:"sharing"` - - // Families are the destination address families this binding has to reach, - // so the shard it binds is one that translates them. + // Families are the destination address families the network declared, + // so the shard on the node is one that translates them. // +listType=set // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=2 @@ -114,46 +74,28 @@ type EgressShardClaimSpec struct { Families []InternetEgressAddressFamily `json:"families"` } -// NetworkRef references a networking.datumapis.com Network by name. -type NetworkRef struct { - // Name of the Network. +// AttachmentRef references a VPCAttachment by name. +type AttachmentRef struct { + // Name of the VPCAttachment. // +kubebuilder:validation:MinLength=1 // +required Name string `json:"name"` } -// NetworkContextRef references a networking.datumapis.com NetworkContext by -// name in the same namespace. -type NetworkContextRef struct { - // Name of the NetworkContext. - // +kubebuilder:validation:MinLength=1 - // +required - Name string `json:"name"` -} - -// EgressShardReference names one egress shard. +// EgressShardReference names the shard an attachment egresses through. type EgressShardReference struct { - // Namespace holding the shard. It is stated rather than assumed: the - // shards are in the namespace an operator gave the serving class, which is - // not the namespace a claim lives in. + // Namespace of the EgressShard. // +kubebuilder:validation:MinLength=1 // +required Namespace string `json:"namespace"` - // Name of the shard. + // Name of the EgressShard. // +kubebuilder:validation:MinLength=1 // +required Name string `json:"name"` } -// EgressShardClaimStatus is the binding. -// -// The binding is recorded here and nowhere else. The shard side carries no -// reference back, unlike the interface and subnet claims this follows in every -// other respect: both of those are strictly one-to-one and the reference on the -// provisioned object is what enforces it, whereas many networks share one -// shard, so a shard-side reference would have to be a list of the networks -// served — which is the state a shard deliberately does not hold. +// EgressShardClaimStatus is the shard an attachment was recorded against. type EgressShardClaimStatus struct { // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` @@ -163,93 +105,69 @@ type EgressShardClaimStatus struct { // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` - // ShardRef is the shard this network egresses through. - // - // Absent means no shard is bound, which is what a location whose cell holds - // no usable shard reads. Nothing publishes an address or a route in that - // state: an address a consumer might allow-list is withheld until the - // platform can state which one their packets leave on. + // ShardRef is the shard on the attachment's node. // - // Present is permanent for this claim's life. It is written once, and - // nothing recomputes it: a rebinding would move a live VPC's egress to a - // different source address, which is the value a consumer allow-listed at - // their destination. + // Absent means the node holds no shard this claim can record, which is + // what an attachment on a node an operator has not commissioned reads. // +optional ShardRef *EgressShardReference `json:"shardRef,omitempty"` } // Reasons reported on an EgressShardClaim's Ready condition. const ( - // EgressShardClaimReasonBound means this network egresses through the + // EgressShardClaimReasonBound means this attachment egresses through the // shard status names. EgressShardClaimReasonBound = "Bound" - // EgressShardClaimReasonParametersUnavailable means the parameters the - // serving class names do not exist in this cell, so which shards serve the - // class is unknown here. - EgressShardClaimReasonParametersUnavailable = "ParametersUnavailable" + // EgressShardClaimReasonNoShardOnNode means no shard names the node the + // attachment landed on. + EgressShardClaimReasonNoShardOnNode = "NoShardOnNode" - // EgressShardClaimReasonNoShardMatchesTheClass means no shard in the - // namespace the class names carries the labels its selector requires. - EgressShardClaimReasonNoShardMatchesTheClass = "NoShardMatchesTheClass" + // EgressShardClaimReasonShardNotReady means the shard on the node has not + // reported the identifier a node routes toward. + EgressShardClaimReasonShardNotReady = "ShardNotReady" - // EgressShardClaimReasonNoShardIdentifier means every shard the class - // selects is still without the SRv6 identifier a node routes toward, so - // there is nothing to bind that would carry a packet. - EgressShardClaimReasonNoShardIdentifier = "NoShardIdentifier" + // EgressShardClaimReasonShardMismatch means the shard's spec and the + // identity its process reported disagree, so which one the node runs is + // unknown and nothing is recorded against it. + EgressShardClaimReasonShardMismatch = "ShardMismatch" - // EgressShardClaimReasonShardMissing means the bound shard no longer - // exists. Nothing rebinds a claim, so this network has no egress and no - // second answer coming; the finalizer is what makes the state reachable - // only by someone removing it. + // EgressShardClaimReasonFamilyUnsupported means the shard on the node + // translates none of a family the network declared. + EgressShardClaimReasonFamilyUnsupported = "FamilyUnsupported" + + // EgressShardClaimReasonShardMissing means the recorded shard no longer + // exists. The node's instances lost their egress with it. EgressShardClaimReasonShardMissing = "ShardMissing" - // EgressShardClaimReasonShardTerminating means the bound shard is being - // deleted. The binding stands, because nothing rebinds a claim, and the - // shard is held until the claim is gone. + // EgressShardClaimReasonShardTerminating means the recorded shard is being + // deleted. The record stands, and the shard is held until the claim goes. EgressShardClaimReasonShardTerminating = "ShardTerminating" - - // EgressShardClaimReasonTermsChanged means the egress this location is - // instructed to provide no longer matches the terms this binding was made - // under. The binding stands and delivers what it always did; changing the - // terms means deleting the claim. - EgressShardClaimReasonTermsChanged = "TermsChanged" ) // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:scope=Namespaced -// +kubebuilder:printcolumn:name="Network",type="string",JSONPath=".spec.network.name" -// +kubebuilder:printcolumn:name="Sharing",type="string",JSONPath=".spec.sharing" +// +kubebuilder:printcolumn:name="Attachment",type="string",JSONPath=".spec.attachment.name" +// +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".spec.nodeName" // +kubebuilder:printcolumn:name="Shard",type="string",JSONPath=".status.shardRef.name" // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=`.status.conditions[?(@.type=="Ready")].status` // +kubebuilder:printcolumn:name="Reason",type="string",JSONPath=`.status.conditions[?(@.type=="Ready")].reason` // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" -// EgressShardClaim is one network being bound to one egress shard in this cell. -// -// There is one claim per network context that declares egress — not one per -// interface and not one per attachment. The binding has to outlive the -// workloads using it: an instance is replaced routinely, and a binding that -// followed an attachment would move a network's source address every time that -// happened, which is the address a consumer allow-listed at their destination. +// EgressShardClaim records one attachment's egress shard: the shard on the +// node the attachment landed on. // -// The claim names no shard, no selector, no address and no pool. It states what -// the network needs and the cell answers with which shard serves it, the same -// division a subnet claim makes. +// There is one claim per attachment, owned by it, so an attachment that goes +// takes its record with it. The claim names no selector, no address and no +// pool: the node is the binding, and the claim writes it down. type EgressShardClaim struct { - metav1.TypeMeta `json:",inline"` - - // metadata is a standard object metadata. The name is the network context's - // own, because there is exactly one claim per context. - // +optional + metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - // spec is the network being bound and the terms the binding satisfies // +required Spec EgressShardClaimSpec `json:"spec"` - // status is the binding // +optional Status EgressShardClaimStatus `json:"status,omitempty"` } diff --git a/api/v1alpha1/vpcattachment_types.go b/api/v1alpha1/vpcattachment_types.go index 674ee0c..40f0968 100644 --- a/api/v1alpha1/vpcattachment_types.go +++ b/api/v1alpha1/vpcattachment_types.go @@ -28,6 +28,12 @@ const ( // NetworkAttachmentDefinition is written. ConditionTypeReady = "Ready" + // ConditionTypeInternetEgressReady reports whether this attachment's + // outbound traffic reaches the destinations its network declared. It is + // written by the controller that records the shard on the attachment's + // node, and read back onto the interface a consumer holds. + ConditionTypeInternetEgressReady = "InternetEgressReady" + // ConditionTypeProgrammed reports that the data plane realized the attachment. ConditionTypeProgrammed = "Programmed" ) @@ -128,6 +134,11 @@ type InternetEgressAddressFamily string // InternetEgressAddressFamilyIPv6 is an IPv6 egress source address. const InternetEgressAddressFamilyIPv6 InternetEgressAddressFamily = "IPv6" +// InternetEgressAddressFamilyIPv4 is an IPv4 egress source address. The +// family type does not admit it yet, so nothing can ask for it; it is defined +// so the binder's family check means today what it will mean then. +const InternetEgressAddressFamilyIPv4 InternetEgressAddressFamily = "IPv4" + // InternetEgressAddressStability is how far a consumer may rely on an egress // source address. It is the consumer-side projection of the serving class's // sharing, derived here so a consumer never reads a class. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index d76dd55..47bb875 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -26,6 +26,143 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AttachmentRef) DeepCopyInto(out *AttachmentRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AttachmentRef. +func (in *AttachmentRef) DeepCopy() *AttachmentRef { + if in == nil { + return nil + } + out := new(AttachmentRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressShardClaim) DeepCopyInto(out *EgressShardClaim) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardClaim. +func (in *EgressShardClaim) DeepCopy() *EgressShardClaim { + if in == nil { + return nil + } + out := new(EgressShardClaim) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressShardClaim) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressShardClaimList) DeepCopyInto(out *EgressShardClaimList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]EgressShardClaim, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardClaimList. +func (in *EgressShardClaimList) DeepCopy() *EgressShardClaimList { + if in == nil { + return nil + } + out := new(EgressShardClaimList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *EgressShardClaimList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressShardClaimSpec) DeepCopyInto(out *EgressShardClaimSpec) { + *out = *in + out.Attachment = in.Attachment + if in.Families != nil { + in, out := &in.Families, &out.Families + *out = make([]InternetEgressAddressFamily, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardClaimSpec. +func (in *EgressShardClaimSpec) DeepCopy() *EgressShardClaimSpec { + if in == nil { + return nil + } + out := new(EgressShardClaimSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressShardClaimStatus) DeepCopyInto(out *EgressShardClaimStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ShardRef != nil { + in, out := &in.ShardRef, &out.ShardRef + *out = new(EgressShardReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardClaimStatus. +func (in *EgressShardClaimStatus) DeepCopy() *EgressShardClaimStatus { + if in == nil { + return nil + } + out := new(EgressShardClaimStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressShardReference) DeepCopyInto(out *EgressShardReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardReference. +func (in *EgressShardReference) DeepCopy() *EgressShardReference { + if in == nil { + return nil + } + out := new(EgressShardReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InternetEgressSourceAddress) DeepCopyInto(out *InternetEgressSourceAddress) { *out = *in diff --git a/config/crd/cloud.datumapis.com_egressshardclaims.yaml b/config/crd/cloud.datumapis.com_egressshardclaims.yaml index 305e831..9d800b2 100644 --- a/config/crd/cloud.datumapis.com_egressshardclaims.yaml +++ b/config/crd/cloud.datumapis.com_egressshardclaims.yaml @@ -15,11 +15,11 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - jsonPath: .spec.network.name - name: Network + - jsonPath: .spec.attachment.name + name: Attachment type: string - - jsonPath: .spec.sharing - name: Sharing + - jsonPath: .spec.nodeName + name: Node type: string - jsonPath: .status.shardRef.name name: Shard @@ -37,17 +37,12 @@ spec: schema: openAPIV3Schema: description: |- - EgressShardClaim is one network being bound to one egress shard in this cell. + EgressShardClaim records one attachment's egress shard: the shard on the + node the attachment landed on. - There is one claim per network context that declares egress — not one per - interface and not one per attachment. The binding has to outlive the - workloads using it: an instance is replaced routinely, and a binding that - followed an attachment would move a network's source address every time that - happened, which is the address a consumer allow-listed at their destination. - - The claim names no shard, no selector, no address and no pool. It states what - the network needs and the cell answers with which shard serves it, the same - division a subnet claim makes. + There is one claim per attachment, owned by it, so an attachment that goes + takes its record with it. The claim names no selector, no address and no + pool: the node is the binding, and the claim writes it down. properties: apiVersion: description: |- @@ -67,21 +62,36 @@ spec: metadata: type: object spec: - description: spec is the network being bound and the terms the binding - satisfies + description: |- + EgressShardClaimSpec is one attachment being recorded against the egress + shard on its node. + + The claim decides nothing. The node already routes toward its own shard from + the moment the attachment exists; the claim records which shard that is, so + the binding is readable, so a node without a usable shard produces a + condition a consumer can see, and so a later tier that does select among + shards binds through the same object. + + The whole spec is immutable. An attachment that lands on a different node is + a different record, so the claim is replaced rather than edited. properties: - className: + attachment: description: |- - ClassName is the InternetEgressClass resolved for this network. It is - recorded rather than read: the class is cluster-scoped upstream and no - copy of it reaches this cell. - maxLength: 253 - minLength: 1 - type: string + Attachment is the attachment this claim records egress for. The claim + carries the attachment's name and namespace, so the two are read by one + key. + properties: + name: + description: Name of the VPCAttachment. + minLength: 1 + type: string + required: + - name + type: object families: description: |- - Families are the destination address families this binding has to reach, - so the shard it binds is one that translates them. + Families are the destination address families the network declared, + so the shard on the node is one that translates them. items: description: |- InternetEgressAddressFamily is the address family of an egress source @@ -98,48 +108,24 @@ spec: minItems: 1 type: array x-kubernetes-list-type: set - network: - description: Network is the network reaching the internet. - properties: - name: - description: Name of the Network. - minLength: 1 - type: string - required: - - name - type: object - networkContext: + nodeName: description: |- - NetworkContext is that network's presence in this cell, which is what - makes the claim one per location. - properties: - name: - description: Name of the NetworkContext. - minLength: 1 - type: string - required: - - name - type: object - sharing: - description: Sharing is how many networks that class allows on one - shard. - enum: - - Shared - - Dedicated + NodeName is the node the attachment landed on, and therefore the node + whose shard serves it. + minLength: 1 type: string required: - - className + - attachment - families - - network - - networkContext - - sharing + - nodeName type: object x-kubernetes-validations: - - message: spec is immutable; a binding is decided once, so delete the - claim to ask for different terms + - message: spec is immutable; an attachment that moved nodes gets a new + claim rule: self == oldSelf status: - description: status is the binding + description: EgressShardClaimStatus is the shard an attachment was recorded + against. properties: conditions: items: @@ -205,27 +191,17 @@ spec: type: integer shardRef: description: |- - ShardRef is the shard this network egresses through. - - Absent means no shard is bound, which is what a location whose cell holds - no usable shard reads. Nothing publishes an address or a route in that - state: an address a consumer might allow-list is withheld until the - platform can state which one their packets leave on. + ShardRef is the shard on the attachment's node. - Present is permanent for this claim's life. It is written once, and - nothing recomputes it: a rebinding would move a live VPC's egress to a - different source address, which is the value a consumer allow-listed at - their destination. + Absent means the node holds no shard this claim can record, which is + what an attachment on a node an operator has not commissioned reads. properties: name: - description: Name of the shard. + description: Name of the EgressShard. minLength: 1 type: string namespace: - description: |- - Namespace holding the shard. It is stated rather than assumed: the - shards are in the namespace an operator gave the serving class, which is - not the namespace a claim lives in. + description: Namespace of the EgressShard. minLength: 1 type: string required: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 01f8cca..ddfa554 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -14,6 +14,7 @@ rules: - apiGroups: - cloud.datumapis.com resources: + - egressshardclaims - networkfabricidentities - vpcattachments - vpcs @@ -28,6 +29,7 @@ rules: - apiGroups: - cloud.datumapis.com resources: + - egressshardclaims/status - vpcattachments/status - vpcs/status verbs: @@ -71,10 +73,19 @@ rules: resources: - bgpadvertisements - bgprouters + verbs: + - get + - list + - watch +- apiGroups: + - network.datumapis.com + resources: - egressshards verbs: - get - list + - patch + - update - watch - apiGroups: - networking.datumapis.com diff --git a/docs/api/vpc.md b/docs/api/vpc.md index 3481582..5deca7b 100644 --- a/docs/api/vpc.md +++ b/docs/api/vpc.md @@ -9,12 +9,118 @@ Package v1alpha1 contains API Schema definitions for the cloud.datumapis.com/v1alpha1 API group. ### Resource Types +- [EgressShardClaim](#egressshardclaim) - [NetworkFabricIdentity](#networkfabricidentity) - [VPC](#vpc) - [VPCAttachment](#vpcattachment) +#### AttachmentRef + + + +AttachmentRef references a VPCAttachment by name. + + + +_Appears in:_ +- [EgressShardClaimSpec](#egressshardclaimspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | Name of the VPCAttachment. | | MinLength: 1
| + + +#### EgressShardClaim + + + +EgressShardClaim records one attachment's egress shard: the shard on the +node the attachment landed on. + +There is one claim per attachment, owned by it, so an attachment that goes +takes its record with it. The claim names no selector, no address and no +pool: the node is the binding, and the claim writes it down. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `cloud.datumapis.com/v1alpha1` | | | +| `kind` _string_ | `EgressShardClaim` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[EgressShardClaimSpec](#egressshardclaimspec)_ | | | | +| `status` _[EgressShardClaimStatus](#egressshardclaimstatus)_ | | | | + + +#### EgressShardClaimSpec + + + +EgressShardClaimSpec is one attachment being recorded against the egress +shard on its node. + +The claim decides nothing. The node already routes toward its own shard from +the moment the attachment exists; the claim records which shard that is, so +the binding is readable, so a node without a usable shard produces a +condition a consumer can see, and so a later tier that does select among +shards binds through the same object. + +The whole spec is immutable. An attachment that lands on a different node is +a different record, so the claim is replaced rather than edited. + + + +_Appears in:_ +- [EgressShardClaim](#egressshardclaim) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `attachment` _[AttachmentRef](#attachmentref)_ | Attachment is the attachment this claim records egress for. The claim
carries the attachment's name and namespace, so the two are read by one
key. | | | +| `nodeName` _string_ | NodeName is the node the attachment landed on, and therefore the node
whose shard serves it. | | MinLength: 1
| +| `families` _[InternetEgressAddressFamily](#internetegressaddressfamily) array_ | Families are the destination address families the network declared,
so the shard on the node is one that translates them. | | Enum: [IPv6]
MaxItems: 2
MinItems: 1
| + + +#### EgressShardClaimStatus + + + +EgressShardClaimStatus is the shard an attachment was recorded against. + + + +_Appears in:_ +- [EgressShardClaim](#egressshardclaim) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `observedGeneration` _integer_ | | | | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | | | | +| `shardRef` _[EgressShardReference](#egressshardreference)_ | ShardRef is the shard on the attachment's node.
Absent means the node holds no shard this claim can record, which is
what an attachment on a node an operator has not commissioned reads. | | | + + +#### EgressShardReference + + + +EgressShardReference names the shard an attachment egresses through. + + + +_Appears in:_ +- [EgressShardClaimStatus](#egressshardclaimstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `namespace` _string_ | Namespace of the EgressShard. | | MinLength: 1
| +| `name` _string_ | Name of the EgressShard. | | MinLength: 1
| + + #### IPAddress _Underlying type:_ _string_ @@ -45,11 +151,13 @@ _Validation:_ - Enum: [IPv6] _Appears in:_ +- [EgressShardClaimSpec](#egressshardclaimspec) - [InternetEgressSourceAddress](#internetegresssourceaddress) | Field | Description | | --- | --- | | `IPv6` | | +| `IPv4` | | #### InternetEgressAddressStability diff --git a/internal/controller/egressshardbinding_controller_test.go b/internal/controller/egressshardbinding_controller_test.go index 4dbd42b..125acf2 100644 --- a/internal/controller/egressshardbinding_controller_test.go +++ b/internal/controller/egressshardbinding_controller_test.go @@ -71,7 +71,7 @@ func heldOpen(t *testing.T, cl client.Client) bool { // every flow on it, and nothing rebinds a claim. func TestShardIsHeldOpenWhileANetworkIsBound(t *testing.T) { r, cl := newShardBinder(t, newEgressClaim("shard-a"), - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) + newEgressShard("shard-a", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) reconcileShard(t, r) @@ -83,7 +83,7 @@ func TestShardIsHeldOpenWhileANetworkIsBound(t *testing.T) { // A drained shard is released, which is what lets an operator decommission the // node it runs on. func TestShardIsReleasedWhenNoNetworkIsBound(t *testing.T) { - shard := newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels()) + shard := newEgressShard("shard-a", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100") shard.Finalizers = []string{cloudv1alpha1.FinalizerEgressShardBinding} r, cl := newShardBinder(t, shard) @@ -99,7 +99,7 @@ func TestShardIsReleasedWhenNoNetworkIsBound(t *testing.T) { func TestShardIsReleasedWhenAClaimHoldsOnlyTheLabel(t *testing.T) { claim := newEgressClaim("shard-a") claim.Status.ShardRef = nil - shard := newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels()) + shard := newEgressShard("shard-a", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100") shard.Finalizers = []string{cloudv1alpha1.FinalizerEgressShardBinding} r, cl := newShardBinder(t, claim, shard) @@ -115,7 +115,7 @@ func TestShardIsReleasedWhenAClaimHoldsOnlyTheLabel(t *testing.T) { func TestShardIgnoresAClaimBoundElsewhere(t *testing.T) { claim := newEgressClaim("shard-a") claim.Status.ShardRef.Namespace = "some-other-namespace" - shard := newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels()) + shard := newEgressShard("shard-a", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100") shard.Finalizers = []string{cloudv1alpha1.FinalizerEgressShardBinding} r, cl := newShardBinder(t, claim, shard) diff --git a/internal/controller/egressshardclaim_controller.go b/internal/controller/egressshardclaim_controller.go index 131de3e..9acd41e 100644 --- a/internal/controller/egressshardclaim_controller.go +++ b/internal/controller/egressshardclaim_controller.go @@ -18,7 +18,6 @@ along with this program. If not, see . package controller import ( - "cmp" "context" "errors" "fmt" @@ -28,9 +27,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/selection" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -45,17 +42,14 @@ import ( bgpv1alpha1 "go.datum.net/network/api/v1alpha1" ) -// egressUnavailableMessage is what a consumer reads when this location -// provides no egress for their network. It names no node, no shard and no -// other consumer: a consumer cannot act on any of those, and each says where -// the platform runs their workload. The cause is logged and lives on the -// claim, which is an operator's object. -const egressUnavailableMessage = "No component in this location provides internet egress for this network" - -// bindingRefusedError is why one binding may not be made, carrying the named -// reason it is reported under. A binding that cannot be made says which -// incompatibility stopped it; a generic failure would leave an operator to work -// that out from the objects. +// egressUnavailableMessage is what a consumer reads when the node serving their +// instance provides no egress. It names no node and no shard: a consumer cannot +// act on either, and each says where the platform runs their workload. The +// cause lives on the claim, which is an operator's object. +const egressUnavailableMessage = "No component serving this instance provides internet egress" + +// bindingRefusedError is why one shard may not be recorded, carrying the named +// reason it is reported under. type bindingRefusedError struct { reason string message string @@ -63,46 +57,36 @@ type bindingRefusedError struct { func (e *bindingRefusedError) Error() string { return e.message } -// EgressShardClaimReconciler binds a network's presence in this cell to one -// egress shard, once. -// -// It is the cell's single decision point for egress. The intent a location was -// instructed with says what the network needs; this decides which shard answers -// it, records that on the claim, and reports the result on the network context -// a consumer already reads. Everything downstream — the route a node installs, -// the address a consumer allow-lists — reads the binding rather than selecting -// again, because two selections made from the same inputs at different moments -// are two answers, and the datapath can hold one. +// EgressShardClaimReconciler records which shard each attachment egresses +// through: the one on the node the attachment landed on. // -// Every claim binds a shared shard. Dedicated capacity is a hand-commissioned -// shard node and no controller can grow it while nothing allocates the -// identifier a shard is unusable without, so the platform withholds the value -// rather than accepting a request that would wait indefinitely — the same way -// it withholds reaching IPv4 destinations until a resolver and a translator -// share a prefix. The claim records the sharing it was created under and -// nothing here branches on it. +// It decides nothing. The node installs its route from its own configuration +// the moment the attachment exists, and this runs after the attachment has +// reported its node. The record exists so the binding is readable, so a node +// without a usable shard produces a condition a consumer can see on the +// attachment, and so a later tier that does select among shards binds through +// the same object. type EgressShardClaimReconciler struct { client.Client Scheme *runtime.Scheme } +// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=vpcattachments,verbs=get;list;watch +// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=vpcattachments/status,verbs=get;update;patch // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkcontexts,verbs=get;list;watch -// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkcontexts/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=egressshardparameters,verbs=get;list;watch // +kubebuilder:rbac:groups=cloud.datumapis.com,resources=egressshardclaims,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=cloud.datumapis.com,resources=egressshardclaims/status,verbs=get;update;patch // +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshards,verbs=get;list;watch;update;patch func (r *EgressShardClaimReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - // A claim carries its network context's name, so one key reads both: the - // instruction this cell was given, and the binding made from it. - var networkContext networkingv1alpha.NetworkContext - contextFound := true - if err := r.Get(ctx, req.NamespacedName, &networkContext); err != nil { + // A claim carries its attachment's name, so one key reads both. + var attachment cloudv1alpha1.VPCAttachment + attachmentFound := true + if err := r.Get(ctx, req.NamespacedName, &attachment); err != nil { if !apierrors.IsNotFound(err) { - return ctrl.Result{}, fmt.Errorf("get NetworkContext %s: %w", req.NamespacedName, err) + return ctrl.Result{}, fmt.Errorf("get VPCAttachment %s: %w", req.NamespacedName, err) } - contextFound = false + attachmentFound = false } var claim cloudv1alpha1.EgressShardClaim @@ -119,170 +103,99 @@ func (r *EgressShardClaimReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, nil } - if !contextFound || !networkContext.DeletionTimestamp.IsZero() { - return ctrl.Result{}, r.releaseClaim(ctx, &claim, claimFound, - "the network is no longer present in this cell") + if !attachmentFound || !attachment.DeletionTimestamp.IsZero() { + return ctrl.Result{}, r.releaseClaim(ctx, &claim, claimFound, "the attachment is gone") } - intent := internetEgressIntent(&networkContext) - terms, err := r.claimTerms(ctx, &networkContext) + terms, err := r.claimTerms(ctx, &attachment) if err != nil { var refused *bindingRefusedError if errors.As(err, &refused) { - // Nothing can be bound and nothing is: no claim, so no route and no - // address, which is what a consumer reads as no egress here. The - // cause is logged rather than published, for the same reason the - // condition's message never carries one. - logf.FromContext(ctx).Info("nothing can be bound for this location's egress", - "networkContext", networkContext.Name, "reason", refused.reason, + logf.FromContext(ctx).Info("nothing can be recorded for this attachment's egress", + "attachment", client.ObjectKeyFromObject(&attachment), "reason", refused.reason, "cause", refused.message) if err := r.releaseClaim(ctx, &claim, claimFound, refused.message); err != nil { return ctrl.Result{}, err } - return ctrl.Result{}, r.reportContext(ctx, &networkContext, metav1.ConditionFalse, - networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, - egressUnavailableMessage) + return ctrl.Result{}, r.reportAttachment(ctx, &attachment, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, egressUnavailableMessage) } return ctrl.Result{}, err } if terms == nil { - // Either this location was told to reach nothing, or its class is - // served by another implementation. Both are ordinary answers and - // neither is this controller's to report on. + // The network reaches nothing, or the attachment has not landed yet. + // Both are ordinary and neither is this controller's to report on. return ctrl.Result{}, r.releaseClaim(ctx, &claim, claimFound, - "this location provides no internet egress this controller serves") + "this attachment declares no internet egress, or has not reported its node") } if !claimFound { - return ctrl.Result{}, r.createClaim(ctx, &networkContext, terms) + return ctrl.Result{}, r.createClaim(ctx, &attachment, terms) } if !equality.Semantic.DeepEqual(claim.Spec, *terms) { - // The spec is immutable, so the terms cannot be brought into line. An - // unbound claim is discarded and rewritten; a bound one keeps - // delivering what it was bound for and says that it no longer matches. - if claim.Status.ShardRef == nil { - return ctrl.Result{}, r.releaseClaim(ctx, &claim, claimFound, - "the terms this location is instructed with changed before a shard was bound") - } - logf.FromContext(ctx).Info("the egress terms changed after a shard was bound; keeping the binding", - "claim", client.ObjectKeyFromObject(&claim), "shard", claim.Status.ShardRef.Name) - if err := r.publishClaimStatus(ctx, &claim, metav1.ConditionFalse, - cloudv1alpha1.EgressShardClaimReasonTermsChanged, - fmt.Sprintf("Network %q is bound to egress shard %q under terms this location no longer states; delete this claim to bind under the new ones", - claim.Spec.Network.Name, claim.Status.ShardRef.Name)); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, r.reportBoundContext(ctx, &networkContext, &claim) + // The spec is immutable and the attachment moved, most often to another + // node. The old record describes a node this attachment is no longer on, + // so it is replaced on the next pass. + return ctrl.Result{}, r.releaseClaim(ctx, &claim, claimFound, + "the attachment no longer matches the record") } if claim.Status.ShardRef != nil { - // Decided once. Nothing here recomputes a binding: a rebinding moves a - // live VPC's egress to a different source address, which is the value a - // consumer allow-listed at their destination. - return ctrl.Result{}, r.reportExistingBinding(ctx, &networkContext, &claim) + return ctrl.Result{}, r.reportExistingBinding(ctx, &attachment, &claim) } - return ctrl.Result{}, r.bind(ctx, &networkContext, &claim, intent.ParametersRef.Name) + return ctrl.Result{}, r.bind(ctx, &attachment, &claim) } -// claimTerms are the terms a claim would be written with for this location, or -// nil when this controller has nothing to bind. -// -// It reads the instruction and never the class: every field it copies was -// resolved upstream, so nothing here selects a class, picks a default, or -// interprets a parameters reference beyond recognizing whether it is this -// controller's to serve. +// claimTerms are the terms a claim would be written with for this attachment, +// or nil when there is nothing to record yet. func (r *EgressShardClaimReconciler) claimTerms( - ctx context.Context, networkContext *networkingv1alpha.NetworkContext, + ctx context.Context, attachment *cloudv1alpha1.VPCAttachment, ) (*cloudv1alpha1.EgressShardClaimSpec, error) { - intent := internetEgressIntent(networkContext) - if intent == nil || intent.Mode != networkingv1alpha.NetworkInternetEgressEnabled { + if attachment.Status.Node == "" { return nil, nil } - ref := intent.ParametersRef - if ref == nil { - return nil, &bindingRefusedError{ - reason: networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, - message: fmt.Sprintf("Internet egress class %q names no parameters, so which shards serve it is unstated", - intent.ClassName), + + var networkContext networkingv1alpha.NetworkContext + key := client.ObjectKey{Namespace: attachment.Namespace, Name: attachment.Spec.VPC.Name} + if err := r.Get(ctx, key, &networkContext); err != nil { + if apierrors.IsNotFound(err) { + return nil, nil } + return nil, fmt.Errorf("get NetworkContext %s: %w", key, err) } - if ref.Group != cloudv1alpha1.GroupVersion.Group || ref.Kind != cloudv1alpha1.KindEgressShardParameters { + intent := internetEgressIntent(&networkContext) + if intent == nil || intent.Mode != networkingv1alpha.NetworkInternetEgressEnabled { return nil, nil } - sharing, err := claimSharing(intent.Sharing) - if err != nil { - return nil, err - } families, err := claimFamilies(intent.Reach) if err != nil { return nil, err } - if networkContext.Spec.Network.Name == "" { - return nil, &bindingRefusedError{ - reason: networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, - message: "This location names no network, so there is nothing to bind to a shard", - } - } - // Read only to establish that the class this cell was pointed at exists - // here. Which shards it selects is read at the moment of binding. - if err := r.Get(ctx, client.ObjectKey{Name: ref.Name}, - &cloudv1alpha1.EgressShardParameters{}); err != nil { - if apierrors.IsNotFound(err) { - return nil, &bindingRefusedError{ - reason: cloudv1alpha1.EgressShardClaimReasonParametersUnavailable, - message: fmt.Sprintf("Internet egress class %q is served by parameters %q, which do not exist in this location", - intent.ClassName, ref.Name), - } - } - return nil, fmt.Errorf("get EgressShardParameters %s: %w", ref.Name, err) - } - return &cloudv1alpha1.EgressShardClaimSpec{ - Network: cloudv1alpha1.NetworkRef{Name: networkContext.Spec.Network.Name}, - NetworkContext: cloudv1alpha1.NetworkContextRef{Name: networkContext.Name}, - ClassName: intent.ClassName, - Sharing: sharing, - Families: families, + Attachment: cloudv1alpha1.AttachmentRef{Name: attachment.Name}, + NodeName: attachment.Status.Node, + Families: families, }, nil } -// claimSharing carries the class's sharing onto the claim. +// claimFamilies carries the families the network declared, copied from the +// network rather than stated per attachment, so two attachments of one network +// on one node can never ask for different shards. // -// An unprojected value is refused rather than defaulted. Sharing decides how -// many networks a shard may take, and there is no safe guess: reading it as -// Shared would place a network promised its own address alongside others, and -// reading it as Dedicated would hold a shard open for a network that asked for -// no such thing. -func claimSharing(sharing networkingv1alpha.InternetEgressSharing) (cloudv1alpha1.EgressSharing, error) { - switch sharing { - case networkingv1alpha.InternetEgressSharingShared: - return cloudv1alpha1.EgressSharingShared, nil - case networkingv1alpha.InternetEgressSharingDedicated: - return cloudv1alpha1.EgressSharingDedicated, nil - default: - return "", &bindingRefusedError{ - reason: networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, - message: "The serving class's sharing did not reach this location, so how many networks may share a shard is unknown here", - } - } -} - -// claimFamilies carries the families this location was told to reach. -// -// Only IPv6 is accepted, because it is the only family the instruction can -// carry and the only one a shard is selected for. A family that arrives anyway -// is refused rather than dropped: silently binding a shard that translates -// nothing for it would report egress a consumer does not have. +// Only IPv6 is accepted, because it is the only family the declaration can +// carry. A family that arrives anyway is refused rather than dropped: silently +// recording a shard that translates nothing for it would report egress a +// consumer does not have. func claimFamilies( reach []networkingv1alpha.IPFamily, ) ([]cloudv1alpha1.InternetEgressAddressFamily, error) { if len(reach) == 0 { return nil, &bindingRefusedError{ reason: networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, - message: "This location was told to reach no address family, so no shard can be selected for it", + message: "The network declares no address family to reach, so no shard can serve it", } } families := make([]cloudv1alpha1.InternetEgressAddressFamily, 0, len(reach)) @@ -290,7 +203,7 @@ func claimFamilies( if family != networkingv1alpha.IPv6Protocol { return nil, &bindingRefusedError{ reason: networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, - message: fmt.Sprintf("This location was told to reach %s destinations, which no shard in it translates", + message: fmt.Sprintf("The network declares %s destinations, which no shard translates", family), } } @@ -299,22 +212,22 @@ func claimFamilies( return families, nil } -// createClaim writes the one claim this location's egress is bound through. It -// is owned by the network context, so a network withdrawn from the cell takes -// its claim with it and the shard it held is released. +// createClaim writes the one claim recording this attachment's egress. It is +// owned by the attachment, so an attachment that goes takes its record with it +// and the shard it held is released. func (r *EgressShardClaimReconciler) createClaim( ctx context.Context, - networkContext *networkingv1alpha.NetworkContext, + attachment *cloudv1alpha1.VPCAttachment, terms *cloudv1alpha1.EgressShardClaimSpec, ) error { claim := &cloudv1alpha1.EgressShardClaim{ ObjectMeta: metav1.ObjectMeta{ - Namespace: networkContext.Namespace, - Name: networkContext.Name, + Namespace: attachment.Namespace, + Name: attachment.Name, }, Spec: *terms, } - if err := controllerutil.SetControllerReference(networkContext, claim, r.Scheme); err != nil { + if err := controllerutil.SetControllerReference(attachment, claim, r.Scheme); err != nil { return fmt.Errorf("set the owner on EgressShardClaim %s: %w", claim.Name, err) } if err := r.Create(ctx, claim); err != nil { @@ -323,16 +236,14 @@ func (r *EgressShardClaimReconciler) createClaim( } return fmt.Errorf("create EgressShardClaim %s: %w", client.ObjectKeyFromObject(claim), err) } - logf.FromContext(ctx).Info("claimed an egress shard for a location", - "claim", client.ObjectKeyFromObject(claim), "network", terms.Network.Name, - "class", terms.ClassName, "sharing", terms.Sharing) + logf.FromContext(ctx).Info("recorded an attachment's egress", + "claim", client.ObjectKeyFromObject(claim), "node", terms.NodeName) return nil } -// releaseClaim deletes the claim for a location that no longer has egress this -// controller provides. Deleting it is what releases the shard: the consumer set -// is a list of claims, so leaving the shard, and takes the route and the -// address with it. +// releaseClaim deletes the claim for an attachment that no longer has egress +// to record. Deleting it is what releases the shard: the consumer set is a +// list of claims. func (r *EgressShardClaimReconciler) releaseClaim( ctx context.Context, claim *cloudv1alpha1.EgressShardClaim, claimFound bool, why string, ) error { @@ -348,80 +259,74 @@ func (r *EgressShardClaimReconciler) releaseClaim( return nil } -// bind chooses the shard this network egresses through and records it. -// -// The candidates are the shards the class selects, in name order, and the first -// usable one wins. A refused candidate carries the reason it was refused, so a -// claim that binds nothing says which incompatibility stopped it rather than -// reporting a generic failure. A claim that found no usable shard waits: it is -// not refused, and nothing provisions a shard for it, because a shard is -// unusable until an operator commissions its identifier. +// bind records the shard on the attachment's node, or why none can be. func (r *EgressShardClaimReconciler) bind( ctx context.Context, - networkContext *networkingv1alpha.NetworkContext, + attachment *cloudv1alpha1.VPCAttachment, claim *cloudv1alpha1.EgressShardClaim, - parametersName string, ) error { - shards, err := egressShards(ctx, r.Client, parametersName) + shard, err := egressShardOnNode(ctx, r.Client, claim.Spec.NodeName) if err != nil { return err } - if len(shards) == 0 { - return r.refuse(ctx, networkContext, claim, &bindingRefusedError{ - reason: cloudv1alpha1.EgressShardClaimReasonNoShardMatchesTheClass, - message: fmt.Sprintf("No egress shard carries the labels internet egress class %q selects", - claim.Spec.ClassName), + if shard == nil { + return r.refuse(ctx, attachment, claim, &bindingRefusedError{ + reason: cloudv1alpha1.EgressShardClaimReasonNoShardOnNode, + message: fmt.Sprintf("No egress shard names node %q, so nothing on it translates this attachment's traffic", + claim.Spec.NodeName), }) } - - var firstRefusal *bindingRefusedError - for i := range shards { - refusal := shardRefusal(&shards[i]) - if refusal == nil { - return r.recordBinding(ctx, networkContext, claim, &shards[i]) - } - if firstRefusal == nil { - firstRefusal = refusal - } + if refusal := shardRefusal(shard, claim.Spec.Families); refusal != nil { + return r.refuse(ctx, attachment, claim, refusal) } - return r.refuse(ctx, networkContext, claim, firstRefusal) + return r.recordBinding(ctx, attachment, claim, shard) } -// shardRefusal is why no claim may bind one shard, or nil if it may be bound. -// -// It reads the shard alone. Nothing about the claim asking narrows the set, -// because every claim takes a shared shard; when dedicated capacity is offered, -// this is where a claim's own terms would start to matter. -func shardRefusal(shard *bgpv1alpha1.EgressShard) *bindingRefusedError { +// shardRefusal is why a shard may not be recorded, or nil if it may be. +func shardRefusal( + shard *bgpv1alpha1.EgressShard, families []cloudv1alpha1.InternetEgressAddressFamily, +) *bindingRefusedError { + if !shard.DeletionTimestamp.IsZero() { + return &bindingRefusedError{ + reason: cloudv1alpha1.EgressShardClaimReasonShardTerminating, + message: fmt.Sprintf("Egress shard %q is being deleted, so it takes no further attachment", + shard.Name), + } + } if shard.Status.ShardSID == "" { return &bindingRefusedError{ - reason: cloudv1alpha1.EgressShardClaimReasonNoShardIdentifier, - message: fmt.Sprintf("Egress shard %q has no identifier a node can route toward, so binding it would carry no packet", + reason: cloudv1alpha1.EgressShardClaimReasonShardNotReady, + message: fmt.Sprintf("Egress shard %q has reported no identifier a node can route toward", shard.Name), } } - if !shard.DeletionTimestamp.IsZero() { + if (shard.Spec.ShardSID != "" && shard.Spec.ShardSID != shard.Status.ShardSID) || + (shard.Spec.ShardAddressIPv6 != "" && shard.Status.ShardAddressIPv6 != "" && + shard.Spec.ShardAddressIPv6 != shard.Status.ShardAddressIPv6) { return &bindingRefusedError{ - reason: cloudv1alpha1.EgressShardClaimReasonShardTerminating, - message: fmt.Sprintf("Egress shard %q is being deleted, so it takes no further network", + reason: cloudv1alpha1.EgressShardClaimReasonShardMismatch, + message: fmt.Sprintf("Egress shard %q runs an identity other than the one its spec states, so which one serves this node is unknown", + shard.Name), + } + } + if slices.Contains(families, cloudv1alpha1.InternetEgressAddressFamilyIPv4) && + shard.Status.ShardAddressIPv4 == "" { + return &bindingRefusedError{ + reason: cloudv1alpha1.EgressShardClaimReasonFamilyUnsupported, + message: fmt.Sprintf("Egress shard %q translates no IPv4 flow, which the network declares it reaches", shard.Name), } } return nil } -// recordBinding writes the binding: the shard's finalizer first, then the label -// that makes this claim part of the shard's consumer set, then the binding -// itself. -// -// The order is what keeps a crash between the writes harmless. A finalizer with -// no binding behind it is removed by the reconciler that watches shards, and a -// label with no binding behind it counts as no consumer and is overwritten by -// the next pass. A binding recorded before either would be a network egressing -// through a shard nothing holds open and nothing counts. +// recordBinding writes the record: the shard's finalizer first, then the label +// that makes this claim part of the shard's consumer set, then the record +// itself, so a crash between the writes leaves nothing that reads as bound +// without being held. func (r *EgressShardClaimReconciler) recordBinding( ctx context.Context, - networkContext *networkingv1alpha.NetworkContext, + attachment *cloudv1alpha1.VPCAttachment, claim *cloudv1alpha1.EgressShardClaim, shard *bgpv1alpha1.EgressShard, ) error { @@ -436,22 +341,22 @@ func (r *EgressShardClaimReconciler) recordBinding( Namespace: shard.Namespace, Name: shard.Name, } - logf.FromContext(ctx).Info("bound a network's egress to a shard", - "claim", client.ObjectKeyFromObject(claim), "network", claim.Spec.Network.Name, - "shard", client.ObjectKeyFromObject(shard), "sharing", claim.Spec.Sharing) + logf.FromContext(ctx).Info("recorded an attachment's egress shard", + "claim", client.ObjectKeyFromObject(claim), "node", claim.Spec.NodeName, + "shard", client.ObjectKeyFromObject(shard)) if err := r.publishClaimStatus(ctx, claim, metav1.ConditionTrue, cloudv1alpha1.EgressShardClaimReasonBound, - fmt.Sprintf("Network %q egresses through egress shard %q", claim.Spec.Network.Name, shard.Name)); err != nil { + fmt.Sprintf("Attachment %q egresses through egress shard %q", claim.Spec.Attachment.Name, shard.Name)); err != nil { return err } - return r.reportBinding(ctx, networkContext, shard) + return r.reportBinding(ctx, attachment, shard) } -// labelClaim stamps the shard a claim is bound to, so the shard's consumer set -// is a list query. It is re-asserted on every pass over a bound claim, because -// a label lost to an edit would hide a network from the count that keeps a -// dedicated shard exclusive. +// labelClaim stamps the shard a claim records, so the shard's consumer set is +// a list query. It is re-asserted on every pass over a bound claim, because a +// label lost to an edit would hide an attachment from the query that holds a +// shard open. func (r *EgressShardClaimReconciler) labelClaim( ctx context.Context, claim *cloudv1alpha1.EgressShardClaim, shardName string, ) error { @@ -470,26 +375,18 @@ func (r *EgressShardClaimReconciler) labelClaim( return nil } -// reportExistingBinding says what a binding already made is delivering, on the -// claim and on the network context, and repairs the label the consumer set is +// reportExistingBinding says what a record already made is delivering, on the +// claim and on the attachment, and repairs the label the consumer set is // counted by. func (r *EgressShardClaimReconciler) reportExistingBinding( ctx context.Context, - networkContext *networkingv1alpha.NetworkContext, + attachment *cloudv1alpha1.VPCAttachment, claim *cloudv1alpha1.EgressShardClaim, ) error { if err := r.labelClaim(ctx, claim, claim.Status.ShardRef.Name); err != nil { return err } - return r.reportBoundContext(ctx, networkContext, claim) -} -// reportBoundContext reports a binding whose shard has to be read back for it. -func (r *EgressShardClaimReconciler) reportBoundContext( - ctx context.Context, - networkContext *networkingv1alpha.NetworkContext, - claim *cloudv1alpha1.EgressShardClaim, -) error { var shard bgpv1alpha1.EgressShard key := client.ObjectKey{ Namespace: claim.Status.ShardRef.Namespace, @@ -497,50 +394,46 @@ func (r *EgressShardClaimReconciler) reportBoundContext( } if err := r.Get(ctx, key, &shard); err != nil { if !apierrors.IsNotFound(err) { - return fmt.Errorf("get the bound EgressShard %s: %w", key, err) + return fmt.Errorf("get the recorded EgressShard %s: %w", key, err) } - // Nothing rebinds, so this network has no egress and no second answer - // coming. The finalizer exists to make this reachable only by someone - // removing it. - message := fmt.Sprintf("Egress shard %q no longer exists, and a binding is not remade", key.Name) + message := fmt.Sprintf("Egress shard %q no longer exists", key.Name) if err := r.publishClaimStatus(ctx, claim, metav1.ConditionFalse, cloudv1alpha1.EgressShardClaimReasonShardMissing, message); err != nil { return err } - return r.reportContext(ctx, networkContext, metav1.ConditionFalse, - networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, - egressUnavailableMessage) + return r.reportAttachment(ctx, attachment, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, egressUnavailableMessage) } - return r.reportBinding(ctx, networkContext, &shard) + return r.reportBinding(ctx, attachment, &shard) } -// reportBinding projects a binding onto the network context condition a +// reportBinding projects a recorded shard onto the attachment condition a // consumer reads. // // Degraded is deliberately never written. It means egress works for some -// declared families and not others, and only one family is accepted anywhere on -// this path, so no state can reach it. +// declared families and not others, and only one family is accepted anywhere +// on this path, so no state can reach it. func (r *EgressShardClaimReconciler) reportBinding( ctx context.Context, - networkContext *networkingv1alpha.NetworkContext, + attachment *cloudv1alpha1.VPCAttachment, shard *bgpv1alpha1.EgressShard, ) error { if shard.Status.ShardAddressIPv6 == "" { - return r.reportContext(ctx, networkContext, metav1.ConditionFalse, + return r.reportAttachment(ctx, attachment, metav1.ConditionFalse, networkingv1alpha.NetworkContextInternetEgressReasonAddressUnavailable, - "No egress address has been allocated for this location yet") + "No egress address has been allocated for the node serving this instance yet") } - return r.reportContext(ctx, networkContext, metav1.ConditionTrue, + return r.reportAttachment(ctx, attachment, metav1.ConditionTrue, networkingv1alpha.NetworkContextInternetEgressReasonReady, - fmt.Sprintf("Instances in this location reach the internet, and %s is the address they reach it from", + fmt.Sprintf("This instance reaches the internet, and %s is the address it reaches it from", shard.Status.ShardAddressIPv6)) } -// refuse records that nothing was bound, and why. The claim stays, unbound, -// and takes the first usable shard that appears. +// refuse records that nothing was recorded, and why. The claim stays, unbound, +// and takes the shard the moment one names its node and is usable. func (r *EgressShardClaimReconciler) refuse( ctx context.Context, - networkContext *networkingv1alpha.NetworkContext, + attachment *cloudv1alpha1.VPCAttachment, claim *cloudv1alpha1.EgressShardClaim, refusal *bindingRefusedError, ) error { @@ -548,11 +441,10 @@ func (r *EgressShardClaimReconciler) refuse( refusal.reason, refusal.message); err != nil { return err } - // The context carries the fact about the consumer's network. Which shard - // refused it, and why, is on the claim, which is an operator's object. - return r.reportContext(ctx, networkContext, metav1.ConditionFalse, - networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, - egressUnavailableMessage) + // The attachment carries the fact about the consumer's instance. Which + // shard refused it, and why, is on the claim, which is an operator's object. + return r.reportAttachment(ctx, attachment, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, egressUnavailableMessage) } func (r *EgressShardClaimReconciler) publishClaimStatus( @@ -576,125 +468,106 @@ func (r *EgressShardClaimReconciler) publishClaimStatus( return nil } -// reportContext writes the readiness a consumer reads for this location. +// reportAttachment writes the egress readiness a consumer reads for this +// instance, on the attachment the interface's status is read from. // -// The reasons were defined with no writer; this is the writer of all of them -// but the class-resolution refusal, which is written upstream where the class -// is read. The message states a fact about the consumer's network and names no -// node, no shard and no other consumer: a consumer cannot act on those, and -// they describe where the platform runs their workload. -func (r *EgressShardClaimReconciler) reportContext( +// It is patched rather than updated. The attachment's status has another +// writer, the controller that renders it, and a whole-object update from a +// copy read before that write would put stale values back over its fields. +func (r *EgressShardClaimReconciler) reportAttachment( ctx context.Context, - networkContext *networkingv1alpha.NetworkContext, + attachment *cloudv1alpha1.VPCAttachment, status metav1.ConditionStatus, reason, message string, ) error { condition := metav1.Condition{ - Type: networkingv1alpha.NetworkContextInternetEgressReady, + Type: cloudv1alpha1.ConditionTypeInternetEgressReady, Status: status, Reason: reason, Message: message, - ObservedGeneration: networkContext.Generation, + ObservedGeneration: attachment.Generation, } - if !meta.SetStatusCondition(&networkContext.Status.Conditions, condition) { + patch := client.MergeFrom(attachment.DeepCopy()) + if !meta.SetStatusCondition(&attachment.Status.Conditions, condition) { return nil } - if err := r.Status().Update(ctx, networkContext); err != nil { - return fmt.Errorf("update NetworkContext %s status: %w", - client.ObjectKeyFromObject(networkContext), err) + if err := r.Status().Patch(ctx, attachment, patch); err != nil { + return fmt.Errorf("report internet egress on VPCAttachment %s: %w", + client.ObjectKeyFromObject(attachment), err) } return nil } -// egressShards lists the shards a class's parameters select, in name order. -// -// It moved here from the interface controller with binding. The order used to -// be what made a candidate list a function of the matched set alone, so two -// attachments of one VPC could not compute different lists; now it is what -// makes the binding itself deterministic over the set of shards it saw. -func egressShards( - ctx context.Context, reader client.Reader, parametersName string, -) ([]bgpv1alpha1.EgressShard, error) { - var parameters cloudv1alpha1.EgressShardParameters - if err := reader.Get(ctx, client.ObjectKey{Name: parametersName}, ¶meters); err != nil { - return nil, fmt.Errorf("get EgressShardParameters %s: %w", parametersName, err) - } - - selector, err := metav1.LabelSelectorAsSelector(¶meters.Spec.ShardSelector) - if err != nil { - return nil, fmt.Errorf("parse the shard selector on EgressShardParameters %s: %w", - parameters.Name, err) - } - // Only IPv6 is reached, so a shard that translates no IPv6 flow is no - // candidate however an operator wrote the selector. The family label is - // matched on presence: absence, not a false value, means the family is - // unserved, so a shard predating the label never reads as serving one. - servesIPv6, err := labels.NewRequirement(bgpv1alpha1.LabelEgressShardIPv6, selection.Exists, nil) - if err != nil { - return nil, fmt.Errorf("build the IPv6 shard requirement: %w", err) - } - - var shards bgpv1alpha1.EgressShardList - if err := reader.List(ctx, &shards, - client.InNamespace(parameters.Spec.ShardNamespace), - client.MatchingLabelsSelector{Selector: selector.Add(*servesIPv6)}, - ); err != nil { - return nil, fmt.Errorf("list egress shards for EgressShardParameters %s: %w", - parameters.Name, err) - } - slices.SortFunc(shards.Items, func(a, b bgpv1alpha1.EgressShard) int { - return cmp.Compare(a.Name, b.Name) - }) - return shards.Items, nil -} - // SetupWithManager registers the reconciler with the manager. func (r *EgressShardClaimReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&cloudv1alpha1.EgressShardClaim{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { - // A bound claim is never reconsidered on its own events, which - // is what makes "decided once" a property of the controller - // rather than a check inside it. + // A bound claim is never reconsidered on its own events. It is + // re-read when its attachment or its shard changes. claim, ok := object.(*cloudv1alpha1.EgressShardClaim) return ok && claim.Status.ShardRef == nil }))). + Watches(&cloudv1alpha1.VPCAttachment{}, + handler.EnqueueRequestsFromMapFunc(claimForAttachment)). Watches(&networkingv1alpha.NetworkContext{}, - handler.EnqueueRequestsFromMapFunc(claimForNetworkContext)). + handler.EnqueueRequestsFromMapFunc(r.claimsForNetworkContext)). Watches(&bgpv1alpha1.EgressShard{}, - handler.EnqueueRequestsFromMapFunc(r.unboundClaimsForEgressShard)). + handler.EnqueueRequestsFromMapFunc(r.claimsForEgressShard)). Named("egressshardclaim"). Complete(r) } -// claimForNetworkContext maps a location to its one claim, which carries the +// claimForAttachment maps an attachment to its one claim, which carries the // same name. -func claimForNetworkContext(_ context.Context, object client.Object) []reconcile.Request { +func claimForAttachment(_ context.Context, object client.Object) []reconcile.Request { return []reconcile.Request{{NamespacedName: client.ObjectKeyFromObject(object)}} } -// unboundClaimsForEgressShard wakes the claims still waiting for a shard when -// one arrives, reports its identifier, or leaves. -// -// Only unbound claims are enqueued. A bound one has nothing to recompute, and a -// shard arriving is exactly the moment a claim that was told there was no free -// dedicated shard can stop waiting. -func (r *EgressShardClaimReconciler) unboundClaimsForEgressShard( - ctx context.Context, _ client.Object, +// claimsForNetworkContext wakes every attachment of a network when its +// declaration changes, so a network disabled releases its records. +func (r *EgressShardClaimReconciler) claimsForNetworkContext( + ctx context.Context, object client.Object, +) []reconcile.Request { + var attachments cloudv1alpha1.VPCAttachmentList + if err := r.List(ctx, &attachments, client.InNamespace(object.GetNamespace())); err != nil { + return nil + } + requests := make([]reconcile.Request, 0, len(attachments.Items)) + for i := range attachments.Items { + if attachments.Items[i].Spec.VPC.Name != object.GetName() { + continue + } + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&attachments.Items[i]), + }) + } + return requests +} + +// claimsForEgressShard wakes the claims on a shard's node when it arrives, +// reports its identity, or leaves, and the claims recorded against it so a +// missing or draining shard is reported. +func (r *EgressShardClaimReconciler) claimsForEgressShard( + ctx context.Context, object client.Object, ) []reconcile.Request { + shard, ok := object.(*bgpv1alpha1.EgressShard) + if !ok { + return nil + } var claims cloudv1alpha1.EgressShardClaimList if err := r.List(ctx, &claims); err != nil { return nil } - requests := make([]reconcile.Request, 0, len(claims.Items)) for i := range claims.Items { - if claims.Items[i].Status.ShardRef != nil { + claim := &claims.Items[i] + onNode := claim.Spec.NodeName == shard.Spec.TargetRef.Name + recorded := claim.Status.ShardRef != nil && claim.Status.ShardRef.Name == shard.Name + if !onNode && !recorded { continue } - requests = append(requests, reconcile.Request{ - NamespacedName: client.ObjectKeyFromObject(&claims.Items[i]), - }) + requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(claim)}) } return requests } diff --git a/internal/controller/egressshardclaim_controller_test.go b/internal/controller/egressshardclaim_controller_test.go index 94cae04..b10b35f 100644 --- a/internal/controller/egressshardclaim_controller_test.go +++ b/internal/controller/egressshardclaim_controller_test.go @@ -33,9 +33,9 @@ import ( bgpv1alpha1 "go.datum.net/network/api/v1alpha1" ) -// egressContextName is the network context every egress test binds, and -// therefore the name of the one claim that binds it. -const egressContextName = "default-us-central-1" +// egressAttachmentName is the attachment every egress test records, and +// therefore the name of the one claim recording it. +const egressAttachmentName = "web-eth0" func newBinder(t *testing.T, objects ...client.Object) (*EgressShardClaimReconciler, client.Client) { t.Helper() @@ -52,22 +52,47 @@ func newBinder(t *testing.T, objects ...client.Object) (*EgressShardClaimReconci } fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...). - WithStatusSubresource(&cloudv1alpha1.EgressShardClaim{}, &networkingv1alpha.NetworkContext{}). + WithStatusSubresource(&cloudv1alpha1.EgressShardClaim{}, &cloudv1alpha1.VPCAttachment{}). Build() return &EgressShardClaimReconciler{Client: fakeClient, Scheme: scheme}, fakeClient } -// newBoundContext is the location the binder works from: the projected intent, -// with the network it belongs to named. -func newBoundContext(mode networkingv1alpha.NetworkInternetEgressMode) *networkingv1alpha.NetworkContext { - networkContext := newEgressContext(mode) - networkContext.Spec.Network = networkingv1alpha.LocalNetworkRef{Name: "default"} - return networkContext +// newLandedAttachment is an attachment of the egress test network that has +// reported the node it landed on. +func newLandedAttachment(node string) *cloudv1alpha1.VPCAttachment { + attachment := &cloudv1alpha1.VPCAttachment{} + attachment.Namespace = egressTestNamespace + attachment.Name = egressAttachmentName + attachment.Spec.VPC = cloudv1alpha1.VPCRef{Name: "default-us-central-1"} + attachment.Spec.Interface.Name = "eth0" + attachment.Status.Node = node + return attachment +} + +// newEgressClaim is a record already written for the test attachment, bound +// to shardName or, with an empty name, still unbound. +func newEgressClaim(shardName string) *cloudv1alpha1.EgressShardClaim { + claim := &cloudv1alpha1.EgressShardClaim{} + claim.Namespace = egressTestNamespace + claim.Name = egressAttachmentName + claim.Spec = cloudv1alpha1.EgressShardClaimSpec{ + Attachment: cloudv1alpha1.AttachmentRef{Name: egressAttachmentName}, + NodeName: egressTestNode, + Families: []cloudv1alpha1.InternetEgressAddressFamily{cloudv1alpha1.InternetEgressAddressFamilyIPv6}, + } + if shardName != "" { + claim.Labels = map[string]string{cloudv1alpha1.LabelEgressShardClaimShard: shardName} + claim.Status.ShardRef = &cloudv1alpha1.EgressShardReference{ + Namespace: egressShardNamespace, + Name: shardName, + } + } + return claim } func reconcileBinding(t *testing.T, r *EgressShardClaimReconciler) { t.Helper() - key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressContextName} + key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressAttachmentName} if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: key}); err != nil { t.Fatalf("reconcile the claim: %v", err) } @@ -76,7 +101,7 @@ func reconcileBinding(t *testing.T, r *EgressShardClaimReconciler) { func readClaim(t *testing.T, cl client.Client) *cloudv1alpha1.EgressShardClaim { t.Helper() var claim cloudv1alpha1.EgressShardClaim - key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressContextName} + key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressAttachmentName} if err := cl.Get(t.Context(), key, &claim); err != nil { t.Fatalf("get the claim: %v", err) } @@ -86,157 +111,133 @@ func readClaim(t *testing.T, cl client.Client) *cloudv1alpha1.EgressShardClaim { func claimExists(t *testing.T, cl client.Client) bool { t.Helper() var claim cloudv1alpha1.EgressShardClaim - key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressContextName} - err := cl.Get(t.Context(), key, &claim) - return err == nil + key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressAttachmentName} + return cl.Get(t.Context(), key, &claim) == nil } -// One claim per network context that declares egress, carrying the terms the -// projection resolved and nothing this controller invented. -func TestBinderClaimsOncePerNetworkContext(t *testing.T) { - r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled), - newEgressParameters(), - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) +// One claim per attachment that has landed on a node, carrying the node and +// the families the network declared and nothing this controller invented. +func TestBinderRecordsOncePerAttachment(t *testing.T) { + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) reconcileBinding(t, r) claim := readClaim(t, cl) - if claim.Spec.Network.Name != "default" { - t.Errorf("network: got %q, want default", claim.Spec.Network.Name) - } - if claim.Spec.NetworkContext.Name != egressContextName { - t.Errorf("network context: got %q, want %q", claim.Spec.NetworkContext.Name, egressContextName) + if claim.Spec.Attachment.Name != egressAttachmentName { + t.Errorf("attachment: got %q, want %q", claim.Spec.Attachment.Name, egressAttachmentName) } - if claim.Spec.ClassName != "shared" { - t.Errorf("class: got %q, want shared", claim.Spec.ClassName) - } - if claim.Spec.Sharing != cloudv1alpha1.EgressSharingShared { - t.Errorf("sharing: got %q, want Shared", claim.Spec.Sharing) + if claim.Spec.NodeName != egressTestNode { + t.Errorf("node: got %q, want %q", claim.Spec.NodeName, egressTestNode) } if len(claim.Spec.Families) != 1 || claim.Spec.Families[0] != cloudv1alpha1.InternetEgressAddressFamilyIPv6 { t.Errorf("families: got %v, want [IPv6]", claim.Spec.Families) } - // The claim names no shard, no selector, no address and no pool: the cell - // answers with the shard, and it answers on status. if claim.Status.ShardRef != nil { - t.Errorf("the claim bound %v in the pass that wrote it", claim.Status.ShardRef) + t.Errorf("the claim recorded %v in the pass that wrote it", claim.Status.ShardRef) + } + if !metav1.IsControlledBy(claim, newLandedAttachment(egressTestNode)) && + len(claim.OwnerReferences) == 0 { + t.Error("the claim is not owned by its attachment") + } +} + +// Nothing is recorded before the attachment reports where it landed. The node +// is the whole content of the record. +func TestBinderWaitsForTheAttachmentsNode(t *testing.T) { + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(""), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) + + reconcileBinding(t, r) + + if claimExists(t, cl) { + t.Error("a claim was written for an attachment on no known node") } } -func TestBinderBindsTheClaimToAShard(t *testing.T) { - r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled), - newEgressParameters(), - newEgressShard("shard-b", "2001:db8:ff02::", "2001:db8:f00d::200", poolLabels()), - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) +// The shard recorded is the one on the attachment's node and no other. +func TestBinderRecordsTheShardOnTheNode(t *testing.T) { + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode), + newEgressShard("worker-2-egress", "worker-2", "2001:db8:ff02::", "2001:db8:f00d::200"), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) reconcileBinding(t, r) reconcileBinding(t, r) claim := readClaim(t, cl) if claim.Status.ShardRef == nil { - t.Fatal("two usable shards bound nothing") + t.Fatal("a shard on the node recorded nothing") } - // Name order, moved here with the selection it belongs to: the binding has - // to be deterministic over the set of shards it saw. - if claim.Status.ShardRef.Name != "shard-a" { - t.Errorf("shard: got %q, want shard-a", claim.Status.ShardRef.Name) + if claim.Status.ShardRef.Name != "worker-3-egress" { + t.Errorf("shard: got %q, want worker-3-egress", claim.Status.ShardRef.Name) } if claim.Status.ShardRef.Namespace != egressShardNamespace { t.Errorf("shard namespace: got %q, want %q", claim.Status.ShardRef.Namespace, egressShardNamespace) } - // The label is what makes the shard's consumer set a list query, which is - // what stands in for the list of networks a shard does not hold. - if got := claim.Labels[cloudv1alpha1.LabelEgressShardClaimShard]; got != "shard-a" { - t.Errorf("shard label: got %q, want shard-a", got) + if got := claim.Labels[cloudv1alpha1.LabelEgressShardClaimShard]; got != "worker-3-egress" { + t.Errorf("shard label: got %q, want worker-3-egress", got) } assertClaimCondition(t, cl, metav1.ConditionTrue, cloudv1alpha1.EgressShardClaimReasonBound) - assertContextCondition(t, cl, metav1.ConditionTrue, + assertAttachmentCondition(t, cl, metav1.ConditionTrue, networkingv1alpha.NetworkContextInternetEgressReasonReady) // The finalizer is the only state a binder puts on a shard, written before - // the binding so a recorded binding is never held by nothing. + // the record so a recorded shard is never held by nothing. var shard bgpv1alpha1.EgressShard - key := client.ObjectKey{Namespace: egressShardNamespace, Name: "shard-a"} + key := client.ObjectKey{Namespace: egressShardNamespace, Name: "worker-3-egress"} if err := cl.Get(t.Context(), key, &shard); err != nil { - t.Fatalf("get the bound shard: %v", err) + t.Fatalf("get the recorded shard: %v", err) } if !controllerutil.ContainsFinalizer(&shard, cloudv1alpha1.FinalizerEgressShardBinding) { - t.Error("the bound shard is not held open") - } - // Nothing else is written to it. A shard holds no list of the networks it - // serves and no count of them. - if len(shard.Labels) != len(poolLabels()) { - t.Errorf("the binder wrote labels onto the shard: %v", shard.Labels) + t.Error("the recorded shard is not held open") } if shard.Spec.ShardAddressIPv6 != "2001:db8:f00d::100" { t.Errorf("the binder rewrote the shard's address: %q", shard.Spec.ShardAddressIPv6) } } -// Many networks bind one shard. Nothing branches on the sharing a claim -// records, because dedicated capacity is not offered. -func TestBinderBindsManyNetworksToOneShard(t *testing.T) { - first := newEgressClaim("shard-a") - first.Name = "other-us-central-1" - first.Spec.Network.Name = "other" - first.Spec.NetworkContext.Name = "other-us-central-1" - - r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled), - newEgressParameters(), first, - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) - - reconcileBinding(t, r) - reconcileBinding(t, r) - - claim := readClaim(t, cl) - if claim.Status.ShardRef == nil || claim.Status.ShardRef.Name != "shard-a" { - t.Fatalf("got %v, want the shard another network already holds", claim.Status.ShardRef) - } -} - -// A shard with no identifier has nothing a node can route toward, so binding it -// would report egress that carries no packet. The claim waits, and the address -// stays unpublished. +// A node whose shard cannot be recorded leaves the claim unbound with the +// reason, and the consumer reads a fact about their own instance. func TestBinderWaitsForAShardItCanUse(t *testing.T) { + mismatched := newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100") + mismatched.Status.ShardSID = "2001:db8:ffff::" + tests := []struct { name string objects []client.Object reason string }{ { - name: "no shard carries the class's labels", - objects: []client.Object{newEgressParameters()}, - reason: cloudv1alpha1.EgressShardClaimReasonNoShardMatchesTheClass, + name: "no shard names the node", + reason: cloudv1alpha1.EgressShardClaimReasonNoShardOnNode, }, { - name: "the only shard reports no identifier", - objects: []client.Object{newEgressParameters(), - newEgressShard("unprogrammed", "", "2001:db8:f00d::100", poolLabels())}, - reason: cloudv1alpha1.EgressShardClaimReasonNoShardIdentifier, + name: "the shard reports no identifier", + objects: []client.Object{ + newEgressShard("worker-3-egress", egressTestNode, "", "2001:db8:f00d::100")}, + reason: cloudv1alpha1.EgressShardClaimReasonShardNotReady, }, { - name: "the only shard translates no IPv6", - objects: []client.Object{newEgressParameters(), - newEgressShard("ipv4-only", "2001:db8:ff01::", "", map[string]string{ - bgpv1alpha1.LabelEgressShardPool: "shared", - bgpv1alpha1.LabelEgressShardCell: "us-central-1", - bgpv1alpha1.LabelEgressShardIPv4: bgpv1alpha1.LabelValueEgressFamilyServed, - })}, - reason: cloudv1alpha1.EgressShardClaimReasonNoShardMatchesTheClass, + name: "the shard runs an identity its spec does not state", + objects: []client.Object{mismatched}, + reason: cloudv1alpha1.EgressShardClaimReasonShardMismatch, }, { - name: "the only shard is being deleted", - objects: []client.Object{newEgressParameters(), - terminatingShard(newEgressShard("draining", "2001:db8:ff01::", - "2001:db8:f00d::100", poolLabels()))}, + name: "the shard is being deleted", + objects: []client.Object{terminatingShard( + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100"))}, reason: cloudv1alpha1.EgressShardClaimReasonShardTerminating, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { objects := append([]client.Object{ - newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled)}, test.objects...) + newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode)}, test.objects...) r, cl := newBinder(t, objects...) reconcileBinding(t, r) @@ -244,12 +245,10 @@ func TestBinderWaitsForAShardItCanUse(t *testing.T) { claim := readClaim(t, cl) if claim.Status.ShardRef != nil { - t.Fatalf("bound %v, want nothing", claim.Status.ShardRef) + t.Fatalf("recorded %v, want nothing", claim.Status.ShardRef) } assertClaimCondition(t, cl, metav1.ConditionFalse, test.reason) - // The consumer reads a fact about their own network. Which shard - // refused it is on the claim, which is an operator's object. - assertContextCondition(t, cl, metav1.ConditionFalse, + assertAttachmentCondition(t, cl, metav1.ConditionFalse, networkingv1alpha.NetworkContextInternetEgressReasonUnavailable) }) } @@ -258,82 +257,62 @@ func TestBinderWaitsForAShardItCanUse(t *testing.T) { // Egress that works and an address that cannot yet be stated are different // facts, and the condition says which. func TestBinderReportsThatNoAddressIsAllocatedYet(t *testing.T) { - r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled), - newEgressParameters(), - newEgressShard("shard-a", "2001:db8:ff01::", "", poolLabels())) + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "")) reconcileBinding(t, r) reconcileBinding(t, r) - claim := readClaim(t, cl) - if claim.Status.ShardRef == nil { - t.Fatal("a shard with an identifier and no address bound nothing") + if readClaim(t, cl).Status.ShardRef == nil { + t.Fatal("a shard with an identifier and no address recorded nothing") } assertClaimCondition(t, cl, metav1.ConditionTrue, cloudv1alpha1.EgressShardClaimReasonBound) - assertContextCondition(t, cl, metav1.ConditionFalse, + assertAttachmentCondition(t, cl, metav1.ConditionFalse, networkingv1alpha.NetworkContextInternetEgressReasonAddressUnavailable) } -// Decided once. A shard that would sort ahead of the bound one arriving later -// does not move a live network's egress, which is the address a consumer -// allow-listed at their destination. +// A record is not remade while the attachment stays where it is. A second +// shard naming the node is an operator error, and the one already recorded +// stands. func TestBinderNeverRebinds(t *testing.T) { - r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled), - newEgressParameters(), - newEgressShard("shard-b", "2001:db8:ff02::", "2001:db8:f00d::200", poolLabels())) + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode), + newEgressShard("worker-3-egress-b", egressTestNode, "2001:db8:ff02::", "2001:db8:f00d::200")) reconcileBinding(t, r) reconcileBinding(t, r) - if got := readClaim(t, cl).Status.ShardRef; got == nil || got.Name != "shard-b" { - t.Fatalf("got %v, want shard-b", got) + if got := readClaim(t, cl).Status.ShardRef; got == nil || got.Name != "worker-3-egress-b" { + t.Fatalf("got %v, want worker-3-egress-b", got) } - earlier := newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels()) + earlier := newEgressShard("worker-3-egress-a", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100") if err := cl.Create(t.Context(), earlier); err != nil { t.Fatalf("commission a shard sorting earlier: %v", err) } reconcileBinding(t, r) - - if got := readClaim(t, cl).Status.ShardRef; got == nil || got.Name != "shard-b" { - t.Errorf("got %v, want the shard it was already bound to", got) + if got := readClaim(t, cl).Status.ShardRef; got == nil || got.Name != "worker-3-egress-b" { + t.Errorf("got %v, want the shard already recorded", got) } } -// Egress withdrawn is a claim released, which is what takes the route and the -// address away and lets the shard go. +// Egress withdrawn is a claim released, which lets the shard go. func TestBinderReleasesTheClaimWhenEgressIsWithdrawn(t *testing.T) { + unprojected := newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled) + unprojected.Spec.Egress = nil + tests := []struct { name string - networkContext func() *networkingv1alpha.NetworkContext + networkContext *networkingv1alpha.NetworkContext }{ - { - name: "egress disabled", - networkContext: func() *networkingv1alpha.NetworkContext { - return newBoundContext(networkingv1alpha.NetworkInternetEgressDisabled) - }, - }, - { - name: "class served by another implementation", - networkContext: func() *networkingv1alpha.NetworkContext { - networkContext := newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled) - networkContext.Spec.Egress.Internet.ParametersRef.Kind = "SomeOtherParameters" - return networkContext - }, - }, - { - name: "intent never projected", - networkContext: func() *networkingv1alpha.NetworkContext { - networkContext := newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled) - networkContext.Spec.Egress = nil - return networkContext - }, - }, + {"egress disabled", newEgressContext(networkingv1alpha.NetworkInternetEgressDisabled)}, + {"intent never projected", unprojected}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - r, cl := newBinder(t, test.networkContext(), newEgressParameters(), - newEgressClaim("shard-a"), - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) + r, cl := newBinder(t, test.networkContext, newLandedAttachment(egressTestNode), + newEgressClaim("worker-3-egress"), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) reconcileBinding(t, r) @@ -344,152 +323,87 @@ func TestBinderReleasesTheClaimWhenEgressIsWithdrawn(t *testing.T) { } } -// A network no longer present in the cell takes its claim with it, rather than -// holding a shard open for a location that does not exist. -func TestBinderReleasesTheClaimWhenTheNetworkLeaves(t *testing.T) { - r, cl := newBinder(t, newEgressParameters(), newEgressClaim("shard-a"), - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) +// An attachment that is gone takes its record with it, rather than holding a +// shard open for an instance that no longer exists. +func TestBinderReleasesTheClaimWhenTheAttachmentLeaves(t *testing.T) { + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newEgressClaim("worker-3-egress"), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) reconcileBinding(t, r) if claimExists(t, cl) { - t.Error("the claim survived its network context") + t.Error("the claim survived its attachment") } } -// Parameters an operator has not written in this cell are an answer, not a -// silent nothing: the class this cell was pointed at does not exist here. -func TestBinderReportsAbsentParameters(t *testing.T) { - r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled)) +// An attachment that moved nodes is a different record. The spec is immutable, +// so the stale one is released and the next pass writes the new one. +func TestBinderReplacesTheRecordWhenTheAttachmentMovesNodes(t *testing.T) { + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment("worker-4"), + newEgressClaim("worker-3-egress"), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100"), + newEgressShard("worker-4-egress", "worker-4", "2001:db8:ff04::", "2001:db8:f00d::400")) reconcileBinding(t, r) - if claimExists(t, cl) { - t.Error("a claim was written for a class this cell cannot serve") + t.Fatal("a record for the node the attachment left was kept") } - assertContextCondition(t, cl, metav1.ConditionFalse, - networkingv1alpha.NetworkContextInternetEgressReasonUnavailable) -} - -// Sharing decides nothing here, but an unprojected value still means the claim -// cannot record what it was created under, and a claim is refused rather than -// written with a guess. -func TestBinderRefusesTermsItCannotRecord(t *testing.T) { - tests := []struct { - name string - amend func(*networkingv1alpha.NetworkContext) - }{ - { - name: "sharing never projected", - amend: func(networkContext *networkingv1alpha.NetworkContext) { - networkContext.Spec.Egress.Internet.Sharing = "" - }, - }, - { - name: "no address family to reach", - amend: func(networkContext *networkingv1alpha.NetworkContext) { - networkContext.Spec.Egress.Internet.Reach = nil - }, - }, - { - name: "location names no network", - amend: func(networkContext *networkingv1alpha.NetworkContext) { - networkContext.Spec.Network = networkingv1alpha.LocalNetworkRef{} - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - networkContext := newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled) - test.amend(networkContext) - r, cl := newBinder(t, networkContext, newEgressParameters(), - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) - - reconcileBinding(t, r) - - if claimExists(t, cl) { - t.Error("a claim was written from terms it could not record") - } - assertContextCondition(t, cl, metav1.ConditionFalse, - networkingv1alpha.NetworkContextInternetEgressReasonUnavailable) - }) - } -} - -// The terms are immutable, so a class change reaching a bound location cannot -// be applied to the binding. The binding keeps delivering what it was made for -// and says it no longer matches. -func TestBinderKeepsABindingWhoseTermsChanged(t *testing.T) { - networkContext := newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled) - networkContext.Spec.Egress.Internet.ClassName = "some-other-class" - - r, cl := newBinder(t, networkContext, newEgressParameters(), newEgressClaim("shard-a"), - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) reconcileBinding(t, r) - + reconcileBinding(t, r) claim := readClaim(t, cl) - if claim.Status.ShardRef == nil || claim.Status.ShardRef.Name != "shard-a" { - t.Fatalf("got %v, want the binding it already had", claim.Status.ShardRef) + if claim.Spec.NodeName != "worker-4" { + t.Errorf("node: got %q, want worker-4", claim.Spec.NodeName) + } + if claim.Status.ShardRef == nil || claim.Status.ShardRef.Name != "worker-4-egress" { + t.Errorf("shard: got %v, want worker-4-egress", claim.Status.ShardRef) } - assertClaimCondition(t, cl, metav1.ConditionFalse, - cloudv1alpha1.EgressShardClaimReasonTermsChanged) } -// An unbound claim whose terms changed is discarded rather than kept, because -// nothing is bound to protect and the next pass writes one that matches. -func TestBinderDiscardsAnUnboundClaimWhoseTermsChanged(t *testing.T) { - networkContext := newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled) - networkContext.Spec.Egress.Internet.ClassName = "some-other-class" - - r, cl := newBinder(t, networkContext, newEgressParameters(), newEgressClaim(""), - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) +// A label lost to an edit would hide an attachment from the query a shard's +// consumer set is counted by, so it is re-asserted on every pass. +func TestBinderRepairsTheShardLabel(t *testing.T) { + claim := newEgressClaim("worker-3-egress") + claim.Labels = nil + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode), claim, + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) reconcileBinding(t, r) - if claimExists(t, cl) { - t.Error("an unbound claim with stale terms was kept") + if got := readClaim(t, cl).Labels[cloudv1alpha1.LabelEgressShardClaimShard]; got != "worker-3-egress" { + t.Errorf("shard label: got %q, want worker-3-egress", got) } } -// A label lost to an edit would hide a network from the query a shard's -// consumer set is counted by, so it is re-asserted on every pass. -func TestBinderRepairsTheShardLabel(t *testing.T) { - claim := newEgressClaim("shard-a") - claim.Labels = nil - - r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled), - newEgressParameters(), claim, - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) +// A recorded shard that vanished is said on both objects. Nothing rebinds. +func TestBinderReportsAMissingShard(t *testing.T) { + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode), + newEgressClaim("worker-3-egress")) reconcileBinding(t, r) - if got := readClaim(t, cl).Labels[cloudv1alpha1.LabelEgressShardClaimShard]; got != "shard-a" { - t.Errorf("shard label: got %q, want shard-a", got) - } + assertClaimCondition(t, cl, metav1.ConditionFalse, cloudv1alpha1.EgressShardClaimReasonShardMissing) + assertAttachmentCondition(t, cl, metav1.ConditionFalse, + networkingv1alpha.NetworkContextInternetEgressReasonUnavailable) } // Degraded means egress works for some declared families and not others. Only -// one family is accepted anywhere on this path, so nothing may write it — the -// reason stays defined and unreachable rather than being given a fabricated -// path to reach it. +// one family is accepted anywhere on this path, so nothing may write it. func TestBinderNeverReportsDegraded(t *testing.T) { - r, cl := newBinder(t, newBoundContext(networkingv1alpha.NetworkInternetEgressEnabled), - newEgressParameters(), - newEgressShard("shard-a", "2001:db8:ff01::", "2001:db8:f00d::100", poolLabels())) + r, cl := newBinder(t, newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), + newLandedAttachment(egressTestNode), + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) reconcileBinding(t, r) reconcileBinding(t, r) - var networkContext networkingv1alpha.NetworkContext - key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressContextName} - if err := cl.Get(t.Context(), key, &networkContext); err != nil { - t.Fatalf("get the network context: %v", err) - } - condition := meta.FindStatusCondition(networkContext.Status.Conditions, - networkingv1alpha.NetworkContextInternetEgressReady) + condition := attachmentEgressCondition(t, cl) if condition == nil { - t.Fatal("the location reports no egress readiness") + t.Fatal("the attachment reports no egress readiness") } if condition.Reason == networkingv1alpha.NetworkContextInternetEgressReasonDegraded { t.Error("Degraded was reported for a path that accepts one address family") @@ -518,22 +432,27 @@ func assertClaimCondition( } } -func assertContextCondition( +func attachmentEgressCondition(t *testing.T, cl client.Client) *metav1.Condition { + t.Helper() + var attachment cloudv1alpha1.VPCAttachment + key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressAttachmentName} + if err := cl.Get(t.Context(), key, &attachment); err != nil { + t.Fatalf("get the attachment: %v", err) + } + return meta.FindStatusCondition(attachment.Status.Conditions, + cloudv1alpha1.ConditionTypeInternetEgressReady) +} + +func assertAttachmentCondition( t *testing.T, cl client.Client, status metav1.ConditionStatus, reason string, ) { t.Helper() - var networkContext networkingv1alpha.NetworkContext - key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressContextName} - if err := cl.Get(t.Context(), key, &networkContext); err != nil { - t.Fatalf("get the network context: %v", err) - } - condition := meta.FindStatusCondition(networkContext.Status.Conditions, - networkingv1alpha.NetworkContextInternetEgressReady) + condition := attachmentEgressCondition(t, cl) if condition == nil { - t.Fatal("the location reports no egress readiness") + t.Fatal("the attachment reports no InternetEgressReady condition") } if condition.Status != status || condition.Reason != reason { - t.Errorf("InternetEgressReady: got %s/%s, want %s/%s", + t.Errorf("attachment InternetEgressReady: got %s/%s, want %s/%s", condition.Status, condition.Reason, status, reason) } } diff --git a/internal/controller/networkinterface_controller.go b/internal/controller/networkinterface_controller.go index c827681..b289375 100644 --- a/internal/controller/networkinterface_controller.go +++ b/internal/controller/networkinterface_controller.go @@ -383,7 +383,7 @@ func (r *NetworkInterfaceReconciler) resolveInternetEgress( if attachment.Status.Node == "" { return resolved, nil } - shard, err := r.egressShardOnNode(ctx, attachment.Status.Node) + shard, err := egressShardOnNode(ctx, r.Client, attachment.Status.Node) if err != nil { return nil, err } @@ -433,11 +433,11 @@ func internetEgressIntent( // operator's to choose, and the node reference is what ties one to a node. Two // shards naming one node is an operator error, and the first by name is taken // so that every attachment on that node computes the same answer. -func (r *NetworkInterfaceReconciler) egressShardOnNode( - ctx context.Context, node string, +func egressShardOnNode( + ctx context.Context, reader client.Reader, node string, ) (*bgpv1alpha1.EgressShard, error) { var shards bgpv1alpha1.EgressShardList - if err := r.List(ctx, &shards, client.InNamespace(galactic.SystemNamespace)); err != nil { + if err := reader.List(ctx, &shards, client.InNamespace(galactic.SystemNamespace)); err != nil { return nil, fmt.Errorf("list egress shards: %w", err) } var found *bgpv1alpha1.EgressShard diff --git a/test/e2e/tests/egress-shard-claim-crd-schema/chainsaw-test.yaml b/test/e2e/tests/egress-shard-claim-crd-schema/chainsaw-test.yaml index ee811b3..3dd45cc 100644 --- a/test/e2e/tests/egress-shard-claim-crd-schema/chainsaw-test.yaml +++ b/test/e2e/tests/egress-shard-claim-crd-schema/chainsaw-test.yaml @@ -5,10 +5,10 @@ metadata: spec: description: > Verify schema validation and storage for the EgressShardClaim CRD. The rule that - matters most is the immutable spec: a binding is decided once from these terms and - never recomputed, so terms that moved underneath it would describe a binding never - made under them. Does not require any controller — CRD schema is enforced by the API - server alone, and the binding itself lives on status, which no consumer writes. + matters most is the immutable spec: a claim records the node an attachment landed + on, and an attachment that moves is a new record rather than an edited one. Does + not require any controller. CRD schema is enforced by the API server alone, and + the recorded shard lives on status, which no consumer writes. steps: - name: create-valid-claim try: @@ -17,14 +17,11 @@ spec: apiVersion: cloud.datumapis.com/v1alpha1 kind: EgressShardClaim metadata: - name: e2e-default-us-central-1 + name: web-eth0 spec: - network: - name: default - networkContext: - name: e2e-default-us-central-1 - className: shared - sharing: Shared + attachment: + name: web-eth0 + nodeName: worker-3 families: - IPv6 - assert: @@ -32,10 +29,9 @@ spec: apiVersion: cloud.datumapis.com/v1alpha1 kind: EgressShardClaim metadata: - name: e2e-default-us-central-1 + name: web-eth0 spec: - className: shared - sharing: Shared + nodeName: worker-3 - name: reject-spec-change try: @@ -46,14 +42,11 @@ spec: apiVersion: cloud.datumapis.com/v1alpha1 kind: EgressShardClaim metadata: - name: e2e-default-us-central-1 + name: web-eth0 spec: - network: - name: default - networkContext: - name: e2e-default-us-central-1 - className: some-other-class - sharing: Shared + attachment: + name: web-eth0 + nodeName: worker-4 families: - IPv6 EOF @@ -61,14 +54,14 @@ spec: EXIT=$? set -e if [ "$EXIT" -eq 0 ]; then - echo "ERROR: the terms a binding was decided from were changed under it" + echo "ERROR: the node a record was written for was changed under it" exit 1 fi echo "$OUTPUT" | grep -q "spec is immutable" || { echo "ERROR: rejected for the wrong reason: $OUTPUT" exit 1 } - echo "OK: the terms are immutable once written" + echo "OK: the record is immutable once written" - name: reject-ipv4-family try: @@ -79,14 +72,11 @@ spec: apiVersion: cloud.datumapis.com/v1alpha1 kind: EgressShardClaim metadata: - name: e2e-reaches-ipv4 + name: web-eth0-v4 spec: - network: - name: default - networkContext: - name: e2e-reaches-ipv4 - className: shared - sharing: Shared + attachment: + name: web-eth0 + nodeName: worker-3 families: - IPv4 EOF @@ -94,12 +84,12 @@ spec: EXIT=$? set -e if [ "$EXIT" -eq 0 ]; then - echo "ERROR: a claim requiring a family no shard translates was accepted" + echo "ERROR: an IPv4 family was accepted, which no shard translates" exit 1 fi - echo "OK: only the family the platform delivers is accepted" + echo "OK: IPv4 is refused" - - name: reject-empty-families + - name: reject-missing-node try: - script: content: | @@ -108,21 +98,18 @@ spec: apiVersion: cloud.datumapis.com/v1alpha1 kind: EgressShardClaim metadata: - name: e2e-reaches-nothing + name: web-eth0-nonode spec: - network: - name: default - networkContext: - name: e2e-reaches-nothing - className: shared - sharing: Shared - families: [] + attachment: + name: web-eth0 + families: + - IPv6 EOF ) EXIT=$? set -e if [ "$EXIT" -eq 0 ]; then - echo "ERROR: a claim reaching no address family was accepted" + echo "ERROR: a claim naming no node was accepted" exit 1 fi - echo "OK: a claim must state what it reaches" + echo "OK: a node is required" From f59b2b7385b82a4153c53bdec0088352f1491fc9 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Sat, 26 Sep 2026 11:28:04 -0500 Subject: [PATCH 4/4] feat: Take the egress shard claim from the network API The claim is now the contract between the cell controller and the node: the node's installer reads the claims naming it to keep each VRF's egress route in step, so the type moves to the API galactic already depends on. The controller keeps creating one per attachment while its network declares egress and deleting it when that stops. Key changes: - Drop the claim type and CRD from this API group - Name the attachment's VPC on the claim so a node can find the VRF without reading anything else - Label the claim with its node so a node lists its own claims server-side - Pin the network API at the commit that adds the claim Co-Authored-By: Claude Fable 5.1 --- api/v1alpha1/egressshardclaim_types.go | 186 --------------- api/v1alpha1/zz_generated.deepcopy.go | 137 ----------- ...cloud.datumapis.com_egressshardclaims.yaml | 218 ------------------ config/crd/kustomization.yaml | 1 - config/rbac/role.yaml | 22 +- docs/api/vpc.md | 107 --------- go.mod | 2 +- go.sum | 4 +- .../egressshardbinding_controller.go | 28 ++- .../egressshardbinding_controller_test.go | 8 +- .../controller/egressshardclaim_controller.go | 68 +++--- .../egressshardclaim_controller_test.go | 47 ++-- test/e2e/Taskfile.yaml | 1 - .../chainsaw-test.yaml | 115 --------- 14 files changed, 102 insertions(+), 842 deletions(-) delete mode 100644 api/v1alpha1/egressshardclaim_types.go delete mode 100644 config/crd/cloud.datumapis.com_egressshardclaims.yaml delete mode 100644 test/e2e/tests/egress-shard-claim-crd-schema/chainsaw-test.yaml diff --git a/api/v1alpha1/egressshardclaim_types.go b/api/v1alpha1/egressshardclaim_types.go deleted file mode 100644 index 749719e..0000000 --- a/api/v1alpha1/egressshardclaim_types.go +++ /dev/null @@ -1,186 +0,0 @@ -/* -Copyright © 2026 Datum Technology, Inc. All rights reserved. - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// LabelEgressShardClaimShard names the shard a claim is bound to. -// -// It is what makes a shard's consumer set a list rather than a number anyone -// has to keep in step: a shard holds no list of the attachments it serves, so -// "what does this shard serve" is answered by listing claims carrying this -// label. The value is the shard's name. The binding itself lives on the claim's -// status, which is what a reader trusts; this label narrows the query that -// finds the claims to ask. -const LabelEgressShardClaimShard = "cloud.datumapis.com/egress-shard" - -// FinalizerEgressShardBinding is the one piece of state a binder adds to a -// shard, held while any claim is bound to it. -// -// It exists so that decommissioning a shard is an act someone takes rather -// than an outcome instances discover. Deleting a shard that is translating -// strands the return traffic of every flow on it. -const FinalizerEgressShardBinding = "cloud.datumapis.com/egress-shard-binding" - -// EgressShardClaimSpec is one attachment being recorded against the egress -// shard on its node. -// -// The claim decides nothing. The node already routes toward its own shard from -// the moment the attachment exists; the claim records which shard that is, so -// the binding is readable, so a node without a usable shard produces a -// condition a consumer can see, and so a later tier that does select among -// shards binds through the same object. -// -// The whole spec is immutable. An attachment that lands on a different node is -// a different record, so the claim is replaced rather than edited. -// -// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec is immutable; an attachment that moved nodes gets a new claim" -type EgressShardClaimSpec struct { - // Attachment is the attachment this claim records egress for. The claim - // carries the attachment's name and namespace, so the two are read by one - // key. - // +required - Attachment AttachmentRef `json:"attachment"` - - // NodeName is the node the attachment landed on, and therefore the node - // whose shard serves it. - // +kubebuilder:validation:MinLength=1 - // +required - NodeName string `json:"nodeName"` - - // Families are the destination address families the network declared, - // so the shard on the node is one that translates them. - // +listType=set - // +kubebuilder:validation:MinItems=1 - // +kubebuilder:validation:MaxItems=2 - // +required - Families []InternetEgressAddressFamily `json:"families"` -} - -// AttachmentRef references a VPCAttachment by name. -type AttachmentRef struct { - // Name of the VPCAttachment. - // +kubebuilder:validation:MinLength=1 - // +required - Name string `json:"name"` -} - -// EgressShardReference names the shard an attachment egresses through. -type EgressShardReference struct { - // Namespace of the EgressShard. - // +kubebuilder:validation:MinLength=1 - // +required - Namespace string `json:"namespace"` - - // Name of the EgressShard. - // +kubebuilder:validation:MinLength=1 - // +required - Name string `json:"name"` -} - -// EgressShardClaimStatus is the shard an attachment was recorded against. -type EgressShardClaimStatus struct { - // +optional - ObservedGeneration int64 `json:"observedGeneration,omitempty"` - - // +listType=map - // +listMapKey=type - // +optional - Conditions []metav1.Condition `json:"conditions,omitempty"` - - // ShardRef is the shard on the attachment's node. - // - // Absent means the node holds no shard this claim can record, which is - // what an attachment on a node an operator has not commissioned reads. - // +optional - ShardRef *EgressShardReference `json:"shardRef,omitempty"` -} - -// Reasons reported on an EgressShardClaim's Ready condition. -const ( - // EgressShardClaimReasonBound means this attachment egresses through the - // shard status names. - EgressShardClaimReasonBound = "Bound" - - // EgressShardClaimReasonNoShardOnNode means no shard names the node the - // attachment landed on. - EgressShardClaimReasonNoShardOnNode = "NoShardOnNode" - - // EgressShardClaimReasonShardNotReady means the shard on the node has not - // reported the identifier a node routes toward. - EgressShardClaimReasonShardNotReady = "ShardNotReady" - - // EgressShardClaimReasonShardMismatch means the shard's spec and the - // identity its process reported disagree, so which one the node runs is - // unknown and nothing is recorded against it. - EgressShardClaimReasonShardMismatch = "ShardMismatch" - - // EgressShardClaimReasonFamilyUnsupported means the shard on the node - // translates none of a family the network declared. - EgressShardClaimReasonFamilyUnsupported = "FamilyUnsupported" - - // EgressShardClaimReasonShardMissing means the recorded shard no longer - // exists. The node's instances lost their egress with it. - EgressShardClaimReasonShardMissing = "ShardMissing" - - // EgressShardClaimReasonShardTerminating means the recorded shard is being - // deleted. The record stands, and the shard is held until the claim goes. - EgressShardClaimReasonShardTerminating = "ShardTerminating" -) - -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status -// +kubebuilder:resource:scope=Namespaced -// +kubebuilder:printcolumn:name="Attachment",type="string",JSONPath=".spec.attachment.name" -// +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".spec.nodeName" -// +kubebuilder:printcolumn:name="Shard",type="string",JSONPath=".status.shardRef.name" -// +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=`.status.conditions[?(@.type=="Ready")].status` -// +kubebuilder:printcolumn:name="Reason",type="string",JSONPath=`.status.conditions[?(@.type=="Ready")].reason` -// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" - -// EgressShardClaim records one attachment's egress shard: the shard on the -// node the attachment landed on. -// -// There is one claim per attachment, owned by it, so an attachment that goes -// takes its record with it. The claim names no selector, no address and no -// pool: the node is the binding, and the claim writes it down. -type EgressShardClaim struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - // +required - Spec EgressShardClaimSpec `json:"spec"` - - // +optional - Status EgressShardClaimStatus `json:"status,omitempty"` -} - -// +kubebuilder:object:root=true - -// EgressShardClaimList contains a list of EgressShardClaim. -type EgressShardClaimList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []EgressShardClaim `json:"items"` -} - -func init() { - SchemeBuilder.Register(&EgressShardClaim{}, &EgressShardClaimList{}) -} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 47bb875..d76dd55 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -26,143 +26,6 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AttachmentRef) DeepCopyInto(out *AttachmentRef) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AttachmentRef. -func (in *AttachmentRef) DeepCopy() *AttachmentRef { - if in == nil { - return nil - } - out := new(AttachmentRef) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EgressShardClaim) DeepCopyInto(out *EgressShardClaim) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardClaim. -func (in *EgressShardClaim) DeepCopy() *EgressShardClaim { - if in == nil { - return nil - } - out := new(EgressShardClaim) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *EgressShardClaim) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EgressShardClaimList) DeepCopyInto(out *EgressShardClaimList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]EgressShardClaim, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardClaimList. -func (in *EgressShardClaimList) DeepCopy() *EgressShardClaimList { - if in == nil { - return nil - } - out := new(EgressShardClaimList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *EgressShardClaimList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EgressShardClaimSpec) DeepCopyInto(out *EgressShardClaimSpec) { - *out = *in - out.Attachment = in.Attachment - if in.Families != nil { - in, out := &in.Families, &out.Families - *out = make([]InternetEgressAddressFamily, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardClaimSpec. -func (in *EgressShardClaimSpec) DeepCopy() *EgressShardClaimSpec { - if in == nil { - return nil - } - out := new(EgressShardClaimSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EgressShardClaimStatus) DeepCopyInto(out *EgressShardClaimStatus) { - *out = *in - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.ShardRef != nil { - in, out := &in.ShardRef, &out.ShardRef - *out = new(EgressShardReference) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardClaimStatus. -func (in *EgressShardClaimStatus) DeepCopy() *EgressShardClaimStatus { - if in == nil { - return nil - } - out := new(EgressShardClaimStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EgressShardReference) DeepCopyInto(out *EgressShardReference) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressShardReference. -func (in *EgressShardReference) DeepCopy() *EgressShardReference { - if in == nil { - return nil - } - out := new(EgressShardReference) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InternetEgressSourceAddress) DeepCopyInto(out *InternetEgressSourceAddress) { *out = *in diff --git a/config/crd/cloud.datumapis.com_egressshardclaims.yaml b/config/crd/cloud.datumapis.com_egressshardclaims.yaml deleted file mode 100644 index 9d800b2..0000000 --- a/config/crd/cloud.datumapis.com_egressshardclaims.yaml +++ /dev/null @@ -1,218 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.18.0 - name: egressshardclaims.cloud.datumapis.com -spec: - group: cloud.datumapis.com - names: - kind: EgressShardClaim - listKind: EgressShardClaimList - plural: egressshardclaims - singular: egressshardclaim - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .spec.attachment.name - name: Attachment - type: string - - jsonPath: .spec.nodeName - name: Node - type: string - - jsonPath: .status.shardRef.name - name: Shard - type: string - - jsonPath: .status.conditions[?(@.type=="Ready")].status - name: Ready - type: string - - jsonPath: .status.conditions[?(@.type=="Ready")].reason - name: Reason - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1alpha1 - schema: - openAPIV3Schema: - description: |- - EgressShardClaim records one attachment's egress shard: the shard on the - node the attachment landed on. - - There is one claim per attachment, owned by it, so an attachment that goes - takes its record with it. The claim names no selector, no address and no - pool: the node is the binding, and the claim writes it down. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - EgressShardClaimSpec is one attachment being recorded against the egress - shard on its node. - - The claim decides nothing. The node already routes toward its own shard from - the moment the attachment exists; the claim records which shard that is, so - the binding is readable, so a node without a usable shard produces a - condition a consumer can see, and so a later tier that does select among - shards binds through the same object. - - The whole spec is immutable. An attachment that lands on a different node is - a different record, so the claim is replaced rather than edited. - properties: - attachment: - description: |- - Attachment is the attachment this claim records egress for. The claim - carries the attachment's name and namespace, so the two are read by one - key. - properties: - name: - description: Name of the VPCAttachment. - minLength: 1 - type: string - required: - - name - type: object - families: - description: |- - Families are the destination address families the network declared, - so the shard on the node is one that translates them. - items: - description: |- - InternetEgressAddressFamily is the address family of an egress source - address. - - Only IPv6 is reported. Reaching an IPv4 destination needs a resolver and a - translator sharing a prefix, which the platform pairs neither of, so the - value is withheld rather than reported and not delivered. An address written - today records IPv6, so accepting IPv4 later changes no attachment. - enum: - - IPv6 - type: string - maxItems: 2 - minItems: 1 - type: array - x-kubernetes-list-type: set - nodeName: - description: |- - NodeName is the node the attachment landed on, and therefore the node - whose shard serves it. - minLength: 1 - type: string - required: - - attachment - - families - - nodeName - type: object - x-kubernetes-validations: - - message: spec is immutable; an attachment that moved nodes gets a new - claim - rule: self == oldSelf - status: - description: EgressShardClaimStatus is the shard an attachment was recorded - against. - properties: - conditions: - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - observedGeneration: - format: int64 - type: integer - shardRef: - description: |- - ShardRef is the shard on the attachment's node. - - Absent means the node holds no shard this claim can record, which is - what an attachment on a node an operator has not commissioned reads. - properties: - name: - description: Name of the EgressShard. - minLength: 1 - type: string - namespace: - description: Namespace of the EgressShard. - minLength: 1 - type: string - required: - - name - - namespace - type: object - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 14b7a3f..8d0a278 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,7 +3,6 @@ kind: Kustomization resources: - cloud.datumapis.com_vpcs.yaml - cloud.datumapis.com_vpcattachments.yaml - - cloud.datumapis.com_egressshardclaims.yaml # Written centrally, federated to the cells that need it, so it is installed # both places. - cloud.datumapis.com_networkfabricidentities.yaml diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ddfa554..b012a84 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -14,7 +14,6 @@ rules: - apiGroups: - cloud.datumapis.com resources: - - egressshardclaims - networkfabricidentities - vpcattachments - vpcs @@ -29,7 +28,6 @@ rules: - apiGroups: - cloud.datumapis.com resources: - - egressshardclaims/status - vpcattachments/status - vpcs/status verbs: @@ -77,6 +75,26 @@ rules: - get - list - watch +- apiGroups: + - network.datumapis.com + resources: + - egressshardclaims + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - network.datumapis.com + resources: + - egressshardclaims/status + verbs: + - get + - patch + - update - apiGroups: - network.datumapis.com resources: diff --git a/docs/api/vpc.md b/docs/api/vpc.md index 5deca7b..a15848e 100644 --- a/docs/api/vpc.md +++ b/docs/api/vpc.md @@ -9,118 +9,12 @@ Package v1alpha1 contains API Schema definitions for the cloud.datumapis.com/v1alpha1 API group. ### Resource Types -- [EgressShardClaim](#egressshardclaim) - [NetworkFabricIdentity](#networkfabricidentity) - [VPC](#vpc) - [VPCAttachment](#vpcattachment) -#### AttachmentRef - - - -AttachmentRef references a VPCAttachment by name. - - - -_Appears in:_ -- [EgressShardClaimSpec](#egressshardclaimspec) - -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `name` _string_ | Name of the VPCAttachment. | | MinLength: 1
| - - -#### EgressShardClaim - - - -EgressShardClaim records one attachment's egress shard: the shard on the -node the attachment landed on. - -There is one claim per attachment, owned by it, so an attachment that goes -takes its record with it. The claim names no selector, no address and no -pool: the node is the binding, and the claim writes it down. - - - - - -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `apiVersion` _string_ | `cloud.datumapis.com/v1alpha1` | | | -| `kind` _string_ | `EgressShardClaim` | | | -| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | -| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | -| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | -| `spec` _[EgressShardClaimSpec](#egressshardclaimspec)_ | | | | -| `status` _[EgressShardClaimStatus](#egressshardclaimstatus)_ | | | | - - -#### EgressShardClaimSpec - - - -EgressShardClaimSpec is one attachment being recorded against the egress -shard on its node. - -The claim decides nothing. The node already routes toward its own shard from -the moment the attachment exists; the claim records which shard that is, so -the binding is readable, so a node without a usable shard produces a -condition a consumer can see, and so a later tier that does select among -shards binds through the same object. - -The whole spec is immutable. An attachment that lands on a different node is -a different record, so the claim is replaced rather than edited. - - - -_Appears in:_ -- [EgressShardClaim](#egressshardclaim) - -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `attachment` _[AttachmentRef](#attachmentref)_ | Attachment is the attachment this claim records egress for. The claim
carries the attachment's name and namespace, so the two are read by one
key. | | | -| `nodeName` _string_ | NodeName is the node the attachment landed on, and therefore the node
whose shard serves it. | | MinLength: 1
| -| `families` _[InternetEgressAddressFamily](#internetegressaddressfamily) array_ | Families are the destination address families the network declared,
so the shard on the node is one that translates them. | | Enum: [IPv6]
MaxItems: 2
MinItems: 1
| - - -#### EgressShardClaimStatus - - - -EgressShardClaimStatus is the shard an attachment was recorded against. - - - -_Appears in:_ -- [EgressShardClaim](#egressshardclaim) - -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `observedGeneration` _integer_ | | | | -| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v/#condition-v1-meta) array_ | | | | -| `shardRef` _[EgressShardReference](#egressshardreference)_ | ShardRef is the shard on the attachment's node.
Absent means the node holds no shard this claim can record, which is
what an attachment on a node an operator has not commissioned reads. | | | - - -#### EgressShardReference - - - -EgressShardReference names the shard an attachment egresses through. - - - -_Appears in:_ -- [EgressShardClaimStatus](#egressshardclaimstatus) - -| Field | Description | Default | Validation | -| --- | --- | --- | --- | -| `namespace` _string_ | Namespace of the EgressShard. | | MinLength: 1
| -| `name` _string_ | Name of the EgressShard. | | MinLength: 1
| - - #### IPAddress _Underlying type:_ _string_ @@ -151,7 +45,6 @@ _Validation:_ - Enum: [IPv6] _Appears in:_ -- [EgressShardClaimSpec](#egressshardclaimspec) - [InternetEgressSourceAddress](#internetegresssourceaddress) | Field | Description | diff --git a/go.mod b/go.mod index 258934f..cc5f779 100644 --- a/go.mod +++ b/go.mod @@ -79,6 +79,6 @@ require ( sigs.k8s.io/yaml v1.6.0 // indirect ) -replace go.datum.net/network => github.com/datum-cloud/network v0.1.1-0.20260923215140-1ed44c853eea +replace go.datum.net/network => github.com/datum-cloud/network v0.1.1-0.20260926162437-e92beadac630 replace go.datum.net/network-services-operator => github.com/datum-cloud/network-services-operator v0.27.2-0.20260924225138-f78ee4cb7f7f diff --git a/go.sum b/go.sum index 298a125..7cb46b9 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/datum-cloud/network v0.1.1-0.20260923215140-1ed44c853eea h1:/a67sX+DD7+tRpaQSDAuoFuOfoMz9hC22wi4AGakiuk= -github.com/datum-cloud/network v0.1.1-0.20260923215140-1ed44c853eea/go.mod h1:dqzM8WZczbiZ9bCvsxjkoI10GJqQ24NVWnc9boXgOkE= +github.com/datum-cloud/network v0.1.1-0.20260926162437-e92beadac630 h1:qy9/S+Y4C67n7zvwgN1rjfTVxJVRziGxCgMWIkBwKL8= +github.com/datum-cloud/network v0.1.1-0.20260926162437-e92beadac630/go.mod h1:dqzM8WZczbiZ9bCvsxjkoI10GJqQ24NVWnc9boXgOkE= github.com/datum-cloud/network-services-operator v0.27.2-0.20260924225138-f78ee4cb7f7f h1:O/pK+GHjZqAQIBWYmkkEJNmxTPp6tmrL60bXMkWzyC0= github.com/datum-cloud/network-services-operator v0.27.2-0.20260924225138-f78ee4cb7f7f/go.mod h1:IyFCDsNxfxvXHFfotZsD8eBBI9EyxpxAbQ7UTg6CNdw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/controller/egressshardbinding_controller.go b/internal/controller/egressshardbinding_controller.go index c8dbd41..5204c4f 100644 --- a/internal/controller/egressshardbinding_controller.go +++ b/internal/controller/egressshardbinding_controller.go @@ -29,7 +29,6 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" - cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" bgpv1alpha1 "go.datum.net/network/api/v1alpha1" ) @@ -52,7 +51,12 @@ type EgressShardBindingReconciler struct { } // +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshards,verbs=get;list;watch;update;patch -// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=egressshardclaims,verbs=get;list;watch +// finalizerEgressShardBinding is held on a shard while any claim is bound to it, +// so decommissioning a shard is an act someone takes rather than an outcome +// instances discover. +const finalizerEgressShardBinding = "cloud.datumapis.com/egress-shard-binding" + +// +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshardclaims,verbs=get;list;watch func (r *EgressShardBindingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { var shard bgpv1alpha1.EgressShard @@ -80,7 +84,7 @@ func (r *EgressShardBindingReconciler) Reconcile(ctx context.Context, req ctrl.R // holdShard adds the binder's finalizer, so the shard cannot go while a network // is bound to it. func holdShard(ctx context.Context, cl client.Client, shard *bgpv1alpha1.EgressShard) error { - if controllerutil.ContainsFinalizer(shard, cloudv1alpha1.FinalizerEgressShardBinding) { + if controllerutil.ContainsFinalizer(shard, finalizerEgressShardBinding) { return nil } // Patched rather than updated. The controller holding the addressing-service @@ -88,7 +92,7 @@ func holdShard(ctx context.Context, cl client.Client, shard *bgpv1alpha1.EgressS // copy read before that write would put the old value back over a field that // is write-once. patch := client.MergeFrom(shard.DeepCopy()) - controllerutil.AddFinalizer(shard, cloudv1alpha1.FinalizerEgressShardBinding) + controllerutil.AddFinalizer(shard, finalizerEgressShardBinding) if err := cl.Patch(ctx, shard, patch); err != nil { return fmt.Errorf("hold egress shard %s open for the networks bound to it: %w", client.ObjectKeyFromObject(shard), err) @@ -99,11 +103,11 @@ func holdShard(ctx context.Context, cl client.Client, shard *bgpv1alpha1.EgressS // releaseShard removes the binder's finalizer from a shard no network is bound // to, which is what lets an operator decommission a drained node. func releaseShard(ctx context.Context, cl client.Client, shard *bgpv1alpha1.EgressShard) error { - if !controllerutil.ContainsFinalizer(shard, cloudv1alpha1.FinalizerEgressShardBinding) { + if !controllerutil.ContainsFinalizer(shard, finalizerEgressShardBinding) { return nil } patch := client.MergeFrom(shard.DeepCopy()) - controllerutil.RemoveFinalizer(shard, cloudv1alpha1.FinalizerEgressShardBinding) + controllerutil.RemoveFinalizer(shard, finalizerEgressShardBinding) if err := cl.Patch(ctx, shard, patch); err != nil { return fmt.Errorf("release egress shard %s: %w", client.ObjectKeyFromObject(shard), err) } @@ -120,15 +124,15 @@ func releaseShard(ctx context.Context, cl client.Client, shard *bgpv1alpha1.Egre // binding that never completed counts as nothing. func boundEgressShardClaims( ctx context.Context, reader client.Reader, shard *bgpv1alpha1.EgressShard, -) ([]cloudv1alpha1.EgressShardClaim, error) { - var claims cloudv1alpha1.EgressShardClaimList +) ([]bgpv1alpha1.EgressShardClaim, error) { + var claims bgpv1alpha1.EgressShardClaimList if err := reader.List(ctx, &claims, client.MatchingLabels{ - cloudv1alpha1.LabelEgressShardClaimShard: shard.Name, + bgpv1alpha1.LabelEgressShardClaimShard: shard.Name, }); err != nil { return nil, fmt.Errorf("list the claims bound to egress shard %s: %w", shard.Name, err) } - bound := make([]cloudv1alpha1.EgressShardClaim, 0, len(claims.Items)) + bound := make([]bgpv1alpha1.EgressShardClaim, 0, len(claims.Items)) for i := range claims.Items { claim := claims.Items[i] held := claim.Status.ShardRef @@ -147,7 +151,7 @@ func boundEgressShardClaims( func (r *EgressShardBindingReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&bgpv1alpha1.EgressShard{}). - Watches(&cloudv1alpha1.EgressShardClaim{}, + Watches(&bgpv1alpha1.EgressShardClaim{}, handler.EnqueueRequestsFromMapFunc(egressShardForClaim)). Named("egressshardbinding"). Complete(r) @@ -160,7 +164,7 @@ func (r *EgressShardBindingReconciler) SetupWithManager(mgr ctrl.Manager) error // A claim holding no binding maps to nothing, and needs to: it was never part // of any shard's consumer set, which is counted from this same field. func egressShardForClaim(_ context.Context, object client.Object) []reconcile.Request { - claim, ok := object.(*cloudv1alpha1.EgressShardClaim) + claim, ok := object.(*bgpv1alpha1.EgressShardClaim) if !ok || claim.Status.ShardRef == nil { return nil } diff --git a/internal/controller/egressshardbinding_controller_test.go b/internal/controller/egressshardbinding_controller_test.go index 125acf2..683bbf4 100644 --- a/internal/controller/egressshardbinding_controller_test.go +++ b/internal/controller/egressshardbinding_controller_test.go @@ -63,7 +63,7 @@ func heldOpen(t *testing.T, cl client.Client) bool { if err := cl.Get(t.Context(), key, &shard); err != nil { t.Fatalf("get the shard: %v", err) } - return controllerutil.ContainsFinalizer(&shard, cloudv1alpha1.FinalizerEgressShardBinding) + return controllerutil.ContainsFinalizer(&shard, finalizerEgressShardBinding) } // The finalizer is the only state a binder adds to a shard, and it holds while @@ -84,7 +84,7 @@ func TestShardIsHeldOpenWhileANetworkIsBound(t *testing.T) { // node it runs on. func TestShardIsReleasedWhenNoNetworkIsBound(t *testing.T) { shard := newEgressShard("shard-a", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100") - shard.Finalizers = []string{cloudv1alpha1.FinalizerEgressShardBinding} + shard.Finalizers = []string{finalizerEgressShardBinding} r, cl := newShardBinder(t, shard) reconcileShard(t, r) @@ -100,7 +100,7 @@ func TestShardIsReleasedWhenAClaimHoldsOnlyTheLabel(t *testing.T) { claim := newEgressClaim("shard-a") claim.Status.ShardRef = nil shard := newEgressShard("shard-a", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100") - shard.Finalizers = []string{cloudv1alpha1.FinalizerEgressShardBinding} + shard.Finalizers = []string{finalizerEgressShardBinding} r, cl := newShardBinder(t, claim, shard) reconcileShard(t, r) @@ -116,7 +116,7 @@ func TestShardIgnoresAClaimBoundElsewhere(t *testing.T) { claim := newEgressClaim("shard-a") claim.Status.ShardRef.Namespace = "some-other-namespace" shard := newEgressShard("shard-a", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100") - shard.Finalizers = []string{cloudv1alpha1.FinalizerEgressShardBinding} + shard.Finalizers = []string{finalizerEgressShardBinding} r, cl := newShardBinder(t, claim, shard) reconcileShard(t, r) diff --git a/internal/controller/egressshardclaim_controller.go b/internal/controller/egressshardclaim_controller.go index 9acd41e..180fcf0 100644 --- a/internal/controller/egressshardclaim_controller.go +++ b/internal/controller/egressshardclaim_controller.go @@ -74,8 +74,8 @@ type EgressShardClaimReconciler struct { // +kubebuilder:rbac:groups=cloud.datumapis.com,resources=vpcattachments,verbs=get;list;watch // +kubebuilder:rbac:groups=cloud.datumapis.com,resources=vpcattachments/status,verbs=get;update;patch // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkcontexts,verbs=get;list;watch -// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=egressshardclaims,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=cloud.datumapis.com,resources=egressshardclaims/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshardclaims,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshardclaims/status,verbs=get;update;patch // +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshards,verbs=get;list;watch;update;patch func (r *EgressShardClaimReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -89,7 +89,7 @@ func (r *EgressShardClaimReconciler) Reconcile(ctx context.Context, req ctrl.Req attachmentFound = false } - var claim cloudv1alpha1.EgressShardClaim + var claim bgpv1alpha1.EgressShardClaim claimFound := true if err := r.Get(ctx, req.NamespacedName, &claim); err != nil { if !apierrors.IsNotFound(err) { @@ -152,7 +152,7 @@ func (r *EgressShardClaimReconciler) Reconcile(ctx context.Context, req ctrl.Req // or nil when there is nothing to record yet. func (r *EgressShardClaimReconciler) claimTerms( ctx context.Context, attachment *cloudv1alpha1.VPCAttachment, -) (*cloudv1alpha1.EgressShardClaimSpec, error) { +) (*bgpv1alpha1.EgressShardClaimSpec, error) { if attachment.Status.Node == "" { return nil, nil } @@ -174,8 +174,9 @@ func (r *EgressShardClaimReconciler) claimTerms( if err != nil { return nil, err } - return &cloudv1alpha1.EgressShardClaimSpec{ - Attachment: cloudv1alpha1.AttachmentRef{Name: attachment.Name}, + return &bgpv1alpha1.EgressShardClaimSpec{ + Attachment: bgpv1alpha1.EgressShardClaimAttachmentRef{Name: attachment.Name}, + VPC: bgpv1alpha1.EgressShardClaimVPCRef{Name: attachment.Spec.VPC.Name}, NodeName: attachment.Status.Node, Families: families, }, nil @@ -191,14 +192,14 @@ func (r *EgressShardClaimReconciler) claimTerms( // consumer does not have. func claimFamilies( reach []networkingv1alpha.IPFamily, -) ([]cloudv1alpha1.InternetEgressAddressFamily, error) { +) ([]bgpv1alpha1.EgressAddressFamily, error) { if len(reach) == 0 { return nil, &bindingRefusedError{ reason: networkingv1alpha.NetworkContextInternetEgressReasonUnavailable, message: "The network declares no address family to reach, so no shard can serve it", } } - families := make([]cloudv1alpha1.InternetEgressAddressFamily, 0, len(reach)) + families := make([]bgpv1alpha1.EgressAddressFamily, 0, len(reach)) for _, family := range reach { if family != networkingv1alpha.IPv6Protocol { return nil, &bindingRefusedError{ @@ -207,7 +208,7 @@ func claimFamilies( family), } } - families = append(families, cloudv1alpha1.InternetEgressAddressFamilyIPv6) + families = append(families, bgpv1alpha1.EgressAddressFamilyIPv6) } return families, nil } @@ -218,12 +219,13 @@ func claimFamilies( func (r *EgressShardClaimReconciler) createClaim( ctx context.Context, attachment *cloudv1alpha1.VPCAttachment, - terms *cloudv1alpha1.EgressShardClaimSpec, + terms *bgpv1alpha1.EgressShardClaimSpec, ) error { - claim := &cloudv1alpha1.EgressShardClaim{ + claim := &bgpv1alpha1.EgressShardClaim{ ObjectMeta: metav1.ObjectMeta{ Namespace: attachment.Namespace, Name: attachment.Name, + Labels: map[string]string{bgpv1alpha1.LabelEgressShardClaimNode: terms.NodeName}, }, Spec: *terms, } @@ -245,7 +247,7 @@ func (r *EgressShardClaimReconciler) createClaim( // to record. Deleting it is what releases the shard: the consumer set is a // list of claims. func (r *EgressShardClaimReconciler) releaseClaim( - ctx context.Context, claim *cloudv1alpha1.EgressShardClaim, claimFound bool, why string, + ctx context.Context, claim *bgpv1alpha1.EgressShardClaim, claimFound bool, why string, ) error { if !claimFound { return nil @@ -263,7 +265,7 @@ func (r *EgressShardClaimReconciler) releaseClaim( func (r *EgressShardClaimReconciler) bind( ctx context.Context, attachment *cloudv1alpha1.VPCAttachment, - claim *cloudv1alpha1.EgressShardClaim, + claim *bgpv1alpha1.EgressShardClaim, ) error { shard, err := egressShardOnNode(ctx, r.Client, claim.Spec.NodeName) if err != nil { @@ -271,7 +273,7 @@ func (r *EgressShardClaimReconciler) bind( } if shard == nil { return r.refuse(ctx, attachment, claim, &bindingRefusedError{ - reason: cloudv1alpha1.EgressShardClaimReasonNoShardOnNode, + reason: bgpv1alpha1.EgressShardClaimReasonNoShardOnNode, message: fmt.Sprintf("No egress shard names node %q, so nothing on it translates this attachment's traffic", claim.Spec.NodeName), }) @@ -284,18 +286,18 @@ func (r *EgressShardClaimReconciler) bind( // shardRefusal is why a shard may not be recorded, or nil if it may be. func shardRefusal( - shard *bgpv1alpha1.EgressShard, families []cloudv1alpha1.InternetEgressAddressFamily, + shard *bgpv1alpha1.EgressShard, families []bgpv1alpha1.EgressAddressFamily, ) *bindingRefusedError { if !shard.DeletionTimestamp.IsZero() { return &bindingRefusedError{ - reason: cloudv1alpha1.EgressShardClaimReasonShardTerminating, + reason: bgpv1alpha1.EgressShardClaimReasonShardTerminating, message: fmt.Sprintf("Egress shard %q is being deleted, so it takes no further attachment", shard.Name), } } if shard.Status.ShardSID == "" { return &bindingRefusedError{ - reason: cloudv1alpha1.EgressShardClaimReasonShardNotReady, + reason: bgpv1alpha1.EgressShardClaimReasonShardNotReady, message: fmt.Sprintf("Egress shard %q has reported no identifier a node can route toward", shard.Name), } @@ -304,15 +306,15 @@ func shardRefusal( (shard.Spec.ShardAddressIPv6 != "" && shard.Status.ShardAddressIPv6 != "" && shard.Spec.ShardAddressIPv6 != shard.Status.ShardAddressIPv6) { return &bindingRefusedError{ - reason: cloudv1alpha1.EgressShardClaimReasonShardMismatch, + reason: bgpv1alpha1.EgressShardClaimReasonShardMismatch, message: fmt.Sprintf("Egress shard %q runs an identity other than the one its spec states, so which one serves this node is unknown", shard.Name), } } - if slices.Contains(families, cloudv1alpha1.InternetEgressAddressFamilyIPv4) && + if slices.Contains(families, bgpv1alpha1.EgressAddressFamilyIPv4) && shard.Status.ShardAddressIPv4 == "" { return &bindingRefusedError{ - reason: cloudv1alpha1.EgressShardClaimReasonFamilyUnsupported, + reason: bgpv1alpha1.EgressShardClaimReasonFamilyUnsupported, message: fmt.Sprintf("Egress shard %q translates no IPv4 flow, which the network declares it reaches", shard.Name), } @@ -327,7 +329,7 @@ func shardRefusal( func (r *EgressShardClaimReconciler) recordBinding( ctx context.Context, attachment *cloudv1alpha1.VPCAttachment, - claim *cloudv1alpha1.EgressShardClaim, + claim *bgpv1alpha1.EgressShardClaim, shard *bgpv1alpha1.EgressShard, ) error { if err := holdShard(ctx, r.Client, shard); err != nil { @@ -337,7 +339,7 @@ func (r *EgressShardClaimReconciler) recordBinding( return err } - claim.Status.ShardRef = &cloudv1alpha1.EgressShardReference{ + claim.Status.ShardRef = &bgpv1alpha1.EgressShardClaimShardRef{ Namespace: shard.Namespace, Name: shard.Name, } @@ -346,7 +348,7 @@ func (r *EgressShardClaimReconciler) recordBinding( "shard", client.ObjectKeyFromObject(shard)) if err := r.publishClaimStatus(ctx, claim, metav1.ConditionTrue, - cloudv1alpha1.EgressShardClaimReasonBound, + bgpv1alpha1.EgressShardClaimReasonBound, fmt.Sprintf("Attachment %q egresses through egress shard %q", claim.Spec.Attachment.Name, shard.Name)); err != nil { return err } @@ -358,16 +360,16 @@ func (r *EgressShardClaimReconciler) recordBinding( // label lost to an edit would hide an attachment from the query that holds a // shard open. func (r *EgressShardClaimReconciler) labelClaim( - ctx context.Context, claim *cloudv1alpha1.EgressShardClaim, shardName string, + ctx context.Context, claim *bgpv1alpha1.EgressShardClaim, shardName string, ) error { - if claim.Labels[cloudv1alpha1.LabelEgressShardClaimShard] == shardName { + if claim.Labels[bgpv1alpha1.LabelEgressShardClaimShard] == shardName { return nil } patch := client.MergeFrom(claim.DeepCopy()) if claim.Labels == nil { claim.Labels = map[string]string{} } - claim.Labels[cloudv1alpha1.LabelEgressShardClaimShard] = shardName + claim.Labels[bgpv1alpha1.LabelEgressShardClaimShard] = shardName if err := r.Patch(ctx, claim, patch); err != nil { return fmt.Errorf("label EgressShardClaim %s with its shard: %w", client.ObjectKeyFromObject(claim), err) @@ -381,7 +383,7 @@ func (r *EgressShardClaimReconciler) labelClaim( func (r *EgressShardClaimReconciler) reportExistingBinding( ctx context.Context, attachment *cloudv1alpha1.VPCAttachment, - claim *cloudv1alpha1.EgressShardClaim, + claim *bgpv1alpha1.EgressShardClaim, ) error { if err := r.labelClaim(ctx, claim, claim.Status.ShardRef.Name); err != nil { return err @@ -398,7 +400,7 @@ func (r *EgressShardClaimReconciler) reportExistingBinding( } message := fmt.Sprintf("Egress shard %q no longer exists", key.Name) if err := r.publishClaimStatus(ctx, claim, metav1.ConditionFalse, - cloudv1alpha1.EgressShardClaimReasonShardMissing, message); err != nil { + bgpv1alpha1.EgressShardClaimReasonShardMissing, message); err != nil { return err } return r.reportAttachment(ctx, attachment, metav1.ConditionFalse, @@ -434,7 +436,7 @@ func (r *EgressShardClaimReconciler) reportBinding( func (r *EgressShardClaimReconciler) refuse( ctx context.Context, attachment *cloudv1alpha1.VPCAttachment, - claim *cloudv1alpha1.EgressShardClaim, + claim *bgpv1alpha1.EgressShardClaim, refusal *bindingRefusedError, ) error { if err := r.publishClaimStatus(ctx, claim, metav1.ConditionFalse, @@ -449,7 +451,7 @@ func (r *EgressShardClaimReconciler) refuse( func (r *EgressShardClaimReconciler) publishClaimStatus( ctx context.Context, - claim *cloudv1alpha1.EgressShardClaim, + claim *bgpv1alpha1.EgressShardClaim, status metav1.ConditionStatus, reason, message string, ) error { @@ -501,11 +503,11 @@ func (r *EgressShardClaimReconciler) reportAttachment( // SetupWithManager registers the reconciler with the manager. func (r *EgressShardClaimReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). - For(&cloudv1alpha1.EgressShardClaim{}, + For(&bgpv1alpha1.EgressShardClaim{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { // A bound claim is never reconsidered on its own events. It is // re-read when its attachment or its shard changes. - claim, ok := object.(*cloudv1alpha1.EgressShardClaim) + claim, ok := object.(*bgpv1alpha1.EgressShardClaim) return ok && claim.Status.ShardRef == nil }))). Watches(&cloudv1alpha1.VPCAttachment{}, @@ -555,7 +557,7 @@ func (r *EgressShardClaimReconciler) claimsForEgressShard( if !ok { return nil } - var claims cloudv1alpha1.EgressShardClaimList + var claims bgpv1alpha1.EgressShardClaimList if err := r.List(ctx, &claims); err != nil { return nil } diff --git a/internal/controller/egressshardclaim_controller_test.go b/internal/controller/egressshardclaim_controller_test.go index b10b35f..102466f 100644 --- a/internal/controller/egressshardclaim_controller_test.go +++ b/internal/controller/egressshardclaim_controller_test.go @@ -52,7 +52,7 @@ func newBinder(t *testing.T, objects ...client.Object) (*EgressShardClaimReconci } fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...). - WithStatusSubresource(&cloudv1alpha1.EgressShardClaim{}, &cloudv1alpha1.VPCAttachment{}). + WithStatusSubresource(&bgpv1alpha1.EgressShardClaim{}, &cloudv1alpha1.VPCAttachment{}). Build() return &EgressShardClaimReconciler{Client: fakeClient, Scheme: scheme}, fakeClient } @@ -71,18 +71,19 @@ func newLandedAttachment(node string) *cloudv1alpha1.VPCAttachment { // newEgressClaim is a record already written for the test attachment, bound // to shardName or, with an empty name, still unbound. -func newEgressClaim(shardName string) *cloudv1alpha1.EgressShardClaim { - claim := &cloudv1alpha1.EgressShardClaim{} +func newEgressClaim(shardName string) *bgpv1alpha1.EgressShardClaim { + claim := &bgpv1alpha1.EgressShardClaim{} claim.Namespace = egressTestNamespace claim.Name = egressAttachmentName - claim.Spec = cloudv1alpha1.EgressShardClaimSpec{ - Attachment: cloudv1alpha1.AttachmentRef{Name: egressAttachmentName}, + claim.Spec = bgpv1alpha1.EgressShardClaimSpec{ + Attachment: bgpv1alpha1.EgressShardClaimAttachmentRef{Name: egressAttachmentName}, + VPC: bgpv1alpha1.EgressShardClaimVPCRef{Name: "default-us-central-1"}, NodeName: egressTestNode, - Families: []cloudv1alpha1.InternetEgressAddressFamily{cloudv1alpha1.InternetEgressAddressFamilyIPv6}, + Families: []bgpv1alpha1.EgressAddressFamily{bgpv1alpha1.EgressAddressFamilyIPv6}, } if shardName != "" { - claim.Labels = map[string]string{cloudv1alpha1.LabelEgressShardClaimShard: shardName} - claim.Status.ShardRef = &cloudv1alpha1.EgressShardReference{ + claim.Labels = map[string]string{bgpv1alpha1.LabelEgressShardClaimShard: shardName} + claim.Status.ShardRef = &bgpv1alpha1.EgressShardClaimShardRef{ Namespace: egressShardNamespace, Name: shardName, } @@ -98,9 +99,9 @@ func reconcileBinding(t *testing.T, r *EgressShardClaimReconciler) { } } -func readClaim(t *testing.T, cl client.Client) *cloudv1alpha1.EgressShardClaim { +func readClaim(t *testing.T, cl client.Client) *bgpv1alpha1.EgressShardClaim { t.Helper() - var claim cloudv1alpha1.EgressShardClaim + var claim bgpv1alpha1.EgressShardClaim key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressAttachmentName} if err := cl.Get(t.Context(), key, &claim); err != nil { t.Fatalf("get the claim: %v", err) @@ -110,7 +111,7 @@ func readClaim(t *testing.T, cl client.Client) *cloudv1alpha1.EgressShardClaim { func claimExists(t *testing.T, cl client.Client) bool { t.Helper() - var claim cloudv1alpha1.EgressShardClaim + var claim bgpv1alpha1.EgressShardClaim key := client.ObjectKey{Namespace: egressTestNamespace, Name: egressAttachmentName} return cl.Get(t.Context(), key, &claim) == nil } @@ -132,7 +133,7 @@ func TestBinderRecordsOncePerAttachment(t *testing.T) { t.Errorf("node: got %q, want %q", claim.Spec.NodeName, egressTestNode) } if len(claim.Spec.Families) != 1 || - claim.Spec.Families[0] != cloudv1alpha1.InternetEgressAddressFamilyIPv6 { + claim.Spec.Families[0] != bgpv1alpha1.EgressAddressFamilyIPv6 { t.Errorf("families: got %v, want [IPv6]", claim.Spec.Families) } if claim.Status.ShardRef != nil { @@ -178,10 +179,10 @@ func TestBinderRecordsTheShardOnTheNode(t *testing.T) { if claim.Status.ShardRef.Namespace != egressShardNamespace { t.Errorf("shard namespace: got %q, want %q", claim.Status.ShardRef.Namespace, egressShardNamespace) } - if got := claim.Labels[cloudv1alpha1.LabelEgressShardClaimShard]; got != "worker-3-egress" { + if got := claim.Labels[bgpv1alpha1.LabelEgressShardClaimShard]; got != "worker-3-egress" { t.Errorf("shard label: got %q, want worker-3-egress", got) } - assertClaimCondition(t, cl, metav1.ConditionTrue, cloudv1alpha1.EgressShardClaimReasonBound) + assertClaimCondition(t, cl, metav1.ConditionTrue, bgpv1alpha1.EgressShardClaimReasonBound) assertAttachmentCondition(t, cl, metav1.ConditionTrue, networkingv1alpha.NetworkContextInternetEgressReasonReady) @@ -192,7 +193,7 @@ func TestBinderRecordsTheShardOnTheNode(t *testing.T) { if err := cl.Get(t.Context(), key, &shard); err != nil { t.Fatalf("get the recorded shard: %v", err) } - if !controllerutil.ContainsFinalizer(&shard, cloudv1alpha1.FinalizerEgressShardBinding) { + if !controllerutil.ContainsFinalizer(&shard, finalizerEgressShardBinding) { t.Error("the recorded shard is not held open") } if shard.Spec.ShardAddressIPv6 != "2001:db8:f00d::100" { @@ -213,24 +214,24 @@ func TestBinderWaitsForAShardItCanUse(t *testing.T) { }{ { name: "no shard names the node", - reason: cloudv1alpha1.EgressShardClaimReasonNoShardOnNode, + reason: bgpv1alpha1.EgressShardClaimReasonNoShardOnNode, }, { name: "the shard reports no identifier", objects: []client.Object{ newEgressShard("worker-3-egress", egressTestNode, "", "2001:db8:f00d::100")}, - reason: cloudv1alpha1.EgressShardClaimReasonShardNotReady, + reason: bgpv1alpha1.EgressShardClaimReasonShardNotReady, }, { name: "the shard runs an identity its spec does not state", objects: []client.Object{mismatched}, - reason: cloudv1alpha1.EgressShardClaimReasonShardMismatch, + reason: bgpv1alpha1.EgressShardClaimReasonShardMismatch, }, { name: "the shard is being deleted", objects: []client.Object{terminatingShard( newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100"))}, - reason: cloudv1alpha1.EgressShardClaimReasonShardTerminating, + reason: bgpv1alpha1.EgressShardClaimReasonShardTerminating, }, } for _, test := range tests { @@ -267,7 +268,7 @@ func TestBinderReportsThatNoAddressIsAllocatedYet(t *testing.T) { if readClaim(t, cl).Status.ShardRef == nil { t.Fatal("a shard with an identifier and no address recorded nothing") } - assertClaimCondition(t, cl, metav1.ConditionTrue, cloudv1alpha1.EgressShardClaimReasonBound) + assertClaimCondition(t, cl, metav1.ConditionTrue, bgpv1alpha1.EgressShardClaimReasonBound) assertAttachmentCondition(t, cl, metav1.ConditionFalse, networkingv1alpha.NetworkContextInternetEgressReasonAddressUnavailable) } @@ -373,7 +374,7 @@ func TestBinderRepairsTheShardLabel(t *testing.T) { reconcileBinding(t, r) - if got := readClaim(t, cl).Labels[cloudv1alpha1.LabelEgressShardClaimShard]; got != "worker-3-egress" { + if got := readClaim(t, cl).Labels[bgpv1alpha1.LabelEgressShardClaimShard]; got != "worker-3-egress" { t.Errorf("shard label: got %q, want worker-3-egress", got) } } @@ -386,7 +387,7 @@ func TestBinderReportsAMissingShard(t *testing.T) { reconcileBinding(t, r) - assertClaimCondition(t, cl, metav1.ConditionFalse, cloudv1alpha1.EgressShardClaimReasonShardMissing) + assertClaimCondition(t, cl, metav1.ConditionFalse, bgpv1alpha1.EgressShardClaimReasonShardMissing) assertAttachmentCondition(t, cl, metav1.ConditionFalse, networkingv1alpha.NetworkContextInternetEgressReasonUnavailable) } @@ -411,7 +412,7 @@ func TestBinderNeverReportsDegraded(t *testing.T) { } func terminatingShard(shard *bgpv1alpha1.EgressShard) *bgpv1alpha1.EgressShard { - shard.Finalizers = []string{cloudv1alpha1.FinalizerEgressShardBinding} + shard.Finalizers = []string{finalizerEgressShardBinding} deletion := metav1.Now() shard.DeletionTimestamp = &deletion return shard diff --git a/test/e2e/Taskfile.yaml b/test/e2e/Taskfile.yaml index 481e01c..ca5048b 100644 --- a/test/e2e/Taskfile.yaml +++ b/test/e2e/Taskfile.yaml @@ -57,7 +57,6 @@ tasks: --for=condition=Established crd/vpcs.cloud.datumapis.com crd/vpcattachments.cloud.datumapis.com - crd/egressshardclaims.cloud.datumapis.com --timeout=60s test: diff --git a/test/e2e/tests/egress-shard-claim-crd-schema/chainsaw-test.yaml b/test/e2e/tests/egress-shard-claim-crd-schema/chainsaw-test.yaml deleted file mode 100644 index 3dd45cc..0000000 --- a/test/e2e/tests/egress-shard-claim-crd-schema/chainsaw-test.yaml +++ /dev/null @@ -1,115 +0,0 @@ -apiVersion: chainsaw.kyverno.io/v1alpha1 -kind: Test -metadata: - name: egress-shard-claim-crd-schema -spec: - description: > - Verify schema validation and storage for the EgressShardClaim CRD. The rule that - matters most is the immutable spec: a claim records the node an attachment landed - on, and an attachment that moves is a new record rather than an edited one. Does - not require any controller. CRD schema is enforced by the API server alone, and - the recorded shard lives on status, which no consumer writes. - steps: - - name: create-valid-claim - try: - - apply: - resource: - apiVersion: cloud.datumapis.com/v1alpha1 - kind: EgressShardClaim - metadata: - name: web-eth0 - spec: - attachment: - name: web-eth0 - nodeName: worker-3 - families: - - IPv6 - - assert: - resource: - apiVersion: cloud.datumapis.com/v1alpha1 - kind: EgressShardClaim - metadata: - name: web-eth0 - spec: - nodeName: worker-3 - - - name: reject-spec-change - try: - - script: - content: | - set +e - OUTPUT=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF' - apiVersion: cloud.datumapis.com/v1alpha1 - kind: EgressShardClaim - metadata: - name: web-eth0 - spec: - attachment: - name: web-eth0 - nodeName: worker-4 - families: - - IPv6 - EOF - ) - EXIT=$? - set -e - if [ "$EXIT" -eq 0 ]; then - echo "ERROR: the node a record was written for was changed under it" - exit 1 - fi - echo "$OUTPUT" | grep -q "spec is immutable" || { - echo "ERROR: rejected for the wrong reason: $OUTPUT" - exit 1 - } - echo "OK: the record is immutable once written" - - - name: reject-ipv4-family - try: - - script: - content: | - set +e - OUTPUT=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF' - apiVersion: cloud.datumapis.com/v1alpha1 - kind: EgressShardClaim - metadata: - name: web-eth0-v4 - spec: - attachment: - name: web-eth0 - nodeName: worker-3 - families: - - IPv4 - EOF - ) - EXIT=$? - set -e - if [ "$EXIT" -eq 0 ]; then - echo "ERROR: an IPv4 family was accepted, which no shard translates" - exit 1 - fi - echo "OK: IPv4 is refused" - - - name: reject-missing-node - try: - - script: - content: | - set +e - OUTPUT=$(kubectl apply -n "$NAMESPACE" -f - 2>&1 <<'EOF' - apiVersion: cloud.datumapis.com/v1alpha1 - kind: EgressShardClaim - metadata: - name: web-eth0-nonode - spec: - attachment: - name: web-eth0 - families: - - IPv6 - EOF - ) - EXIT=$? - set -e - if [ "$EXIT" -eq 0 ]; then - echo "ERROR: a claim naming no node was accepted" - exit 1 - fi - echo "OK: a node is required"