diff --git a/api/v1alpha1/vpcattachment_types.go b/api/v1alpha1/vpcattachment_types.go index 13cbe33..674ee0c 100644 --- a/api/v1alpha1/vpcattachment_types.go +++ b/api/v1alpha1/vpcattachment_types.go @@ -114,6 +114,85 @@ type VPCAttachmentInterface struct { Addresses []IPAddress `json:"addresses,omitempty"` } +// InternetEgressAddressFamily is the address family of an egress source +// address. +// +// Only IPv6 is reported. Reaching an IPv4 destination needs a resolver and a +// translator sharing a prefix, which the platform pairs neither of, so the +// value is withheld rather than reported and not delivered. An address written +// today records IPv6, so accepting IPv4 later changes no attachment. +// +// +kubebuilder:validation:Enum=IPv6 +type InternetEgressAddressFamily string + +// InternetEgressAddressFamilyIPv6 is an IPv6 egress source address. +const InternetEgressAddressFamilyIPv6 InternetEgressAddressFamily = "IPv6" + +// InternetEgressAddressStability is how far a consumer may rely on an egress +// source address. It is the consumer-side projection of the serving class's +// sharing, derived here so a consumer never reads a class. +// +// +kubebuilder:validation:Enum=None;Network +type InternetEgressAddressStability string + +const ( + // InternetEgressAddressStabilityNone means the address may change and + // other networks share it. Allow-listing it admits traffic from other + // networks and loses access when the address changes. + InternetEgressAddressStabilityNone InternetEgressAddressStability = "None" + + // InternetEgressAddressStabilityNetwork means the address belongs to this + // network and persists. Allow-listing it is safe. + InternetEgressAddressStabilityNetwork InternetEgressAddressStability = "Network" +) + +// InternetEgressSourceAddress is one address outbound traffic leaves on. +// +// +kubebuilder:validation:XValidation:rule="self.family != 'IPv6' || (isIP(self.address) && ip(self.address).family() == 6)",message="an IPv6 source address must be a valid IPv6 address" +type InternetEgressSourceAddress struct { + // Family is the address family of this source address. + // +required + Family InternetEgressAddressFamily `json:"family"` + + // Address is the source address translation writes, without a prefix + // length. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=39 + // +required + Address string `json:"address"` + + // Stability states how far a consumer may rely on this address before + // they act on it. + // +required + Stability InternetEgressAddressStability `json:"stability"` +} + +// VPCAttachmentInternetEgressStatus reports the outbound path this attachment +// leaves the platform on. +type VPCAttachmentInternetEgressStatus struct { + // SourceAddresses are the addresses translation writes for this + // attachment, one per family reached. + // + // Absent means this attachment reaches nothing outside the platform, or + // that no address has been reported for a path that does. An absent list + // is never a placeholder: a consumer that allow-listed a guessed address + // would admit the wrong traffic and believe otherwise. + // + // +listType=map + // +listMapKey=family + // +kubebuilder:validation:MaxItems=2 + // +optional + SourceAddresses []InternetEgressSourceAddress `json:"sourceAddresses,omitempty"` +} + +// VPCAttachmentEgressStatus reports what this attachment reaches outside the +// platform. +type VPCAttachmentEgressStatus struct { + // Internet is the internet egress realized for this attachment. + // +optional + Internet *VPCAttachmentInternetEgressStatus `json:"internet,omitempty"` +} + // VPCAttachmentStatus defines the observed state of VPCAttachment. // // Every field but Conditions is optional: an identifier is recorded before a pod @@ -181,6 +260,15 @@ type VPCAttachmentStatus struct { // +kubebuilder:validation:MinLength=1 // +optional NetworkAttachmentDefinition string `json:"networkAttachmentDefinition,omitempty"` + + // Egress reports what this attachment reaches outside the platform. + // + // It is reported per attachment rather than on the network, because the + // interface is what a workload holds and what a consumer reads back + // through. This controller is the only component that resolved which shard + // the network bound to, so it is the only one that can report the answer. + // +optional + Egress *VPCAttachmentEgressStatus `json:"egress,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index b31bf5c..d76dd55 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -26,6 +26,21 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InternetEgressSourceAddress) DeepCopyInto(out *InternetEgressSourceAddress) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InternetEgressSourceAddress. +func (in *InternetEgressSourceAddress) DeepCopy() *InternetEgressSourceAddress { + if in == nil { + return nil + } + out := new(InternetEgressSourceAddress) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkFabricIdentity) DeepCopyInto(out *NetworkFabricIdentity) { *out = *in @@ -184,6 +199,26 @@ func (in *VPCAttachment) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VPCAttachmentEgressStatus) DeepCopyInto(out *VPCAttachmentEgressStatus) { + *out = *in + if in.Internet != nil { + in, out := &in.Internet, &out.Internet + *out = new(VPCAttachmentInternetEgressStatus) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPCAttachmentEgressStatus. +func (in *VPCAttachmentEgressStatus) DeepCopy() *VPCAttachmentEgressStatus { + if in == nil { + return nil + } + out := new(VPCAttachmentEgressStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VPCAttachmentInterface) DeepCopyInto(out *VPCAttachmentInterface) { *out = *in @@ -204,6 +239,26 @@ func (in *VPCAttachmentInterface) DeepCopy() *VPCAttachmentInterface { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VPCAttachmentInternetEgressStatus) DeepCopyInto(out *VPCAttachmentInternetEgressStatus) { + *out = *in + if in.SourceAddresses != nil { + in, out := &in.SourceAddresses, &out.SourceAddresses + *out = make([]InternetEgressSourceAddress, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPCAttachmentInternetEgressStatus. +func (in *VPCAttachmentInternetEgressStatus) DeepCopy() *VPCAttachmentInternetEgressStatus { + if in == nil { + return nil + } + out := new(VPCAttachmentInternetEgressStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VPCAttachmentList) DeepCopyInto(out *VPCAttachmentList) { *out = *in @@ -268,6 +323,11 @@ func (in *VPCAttachmentStatus) DeepCopyInto(out *VPCAttachmentStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Egress != nil { + in, out := &in.Egress, &out.Egress + *out = new(VPCAttachmentEgressStatus) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VPCAttachmentStatus. diff --git a/config/crd/cloud.datumapis.com_vpcattachments.yaml b/config/crd/cloud.datumapis.com_vpcattachments.yaml index d49ec0b..e030fa9 100644 --- a/config/crd/cloud.datumapis.com_vpcattachments.yaml +++ b/config/crd/cloud.datumapis.com_vpcattachments.yaml @@ -164,6 +164,69 @@ spec: maxLength: 46 minLength: 46 type: string + egress: + description: |- + Egress reports what this attachment reaches outside the platform. + + It is reported per attachment rather than on the network, because the + interface is what a workload holds and what a consumer reads back + through. This controller is the only component that resolved which shard + the network bound to, so it is the only one that can report the answer. + properties: + internet: + description: Internet is the internet egress realized for this + attachment. + properties: + sourceAddresses: + description: |- + SourceAddresses are the addresses translation writes for this + attachment, one per family reached. + + Absent means this attachment reaches nothing outside the platform, or + that no address has been reported for a path that does. An absent list + is never a placeholder: a consumer that allow-listed a guessed address + would admit the wrong traffic and believe otherwise. + items: + description: InternetEgressSourceAddress is one address + outbound traffic leaves on. + properties: + address: + description: |- + Address is the source address translation writes, without a prefix + length. + maxLength: 39 + minLength: 1 + type: string + family: + description: Family is the address family of this source + address. + enum: + - IPv6 + type: string + stability: + description: |- + Stability states how far a consumer may rely on this address before + they act on it. + enum: + - None + - Network + type: string + required: + - address + - family + - stability + type: object + x-kubernetes-validations: + - message: an IPv6 source address must be a valid IPv6 address + rule: self.family != 'IPv6' || (isIP(self.address) && + ip(self.address).family() == 6) + maxItems: 2 + type: array + x-kubernetes-list-map-keys: + - family + x-kubernetes-list-type: map + type: object + type: object guestInterface: description: Guest-side veth device name (e.g., "G000000010013G"). minLength: 1 diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index bbfc2ec..8d0a278 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -6,3 +6,5 @@ resources: # Written centrally, federated to the cells that need it, so it is installed # both places. - cloud.datumapis.com_networkfabricidentities.yaml + # Cell-local: an operator writes one per InternetEgressClass this cell serves, + # and only the controller in the cell reads it. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 8600ec8..01f8cca 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -71,6 +71,7 @@ rules: resources: - bgpadvertisements - bgprouters + - egressshards verbs: - get - list diff --git a/docs/api/vpc.md b/docs/api/vpc.md index 537f5ee..3481582 100644 --- a/docs/api/vpc.md +++ b/docs/api/vpc.md @@ -29,6 +29,67 @@ _Appears in:_ +#### InternetEgressAddressFamily + +_Underlying type:_ _string_ + +InternetEgressAddressFamily is the address family of an egress source +address. + +Only IPv6 is reported. Reaching an IPv4 destination needs a resolver and a +translator sharing a prefix, which the platform pairs neither of, so the +value is withheld rather than reported and not delivered. An address written +today records IPv6, so accepting IPv4 later changes no attachment. + +_Validation:_ +- Enum: [IPv6] + +_Appears in:_ +- [InternetEgressSourceAddress](#internetegresssourceaddress) + +| Field | Description | +| --- | --- | +| `IPv6` | | + + +#### InternetEgressAddressStability + +_Underlying type:_ _string_ + +InternetEgressAddressStability is how far a consumer may rely on an egress +source address. It is the consumer-side projection of the serving class's +sharing, derived here so a consumer never reads a class. + +_Validation:_ +- Enum: [None Network] + +_Appears in:_ +- [InternetEgressSourceAddress](#internetegresssourceaddress) + +| Field | Description | +| --- | --- | +| `None` | InternetEgressAddressStabilityNone means the address may change and
other networks share it. Allow-listing it admits traffic from other
networks and loses access when the address changes.
| +| `Network` | InternetEgressAddressStabilityNetwork means the address belongs to this
network and persists. Allow-listing it is safe.
| + + +#### InternetEgressSourceAddress + + + +InternetEgressSourceAddress is one address outbound traffic leaves on. + + + +_Appears in:_ +- [VPCAttachmentInternetEgressStatus](#vpcattachmentinternetegressstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `family` _[InternetEgressAddressFamily](#internetegressaddressfamily)_ | Family is the address family of this source address. | | Enum: [IPv6]
| +| `address` _string_ | Address is the source address translation writes, without a prefix
length. | | MaxLength: 39
MinLength: 1
| +| `stability` _[InternetEgressAddressStability](#internetegressaddressstability)_ | Stability states how far a consumer may rely on this address before
they act on it. | | Enum: [None Network]
| + + #### Network _Underlying type:_ _string_ @@ -172,6 +233,23 @@ VPCAttachment is the Schema for the vpcattachments API | `status` _[VPCAttachmentStatus](#vpcattachmentstatus)_ | status defines the observed state of VPCAttachment | | | +#### VPCAttachmentEgressStatus + + + +VPCAttachmentEgressStatus reports what this attachment reaches outside the +platform. + + + +_Appears in:_ +- [VPCAttachmentStatus](#vpcattachmentstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `internet` _[VPCAttachmentInternetEgressStatus](#vpcattachmentinternetegressstatus)_ | Internet is the internet egress realized for this attachment. | | | + + #### VPCAttachmentInterface @@ -211,6 +289,23 @@ _Appears in:_ | `HypervisorDeclared` | VPCAttachmentInterfaceModeHypervisorDeclared also hands the interface to a
hypervisor as a device. It differs from Hypervisor in who tells the
hypervisor that the device exists. Under Hypervisor the hypervisor finds
the device from what the node publishes. Under HypervisorDeclared the data
plane states the device, its addresses, and its MTU to the hypervisor
directly, which is what a guest whose hypervisor reads no node state
needs.
| +#### VPCAttachmentInternetEgressStatus + + + +VPCAttachmentInternetEgressStatus reports the outbound path this attachment +leaves the platform on. + + + +_Appears in:_ +- [VPCAttachmentEgressStatus](#vpcattachmentegressstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `sourceAddresses` _[InternetEgressSourceAddress](#internetegresssourceaddress) array_ | SourceAddresses are the addresses translation writes for this
attachment, one per family reached.
Absent means this attachment reaches nothing outside the platform, or
that no address has been reported for a path that does. An absent list
is never a placeholder: a consumer that allow-listed a guessed address
would admit the wrong traffic and believe otherwise. | | MaxItems: 2
| + + #### VPCAttachmentSpec @@ -257,6 +352,7 @@ _Appears in:_ | `guestInterface` _string_ | Guest-side veth device name (e.g., "G000000010013G"). | | MinLength: 1
| | `podSubnet` _string_ | Allocated subnet in CIDR notation (e.g., "fd00:10:ff01:0:1::/80"). | | MinLength: 1
| | `networkAttachmentDefinition` _string_ | NetworkAttachmentDefinition rendered for this attachment. | | MinLength: 1
| +| `egress` _[VPCAttachmentEgressStatus](#vpcattachmentegressstatus)_ | Egress reports what this attachment reaches outside the platform.
It is reported per attachment rather than on the network, because the
interface is what a workload holds and what a consumer reads back
through. This controller is the only component that resolved which shard
the network bound to, so it is the only one that can report the answer. | | | #### VPCRef diff --git a/go.mod b/go.mod index 0431086..258934f 100644 --- a/go.mod +++ b/go.mod @@ -78,3 +78,7 @@ require ( sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) + +replace go.datum.net/network => github.com/datum-cloud/network v0.1.1-0.20260923215140-1ed44c853eea + +replace go.datum.net/network-services-operator => github.com/datum-cloud/network-services-operator v0.27.2-0.20260924225138-f78ee4cb7f7f diff --git a/go.sum b/go.sum index 6c1b0e7..298a125 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/datum-cloud/network v0.1.1-0.20260923215140-1ed44c853eea h1:/a67sX+DD7+tRpaQSDAuoFuOfoMz9hC22wi4AGakiuk= +github.com/datum-cloud/network v0.1.1-0.20260923215140-1ed44c853eea/go.mod h1:dqzM8WZczbiZ9bCvsxjkoI10GJqQ24NVWnc9boXgOkE= +github.com/datum-cloud/network-services-operator v0.27.2-0.20260924225138-f78ee4cb7f7f h1:O/pK+GHjZqAQIBWYmkkEJNmxTPp6tmrL60bXMkWzyC0= +github.com/datum-cloud/network-services-operator v0.27.2-0.20260924225138-f78ee4cb7f7f/go.mod h1:IyFCDsNxfxvXHFfotZsD8eBBI9EyxpxAbQ7UTg6CNdw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -121,10 +125,6 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= go.datum.net/compute v0.8.0 h1:/v1lni/oO4KwthPeXhhS0+VFRZjWYbU6CXiF3RgCMzY= go.datum.net/compute v0.8.0/go.mod h1:u7YIQX4+Wgts5XJwtvlz5NMs23g7+R/4TkAmv3DRycw= -go.datum.net/network v0.1.0 h1:AmYSwxUWOk26UnK6S6NA7OuucGJniKo/CWqjs+VcSCs= -go.datum.net/network v0.1.0/go.mod h1:dqzM8WZczbiZ9bCvsxjkoI10GJqQ24NVWnc9boXgOkE= -go.datum.net/network-services-operator v0.27.0 h1:LYCUjc6i0/f3c5YXSgisnbV8gB3R8emDwNfiYapdCAo= -go.datum.net/network-services-operator v0.27.0/go.mod h1:9nuuBWdrkdnIBMaWJsWM3j4CcbKJIF1GDuCmAjdpIHo= go.miloapis.com/ipam v0.4.0 h1:U+mg3RMFXj0c2eQ0Lm15CI0bYBPFDqy8IsGLN8v/eGs= go.miloapis.com/ipam v0.4.0/go.mod h1:Jj7xg4lJi9psE0+4PuOg/GQOG8rG13h112xYoM994rc= go.miloapis.com/locations v0.0.1 h1:voJKqBzyLX5x96M3+5y/ga7fgq9/+vypTizd+bBvqUY= diff --git a/internal/controller/networkinterface_controller.go b/internal/controller/networkinterface_controller.go index 5b0a370..c827681 100644 --- a/internal/controller/networkinterface_controller.go +++ b/internal/controller/networkinterface_controller.go @@ -32,11 +32,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" "go.datum.net/cloud/internal/galactic" "go.datum.net/cloud/internal/identifier" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" ) const ( @@ -93,7 +96,9 @@ type NetworkInterfaceReconciler struct { // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaces/status,verbs=get;update;patch // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaceclaims,verbs=get;list;watch // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkinterfaceclaims/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkcontexts,verbs=get;list;watch // +kubebuilder:rbac:groups=cloud.datumapis.com,resources=vpcs,verbs=get;list;watch +// +kubebuilder:rbac:groups=network.datumapis.com,resources=egressshards,verbs=get;list;watch // +kubebuilder:rbac:groups=cloud.datumapis.com,resources=vpcattachments,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=cloud.datumapis.com,resources=vpcattachments/status,verbs=get;update;patch // +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch;create;update;patch;delete @@ -110,6 +115,9 @@ func (r *NetworkInterfaceReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, nil } + // The VPC is named after the NetworkContext it realizes, so one key reads + // both: the identity the fabric keys on, and the egress intent projected + // onto this location. var vpc cloudv1alpha1.VPC vpcKey := types.NamespacedName{ Namespace: networkInterface.Namespace, @@ -128,15 +136,36 @@ func (r *NetworkInterfaceReconciler) Reconcile(ctx context.Context, req ctrl.Req fmt.Sprintf("VPC %s has no identifier yet", vpc.Name)) } + var networkContext networkingv1alpha.NetworkContext + if err := r.Get(ctx, vpcKey, &networkContext); err != nil { + if apierrors.IsNotFound(err) { + // The VPC exists only because this context did, so a context that + // is gone is a network being withdrawn from the cell rather than a + // race worth rendering through. + return ctrl.Result{RequeueAfter: 10 * time.Second}, r.markPrepared(ctx, &networkInterface, + metav1.ConditionFalse, "AwaitingNetworkContext", + fmt.Sprintf("NetworkContext %s does not exist yet", vpcKey.Name)) + } + return ctrl.Result{}, fmt.Errorf("get NetworkContext %s: %w", vpcKey, err) + } + attachment, err := r.reconcileAttachment(ctx, &networkInterface, &vpc) if err != nil { return ctrl.Result{}, err } - nad, err := r.reconcileNAD(ctx, attachment, &vpc, &networkInterface) + // Resolved once for the whole pass, after the attachment exists, because + // the node it reports is what the address is read for. The conflist the + // node reads and the address a consumer reads back have to be the same + // answer, and resolving twice could produce two. + egress, err := r.resolveInternetEgress(ctx, &networkContext, attachment) if err != nil { return ctrl.Result{}, err } - if err := r.publishAttachmentStatus(ctx, attachment, &vpc, nad); err != nil { + nad, err := r.reconcileNAD(ctx, attachment, &vpc, &networkInterface, egress) + if err != nil { + return ctrl.Result{}, err + } + if err := r.publishAttachmentStatus(ctx, attachment, &vpc, nad, egress); err != nil { return ctrl.Result{}, err } return ctrl.Result{}, r.publishToInterface(ctx, &networkInterface, attachment, &vpc) @@ -183,6 +212,7 @@ func (r *NetworkInterfaceReconciler) reconcileNAD( attachment *cloudv1alpha1.VPCAttachment, vpc *cloudv1alpha1.VPC, networkInterface *networkingv1alpha.NetworkInterface, + egress *internetEgress, ) (*nadv1.NetworkAttachmentDefinition, error) { nad := &nadv1.NetworkAttachmentDefinition{ ObjectMeta: metav1.ObjectMeta{Name: attachment.Name, Namespace: attachment.Namespace}, @@ -211,7 +241,7 @@ func (r *NetworkInterfaceReconciler) reconcileNAD( } config, err := galactic.ConflistJSON(attachment.Name, masterPlugin(attachment.Spec.Interface.Mode), vpc.Status.VPC, attachmentID, networkInterface.Spec.MTU, addresses, - declaresDevice(attachment.Spec.Interface.Mode)) + declaresDevice(attachment.Spec.Interface.Mode), egress.conflist()) if err != nil { return nil, err } @@ -281,6 +311,148 @@ func interfaceAddresses(networkInterface *networkingv1alpha.NetworkInterface) [] return addresses } +// internetEgress is what one attachment's egress intent resolved to: whether +// the node installs a route, and the address a consumer reads back. +// +// A nil internetEgress is a network that reaches nothing outside the platform. +// It is not an empty one: absence is the instruction, in the conflist and on +// the attachment alike. +type internetEgress struct { + // sourceAddress is what translation writes, read from the shard on the + // node this attachment landed on. Empty until the attachment reports its + // node and that node's shard reports an address. + sourceAddress *cloudv1alpha1.InternetEgressSourceAddress +} + +// conflist renders the block the node reads, or nothing. It carries the +// declaration alone: the node routes toward its own shard, so no shard +// identity travels here. +func (e *internetEgress) conflist() *galactic.Egress { + if e == nil { + return nil + } + return &galactic.Egress{Internet: &galactic.InternetEgress{Mode: galactic.InternetEgressEnabled}} +} + +// status renders what a consumer reads back, or nothing. An address the +// platform cannot state is reported as no egress rather than as a guess: a +// consumer allow-listing the wrong address admits the wrong traffic and has no +// way to tell. +func (e *internetEgress) status() *cloudv1alpha1.VPCAttachmentEgressStatus { + if e == nil || e.sourceAddress == nil { + return nil + } + return &cloudv1alpha1.VPCAttachmentEgressStatus{ + Internet: &cloudv1alpha1.VPCAttachmentInternetEgressStatus{ + SourceAddresses: []cloudv1alpha1.InternetEgressSourceAddress{*e.sourceAddress}, + }, + } +} + +// resolveInternetEgress reads the egress this attachment provides: the +// declaration projected onto its network context, and the address of the shard +// on the node it landed on. +// +// Translation runs on the node the instance attached to, so the shard is the +// node's and nothing here selects one. The declaration is what the node reads. +// The address is what the consumer reads, and it is stated only once the +// attachment has reported its node and that node's shard has reported an +// address. A wrong address is worse than an absent one, because a consumer +// allow-lists it at their destination. +func (r *NetworkInterfaceReconciler) resolveInternetEgress( + ctx context.Context, + networkContext *networkingv1alpha.NetworkContext, + attachment *cloudv1alpha1.VPCAttachment, +) (*internetEgress, error) { + log := logf.FromContext(ctx) + + intent := internetEgressIntent(networkContext) + if intent == nil { + // A context written before this field existed carries no intent, which + // is not the same as a network that reaches nothing. Both render no + // egress; only this one is worth saying out loud. + log.V(1).Info("network context carries no projected egress intent", + "networkContext", networkContext.Name) + return nil, nil + } + if intent.Mode != networkingv1alpha.NetworkInternetEgressEnabled { + return nil, nil + } + + resolved := &internetEgress{} + if attachment.Status.Node == "" { + return resolved, nil + } + shard, err := r.egressShardOnNode(ctx, attachment.Status.Node) + if err != nil { + return nil, err + } + if shard == nil { + log.Info("internet egress is enabled but the node serving this attachment has no shard", + "attachment", attachment.Name, "node", attachment.Status.Node) + return resolved, nil + } + resolved.sourceAddress = sourceAddress(shard) + if resolved.sourceAddress == nil { + log.Info("internet egress is enabled but the node's shard reports no source address", + "attachment", attachment.Name, "node", attachment.Status.Node, "shard", shard.Name) + } + return resolved, nil +} + +// sourceAddress is what a consumer reads back for the shard on their node. +// +// The address itself is write-once and immutable upstream, so a reported value +// that changes means the instance moved nodes or the shard was replaced, not +// that the platform renumbered a live one. It is shared by every network on the +// node and follows the node, which is what stability None states. +func sourceAddress(shard *bgpv1alpha1.EgressShard) *cloudv1alpha1.InternetEgressSourceAddress { + if shard == nil || shard.Status.ShardAddressIPv6 == "" { + return nil + } + return &cloudv1alpha1.InternetEgressSourceAddress{ + Family: cloudv1alpha1.InternetEgressAddressFamilyIPv6, + Address: shard.Status.ShardAddressIPv6, + Stability: cloudv1alpha1.InternetEgressAddressStabilityNone, + } +} + +// internetEgressIntent reads the internet egress a location was instructed to +// provide, or nil when the context carries no instruction at all. +func internetEgressIntent( + networkContext *networkingv1alpha.NetworkContext, +) *networkingv1alpha.NetworkContextInternetEgress { + if networkContext.Spec.Egress == nil { + return nil + } + return networkContext.Spec.Egress.Internet +} + +// egressShardOnNode is the shard running on a node, or nil when the node has +// none. Shards are listed rather than named because a shard's name is the +// operator's to choose, and the node reference is what ties one to a node. Two +// shards naming one node is an operator error, and the first by name is taken +// so that every attachment on that node computes the same answer. +func (r *NetworkInterfaceReconciler) egressShardOnNode( + ctx context.Context, node string, +) (*bgpv1alpha1.EgressShard, error) { + var shards bgpv1alpha1.EgressShardList + if err := r.List(ctx, &shards, client.InNamespace(galactic.SystemNamespace)); err != nil { + return nil, fmt.Errorf("list egress shards: %w", err) + } + var found *bgpv1alpha1.EgressShard + for i := range shards.Items { + shard := &shards.Items[i] + if shard.Spec.TargetRef.Name != node { + continue + } + if found == nil || shard.Name < found.Name { + found = shard + } + } + return found, nil +} + // allocateAttachmentIdentifier draws a random identifier unused within the VPC. // Random rather than lowest-free, so a freed identifier is not immediately // reissued while its BGPAdvertisement is still being garbage collected. @@ -309,16 +481,38 @@ func (r *NetworkInterfaceReconciler) allocateAttachmentIdentifier(ctx context.Co vpc, maxIdentifierAttempts) } -// publishAttachmentStatus records the allocated identifiers on the attachment. +// publishAttachmentStatus records the allocated identifiers on the attachment, +// and the egress address a consumer reads back through the interface that holds +// it. +// +// Two reconcilers write this status, over disjoint field sets: this one writes +// the identifiers, the attachment definition and now the egress address, and +// the BGPAdvertisement reconciler writes what the node programmed. Both do a +// whole-object Status().Update, which carries the resourceVersion it was read +// at, so a writer working from a copy the other has since superseded is +// rejected with a conflict and retries — it does not overwrite fields it never +// set. Adding a field set to a reconciler that already writes here keeps the +// writer count at two and that property intact. +// +// Server-side apply was considered and rejected. It would have to convert both +// writers to be coherent: a status subresource written by SSA on one side and +// replaced wholesale on the other is worse than either alone, because the +// wholesale writer drops whatever it did not read. Converting both means +// generated apply configurations this repository does not produce, or the +// deprecated unstructured apply path whose single use here is a foreign object. +// That is a change to make deliberately, for the type as a whole, and not as a +// side effect of adding three fields. func (r *NetworkInterfaceReconciler) publishAttachmentStatus( ctx context.Context, attachment *cloudv1alpha1.VPCAttachment, vpc *cloudv1alpha1.VPC, nad *nadv1.NetworkAttachmentDefinition, + egress *internetEgress, ) error { attachment.Status.VPC = vpc.Status.VPC attachment.Status.VPCAttachment = nad.Labels[LabelVPCAttachment] attachment.Status.NetworkAttachmentDefinition = nad.Name + attachment.Status.Egress = egress.status() attachment.Status.ObservedGeneration = attachment.Generation meta.SetStatusCondition(&attachment.Status.Conditions, metav1.Condition{ Type: cloudv1alpha1.ConditionTypeReady, @@ -402,6 +596,58 @@ func (r *NetworkInterfaceReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&networkingv1alpha.NetworkInterface{}). Owns(&cloudv1alpha1.VPCAttachment{}). Watches(&nadv1.NetworkAttachmentDefinition{}, handler.EnqueueRequestsFromMapFunc(nadToInterface)). + Watches(&networkingv1alpha.NetworkContext{}, + handler.EnqueueRequestsFromMapFunc(r.interfacesForNetworkContext)). + Watches(&bgpv1alpha1.EgressShard{}, + handler.EnqueueRequestsFromMapFunc(r.interfacesForEgressShard)). Named("networkinterface"). Complete(r) } + +// interfacesForNetworkContext re-renders a location's attachments when the +// egress a consumer declared reaches it, so a network that was enabled does not +// wait out a poll interval it does not have. +func (r *NetworkInterfaceReconciler) interfacesForNetworkContext( + ctx context.Context, object client.Object, +) []reconcile.Request { + var interfaces networkingv1alpha.NetworkInterfaceList + if err := r.List(ctx, &interfaces, client.InNamespace(object.GetNamespace())); err != nil { + return nil + } + + requests := make([]reconcile.Request, 0, len(interfaces.Items)) + for i := range interfaces.Items { + reference := interfaces.Items[i].Status.NetworkContextRef + if reference == nil || reference.Name != object.GetName() { + continue + } + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&interfaces.Items[i]), + }) + } + return requests +} + +// interfacesForEgressShard re-renders every attachment in the cell when a shard +// arrives, reports its address, or leaves. +// +// It enqueues everything rather than working out which attachments sit on the +// shard's node: the sweep is affordable because a shard is an operator-written +// object in one namespace with one per node, and re-rendering an unaffected +// attachment writes nothing. +func (r *NetworkInterfaceReconciler) interfacesForEgressShard( + ctx context.Context, _ client.Object, +) []reconcile.Request { + var interfaces networkingv1alpha.NetworkInterfaceList + if err := r.List(ctx, &interfaces); err != nil { + return nil + } + + requests := make([]reconcile.Request, 0, len(interfaces.Items)) + for i := range interfaces.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKeyFromObject(&interfaces.Items[i]), + }) + } + return requests +} diff --git a/internal/controller/networkinterface_controller_test.go b/internal/controller/networkinterface_controller_test.go index 6ce4090..6bad503 100644 --- a/internal/controller/networkinterface_controller_test.go +++ b/internal/controller/networkinterface_controller_test.go @@ -20,11 +20,16 @@ package controller import ( "testing" + nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1" "go.datum.net/cloud/internal/galactic" networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" ) func TestMasterPlugin(t *testing.T) { @@ -139,3 +144,319 @@ func TestDeclaresDevice(t *testing.T) { t.Error("a declared hypervisor attachment must ask for a description") } } + +const ( + egressTestNamespace = "project-egress" + egressShardNamespace = "galactic-system" + egressTestNode = "worker-3" +) + +// newEgressReconciler builds a reconciler over a cell holding the shards +// given, so a test states only what it is about. +func newEgressReconciler(t *testing.T, objects ...client.Object) *NetworkInterfaceReconciler { + t.Helper() + + scheme := runtime.NewScheme() + if err := networkingv1alpha.AddToScheme(scheme); err != nil { + t.Fatalf("build the networking scheme: %v", err) + } + if err := cloudv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("build the cloud scheme: %v", err) + } + if err := bgpv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("build the fabric scheme: %v", err) + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + return &NetworkInterfaceReconciler{Client: fakeClient, Scheme: scheme, APIReader: fakeClient} +} + +// newEgressShard is the shard an operator wrote for one node, whose process +// reported its address. +func newEgressShard(name, node, sid, address string) *bgpv1alpha1.EgressShard { + shard := &bgpv1alpha1.EgressShard{} + shard.Namespace = egressShardNamespace + shard.Name = name + shard.Spec.TargetRef.Name = node + shard.Spec.ShardSID = sid + shard.Spec.ShardAddressIPv6 = address + shard.Status.ShardSID = sid + shard.Status.ShardAddressIPv6 = address + return shard +} + +// newEgressContext is a location carrying the declaration projected onto it. +func newEgressContext(mode networkingv1alpha.NetworkInternetEgressMode) *networkingv1alpha.NetworkContext { + networkContext := &networkingv1alpha.NetworkContext{} + networkContext.Namespace = egressTestNamespace + networkContext.Name = "default-us-central-1" + networkContext.Spec.Egress = &networkingv1alpha.NetworkContextEgress{ + Internet: &networkingv1alpha.NetworkContextInternetEgress{ + Mode: mode, + Reach: []networkingv1alpha.IPFamily{networkingv1alpha.IPv6Protocol}, + }, + } + return networkContext +} + +// newEgressAttachment is an attachment that has, or has not yet, reported the +// node it landed on. +func newEgressAttachment(node string) *cloudv1alpha1.VPCAttachment { + attachment := &cloudv1alpha1.VPCAttachment{} + attachment.Namespace = egressTestNamespace + attachment.Name = "web-eth0" + attachment.Status.Node = node + return attachment +} + +// The node reads the declaration and nothing else. Whether it has a shard, and +// whether that shard has an address, is the node's to know and the consumer's +// to read back; neither changes what the node is told. +func TestResolveInternetEgressInstallsARouteWhenEnabled(t *testing.T) { + r := newEgressReconciler(t) + + egress, err := r.resolveInternetEgress(t.Context(), + newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), newEgressAttachment("")) + if err != nil { + t.Fatalf("resolveInternetEgress: %v", err) + } + block := egress.conflist() + if block == nil || block.Internet == nil || block.Internet.Mode != galactic.InternetEgressEnabled { + t.Fatalf("an enabled network rendered %v, want the Enabled declaration", block) + } + if egress.status() != nil { + t.Error("an attachment on no known node published an address") + } +} + +// Every reason a network reaches nothing renders the same absent block. A node +// that receives no block installs no route. +func TestResolveInternetEgressYieldsNothingWhenNotEnabled(t *testing.T) { + unprojected := newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled) + unprojected.Spec.Egress = nil + + noInternet := newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled) + noInternet.Spec.Egress.Internet = nil + + tests := []struct { + name string + networkContext *networkingv1alpha.NetworkContext + }{ + {"disabled", newEgressContext(networkingv1alpha.NetworkInternetEgressDisabled)}, + {"mode never projected", newEgressContext("")}, + {"egress never projected", unprojected}, + {"no internet egress projected", noInternet}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r := newEgressReconciler(t, + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:1::1")) + + egress, err := r.resolveInternetEgress(t.Context(), test.networkContext, + newEgressAttachment(egressTestNode)) + if err != nil { + t.Fatalf("resolveInternetEgress: %v", err) + } + if egress != nil { + t.Errorf("resolved %v, want nothing", egress) + } + if egress.conflist() != nil { + t.Error("no egress resolved but a block was rendered") + } + if egress.status() != nil { + t.Error("no egress resolved but an address was published") + } + }) + } +} + +// The address a consumer reads back is the one on the node their instance +// landed on, shared by every network there, which is what stability None +// states. +func TestResolveInternetEgressReportsTheNodesShardAddress(t *testing.T) { + r := newEgressReconciler(t, + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) + + egress, err := r.resolveInternetEgress(t.Context(), + newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), newEgressAttachment(egressTestNode)) + if err != nil { + t.Fatalf("resolveInternetEgress: %v", err) + } + status := egress.status() + if status == nil || status.Internet == nil { + t.Fatal("a shard reporting an address published nothing") + } + addresses := status.Internet.SourceAddresses + if len(addresses) != 1 { + t.Fatalf("source addresses: got %d, want 1", len(addresses)) + } + if addresses[0].Family != cloudv1alpha1.InternetEgressAddressFamilyIPv6 { + t.Errorf("family: got %q, want IPv6", addresses[0].Family) + } + if addresses[0].Address != "2001:db8:f00d::100" { + t.Errorf("address: got %q, want %q", addresses[0].Address, "2001:db8:f00d::100") + } + if addresses[0].Stability != cloudv1alpha1.InternetEgressAddressStabilityNone { + t.Errorf("stability: got %q, want None", addresses[0].Stability) + } +} + +// Egress that is declared and an address that cannot yet be stated are +// different facts. The node is told to route; the consumer is told nothing +// rather than a value they might allow-list. +func TestResolveInternetEgressWithholdsAnAddressItCannotState(t *testing.T) { + tests := []struct { + name string + node string + objects []client.Object + }{ + {"attachment has reported no node", "", []client.Object{ + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")}}, + {"node has no shard", egressTestNode, nil}, + {"shard has reported no address", egressTestNode, []client.Object{ + newEgressShard("worker-3-egress", egressTestNode, "2001:db8:ff01::", "")}}, + {"shard is on another node", egressTestNode, []client.Object{ + newEgressShard("worker-4-egress", "worker-4", "2001:db8:ff02::", "2001:db8:f00d::200")}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + r := newEgressReconciler(t, test.objects...) + + egress, err := r.resolveInternetEgress(t.Context(), + newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), newEgressAttachment(test.node)) + if err != nil { + t.Fatalf("resolveInternetEgress: %v", err) + } + if egress == nil || egress.conflist() == nil { + t.Fatal("an enabled network rendered no declaration for the node") + } + if status := egress.status(); status != nil { + t.Errorf("published %v, want no address", status.Internet.SourceAddresses) + } + }) + } +} + +// Two shards naming one node is an operator error, and every attachment on that +// node has to compute the same answer from it. +func TestResolveInternetEgressIsDeterministicForOneNode(t *testing.T) { + r := newEgressReconciler(t, + newEgressShard("worker-3-egress-b", egressTestNode, "2001:db8:ff02::", "2001:db8:f00d::200"), + newEgressShard("worker-3-egress-a", egressTestNode, "2001:db8:ff01::", "2001:db8:f00d::100")) + + for range 2 { + egress, err := r.resolveInternetEgress(t.Context(), + newEgressContext(networkingv1alpha.NetworkInternetEgressEnabled), newEgressAttachment(egressTestNode)) + if err != nil { + t.Fatalf("resolveInternetEgress: %v", err) + } + addresses := egress.status().Internet.SourceAddresses + if len(addresses) != 1 || addresses[0].Address != "2001:db8:f00d::100" { + t.Errorf("got %v, want the first shard by name", addresses) + } + } +} + +// Egress withdrawn has to be egress unreported. An address left behind on the +// attachment is one a consumer keeps allow-listing after the path is gone. +func TestPublishAttachmentStatusWithdrawsAnUnboundAddress(t *testing.T) { + attachment := &cloudv1alpha1.VPCAttachment{} + attachment.Namespace = egressTestNamespace + attachment.Name = "web-eth0" + attachment.Spec.VPC = cloudv1alpha1.VPCRef{Name: "default-us-central-1"} + attachment.Spec.Interface.Name = "eth0" + attachment.Status.Egress = &cloudv1alpha1.VPCAttachmentEgressStatus{ + Internet: &cloudv1alpha1.VPCAttachmentInternetEgressStatus{ + SourceAddresses: []cloudv1alpha1.InternetEgressSourceAddress{{ + Family: cloudv1alpha1.InternetEgressAddressFamilyIPv6, + Address: "2001:db8:f00d::100", + Stability: cloudv1alpha1.InternetEgressAddressStabilityNone, + }}, + }, + } + + scheme := runtime.NewScheme() + if err := cloudv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("build the cloud scheme: %v", err) + } + fakeClient := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&cloudv1alpha1.VPCAttachment{}). + WithObjects(attachment).Build() + r := &NetworkInterfaceReconciler{Client: fakeClient, Scheme: scheme, APIReader: fakeClient} + + vpc := &cloudv1alpha1.VPC{} + vpc.Status.VPC = "0000000jU" + nad := &nadv1.NetworkAttachmentDefinition{} + nad.Name = attachment.Name + nad.Labels = map[string]string{LabelVPCAttachment: "01a"} + + if err := r.publishAttachmentStatus(t.Context(), attachment, vpc, nad, nil); err != nil { + t.Fatalf("publishAttachmentStatus: %v", err) + } + + stored := &cloudv1alpha1.VPCAttachment{} + if err := fakeClient.Get(t.Context(), client.ObjectKeyFromObject(attachment), stored); err != nil { + t.Fatalf("read the attachment back: %v", err) + } + if stored.Status.Egress != nil { + t.Errorf("egress still reported after it was withdrawn: %v", stored.Status.Egress) + } + // The other field set this reconciler owns still has to land. + if stored.Status.VPC != "0000000jU" || stored.Status.VPCAttachment != "01a" { + t.Errorf("identifiers: got %q/%q", stored.Status.VPC, stored.Status.VPCAttachment) + } +} + +// The BGPAdvertisement reconciler writes a disjoint field set on this same +// status, and both writers do a whole-object update. Neither may drop the +// other's fields, which is the property that makes two writers safe without +// server-side apply. +func TestPublishAttachmentStatusKeepsTheOtherWritersFields(t *testing.T) { + attachment := &cloudv1alpha1.VPCAttachment{} + attachment.Namespace = egressTestNamespace + attachment.Name = "web-eth0" + attachment.Spec.VPC = cloudv1alpha1.VPCRef{Name: "default-us-central-1"} + attachment.Spec.Interface.Name = "eth0" + attachment.Status.Node = "node-1" + attachment.Status.HostInterface = "G0000000jU01aH" + attachment.Status.PodSubnet = "fd00:10:ff01:0:1::/80" + + scheme := runtime.NewScheme() + if err := cloudv1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("build the cloud scheme: %v", err) + } + fakeClient := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&cloudv1alpha1.VPCAttachment{}). + WithObjects(attachment).Build() + r := &NetworkInterfaceReconciler{Client: fakeClient, Scheme: scheme, APIReader: fakeClient} + + vpc := &cloudv1alpha1.VPC{} + vpc.Status.VPC = "0000000jU" + nad := &nadv1.NetworkAttachmentDefinition{} + nad.Name = attachment.Name + nad.Labels = map[string]string{LabelVPCAttachment: "01a"} + egress := &internetEgress{ + sourceAddress: &cloudv1alpha1.InternetEgressSourceAddress{ + Family: cloudv1alpha1.InternetEgressAddressFamilyIPv6, + Address: "2001:db8:f00d::100", + Stability: cloudv1alpha1.InternetEgressAddressStabilityNone, + }, + } + + if err := r.publishAttachmentStatus(t.Context(), attachment, vpc, nad, egress); err != nil { + t.Fatalf("publishAttachmentStatus: %v", err) + } + + stored := &cloudv1alpha1.VPCAttachment{} + if err := fakeClient.Get(t.Context(), client.ObjectKeyFromObject(attachment), stored); err != nil { + t.Fatalf("read the attachment back: %v", err) + } + if stored.Status.Node != "node-1" || stored.Status.HostInterface != "G0000000jU01aH" || + stored.Status.PodSubnet != "fd00:10:ff01:0:1::/80" { + t.Errorf("the data plane's field set was dropped: %+v", stored.Status) + } + if stored.Status.Egress == nil || + stored.Status.Egress.Internet.SourceAddresses[0].Address != "2001:db8:f00d::100" { + t.Errorf("egress address: got %v", stored.Status.Egress) + } +} diff --git a/internal/galactic/galactic.go b/internal/galactic/galactic.go index 4e82cf9..fdb69cb 100644 --- a/internal/galactic/galactic.go +++ b/internal/galactic/galactic.go @@ -98,6 +98,28 @@ type BGPPlugin struct { VPC string `json:"vpc"` VPCAttachment string `json:"vpcattachment"` Namespace string `json:"namespace"` + // Egress is what this attachment reaches outside the platform. It is omitted + // whenever egress is not enabled, which is what keeps every conflist rendered + // until now byte-identical, the same property DAN was given. A node that + // receives no block installs no route, so a network reaches nothing outside + // the platform until a consumer asks for it. + Egress *Egress `json:"egress,omitempty"` +} + +// Egress is the outbound declaration this attachment carries to its node. It +// names no shard: translation runs on the node the instance attaches to, so +// the node routes toward its own shard and reads only whether to. +type Egress struct { + Internet *InternetEgress `json:"internet,omitempty"` +} + +// InternetEgressEnabled is the one mode a node acts on. Anything else, and an +// absent block, installs no route. +const InternetEgressEnabled = "Enabled" + +// InternetEgress is whether this attachment reaches the internet. +type InternetEgress struct { + Mode string `json:"mode"` } // IPAM is the delegated IPAM block. Presence alone decides whether IPAM runs. @@ -118,8 +140,10 @@ type Address struct { // Conflist renders the conflist for one attachment. Addresses are the addresses // NSO already allocated; an empty list means the guest addresses itself and no // IPAM block is emitted. Set dan for a guest whose hypervisor is handed the -// device rather than discovering it. -func Conflist(name, plugin, vpc, vpcAttachment string, mtu int32, addresses []Address, dan bool) NetConfList { +// device rather than discovering it. A nil egress renders no egress block, so +// the attachment reaches nothing outside the platform. +func Conflist(name, plugin, vpc, vpcAttachment string, mtu int32, addresses []Address, dan bool, + egress *Egress) NetConfList { master := MasterPlugin{ Type: plugin, VPC: vpc, @@ -131,19 +155,32 @@ func Conflist(name, plugin, vpc, vpcAttachment string, mtu int32, addresses []Ad if len(addresses) > 0 { master.IPAM = &IPAM{Type: PluginIPAM, Addresses: addresses} } + // A block that does not declare Enabled is one a node can do nothing with, + // and it is not the same instruction as no block: absence is what tells the + // node to install no route. + if egress != nil && (egress.Internet == nil || egress.Internet.Mode != InternetEgressEnabled) { + egress = nil + } return NetConfList{ CNIVersion: CNIVersion, Name: name, Plugins: []any{ master, - BGPPlugin{Type: PluginBGP, VPC: vpc, VPCAttachment: vpcAttachment, Namespace: SystemNamespace}, + BGPPlugin{ + Type: PluginBGP, + VPC: vpc, + VPCAttachment: vpcAttachment, + Namespace: SystemNamespace, + Egress: egress, + }, }, } } // ConflistJSON renders the conflist as the string a NAD's spec.config holds. -func ConflistJSON(name, plugin, vpc, vpcAttachment string, mtu int32, addresses []Address, dan bool) (string, error) { - raw, err := json.Marshal(Conflist(name, plugin, vpc, vpcAttachment, mtu, addresses, dan)) +func ConflistJSON(name, plugin, vpc, vpcAttachment string, mtu int32, addresses []Address, dan bool, + egress *Egress) (string, error) { + raw, err := json.Marshal(Conflist(name, plugin, vpc, vpcAttachment, mtu, addresses, dan, egress)) if err != nil { return "", fmt.Errorf("marshal CNI conflist: %w", err) } diff --git a/internal/galactic/galactic_test.go b/internal/galactic/galactic_test.go index d1ac56a..a34bba7 100644 --- a/internal/galactic/galactic_test.go +++ b/internal/galactic/galactic_test.go @@ -27,7 +27,7 @@ func TestConflistChainIsComplete(t *testing.T) { []Address{ {Address: "fd00:10:ff01:0:1::1/96", Gateway: "fd00:10:ff01::1"}, {Address: "172.20.1.7/32", Gateway: "172.20.1.1"}, - }, false) + }, false, nil) if conflist.CNIVersion != "1.0.0" { t.Errorf("cniVersion: got %q, want %q", conflist.CNIVersion, "1.0.0") @@ -64,7 +64,7 @@ func TestConflistChainIsComplete(t *testing.T) { } func TestConflistOmitsIPAMForSelfAddressingGuest(t *testing.T) { - raw, err := ConflistJSON("web-eth0", PluginTap, "0000000jU", "01a", 0, nil, false) + raw, err := ConflistJSON("web-eth0", PluginTap, "0000000jU", "01a", 0, nil, false, nil) if err != nil { t.Fatalf("ConflistJSON: %v", err) } @@ -131,11 +131,11 @@ func TestSplitAdvertisementName(t *testing.T) { // to the hypervisor. Every attachment rendered until now leaves it out, so its // absence has to stay the default. func TestConflistCarriesTheDeclaredDeviceRequest(t *testing.T) { - declared, err := ConflistJSON("vm-eth0", PluginTap, "0000000jU", "01a", 1400, nil, true) + declared, err := ConflistJSON("vm-eth0", PluginTap, "0000000jU", "01a", 1400, nil, true, nil) if err != nil { t.Fatalf("ConflistJSON: %v", err) } - discovered, err := ConflistJSON("vm-eth0", PluginTap, "0000000jU", "01a", 1400, nil, false) + discovered, err := ConflistJSON("vm-eth0", PluginTap, "0000000jU", "01a", 1400, nil, false, nil) if err != nil { t.Fatalf("ConflistJSON: %v", err) } @@ -158,3 +158,100 @@ func masterStanza(t *testing.T, raw string) map[string]any { } return decoded.Plugins[0] } + +// The egress block is the only new field in the conflist, and a node that +// receives none installs no egress route. Every conflist rendered before it +// existed has to stay byte-identical, so these are the exact strings the +// renderer produced at the commit that introduced the field. +func TestConflistWithoutEgressIsByteIdentical(t *testing.T) { + tests := []struct { + name string + got func() (string, error) + want string + }{ + { + name: "addressed container", + got: func() (string, error) { + return ConflistJSON("web-eth0", PluginVeth, "0000000jU", "01a", 1400, + []Address{{Address: "fd00:10:ff01:0:1::1/96", Gateway: "fd00:10:ff01::1"}}, false, nil) + }, + want: `{"cniVersion":"1.0.0","name":"web-eth0","plugins":[{"type":"galactic-veth","vpc":"0000000jU","vpcattachment":"01a","namespace":"galactic-system","mtu":1400,"ipam":{"type":"galactic-ipam","addresses":[{"address":"fd00:10:ff01:0:1::1/96","gateway":"fd00:10:ff01::1"}]}},{"type":"galactic-bgp","vpc":"0000000jU","vpcattachment":"01a","namespace":"galactic-system"}]}`, + }, + { + name: "declared guest", + got: func() (string, error) { + return ConflistJSON("web-eth0", PluginVeth, "0000000jU", "01a", 1400, + []Address{{Address: "fd00:10:ff01:0:1::1/96", Gateway: "fd00:10:ff01::1"}}, true, nil) + }, + want: `{"cniVersion":"1.0.0","name":"web-eth0","plugins":[{"type":"galactic-veth","vpc":"0000000jU","vpcattachment":"01a","namespace":"galactic-system","mtu":1400,"dan":true,"ipam":{"type":"galactic-ipam","addresses":[{"address":"fd00:10:ff01:0:1::1/96","gateway":"fd00:10:ff01::1"}]}},{"type":"galactic-bgp","vpc":"0000000jU","vpcattachment":"01a","namespace":"galactic-system"}]}`, + }, + { + name: "self addressing guest", + got: func() (string, error) { + return ConflistJSON("vm-eth0", PluginTap, "0000000jU", "01a", 0, nil, false, nil) + }, + want: `{"cniVersion":"1.0.0","name":"vm-eth0","plugins":[{"type":"galactic-tap","vpc":"0000000jU","vpcattachment":"01a","namespace":"galactic-system"},{"type":"galactic-bgp","vpc":"0000000jU","vpcattachment":"01a","namespace":"galactic-system"}]}`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.got() + if err != nil { + t.Fatalf("ConflistJSON: %v", err) + } + if got != test.want { + t.Errorf("conflist changed:\n got %s\nwant %s", got, test.want) + } + }) + } +} + +// The egress block is the whole contract with the node: it hangs off the +// galactic-bgp stanza, under one key, and carries the declaration alone. The +// node routes toward its own shard, so no shard identity travels here. +func TestConflistCarriesTheEgressDeclaration(t *testing.T) { + const want = `{"cniVersion":"1.0.0","name":"vm-eth0","plugins":[` + + `{"type":"galactic-tap","vpc":"0000000jU","vpcattachment":"01a","namespace":"galactic-system"},` + + `{"type":"galactic-bgp","vpc":"0000000jU","vpcattachment":"01a","namespace":"galactic-system",` + + `"egress":{"internet":{"mode":"Enabled"}}}]}` + + got, err := ConflistJSON("vm-eth0", PluginTap, "0000000jU", "01a", 0, nil, false, + &Egress{Internet: &InternetEgress{Mode: InternetEgressEnabled}}) + if err != nil { + t.Fatalf("ConflistJSON: %v", err) + } + if got != want { + t.Errorf("conflist:\n got %s\nwant %s", got, want) + } +} + +// A block that declares anything but Enabled is one a node can do nothing +// with, so it renders as no block at all rather than as an empty one. +func TestConflistOmitsAnEgressBlockThatDeclaresNothing(t *testing.T) { + for name, egress := range map[string]*Egress{ + "empty": {}, + "no mode": {Internet: &InternetEgress{}}, + "disabled": {Internet: &InternetEgress{Mode: "Disabled"}}, + } { + t.Run(name, func(t *testing.T) { + raw, err := ConflistJSON("vm-eth0", PluginTap, "0000000jU", "01a", 0, nil, false, egress) + if err != nil { + t.Fatalf("ConflistJSON: %v", err) + } + if _, present := bgpStanza(t, raw)["egress"]; present { + t.Errorf("egress block present with nothing enabled: %s", raw) + } + }) + } +} + +func bgpStanza(t *testing.T, raw string) map[string]any { + t.Helper() + var decoded struct { + Plugins []map[string]any `json:"plugins"` + } + if err := json.Unmarshal([]byte(raw), &decoded); err != nil { + t.Fatalf("unmarshal conflist: %v", err) + } + return decoded.Plugins[1] +}