feat: Turn a network's egress on or off under a running workload

config/galactic-cni/rbac.yaml

@@ -7,6 +7,13 @@ rules:
     resources:
       - bgprouters
     verbs: ["get", "list"]
+  # egressshardclaims (read only): the installer's claim sweep lists the
+  # claims the cell recorded against this node to keep each tenant VRF's
+  # egress route in step with what its network declares.
+  - apiGroups: ["network.datumapis.com"]
+    resources:
+      - egressshardclaims
+    verbs: ["get", "list", "watch"]
   - apiGroups: ["network.datumapis.com"]
     resources:
       - bgpadvertisements

docs/cni/conflist-reference.md

@@ -266,6 +266,15 @@ names no shard or none of its shards resolves. Anything else, including an
 absent `egress` key, installs no route and withdraws one that is there, so a
 network that declared no egress gets none even on a node that has shards.
 
+The declaration is read once, at ADD. After that the node follows the
+`EgressShardClaim` the cell records against it for each attachment whose
+network declares egress: the installer lists the claims naming its node every
+30 seconds and installs the route for a claimed VRF that lacks one, or
+withdraws it from a VRF whose claim is gone. That is how egress is turned on
+or off for a running workload without re-attaching it. A route ADD wrote is
+left alone for two minutes before the sweep will withdraw it, which covers the
+gap between ADD returning and the cell recording the claim.
+
 ### EndpointSlice publish (HTTP ingress backend discovery)
 
 Alongside the `BGPVRFInstance`/`BGPAdvertisement` CRDs, `galactic-bgp`

docs/nat/configuration.md

@@ -218,6 +218,15 @@ conflist stanza, rendered from what the network declared; a network that
 declares none gets no route however the node is configured. See the
 [conflist reference](../cni/conflist-reference.md#egress-declaration).
 
+After ADD, the declaration is carried by the `EgressShardClaim` the cell
+records against the node for each attachment whose network declares egress.
+The `galactic-cni` installer lists the claims naming its node on the same
+30-second sweep that re-resolves egress routes, installing the route for a
+claimed VRF and withdrawing it from an unclaimed one, so enabling or disabling
+egress on a network takes effect on running workloads within one sweep. The
+installer needs `get`/`list`/`watch` on `egressshardclaims` for this, granted
+in `config/galactic-cni/rbac.yaml`.
+
 A tenant's compute node needs to know the fabric-wide list of live shard
 SIDs to install its own tenant VRFs' egress routes, and the NAT64 prefix to
 install a route toward IPv4 reachability. Both are separate from anything

go.mod

@@ -111,3 +111,5 @@ require (
 )
 
 tool github.com/cilium/ebpf/cmd/bpf2go
+
+replace go.datum.net/network => github.com/datum-cloud/network v0.1.1-0.20260926162437-e92beadac630

go.sum

@@ -16,6 +16,8 @@ github.com/coreos/go-iptables v0.8.0 h1:MPc2P89IhuVpLI7ETL/2tx3XZ61VeICZjYqDEgNs
 github.com/coreos/go-iptables v0.8.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/datum-cloud/network v0.1.1-0.20260926162437-e92beadac630 h1:qy9/S+Y4C67n7zvwgN1rjfTVxJVRziGxCgMWIkBwKL8=
+github.com/datum-cloud/network v0.1.1-0.20260926162437-e92beadac630/go.mod h1:dqzM8WZczbiZ9bCvsxjkoI10GJqQ24NVWnc9boXgOkE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
@@ -108,8 +110,6 @@ github.com/mdlayher/genetlink v1.4.0 h1:f/Xs7Y2T+GyX9b3dbiUhnLE9InGs5F9RxJ2JwBMl
 github.com/mdlayher/genetlink v1.4.0/go.mod h1:d1hrKr8fwZU2JkcAtQUAzeTrI7nbgQSl+5k1cC0biSA=
 github.com/mdlayher/ndp v1.1.0 h1:QylGKGVtH60sKZUE88+IW5ila1Z/M9/OXhWdsVKuscs=
 github.com/mdlayher/ndp v1.1.0/go.mod h1:FmgESgemgjl38vuOIyAHWUUL6vQKA/pQNkvXdWsdQFM=
-github.com/mdlayher/netlink v1.9.0 h1:G8+GLq2x3v4D4MVIqDdNUhTUC7TKiCy/6MDkmItfKco=
-github.com/mdlayher/netlink v1.9.0/go.mod h1:YBnl5BXsCoRuwBjKKlZ+aYmEoq0r12FDA/3JC+94KDg=
 github.com/mdlayher/netlink v1.11.2 h1:HKh2jqe+omdSWcQ88nrT7INE61B0NXfiSPFdgL4YbNI=
 github.com/mdlayher/netlink v1.11.2/go.mod h1:uT2Yc/QLaZubzDpZIBi9d4GoeLwtp3x1AMeqSRrK2sA=
 github.com/mdlayher/socket v0.6.0 h1:ScZPaAGyO1icQnbFrhPM8mnXyMu9qukC1K4ZoM2IQKU=
@@ -186,18 +186,12 @@ github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zd
 github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
-go.datum.net/network v0.1.0 h1:AmYSwxUWOk26UnK6S6NA7OuucGJniKo/CWqjs+VcSCs=
-go.datum.net/network v0.1.0/go.mod h1:dqzM8WZczbiZ9bCvsxjkoI10GJqQ24NVWnc9boXgOkE=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
 go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
 go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
 go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
-go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
-go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
-go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
-go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
 go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
 go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@@ -249,12 +243,8 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw
 gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
 gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
 gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 h1:qEHAMpSaUhtD0p3NbEEI83HwNGFxEwaSJ1G9PLnCBZE=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
-google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
-google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
 google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0=
 google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0=
 google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=

internal/cnibgp/bgp.go

@@ -46,6 +46,7 @@ import (
 	"go.datum.net/galactic/internal/cniipam"
 	"go.datum.net/galactic/internal/config"
 	"go.datum.net/galactic/internal/crdnames"
+	"go.datum.net/galactic/internal/egressroutes"
 	"go.datum.net/galactic/internal/gc"
 	"go.datum.net/galactic/internal/plumbing/ebpf/attach"
 	"go.datum.net/galactic/internal/plumbing/ebpf/egressroutemap"
@@ -741,38 +742,24 @@ func registerLocalEgressRoutes(pinDir string, vrfTableID uint32, prefixes []stri
 	return nil
 }
 
-// installEgressRoutes installs or refreshes vrfTableID's egress routes toward
-// the configured shards: the ::/0 default that reaches the IPv6 internet, and,
-// where this fabric has NAT64, a more-specific route for the NAT64 prefix.
+// installEgressRoutes installs, refreshes or withdraws vrfTableID's egress
+// routes from this network's own declaration, taken from its conflist stanza.
 // Idempotent, so it is safe on every attachment ADD sharing this VRF.
 //
-// argument is this attachment's VRFID, written into every shard SID these
-// routes encapsulate toward. It is what makes a shard able to tell one tenant
-// on this node from another: the shard reads it back out of the outer
-// destination and composes it with the encapsulation source into its session
-// table key. Without it every VRF on this node encapsulates toward a byte-
-// identical destination, and two tenants whose inner tuples also match -- an
-// ordinary occurrence with overlapping RFC 4193 ULAs -- share one connection
-// row and one masquerade port, so the second tenant's replies are delivered to
-// the first. See struct conn_key in internal/plumbing/ebpf/natprog/nat.c.
-//
-// Both routes point at the same shard SID, argument included. A shard decides
-// which translation a packet gets from its inner destination, so the second
-// route exists to make the NAT64 prefix reachable at all rather than to steer
-// it somewhere else -- which matters because the two are independent: a fabric
-// may offer NAT64 without NAT66, and then no default route exists for this
-// traffic to fall into.
-//
-// egress is this network's own declaration, from its own conflist stanza. A
-// network that declares no egress gets no route, and loses one it has, which is
-// what makes a declaration of no egress mean anything: the node-wide list on
-// its own handed a default route out to every network on the node.
+// egress is this network's own declaration. A network that declares no
+// egress gets no route, and loses one it has, which is what makes a
+// declaration of no egress mean anything: the node-wide list on its own
+// handed a default route out to every network on the node.
 //
 // A network that declares egress on a node naming no shard fails this
 // attachment's ADD. A node without a shard is an operator error, and failing
 // the first instance surfaces it where an attach that succeeded without egress
 // would hide it. A shard SID that is invalid, or that has no reachable route
 // yet, fails the ADD for the same reason.
+//
+// This is the first of the two writers of these routes. The installer's claim
+// sweep is the second, and keeps a running VRF in step with the claims the
+// cell records against this node after ADD has returned.
 func installEgressRoutes(vrfTableID uint32, argument uint16, egress *Egress) error {
 	if !egress.Enabled() {
 		return withdrawEgressRoutes(vrfTableID)
@@ -791,68 +778,26 @@ func installEgressRoutes(vrfTableID uint32, argument uint16, egress *Egress) err
 		return fmt.Errorf("this network declares internet egress and this node names no egress shard (%s is empty)",
 			config.EnvCNIEgressShardSIDs)
 	}
-	tenantSIDs, err := shardSIDsForTenant(shardSIDs, argument)
+	nat64Prefix, err := nat64EgressPrefix()
 	if err != nil {
-		return fmt.Errorf("apply tenant argument to %s: %w", config.EnvCNIEgressShardSIDs, err)
-	}
-	if err := srv6.EgressDefaultRouteAdd(vrfTableID, tenantSIDs); err != nil {
 		return err
 	}
-	return installNAT64EgressRoute(vrfTableID, tenantSIDs)
-}
-
-// shardSIDsForTenant returns sids with each SID's 12-bit Argument replaced by
-// argument, leaving Block, Node-ID and Function as the operator configured
-// them. Whatever Argument an operator baked into a configured SID is therefore
-// overwritten rather than honoured; it identifies no tenant and never could,
-// one configured value being shared by every VRF on every node.
-//
-// A SID that is not a well-formed uFMT 48+16 address fails here rather than
-// being passed through unchanged. uformat.Decode's padding check is what
-// catches it -- an address with anything in bits 81-128 is not a uSID, and
-// writing an Argument into it would produce a plausible-looking destination
-// that addresses nothing. An IPv4 entry is unmapped first so it fails as "not
-// an IPv6 address" rather than as stray padding, which is what it actually is.
-//
-// The shard must have a route covering its whole Block and Node-ID for these
-// destinations to be reachable, not just a host route for the one SID the
-// operator configured. EgressShardReconciler advertises that /64; see
-// shardAdvertisementPrefixes.
-func shardSIDsForTenant(sids []net.IP, argument uint16) ([]net.IP, error) {
-	out := make([]net.IP, 0, len(sids))
-	for _, sid := range sids {
-		addr, ok := netip.AddrFromSlice(sid.To16())
-		if !ok {
-			return nil, fmt.Errorf("egress shard SID %s is not a 16-byte address", sid)
-		}
-		fields, err := uformat.Decode(addr.Unmap())
-		if err != nil {
-			return nil, fmt.Errorf("decode egress shard SID %s: %w", sid, err)
-		}
-		fields.Argument = argument
-		tenant, err := uformat.Encode(fields)
-		if err != nil {
-			return nil, fmt.Errorf("encode egress shard SID %s with argument %#x: %w", sid, argument, err)
-		}
-		out = append(out, net.IP(tenant.AsSlice()))
-	}
-	return out, nil
+	return egressroutes.Install(vrfTableID, argument, shardSIDs, nat64Prefix)
 }
 
-// installNAT64EgressRoute installs vrfTableID's route for the fabric's NAT64
-// prefix. An unset prefix means this fabric has no NAT64 and is not an error; a
-// set but unparseable one is a misconfiguration and fails the ADD, since
-// silently skipping it would leave the VRF with no IPv4 reachability and
-// nothing to say why.
-func installNAT64EgressRoute(vrfTableID uint32, shardSIDs []net.IP) error {
-	if cniConfig.NAT64Prefix == "" {
-		return nil
+// nat64EgressPrefix is the fabric's NAT64 prefix, or nil when this fabric has
+// none. An unset prefix is not an error; a set but unparseable one is a
+// misconfiguration and fails the ADD, since silently skipping it would leave
+// the VRF with no IPv4 reachability and nothing to say why.
+func nat64EgressPrefix() (*net.IPNet, error) {
+	if cniConfig == nil || cniConfig.NAT64Prefix == "" {
+		return nil, nil
 	}
 	_, prefix, err := net.ParseCIDR(cniConfig.NAT64Prefix)
 	if err != nil {
-		return fmt.Errorf("parse %s %q: %w", config.EnvCNINAT64Prefix, cniConfig.NAT64Prefix, err)
+		return nil, fmt.Errorf("parse %s %q: %w", config.EnvCNINAT64Prefix, cniConfig.NAT64Prefix, err)
 	}
-	return srv6.EgressPrefixRouteAdd(vrfTableID, prefix, shardSIDs)
+	return prefix, nil
 }
 
 // hostInterfaceIndex resolves this attachment's host-side veth or tap
@@ -915,18 +860,9 @@ func egressKindForInterfaceType(ifaceType string) (uint32, error) {
 // node is configured. Idempotent, and a no-op on a node whose datapath has not
 // loaded, so an attachment there never fails its ADD over a route it never had.
 func withdrawEgressRoutes(vrfTableID uint32) error {
-	if err := srv6.EgressDefaultRouteWithdraw(vrfTableID); err != nil {
-		return fmt.Errorf("withdraw default egress route: %w", err)
-	}
-	if cniConfig == nil || cniConfig.NAT64Prefix == "" {
-		return nil
-	}
-	_, prefix, err := net.ParseCIDR(cniConfig.NAT64Prefix)
+	nat64Prefix, err := nat64EgressPrefix()
 	if err != nil {
-		return fmt.Errorf("parse %s %q: %w", config.EnvCNINAT64Prefix, cniConfig.NAT64Prefix, err)
-	}
-	if err := srv6.EgressPrefixRouteWithdraw(vrfTableID, prefix); err != nil {
-		return fmt.Errorf("withdraw NAT64 egress route: %w", err)
+		return err
 	}
-	return nil
+	return egressroutes.Withdraw(vrfTableID, nat64Prefix)
 }