diff --git a/cmd/generate-config/config/config-openapi-spec.json b/cmd/generate-config/config/config-openapi-spec.json index 3d51b1cc75..d303b11e96 100755 --- a/cmd/generate-config/config/config-openapi-spec.json +++ b/cmd/generate-config/config/config-openapi-spec.json @@ -182,7 +182,8 @@ "clusterToCluster": { "type": "object", "required": [ - "dns" + "dns", + "routing" ], "properties": { "dns": { @@ -243,6 +244,26 @@ } } } + }, + "routing": { + "description": "Linux policy routing table settings for C2CC routes.", + "type": "object", + "properties": { + "routeTableID": { + "description": "Linux policy routing table ID for direct routes to remote cluster CIDRs.\nThe route protocol number is set to the same value.\nMust be between 1 and 252 (253-255 are reserved by the kernel).\nMust differ from serviceRouteTableID.", + "type": "integer", + "default": 200, + "maximum": 252, + "minimum": 1 + }, + "serviceRouteTableID": { + "description": "Linux policy routing table ID for service routes via the OVN management port.\nThe route protocol number is set to the same value.\nMust be between 1 and 252 (253-255 are reserved by the kernel).\nMust differ from routeTableID.", + "type": "integer", + "default": 201, + "maximum": 252, + "minimum": 1 + } + } } } }, diff --git a/docs/user/howto_config.md b/docs/user/howto_config.md index 5ad9667d84..1e7fd62732 100644 --- a/docs/user/howto_config.md +++ b/docs/user/howto_config.md @@ -40,6 +40,9 @@ clusterToCluster: domain: "" nextHop: "" serviceNetwork: [] + routing: + routeTableID: 0 + serviceRouteTableID: 0 debugging: logLevel: "" dns: @@ -206,6 +209,9 @@ clusterToCluster: domain: "" nextHop: "" serviceNetwork: [] + routing: + routeTableID: 200 + serviceRouteTableID: 201 debugging: logLevel: Normal dns: diff --git a/packaging/microshift/config.yaml b/packaging/microshift/config.yaml index e9fcfdae4a..bd42e27fe4 100644 --- a/packaging/microshift/config.yaml +++ b/packaging/microshift/config.yaml @@ -66,6 +66,18 @@ clusterToCluster: nextHop: "" # Service CIDRs of the remote cluster. Must not overlap with local cluster or other remotes. serviceNetwork: [] + # Linux policy routing table settings for C2CC routes. + routing: + # Linux policy routing table ID for direct routes to remote cluster CIDRs. + # The route protocol number is set to the same value. + # Must be between 1 and 252 (253-255 are reserved by the kernel). + # Must differ from serviceRouteTableID. + routeTableID: 200 + # Linux policy routing table ID for service routes via the OVN management port. + # The route protocol number is set to the same value. + # Must be between 1 and 252 (253-255 are reserved by the kernel). + # Must differ from routeTableID. + serviceRouteTableID: 201 debugging: # Valid values are: "Normal", "Debug", "Trace", "TraceAll". # Defaults to "Normal". diff --git a/pkg/config/c2cc.go b/pkg/config/c2cc.go index c742b732bb..3afebac082 100644 --- a/pkg/config/c2cc.go +++ b/pkg/config/c2cc.go @@ -26,9 +26,30 @@ type C2CCDNS struct { CacheNegativeTTL *int `json:"cacheNegativeTTL,omitempty"` } +type C2CCRouting struct { + // Linux policy routing table ID for direct routes to remote cluster CIDRs. + // The route protocol number is set to the same value. + // Must be between 1 and 252 (253-255 are reserved by the kernel). + // Must differ from serviceRouteTableID. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=252 + // +kubebuilder:default=200 + RouteTableID *int `json:"routeTableID,omitempty"` + // Linux policy routing table ID for service routes via the OVN management port. + // The route protocol number is set to the same value. + // Must be between 1 and 252 (253-255 are reserved by the kernel). + // Must differ from routeTableID. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=252 + // +kubebuilder:default=201 + ServiceRouteTableID *int `json:"serviceRouteTableID,omitempty"` +} + type C2CC struct { // DNS cache settings for CoreDNS server blocks generated for remote clusters. DNS C2CCDNS `json:"dns"` + // Linux policy routing table settings for C2CC routes. + Routing C2CCRouting `json:"routing"` // List of remote clusters to establish connectivity with. // C2CC is disabled when this list is empty. RemoteClusters []RemoteCluster `json:"remoteClusters,omitempty"` @@ -39,9 +60,11 @@ type C2CC struct { ProbeInterval string `json:"probeInterval,omitempty"` // Populated during validation with parsed network objects. - Resolved []ResolvedRemoteCluster `json:"-"` - ResolvedAllCIDRs []*net.IPNet `json:"-"` - ResolvedProbeInterval time.Duration `json:"-"` + Resolved []ResolvedRemoteCluster `json:"-"` + ResolvedAllCIDRs []*net.IPNet `json:"-"` + ResolvedProbeInterval time.Duration `json:"-"` + ResolvedRouteTableID int `json:"-"` + ResolvedServiceRouteTableID int `json:"-"` } type RemoteCluster struct { @@ -193,16 +216,58 @@ func parseAndValidateCIDR(cidr, field string, errs *[]error) *net.IPNet { return ipNet } +func (c *C2CC) resolveRoutingDefaults() { + routeTable := 200 + if c.Routing.RouteTableID != nil { + routeTable = *c.Routing.RouteTableID + } + svcRouteTable := 201 + if c.Routing.ServiceRouteTableID != nil { + svcRouteTable = *c.Routing.ServiceRouteTableID + } + c.ResolvedRouteTableID = routeTable + c.ResolvedServiceRouteTableID = svcRouteTable +} + +func (c *C2CC) validateRouting() error { + c.resolveRoutingDefaults() + + var errs []error + if c.ResolvedRouteTableID < 1 || c.ResolvedRouteTableID > 252 { + errs = append(errs, fmt.Errorf("routing.routeTableID must be between 1 and 252, got %d", c.ResolvedRouteTableID)) + } + if c.ResolvedServiceRouteTableID < 1 || c.ResolvedServiceRouteTableID > 252 { + errs = append(errs, fmt.Errorf("routing.serviceRouteTableID must be between 1 and 252, got %d", c.ResolvedServiceRouteTableID)) + } + if c.ResolvedRouteTableID == c.ResolvedServiceRouteTableID { + errs = append(errs, fmt.Errorf("routing.routeTableID (%d) and routing.serviceRouteTableID (%d) must differ", + c.ResolvedRouteTableID, c.ResolvedServiceRouteTableID)) + } + return errors.Join(errs...) +} + +func (d *C2CCDNS) validate() error { + var errs []error + if d.CacheTTL != nil && *d.CacheTTL < 0 { + errs = append(errs, fmt.Errorf("dns.cacheTTL must be >= 0, got %d", *d.CacheTTL)) + } + if d.CacheNegativeTTL != nil && *d.CacheNegativeTTL < 0 { + errs = append(errs, fmt.Errorf("dns.cacheNegativeTTL must be >= 0, got %d", *d.CacheNegativeTTL)) + } + return errors.Join(errs...) +} + func (c *C2CC) validate(cfg *Config) error { if cfg.Network.CNIPlugin != CniPluginUnset && cfg.Network.CNIPlugin != CniPluginOVNK { return fmt.Errorf("cluster to cluster requires OVN-Kubernetes CNI (network.cniPlugin must be \"\" or \"ovnk\", got %q)", cfg.Network.CNIPlugin) } - if c.DNS.CacheTTL != nil && *c.DNS.CacheTTL < 0 { - return fmt.Errorf("dns.cacheTTL must be >= 0, got %d", *c.DNS.CacheTTL) + if err := c.DNS.validate(); err != nil { + return err } - if c.DNS.CacheNegativeTTL != nil && *c.DNS.CacheNegativeTTL < 0 { - return fmt.Errorf("dns.cacheNegativeTTL must be >= 0, got %d", *c.DNS.CacheNegativeTTL) + + if err := c.Routing.validate(c); err != nil { + return err } resolved, parseErrs := c.parseRemoteClusters() diff --git a/pkg/config/c2cc_test.go b/pkg/config/c2cc_test.go index 9a87530392..f0ea1c7852 100644 --- a/pkg/config/c2cc_test.go +++ b/pkg/config/c2cc_test.go @@ -69,6 +69,16 @@ func withDNSDefaults(c2cc C2CC) C2CC { return c2cc } +func withRoutingDefaults(c2cc C2CC) C2CC { + if c2cc.Routing.RouteTableID == nil { + c2cc.Routing.RouteTableID = ptr.To(200) + } + if c2cc.Routing.ServiceRouteTableID == nil { + c2cc.Routing.ServiceRouteTableID = ptr.To(201) + } + return c2cc +} + func mkC2CCConfig(c2cc C2CC) *Config { if c2cc.ProbeInterval == "" { c2cc.ProbeInterval = "10s" @@ -82,7 +92,7 @@ func mkC2CCConfig(c2cc C2CC) *Config { Node: Node{ NodeIP: "10.100.0.1", }, - C2CC: withDNSDefaults(c2cc), + C2CC: withRoutingDefaults(withDNSDefaults(c2cc)), } } @@ -100,7 +110,7 @@ func mkDualStackC2CCConfig(c2cc C2CC) *Config { NodeIP: "10.100.0.1", NodeIPV6: "fd00::1", }, - C2CC: withDNSDefaults(c2cc), + C2CC: withRoutingDefaults(withDNSDefaults(c2cc)), } } @@ -117,7 +127,7 @@ func mkIPv6OnlyC2CCConfig(c2cc C2CC) *Config { Node: Node{ NodeIP: "fd00::1", }, - C2CC: withDNSDefaults(c2cc), + C2CC: withRoutingDefaults(withDNSDefaults(c2cc)), } } @@ -859,6 +869,82 @@ func TestRenderC2CCDNSBlocks(t *testing.T) { }) } +func TestC2CC_RoutingTableValidation(t *testing.T) { + stubHostIPs(t, nil) + + validRemote := []RemoteCluster{{ + NextHop: "10.100.0.2", + ClusterNetwork: []string{"10.45.0.0/16"}, + ServiceNetwork: []string{"10.46.0.0/16"}, + }} + + t.Run("valid custom routing table IDs", func(t *testing.T) { + cfg := mkC2CCConfig(C2CC{ + Routing: C2CCRouting{RouteTableID: ptr.To(100), ServiceRouteTableID: ptr.To(101)}, + RemoteClusters: validRemote, + }) + require.NoError(t, cfg.C2CC.validate(cfg)) + assert.Equal(t, 100, cfg.C2CC.ResolvedRouteTableID) + assert.Equal(t, 101, cfg.C2CC.ResolvedServiceRouteTableID) + }) + + t.Run("boundary values 1 and 252", func(t *testing.T) { + cfg := mkC2CCConfig(C2CC{ + Routing: C2CCRouting{RouteTableID: ptr.To(1), ServiceRouteTableID: ptr.To(252)}, + RemoteClusters: validRemote, + }) + require.NoError(t, cfg.C2CC.validate(cfg)) + assert.Equal(t, 1, cfg.C2CC.ResolvedRouteTableID) + assert.Equal(t, 252, cfg.C2CC.ResolvedServiceRouteTableID) + }) + + t.Run("routeTableID below range", func(t *testing.T) { + cfg := mkC2CCConfig(C2CC{ + Routing: C2CCRouting{RouteTableID: ptr.To(0), ServiceRouteTableID: ptr.To(201)}, + RemoteClusters: validRemote, + }) + err := cfg.C2CC.validate(cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "routing.routeTableID must be between 1 and 252") + }) + + t.Run("serviceRouteTableID above range", func(t *testing.T) { + cfg := mkC2CCConfig(C2CC{ + Routing: C2CCRouting{RouteTableID: ptr.To(200), ServiceRouteTableID: ptr.To(253)}, + RemoteClusters: validRemote, + }) + err := cfg.C2CC.validate(cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "routing.serviceRouteTableID must be between 1 and 252") + }) + + t.Run("duplicate table IDs", func(t *testing.T) { + cfg := mkC2CCConfig(C2CC{ + Routing: C2CCRouting{RouteTableID: ptr.To(150), ServiceRouteTableID: ptr.To(150)}, + RemoteClusters: validRemote, + }) + err := cfg.C2CC.validate(cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "must differ") + }) + + t.Run("defaults are used when nil", func(t *testing.T) { + c2cc := C2CC{RemoteClusters: validRemote, ProbeInterval: "10s"} + cfg := &Config{ + Network: Network{ + CNIPlugin: CniPluginOVNK, + ClusterNetwork: []string{"10.42.0.0/16"}, + ServiceNetwork: []string{"10.43.0.0/16"}, + }, + Node: Node{NodeIP: "10.100.0.1"}, + C2CC: withDNSDefaults(c2cc), + } + require.NoError(t, cfg.C2CC.validate(cfg)) + assert.Equal(t, 200, cfg.C2CC.ResolvedRouteTableID) + assert.Equal(t, 201, cfg.C2CC.ResolvedServiceRouteTableID) + }) +} + func TestC2CC_ProbeIntervalDefault(t *testing.T) { cfg := &Config{} require.NoError(t, cfg.fillDefaults()) @@ -879,6 +965,39 @@ func TestC2CC_IncorporateUserSettings(t *testing.T) { assert.Equal(t, "30s", cfg.C2CC.ProbeInterval) }) + t.Run("user overrides routing table IDs", func(t *testing.T) { + cfg := &Config{} + require.NoError(t, cfg.fillDefaults()) + + user := &Config{ + C2CC: C2CC{ + Routing: C2CCRouting{ + RouteTableID: ptr.To(100), + ServiceRouteTableID: ptr.To(101), + }, + }, + } + cfg.incorporateUserSettings(user) + assert.Equal(t, 100, *cfg.C2CC.Routing.RouteTableID) + assert.Equal(t, 101, *cfg.C2CC.Routing.ServiceRouteTableID) + }) + + t.Run("user overrides only one routing table ID preserves other default", func(t *testing.T) { + cfg := &Config{} + require.NoError(t, cfg.fillDefaults()) + + user := &Config{ + C2CC: C2CC{ + Routing: C2CCRouting{ + RouteTableID: ptr.To(100), + }, + }, + } + cfg.incorporateUserSettings(user) + assert.Equal(t, 100, *cfg.C2CC.Routing.RouteTableID) + assert.Equal(t, 201, *cfg.C2CC.Routing.ServiceRouteTableID) + }) + t.Run("user sets remoteClusters without probeInterval preserves default", func(t *testing.T) { cfg := &Config{} require.NoError(t, cfg.fillDefaults()) diff --git a/pkg/config/config.go b/pkg/config/config.go index c4902cae86..8fea3f4a3b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -201,6 +201,7 @@ func (c *Config) fillDefaults() error { c.DNS = dnsDefaults() c.C2CC = C2CC{ DNS: C2CCDNS{CacheTTL: ptr.To(10), CacheNegativeTTL: ptr.To(10)}, + Routing: C2CCRouting{RouteTableID: ptr.To(200), ServiceRouteTableID: ptr.To(201)}, ProbeInterval: "10s", } return nil @@ -491,6 +492,12 @@ func (c *Config) incorporateUserSettings(u *Config) { if u.C2CC.ProbeInterval != "" { c.C2CC.ProbeInterval = u.C2CC.ProbeInterval } + if u.C2CC.Routing.RouteTableID != nil { + c.C2CC.Routing.RouteTableID = u.C2CC.Routing.RouteTableID + } + if u.C2CC.Routing.ServiceRouteTableID != nil { + c.C2CC.Routing.ServiceRouteTableID = u.C2CC.Routing.ServiceRouteTableID + } } // updateComputedValues examins the existing settings and converts any @@ -578,6 +585,7 @@ func (c *Config) updateComputedValues() error { } c.C2CC.stripEmptyRemoteClusters() + c.C2CC.resolveRoutingDefaults() return nil } diff --git a/pkg/controllers/c2cc/controller.go b/pkg/controllers/c2cc/controller.go index 397edd98f7..91fa72087f 100644 --- a/pkg/controllers/c2cc/controller.go +++ b/pkg/controllers/c2cc/controller.go @@ -83,13 +83,13 @@ func (c *C2CCRouteManager) Run(ctx context.Context, ready chan<- struct{}, stopp c.ovn.subscribe(ctx, reconcileCh) if routeDone, err := c.routes.subscribe(reconcileCh, "linux-route-change"); err != nil { - klog.Warningf("Could not subscribe to route events for table %d: %v", c2ccRouteTable, err) + klog.Warningf("Could not subscribe to route events for table %d: %v", c.routes.table, err) } else { defer close(routeDone) } if svcRouteDone, err := c.svcRoutes.subscribe(reconcileCh, "service-route-change"); err != nil { - klog.Warningf("Could not subscribe to route events for table %d: %v", c2ccSvcRouteTable, err) + klog.Warningf("Could not subscribe to route events for table %d: %v", c.svcRoutes.table, err) } else { defer close(svcRouteDone) } diff --git a/pkg/controllers/c2cc/helpers_test.go b/pkg/controllers/c2cc/helpers_test.go index 474840488c..b00f12bfcc 100644 --- a/pkg/controllers/c2cc/helpers_test.go +++ b/pkg/controllers/c2cc/helpers_test.go @@ -60,5 +60,8 @@ func testConfigWithRemotes(t *testing.T, remotes ...testRemoteConfig) *config.Co cfg.C2CC.ResolvedAllCIDRs = append(cfg.C2CC.ResolvedAllCIDRs, resolved.AllCIDRs()...) } + cfg.C2CC.ResolvedRouteTableID = 200 + cfg.C2CC.ResolvedServiceRouteTableID = 201 + return cfg } diff --git a/pkg/controllers/c2cc/routes.go b/pkg/controllers/c2cc/routes.go index 5dc4c61fe7..bd1b542792 100644 --- a/pkg/controllers/c2cc/routes.go +++ b/pkg/controllers/c2cc/routes.go @@ -13,8 +13,6 @@ import ( ) const ( - c2ccRouteTable = 200 - c2ccRouteProto = 200 c2ccRulePriority = 100 ) @@ -28,8 +26,8 @@ type linuxRouteManager struct { func newLinuxRouteManager(cfg *config.Config) *linuxRouteManager { m := &linuxRouteManager{ policyRouteTable: policyRouteTable{ - table: c2ccRouteTable, - proto: c2ccRouteProto, + table: cfg.C2CC.ResolvedRouteTableID, + proto: cfg.C2CC.ResolvedRouteTableID, priority: c2ccRulePriority, }, desiredGWs: make(map[string]net.IP), diff --git a/pkg/controllers/c2cc/service_routes.go b/pkg/controllers/c2cc/service_routes.go index 63fbe363b8..971b9e8842 100644 --- a/pkg/controllers/c2cc/service_routes.go +++ b/pkg/controllers/c2cc/service_routes.go @@ -13,8 +13,6 @@ import ( ) const ( - c2ccSvcRouteTable = 201 - c2ccSvcRouteProto = 201 c2ccSvcRulePriority = 99 mgmtPortInterface = "ovn-k8s-mp0" ) @@ -41,8 +39,8 @@ func newServiceRouteManager(cfg *config.Config) *serviceRouteManager { return &serviceRouteManager{ policyRouteTable: policyRouteTable{ - table: c2ccSvcRouteTable, - proto: c2ccSvcRouteProto, + table: cfg.C2CC.ResolvedServiceRouteTableID, + proto: cfg.C2CC.ResolvedServiceRouteTableID, priority: c2ccSvcRulePriority, }, remoteCIDRs: remoteCIDRs, diff --git a/test/resources/c2cc.resource b/test/resources/c2cc.resource index c47526a22e..430f75833a 100644 --- a/test/resources/c2cc.resource +++ b/test/resources/c2cc.resource @@ -139,6 +139,11 @@ Oc Apply On Cluster ${stdout}= Oc On Cluster ${alias} oc apply ${ns_arg} -f ${file} RETURN ${stdout} +Verify Cluster Is Healthy + [Documentation] Verify MicroShift and all core workloads are ready. + [Arguments] ${alias} + Command On Cluster ${alias} microshift healthcheck --timeout=300s + Verify Routes In Table 200 [Documentation] Check that routes for the given CIDRs exist in table 200. [Arguments] ${alias} ${remote_pod_cidr} ${remote_svc_cidr} diff --git a/test/suites/c2cc/cleanup.robot b/test/suites/c2cc/cleanup.robot index d38c92ef5b..59af6e8ea6 100644 --- a/test/suites/c2cc/cleanup.robot +++ b/test/suites/c2cc/cleanup.robot @@ -126,8 +126,3 @@ Enable C2CC On Cluster Command On Cluster ${alias} systemctl restart microshift Wait Until Keyword Succeeds ${CLEANUP_TIMEOUT} ${CLEANUP_RETRY} ... Verify Cluster Is Healthy ${alias} - -Verify Cluster Is Healthy - [Documentation] Verify MicroShift and all core workloads (including OVN-K) are ready. - [Arguments] ${alias} - Command On Cluster ${alias} microshift healthcheck --timeout=300s diff --git a/test/suites/c2cc/custom-route-tables.robot b/test/suites/c2cc/custom-route-tables.robot new file mode 100644 index 0000000000..dc3f1c6538 --- /dev/null +++ b/test/suites/c2cc/custom-route-tables.robot @@ -0,0 +1,132 @@ +*** Settings *** +Documentation Verify C2CC respects custom routing table IDs from configuration. +... Applies a drop-in config with non-default table IDs (150/151) on +... Cluster A, restarts MicroShift, and verifies routes and IP rules +... appear in the custom tables. +... +... Note: MicroShift does not clean up C2CC state on exit. This is by +... design - in case of a crash, pods keep running under kubepods-slice +... managed by crio, so C2CC connectivity stays online. Changing routing +... table IDs while C2CC is enabled results in duplicated rules in both +... old and new tables. The old tables are only cleaned up by rebooting. + +Resource ../../resources/microshift-process.resource +Resource ../../resources/kubeconfig.resource +Resource ../../resources/oc.resource +Resource ../../resources/c2cc.resource + +Suite Setup Setup +Suite Teardown Teardown + +Test Tags c2cc + + +*** Variables *** +${CUSTOM_ROUTE_TABLE} 150 +${CUSTOM_SVC_ROUTE_TABLE} 151 +${CUSTOM_ROUTING_DROPIN} 60-custom-routing +${RESTART_TIMEOUT} 360s +${RESTART_RETRY} 30s + + +*** Test Cases *** +Routes Exist In Custom Table + [Documentation] Verify routes to remote CIDRs exist in the custom policy routing table. + ${stdout}= Command On Cluster cluster-a + ... ${IP_CMD} route show table ${CUSTOM_ROUTE_TABLE} + Should Contain ${stdout} ${CLUSTER_B_POD_CIDR} + Should Contain ${stdout} ${CLUSTER_B_SVC_CIDR} + +IP Rules Point To Custom Table + [Documentation] Verify IP rules direct remote CIDRs to the custom table. + ${stdout}= Command On Cluster cluster-a ${IP_CMD} rule show + Should Contain ${stdout} to ${CLUSTER_B_POD_CIDR} lookup ${CUSTOM_ROUTE_TABLE} + Should Contain ${stdout} to ${CLUSTER_B_SVC_CIDR} lookup ${CUSTOM_ROUTE_TABLE} + +Service Routes Exist In Custom Service Table + [Documentation] Verify service routes exist in the custom service routing table. + ${stdout}= Command On Cluster cluster-a + ... ${IP_CMD} route show table ${CUSTOM_SVC_ROUTE_TABLE} + Should Contain ${stdout} ${CLUSTER_A_SVC_CIDR} + +Service IP Rules Point To Custom Service Table + [Documentation] Verify service IP rules point to the custom service table. + ${stdout}= Command On Cluster cluster-a ${IP_CMD} rule show + Should Contain ${stdout} + ... from ${CLUSTER_B_POD_CIDR} to ${CLUSTER_A_SVC_CIDR} lookup ${CUSTOM_SVC_ROUTE_TABLE} + Should Contain ${stdout} + ... from ${CLUSTER_B_SVC_CIDR} to ${CLUSTER_A_SVC_CIDR} lookup ${CUSTOM_SVC_ROUTE_TABLE} + +Old Default Tables Are Not Cleaned Up + [Documentation] Verify that the old default tables (200/201) still contain routes + ... from before the config change. MicroShift intentionally does not clean up + ... C2CC state on exit - in case of a crash, pods keep running and C2CC + ... connectivity stays online. Changing table IDs produces duplicated rules + ... which are resolved by rebooting the system. + ${routes}= Command On Cluster cluster-a ${IP_CMD} route show table 200 + Should Contain ${routes} ${CLUSTER_B_POD_CIDR} + ${rules}= Command On Cluster cluster-a ${IP_CMD} rule show + Should Contain ${rules} lookup 200 + + +*** Keywords *** +Setup + [Documentation] Register clusters, apply custom routing drop-in, restart MicroShift. + Check Required Env Variables + Login MicroShift Host + Setup Kubeconfig + Register Local Cluster cluster-a + Register Remote Cluster cluster-b ${HOST2_IP} ${HOST2_SSH_PORT} ${KUBECONFIG_B} + Apply Custom Routing Config + Restart And Wait For Healthy + +Teardown + [Documentation] Remove custom routing drop-in, flush custom tables, restore defaults. + Remove Custom Routing Config + Flush Custom Tables + Restart And Wait For Healthy + Teardown All Remote Clusters + Remove Kubeconfig + Logout MicroShift Host + +Apply Custom Routing Config + [Documentation] Write a drop-in that overrides routing table IDs. + VAR ${dropin}= /etc/microshift/config.d/${CUSTOM_ROUTING_DROPIN}.yaml + ${yaml}= Catenate SEPARATOR=\n + ... clusterToCluster: + ... ${SPACE}${SPACE}routing: + ... ${SPACE}${SPACE}${SPACE}${SPACE}routeTableID: ${CUSTOM_ROUTE_TABLE} + ... ${SPACE}${SPACE}${SPACE}${SPACE}serviceRouteTableID: ${CUSTOM_SVC_ROUTE_TABLE} + ${conn_id}= Get From Dictionary ${C2CC_SSH_IDS} cluster-a + SSHLibrary.Switch Connection ${conn_id} + Upload String To File ${yaml} ${dropin} + +Remove Custom Routing Config + [Documentation] Remove the custom routing drop-in file. + Command On Cluster cluster-a + ... rm -f /etc/microshift/config.d/${CUSTOM_ROUTING_DROPIN}.yaml + +Flush Custom Tables + [Documentation] Remove routes and rules from custom tables to avoid interfering + ... with subsequent tests that expect only tables 200/201. + Disruptive Command On Cluster cluster-a + ... ${IP_CMD} route flush table ${CUSTOM_ROUTE_TABLE} + Disruptive Command On Cluster cluster-a + ... ${IP_CMD} route flush table ${CUSTOM_SVC_ROUTE_TABLE} + ${rules}= Disruptive Command On Cluster cluster-a ${IP_CMD} rule show + ${lines}= Split To Lines ${rules} + FOR ${line} IN @{lines} + IF 'lookup ${CUSTOM_ROUTE_TABLE}' in $line or 'lookup ${CUSTOM_SVC_ROUTE_TABLE}' in $line + ${priority}= Fetch From Left ${line} : + ${priority}= Strip String ${priority} + ${selector}= Fetch From Right ${line} :\t + Disruptive Command On Cluster cluster-a + ... ${IP_CMD} rule del prio ${priority} ${selector} + END + END + +Restart And Wait For Healthy + [Documentation] Restart MicroShift and wait until the cluster is healthy. + Command On Cluster cluster-a systemctl restart microshift + Wait Until Keyword Succeeds ${RESTART_TIMEOUT} ${RESTART_RETRY} + ... Verify Cluster Is Healthy cluster-a