diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index 3d52657af..5869805a8 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -66,6 +66,12 @@ const ( RouteName = "substrate_routes" ClusterName = "ate-cluster" OtlpClusterName = "otel_collector_cluster" + + // httpProtocolOptionsName is the well-known extension key Envoy looks for in + // a cluster's typed_extension_protocol_options. It must match the message's + // full proto type name exactly; a typo is silently ignored rather than + // rejected, so the options simply never take effect. + httpProtocolOptionsName = "envoy.extensions.upstreams.http.v3.HttpProtocolOptions" ) // XdsServer implements an aggregated discovery service server for dynamic Envoy router nodes. @@ -262,7 +268,7 @@ func (x *XdsServer) buildCluster() *clusterv3.Cluster { }, }, TypedExtensionProtocolOptions: map[string]*anypb.Any{ - "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": h2Opts, + httpProtocolOptionsName: h2Opts, }, } } @@ -324,7 +330,7 @@ func (x *XdsServer) buildOtlpCollectorCluster() *clusterv3.Cluster { }, }, TypedExtensionProtocolOptions: map[string]*anypb.Any{ - "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": h2Opts, + httpProtocolOptionsName: h2Opts, }, } } @@ -334,10 +340,32 @@ func (x *XdsServer) buildDynamicForwardProxyCluster() *clusterv3.Cluster { ClusterImplementationSpecifier: &dfpclusterv3.ClusterConfig_DnsCacheConfig{ DnsCacheConfig: buildDnsCacheConfig(), }, + // A DFP cluster rejects HttpProtocolOptions unless auto_sni and + // auto_san_validation are on or this is set. This cluster has no + // transport socket — plaintext to worker pod IPs, with an IP literal for + // the authority — so there is no certificate to validate against. + AllowInsecureClusterOptions: true, } clusterConfigAny, _ := anypb.New(dfpClusterConfig) + // One request per connection. Envoy pools by destination address, which + // assumes an address means one stable server. A worker pod's IP is stable + // but the actor sandbox behind port 80 is destroyed on every Suspend and a + // different actor takes the slot, so a pooled connection can belong to an + // actor that is already gone and the request 503s. + httpOpts, _ := anypb.New(&httpv3.HttpProtocolOptions{ + CommonHttpProtocolOptions: &corev3.HttpProtocolOptions{ + MaxRequestsPerConnection: wrapperspb.UInt32(1), + }, + UpstreamProtocolOptions: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_{ + ExplicitHttpConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig{ + // HTTP/1.1 upstream, matching what actors serve on port 80. + ProtocolConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_HttpProtocolOptions{}, + }, + }, + }) + return &clusterv3.Cluster{ Name: "dynamic_forward_proxy_cluster", LbPolicy: clusterv3.Cluster_CLUSTER_PROVIDED, @@ -347,6 +375,9 @@ func (x *XdsServer) buildDynamicForwardProxyCluster() *clusterv3.Cluster { TypedConfig: clusterConfigAny, }, }, + TypedExtensionProtocolOptions: map[string]*anypb.Any{ + httpProtocolOptionsName: httpOpts, + }, } } diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 9a2c2460f..04f20a764 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -24,6 +24,8 @@ import ( clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + dfpclusterv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dynamic_forward_proxy/v3" + httpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3" resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" ) @@ -210,3 +212,50 @@ func TestXdsServer_Serve_Shutdown(t *testing.T) { t.Error("Timeout exceeded waiting for Serve to finish graceful closure") } } + +// TestDynamicForwardProxyCluster_EnvoyAcceptsHttpProtocolOptions guards a +// coupling that is invisible on the Go side but fatal at runtime. +// +// Envoy refuses a dynamic_forward_proxy cluster that carries +// HttpProtocolOptions unless the cluster config either turns on both auto_sni +// and auto_san_validation or sets allow_insecure_cluster_options. A snapshot +// that breaks the rule is a perfectly well-formed proto and passes +// Consistent(), so nothing here fails: the only symptom is Envoy NACKing every +// CDS push and the cluster silently missing from its config dump, which reads +// as "all actor traffic 503s" rather than as a config error. +func TestDynamicForwardProxyCluster_EnvoyAcceptsHttpProtocolOptions(t *testing.T) { + cluster := NewXdsServer(18000).buildDynamicForwardProxyCluster() + + var clusterConfig dfpclusterv3.ClusterConfig + if err := cluster.GetClusterType().GetTypedConfig().UnmarshalTo(&clusterConfig); err != nil { + t.Fatalf("Failed to unmarshal dynamic forward proxy cluster config: %v", err) + } + + if !clusterConfig.GetAllowInsecureClusterOptions() { + t.Errorf("Cluster carries %s but neither allows insecure cluster options nor "+ + "enables auto_sni and auto_san_validation; Envoy will reject every CDS push", + httpProtocolOptionsName) + } +} + +// TestDynamicForwardProxyCluster_DisablesConnectionReuse pins the fix for the +// 503 storm seen under actor churn. Worker pod IPs are stable while the actor +// sandbox behind them is destroyed on every Suspend, so a pooled connection +// outlives the actor that owned it. +func TestDynamicForwardProxyCluster_DisablesConnectionReuse(t *testing.T) { + cluster := NewXdsServer(18000).buildDynamicForwardProxyCluster() + + raw, ok := cluster.GetTypedExtensionProtocolOptions()[httpProtocolOptionsName] + if !ok { + t.Fatalf("Cluster is missing %s", httpProtocolOptionsName) + } + + var opts httpv3.HttpProtocolOptions + if err := raw.UnmarshalTo(&opts); err != nil { + t.Fatalf("Failed to unmarshal HttpProtocolOptions: %v", err) + } + + if got := opts.GetCommonHttpProtocolOptions().GetMaxRequestsPerConnection().GetValue(); got != 1 { + t.Errorf("Expected max_requests_per_connection 1, got %d", got) + } +}