From 0769a12290619e83e6014682afb85f3fa0611c48 Mon Sep 17 00:00:00 2001 From: Chuang Wang Date: Mon, 27 Jul 2026 09:37:38 -0700 Subject: [PATCH] Stop Envoy reusing upstream connections to actors Envoy pools HTTP/1.1 connections by destination address, which is safe when an address means one stable server. For actors it does not. A worker pod keeps its IP for its whole lifetime, but the actor sandbox behind port 80 is torn down on every Suspend and a different actor takes the slot. A pooled connection therefore belongs to an actor that may already be gone, and reusing it races the far end's close: the request goes into a dead socket and Envoy synthesizes a 503. At 50 actors churning across 50 workers this was ~42% of pings failing, with Envoy reporting 450 upstream_cx_destroy_remote_with_active_rq. Setting max_requests_per_connection to 1 took it to 0% and 0. The handshake this costs is sub-millisecond in-cluster against pings that take ~22ms, and it removes the race rather than recovering from it after the fact. Carrying HttpProtocolOptions on a dynamic_forward_proxy cluster also needs allow_insecure_cluster_options. Envoy guards that because a DFP cluster forwards to whatever host the request names, so a TLS upstream must validate against it; this cluster has no transport socket at all and the authority is an IP literal. Without the flag Envoy NACKs every CDS push and drops the cluster, which looks like "all actor traffic 503s" rather than a config error, so there is a test pinning it. --- cmd/atenet/internal/router/xds.go | 35 ++++++++++++++++-- cmd/atenet/internal/router/xds_test.go | 49 ++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index 3d52657af..5869805a8 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -66,6 +66,12 @@ const ( RouteName = "substrate_routes" ClusterName = "ate-cluster" OtlpClusterName = "otel_collector_cluster" + + // httpProtocolOptionsName is the well-known extension key Envoy looks for in + // a cluster's typed_extension_protocol_options. It must match the message's + // full proto type name exactly; a typo is silently ignored rather than + // rejected, so the options simply never take effect. + httpProtocolOptionsName = "envoy.extensions.upstreams.http.v3.HttpProtocolOptions" ) // XdsServer implements an aggregated discovery service server for dynamic Envoy router nodes. @@ -262,7 +268,7 @@ func (x *XdsServer) buildCluster() *clusterv3.Cluster { }, }, TypedExtensionProtocolOptions: map[string]*anypb.Any{ - "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": h2Opts, + httpProtocolOptionsName: h2Opts, }, } } @@ -324,7 +330,7 @@ func (x *XdsServer) buildOtlpCollectorCluster() *clusterv3.Cluster { }, }, TypedExtensionProtocolOptions: map[string]*anypb.Any{ - "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": h2Opts, + httpProtocolOptionsName: h2Opts, }, } } @@ -334,10 +340,32 @@ func (x *XdsServer) buildDynamicForwardProxyCluster() *clusterv3.Cluster { ClusterImplementationSpecifier: &dfpclusterv3.ClusterConfig_DnsCacheConfig{ DnsCacheConfig: buildDnsCacheConfig(), }, + // A DFP cluster rejects HttpProtocolOptions unless auto_sni and + // auto_san_validation are on or this is set. This cluster has no + // transport socket — plaintext to worker pod IPs, with an IP literal for + // the authority — so there is no certificate to validate against. + AllowInsecureClusterOptions: true, } clusterConfigAny, _ := anypb.New(dfpClusterConfig) + // One request per connection. Envoy pools by destination address, which + // assumes an address means one stable server. A worker pod's IP is stable + // but the actor sandbox behind port 80 is destroyed on every Suspend and a + // different actor takes the slot, so a pooled connection can belong to an + // actor that is already gone and the request 503s. + httpOpts, _ := anypb.New(&httpv3.HttpProtocolOptions{ + CommonHttpProtocolOptions: &corev3.HttpProtocolOptions{ + MaxRequestsPerConnection: wrapperspb.UInt32(1), + }, + UpstreamProtocolOptions: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_{ + ExplicitHttpConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig{ + // HTTP/1.1 upstream, matching what actors serve on port 80. + ProtocolConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_HttpProtocolOptions{}, + }, + }, + }) + return &clusterv3.Cluster{ Name: "dynamic_forward_proxy_cluster", LbPolicy: clusterv3.Cluster_CLUSTER_PROVIDED, @@ -347,6 +375,9 @@ func (x *XdsServer) buildDynamicForwardProxyCluster() *clusterv3.Cluster { TypedConfig: clusterConfigAny, }, }, + TypedExtensionProtocolOptions: map[string]*anypb.Any{ + httpProtocolOptionsName: httpOpts, + }, } } diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 9a2c2460f..04f20a764 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -24,6 +24,8 @@ import ( clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + dfpclusterv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dynamic_forward_proxy/v3" + httpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3" resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" ) @@ -210,3 +212,50 @@ func TestXdsServer_Serve_Shutdown(t *testing.T) { t.Error("Timeout exceeded waiting for Serve to finish graceful closure") } } + +// TestDynamicForwardProxyCluster_EnvoyAcceptsHttpProtocolOptions guards a +// coupling that is invisible on the Go side but fatal at runtime. +// +// Envoy refuses a dynamic_forward_proxy cluster that carries +// HttpProtocolOptions unless the cluster config either turns on both auto_sni +// and auto_san_validation or sets allow_insecure_cluster_options. A snapshot +// that breaks the rule is a perfectly well-formed proto and passes +// Consistent(), so nothing here fails: the only symptom is Envoy NACKing every +// CDS push and the cluster silently missing from its config dump, which reads +// as "all actor traffic 503s" rather than as a config error. +func TestDynamicForwardProxyCluster_EnvoyAcceptsHttpProtocolOptions(t *testing.T) { + cluster := NewXdsServer(18000).buildDynamicForwardProxyCluster() + + var clusterConfig dfpclusterv3.ClusterConfig + if err := cluster.GetClusterType().GetTypedConfig().UnmarshalTo(&clusterConfig); err != nil { + t.Fatalf("Failed to unmarshal dynamic forward proxy cluster config: %v", err) + } + + if !clusterConfig.GetAllowInsecureClusterOptions() { + t.Errorf("Cluster carries %s but neither allows insecure cluster options nor "+ + "enables auto_sni and auto_san_validation; Envoy will reject every CDS push", + httpProtocolOptionsName) + } +} + +// TestDynamicForwardProxyCluster_DisablesConnectionReuse pins the fix for the +// 503 storm seen under actor churn. Worker pod IPs are stable while the actor +// sandbox behind them is destroyed on every Suspend, so a pooled connection +// outlives the actor that owned it. +func TestDynamicForwardProxyCluster_DisablesConnectionReuse(t *testing.T) { + cluster := NewXdsServer(18000).buildDynamicForwardProxyCluster() + + raw, ok := cluster.GetTypedExtensionProtocolOptions()[httpProtocolOptionsName] + if !ok { + t.Fatalf("Cluster is missing %s", httpProtocolOptionsName) + } + + var opts httpv3.HttpProtocolOptions + if err := raw.UnmarshalTo(&opts); err != nil { + t.Fatalf("Failed to unmarshal HttpProtocolOptions: %v", err) + } + + if got := opts.GetCommonHttpProtocolOptions().GetMaxRequestsPerConnection().GetValue(); got != 1 { + t.Errorf("Expected max_requests_per_connection 1, got %d", got) + } +}