Stop Envoy reusing upstream connections to actors

cmd/atenet/internal/router/xds.go

@@ -66,6 +66,12 @@ const (
 	RouteName            = "substrate_routes"
 	ClusterName          = "ate-cluster"
 	OtlpClusterName      = "otel_collector_cluster"
+
+	// httpProtocolOptionsName is the well-known extension key Envoy looks for in
+	// a cluster's typed_extension_protocol_options. It must match the message's
+	// full proto type name exactly; a typo is silently ignored rather than
+	// rejected, so the options simply never take effect.
+	httpProtocolOptionsName = "envoy.extensions.upstreams.http.v3.HttpProtocolOptions"
 )
 
 // XdsServer implements an aggregated discovery service server for dynamic Envoy router nodes.
@@ -334,10 +340,46 @@ func (x *XdsServer) buildDynamicForwardProxyCluster() *clusterv3.Cluster {
 		ClusterImplementationSpecifier: &dfpclusterv3.ClusterConfig_DnsCacheConfig{
 			DnsCacheConfig: buildDnsCacheConfig(),
 		},
+		// Envoy refuses a dynamic_forward_proxy cluster that carries
+		// HttpProtocolOptions unless auto_sni and auto_san_validation are on, or
+		// this flag is set. That guard exists because a DFP cluster forwards to
+		// whatever host the request names, so a TLS upstream must validate the
+		// certificate against that host. This cluster has no transport socket at
+		// all — it speaks plaintext HTTP to worker pod IPs inside the cluster —
+		// and the authority is an IP literal, which cannot produce a meaningful
+		// SNI or SAN match. Saying so is more honest than enabling two TLS knobs
+		// that would do nothing.
+		AllowInsecureClusterOptions: true,
 	}
 
 	clusterConfigAny, _ := anypb.New(dfpClusterConfig)
 
+	// Disable upstream connection reuse.
+	//
+	// Envoy pools HTTP/1.1 connections by destination address, which is safe
+	// when an address means one stable server. Here it does not. A worker pod
+	// keeps its IP for its whole lifetime, but the actor sandbox behind port 80
+	// is destroyed on every Suspend and a different actor takes the slot. A
+	// pooled connection therefore belongs to an actor that may already be gone,
+	// and reusing it races the far end's close: the request goes into a dead
+	// socket and Envoy synthesizes a 503. Measured at ~40% of pings with 50
+	// actors churning across 50 workers.
+	//
+	// One request per connection costs a TCP handshake — sub-millisecond
+	// in-cluster, against pings that take ~22ms — and removes the race outright
+	// rather than recovering from it after the fact.
+	httpOpts, _ := anypb.New(&httpv3.HttpProtocolOptions{
+		CommonHttpProtocolOptions: &corev3.HttpProtocolOptions{
+			MaxRequestsPerConnection: wrapperspb.UInt32(1),
+		},
+		UpstreamProtocolOptions: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_{
+			ExplicitHttpConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig{
+				// HTTP/1.1 upstream, matching what actors serve on port 80.
+				ProtocolConfig: &httpv3.HttpProtocolOptions_ExplicitHttpConfig_HttpProtocolOptions{},
+			},
+		},
+	})
+
 	return &clusterv3.Cluster{
 		Name:     "dynamic_forward_proxy_cluster",
 		LbPolicy: clusterv3.Cluster_CLUSTER_PROVIDED,
@@ -347,6 +389,9 @@ func (x *XdsServer) buildDynamicForwardProxyCluster() *clusterv3.Cluster {
 				TypedConfig: clusterConfigAny,
 			},
 		},
+		TypedExtensionProtocolOptions: map[string]*anypb.Any{
+			httpProtocolOptionsName: httpOpts,
+		},
 	}
 }
 

cmd/atenet/internal/router/xds_test.go

@@ -24,6 +24,8 @@ import (
 	clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	listenerv3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
 	routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	dfpclusterv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dynamic_forward_proxy/v3"
+	httpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3"
 	cachev3 "github.com/envoyproxy/go-control-plane/pkg/cache/v3"
 	resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3"
 )
@@ -210,3 +212,63 @@ func TestXdsServer_Serve_Shutdown(t *testing.T) {
 		t.Error("Timeout exceeded waiting for Serve to finish graceful closure")
 	}
 }
+
+// TestDynamicForwardProxyCluster_EnvoyAcceptsHttpProtocolOptions guards a
+// coupling that is invisible on the Go side but fatal at runtime.
+//
+// Envoy refuses a dynamic_forward_proxy cluster that carries
+// HttpProtocolOptions unless the cluster config either turns on both auto_sni
+// and auto_san_validation or sets allow_insecure_cluster_options. A snapshot
+// that breaks the rule is a perfectly well-formed proto and passes
+// Consistent(), so nothing here fails: the only symptom is Envoy NACKing every
+// CDS push and the cluster silently missing from its config dump, which reads
+// as "all actor traffic 503s" rather than as a config error.
+func TestDynamicForwardProxyCluster_EnvoyAcceptsHttpProtocolOptions(t *testing.T) {
+	server := NewXdsServer(18000)
+	server.SetConfig(8081, 50052, "10.0.0.1")
+
+	cluster := server.buildDynamicForwardProxyCluster()
+
+	if _, ok := cluster.GetTypedExtensionProtocolOptions()[httpProtocolOptionsName]; !ok {
+		// No HttpProtocolOptions means the constraint does not apply.
+		return
+	}
+
+	var clusterConfig dfpclusterv3.ClusterConfig
+	if err := cluster.GetClusterType().GetTypedConfig().UnmarshalTo(&clusterConfig); err != nil {
+		t.Fatalf("Failed to unmarshal dynamic forward proxy cluster config: %v", err)
+	}
+
+	if clusterConfig.GetAllowInsecureClusterOptions() {
+		return
+	}
+
+	t.Errorf("Cluster carries %s but neither allows insecure cluster options nor "+
+		"enables auto_sni and auto_san_validation; Envoy will reject every CDS push",
+		httpProtocolOptionsName)
+}
+
+// TestDynamicForwardProxyCluster_DisablesConnectionReuse pins the fix for the
+// 503 storm seen under actor churn. Worker pod IPs are stable while the actor
+// sandbox behind them is destroyed on every Suspend, so a pooled connection
+// outlives the actor that owned it.
+func TestDynamicForwardProxyCluster_DisablesConnectionReuse(t *testing.T) {
+	server := NewXdsServer(18000)
+	server.SetConfig(8081, 50052, "10.0.0.1")
+
+	cluster := server.buildDynamicForwardProxyCluster()
+
+	raw, ok := cluster.GetTypedExtensionProtocolOptions()[httpProtocolOptionsName]
+	if !ok {
+		t.Fatalf("Cluster is missing %s", httpProtocolOptionsName)
+	}
+
+	var opts httpv3.HttpProtocolOptions
+	if err := raw.UnmarshalTo(&opts); err != nil {
+		t.Fatalf("Failed to unmarshal HttpProtocolOptions: %v", err)
+	}
+
+	if got := opts.GetCommonHttpProtocolOptions().GetMaxRequestsPerConnection().GetValue(); got != 1 {
+		t.Errorf("Expected max_requests_per_connection 1, got %d", got)
+	}
+}