From 30004dfa7daf831def27360be05ae55dd19024ed Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 03:10:23 +0900 Subject: [PATCH 1/6] Tune redis proxy connection capacity --- adapter/redis_peer_limiter.go | 2 +- adapter/redis_peer_limiter_test.go | 6 +++--- cmd/redis-proxy/main_test.go | 8 ++++---- deploy/redis-proxy/docker-compose.ha.yml | 4 ++-- ...26_04_24_implemented_workload_isolation.md | 4 ++-- docs/redis-proxy-deployment.md | 20 +++++++++---------- proxy/backend.go | 2 +- proxy/dualwrite.go | 2 +- proxy/proxy_test.go | 4 ++-- 9 files changed, 26 insertions(+), 26 deletions(-) diff --git a/adapter/redis_peer_limiter.go b/adapter/redis_peer_limiter.go index d1bace22c..c9d44329a 100644 --- a/adapter/redis_peer_limiter.go +++ b/adapter/redis_peer_limiter.go @@ -10,7 +10,7 @@ import ( const ( redisPerPeerLimitEnv = "ELASTICKV_REDIS_PER_PEER_CONNECTIONS" - defaultRedisPerPeerConnectionCap = 8 + defaultRedisPerPeerConnectionCap = 64 redisPeerLimitError = "ERR max connections per client exceeded" unknownRedisPeer = "unknown" ) diff --git a/adapter/redis_peer_limiter_test.go b/adapter/redis_peer_limiter_test.go index ea631375f..65e9ff944 100644 --- a/adapter/redis_peer_limiter_test.go +++ b/adapter/redis_peer_limiter_test.go @@ -78,7 +78,7 @@ func TestRedisLeaderClientPoolStaysBelowPeerLimit(t *testing.T) { } func TestRedisLeaderClientPoolUsesSmallDefault(t *testing.T) { - server := NewRedisServer(nil, "", nil, nil, nil, nil, WithRedisPerPeerConnectionLimit(8)) + server := NewRedisServer(nil, "", nil, nil, nil, nil, WithRedisPerPeerConnectionLimit(defaultRedisPerPeerConnectionCap)) client := server.getOrCreateLeaderClient("127.0.0.1:6379") defer client.Close() @@ -95,7 +95,7 @@ func TestRedisLeaderClientPoolsSharePeerBudget(t *testing.T) { }{ {name: "low cap", limit: 2, wantNormal: 1, wantBlocking: 1}, {name: "four cap", limit: 4, wantNormal: 2, wantBlocking: 2}, - {name: "default cap", limit: 8, wantNormal: 4, wantBlocking: 4}, + {name: "default cap", limit: defaultRedisPerPeerConnectionCap, wantNormal: defaultRedisLeaderClientPoolSize, wantBlocking: defaultRedisBlockingLeaderClientPoolSize}, } { t.Run(tc.name, func(t *testing.T) { server := NewRedisServer(nil, "", nil, nil, nil, nil, WithRedisPerPeerConnectionLimit(tc.limit)) @@ -107,7 +107,7 @@ func TestRedisLeaderClientPoolsSharePeerBudget(t *testing.T) { } func TestRedisBlockingLeaderClientUsesDedicatedBudgetedPool(t *testing.T) { - server := NewRedisServer(nil, "", nil, nil, nil, nil, WithRedisPerPeerConnectionLimit(8)) + server := NewRedisServer(nil, "", nil, nil, nil, nil, WithRedisPerPeerConnectionLimit(defaultRedisPerPeerConnectionCap)) shared := server.getOrCreateLeaderClient("127.0.0.1:6379") defer shared.Close() blocking := server.getOrCreateBlockingLeaderClient("127.0.0.1:6379") diff --git a/cmd/redis-proxy/main_test.go b/cmd/redis-proxy/main_test.go index 0330ae8eb..485fd31a2 100644 --- a/cmd/redis-proxy/main_test.go +++ b/cmd/redis-proxy/main_test.go @@ -72,10 +72,10 @@ func TestDeriveSecondaryConcurrency(t *testing.T) { name: "dual write derives from ElasticKV pool", mode: proxy.ModeDualWrite, primaryPoolSize: 128, - elasticKVPoolSize: 4, - wantWriteConcurrency: 2, - wantScriptConcurrency: 1, - wantBlockingConcurrency: 2, + elasticKVPoolSize: 16, + wantWriteConcurrency: 8, + wantScriptConcurrency: 4, + wantBlockingConcurrency: 8, }, { name: "shadow mode derives from ElasticKV pool", diff --git a/deploy/redis-proxy/docker-compose.ha.yml b/deploy/redis-proxy/docker-compose.ha.yml index c150fd9d1..b89e30453 100644 --- a/deploy/redis-proxy/docker-compose.ha.yml +++ b/deploy/redis-proxy/docker-compose.ha.yml @@ -26,7 +26,7 @@ services: - -listen=:6379 - -primary=${REDIS_PROXY_PRIMARY:-redis:6379} - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380} - - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-4} + - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-16} - -mode=${REDIS_PROXY_MODE:-dual-write-shadow} - -metrics=:9191 networks: @@ -46,7 +46,7 @@ services: - -listen=:6379 - -primary=${REDIS_PROXY_PRIMARY:-redis:6379} - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380} - - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-4} + - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-16} - -mode=${REDIS_PROXY_MODE:-dual-write-shadow} - -metrics=:9191 networks: diff --git a/docs/design/2026_04_24_implemented_workload_isolation.md b/docs/design/2026_04_24_implemented_workload_isolation.md index a929b0c2d..dc29e7c6f 100644 --- a/docs/design/2026_04_24_implemented_workload_isolation.md +++ b/docs/design/2026_04_24_implemented_workload_isolation.md @@ -24,7 +24,7 @@ Implementation status: gated `EVAL`/`EVALSHA` path. - Shipped: Layer 3 per-peer Redis connection admission in `adapter/redis_peer_limiter.go`, wired through `RedisServer.Run` accept and - close hooks. Default cap is 8 per peer IP and is configurable via + close hooks. Default cap is 64 per peer IP and is configurable via `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` / `WithRedisPerPeerConnectionLimit`. Redis leader-proxy clients use a small explicit go-redis pool below that default cap, and Pub/Sub detached sockets @@ -374,7 +374,7 @@ one check per accept, not per command. ### Recommended v1 shape -**Per-peer-IP connection cap, default `N=8`, env-configurable, +**Per-peer-IP connection cap, default `N=64`, env-configurable, enforced at accept.** On reject, accept the TCP connection, write a `-ERR max connections per client exceeded` RESP error, then close — so the client sees a protocol-level message instead of a bare diff --git a/docs/redis-proxy-deployment.md b/docs/redis-proxy-deployment.md index 003597f2d..32c89d54d 100644 --- a/docs/redis-proxy-deployment.md +++ b/docs/redis-proxy-deployment.md @@ -35,7 +35,7 @@ go build -o redis-proxy ./cmd/redis-proxy/ | `-secondary-db` | `0` | Secondary Redis DB number | | `-secondary-password` | (empty) | Secondary Redis password | | `-primary-pool-size` | `128` | Primary Redis backend connection pool size | -| `-elastickv-pool-size` | `4` | ElasticKV backend connection pool size | +| `-elastickv-pool-size` | `16` | ElasticKV backend connection pool size | | `-secondary-write-concurrency` | `0` | Shared maximum for all asynchronous secondary writes, including scripts. `0` derives half of the secondary backend pool size, minimum `1` | | `-secondary-script-concurrency` | `0` | Lua-script sublimit within `-secondary-write-concurrency`. `0` derives half of the shared write limit, minimum `1` | | `-secondary-write-queue-size` | `0` | Bounded queue for non-script secondary writes. `0` derives `64 * concurrency`, clamped to `64..8192` | @@ -94,9 +94,9 @@ docker run --rm \ -primary redis.internal:6379 \ -primary-password "${REDIS_PASSWORD}" \ -secondary elastickv.internal:6380 \ - -elastickv-pool-size 4 \ - -secondary-write-concurrency 2 \ - -secondary-script-concurrency 1 \ + -elastickv-pool-size 16 \ + -secondary-write-concurrency 8 \ + -secondary-script-concurrency 4 \ -mode dual-write-shadow \ -secondary-timeout 5s \ -shadow-timeout 3s \ @@ -118,9 +118,9 @@ services: - -listen=:6479 - -primary=redis:6379 - -secondary=elastickv:6380 - - -elastickv-pool-size=4 - - -secondary-write-concurrency=2 - - -secondary-script-concurrency=1 + - -elastickv-pool-size=16 + - -secondary-write-concurrency=8 + - -secondary-script-concurrency=4 - -mode=dual-write-shadow - -metrics=:9191 depends_on: @@ -213,7 +213,7 @@ Override backend wiring via env vars before `docker compose up`: ```bash REDIS_PROXY_PRIMARY=redis.prod.internal:6379 \ REDIS_PROXY_SECONDARY=elastickv-1.prod.internal:6380,elastickv-2.prod.internal:6380,elastickv-3.prod.internal:6380 \ -REDIS_PROXY_ELASTICKV_POOL_SIZE=4 \ +REDIS_PROXY_ELASTICKV_POOL_SIZE=16 \ REDIS_PROXY_MODE=dual-write-shadow \ docker compose -f docker-compose.ha.yml up -d ``` @@ -414,7 +414,7 @@ groups: | Parameter | Value | Description | |-----------|-------|-------------| | Redis connection pool size | 128 | Default go-redis pool size for Redis | -| ElasticKV connection pool size | 4 | Default per-leader pool; keep within the server per-peer connection limit | +| ElasticKV connection pool size | 16 | Default per-leader pool; keep within the server per-peer connection limit | | Dial timeout | 5s | Backend connection timeout | | Read timeout | 3s | Backend read timeout | | Write timeout | 3s | Backend write timeout | @@ -439,7 +439,7 @@ Recommended shutdown order: `redis-proxy -> application -> Redis / ElasticKV`. ### Secondary writes are falling behind - Check `proxy_async_queue_depth`, `proxy_async_queue_delay_seconds`, and `proxy_async_drops_by_queue_total` before increasing concurrency. - Check `proxy_backend_pool_pending_requests` and the `waits`/`timeouts` pool events. Pool waits mean concurrency is too high for the configured pool. -- Increase the ElasticKV pool only together with `ELASTICKV_REDIS_PER_PEER_CONNECTIONS`; keep `-secondary-write-concurrency` at or below the pool size. +- Increase the ElasticKV pool only together with `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` when the proxy pool can exceed the server's per-peer cap; keep `-secondary-write-concurrency` at or below the pool size. - A sustained `expired` rate means secondary throughput is below ingress. Increasing queue size only delays the loss; profile ElasticKV before raising concurrency. ### High divergence count diff --git a/proxy/backend.go b/proxy/backend.go index 694155dba..d399ec155 100644 --- a/proxy/backend.go +++ b/proxy/backend.go @@ -11,7 +11,7 @@ import ( const ( defaultPoolSize = 128 - defaultElasticKVPoolSize = 4 + defaultElasticKVPoolSize = 16 defaultDialTimeout = 5 * time.Second defaultReadTimeout = 3 * time.Second defaultWriteTimeout = 3 * time.Second diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index dc1e16bf7..d28956431 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -57,7 +57,7 @@ const ( // producer-before-consumer ordering settle; a bounded no-effect window keeps // BZPOP load from turning into a long retry backlog. blockingReplayInitialDelay = 250 * time.Millisecond - blockingReplayNoEffectRetryWindow = 500 * time.Millisecond + blockingReplayNoEffectRetryWindow = 100 * time.Millisecond // compactedRetryInitialBackoff is the first delay before retrying a secondary // command that failed with a compacted-read error. compactedRetryInitialBackoff = 10 * time.Millisecond diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 2a5cd808f..a42590ef3 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -784,7 +784,7 @@ func TestDualWriter_Blocking_BZPopReplayShortTimeoutStillAttemptsZRem(t *testing func TestNoEffectReplayRetryLimitIncludesJitterBudget(t *testing.T) { limit := noEffectReplayRetryLimit(context.Background(), blockingReplayNoEffectRetryWindow) - assert.Positive(t, limit) + assert.Equal(t, 2, limit) var spent time.Duration backoff := compactedRetryInitialBackoff @@ -1334,7 +1334,7 @@ func TestDefaultBackendOptions(t *testing.T) { func TestDefaultElasticKVBackendOptions(t *testing.T) { opts := DefaultElasticKVBackendOptions() - assert.Equal(t, 4, opts.PoolSize) + assert.Equal(t, 16, opts.PoolSize) assert.Equal(t, 5*time.Second, opts.DialTimeout) assert.Equal(t, 3*time.Second, opts.ReadTimeout) assert.Equal(t, 3*time.Second, opts.WriteTimeout) From e4ce5d038c88f42bc2dffbe69378f9d355fe87ea Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 03:58:32 +0900 Subject: [PATCH 2/6] Tune redis proxy capacity defaults --- adapter/redis_peer_limiter.go | 2 +- adapter/redis_peer_limiter_test.go | 17 +++++++++++++++++ deploy/redis-proxy/docker-compose.ha.yml | 4 ++-- ...2026_04_24_implemented_workload_isolation.md | 10 ++++++---- docs/redis-proxy-deployment.md | 11 ++++++----- proxy/backend.go | 9 +++++---- proxy/dualwrite.go | 2 +- proxy/proxy_test.go | 4 ++-- 8 files changed, 40 insertions(+), 19 deletions(-) diff --git a/adapter/redis_peer_limiter.go b/adapter/redis_peer_limiter.go index c9d44329a..d1bace22c 100644 --- a/adapter/redis_peer_limiter.go +++ b/adapter/redis_peer_limiter.go @@ -10,7 +10,7 @@ import ( const ( redisPerPeerLimitEnv = "ELASTICKV_REDIS_PER_PEER_CONNECTIONS" - defaultRedisPerPeerConnectionCap = 64 + defaultRedisPerPeerConnectionCap = 8 redisPeerLimitError = "ERR max connections per client exceeded" unknownRedisPeer = "unknown" ) diff --git a/adapter/redis_peer_limiter_test.go b/adapter/redis_peer_limiter_test.go index 65e9ff944..a387d0000 100644 --- a/adapter/redis_peer_limiter_test.go +++ b/adapter/redis_peer_limiter_test.go @@ -10,6 +10,23 @@ const ( testPeerLimit = 2 ) +func TestDefaultRedisPeerLimiterUsesNoisyClientBoundedDefault(t *testing.T) { + t.Setenv(redisPerPeerLimitEnv, "") + + limiter := newDefaultRedisPeerLimiter() + require.NotNil(t, limiter) + require.Equal(t, 8, limiter.limit) + require.Equal(t, defaultRedisPerPeerConnectionCap, limiter.limit) +} + +func TestDefaultRedisPeerLimiterCanRaiseProxyDeploymentCap(t *testing.T) { + t.Setenv(redisPerPeerLimitEnv, "64") + + limiter := newDefaultRedisPeerLimiter() + require.NotNil(t, limiter) + require.Equal(t, 64, limiter.limit) +} + func TestRedisPeerLimiterRejectsAndReleases(t *testing.T) { server := NewRedisServer(nil, "", nil, nil, nil, nil, WithRedisPerPeerConnectionLimit(testPeerLimit)) c1 := &remoteCommandRecorder{remote: "192.168.0.64:10001"} diff --git a/deploy/redis-proxy/docker-compose.ha.yml b/deploy/redis-proxy/docker-compose.ha.yml index b89e30453..6c147c638 100644 --- a/deploy/redis-proxy/docker-compose.ha.yml +++ b/deploy/redis-proxy/docker-compose.ha.yml @@ -26,7 +26,7 @@ services: - -listen=:6379 - -primary=${REDIS_PROXY_PRIMARY:-redis:6379} - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380} - - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-16} + - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-8} - -mode=${REDIS_PROXY_MODE:-dual-write-shadow} - -metrics=:9191 networks: @@ -46,7 +46,7 @@ services: - -listen=:6379 - -primary=${REDIS_PROXY_PRIMARY:-redis:6379} - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380} - - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-16} + - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-8} - -mode=${REDIS_PROXY_MODE:-dual-write-shadow} - -metrics=:9191 networks: diff --git a/docs/design/2026_04_24_implemented_workload_isolation.md b/docs/design/2026_04_24_implemented_workload_isolation.md index dc29e7c6f..049743519 100644 --- a/docs/design/2026_04_24_implemented_workload_isolation.md +++ b/docs/design/2026_04_24_implemented_workload_isolation.md @@ -24,11 +24,13 @@ Implementation status: gated `EVAL`/`EVALSHA` path. - Shipped: Layer 3 per-peer Redis connection admission in `adapter/redis_peer_limiter.go`, wired through `RedisServer.Run` accept and - close hooks. Default cap is 64 per peer IP and is configurable via + close hooks. Default cap is 8 per peer IP and is configurable via `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` / - `WithRedisPerPeerConnectionLimit`. Redis leader-proxy clients use a small - explicit go-redis pool below that default cap, and Pub/Sub detached sockets - stay counted until their Pub/Sub cleanup path closes. + `WithRedisPerPeerConnectionLimit`. The redis-proxy deployment can raise the + cap explicitly on ElasticKV nodes before increasing the proxy's ElasticKV + pool size. Redis leader-proxy clients use a small explicit go-redis pool + below that default cap, and Pub/Sub detached sockets stay counted until their + Pub/Sub cleanup path closes. - Shipped: Layer 4 stream entry-per-key layout in `store/stream_helpers.go`, `adapter/redis_stream_cmds.go`, and `adapter/redis_compat_helpers.go`. XREAD now range-scans `!stream|entry|...` after the requested ID instead of diff --git a/docs/redis-proxy-deployment.md b/docs/redis-proxy-deployment.md index 32c89d54d..c35313632 100644 --- a/docs/redis-proxy-deployment.md +++ b/docs/redis-proxy-deployment.md @@ -35,7 +35,7 @@ go build -o redis-proxy ./cmd/redis-proxy/ | `-secondary-db` | `0` | Secondary Redis DB number | | `-secondary-password` | (empty) | Secondary Redis password | | `-primary-pool-size` | `128` | Primary Redis backend connection pool size | -| `-elastickv-pool-size` | `16` | ElasticKV backend connection pool size | +| `-elastickv-pool-size` | `8` | ElasticKV backend connection pool size | | `-secondary-write-concurrency` | `0` | Shared maximum for all asynchronous secondary writes, including scripts. `0` derives half of the secondary backend pool size, minimum `1` | | `-secondary-script-concurrency` | `0` | Lua-script sublimit within `-secondary-write-concurrency`. `0` derives half of the shared write limit, minimum `1` | | `-secondary-write-queue-size` | `0` | Bounded queue for non-script secondary writes. `0` derives `64 * concurrency`, clamped to `64..8192` | @@ -94,7 +94,7 @@ docker run --rm \ -primary redis.internal:6379 \ -primary-password "${REDIS_PASSWORD}" \ -secondary elastickv.internal:6380 \ - -elastickv-pool-size 16 \ + -elastickv-pool-size 8 \ -secondary-write-concurrency 8 \ -secondary-script-concurrency 4 \ -mode dual-write-shadow \ @@ -118,7 +118,7 @@ services: - -listen=:6479 - -primary=redis:6379 - -secondary=elastickv:6380 - - -elastickv-pool-size=16 + - -elastickv-pool-size=8 - -secondary-write-concurrency=8 - -secondary-script-concurrency=4 - -mode=dual-write-shadow @@ -213,7 +213,7 @@ Override backend wiring via env vars before `docker compose up`: ```bash REDIS_PROXY_PRIMARY=redis.prod.internal:6379 \ REDIS_PROXY_SECONDARY=elastickv-1.prod.internal:6380,elastickv-2.prod.internal:6380,elastickv-3.prod.internal:6380 \ -REDIS_PROXY_ELASTICKV_POOL_SIZE=16 \ +REDIS_PROXY_ELASTICKV_POOL_SIZE=8 \ REDIS_PROXY_MODE=dual-write-shadow \ docker compose -f docker-compose.ha.yml up -d ``` @@ -414,7 +414,7 @@ groups: | Parameter | Value | Description | |-----------|-------|-------------| | Redis connection pool size | 128 | Default go-redis pool size for Redis | -| ElasticKV connection pool size | 16 | Default per-leader pool; keep within the server per-peer connection limit | +| ElasticKV connection pool size | 8 | Default per-leader pool; keep within the server per-peer connection limit | | Dial timeout | 5s | Backend connection timeout | | Read timeout | 3s | Backend read timeout | | Write timeout | 3s | Backend write timeout | @@ -440,6 +440,7 @@ Recommended shutdown order: `redis-proxy -> application -> Redis / ElasticKV`. - Check `proxy_async_queue_depth`, `proxy_async_queue_delay_seconds`, and `proxy_async_drops_by_queue_total` before increasing concurrency. - Check `proxy_backend_pool_pending_requests` and the `waits`/`timeouts` pool events. Pool waits mean concurrency is too high for the configured pool. - Increase the ElasticKV pool only together with `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` when the proxy pool can exceed the server's per-peer cap; keep `-secondary-write-concurrency` at or below the pool size. +- For the production proxy shape that needs more backend connections, first deploy ElasticKV nodes with `ELASTICKV_REDIS_PER_PEER_CONNECTIONS=64`, then raise the proxy with `REDIS_PROXY_ELASTICKV_POOL_SIZE=16` or an explicit `-elastickv-pool-size=16`. Do not roll a larger proxy pool before the server-side cap is active on every node. - A sustained `expired` rate means secondary throughput is below ingress. Increasing queue size only delays the loss; profile ElasticKV before raising concurrency. ### High divergence count diff --git a/proxy/backend.go b/proxy/backend.go index d399ec155..3f4cb4eef 100644 --- a/proxy/backend.go +++ b/proxy/backend.go @@ -11,7 +11,7 @@ import ( const ( defaultPoolSize = 128 - defaultElasticKVPoolSize = 16 + defaultElasticKVPoolSize = 8 defaultDialTimeout = 5 * time.Second defaultReadTimeout = 3 * time.Second defaultWriteTimeout = 3 * time.Second @@ -72,9 +72,10 @@ func DefaultBackendOptions() BackendOptions { } // DefaultElasticKVBackendOptions returns defaults for proxy backends that -// connect to ElasticKV's Redis adapter. ElasticKV limits concurrent Redis -// connections per peer by default, so keep the pool below that cap unless the -// operator also raises ELASTICKV_REDIS_PER_PEER_CONNECTIONS on the cluster. +// connect to ElasticKV's Redis adapter. Keep the default within ElasticKV's +// server-side per-peer cap so mixed-version proxy rollouts do not exceed older +// servers; operators can still raise this together with +// ELASTICKV_REDIS_PER_PEER_CONNECTIONS after the cluster is configured for it. func DefaultElasticKVBackendOptions() BackendOptions { opts := DefaultBackendOptions() opts.PoolSize = defaultElasticKVPoolSize diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index d28956431..dc1e16bf7 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -57,7 +57,7 @@ const ( // producer-before-consumer ordering settle; a bounded no-effect window keeps // BZPOP load from turning into a long retry backlog. blockingReplayInitialDelay = 250 * time.Millisecond - blockingReplayNoEffectRetryWindow = 100 * time.Millisecond + blockingReplayNoEffectRetryWindow = 500 * time.Millisecond // compactedRetryInitialBackoff is the first delay before retrying a secondary // command that failed with a compacted-read error. compactedRetryInitialBackoff = 10 * time.Millisecond diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index a42590ef3..5e7a4920a 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -784,7 +784,7 @@ func TestDualWriter_Blocking_BZPopReplayShortTimeoutStillAttemptsZRem(t *testing func TestNoEffectReplayRetryLimitIncludesJitterBudget(t *testing.T) { limit := noEffectReplayRetryLimit(context.Background(), blockingReplayNoEffectRetryWindow) - assert.Equal(t, 2, limit) + assert.Equal(t, 5, limit) var spent time.Duration backoff := compactedRetryInitialBackoff @@ -1334,7 +1334,7 @@ func TestDefaultBackendOptions(t *testing.T) { func TestDefaultElasticKVBackendOptions(t *testing.T) { opts := DefaultElasticKVBackendOptions() - assert.Equal(t, 16, opts.PoolSize) + assert.Equal(t, 8, opts.PoolSize) assert.Equal(t, 5*time.Second, opts.DialTimeout) assert.Equal(t, 3*time.Second, opts.ReadTimeout) assert.Equal(t, 3*time.Second, opts.WriteTimeout) From 41be10a1fd570475f81d002f5024d4c7d6604d83 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 04:08:31 +0900 Subject: [PATCH 3/6] Leave redis proxy pool headroom --- deploy/redis-proxy/docker-compose.ha.yml | 4 ++-- docs/redis-proxy-deployment.md | 20 ++++++++++---------- proxy/backend.go | 6 +++--- proxy/proxy_test.go | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/deploy/redis-proxy/docker-compose.ha.yml b/deploy/redis-proxy/docker-compose.ha.yml index 6c147c638..c150fd9d1 100644 --- a/deploy/redis-proxy/docker-compose.ha.yml +++ b/deploy/redis-proxy/docker-compose.ha.yml @@ -26,7 +26,7 @@ services: - -listen=:6379 - -primary=${REDIS_PROXY_PRIMARY:-redis:6379} - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380} - - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-8} + - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-4} - -mode=${REDIS_PROXY_MODE:-dual-write-shadow} - -metrics=:9191 networks: @@ -46,7 +46,7 @@ services: - -listen=:6379 - -primary=${REDIS_PROXY_PRIMARY:-redis:6379} - -secondary=${REDIS_PROXY_SECONDARY:-elastickv:6380} - - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-8} + - -elastickv-pool-size=${REDIS_PROXY_ELASTICKV_POOL_SIZE:-4} - -mode=${REDIS_PROXY_MODE:-dual-write-shadow} - -metrics=:9191 networks: diff --git a/docs/redis-proxy-deployment.md b/docs/redis-proxy-deployment.md index c35313632..6cdc1f8b1 100644 --- a/docs/redis-proxy-deployment.md +++ b/docs/redis-proxy-deployment.md @@ -35,7 +35,7 @@ go build -o redis-proxy ./cmd/redis-proxy/ | `-secondary-db` | `0` | Secondary Redis DB number | | `-secondary-password` | (empty) | Secondary Redis password | | `-primary-pool-size` | `128` | Primary Redis backend connection pool size | -| `-elastickv-pool-size` | `8` | ElasticKV backend connection pool size | +| `-elastickv-pool-size` | `4` | ElasticKV backend connection pool size | | `-secondary-write-concurrency` | `0` | Shared maximum for all asynchronous secondary writes, including scripts. `0` derives half of the secondary backend pool size, minimum `1` | | `-secondary-script-concurrency` | `0` | Lua-script sublimit within `-secondary-write-concurrency`. `0` derives half of the shared write limit, minimum `1` | | `-secondary-write-queue-size` | `0` | Bounded queue for non-script secondary writes. `0` derives `64 * concurrency`, clamped to `64..8192` | @@ -94,9 +94,9 @@ docker run --rm \ -primary redis.internal:6379 \ -primary-password "${REDIS_PASSWORD}" \ -secondary elastickv.internal:6380 \ - -elastickv-pool-size 8 \ - -secondary-write-concurrency 8 \ - -secondary-script-concurrency 4 \ + -elastickv-pool-size 4 \ + -secondary-write-concurrency 2 \ + -secondary-script-concurrency 1 \ -mode dual-write-shadow \ -secondary-timeout 5s \ -shadow-timeout 3s \ @@ -118,9 +118,9 @@ services: - -listen=:6479 - -primary=redis:6379 - -secondary=elastickv:6380 - - -elastickv-pool-size=8 - - -secondary-write-concurrency=8 - - -secondary-script-concurrency=4 + - -elastickv-pool-size=4 + - -secondary-write-concurrency=2 + - -secondary-script-concurrency=1 - -mode=dual-write-shadow - -metrics=:9191 depends_on: @@ -213,7 +213,7 @@ Override backend wiring via env vars before `docker compose up`: ```bash REDIS_PROXY_PRIMARY=redis.prod.internal:6379 \ REDIS_PROXY_SECONDARY=elastickv-1.prod.internal:6380,elastickv-2.prod.internal:6380,elastickv-3.prod.internal:6380 \ -REDIS_PROXY_ELASTICKV_POOL_SIZE=8 \ +REDIS_PROXY_ELASTICKV_POOL_SIZE=4 \ REDIS_PROXY_MODE=dual-write-shadow \ docker compose -f docker-compose.ha.yml up -d ``` @@ -414,7 +414,7 @@ groups: | Parameter | Value | Description | |-----------|-------|-------------| | Redis connection pool size | 128 | Default go-redis pool size for Redis | -| ElasticKV connection pool size | 8 | Default per-leader pool; keep within the server per-peer connection limit | +| ElasticKV connection pool size | 4 | Default per-leader pool; keep within the server per-peer connection limit and leave room for dedicated sockets | | Dial timeout | 5s | Backend connection timeout | | Read timeout | 3s | Backend read timeout | | Write timeout | 3s | Backend write timeout | @@ -440,7 +440,7 @@ Recommended shutdown order: `redis-proxy -> application -> Redis / ElasticKV`. - Check `proxy_async_queue_depth`, `proxy_async_queue_delay_seconds`, and `proxy_async_drops_by_queue_total` before increasing concurrency. - Check `proxy_backend_pool_pending_requests` and the `waits`/`timeouts` pool events. Pool waits mean concurrency is too high for the configured pool. - Increase the ElasticKV pool only together with `ELASTICKV_REDIS_PER_PEER_CONNECTIONS` when the proxy pool can exceed the server's per-peer cap; keep `-secondary-write-concurrency` at or below the pool size. -- For the production proxy shape that needs more backend connections, first deploy ElasticKV nodes with `ELASTICKV_REDIS_PER_PEER_CONNECTIONS=64`, then raise the proxy with `REDIS_PROXY_ELASTICKV_POOL_SIZE=16` or an explicit `-elastickv-pool-size=16`. Do not roll a larger proxy pool before the server-side cap is active on every node. +- For the production proxy shape that needs more backend connections, first deploy ElasticKV nodes with `ELASTICKV_REDIS_PER_PEER_CONNECTIONS=64`, then raise the proxy with `REDIS_PROXY_ELASTICKV_POOL_SIZE=16` or an explicit `-elastickv-pool-size=16`, keeping `-secondary-write-concurrency` below the pool size, for example `8`. Do not roll a larger proxy pool before the server-side cap is active on every node. - A sustained `expired` rate means secondary throughput is below ingress. Increasing queue size only delays the loss; profile ElasticKV before raising concurrency. ### High divergence count diff --git a/proxy/backend.go b/proxy/backend.go index 3f4cb4eef..bba589f79 100644 --- a/proxy/backend.go +++ b/proxy/backend.go @@ -11,7 +11,7 @@ import ( const ( defaultPoolSize = 128 - defaultElasticKVPoolSize = 8 + defaultElasticKVPoolSize = 4 defaultDialTimeout = 5 * time.Second defaultReadTimeout = 3 * time.Second defaultWriteTimeout = 3 * time.Second @@ -73,8 +73,8 @@ func DefaultBackendOptions() BackendOptions { // DefaultElasticKVBackendOptions returns defaults for proxy backends that // connect to ElasticKV's Redis adapter. Keep the default within ElasticKV's -// server-side per-peer cap so mixed-version proxy rollouts do not exceed older -// servers; operators can still raise this together with +// server-side per-peer cap while leaving room for dedicated Pub/Sub and +// blocking-command sockets. Operators can still raise this together with // ELASTICKV_REDIS_PER_PEER_CONNECTIONS after the cluster is configured for it. func DefaultElasticKVBackendOptions() BackendOptions { opts := DefaultBackendOptions() diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 5e7a4920a..de8ce6af8 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -1334,7 +1334,7 @@ func TestDefaultBackendOptions(t *testing.T) { func TestDefaultElasticKVBackendOptions(t *testing.T) { opts := DefaultElasticKVBackendOptions() - assert.Equal(t, 8, opts.PoolSize) + assert.Equal(t, 4, opts.PoolSize) assert.Equal(t, 5*time.Second, opts.DialTimeout) assert.Equal(t, 3*time.Second, opts.ReadTimeout) assert.Equal(t, 3*time.Second, opts.WriteTimeout) From fa52ca709bccf3bc45656db73bdb692ade2f269f Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 04:16:38 +0900 Subject: [PATCH 4/6] Fix redis peer cap documentation --- docs/design/2026_04_24_implemented_workload_isolation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/2026_04_24_implemented_workload_isolation.md b/docs/design/2026_04_24_implemented_workload_isolation.md index 049743519..8ee69b3ee 100644 --- a/docs/design/2026_04_24_implemented_workload_isolation.md +++ b/docs/design/2026_04_24_implemented_workload_isolation.md @@ -376,7 +376,7 @@ one check per accept, not per command. ### Recommended v1 shape -**Per-peer-IP connection cap, default `N=64`, env-configurable, +**Per-peer-IP connection cap, default `N=8`, env-configurable, enforced at accept.** On reject, accept the TCP connection, write a `-ERR max connections per client exceeded` RESP error, then close — so the client sees a protocol-level message instead of a bare From 1f3b80d715f64e9b85192b999fb1b346c5d61d3c Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 07:00:59 +0900 Subject: [PATCH 5/6] redis-proxy: discover leader before first command --- proxy/leader_aware_backend.go | 23 ++++++++++----- proxy/leader_aware_backend_test.go | 45 ++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/proxy/leader_aware_backend.go b/proxy/leader_aware_backend.go index 05bd6e6b0..52fdc3b4c 100644 --- a/proxy/leader_aware_backend.go +++ b/proxy/leader_aware_backend.go @@ -75,8 +75,8 @@ type LeaderAwareRedisBackend struct { } // NewLeaderAwareRedisBackend creates a LeaderAwareRedisBackend with the given -// seed addresses. The first seed is used as the initial target until the -// first refresh completes. At least one seed is required. +// seed addresses. The first command waits for leader discovery instead of +// sending traffic to a seed that may be down. At least one seed is required. func NewLeaderAwareRedisBackend(seeds []string, name string, opts BackendOptions, logger *slog.Logger) *LeaderAwareRedisBackend { return NewLeaderAwareRedisBackendWithInterval(seeds, name, opts, defaultLeaderRefreshInterval, defaultLeaderRefreshTimeout, logger) } @@ -106,7 +106,7 @@ func NewLeaderAwareRedisBackendWithInterval(seeds []string, name string, opts Ba clients: make(map[string]*redis.Client, len(normalized)), clientOrder: make([]string, 0, len(normalized)), seedProtect: seedProtect, - leader: normalized[0], + leader: "", stopCh: make(chan struct{}), done: make(chan struct{}), refreshCh: make(chan struct{}, 1), @@ -389,6 +389,15 @@ func (b *LeaderAwareRedisBackend) currentClient() *redis.Client { return b.clients[b.leader] } +func (b *LeaderAwareRedisBackend) currentClientOrRefresh(ctx context.Context) *redis.Client { + cli := b.currentClient() + if cli != nil { + return cli + } + b.RefreshLeaderNow(ctx) + return b.currentClient() +} + // Do forwards a single command to the current leader. NOTLEADER refreshes the // cached leader for the next command, but the current command is not replayed: // leadership-loss errors can be returned after an operation has already applied. @@ -404,7 +413,7 @@ func (b *LeaderAwareRedisBackend) Do(ctx context.Context, args ...any) *redis.Cm } func (b *LeaderAwareRedisBackend) doOnce(ctx context.Context, args ...any) *redis.Cmd { - cli := b.currentClient() + cli := b.currentClientOrRefresh(ctx) if cli == nil { cmd := redis.NewCmd(ctx, args...) cmd.SetErr(ErrNoLeaderBackend) @@ -426,7 +435,7 @@ func (b *LeaderAwareRedisBackend) DoWithTimeout(ctx context.Context, timeout tim } func (b *LeaderAwareRedisBackend) doWithTimeoutOnce(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd { - cli := b.currentClient() + cli := b.currentClientOrRefresh(ctx) if cli == nil { cmd := redis.NewCmd(ctx, args...) cmd.SetErr(ErrNoLeaderBackend) @@ -437,7 +446,7 @@ func (b *LeaderAwareRedisBackend) doWithTimeoutOnce(ctx context.Context, timeout // Pipeline forwards a batch to the current leader. func (b *LeaderAwareRedisBackend) Pipeline(ctx context.Context, cmds [][]any) ([]*redis.Cmd, error) { - cli := b.currentClient() + cli := b.currentClientOrRefresh(ctx) if cli == nil { return nil, ErrNoLeaderBackend } @@ -517,7 +526,7 @@ func isLeaderRefreshTransportError(err error) bool { // NewPubSub opens a subscribe connection on the current leader. func (b *LeaderAwareRedisBackend) NewPubSub(ctx context.Context) *redis.PubSub { - cli := b.currentClient() + cli := b.currentClientOrRefresh(ctx) if cli == nil { return nil } diff --git a/proxy/leader_aware_backend_test.go b/proxy/leader_aware_backend_test.go index c42cfbeed..80171d246 100644 --- a/proxy/leader_aware_backend_test.go +++ b/proxy/leader_aware_backend_test.go @@ -553,6 +553,51 @@ func TestLeaderAwareRedisBackend_EvictsOldestNonProtectedClient(t *testing.T) { assert.Contains(t, addrs, seed.addr, "seed must never be evicted") } +func TestLeaderAwareRedisBackend_InitialCommandWaitsForLeaderDiscovery(t *testing.T) { + aliveLeader := newFakeElasticKVNode(t) + aliveLeader.SetLeader(aliveLeader.addr) + gate := make(chan struct{}) + aliveLeader.SetInfoGate(gate) + + backend := NewLeaderAwareRedisBackendWithInterval( + []string{"127.0.0.1:1", aliveLeader.addr}, + "elastickv", + DefaultBackendOptions(), + time.Hour, 200*time.Millisecond, + testLogger, + ) + t.Cleanup(func() { _ = backend.Close() }) + + require.Empty(t, backend.CurrentLeader(), "initial commands must not target the first seed before discovery") + + done := make(chan error, 1) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + done <- backend.Do(ctx, "SET", "k", "v").Err() + }() + + select { + case err := <-done: + t.Fatalf("command returned before leader discovery completed: %v", err) + case <-time.After(50 * time.Millisecond): + } + + close(gate) + var err error + require.Eventually(t, func() bool { + select { + case err = <-done: + return true + default: + return false + } + }, 2*time.Second, 10*time.Millisecond) + require.NoError(t, err) + require.Equal(t, int64(1), aliveLeader.commands.Load(), "initial command must reach the discovered leader") + require.Equal(t, aliveLeader.addr, backend.CurrentLeader()) +} + func TestLeaderAwareRedisBackend_FallsBackToSeedOnProbeFailure(t *testing.T) { // Seed 1 is unreachable; seed 2 is alive and reports itself as leader. aliveLeader := newFakeElasticKVNode(t) From 6c805f01409185ba39773198a01693f355331aaf Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 20:56:13 +0900 Subject: [PATCH 6/6] Throttle leader discovery without known leader --- proxy/leader_aware_backend.go | 28 +++++++++++++++++++++++++++- proxy/leader_aware_backend_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/proxy/leader_aware_backend.go b/proxy/leader_aware_backend.go index 2798de381..0b83aebc2 100644 --- a/proxy/leader_aware_backend.go +++ b/proxy/leader_aware_backend.go @@ -59,6 +59,7 @@ type LeaderAwareRedisBackend struct { refreshMu sync.Mutex refreshDone chan struct{} refreshClosed bool + lastRefreshAt time.Time refreshCtx context.Context refreshCancel context.CancelFunc @@ -209,6 +210,7 @@ func (b *LeaderAwareRedisBackend) runLeaderRefresh(done chan struct{}) { b.refreshMu.Lock() b.refreshDone = nil + b.lastRefreshAt = time.Now() close(done) b.refreshMu.Unlock() } @@ -244,6 +246,30 @@ func (b *LeaderAwareRedisBackend) RefreshLeaderNow(ctx context.Context) { b.refreshLeader(ctx) } +func (b *LeaderAwareRedisBackend) refreshLeaderNowIfDue(ctx context.Context) { + b.refreshMu.Lock() + if b.refreshClosed { + b.refreshMu.Unlock() + return + } + done := b.refreshDone + if done == nil { + if !b.lastRefreshAt.IsZero() && time.Since(b.lastRefreshAt) < b.refreshInterval { + b.refreshMu.Unlock() + return + } + done = make(chan struct{}) + b.refreshDone = done + go b.runLeaderRefresh(done) + } + b.refreshMu.Unlock() + + select { + case <-done: + case <-ctx.Done(): + } +} + func (b *LeaderAwareRedisBackend) probeLeader(ctx context.Context, addr string) (string, error) { cli := b.getOrCreateClient(addr) if cli == nil { @@ -394,7 +420,7 @@ func (b *LeaderAwareRedisBackend) currentClientOrRefresh(ctx context.Context) *r if cli != nil { return cli } - b.RefreshLeaderNow(ctx) + b.refreshLeaderNowIfDue(ctx) return b.currentClient() } diff --git a/proxy/leader_aware_backend_test.go b/proxy/leader_aware_backend_test.go index 53dadb8cb..5d40cefd7 100644 --- a/proxy/leader_aware_backend_test.go +++ b/proxy/leader_aware_backend_test.go @@ -682,6 +682,33 @@ func TestLeaderAwareRedisBackend_NewPubSubFallsBackToSeedWhenNoLeaderDiscovered( require.Empty(t, backend.CurrentLeader()) } +func TestLeaderAwareRedisBackend_UnknownLeaderDoesNotProbeOnEveryCommand(t *testing.T) { + seed := newFakeElasticKVNode(t) + + backend := NewLeaderAwareRedisBackendWithInterval( + []string{seed.addr}, + "elastickv", + DefaultBackendOptions(), + time.Hour, 50*time.Millisecond, + testLogger, + ) + t.Cleanup(func() { _ = backend.Close() }) + + require.Eventually(t, func() bool { + return seed.infoCalls.Load() > 0 + }, time.Second, 10*time.Millisecond, "initial background probe should run") + require.Empty(t, backend.CurrentLeader()) + before := seed.infoCalls.Load() + + for i := 0; i < 3; i++ { + err := backend.Do(context.Background(), "SET", "k", "v").Err() + require.ErrorIs(t, err, ErrNoLeaderBackend) + } + + require.Equal(t, before, seed.infoCalls.Load(), "commands must not start a fresh probe inside the refresh interval") + require.Equal(t, int64(0), seed.commands.Load(), "commands must not fall back to a seed when leader discovery returned no leader") +} + func TestLeaderAwareRedisBackend_FallsBackToSeedOnProbeFailure(t *testing.T) { // Seed 1 is unreachable; seed 2 is alive and reports itself as leader. aliveLeader := newFakeElasticKVNode(t)