diff --git a/benchmarks/single_node/dsr1_fp4_b300.sh b/benchmarks/single_node/dsr1_fp4_b300.sh index 917f4f5f3..56b5f6256 100644 --- a/benchmarks/single_node/dsr1_fp4_b300.sh +++ b/benchmarks/single_node/dsr1_fp4_b300.sh @@ -16,11 +16,22 @@ check_env_vars \ RESULT_FILENAME \ EP_SIZE +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi nvidia-smi @@ -44,8 +55,8 @@ fi start_gpu_monitor set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL --host 0.0.0.0 --port $PORT --trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 \ +PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT --trust-remote-code \ +--tensor-parallel-size $TP --data-parallel-size 1 \ --cuda-graph-max-bs 256 --max-running-requests 256 --mem-fraction-static 0.85 --kv-cache-dtype fp8_e4m3 \ --chunked-prefill-size 16384 \ --ep-size $EP_SIZE --quantization modelopt_fp4 --enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ diff --git a/benchmarks/single_node/dsr1_fp8_b300.sh b/benchmarks/single_node/dsr1_fp8_b300.sh index 2d475bc0b..e84abfe64 100644 --- a/benchmarks/single_node/dsr1_fp8_b300.sh +++ b/benchmarks/single_node/dsr1_fp8_b300.sh @@ -16,13 +16,24 @@ check_env_vars \ RESULT_FILENAME \ EP_SIZE +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi export SGL_ENABLE_JIT_DEEPGEMM=false export SGLANG_ENABLE_FLASHINFER_GEMM=true @@ -76,8 +87,8 @@ fi start_gpu_monitor set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---tensor-parallel-size=$TP --data-parallel-size=1 \ +PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ +--tensor-parallel-size $TP --data-parallel-size 1 \ --cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ --mem-fraction-static $MEM_FRAC_STATIC --kv-cache-dtype fp8_e4m3 --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ --enable-flashinfer-allreduce-fusion --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL --disable-radix-cache \ diff --git a/benchmarks/single_node/dsr1_fp8_b300_mtp.sh b/benchmarks/single_node/dsr1_fp8_b300_mtp.sh index d16cbcf8e..0e132a4c0 100755 --- a/benchmarks/single_node/dsr1_fp8_b300_mtp.sh +++ b/benchmarks/single_node/dsr1_fp8_b300_mtp.sh @@ -16,13 +16,24 @@ check_env_vars \ RESULT_FILENAME \ EP_SIZE +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi export SGLANG_ENABLE_JIT_DEEPGEMM=false @@ -70,11 +81,11 @@ start_gpu_monitor set -x PYTHONNOUSERSITE=1 python3 -m sglang.launch_server \ - --model-path=$MODEL \ - --host=0.0.0.0 \ - --port=$PORT \ - --tensor-parallel-size=$TP \ - --data-parallel-size=1 \ + --model-path $MODEL_PATH --served-model-name $MODEL \ + --host 0.0.0.0 \ + --port $PORT \ + --tensor-parallel-size $TP \ + --data-parallel-size 1 \ --cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE \ --max-running-requests $MAX_RUNNING_REQUESTS \ --mem-fraction-static $MEM_FRAC_STATIC \ @@ -84,7 +95,7 @@ PYTHONNOUSERSITE=1 python3 -m sglang.launch_server \ --enable-flashinfer-allreduce-fusion \ --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ --disable-radix-cache \ - --fp8-gemm-backend=flashinfer_trtllm \ + --fp8-gemm-backend flashinfer_trtllm \ --attention-backend trtllm_mla \ --stream-interval 30 \ --ep-size $EP_SIZE \ diff --git a/benchmarks/single_node/dsv4_fp4_b300_sglang.sh b/benchmarks/single_node/dsv4_fp4_b300_sglang.sh index 8f43ea8a3..b6ced157c 100755 --- a/benchmarks/single_node/dsv4_fp4_b300_sglang.sh +++ b/benchmarks/single_node/dsv4_fp4_b300_sglang.sh @@ -12,14 +12,20 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" fi -# The B300 runner overrides MODEL to a pre-staged /data/models path, so skip -# `hf download`. Only fetch when MODEL looks like a HF repo ID. -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi @@ -172,7 +178,7 @@ fi set -x PYTHONNOUSERSITE=1 sglang serve \ - --model-path $MODEL \ + --model-path $MODEL_PATH --served-model-name $MODEL \ --host 0.0.0.0 \ --port $PORT \ --trust-remote-code \ diff --git a/benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh index 03102778d..3f0e24cee 100755 --- a/benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh +++ b/benchmarks/single_node/dsv4_fp4_b300_sglang_mtp.sh @@ -23,14 +23,20 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" fi -# The B300 runner overrides MODEL to a pre-staged /data/models path, so skip -# `hf download`. Only fetch when MODEL looks like a HF repo ID. -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi @@ -121,7 +127,7 @@ fi set -x PYTHONNOUSERSITE=1 sglang serve \ - --model-path $MODEL \ + --model-path $MODEL_PATH --served-model-name $MODEL \ --host 0.0.0.0 \ --port $PORT \ --trust-remote-code \ diff --git a/benchmarks/single_node/dsv4_fp4_b300_trt.sh b/benchmarks/single_node/dsv4_fp4_b300_trt.sh index 754846912..99be66eac 100644 --- a/benchmarks/single_node/dsv4_fp4_b300_trt.sh +++ b/benchmarks/single_node/dsv4_fp4_b300_trt.sh @@ -18,6 +18,18 @@ check_env_vars \ DP_ATTENTION \ EP_SIZE +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi @@ -47,10 +59,6 @@ sanitize_slurm_mpi_env_for_trtllm export NCCL_NVLS_ENABLE="${NCCL_NVLS_ENABLE:-0}" echo "NCCL_NVLS_ENABLE: $NCCL_NVLS_ENABLE" -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" -fi - nvidia-smi SERVER_LOG="$PWD/server.log" @@ -108,7 +116,7 @@ start_gpu_monitor --output "$PWD/gpu_metrics.csv" set -x SERVE_CMD=( - trtllm-serve "$MODEL" \ + trtllm-serve "$MODEL_PATH" \ --host 0.0.0.0 \ --port "$PORT" \ --trust_remote_code \ diff --git a/benchmarks/single_node/dsv4_fp4_b300_trt_mtp.sh b/benchmarks/single_node/dsv4_fp4_b300_trt_mtp.sh index 8aa9d0e78..6f617ddf0 100644 --- a/benchmarks/single_node/dsv4_fp4_b300_trt_mtp.sh +++ b/benchmarks/single_node/dsv4_fp4_b300_trt_mtp.sh @@ -17,6 +17,18 @@ check_env_vars \ DP_ATTENTION \ EP_SIZE +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi @@ -46,10 +58,6 @@ sanitize_slurm_mpi_env_for_trtllm export NCCL_NVLS_ENABLE="${NCCL_NVLS_ENABLE:-0}" echo "NCCL_NVLS_ENABLE: $NCCL_NVLS_ENABLE" -if [[ "$MODEL" != /* ]]; then - hf download "$MODEL" -fi - nvidia-smi SERVER_LOG="$PWD/server.log" @@ -111,7 +119,7 @@ start_gpu_monitor --output "$PWD/gpu_metrics.csv" set -x SERVE_CMD=( - trtllm-serve "$MODEL" \ + trtllm-serve "$MODEL_PATH" \ --host 0.0.0.0 \ --port "$PORT" \ --trust_remote_code \ diff --git a/benchmarks/single_node/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/dsv4_fp4_b300_vllm.sh index 92d4bf4ad..fbdacb120 100755 --- a/benchmarks/single_node/dsv4_fp4_b300_vllm.sh +++ b/benchmarks/single_node/dsv4_fp4_b300_vllm.sh @@ -17,13 +17,24 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi SERVER_LOG=/workspace/server.log PORT=${PORT:-8888} @@ -67,7 +78,7 @@ fi start_gpu_monitor set -x -vllm serve "$MODEL" --host 0.0.0.0 --port "$PORT" \ +vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port "$PORT" \ "${PARALLEL_ARGS[@]}" \ --pipeline-parallel-size 1 \ --kv-cache-dtype fp8 \ diff --git a/benchmarks/single_node/dsv4_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/dsv4_fp4_b300_vllm_mtp.sh index cb41a9eb1..3a35a84c3 100755 --- a/benchmarks/single_node/dsv4_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/dsv4_fp4_b300_vllm_mtp.sh @@ -13,13 +13,24 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi SERVER_LOG=/workspace/server.log PORT=${PORT:-8888} @@ -60,7 +71,7 @@ NUM_SPEC_TOKENS=2 start_gpu_monitor set -x -vllm serve "$MODEL" --host 0.0.0.0 --port "$PORT" \ +vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 --port "$PORT" \ "${PARALLEL_ARGS[@]}" \ --pipeline-parallel-size 1 \ --kv-cache-dtype fp8 \ diff --git a/benchmarks/single_node/glm5_fp4_b300.sh b/benchmarks/single_node/glm5_fp4_b300.sh index b751ddf7a..64c3551b2 100755 --- a/benchmarks/single_node/glm5_fp4_b300.sh +++ b/benchmarks/single_node/glm5_fp4_b300.sh @@ -16,13 +16,24 @@ check_env_vars \ RESULT_FILENAME \ EP_SIZE +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi SERVER_LOG=/workspace/server.log PORT=${PORT:-8888} @@ -38,9 +49,9 @@ fi start_gpu_monitor set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ +PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ --trust-remote-code \ ---tensor-parallel-size=$TP \ +--tensor-parallel-size $TP \ --data-parallel-size 1 --expert-parallel-size $EP_SIZE \ --disable-radix-cache \ --quantization modelopt_fp4 \ @@ -56,7 +67,7 @@ PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0. --stream-interval 30 \ --scheduler-recv-interval 10 \ --tokenizer-worker-num 6 \ ---tokenizer-path $MODEL $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & +--tokenizer-path $MODEL_PATH $EVAL_CONTEXT_ARGS > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/benchmarks/single_node/glm5_fp4_b300_mtp.sh b/benchmarks/single_node/glm5_fp4_b300_mtp.sh index db586dad8..546bb429b 100755 --- a/benchmarks/single_node/glm5_fp4_b300_mtp.sh +++ b/benchmarks/single_node/glm5_fp4_b300_mtp.sh @@ -15,13 +15,24 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi pip install --no-deps "transformers==5.2.0" "huggingface-hub==1.4.1" @@ -43,9 +54,9 @@ fi start_gpu_monitor set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ +PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ --trust-remote-code \ ---tensor-parallel-size=$TP \ +--tensor-parallel-size $TP \ --data-parallel-size 1 --expert-parallel-size 1 \ --tool-call-parser glm47 \ --reasoning-parser glm45 \ diff --git a/benchmarks/single_node/glm5_fp8_b300.sh b/benchmarks/single_node/glm5_fp8_b300.sh index 730cc3950..1bdfbdfda 100644 --- a/benchmarks/single_node/glm5_fp8_b300.sh +++ b/benchmarks/single_node/glm5_fp8_b300.sh @@ -15,13 +15,24 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi pip install --no-deps "transformers==5.2.0" "huggingface-hub==1.4.1" @@ -46,9 +57,9 @@ fi start_gpu_monitor set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ +PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ --trust-remote-code \ ---tensor-parallel-size=$TP \ +--tensor-parallel-size $TP \ --data-parallel-size 1 --expert-parallel-size 1 \ --tool-call-parser glm47 \ --reasoning-parser glm45 \ diff --git a/benchmarks/single_node/glm5_fp8_b300_mtp.sh b/benchmarks/single_node/glm5_fp8_b300_mtp.sh index 0d4290dd3..00137c62a 100755 --- a/benchmarks/single_node/glm5_fp8_b300_mtp.sh +++ b/benchmarks/single_node/glm5_fp8_b300_mtp.sh @@ -15,13 +15,24 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi pip install --no-deps "transformers==5.2.0" "huggingface-hub==1.4.1" @@ -47,9 +58,9 @@ fi start_gpu_monitor set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ +PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ --trust-remote-code \ ---tensor-parallel-size=$TP \ +--tensor-parallel-size $TP \ --data-parallel-size 1 --expert-parallel-size 1 \ --tool-call-parser glm47 \ --reasoning-parser glm45 \ diff --git a/benchmarks/single_node/kimik2.5_fp4_b300.sh b/benchmarks/single_node/kimik2.5_fp4_b300.sh index 44a06ebd5..65cc13e30 100755 --- a/benchmarks/single_node/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/kimik2.5_fp4_b300.sh @@ -16,11 +16,22 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi nvidia-smi @@ -38,8 +49,8 @@ fi start_gpu_monitor set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---tensor-parallel-size=$TP \ +vllm serve $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ +--tensor-parallel-size $TP \ --gpu-memory-utilization 0.90 \ --max-model-len $MAX_MODEL_LEN \ --max-num-seqs $CONC \ diff --git a/benchmarks/single_node/kimik2.5_int4_b300.sh b/benchmarks/single_node/kimik2.5_int4_b300.sh index 6674ad8dd..105543e5f 100755 --- a/benchmarks/single_node/kimik2.5_int4_b300.sh +++ b/benchmarks/single_node/kimik2.5_int4_b300.sh @@ -16,11 +16,22 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi nvidia-smi @@ -38,7 +49,7 @@ fi start_gpu_monitor set -x -vllm serve $MODEL --host 0.0.0.0 --port $PORT \ +vllm serve $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ --gpu-memory-utilization 0.95 \ --tensor-parallel-size $TP \ --max-model-len $MAX_MODEL_LEN \ diff --git a/benchmarks/single_node/minimaxm2.5_fp4_b300.sh b/benchmarks/single_node/minimaxm2.5_fp4_b300.sh index a2861b441..446472e35 100755 --- a/benchmarks/single_node/minimaxm2.5_fp4_b300.sh +++ b/benchmarks/single_node/minimaxm2.5_fp4_b300.sh @@ -18,13 +18,24 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi SERVER_LOG=/workspace/server.log PORT=${PORT:-8888} @@ -32,11 +43,11 @@ PORT=${PORT:-8888} export VLLM_FLOAT32_MATMUL_PRECISION=high if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS="--tensor-parallel-size=1 --data-parallel-size=$TP --enable-expert-parallel" + PARALLEL_ARGS="--tensor-parallel-size 1 --data-parallel-size $TP --enable-expert-parallel" elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS="--tensor-parallel-size=$TP --enable-expert-parallel" + PARALLEL_ARGS="--tensor-parallel-size $TP --enable-expert-parallel" else - PARALLEL_ARGS="--tensor-parallel-size=$TP" + PARALLEL_ARGS="--tensor-parallel-size $TP" fi if [ "${EVAL_ONLY}" = "true" ]; then @@ -47,7 +58,7 @@ fi start_gpu_monitor set -x -vllm serve $MODEL --port $PORT \ +vllm serve $MODEL_PATH --served-model-name $MODEL --port $PORT \ $PARALLEL_ARGS \ --gpu-memory-utilization 0.90 \ --max-model-len $MAX_MODEL_LEN \ diff --git a/benchmarks/single_node/minimaxm2.5_fp8_b300.sh b/benchmarks/single_node/minimaxm2.5_fp8_b300.sh index 30821961f..11b48e848 100755 --- a/benchmarks/single_node/minimaxm2.5_fp8_b300.sh +++ b/benchmarks/single_node/minimaxm2.5_fp8_b300.sh @@ -17,13 +17,24 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi SERVER_LOG=/workspace/server.log PORT=${PORT:-8888} @@ -44,12 +55,12 @@ fi start_gpu_monitor set -x -vllm serve $MODEL --port $PORT \ ---tensor-parallel-size=$TP \ +vllm serve $MODEL_PATH --served-model-name $MODEL --port $PORT \ +--tensor-parallel-size $TP \ $EP \ --gpu-memory-utilization 0.90 \ --max-model-len $MAX_MODEL_LEN \ ---block-size=32 \ +--block-size 32 \ --kv-cache-dtype fp8 \ --max-cudagraph-capture-size 2048 \ --max-num-batched-tokens "$((ISL * 2 ))" \ diff --git a/benchmarks/single_node/qwen3.5_bf16_b300.sh b/benchmarks/single_node/qwen3.5_bf16_b300.sh index f1056c896..29c3f7551 100755 --- a/benchmarks/single_node/qwen3.5_bf16_b300.sh +++ b/benchmarks/single_node/qwen3.5_bf16_b300.sh @@ -12,13 +12,24 @@ check_env_vars \ RESULT_FILENAME \ EP_SIZE +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi export NCCL_NVLS_ENABLE=1 export SGL_ENABLE_JIT_DEEPGEMM=false @@ -52,9 +63,9 @@ echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, start_gpu_monitor set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---served-model-name "Qwen/Qwen3.5-397B-A17B" --trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 --ep-size $EP_SIZE \ +PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --host 0.0.0.0 --port $PORT \ +--served-model-name $MODEL --trust-remote-code \ +--tensor-parallel-size $TP --data-parallel-size 1 --ep-size $EP_SIZE \ --cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ --mem-fraction-static $MEM_FRAC_STATIC --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ --context-length $CONTEXT_LENGTH --disable-radix-cache \ diff --git a/benchmarks/single_node/qwen3.5_bf16_b300_mtp.sh b/benchmarks/single_node/qwen3.5_bf16_b300_mtp.sh index 705ca9775..91678a12e 100755 --- a/benchmarks/single_node/qwen3.5_bf16_b300_mtp.sh +++ b/benchmarks/single_node/qwen3.5_bf16_b300_mtp.sh @@ -12,13 +12,24 @@ check_env_vars \ RESULT_FILENAME \ EP_SIZE +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi export NCCL_NVLS_ENABLE=1 export SGL_ENABLE_JIT_DEEPGEMM=false @@ -52,9 +63,9 @@ echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, start_gpu_monitor set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ ---served-model-name "Qwen/Qwen3.5-397B-A17B" --trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 --ep-size $EP_SIZE \ +PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --host 0.0.0.0 --port $PORT \ +--served-model-name $MODEL --trust-remote-code \ +--tensor-parallel-size $TP --data-parallel-size 1 --ep-size $EP_SIZE \ --cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ --mem-fraction-static $MEM_FRAC_STATIC --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ --context-length $CONTEXT_LENGTH --disable-radix-cache \ diff --git a/benchmarks/single_node/qwen3.5_fp4_b300.sh b/benchmarks/single_node/qwen3.5_fp4_b300.sh index 18b6cda09..41c891e05 100755 --- a/benchmarks/single_node/qwen3.5_fp4_b300.sh +++ b/benchmarks/single_node/qwen3.5_fp4_b300.sh @@ -15,13 +15,24 @@ check_env_vars \ RESULT_FILENAME \ EP_SIZE +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi export NCCL_NVLS_ENABLE=1 export SGL_ENABLE_JIT_DEEPGEMM=false @@ -61,9 +72,9 @@ echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, start_gpu_monitor set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ +PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ --trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 --ep-size $EP_SIZE \ +--tensor-parallel-size $TP --data-parallel-size 1 --ep-size $EP_SIZE \ --reasoning-parser qwen3 \ --tool-call-parser qwen3_coder \ --mamba-scheduler-strategy no_buffer \ diff --git a/benchmarks/single_node/qwen3.5_fp4_b300_mtp.sh b/benchmarks/single_node/qwen3.5_fp4_b300_mtp.sh index 9cb5d5464..cc957a623 100755 --- a/benchmarks/single_node/qwen3.5_fp4_b300_mtp.sh +++ b/benchmarks/single_node/qwen3.5_fp4_b300_mtp.sh @@ -15,13 +15,24 @@ check_env_vars \ RESULT_FILENAME \ EP_SIZE +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi nvidia-smi -if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi export NCCL_NVLS_ENABLE=1 export SGL_ENABLE_JIT_DEEPGEMM=false @@ -61,9 +72,9 @@ echo "SCHEDULER_RECV_INTERVAL: $SCHEDULER_RECV_INTERVAL, CONC: $CONC, ISL: $ISL, start_gpu_monitor set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ +PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ --trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 --ep-size $EP_SIZE \ +--tensor-parallel-size $TP --data-parallel-size 1 --ep-size $EP_SIZE \ --reasoning-parser qwen3 \ --tool-call-parser qwen3_coder \ --mamba-scheduler-strategy no_buffer \ diff --git a/benchmarks/single_node/qwen3.5_fp8_b300.sh b/benchmarks/single_node/qwen3.5_fp8_b300.sh index cbceb6f1b..45c0900eb 100644 --- a/benchmarks/single_node/qwen3.5_fp8_b300.sh +++ b/benchmarks/single_node/qwen3.5_fp8_b300.sh @@ -12,6 +12,18 @@ check_env_vars \ RESULT_FILENAME \ EP_SIZE +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi @@ -31,9 +43,9 @@ fi start_gpu_monitor set -x -PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ +PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ --trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 --expert-parallel-size=$EP_SIZE \ +--tensor-parallel-size $TP --data-parallel-size 1 --expert-parallel-size $EP_SIZE \ --enable-symm-mem \ --disable-radix-cache \ --quantization fp8 \ @@ -50,7 +62,7 @@ PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0. --stream-interval 50 \ --scheduler-recv-interval 10 \ --tokenizer-worker-num 6 \ ---tokenizer-path $MODEL \ +--tokenizer-path $MODEL_PATH \ --context-length $CONTEXT_LENGTH > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/benchmarks/single_node/qwen3.5_fp8_b300_mtp.sh b/benchmarks/single_node/qwen3.5_fp8_b300_mtp.sh index ca3b87120..db3b0f03c 100644 --- a/benchmarks/single_node/qwen3.5_fp8_b300_mtp.sh +++ b/benchmarks/single_node/qwen3.5_fp8_b300_mtp.sh @@ -12,6 +12,18 @@ check_env_vars \ RESULT_FILENAME \ EP_SIZE +# `hf download` creates the target dir if missing and is itself idempotent. +# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE +# Either way, MODEL_PATH is what the server is launched with. +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi @@ -31,9 +43,9 @@ fi start_gpu_monitor set -x -SGLANG_ENABLE_SPEC_V2=1 PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0.0.0.0 --port=$PORT \ +SGLANG_ENABLE_SPEC_V2=1 PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ --trust-remote-code \ ---tensor-parallel-size=$TP --data-parallel-size=1 --expert-parallel-size=$EP_SIZE \ +--tensor-parallel-size $TP --data-parallel-size 1 --expert-parallel-size $EP_SIZE \ --enable-symm-mem \ --disable-radix-cache \ --quantization fp8 \ @@ -50,7 +62,7 @@ SGLANG_ENABLE_SPEC_V2=1 PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --mod --stream-interval 50 \ --scheduler-recv-interval 10 \ --tokenizer-worker-num 6 \ ---tokenizer-path $MODEL \ +--tokenizer-path $MODEL_PATH \ --speculative-algorithm EAGLE \ --speculative-num-steps 3 \ --speculative-eagle-topk 1 \ diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index cca8b4ab0..7b4a712f9 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -279,17 +279,48 @@ done find . -name '.nfs*' -delete 2>/dev/null || true else + # HF_HUB_CACHE is set to help with dataset download inside the container + # for eval jobs. Can be updated to some other path on the cluster and + # mounted just like HF_HUB_CACHE_MOUNT. + export HF_HUB_CACHE="$HOME/.cache/huggingface" + + # HF_HUB_CACHE_MOUNT is read-only and holds the pre-staged weights below. + # WRITABLE_MODELS_DIR is writable; the benchmark script downloads anything not + # in the staged list there. + HF_HUB_CACHE_MOUNT="/scratch/models/" + WRITABLE_MODELS_DIR="/data/models/" + + # Pre-staged model + STAGED_MODELS=( + DeepSeek-R1-0528 + DeepSeek-R1-0528-NVFP4-v2 + DeepSeek-V4-Flash + DeepSeek-V4-Pro + GLM-5-FP8 + GLM-5-NVFP4 + GLM-5.1 + Kimi-K2.5 + Kimi-K2.5-NVFP4 + Kimi-K2.6 + MiniMax-M2.5 + MiniMax-M2.5-NVFP4 + MiniMax-M2.7 + MiniMax-M2.7-NVFP4 + Qwen3.5-397B-A17B + Qwen3.5-397B-A17B-FP8 + Qwen3.5-397B-A17B-NVFP4 + gpt-oss-120b + ) - # Pre-staged models on the B300 cluster live under /data/models. Point MODEL - # at the local copy so the benchmark skips `hf download` and reads from the - # mounted dir. Other models fall through and use `hf download` from their - # benchmark script. - HF_HUB_CACHE_MOUNT="/data/models" - if [[ "$MODEL" == "Qwen/Qwen3.5-397B-A17B-FP8" ]]; then - export MODEL="$HF_HUB_CACHE_MOUNT/${MODEL#*/}" - elif [[ "$MODEL_PREFIX" == "dsv4" ]]; then - export MODEL="$HF_HUB_CACHE_MOUNT/dsv4-pro" + # MODEL stays as the HF id for the client (--served-model-name, tokenizer); + # MODEL_PATH is what the server reads weights from. + MODEL_BASENAME="${MODEL##*/}" + if [[ " ${STAGED_MODELS[*]} " == *" ${MODEL_BASENAME} "* ]]; then + export MODEL_PATH="${HF_HUB_CACHE_MOUNT%/}/${MODEL_BASENAME}" + else + export MODEL_PATH="${WRITABLE_MODELS_DIR%/}/${MODEL_BASENAME}" fi + SQUASH_FILE="/data/home/sa-shared/gharunners/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '') # Prefer a framework-tagged script (e.g. dsv4_fp4_b300_sglang.sh) so models @@ -337,7 +368,7 @@ else srun --jobid=$JOB_ID \ --mpi=none \ --container-image=$SQUASH_FILE \ - --container-mounts=$GITHUB_WORKSPACE:$CONTAINER_MOUNT_DIR,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE_MOUNT \ + --container-mounts=$GITHUB_WORKSPACE:$CONTAINER_MOUNT_DIR,$HF_HUB_CACHE_MOUNT:$HF_HUB_CACHE_MOUNT,$WRITABLE_MODELS_DIR:$WRITABLE_MODELS_DIR \ --no-container-mount-home \ --container-workdir=$CONTAINER_MOUNT_DIR \ --no-container-entrypoint --export=ALL,PORT=8888 \