From 7648b2f856ec489b313763d7abf1e76128fc8a90 Mon Sep 17 00:00:00 2001 From: apinge Date: Wed, 26 Aug 2026 09:54:40 +0000 Subject: [PATCH] add 122b scripts --- ...h_qwen3.5-122B_tp8_disable_prefix_cache.sh | 43 +++++++++++++++++ ...bf16_tp1_Dflash_radix_cache_mfs0.7_bs32.sh | 47 +++++++++++++++++++ ...-fp8_tp1_Dflash_radix_cache_mfs0.7_bs32.sh | 44 +++++++++++++++++ 3 files changed, 134 insertions(+) create mode 100644 evaluation/launch_qwen3.5-122B_tp8_disable_prefix_cache.sh create mode 100644 evaluation/launch_qwen3.5-35B-bf16_tp1_Dflash_radix_cache_mfs0.7_bs32.sh create mode 100644 evaluation/launch_qwen3.5-35B-fp8_tp1_Dflash_radix_cache_mfs0.7_bs32.sh diff --git a/evaluation/launch_qwen3.5-122B_tp8_disable_prefix_cache.sh b/evaluation/launch_qwen3.5-122B_tp8_disable_prefix_cache.sh new file mode 100644 index 000000000000..df6dfced8b8f --- /dev/null +++ b/evaluation/launch_qwen3.5-122B_tp8_disable_prefix_cache.sh @@ -0,0 +1,43 @@ +export SGLANG_DISABLE_CUDNN_CHECK=1 +export SGLANG_USE_CUDA_IPC_TRANSPORT=1 +export SGLANG_VLM_CACHE_SIZE_MB=8192 #阿里用0 + +export SGLANG_USE_AITER=1 +export SGLANG_ROCM_USE_AITER_LINEAR_SHUFFLE=1 +export SGLANG_ROCM_USE_AITER_LINEAR_FP8HIPB=1 +#export AITER_QUICK_REDUCE_QUANTIZATION=INT6 #disable for 122B +export USE_AITER_COMM=1 + +export SGLANG_USE_AITER_NEW_CA=false +export SGLANG_USE_IPC_POOL_HANDLE_CACHE=1 +export HIP_GDN_SORT_IDX_BS=32768 + + +export TVM_FFI_DISABLE_TORCH_C_DLPACK=1 # pip uninstall torch-c-dlpack-ext + +# --watchdog-timeout 1200 for profile +model=/models/Qwen/Qwen3.5-122B-A10B/ +python3 -m sglang.launch_server \ + --port 7080 \ + --model-path ${model} \ + --tp-size 8 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder \ + --enable-multimodal \ + --trust-remote-code \ + --chunked-prefill-size 32768 \ + --mem-fraction-static 0.9 \ + --max-prefill-tokens 32768 \ + --max-running-requests 128 \ + --attention-backend aiter \ + --mm-attention-backend aiter_attn \ + --kv-cache-dtype fp8_e4m3 \ + --cuda-graph-max-bs 128 \ + --linear-attn-backend aiter \ + --linear-attn-decode-backend aiter \ + --linear-attn-prefill-backend aiter \ + --watchdog-timeout 1200 \ + --disable-radix-cache 2>&1 | tee launch_qwen3.5-122B-bf16_tp8_disable_prefix_cache.sh.log + # --mamba-scheduler-strategy extra_buffer \ + # --page-size 64 2>&1 | tee launch_qwen3.5-397B-fp8_tp8_prefix_cache_origin_ali2.sh.log + \ No newline at end of file diff --git a/evaluation/launch_qwen3.5-35B-bf16_tp1_Dflash_radix_cache_mfs0.7_bs32.sh b/evaluation/launch_qwen3.5-35B-bf16_tp1_Dflash_radix_cache_mfs0.7_bs32.sh new file mode 100644 index 000000000000..5d2a000ff9eb --- /dev/null +++ b/evaluation/launch_qwen3.5-35B-bf16_tp1_Dflash_radix_cache_mfs0.7_bs32.sh @@ -0,0 +1,47 @@ +export CUDA_VISIBLE_DEVICES=4,5,6,7 +export HIP_VISIBLE_DEVICES=4,5,6,7 + +export SGLANG_DISABLE_CUDNN_CHECK=1 +export SGLANG_USE_CUDA_IPC_TRANSPORT=1 +export SGLANG_VLM_CACHE_SIZE_MB=8192 #阿里用0 +export SGLANG_USE_AITER=1 +export SGLANG_ROCM_USE_AITER_LINEAR_SHUFFLE=1 # internal branch +export SGLANG_ROCM_USE_AITER_LINEAR_FP8HIPB=1 +export USE_AITER_COMM=1 +export AITER_QUICK_REDUCE_QUANTIZATION=INT6 + +export SGLANG_USE_AITER_NEW_CA=false +export SGLANG_USE_IPC_POOL_HANDLE_CACHE=1 + +export HIP_GDN_SORT_IDX_BS=32768 +export TVM_FFI_DISABLE_TORCH_C_DLPACK=1 # pip uninstall torch-c-dlpack-ext + +model=/models/Qwen/Qwen3.5-35B-A3B +model1=/models/Qwen/Qwen3.5-35B-A3B-DFlash + +python3 -m sglang.launch_server \ + --port 10080 \ + --model-path ${model} \ + --tp-size 1 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder \ + --speculative-algorithm DFLASH \ + --speculative-draft-model-path ${model1} \ + --speculative-num-draft-tokens 16 \ + --speculative-draft-attention-backend triton \ + --enable-multimodal \ + --trust-remote-code \ + --chunked-prefill-size 32768 \ + --mem-fraction-static 0.7 \ + --max-prefill-tokens 32768 \ + --max-running-requests 32 \ + --cuda-graph-max-bs 32 \ + --attention-backend aiter \ + --mm-attention-backend aiter_attn \ + --linear-attn-backend aiter \ + --linear-attn-decode-backend aiter \ + --linear-attn-prefill-backend aiter \ + --kv-cache-dtype fp8_e4m3 \ + --watchdog-timeout 1200 \ + --mamba-scheduler-strategy extra_buffer \ + --page-size 64 2>&1 | tee qwen3.5-35B-bf16_tp1_dflash_radix_cache.log diff --git a/evaluation/launch_qwen3.5-35B-fp8_tp1_Dflash_radix_cache_mfs0.7_bs32.sh b/evaluation/launch_qwen3.5-35B-fp8_tp1_Dflash_radix_cache_mfs0.7_bs32.sh new file mode 100644 index 000000000000..cae0edaa8aff --- /dev/null +++ b/evaluation/launch_qwen3.5-35B-fp8_tp1_Dflash_radix_cache_mfs0.7_bs32.sh @@ -0,0 +1,44 @@ +export SGLANG_DISABLE_CUDNN_CHECK=1 +export SGLANG_USE_CUDA_IPC_TRANSPORT=1 +export SGLANG_VLM_CACHE_SIZE_MB=8192 #阿里用0 +export SGLANG_USE_AITER=1 +export SGLANG_ROCM_USE_AITER_LINEAR_SHUFFLE=1 # internal branch +export SGLANG_ROCM_USE_AITER_LINEAR_FP8HIPB=1 +export USE_AITER_COMM=1 +export AITER_QUICK_REDUCE_QUANTIZATION=INT6 + +export SGLANG_USE_AITER_NEW_CA=false +export SGLANG_USE_IPC_POOL_HANDLE_CACHE=1 + +export HIP_GDN_SORT_IDX_BS=32768 +export TVM_FFI_DISABLE_TORCH_C_DLPACK=1 # pip uninstall torch-c-dlpack-ext + +model=/models/Qwen/Qwen3.5-35B-A3B-PTPC-FP8/ +model1=/models/Qwen/Qwen3.5-35B-A3B-DFlash + +python3 -m sglang.launch_server \ + --port 7080 \ + --model-path ${model} \ + --tp-size 1 \ + --reasoning-parser qwen3 \ + --tool-call-parser qwen3_coder \ + --speculative-algorithm DFLASH \ + --speculative-draft-model-path ${model1} \ + --speculative-num-draft-tokens 16 \ + --speculative-draft-attention-backend triton \ + --enable-multimodal \ + --trust-remote-code \ + --chunked-prefill-size 32768 \ + --mem-fraction-static 0.7 \ + --max-prefill-tokens 32768 \ + --max-running-requests 32 \ + --cuda-graph-max-bs 32 \ + --attention-backend aiter \ + --mm-attention-backend aiter_attn \ + --linear-attn-backend aiter \ + --linear-attn-decode-backend aiter \ + --linear-attn-prefill-backend aiter \ + --kv-cache-dtype fp8_e4m3 \ + --mamba-scheduler-strategy extra_buffer \ + --watchdog-timeout 1200 \ + --page-size 64 2>&1 | tee qwen3.5-35B-fp8_tp1_dflash_radix_cache.log