diff --git a/components/MoeExpertRoutingAnimation.jsx b/components/MoeExpertRoutingAnimation.jsx new file mode 100644 index 000000000..2c6424166 --- /dev/null +++ b/components/MoeExpertRoutingAnimation.jsx @@ -0,0 +1,290 @@ +const gpus = [0, 1, 2, 3]; +const PER_GPU = 32; + +export default function MoeExpertRoutingAnimation() { + return ( +
+ +
+

Expert routing animation

+

+ Why a 235B model only does 22B of work per token +

+

+ Every layer of this model has 128 small expert networks, and a tiny router picks just 8 of + them for each token. The other 120 sit still. That is the whole trick of a mixture of + experts: you pay for 235B parameters in memory, but only about 22B of arithmetic per token. +

+ +
+
+ one token arrives + it has already been through attention for this layer +
+ +
router scores all 128 experts, keeps the top 8
+ +
+ {gpus.map((gpu) => { + // 2 of this GPU's 32 experts are picked, so 8 across 4 GPUs + const hot = [3 + gpu, 18 + ((gpu * 5) % 10)]; + return ( +
+

+ GPU {gpu} + experts {gpu * PER_GPU}-{gpu * PER_GPU + PER_GPU - 1} +

+ + ); + })} +
+ +
+
+ In memory + 235B params + + All 128 experts per layer must be resident, which is why the model is big + +
+
+ Active per token + 22B params + + Only the 8 chosen experts do arithmetic, so it runs like a much smaller model + +
+
+ What this costs you + a network hop + + With expert parallelism the token travels to whichever GPU owns its expert, then the + answer travels back + +
+
+
+
+
+ Counts are from the model config: 128 experts per layer, 8 per token, 94 layers. Splitting 128 + experts over 4 GPUs gives 32 each, so on average 2 experts per GPU fire for any given token. + That average is the catch, because routing is not guaranteed to be even. +
+
+ ); +} diff --git a/components/MultiGpuMemoryFitAnimation.jsx b/components/MultiGpuMemoryFitAnimation.jsx new file mode 100644 index 000000000..54032e176 --- /dev/null +++ b/components/MultiGpuMemoryFitAnimation.jsx @@ -0,0 +1,390 @@ +const cases = [ + { + key: 'one', + verdict: 'fail', + title: '1 GPU', + flag: 'will not start', + note: '221 GiB of weights against an 85.51 GiB budget', + parts: [{ label: 'weights', value: '221 GiB', width: '92.08%', color: '#ef4444' }], + log: 'the model is 2.6x larger than the whole budget\nthere is no flag that fixes this', + }, + { + key: 'two', + verdict: 'fail', + title: '2 GPUs', + flag: 'CUDA out of memory', + note: 'about 110 GiB per card, still too much', + parts: [{ label: 'weights per card', value: '110 GiB', width: '46.04%', color: '#f59e0b' }], + log: 'Failed to load model - not enough GPU memory\n95.01 GiB total, of which 438.31 MiB is free', + }, + { + key: 'four', + verdict: 'pass', + title: '4 GPUs', + flag: '621,392 tokens', + note: 'weights fit, with room for about 19 concurrent 32k conversations', + parts: [ + { label: 'weights', value: '55.19 GiB', width: '23.00%', color: '#0098cc' }, + { label: 'KV cache', value: '27.85 GiB', width: '11.60%', color: '#2bb534' }, + ], + log: 'Worker_TP0 Model loading took 55.19 GiB\nAvailable KV cache memory: 27.85 GiB\nGPU KV cache size: 621,392 tokens', + }, +]; + +export default function MultiGpuMemoryFitAnimation() { + return ( +
+ +
+

Memory fit animation

+

+ The same model on 1, 2 and 4 GPUs +

+

+ Every number here came out of a real run. All three bars are drawn to the same scale, and + the dashed line is the 85.51 GiB that vLLM may use on one card at + --gpu-memory-utilization 0.90. A bar reaching past that line means the model does not fit. + Watch it shrink as GPUs are added, and note that it takes 4 before the bar finally lands to + the left of the line. +

+ +
+ {cases.map((c) => ( +
+
+ + {c.title} + {c.note} + + {c.flag} +
+ +
+ what one card must hold + full axis = 240 GiB +
+ +
+ +
+ {c.parts.map((p, i) => ( +
+ + {p.label} {p.value} + +
+ ))} +
+
+ +
{c.log}
+
+ ))} + +
+
+ KV per token, whole model + 188 KiB + 2 x 94 layers x 4 kv heads x 128 head_dim x 2 bytes +
+
+ Per card at TP=4 + 47 KiB + + each card keeps 1 of the 4 kv heads, so the cache divides rather than repeats + +
+
+ Predicted vs reported + 621,337 / 621,392 + + 27.85 GiB divided by 47 KiB, against what vLLM actually printed + +
+
+
+
+
+ Measured on 4x RTX PRO 6000 Blackwell with Qwen3-235B-A22B-Instruct-2507-FP8 on vLLM 0.27.1. + The 1 GPU and 2 GPU bars are what the run actually attempted before failing, not estimates. + Because this model has only 4 key/value heads, its cache is unusually cheap, which is why 4 + cards leave room for about 19 concurrent conversations at the 32,768-token limit we set. +
+
+ ); +} diff --git a/components/MultiGpuSplitModesAnimation.jsx b/components/MultiGpuSplitModesAnimation.jsx new file mode 100644 index 000000000..e58e6c224 --- /dev/null +++ b/components/MultiGpuSplitModesAnimation.jsx @@ -0,0 +1,279 @@ +const modes = [ + { + key: 'tp', + name: 'Tensor parallelism', + flag: '--tensor-parallel-size', + plain: 'Cut every layer into vertical strips. Each GPU holds a strip of all 94 layers.', + talks: 'A lot. Twice per layer, so 188 times per token.', + good: 'Fastest for a single user, because all 4 GPUs work on the same token.', + color: '#0098cc', + }, + { + key: 'pp', + name: 'Pipeline parallelism', + flag: '--pipeline-parallel-size', + plain: 'Cut the stack into horizontal blocks. With 94 layers over 4 GPUs, each one owns about 23 of them.', + talks: 'Barely. One handoff between neighbours per token.', + good: 'Kind to a slow network between GPUs, but a GPU waits its turn.', + color: '#2bb534', + }, + { + key: 'ep', + name: 'Expert parallelism', + flag: '--enable-expert-parallel', + plain: 'Deal the 128 experts out like cards. Each GPU keeps 32 of them, whole.', + talks: 'Medium. Tokens are shipped to whichever GPU owns the expert they need.', + good: 'Only exists for MoE models, and it is how the really big ones are served.', + color: '#a855f7', + }, +]; + +export default function MultiGpuSplitModesAnimation() { + return ( +
+ +
+

Three ways to split animation

+

+ The same model, cut three different ways across four GPUs +

+

+ These are not competing products, they are three different cuts through the same pile of + weights, and you can combine them. Each box below is one GPU. Watch which parts light up, + because that tells you which GPUs are doing work at the same moment. +

+ +
+ {modes.map((mode) => ( +
+

{mode.name}

+ {mode.flag} + + + +
+

+ What it does + {mode.plain} +

+

+ How much it talks + {mode.talks} +

+

+ When it wins + {mode.good} +

+
+
+ ))} +
+
+
+ Layer and expert counts are Qwen3-235B-A22B: 94 layers, 128 experts with 8 picked per token. + Under tensor parallelism all four GPUs light up together on every token. Under pipeline + parallelism they light up in turn, which is the idle time you are trading away. +
+
+ ); +} diff --git a/components/MultiGpuTensorSplitAnimation.jsx b/components/MultiGpuTensorSplitAnimation.jsx new file mode 100644 index 000000000..2e9bbe573 --- /dev/null +++ b/components/MultiGpuTensorSplitAnimation.jsx @@ -0,0 +1,374 @@ +const steps = [ + { label: 'A token arrives', detail: 'all 4 GPUs get the same copy of it' }, + { label: 'Split sideways', detail: 'each GPU owns 16 of the 64 attention heads' }, + { label: 'Work alone', detail: 'no GPU needs to ask the others anything yet' }, + { label: 'Partial answers', detail: 'each GPU has a quarter of the answer' }, + { label: 'Add them up', detail: 'one all-reduce, and all 4 hold the full result' }, +]; + +export default function MultiGpuTensorSplitAnimation() { + return ( +
+ +
+

Tensor parallelism animation

+

+ One layer, sliced four ways +

+

+ This is the part people usually get wrong, so it is worth being precise. The weights get + divided, and the thing flowing through them does not. Every GPU starts each layer holding + an identical copy of the token, does a quarter of the arithmetic on its own slice of the + weights, and ends up with a quarter of an answer. Then they add their quarters together. +

+ +
+
+ the token, 4096 numbers wide + copied to all four GPUs, not divided +
+ +
+ {[0, 1, 2, 3].map((gpu) => ( +
+

+ GPU {gpu} + heads {gpu * 16}-{gpu * 16 + 15} +

+ + ))} +
+ +
+ + +
+ the finished layer output, now identical on all four GPUs + and the next layer does the whole dance again +
+
+ +
+ {steps.map((step, i) => ( +
+ Step {i + 1} + {step.label} + {step.detail} +
+ ))} +
+
+
+ Shapes are Qwen3-235B-A22B: hidden size 4096, 64 attention heads, 4 key/value heads, 94 + layers. Those 4 key/value heads are the reason this model cannot be split cleanly more than 4 + ways, which we come back to later. +
+
+ ); +} diff --git a/content/blog/running-a-big-llm-across-multiple-gpus-with-vllm.md b/content/blog/running-a-big-llm-across-multiple-gpus-with-vllm.md new file mode 100644 index 000000000..644fe69e7 --- /dev/null +++ b/content/blog/running-a-big-llm-across-multiple-gpus-with-vllm.md @@ -0,0 +1,778 @@ +--- +title: "Running a big LLM across multiple GPUs with vLLM" +seoTitle: "Running a big LLM across multiple GPUs with vLLM" +seoDescription: "A plain-English guide to serving a model too big for one GPU, in two tracks: a runbook from download to serving with every flag and error explained, and a deep dive into how tensor, pipeline, and expert parallelism split the model, with measured numbers from a 235B model on four RTX PRO 6000 cards." +datePublished: 2026-08-18T10:00:00.000Z +slug: running-a-big-llm-across-multiple-gpus-with-vllm +author: shubham-katara +authors: ["shubham-katara", "saiyam-pathak"] +cover: /img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.png +tags: ["vllm", "gpu", "nvidia", "llm", "platform-engineering"] +--- + +Sooner or later everyone running models locally hits the same wall. You find a model you want, you look at the download size, and it is bigger than the GPU you own. A 235B model needs roughly 236 GB just for its weights. The card we have holds 96 GB, and even the largest data-centre GPUs available today top out well below 236 GB. So the model does not fit, and no amount of clever flags will make 236 GB squeeze into 96 GB. + +The answer is to use more than one GPU. That part everybody knows. The part that is genuinely confusing is what "use more than one GPU" actually means. Does each GPU get a copy of the model? Does the model get cut in half? Do the GPUs take turns? Which of those is happening, and what does it cost you? + +Let's answer that properly, with a real model on real hardware. And let's be honest that not everyone is here for the same reason. + +## How to read this post + +This post is split into two tracks: a runbook for getting a big model serving, and a deep dive explaining how multi-GPU model splitting actually works. The runbook is for readers who need the commands and configs fast. + +The deep dive is for those who want to understand the mechanics, tradeoffs, and numbers. Jump to the track that fits your need, or read both: the post is structured so each section clearly points to the other right when extra context is helpful. + +So: one post, two tracks, each with a clear exit. Pick your entrance based on the job in front of you: + +| You are | You want | Read | +| ------------------------------------------------------------------- | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Platform engineer, SRE, MLOps**: you have the GPUs and a deadline | The model serving today | **The runbook, Steps 1-8** (~20 min). Every command, flag, log line and error. Each step links into the deep dive at exactly the point a "why" earns its keep; follow those links only when something surprises you. | +| **ML engineer, or just curious**: no root access required | The mental model | **The deep dive, sections 1-7** (~18 min). How the splitting actually works, and measured proof of when each method wins. Jump [straight there](#the-deep-dive-what-splitting-actually-means). | +| **Both** | Everything | Read straight through. The runbook comes first because you cannot benchmark a server that is not running. | + +New to the jargon? Every term, flag, and benchmark number here is explained in plain English in the [local LLM glossary](https://blog.kubesimplify.com/local-llm-glossary). + +## The machine and the model + +Both tracks lean on this section, so here it is once. Numbers mean nothing without the hardware attached. + +**The machine:** a server with 8x NVIDIA RTX PRO 6000 Blackwell Server Edition cards. Each card has 96 GB of memory, and the machine reports 95.01 GiB of that as usable. We borrowed 4 of the 8 cards for this work. + +One detail that matters more than it looks: these GPUs are **not** connected by NVLink. NVLink is NVIDIA's fast direct GPU-to-GPU cable. Without it, GPUs talk to each other over PCIe and through the CPU, which is slower. You can check what you have with one command: + +```bash +root@utho-gpu-rtxpro6000-8-62383:~# nvidia-smi topo -m + +| Device | GPU0 | GPU1 | GPU2 | GPU3 | GPU4 | GPU5 | GPU6 | GPU7 | NIC0 | CPU Affinity | NUMA Affinity | GPU NUMA ID | +| :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :--- | :---: | :---: | +| **GPU0** | **X** | SYS | SYS | SYS | SYS | SYS | SYS | SYS | SYS | 48-55,176-183 | 6 | N/A | +| **GPU1** | SYS | **X** | SYS | SYS | SYS | SYS | SYS | SYS | PHB | 32-39,160-167 | 4 | N/A | +| **GPU2** | SYS | SYS | **X** | SYS | SYS | SYS | SYS | SYS | SYS | 0-7,128-135 | 0 | N/A | +| **GPU3** | SYS | SYS | SYS | **X** | SYS | SYS | SYS | SYS | SYS | 16-23,144-151 | 2 | N/A | +| **GPU4** | SYS | SYS | SYS | SYS | **X** | SYS | SYS | SYS | SYS | 112-119,240-247 | 14 | N/A | +| **GPU5** | SYS | SYS | SYS | SYS | SYS | **X** | SYS | SYS | SYS | 96-103,224-231 | 12 | N/A | +| **GPU6** | SYS | SYS | SYS | SYS | SYS | SYS | **X** | SYS | SYS | 64-71,192-199 | 8 | N/A | +| **GPU7** | SYS | SYS | SYS | SYS | SYS | SYS | SYS | **X** | SYS | 80-87,208-215 | 10 | N/A | +| **NIC0** | SYS | PHB | SYS | SYS | SYS | SYS | SYS | SYS | **X** | | | | + +**Legend:** + +| Symbol | Description | +| :--- | :--- | +| **X** | Self | +| **SYS** | Connection traversing PCIe as well as the SMP interconnect between NUMA nodes (e.g., QPI/UPI) | +| **NODE** | Connection traversing PCIe as well as the interconnect between PCIe Host Bridges within a NUMA node | +| **PHB** | Connection traversing PCIe as well as a PCIe Host Bridge (typically the CPU) | +| **PXB** | Connection traversing multiple PCIe bridges (without traversing the PCIe Host Bridge) | +| **PIX** | Connection traversing at most a single PCIe bridge | +| **NV#** | Connection traversing a bonded set of `#` NVLinks | +| **NIC0** | `mlx4_0` | +``` + +On our machine every pair of GPUs reports `SYS`, which means the traffic goes across PCIe and then across the link between the CPU sockets. If you had NVLink you would see `NV1`, `NV2` and so on instead. Keep this in mind, because it changes which splitting method is fastest. + +**The model:** `Qwen/Qwen3-235B-A22B-Instruct-2507-FP8`. Let's unpack that name, because it is doing a lot of work: + +- **235B** is the total parameter count, 235 billion. +- **A22B** means 22 billion **active** parameters. This is a mixture-of-experts model: each layer holds 128 small expert networks and a router picks just 8 of them per token, so you pay for 235B in memory but only about 22B in arithmetic. [Deep dive 5 tells the full story.](#deep-dive-5-the-expert-part) +- **FP8** is the number format the weights are stored in, 8 bits each, so one byte per parameter. + +**The software:** vLLM 0.27.1 running in the official container, with PyTorch 2.13.0 and CUDA 13.0, on driver 610.43.02. + +--- + +## The runbook: from download to serving + +Written for the person with root on the box. Eight steps, and at the end of them a 235B model is answering requests on four GPUs. No prior knowledge of distributed computing is assumed: if you know what a GPU is and you have run a model locally once, you are qualified. + +## Step 1: Getting the model onto the machine + +Before anything can be split across GPUs it has to be on the disk, and with a model this size that step is not a formality. It is the step that bit us hardest, so let's do it properly. + +You download it with the Hugging Face CLI: + +```bash +root@utho-gpu-rtxpro6000-8-62383:~# pip install huggingface_hub hf_transfer +root@utho-gpu-rtxpro6000-8-62383:~# HF_XET_HIGH_PERFORMANCE=1 hf download Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 +Downloading bytes: ████████████████████████████████████████████████▏ | 24.4GB, 234MB/s +Reconstructing (incomplete total...): 13%|███████████████▋ | 10.0GB / 80.0GB, 104MB/s +Fetching 34 files: 0%| | 0/34 [00:00 https://blog.kubesimplify.com/ - 2026-08-18T08:33:37.156Z + 2026-08-18T11:45:29.744Z Kubesimplify hello@kubesimplify.com + + Running a big LLM across multiple GPUs with vLLM + + https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm + 2026-08-18T10:00:00.000Z + 2026-08-18T10:00:00.000Z + A plain-English guide to serving a model that is too big for one GPU: how tensor, pipeline, and expert parallelism split it up, what every vLLM flag does, and measured numbers from a 235B model on four RTX PRO 6000 cards. + + + + + + The Local LLM Glossary: Every Term, Flag, and Number in Plain English diff --git a/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.png b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.png new file mode 100644 index 000000000..6ed8b13f6 Binary files /dev/null and b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.png differ diff --git a/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.svg b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.svg new file mode 100644 index 000000000..9c6a3d465 --- /dev/null +++ b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.svg @@ -0,0 +1,55 @@ + + + +One big model, four GPUs +how a 235B model is cut up so it fits, and what that costs + +ONE CARD + + + +95 GiB +usable + + + +236 GB of weights +2.3x too big +no flag fixes this + +FOUR CARDS, --tensor-parallel-size 4 + + + +GPU 0 +59 GB +weights +16 of 64 heads + + + +GPU 1 +59 GB +weights +16 of 64 heads + + + +GPU 2 +59 GB +weights +16 of 64 heads + + + +GPU 3 +59 GB +weights +16 of 64 heads + +188 all-reduces per token + +QWEN3-235B-A22B FP8 - 128 EXPERTS, 8 PER TOKEN - vLLM 0.27.1 +tensor, pipeline and expert parallelism explained in plain english +blog.kubesimplify.com + \ No newline at end of file diff --git a/public/llms-full.txt b/public/llms-full.txt index df6842053..150745f31 100644 --- a/public/llms-full.txt +++ b/public/llms-full.txt @@ -5,6 +5,608 @@ --- +# Running a big LLM across multiple GPUs with vLLM + +- Canonical: https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm +- Published: 2026-08-18 +- Summary: A plain-English guide to serving a model that is too big for one GPU: how tensor, pipeline, and expert parallelism split it up, what every vLLM flag does, and measured numbers from a 235B model on four RTX PRO 6000 cards. + +Sooner or later everyone running models locally hits the same wall. You find a model you want, you look at the download size, and it is bigger than the GPU you own. A 235B model needs roughly 236 GB just for its weights. The card we have holds 96 GB, and even the largest data-centre GPUs available today top out well below 236 GB. So the model does not fit, and no amount of clever flags will make 236 GB squeeze into 96 GB. + +The answer is to use more than one GPU. That part everybody knows. The part that is genuinely confusing is what "use more than one GPU" actually means. Does each GPU get a copy of the model? Does the model get cut in half? Do the GPUs take turns? Which of those is happening, and what does it cost you? + +Let's answer that properly, with a real model on real hardware, and let's explain every single flag and command along the way rather than pasting a magic incantation and moving on. + +## What you will learn + +- How to download a 236 GB model, what the 24 files you get actually are, and how they sit on disk +- How to work out on paper whether it fits on your GPUs, before you spend an hour downloading it +- What inference really is: the two completely different phases behind "time to first token" and "tokens per second" +- The three different ways a model can be split across GPUs, in plain English, and when each is used +- What every flag in our vLLM command does, and why it has the value it has +- How to read the startup log, which tells you more than any tutorial can +- The rules that limit how far you can split, and the real errors you get when you break them +- Measured numbers for all three splitting modes on the same model and the same four GPUs + +No prior knowledge of distributed computing is assumed. If you know what a GPU is and you have run a model locally once, you are qualified. + +## The machine and the model + +Here is what we tested on, because numbers mean nothing without the hardware attached. + +**The machine:** a server with 8x NVIDIA RTX PRO 6000 Blackwell Server Edition cards. Each card has 96 GB of memory, and the machine reports 95.01 GiB of that as usable. We borrowed 4 of the 8 cards for this work. + +One detail that matters more than it looks: these GPUs are **not** connected by NVLink. NVLink is NVIDIA's fast direct GPU-to-GPU cable. Without it, GPUs talk to each other over PCIe and through the CPU, which is slower. You can check what you have with one command: + +```bash +nvidia-smi topo -m +``` + +On our machine every pair of GPUs reports `SYS`, which means the traffic goes across PCIe and then across the link between the CPU sockets. If you had NVLink you would see `NV1`, `NV2` and so on instead. Keep this in mind, because it changes which splitting method is fastest. + +**The model:** `Qwen/Qwen3-235B-A22B-Instruct-2507-FP8`. Let's unpack that name, because it is doing a lot of work: + +- **235B** is the total parameter count, 235 billion. +- **A22B** means 22 billion **active** parameters. This is a mixture-of-experts model, and only a fraction of it runs for any given token. More on this shortly, because it is the most interesting thing about serving big models today. +- **FP8** is the number format the weights are stored in, 8 bits each, so one byte per parameter. + +**The software:** vLLM 0.27.1 running in the official container, with PyTorch 2.13.0 and CUDA 13.0, on driver 610.43.02. + +## Part 1: Getting the model onto the machine + +Before anything can be split across GPUs it has to be on the disk, and with a model this size that step is not a formality. It is the step that bit us hardest, so let's do it properly. + +You download it with the Hugging Face CLI: + +```bash +pip install huggingface_hub hf_transfer + +HF_HUB_ENABLE_HF_TRANSFER=1 hf download Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 +``` + +`HF_HUB_ENABLE_HF_TRANSFER=1` switches on a Rust downloader that parallelises across connections. On a 236 GB download that is the difference between an hour and most of an afternoon, so it is worth the extra package. + +### What you actually get + +The download is not one giant file. It arrives as **24 shards**, plus the small text files that describe the model: + +``` +config.json +generation_config.json +model-00001-of-00024.safetensors +model-00002-of-00024.safetensors +... +model-00024-of-00024.safetensors +model.safetensors.index.json +tokenizer.json +``` + +A few things worth understanding here: + +- **`.safetensors`** is the modern format for weights. It is a flat file with a small JSON header at the front listing every tensor's name, dtype, shape and byte range, then the raw bytes. That layout matters for us, because it means a loader can memory-map the file and read exactly the byte ranges it wants without parsing the whole thing, and without the security problems of the old pickle-based `.bin` format. +- **`model.safetensors.index.json`** is the map that says which tensor lives in which shard. This is how vLLM knows to open shard 17 to find layer 62's weights. +- **`config.json`** is the architecture file we keep coming back to: layer count, head counts, expert count. It is a few kilobytes and it determines almost every decision in this post. +- For an FP8 model like this one, the weight tensors are joined by **scale tensors**. FP8 has very little numeric range, so the checkpoint stores a scaling factor per 128x128 block of each weight matrix, and the real value is the 8-bit number multiplied by its block's scale. You can see that arrangement declared in `config.json`: + +```json +"quantization_config": { + "quant_method": "fp8", + "fmt": "e4m3", + "weight_block_size": [128, 128], + "activation_scheme": "dynamic" +} +``` + +Remember those block scales. They are the reason for the most annoying crash we hit, back in Part 12. + +### Can you change the number of shards? + +Worth answering because it is a natural question: no, not at download time. The shard layout is decided by whoever uploaded the model and is baked into `model.safetensors.index.json`. `hf download` just fetches the files that exist in the repo, so there is no flag to ask for more or fewer of them. + +You can only re-shard by loading the model yourself and saving it again, which is a local operation on your own copy: + +```python +from transformers import AutoModelForCausalLM + +model = AutoModelForCausalLM.from_pretrained("some/model") +model.save_pretrained("./resharded", max_shard_size="5GB") +``` + +`max_shard_size` is the knob, and in current `transformers` it defaults to `"50GB"`. One caveat straight from its docs, because it surprises people: "If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard which will be bigger than `max_shard_size`." A giant embedding matrix can therefore blow past whatever cap you set. + +For a 235B model this is almost never worth doing, since you would have to load the whole thing to write it back out. Just take the shards you are given. + +### Is there a standard shard size? + +Not a formal one, but there are firm conventions and real limits. + +**The conventions** are the naming pattern (`model-00001-of-00024.safetensors`) and the index file next to it. Both are produced automatically by the saving code, which is why nearly every model on the Hub looks the same. + +**The limits** come from the Hub. Its guidance is to split large files "into chunks <200GB each", and it states that "500GB is the hard limit for a single file size". The reasoning is practical and worth knowing, because it is the same reasoning that should shape your own thinking about big files: + +- A failed download of a smaller file resumes cheaply. A failed download of one enormous file can mean starting over. +- Files are served through a CDN, and per the Hub's docs "huge files are not cached by this service leading to a slower download speed". So one 236 GB file would genuinely download slower than 24 pieces of it. + +**What publishers actually pick** sits far below those limits. Our model uses a 10 GB cap: 23 shards of exactly 10.00 GB and a 24th holding the remaining 6.45 GB. Somewhere in the 5 to 10 GB range is the common choice across the Hub. + +**Does any of this affect serving?** Essentially no. Shard count does not change how much GPU memory you need or how fast the model runs, because the weights are identical either way and safetensors are memory-mapped, so the loader reads the byte ranges it wants regardless of how they are grouped into files. Shard size is a distribution question, not an inference question. Where it does matter is download throughput and resumability, which is exactly why the convention landed where it did. + +### Where it gets stored + +By default everything lands under `~/.cache/huggingface/hub`, in a layout that looks strange the first time you see it: + +``` +~/.cache/huggingface/hub/models--Qwen--Qwen3-235B-A22B-Instruct-2507-FP8/ +├── blobs/ <- the real files, named by hash +├── refs/ <- which commit "main" points at +└── snapshots/ + └── e156cb4e.../ <- symlinks with friendly names, pointing into blobs/ +``` + +The content lives once in `blobs/` under its hash, and `snapshots/` holds human-readable symlinks into it. That is why pulling two revisions of a model does not always double your disk usage, and it is also why `du` and `df` can disagree with your intuition. + +The practical consequence for serving: mount that whole directory into your container and set `HF_HOME` to it, which is exactly what the `-v` and `-e HF_HOME` flags in Part 8 are doing. Otherwise the container downloads its own copy. + +### The disk trap, which is a real production hazard + +Two things about disk that the model card will not tell you. + +**Each tensor-parallel worker reads the entire checkpoint.** vLLM's own docs say that with tensor parallelism "each process will read the whole model and split it into chunks". So at `-tp 4` the machine performs roughly 4 x 221 GiB of reads at startup, not 221 GiB divided four ways. That is why a big model takes minutes to load even off fast storage, and it is why our first `Model loading took` line reported 45 seconds only because a lot of the file was still in the operating system's page cache from the download. + +**On a shared machine, filling the disk can take down everything else on it.** This is the part we learned the hard way, and it is worth more than a footnote. Our test box also runs a Kubernetes inference platform. Kubernetes treats free disk as a managed resource called ephemeral-storage, and when free space fell below its eviction threshold, the kubelet did exactly what it is designed to do: it evicted pods to reclaim space, tainted the node so nothing new could schedule, and garbage-collected container images. Several of those images had been built locally and existed in no registry, so they could not simply be pulled again. + +Nothing about that is a Kubernetes bug, and nothing about it is specific to our setup. The lesson generalises: **before you download a quarter of a terabyte onto a machine, check what else lives on that disk and what will happen when it fills.** `df -h` before you start, and know your platform's eviction threshold, which is often far higher than "0 bytes free". If the machine is shared, keeping a couple of hundred gigabytes of headroom is not paranoia. + +## Part 2: Why one GPU is not enough + +Let's do the arithmetic, because it is simpler than people expect and it saves you a lot of wasted download time. + +A model is mostly a big pile of numbers called **parameters** or **weights**. To run the model, those numbers have to sit in GPU memory. So the first question is always: how many bytes is one parameter? + +| Format | Bits per parameter | Bytes per parameter | +| --- | --- | --- | +| FP32 | 32 | 4 | +| BF16 or FP16 | 16 | 2 | +| FP8 | 8 | 1 | +| FP4 or NVFP4 | 4 | 0.5 | + +So the weights alone take `number of parameters x bytes per parameter`. For our model that is 235 billion parameters at 1 byte each, which is about 236 GB. Our GPU holds 95.01 GiB. The model is roughly 2.3 times too big for one card. + +But weights are only the first of **three** things that need to fit. This is where most people's mental model is incomplete: + +1. **The weights.** Fixed size. You know it before you start. +2. **The KV cache.** This is the model's memory of the conversation so far. Every token you feed in, and every token the model writes, leaves behind a small record that has to be kept for as long as that request is alive. It grows with how long your prompts are and how many users you serve at once. +3. **Working space.** Temporary scratch memory for the actual calculations, plus some overhead the framework reserves for itself. + +The KV cache is the one that surprises people, so let's size it. The formula looks intimidating but every term is just a number from the model's config file: + +``` +bytes per token = 2 x layers x kv_heads x head_dim x bytes_per_number +``` + +The `2` is because you store two things per token, a key and a value, which is where "KV" comes from. For our model, `layers` is 94, `kv_heads` is 4, `head_dim` is 128, and the cache is kept in BF16 so that is 2 bytes: + +``` +2 x 94 x 4 x 128 x 2 = 192,512 bytes = 188 KiB per token +``` + +188 KiB does not sound like much. But this model supports a 262,144 token context, so one single conversation at full length would need `262,144 x 188 KiB`, which is about **47 GiB**. That is half a GPU for one user. Serving ten users at once with long prompts is where all your leftover memory goes, and it is why "the weights fit, so I am fine" is wrong. + +{{multi-gpu-memory-fit-animation}} + +## Part 3: What actually happens when a request arrives + +Before splitting anything, it helps to know what the work being split actually is, because inference is really two different jobs wearing one coat. Almost everything confusing about multi-GPU performance comes from this split. + +### Phase one: prefill, reading your prompt + +When your prompt arrives, the model has to read all of it. If you send 1,000 tokens, all 1,000 go through every layer **at once**, as one big batch of work. This is called **prefill**, and it is the phase that decides your time to first token. + +Prefill is *compute-heavy*. There is a lot of arithmetic to do and the GPU's matrix engines are the bottleneck. It also produces the keys and values for every one of those 1,000 tokens, which get written into the KV cache and kept. + +### Phase two: decode, writing the answer + +Then the model writes its reply, and here is the part that surprises people: **it can only produce one token at a time.** To write token 2 it needs to have written token 1, because it feeds its own output back in. There is no way around that, it is what "autoregressive" means. + +So decode is a loop. Each pass through it produces exactly one token, reads the entire KV cache built so far, and appends one more entry to that cache. + +Decode is *memory-heavy* rather than compute-heavy. For a single token there is barely any arithmetic to do, but the GPU still has to stream the relevant weights and the whole KV cache past its compute units. The bottleneck is memory bandwidth, not maths. That is why decode speed tracks memory bandwidth so closely, and why giving a single request more GPUs to read from in parallel actually helps. + +Two phases, two different bottlenecks, and they respond differently to everything you tune: + +| | Prefill | Decode | +| --- | --- | --- | +| Work per step | your whole prompt at once | exactly one token | +| Bottleneck | compute | memory bandwidth | +| Metric it drives | time to first token | time per output token | +| Data moved between GPUs | large, whole prompt's worth | tiny, one token's worth | + +That last row is the one to hold on to. It is the reason, later, that pipeline parallelism wins on first-token latency while tensor parallelism wins on tokens per second. The same all-reduce that is trivially cheap during decode is expensive during prefill, because it is carrying a thousand times more data. + +### How the server juggles many users + +A real server is not doing one request at a time. vLLM uses **continuous batching**, which means it does not wait for a batch to fill up or finish. On every step it looks at everything currently in flight and assembles whatever work is ready, so a request that arrives mid-flight joins the very next step rather than queueing behind a whole batch. + +Two consequences worth knowing: + +- **Prefill and decode get mixed together.** A step might carry one user's fresh 1,000-token prompt alongside twenty other users' single decode tokens. That mixing is why a burst of long prompts makes everyone else's tokens arrive more slowly, and it is why `--max-num-batched-tokens` exists as a lever. +- **Capacity is set by the KV cache, not by CPU or queue length.** Every in-flight request is holding cache proportional to its length. When the cache is full, vLLM has to **preempt** somebody: it evicts a request's cache and recomputes it later. That is the real meaning of the `Maximum concurrency` line in the startup log, and it is why we spend so much of this post counting cache bytes. + +Now that the work itself is clear, let's look at the three ways to spread it over more than one GPU. + +## Part 4: The three ways to split a model + +Here is the heart of it. When people say "split the model across GPUs" they could mean three genuinely different things, and mixing them up is the source of most confusion. + +An analogy first, because it makes the rest much easier to hold in your head. Imagine a large restaurant kitchen that has to produce one dish: + +- **Tensor parallelism** is four chefs all working on the same dish at the same time, one chopping, one on sauce, one on protein, one plating. They constantly have to coordinate, but the dish is done fast. +- **Pipeline parallelism** is four chefs at four stations, where the dish moves down the line. Station two cannot start until station one is finished. Very little talking, but three chefs are idle at any moment unless you have several dishes in flight. +- **Expert parallelism** is a kitchen with 128 specialist chefs where each dish only needs 8 of them. You spread those 128 chefs across four rooms, and each dish gets walked to whichever rooms hold the specialists it needs. + +{{multi-gpu-split-modes-animation}} + +All three can be combined, and in production they usually are. Now let's look at each one properly. + +## Part 5: Tensor parallelism, up close + +Tensor parallelism cuts **inside** every layer. This is the important distinction: it does not give GPU 0 the first half of the model and GPU 1 the second half. Every GPU holds a thin slice of **all 94 layers**. + +How can you cut a layer? Because the work a layer does is mostly one big multiplication table, and multiplication tables can be cut up. The technique comes from a 2019 NVIDIA paper called Megatron-LM, and it works in two moves. + +**Move one, cut the first matrix into vertical strips.** Each GPU takes some of the columns. Because each GPU has complete columns, it can finish its part, including the activation function in the middle, without asking anyone anything. In our model the attention block has 64 heads, so with 4 GPUs each one owns 16 whole heads and computes them start to finish alone. + +**Move two, cut the second matrix into horizontal strips.** These line up exactly with the vertical cuts from move one. Each GPU multiplies its slice and gets a **partial answer**, a quarter of the real result. + +Now, and only now, the GPUs have to talk. They add their four partial answers together so that everyone ends up with the complete result. That single operation is called an **all-reduce**: everyone contributes a piece, everyone gets the total back. + +The Megatron paper puts the cost plainly, saying this design lets you run a transformer layer "using only two all-reduces in the forward path". Generating text only uses the forward path, so: + +- 2 all-reduces per layer +- 94 layers +- **188 all-reduces to produce one single token** + +And they happen strictly one after another, because layer 5 cannot begin until layer 4 has finished comparing notes. + +{{multi-gpu-tensor-split-animation}} + +### The KV cache gets divided too, which is a bonus + +Because each GPU owns only some of the attention heads, it only needs to remember keys and values for its own heads. So the KV cache is divided across GPUs rather than duplicated. Four GPUs give you roughly four times the room for conversations, on top of making the weights fit. This is a real and often unmentioned benefit of tensor parallelism. + +## Part 6: The expert part, which is why this model is only 22B of work + +Our model is a **mixture of experts**, and this is the single biggest idea in how large models are served today, so it is worth slowing down for. + +In an ordinary model, every parameter is used for every token. In a mixture-of-experts model, each layer contains many small networks called **experts**, and a tiny component called a **router** decides which few of them each token should visit. Our model has **128 experts per layer** and the router picks **8** of them per token. + +So the model holds 235B parameters in memory, but only about 22B of them do any arithmetic for a given token. That is what "235B-A22B" means, and it is why this model runs far faster than its size suggests. You pay for the full 235B in memory and you pay for only 22B in speed. + +{{moe-expert-routing-animation}} + +This gives you a third way to split. Instead of slicing every expert into strips, you hand out whole experts: with 128 experts and 4 GPUs, each GPU keeps 32 of them intact. That is **expert parallelism**, and in vLLM you switch it on with `--enable-expert-parallel`. + +The trade is different from tensor parallelism. Nothing needs adding up at the end, but tokens have to travel to whichever GPU owns the expert they were routed to, and the answers travel back. It also has a fairness problem: the router does not promise to spread work evenly, so one GPU can end up with more popular experts and become the slow one holding everybody up. + +## Part 7: Every flag, explained + +Before the command, the vocabulary. Here is every flag we use and why it has the value it has. If you only remember one thing from this post, make it this table. + +| Flag | What it does | Why our value | +| --- | --- | --- | +| `--tensor-parallel-size 4` | How many GPUs to slice each layer across. Often shortened to `-tp`. | 236 GB of weights needs at least 3 cards of 95 GiB, and 4 divides the model's head counts cleanly. | +| `--pipeline-parallel-size 1` | How many groups to cut the layer stack into. Often `-pp`. | 1 means off. We test a version with 2 later. | +| `--enable-expert-parallel` | Hand out whole experts per GPU instead of slicing every expert. Mixture-of-experts models only. | Tested both ways, since this is exactly the choice a big MoE forces on you. | +| `--gpu-memory-utilization 0.90` | The fraction of each GPU's memory vLLM is allowed to claim, for weights plus KV cache plus working space. | 0.90 leaves a little headroom. Push it to 0.95 for more cache, but leave room or startup fails. | +| `--max-model-len 32768` | The longest single request, prompt plus reply, in tokens. | The model supports 262,144, but that would eat 47 GiB of cache for one user. 32,768 is a sane serving value. | +| `--max-num-seqs 32` | How many requests may be in flight at once. | Caps how much KV cache can be demanded simultaneously. Lower it if you see requests being preempted. | +| `--served-model-name qwen3-235b` | The name clients use in the API. | Otherwise clients must send the full checkpoint path. | +| `--port 8000` | Port for the OpenAI-compatible API. | Convention. | +| `--distributed-executor-backend mp` | How the GPU worker processes are managed: `mp` for plain Python multiprocessing, `ray` for a Ray cluster. | All 4 GPUs are in one machine, so `mp` is the simpler choice. `ray` is for multiple machines. | +| `--enforce-eager` | Skips building optimised CUDA graphs at startup. | We do **not** use it. It saves memory and starts faster, but generation is slower. Reach for it only if you are out of memory. | +| `--kv-cache-dtype fp8` | Stores the conversation cache at 8 bits instead of 16, roughly halving cache memory. | We left it at the default so our cache numbers are easy to check by hand. It is a good lever if you need more concurrency. | + +Two container flags matter just as much, and neither is a vLLM flag: + +| Docker flag | Why you need it | +| --- | --- | +| `--ipc=host` | The GPU workers are separate processes that pass data through shared memory. Docker's default 64 MB of shared memory is far too small, and leaving this out gives you a confusing hang at startup. | +| `--gpus '"device=1,4,5,6"'` | Hands specific GPUs to the container. The nested quoting is fussy but required. Inside the container they are renumbered 0 to 3. | + +## Part 8: The command, line by line + +Here is the whole thing. Every line is explained above, and we will walk the structure below it. + +```bash +docker run -d --name vllm-tp4 \ + --gpus '"device=1,4,5,6"' \ + --ipc=host \ + -p 8000:8000 \ + -v /root/.cache/huggingface:/root/.cache/huggingface \ + -e HF_HUB_OFFLINE=1 \ + -e HF_HOME=/root/.cache/huggingface \ + -e VLLM_USE_DEEP_GEMM=0 \ + vllm/vllm-openai:latest \ + Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 \ + --served-model-name qwen3-235b \ + --tensor-parallel-size 4 \ + --gpu-memory-utilization 0.90 \ + --max-model-len 32768 \ + --max-num-seqs 32 \ + --port 8000 +``` + +Reading it top to bottom: + +- `docker run -d` starts the container in the background and prints its id. Drop the `-d` if you would rather watch the logs scroll past. +- `--name vllm-tp4` gives it a name so you can say `docker logs vllm-tp4` instead of copying an id. +- `-p 8000:8000` maps the container's port 8000 to the host's port 8000, so you can reach the API from outside. +- `-v /root/.cache/huggingface:/root/.cache/huggingface` shares your downloaded models with the container. Without it the container would download all 236 GB again. +- `-e HF_HUB_OFFLINE=1` tells the Hugging Face library not to phone home. It uses the local copy, which also means startup does not fail if the network is down. +- `vllm/vllm-openai:latest` is the image. Everything after it is passed to vLLM, because the image's entrypoint is already `vllm serve`. +- The first argument after the image is the model. Everything after that is a vLLM flag from the table above. +- `-e VLLM_USE_DEEP_GEMM=0` is here because without it this exact model would not start on these exact GPUs. It is not a general recommendation, and Part 12 explains the crash it avoids. If you are on different hardware, try without it first. + +One thing worth knowing about that entrypoint: because it is already `vllm serve`, running `docker run ... vllm/vllm-openai:latest python3 -c "..."` does **not** work the way you expect. Your Python gets handed to `vllm serve` as arguments and you get a confusing parse error. To run something else inside the image, override it: + +```bash +docker run --rm --gpus '"device=1,4"' --entrypoint python3 vllm/vllm-openai:latest -c " +import torch +print('GPUs visible:', torch.cuda.device_count()) +print('can GPU 0 talk to GPU 1 directly:', torch.cuda.can_device_access_peer(0, 1)) +" +``` + +That is a genuinely useful sanity check before you start a long model load, because it confirms the container can see the cards and that direct GPU-to-GPU access is available. + +## Part 9: How to read the startup log + +The startup log is the best teaching tool in the whole stack, and almost nobody reads it. Four lines tell you everything about whether your configuration is sensible. + +**Line one, how big the weights are per GPU.** You get one of these per worker: + +``` +(Worker_TP0) Model loading took X GiB +``` + +If you divide the full model size by your `--tensor-parallel-size` and get roughly this number, the split worked. If this number equals the **whole** model, something is wrong and you are not actually splitting. + +**Line two, what is left for conversations:** + +``` +Available KV cache memory: X GiB +``` + +If this is **negative**, your weights plus overhead already exceeded the budget, and vLLM will refuse to start. That is the clearest possible signal that you need more GPUs, a smaller number format, or a lower `--max-model-len`. + +**Line three, the cache in tokens:** + +``` +GPU KV cache size: N tokens +``` + +This is the total number of tokens the server can remember across all users at once. You can predict it: take the available cache memory, divide by the bytes-per-token figure we calculated in Part 2. + +**Line four, how many users that really means:** + +``` +Maximum concurrency for 32,768 tokens per request: N.NNx +``` + +This is the one to show your capacity planner. If it says `2.05x`, then two users can each have a full-length 32k conversation, and a third will have to wait or be preempted. It is simply the previous line divided by `--max-model-len`. + +## Part 10: The rules that limit how far you can split + +You cannot pick any number for `--tensor-parallel-size`. There are hard divisibility rules, and hitting them is a common early frustration. + +Because attention heads are handed out whole, **your tensor parallel size must divide the head counts**. Open the model's `config.json` and look: + +```json +{ + "num_hidden_layers": 94, + "hidden_size": 4096, + "num_attention_heads": 64, + "num_key_value_heads": 4, + "head_dim": 128, + "num_experts": 128, + "num_experts_per_tok": 8 +} +``` + +For our model: + +- `num_attention_heads` is 64, so 2, 4, 8, 16 all divide it cleanly. +- `num_key_value_heads` is **4**. This is the binding constraint. At `-tp 4` each GPU gets exactly one key/value head. At `-tp 8` there are not enough to go around, and vLLM has to duplicate them across GPUs, which wastes memory and gives you less benefit than you would hope. +- `num_experts` is 128, which divides evenly by 4 and by 8, so expert parallelism has more freedom than tensor parallelism here. + +That is the real lesson: **the KV head count, not the parameter count, usually decides how wide you can go.** It is the first thing we check on any new model, and it takes ten seconds. + +## Part 11: What we measured + +Once it was running, we compared all three ways of splitting the same model over the same 4 GPUs: tensor parallelism on its own, tensor parallelism plus expert parallelism, and pure pipeline parallelism. Same hardware, same flags otherwise, same benchmark. + +The benchmark is vLLM's own, 1024 tokens in and 256 tokens out per request, with `--ignore-eos` so every request generates exactly 256 tokens and the comparison is fair: + +```bash +docker exec vllm-tp4 vllm bench serve \ + --model Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 \ + --served-model-name qwen3-235b \ + --base-url http://localhost:8000 \ + --dataset-name random --random-input-len 1024 --random-output-len 256 \ + --max-concurrency 1 --num-prompts 12 --seed 42 --ignore-eos +``` + +and then again with 32 requests in flight, which is the same command with two numbers changed: + +```bash + --max-concurrency 32 --num-prompts 128 +``` + +We ran both for every setup, because a single request at a time and 32 at a time behave completely differently, and a configuration that wins one can lose the other. + +### The memory side + +| | TP=4 | TP=4 plus EP | PP=4 | +| --- | --- | --- | --- | +| Weights per GPU | 55.19 GiB | 55.19 GiB | 55.70 GiB | +| KV cache per GPU | 27.85 GiB | 27.96 GiB | 26.84 GiB | +| Total KV cache | 621,392 tokens | **623,696 tokens** | 555,680 tokens | +| Max concurrency at 32k | 18.96x | **19.03x** | 16.96x | +| GPU memory used | 88,211 MiB on all 4 | 88,209 MiB on all 4 | 84,283 / 87,899 / 87,899 / 84,507 | + +Two things to pull out of that table. + +**Expert parallelism did not save memory.** It moved 0.37% of extra room into the cache, which is noise. If you were hoping expert parallelism would let you fit a model that otherwise does not fit, this is your warning that it will not. + +**Pipeline parallelism cost us 11.8% of the cache**, dropping from 621,392 tokens to 555,680, because a pipeline needs extra buffers for the activations travelling between stages, and that comes straight out of your conversation capacity. + +Look at the last row too. Under tensor parallelism all four cards sat at **exactly 88,211 MiB**, the same number on every one of them. Under pipeline parallelism they ranged from 84,283 to 87,899 MiB, about 3.6 GB apart, because a layer split cannot be perfectly even when 94 layers go over 4 GPUs and the ends of the model are not symmetric: the first stage carries the token embedding and the last carries the output head. That evenness check is the quickest sanity test you have that a tensor-parallel split is behaving. + +### The speed side + +| Measurement | TP=4 | TP=4 plus EP | PP=4 | Winner | +| --- | --- | --- | --- | --- | +| Median time per token, 1 request | **17.14 ms** | 18.83 ms | 21.19 ms | TP | +| Output tokens/sec, 32 requests | **503.68** | 470.93 | 296.48 | TP, by 70% over PP | +| Median time to first token, 32 requests | 3,233 ms | 3,705 ms | **2,735 ms** | PP, by 15% | +| Benchmark duration, 32 requests | **65.06 s** | 69.58 s | 110.52 s | TP | + +Tensor parallelism won nearly everything, and the size of one gap deserves attention: at 32 concurrent requests it produced **70% more tokens per second than pipeline parallelism**. That is not a rounding error, that is a different class of performance, and it lines up exactly with the theory from Part 4. Tensor parallelism has all four GPUs working on every token. Pipeline parallelism has each GPU working on a different request's stage, and with only 32 requests spread over 4 stages there is not enough in flight to keep everyone busy, so cards sit idle waiting for their turn. Its median time per token was 24% worse for the same reason. + +**Pipeline parallelism did win one thing, and it is the one theory predicts:** time to first token, by 15%. Processing your 1024-token prompt is where tensor parallelism's chatter gets expensive, because each of those 188 all-reduces is carrying the whole prompt's worth of data rather than a single token's. Pipeline parallelism just hands one activation tensor to the next stage and skips all of it. If your users judge you on how fast the first word appears, that is a real and measurable advantage. + +That is not a knock on expert parallelism, and it is important not to over-read it. Expert parallelism exists to solve a problem we do not have here: models so large that even a tensor-parallel split cannot hold all the experts, and clusters big enough that duplicating experts everywhere would be wasteful. With 4 GPUs and a model that already fits, we are asking it to do a job it was not designed for, and paying an extra network hop per token for nothing. On a 32 or 64 GPU deployment of a trillion-parameter model the answer would very likely flip. + +### The number we are throwing away, and why + +Being straight about this because it is a good lesson in reading your own benchmarks. The very first expert-parallel run at one-request-at-a-time reported **28.71 output tokens per second**, which would have made expert parallelism look catastrophic. It was not real. Look at the two TTFT figures from that run: + +``` +Mean TTFT (ms): 3987.38 +Median TTFT (ms): 265.56 +``` + +A mean fifteen times the median means one request behaved completely differently from the other eleven. One request stalled for about 45 seconds, almost certainly a one-off kernel compilation on the first pass through a code path, and that single stall stretched the whole benchmark from 63 seconds to 107 seconds. Since throughput is just tokens divided by wall-clock, one stall wrecked the headline number. + +This is why the table above uses **median time per token** as the decode measurement rather than aggregate throughput. Median per-token latency does not care that one request had a bad start. + +One more benchmarking trap while we are here. When we re-ran that same benchmark on the warm server, time to first token dropped from 265 ms to **61 ms**, which looks like a wonderful improvement and is actually meaningless: vLLM caches prompt prefixes by default, and we had just sent it those exact prompts with the same `--seed 42`. If you are comparing configurations, either vary the seed or turn prefix caching off, otherwise your second measurement is mostly measuring your cache. + +### What we would actually run + +For a 235B MoE on 4 GPUs with no NVLink between them, we would use plain `--tensor-parallel-size 4` and leave both of the others off. It was faster nearly everywhere, it gives the most conversation capacity, it splits memory perfectly evenly, and it is one less thing to reason about. + +We would reach for the other two in specific situations, not as general upgrades: + +- **Pipeline parallelism** if time to first token is the metric you are judged on, or if you are spanning multiple machines where the network between them is genuinely slow. It was 15% better at first-token latency and it barely uses the interconnect. +- **Expert parallelism** when the model is so large that even a tensor-parallel split cannot hold all the experts, which is a real problem at trillion-parameter scale and simply is not our problem at 235B on 4 cards. Here it cost 7% and returned nothing. + + +## Part 12: Errors you will actually hit + +Every one of these is a real message we collected while doing this, not a hypothetical. + +### "must be divisible by tensor parallel size" + +We asked for 3 GPUs, which is a perfectly reasonable-sounding thing to want, and got: + +``` +pydantic_core._pydantic_core.ValidationError: 1 validation error for VllmConfig + Value error, Total number of attention heads (64) must be divisible by tensor + parallel size (3). +``` + +**What it means:** the rule from Part 10. 64 heads cannot be shared out evenly among 3 GPUs. Good news, it fails in about a second, before loading a single byte of weights. + +**The fix:** pick a `--tensor-parallel-size` that divides your head count. Powers of two are the safe habit. + +### "Failed to load model - not enough GPU memory" + +Then we tried 2 GPUs, which puts about 110 GiB of weights on a 95 GiB card. It got most of the way through loading and then died: + +``` +ERROR [gpu_model_runner.py:5403] Failed to load model - not enough GPU memory. +Try lowering --gpu-memory-utilization to free memory for weights, increasing +--tensor-parallel-size, or using --quantization. +(original error: CUDA out of memory. Tried to allocate 768.00 MiB. GPU 0 has a +total capacity of 95.01 GiB of which 438.31 MiB is free. Including non-PyTorch +memory, this process has 94.57 GiB memory in use.) +``` + +**What it means:** exactly what it says. The weights for half this model do not fit on one of these cards. Note the useful detail in there, `438.31 MiB is free` out of `95.01 GiB`, so it filled the card almost exactly and then had nowhere to put the next 768 MiB chunk. + +**The fix:** vLLM lists the three real options itself, and for our case only one of them helps. Lowering `--gpu-memory-utilization` would make things worse, not better, because it reduces the space available for weights. Quantizing further would work but changes the model. So the answer is more GPUs, which is the whole point of this post. + +Worth knowing: this one is slow to fail, because it has to read and place most of the weights before it runs out. Budget several minutes, unlike the divisibility error which fails instantly. + +### "Unknown SF transformation", the one that cost us the most time + +This is the error we did not see coming, and it is worth the whole section. With 4 GPUs and everything sized correctly, all four workers died during startup: + +``` +RuntimeError: Assertion error (/workspace/.deps/deepgemm-src/csrc/apis/layout.hpp:60): +Unknown SF transformation +``` + +**What it means:** this model stores its FP8 weights in blocks, with a separate scale factor per 128x128 block, which you can see in its config as `"weight_block_size": [128, 128]`. vLLM hands that kind of matrix multiplication to a library called DeepGEMM, and DeepGEMM did not know how to lay out those scale factors ("SF" is scale factor) on our particular GPU. The RTX PRO 6000 is Blackwell, but it reports as `sm_120`, which is not the same silicon target as the data-centre Blackwell parts that DeepGEMM is usually exercised on. + +Notice how unhelpful the message is if you do not know that background. Nothing in it mentions FP8, quantization, or your GPU. + +**The fix**, which is one environment variable: + +```bash +docker run -d ... -e VLLM_USE_DEEP_GEMM=0 ... vllm/vllm-openai:latest ... +``` + +That tells vLLM to use its own FP8 kernels instead of DeepGEMM. Startup then went through cleanly. There is a performance cost to giving up a specialised kernel, so on hardware where DeepGEMM works you would leave it on. + +**The general lesson:** a quantized model is a contract between the checkpoint's format and a kernel that understands it. When a big quantized model fails to start on hardware that clearly has enough memory, suspect the kernel and the number format before you suspect your parallelism settings. + +### A confusing parse error when you try to run something else in the container + +``` +vllm serve: error: argument --compilation-config/-cc: Invalid JSON: expected value at line 2 +``` + +**What it means:** you ran `docker run ... vllm/vllm-openai:latest python3 -c "..."`, but the image's entrypoint is already `vllm serve`, so your Python source got handed to vLLM as a command-line argument. + +**The fix:** `--entrypoint python3`, as shown in Part 8. + +### "No available shared memory broadcast block found in 60 seconds" + +**What it means:** usually nothing. It shows up while vLLM is busy compiling or capturing CUDA graphs and the worker processes have not checked in for a minute. If it repeats forever and startup never finishes, then you probably forgot `--ipc=host` and the workers cannot pass data to each other through shared memory. + +**The fix:** add `--ipc=host`. If you already have it, wait a bit longer, because CUDA graph capture on a big model is genuinely slow. + + +## Wrapping up + +If you take five things away from this, let them be these. + +**One.** Inference is two jobs, not one. Prefill reads your whole prompt at once and is limited by compute; decode writes one token at a time and is limited by memory bandwidth. Every confusing multi-GPU result in this post traces back to that split, so when a change helps one metric and hurts the other, this is why. + +**Two.** Work out the memory on paper first. Parameters times bytes-per-parameter gives you the weights, and then remember that the weights are only one of three things that must fit, alongside the conversation cache and the working space. A model whose weights just barely fit is a model that cannot serve anybody. + +**Three.** "Splitting across GPUs" is three different things. Tensor parallelism slices every layer and makes all your GPUs work on the same token, at the cost of constant chatter. Pipeline parallelism cuts the layer stack into blocks and barely communicates, at the cost of GPUs waiting their turn. Expert parallelism only exists for mixture-of-experts models and hands out whole experts. You can combine them, and for big models you usually do. + +**Four.** Read the startup log. `Model loading took`, `Available KV cache memory`, `GPU KV cache size` and `Maximum concurrency` tell you, in four lines, whether your setup is sane and how many users it can actually hold. A negative cache number is the clearest error message in the whole stack. + +**Five.** Check `num_key_value_heads` in `config.json` before you plan your hardware. It, not the parameter count, is usually what limits how many GPUs you can split across cleanly. + +One last practical warning, because it cost us more than any GPU problem did. **Check your disk before you download.** A quarter of a terabyte of model weights on a shared machine is not just a storage question, it is a question about everything else living on that disk. Ours was a Kubernetes node, free space crossed the kubelet's eviction threshold, and it evicted the platform's own pods and garbage-collected locally-built images that no registry could replace. `df -h` first, and leave real headroom. + +Try it on whatever you have. Two GPUs are enough to see every concept in this post in action, and the log lines mean the same thing whether you are running 4 GPUs or 40. If you hit something we did not cover, tell us and we will add it. + +## Credits and references + +- The tensor parallel scheme is from **Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism** by Mohammad Shoeybi, Mostofa Patwary, Raul Puri, Patrick LeGresley, Jared Casper and Bryan Catanzaro: [arxiv.org/abs/1909.08053](https://arxiv.org/abs/1909.08053) +- vLLM parallelism and scaling guide: [docs.vllm.ai/en/latest/serving/parallelism_scaling.html](https://docs.vllm.ai/en/latest/serving/parallelism_scaling.html) +- vLLM memory and optimization docs: [conserving_memory](https://docs.vllm.ai/en/latest/configuration/conserving_memory.html) and [optimization](https://docs.vllm.ai/en/latest/configuration/optimization.html) +- Model card and config: [huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507-FP8](https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507-FP8) +- Thanks to the vLLM maintainers, whose startup logging is the best free lesson in distributed inference available anywhere. + +--- + # The Local LLM Glossary: Every Term, Flag, and Number in Plain English - Canonical: https://blog.kubesimplify.com/local-llm-glossary diff --git a/public/llms.txt b/public/llms.txt index 6eae55e4c..b3ca302f7 100644 --- a/public/llms.txt +++ b/public/llms.txt @@ -4,7 +4,7 @@ ## About -Kubesimplify is a community-driven publication on cloud-native technologies, with 198 in-depth technical articles by 62 practitioner authors. We cover Kubernetes (kubelet internals, scheduling, networking, operators), container runtimes (containerd, CRI-O, Docker), GitOps (Argo CD, Flux), service meshes, observability, AI/ML infrastructure on Kubernetes, GPU workloads, platform engineering, and the broader CNCF ecosystem. +Kubesimplify is a community-driven publication on cloud-native technologies, with 199 in-depth technical articles by 62 practitioner authors. We cover Kubernetes (kubelet internals, scheduling, networking, operators), container runtimes (containerd, CRI-O, Docker), GitOps (Argo CD, Flux), service meshes, observability, AI/ML infrastructure on Kubernetes, GPU workloads, platform engineering, and the broader CNCF ecosystem. Authoritative, practitioner-written, citation-friendly. Articles include code examples, diagrams, and references. @@ -34,8 +34,9 @@ Authoritative, practitioner-written, citation-friendly. Articles include code ex - Cloud Native Security: https://blog.kubesimplify.com/hub/security (network policies, Falco, Kyverno, SLSA supply-chain) - Linux Fundamentals: https://blog.kubesimplify.com/hub/linux (shell, sysadmin, networking primitives) -## Recent posts (most recent 30 of 198) +## Recent posts (most recent 30 of 199) +- [Running a big LLM across multiple GPUs with vLLM](https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm) (2026-08-18). A plain-English guide to serving a model that is too big for one GPU: how tensor, pipeline, and expert parallelism split it up, what every vLLM flag does, and measured numbers from a 235B model on four RTX PRO 6000 cards. - [The Local LLM Glossary: Every Term, Flag, and Number in Plain English](https://blog.kubesimplify.com/local-llm-glossary) (2026-08-18). Plain-English definitions for every term you hit in local LLM posts: prefill and decode, tokens per second, FP8 and NVFP4, Q4_K_M, KV cache, YaRN, Gated DeltaNet, speculative decoding, and every vLLM, llama.cpp, and Ollama flag worth knowing. - [Running Qwen3.8-27B on DGX Spark](https://blog.kubesimplify.com/qwen3-8-27b-on-dgx-spark) (2026-08-17). Qwen3.8-27B on DGX Spark with llama.cpp, Ollama, vLLM, and SGLang: the recipes, the tokens per second I measured, MTP speculative decoding, and the sharp edges I hit along the way. - [I Ran an AI SRE Copilot on My Own Hardware. Here Is What It Actually Does.](https://blog.kubesimplify.com/nudgebee-ai-sre-copilot-hands-on) (2026-08-17). Running NudgeBee v1.4.0 end to end - a self-hosted AIOps platform behind AI-SRE, AI-FinOps, AI-K8sOps, and agentic automation - on a Mac, a kiac cluster, and a DGX Spark. @@ -65,7 +66,6 @@ Authoritative, practitioner-written, citation-friendly. Articles include code ex - [Day 5: Docker Compose - How Docker Actually Gets Used](https://blog.kubesimplify.com/day-5-docker-compose-how-docker-actually-gets-used) (2026-04-28) - [What Actually Happens When kube-scheduler Picks a Node (13 Stages Inside Kubernetes)](https://blog.kubesimplify.com/kube-scheduler-deep-dive) (2026-04-28). How kube-scheduler picks a node: 13 framework stages, 14 Filter plugins, 9 Score plugins, live preemption demo. - [Day 4: Breaking Isolation on Purpose - Volumes, Networks, and the Real World](https://blog.kubesimplify.com/day-4-breaking-isolation-on-purpose-volumes-networks-and-the-real-world) (2026-04-27) -- [Day 3: Stop Writing Dockerfiles From Scratch](https://blog.kubesimplify.com/day-3-stop-writing-dockerfiles-from-scratch) (2026-04-24). Stop writing Dockerfiles from scratch. A Docker Captain walks through docker init, layer caching, multi-stage builds, and docker debug for 2026. ## Topics covered (auto-derived from tags) @@ -76,8 +76,8 @@ Authoritative, practitioner-written, citation-friendly. Articles include code ex - linux (19 articles): https://blog.kubesimplify.com/tag/linux - containers (17 articles): https://blog.kubesimplify.com/tag/containers - cloud (16 articles): https://blog.kubesimplify.com/tag/cloud -- nvidia (14 articles): https://blog.kubesimplify.com/tag/nvidia -- llm (12 articles): https://blog.kubesimplify.com/tag/llm +- nvidia (15 articles): https://blog.kubesimplify.com/tag/nvidia +- llm (13 articles): https://blog.kubesimplify.com/tag/llm - aws (12 articles): https://blog.kubesimplify.com/tag/aws - cloud-native (11 articles): https://blog.kubesimplify.com/tag/cloud-native - security (11 articles): https://blog.kubesimplify.com/tag/security @@ -85,15 +85,15 @@ Authoritative, practitioner-written, citation-friendly. Articles include code ex - go (9 articles): https://blog.kubesimplify.com/tag/go - git (9 articles): https://blog.kubesimplify.com/tag/git - linux-for-beginners (9 articles): https://blog.kubesimplify.com/tag/linux-for-beginners +- platform-engineering (8 articles): https://blog.kubesimplify.com/tag/platform-engineering - local-ai (8 articles): https://blog.kubesimplify.com/tag/local-ai - github (8 articles): https://blog.kubesimplify.com/tag/github - terraform (8 articles): https://blog.kubesimplify.com/tag/terraform - ai (7 articles): https://blog.kubesimplify.com/tag/ai -- platform-engineering (7 articles): https://blog.kubesimplify.com/tag/platform-engineering - docker-images (7 articles): https://blog.kubesimplify.com/tag/docker-images - kubesimplify (7 articles): https://blog.kubesimplify.com/tag/kubesimplify - linux-basics (7 articles): https://blog.kubesimplify.com/tag/linux-basics -- ollama (6 articles): https://blog.kubesimplify.com/tag/ollama +- gpu (6 articles): https://blog.kubesimplify.com/tag/gpu ## Top contributors @@ -102,8 +102,8 @@ Authoritative, practitioner-written, citation-friendly. Articles include code ex - [Kunal Verma](https://blog.kubesimplify.com/author/kunal-verma) (12 posts) - [Dipankar Das](https://blog.kubesimplify.com/author/dipankar-das) (9 posts) - [Anurag Kumar](https://blog.kubesimplify.com/author/anurag-kumar) (8 posts) +- [Shubham Katara](https://blog.kubesimplify.com/author/shubham-katara) (6 posts) - [sysxplore](https://blog.kubesimplify.com/author/sysxplore) (6 posts) -- [Shubham Katara](https://blog.kubesimplify.com/author/shubham-katara) (5 posts) - [Arnav Barman](https://blog.kubesimplify.com/author/arnav-barman) (5 posts) - [Srinivas Karnati](https://blog.kubesimplify.com/author/srinivas-karnati) (4 posts) - [Barkatul Mujauddin](https://blog.kubesimplify.com/author/barkatul-mujauddin) (4 posts) diff --git a/public/rss.xml b/public/rss.xml index 2300c9b4a..c59f965e1 100644 --- a/public/rss.xml +++ b/public/rss.xml @@ -6,8 +6,16 @@ Deep dives on Kubernetes, AI infrastructure, GitOps, and the cloud-native stack, written by practitioners. en-us - Tue, 18 Aug 2026 09:00:00 GMT + Tue, 18 Aug 2026 10:00:00 GMT Kubesimplify static blog + + Running a big LLM across multiple GPUs with vLLM + https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm + https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm + Tue, 18 Aug 2026 10:00:00 GMT + A plain-English guide to serving a model that is too big for one GPU: how tensor, pipeline, and expert parallelism split it up, what every vLLM flag does, and measured numbers from a 235B model on four RTX PRO 6000 cards. + vllmgpunvidiallmplatform-engineering + The Local LLM Glossary: Every Term, Flag, and Number in Plain English https://blog.kubesimplify.com/local-llm-glossary diff --git a/scripts/gen-local-llm-glossary-cover.mjs b/scripts/gen-local-llm-glossary-cover.mjs index 803b42205..45c0ec60a 100644 --- a/scripts/gen-local-llm-glossary-cover.mjs +++ b/scripts/gen-local-llm-glossary-cover.mjs @@ -1,5 +1,5 @@ // Excalidraw-style cover for the local LLM glossary post. -// Sketch helpers shared with scripts/gen-two-gpu-vllm-cover.mjs. +// Sketch helpers shared with scripts/gen-multi-gpu-vllm-cover.mjs. import { mkdirSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; diff --git a/scripts/gen-multi-gpu-vllm-cover.mjs b/scripts/gen-multi-gpu-vllm-cover.mjs new file mode 100644 index 000000000..6c1be76af --- /dev/null +++ b/scripts/gen-multi-gpu-vllm-cover.mjs @@ -0,0 +1,228 @@ +// Excalidraw-style cover for the multi-GPU vLLM article. +// Sketch helpers shared with scripts/gen-hami-diagrams.mjs. +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +let seed = 42; +const random = () => { + seed = (seed * 16807) % 2147483647; + return seed / 2147483647; +}; +const jitter = (amount) => (random() - 0.5) * amount * 2; + +const COLORS = { + ink: '#172033', + muted: '#5c677d', + green: { stroke: '#5d8f00', fill: '#d8f5a2' }, + blue: { stroke: '#1971c2', fill: '#a5d8ff' }, + violet: { stroke: '#862e9c', fill: '#eebefa' }, + orange: { stroke: '#d9480f', fill: '#ffd8a8' }, + red: { stroke: '#c92a2a', fill: '#ffc9c9' }, + teal: { stroke: '#087f5b', fill: '#b2f2bb' }, + gray: { stroke: '#495057', fill: '#e9ecef' }, +}; + +const FONT = 'Chalkboard SE, Comic Sans MS, sans-serif'; + +function roughLine(x1, y1, x2, y2, amount = 1.8) { + const middleX = (x1 + x2) / 2 + jitter(amount * 1.5); + const middleY = (y1 + y2) / 2 + jitter(amount * 1.5); + return `M ${(x1 + jitter(amount)).toFixed(1)} ${(y1 + jitter(amount)).toFixed(1)} Q ${middleX.toFixed(1)} ${middleY.toFixed(1)} ${(x2 + jitter(amount)).toFixed(1)} ${(y2 + jitter(amount)).toFixed(1)}`; +} + +class Sketch { + constructor(width, height, background = '#ffffff') { + this.width = width; + this.height = height; + this.background = background; + this.parts = []; + this.defs = []; + this.clipId = 0; + } + + add(value) { + this.parts.push(value); + } + + rect(x, y, width, height, options = {}) { + const { + stroke = COLORS.ink, + fill, + strokeWidth = 2.4, + dashed = false, + hachure = true, + radius = 7, + } = options; + + if (fill) { + if (hachure) { + // Hatch lines are clipped in math rather than with an SVG clipPath so + // the file renders identically in renderers without clipPath support. + const hatch = []; + for (let offset = -height; offset < width; offset += 11) { + const tMin = Math.max(0, -offset / height); + const tMax = Math.min(1, (width - offset) / height); + if (tMax - tMin < 0.05) continue; + const x1 = x + offset + height * tMin; + const y1 = y + height - height * tMin; + const x2 = x + offset + height * tMax; + const y2 = y + height - height * tMax; + hatch.push(roughLine(x1, y1, x2, y2, 1)); + } + this.add(``); + } else { + this.add(``); + } + } + + const points = [[x, y], [x + width, y], [x + width, y + height], [x, y + height]]; + for (let pass = 0; pass < 2; pass += 1) { + const path = points.map((point, index) => { + const next = points[(index + 1) % points.length]; + return roughLine(point[0], point[1], next[0], next[1], pass === 0 ? 2 : 1.2); + }).join(' '); + this.add(``); + } + } + + line(x1, y1, x2, y2, options = {}) { + const { stroke = COLORS.ink, strokeWidth = 2.4, dashed = false } = options; + this.add(``); + } + + arrow(x1, y1, x2, y2, options = {}) { + const { stroke = COLORS.ink, strokeWidth = 2.6, dashed = false } = options; + this.line(x1, y1, x2, y2, { stroke, strokeWidth, dashed }); + const angle = Math.atan2(y2 - y1, x2 - x1); + const length = 14; + for (const offset of [Math.PI * 0.82, -Math.PI * 0.82]) { + this.line( + x2, + y2, + x2 + length * Math.cos(angle + offset), + y2 + length * Math.sin(angle + offset), + { stroke, strokeWidth } + ); + } + } + + text(x, y, value, options = {}) { + const { + size = 22, + color = COLORS.ink, + anchor = 'middle', + weight = 500, + family = FONT, + } = options; + const safe = String(value) + .replace(/&/g, '&') + .replace(//g, '>'); + this.add(`${safe}`); + } + + lines(x, y, values, options = {}) { + const lineHeight = (options.size || 22) * (options.lineHeight || 1.28); + values.forEach((value, index) => this.text(x, y + index * lineHeight, value, options)); + } + + save(path) { + const svg = ` +${this.defs.join('')} + +${this.parts.join('\n')} +`; + writeFileSync(path, svg); + } +} + +const output = process.argv[2] || '.'; +mkdirSync(output, { recursive: true }); + + +const W = 1200; +const H = 630; +const sketch = new Sketch(W, H, '#fdfdfb'); + +sketch.text(64, 82, 'One big model, four GPUs', { size: 50, weight: 800, anchor: 'start' }); +sketch.text(64, 119, 'how a 235B model is cut up so it fits, and what that costs', { + size: 22, + color: COLORS.muted, + anchor: 'start', +}); +sketch.line(64, 139, 760, 139, { stroke: COLORS.muted, strokeWidth: 1.6, dashed: true }); + +// ── left: the model does not fit on one card ────────────── +sketch.text(64, 186, 'ONE CARD', { size: 18, weight: 800, anchor: 'start', color: COLORS.red.stroke }); + +const bY = 206; +sketch.rect(64, bY, 210, 132, { stroke: COLORS.gray.stroke, fill: '#ffffff', hachure: false, dashed: true }); +sketch.text(169, bY + 30, '95 GiB', { size: 19, color: COLORS.muted }); +sketch.text(169, bY + 54, 'usable', { size: 15, color: COLORS.muted }); + +// overflowing weights bar +sketch.rect(78, bY + 72, 330, 46, { stroke: COLORS.red.stroke, fill: COLORS.red.fill }); +sketch.text(200, bY + 95, '236 GB of weights', { size: 19, weight: 800, color: COLORS.red.stroke }); + +sketch.text(64, bY + 164, '2.3x too big', { size: 26, weight: 800, anchor: 'start', color: COLORS.red.stroke }); +sketch.text(64, bY + 192, 'no flag fixes this', { size: 16, anchor: 'start', color: COLORS.muted }); + +// ── divider ─────────────────────────────────────────────── +sketch.line(452, 186, 452, 452, { stroke: COLORS.muted, strokeWidth: 1.6, dashed: true }); + +// ── right: four cards, each holds a quarter ─────────────── +sketch.text(516, 186, 'FOUR CARDS, --tensor-parallel-size 4', { + size: 18, + weight: 800, + anchor: 'start', + color: COLORS.teal.stroke, +}); + +const cw = 145; +const gap = 10; +const gY = 206; +const palette = [COLORS.blue, COLORS.green, COLORS.violet, COLORS.orange]; +[0, 1, 2, 3].forEach((gpu) => { + const x = 516 + gpu * (cw + gap); + const c = palette[gpu]; + sketch.rect(x, gY, cw, 132, { stroke: c.stroke, fill: c.fill }); + sketch.text(x + cw / 2, gY + 30, `GPU ${gpu}`, { size: 20, weight: 800, color: c.stroke }); + sketch.text(x + cw / 2, gY + 60, '59 GB', { size: 18, weight: 700 }); + sketch.text(x + cw / 2, gY + 84, 'weights', { size: 14, color: COLORS.muted }); + sketch.text(x + cw / 2, gY + 112, '16 of 64 heads', { size: 13, color: COLORS.muted }); +}); + +// all-reduce arrows under the row of cards +const arrowY = gY + 154; +sketch.line(516 + 40, arrowY, 516 + 3 * (cw + gap) + cw - 40, arrowY, { + stroke: COLORS.violet.stroke, + dashed: true, +}); +sketch.text(516 + (3 * (cw + gap) + cw) / 2, arrowY + 30, '188 all-reduces per token', { + size: 18, + weight: 800, + color: COLORS.violet.stroke, +}); + +// ── footer ──────────────────────────────────────────────── +sketch.line(64, 516, W - 64, 516, { stroke: COLORS.muted, strokeWidth: 1.6 }); +sketch.text(64, 552, 'QWEN3-235B-A22B FP8 - 128 EXPERTS, 8 PER TOKEN - vLLM 0.27.1', { + size: 18, + weight: 800, + anchor: 'start', + color: COLORS.ink, +}); +sketch.text(64, 582, 'tensor, pipeline and expert parallelism explained in plain english', { + size: 16, + anchor: 'start', + color: COLORS.muted, +}); +sketch.text(W - 64, 582, 'blog.kubesimplify.com', { + size: 16, + weight: 700, + anchor: 'end', + color: COLORS.muted, +}); + +sketch.save(join(output, 'cover.svg')); +console.log(`Wrote multi-GPU vLLM cover to ${output}`); diff --git a/vercel.json b/vercel.json index eaa7bd155..2e0044f80 100644 --- a/vercel.json +++ b/vercel.json @@ -906,6 +906,11 @@ "destination": "https://blog.kubesimplify.com/ready-for-wasm-day-2023", "permanent": true }, + { + "source": "/blog/running-a-big-llm-across-multiple-gpus-with-vllm", + "destination": "https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm", + "permanent": true + }, { "source": "/blog/sharing-gpus-in-kubernetes-with-hami", "destination": "https://blog.kubesimplify.com/sharing-gpus-in-kubernetes-with-hami", @@ -2916,6 +2921,17 @@ } ] }, + { + "source": "/running-a-big-llm-across-multiple-gpus-with-vllm", + "destination": "https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm", + "permanent": true, + "has": [ + { + "type": "host", + "value": "kubesimplify.com" + } + ] + }, { "source": "/sharing-gpus-in-kubernetes-with-hami", "destination": "https://blog.kubesimplify.com/sharing-gpus-in-kubernetes-with-hami",