Fa4 support

.dockerignore

@@ -0,0 +1,30 @@
+.git
+.github
+.conda
+.venv
+.idea
+.vscode
+
+__pycache__
+*.py[cod]
+.pytest_cache
+.mypy_cache
+.ruff_cache
+
+build
+dist
+*.egg-info
+docs
+test
+unit_tests
+benchmark
+logs
+tmp
+
+*.bin
+*.ckpt
+*.gguf
+*.onnx
+*.pt
+*.pth
+*.safetensors

benchmark_results/glm52-static-matrix-4n-mtp/cost_analysis.md

@@ -0,0 +1,48 @@
+# GLM5.2 MTP 静态 PD 成本测算
+
+- 机器整机月租：400000 元/月
+- 每月按 30 天计算
+- 机器整机每秒成本：`400000 / 30 / 24 / 3600 = 0.154321 元/s`
+- Prefill 使用 prefill 表中的 `logical_tok/s` 计算
+- Decode 使用 decode 表中对应 `itl_ms约束` 档位的 `tok/s` 计算
+
+## 计算公式
+
+```text
+机器整机每秒成本 = 机器整机月租 / 30 / 24 / 3600
+输入成本 元/M tokens = 机器整机每秒成本 / prefill_logical_tok_s * 10^6
+输出成本 元/M tokens = 机器整机每秒成本 / decode_tok_s * 10^6
+```
+
+## Decode <= 20ms 成本表
+
+| ctx | 输入cache miss成本 元/M tokens | 输出成本 元/M tokens | 输入@50% hit平均成本 元/M logical tokens | 输入@70% hit平均成本 元/M logical tokens | 输入@80% hit平均成本 元/M logical tokens | 输入@90% hit平均成本 元/M logical tokens |
+|---|---:|---:|---:|---:|---:|---:|
+| 16K | 1.952 | 15.369 | 1.019 | 0.584 | 0.419 | 0.256 |
+| 32K | 1.975 | 15.430 | 0.988 | 0.612 | 0.444 | 0.230 |
+| 64K | 1.920 | 21.043 | 0.977 | 0.594 | 0.401 | 0.210 |
+| 128K | 2.644 | 38.625 | 1.362 | 0.832 | 0.561 | 0.289 |
+| 200K | 2.718 | 82.402 | 1.398 | 0.864 | 0.583 | 0.289 |
+| 250K | 2.781 | 82.810 | 1.442 | 0.911 | 0.603 | 0.304 |
+
+## Decode <= 33ms 成本表
+
+| ctx | 输入cache miss成本 元/M tokens | 输出成本 元/M tokens | 输入@50% hit平均成本 元/M logical tokens | 输入@70% hit平均成本 元/M logical tokens | 输入@80% hit平均成本 元/M logical tokens | 输入@90% hit平均成本 元/M logical tokens |
+|---|---:|---:|---:|---:|---:|---:|
+| 16K | 1.952 | 9.198 | 1.019 | 0.584 | 0.419 | 0.256 |
+| 32K | 1.975 | 12.665 | 0.988 | 0.612 | 0.444 | 0.230 |
+| 64K | 1.920 | 21.043 | 0.977 | 0.594 | 0.401 | 0.210 |
+| 128K | 2.644 | 38.625 | 1.362 | 0.832 | 0.561 | 0.289 |
+| 200K | 2.718 | 82.402 | 1.398 | 0.864 | 0.583 | 0.289 |
+| 250K | 2.781 | 82.810 | 1.442 | 0.911 | 0.603 | 0.304 |
+
+## Decode <= 50ms 成本表
+
+| ctx | 输入cache miss成本 元/M tokens | 输出成本 元/M tokens | 输入@50% hit平均成本 元/M logical tokens | 输入@70% hit平均成本 元/M logical tokens | 输入@80% hit平均成本 元/M logical tokens | 输入@90% hit平均成本 元/M logical tokens |
+|---|---:|---:|---:|---:|---:|---:|
+| 16K | 1.952 | 9.198 | 1.019 | 0.584 | 0.419 | 0.256 |
+| 32K | 1.975 | 12.665 | 0.988 | 0.612 | 0.444 | 0.230 |
+| 64K | 1.920 | 21.043 | 0.977 | 0.594 | 0.401 | 0.210 |
+| 128K | 2.644 | 38.625 | 1.362 | 0.832 | 0.561 | 0.289 |
+| 200K | 2.718 | 82.402 | 1.398 | 0.864 | 0.583 | 0.289 |
+| 250K | 2.781 | 82.810 | 1.442 | 0.911 | 0.603 | 0.304 |

benchmark_results/glm52-static-matrix-4n-mtp/decode_cap19_ctx16384-32768-65536-131072-204800-256000_n4_btok4096_mf0p76_mtp.manifest.json

@@ -0,0 +1,282 @@
+{
+  "started_utc": "2026-07-16T19:02:57.239893+00:00",
+  "command": [
+    "/opt/conda/bin/python",
+    "test/benchmark/static_inference/test_model.py",
+    "--model_dir",
+    "/root/models/GLM-5.2/",
+    "--tp",
+    "32",
+    "--llm_kv_type",
+    "fp8kv_dsa",
+    "--batch_max_tokens",
+    "4096",
+    "--output_lens",
+    "600",
+    "--decode_batch_size_mode",
+    "profile",
+    "--quant_type",
+    "deepgemm-fp8w8a8-b128",
+    "--dp",
+    "32",
+    "--enable_ep_moe",
+    "--expert_dtype",
+    "fp8",
+    "--graph_max_batch_size",
+    "19",
+    "--mem_fraction",
+    "0.76",
+    "--nnodes",
+    "4",
+    "--node_rank",
+    "0",
+    "--nccl_host",
+    "10.119.17.190",
+    "--nccl_port",
+    "28817",
+    "--mtp_mode",
+    "eagle_with_att",
+    "--mtp_step",
+    "2",
+    "--mtp_draft_model_dir",
+    "/root/models/GLM-5.2/",
+    "--mtp_accept_rate",
+    "0.85",
+    "--benchmark",
+    "decode",
+    "--context_lens",
+    "16384,32768,65536,131072,204800,256000",
+    "--max_batch_size",
+    "19"
+  ],
+  "mem_fraction": 0.76,
+  "batch_max_tokens": 4096,
+  "mtp": true,
+  "prefill_chunk_size": null,
+  "prefill_microbatch_overlap": false,
+  "returncode": 1,
+  "log": "/mnt/afs/niushengxiao/LightLLM/benchmark_results/glm52-static-matrix-4n-mtp/decode_cap19_ctx16384-32768-65536-131072-204800-256000_n4_btok4096_mf0p76_mtp.log",
+  "row_count": 0,
+  "nodes": [
+    {
+      "host": "10.119.17.190",
+      "node_rank": 0,
+      "command": [
+        "/opt/conda/bin/python",
+        "test/benchmark/static_inference/test_model.py",
+        "--model_dir",
+        "/root/models/GLM-5.2/",
+        "--tp",
+        "32",
+        "--llm_kv_type",
+        "fp8kv_dsa",
+        "--batch_max_tokens",
+        "4096",
+        "--output_lens",
+        "600",
+        "--decode_batch_size_mode",
+        "profile",
+        "--quant_type",
+        "deepgemm-fp8w8a8-b128",
+        "--dp",
+        "32",
+        "--enable_ep_moe",
+        "--expert_dtype",
+        "fp8",
+        "--graph_max_batch_size",
+        "19",
+        "--mem_fraction",
+        "0.76",
+        "--nnodes",
+        "4",
+        "--node_rank",
+        "0",
+        "--nccl_host",
+        "10.119.17.190",
+        "--nccl_port",
+        "28817",
+        "--mtp_mode",
+        "eagle_with_att",
+        "--mtp_step",
+        "2",
+        "--mtp_draft_model_dir",
+        "/root/models/GLM-5.2/",
+        "--mtp_accept_rate",
+        "0.85",
+        "--benchmark",
+        "decode",
+        "--context_lens",
+        "16384,32768,65536,131072,204800,256000",
+        "--max_batch_size",
+        "19"
+      ],
+      "log": "/mnt/afs/niushengxiao/LightLLM/benchmark_results/glm52-static-matrix-4n-mtp/decode_cap19_ctx16384-32768-65536-131072-204800-256000_n4_btok4096_mf0p76_mtp.log",
+      "returncode": 1
+    },
+    {
+      "host": "10.119.17.202",
+      "node_rank": 1,
+      "command": [
+        "/opt/conda/bin/python",
+        "test/benchmark/static_inference/test_model.py",
+        "--model_dir",
+        "/root/models/GLM-5.2/",
+        "--tp",
+        "32",
+        "--llm_kv_type",
+        "fp8kv_dsa",
+        "--batch_max_tokens",
+        "4096",
+        "--output_lens",
+        "600",
+        "--decode_batch_size_mode",
+        "profile",
+        "--quant_type",
+        "deepgemm-fp8w8a8-b128",
+        "--dp",
+        "32",
+        "--enable_ep_moe",
+        "--expert_dtype",
+        "fp8",
+        "--graph_max_batch_size",
+        "19",
+        "--mem_fraction",
+        "0.76",
+        "--nnodes",
+        "4",
+        "--node_rank",
+        "1",
+        "--nccl_host",
+        "10.119.17.190",
+        "--nccl_port",
+        "28817",
+        "--mtp_mode",
+        "eagle_with_att",
+        "--mtp_step",
+        "2",
+        "--mtp_draft_model_dir",
+        "/root/models/GLM-5.2/",
+        "--mtp_accept_rate",
+        "0.85",
+        "--benchmark",
+        "decode",
+        "--context_lens",
+        "16384,32768,65536,131072,204800,256000",
+        "--max_batch_size",
+        "19"
+      ],
+      "log": "/mnt/afs/niushengxiao/LightLLM/benchmark_results/glm52-static-matrix-4n-mtp/decode_cap19_ctx16384-32768-65536-131072-204800-256000_n4_btok4096_mf0p76_mtp.rank1.log",
+      "returncode": 255
+    },
+    {
+      "host": "10.119.17.179",
+      "node_rank": 2,
+      "command": [
+        "/opt/conda/bin/python",
+        "test/benchmark/static_inference/test_model.py",
+        "--model_dir",
+        "/root/models/GLM-5.2/",
+        "--tp",
+        "32",
+        "--llm_kv_type",
+        "fp8kv_dsa",
+        "--batch_max_tokens",
+        "4096",
+        "--output_lens",
+        "600",
+        "--decode_batch_size_mode",
+        "profile",
+        "--quant_type",
+        "deepgemm-fp8w8a8-b128",
+        "--dp",
+        "32",
+        "--enable_ep_moe",
+        "--expert_dtype",
+        "fp8",
+        "--graph_max_batch_size",
+        "19",
+        "--mem_fraction",
+        "0.76",
+        "--nnodes",
+        "4",
+        "--node_rank",
+        "2",
+        "--nccl_host",
+        "10.119.17.190",
+        "--nccl_port",
+        "28817",
+        "--mtp_mode",
+        "eagle_with_att",
+        "--mtp_step",
+        "2",
+        "--mtp_draft_model_dir",
+        "/root/models/GLM-5.2/",
+        "--mtp_accept_rate",
+        "0.85",
+        "--benchmark",
+        "decode",
+        "--context_lens",
+        "16384,32768,65536,131072,204800,256000",
+        "--max_batch_size",
+        "19"
+      ],
+      "log": "/mnt/afs/niushengxiao/LightLLM/benchmark_results/glm52-static-matrix-4n-mtp/decode_cap19_ctx16384-32768-65536-131072-204800-256000_n4_btok4096_mf0p76_mtp.rank2.log",
+      "returncode": 255
+    },
+    {
+      "host": "10.119.17.169",
+      "node_rank": 3,
+      "command": [
+        "/opt/conda/bin/python",
+        "test/benchmark/static_inference/test_model.py",
+        "--model_dir",
+        "/root/models/GLM-5.2/",
+        "--tp",
+        "32",
+        "--llm_kv_type",
+        "fp8kv_dsa",
+        "--batch_max_tokens",
+        "4096",
+        "--output_lens",
+        "600",
+        "--decode_batch_size_mode",
+        "profile",
+        "--quant_type",
+        "deepgemm-fp8w8a8-b128",
+        "--dp",
+        "32",
+        "--enable_ep_moe",
+        "--expert_dtype",
+        "fp8",
+        "--graph_max_batch_size",
+        "19",
+        "--mem_fraction",
+        "0.76",
+        "--nnodes",
+        "4",
+        "--node_rank",
+        "3",
+        "--nccl_host",
+        "10.119.17.190",
+        "--nccl_port",
+        "28817",
+        "--mtp_mode",
+        "eagle_with_att",
+        "--mtp_step",
+        "2",
+        "--mtp_draft_model_dir",
+        "/root/models/GLM-5.2/",
+        "--mtp_accept_rate",
+        "0.85",
+        "--benchmark",
+        "decode",
+        "--context_lens",
+        "16384,32768,65536,131072,204800,256000",
+        "--max_batch_size",
+        "19"
+      ],
+      "log": "/mnt/afs/niushengxiao/LightLLM/benchmark_results/glm52-static-matrix-4n-mtp/decode_cap19_ctx16384-32768-65536-131072-204800-256000_n4_btok4096_mf0p76_mtp.rank3.log",
+      "returncode": 255
+    }
+  ]
+}