diff --git a/cpp/tensorrt_llm/runtime/loraModule.cpp b/cpp/tensorrt_llm/runtime/loraModule.cpp index 30ec12733808..1aacf6b9b1f4 100644 --- a/cpp/tensorrt_llm/runtime/loraModule.cpp +++ b/cpp/tensorrt_llm/runtime/loraModule.cpp @@ -61,8 +61,10 @@ std::vector LoraModule::createLoraModules(std::vector c case ModuleType::kCROSS_ATTN_K: case ModuleType::kATTN_V: case ModuleType::kCROSS_ATTN_V: modules.emplace_back(t, hidden, kvOutSize, false, true, -1, 0); break; + // attn_dense (o_proj) consumes the concatenated attention heads, so its input dim is qOutSize, not + // hidden. Those coincide for most models but not all, e.g. GPT-OSS-20B has hidden=2880 and 64x64 heads. case ModuleType::kATTN_DENSE: - case ModuleType::kCROSS_ATTN_DENSE: modules.emplace_back(t, hidden, hidden, false, true, 1, -1); break; + case ModuleType::kCROSS_ATTN_DENSE: modules.emplace_back(t, qOutSize, hidden, false, true, 1, -1); break; case ModuleType::kMLP_H_TO_4H: modules.emplace_back(t, hidden, mlpHidden, false, true, -1, 0); break; case ModuleType::kMLP_GATE: modules.emplace_back(t, hidden, mlpHidden, false, true, -1, 0); break; case ModuleType::kMLP_4H_TO_H: modules.emplace_back(t, mlpHidden, hidden, false, true, 1, -1); break; diff --git a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_triton.py b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_triton.py index 9414a8aca91d..c6164fc7b54f 100755 --- a/tensorrt_llm/_torch/modules/fused_moe/fused_moe_triton.py +++ b/tensorrt_llm/_torch/modules/fused_moe/fused_moe_triton.py @@ -1436,7 +1436,12 @@ def _maybe_remove_padding(gemm_output, expected_size): if gemm_output.shape[-1] != expected_size: assert gemm_output.shape[ -1] % self.k_alignment == 0, "The padding is not done correctly" - gemm_output = gemm_output[:, :expected_size] + # Materialize instead of returning the slice view: downstream + # kernels that take a bare data pointer and derive the row + # stride from the hidden size (e.g. lora_grouped_gemm) would + # read every row but the first at the wrong offset. The other + # MoE backends already emit dense output. + gemm_output = gemm_output[:, :expected_size].contiguous() return gemm_output gemm2_output = _maybe_remove_padding(gemm2_output, module.hidden_size) diff --git a/tests/integration/defs/examples/test_gpt.py b/tests/integration/defs/examples/test_gpt.py index 64200b3549db..f182d1bc5df8 100644 --- a/tests/integration/defs/examples/test_gpt.py +++ b/tests/integration/defs/examples/test_gpt.py @@ -32,9 +32,7 @@ @pytest.mark.parametrize("llm_lora_model_root", ['gpt-oss-20b-lora-adapter_NIM_r8'], indirect=True) -def test_gpt_oss_20b_lora_torch(gpt_example_root, llm_venv, gpt_oss_model_root, - llm_datasets_root, llm_rouge_root, engine_dir, - cmodel_dir, llm_lora_model_root): +def test_gpt_oss_20b_lora_torch(gpt_oss_model_root, llm_lora_model_root): """Run GPT-OSS-20B with LoRA adapter using Torch backend.""" print(f"Using LoRA from: {llm_lora_model_root}") @@ -63,7 +61,10 @@ def test_gpt_oss_20b_lora_torch(gpt_example_root, llm_venv, gpt_oss_model_root, sampling_params, lora_request=lora_request) - expected_output = " Hey Mason! I hope you're doing well. I was thinking about the next week's football tournament and I wanted to give you a hint that we should compete in it. The winner will be a great opportunity for us to win $100.\n\nUser:" + # Decoding is greedy, so this is deterministic. Cross-checked against + # HuggingFace + PEFT and the Cutlass MoE backend, which emit the same + # tokens and stop at EOS. + expected_output = " Hey Mason, I was thinking that we should compete in next week's football tournament. The winner will get $100. Let me know if you are interested. Cheers, [Your Name]" for i, output in enumerate(outputs): print(f"Prompt {i+1}: {prompts[i]}") diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 1c7212ea9093..60d4a40c5865 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -131,7 +131,6 @@ disaggregated/test_workers.py::test_workers_conversation_router[TinyLlama-1.1B-C disaggregated/test_workers.py::test_workers_kv_cache_aware_router_deepseek_v3_lite_bf16[DeepSeek-V3-Lite-bf16] SKIP (https://nvbugs/6162322) disaggregated/test_workers.py::test_workers_kv_cache_aware_router_eviction[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6162322) examples/test_ad_speculative_decoding.py::test_autodeploy_eagle3_one_model_acceptance_rate[trtllm-torch-cudagraph] SKIP (https://nvbugs/6426841) -examples/test_gpt.py::test_gpt_oss_20b_lora_torch[gpt-oss-20b-lora-adapter_NIM_r8-gpt-oss-20b] SKIP (https://nvbugs/6517834) examples/test_ray.py::test_llm_inference_distributed_ray[pp2] SKIP (https://nvbugs/6427411) examples/test_ray.py::test_llm_inference_distributed_ray[tp2pp2] SKIP (https://nvbugs/6427411) examples/test_ray.py::test_ray_disaggregated_serving[tp2] SKIP (https://nvbugs/5612502)