From a6a9c30d9d429373c7d9c10752ce47c23d34dae2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20du=20Hamel?= Date: Thu, 30 Jul 2026 14:52:36 +0200 Subject: [PATCH 1/3] lora: fix split qkv compatibility check --- src/model/adapter/lora.hpp | 123 ++++++++++++++++++------------------- 1 file changed, 60 insertions(+), 63 deletions(-) diff --git a/src/model/adapter/lora.hpp b/src/model/adapter/lora.hpp index 71a36f91b..ac34368c5 100644 --- a/src/model/adapter/lora.hpp +++ b/src/model/adapter/lora.hpp @@ -588,6 +588,10 @@ struct LoraModel : public GGMLRunner { const std::string& model_tensor_name) { ggml_tensor* out_diff = nullptr; int index = 0; + + std::vector used_tensors; + bool is_conv2d = forward_params.op_type == WeightAdapter::ForwardParams::op_type_t::OP_CONV2D; + while (true) { std::string key; if (index == 0) { @@ -595,7 +599,6 @@ struct LoraModel : public GGMLRunner { } else { key = model_tensor_name + "." + std::to_string(index); } - bool is_conv2d = forward_params.op_type == WeightAdapter::ForwardParams::op_type_t::OP_CONV2D; std::string lokr_w1_name = "lora." + key + ".lokr_w1"; std::string lokr_w1_a_name = "lora." + key + ".lokr_w1_a"; @@ -663,7 +666,6 @@ struct LoraModel : public GGMLRunner { if (iter != lora_tensors.end()) { float alpha = ggml_ext_backend_tensor_get_f32(iter->second); scale_value = alpha / rank; - applied_lora_tensors.insert(alpha_name); } if (rank == 1) { @@ -678,19 +680,27 @@ struct LoraModel : public GGMLRunner { out_diff = ggml_concat(ctx, out_diff, curr_out_diff, 0); } - if (lokr_w1) - applied_lora_tensors.insert(lokr_w1_name); - if (lokr_w1_a) - applied_lora_tensors.insert(lokr_w1_a_name); - if (lokr_w1_b) - applied_lora_tensors.insert(lokr_w1_b_name); - if (lokr_w2) - applied_lora_tensors.insert(lokr_w2_name); - if (lokr_w2_a) - applied_lora_tensors.insert(lokr_w2_a_name); - if (lokr_w2_b) - applied_lora_tensors.insert(lokr_w2_b_name); - applied_lora_tensors.insert(alpha_name); + if (lokr_w1) { + used_tensors.push_back(lokr_w1_name); + } + if (lokr_w1_a) { + used_tensors.push_back(lokr_w1_a_name); + } + if (lokr_w1_b) { + used_tensors.push_back(lokr_w1_b_name); + } + if (lokr_w2) { + used_tensors.push_back(lokr_w2_name); + } + if (lokr_w2_a) { + used_tensors.push_back(lokr_w2_a_name); + } + if (lokr_w2_b) { + used_tensors.push_back(lokr_w2_b_name); + } + if (iter != lora_tensors.end()) { + used_tensors.push_back(alpha_name); + } index++; continue; @@ -736,64 +746,18 @@ struct LoraModel : public GGMLRunner { break; } - if (!is_conv2d) { - const int64_t down_in = lora_down->ne[0]; - const int64_t down_out = lora_down->ne[1]; - const int64_t up_in = lora_up->ne[0]; - const int64_t up_out = lora_up->ne[1]; - - bool compatible = down_in == model_weight->ne[0] && - up_out == model_weight->ne[1]; - if (lora_mid != nullptr) { - compatible = compatible && - lora_mid->ne[0] == down_out && - up_in == lora_mid->ne[1]; - } else { - compatible = compatible && up_in == down_out; - } - - if (!compatible) { - skipped_incompatible_lora_tensors.insert(lora_down_name); - skipped_incompatible_lora_tensors.insert(lora_up_name); - skipped_incompatible_lora_tensors.insert(lora_mid_name); - skipped_incompatible_lora_tensors.insert(scale_name); - skipped_incompatible_lora_tensors.insert(alpha_name); - if (warned_incompatible_model_tensors.insert(model_tensor_name).second) { - LOG_WARN("skip incompatible LoRA tensor |%s|: model shape = [%lld, %lld], down shape = [%lld, %lld], up shape = [%lld, %lld]", - model_tensor_name.c_str(), - static_cast(model_weight->ne[0]), - static_cast(model_weight->ne[1]), - static_cast(down_in), - static_cast(down_out), - static_cast(up_in), - static_cast(up_out)); - } - index++; - continue; - } - } - - applied_lora_tensors.insert(lora_up_name); - applied_lora_tensors.insert(lora_down_name); - - if (lora_mid) { - applied_lora_tensors.insert(lora_mid_name); - } - float scale_value = 1.0f; int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1]; iter = lora_tensors.find(scale_name); if (iter != lora_tensors.end()) { scale_value = ggml_ext_backend_tensor_get_f32(iter->second); - applied_lora_tensors.insert(scale_name); } else { iter = lora_tensors.find(alpha_name); if (iter != lora_tensors.end()) { float alpha = ggml_ext_backend_tensor_get_f32(iter->second); scale_value = alpha / rank; // LOG_DEBUG("rank %s %ld %.2f %.2f", alpha_name.c_str(), rank, alpha, scale_value); - applied_lora_tensors.insert(alpha_name); } } scale_value *= multiplier; @@ -853,15 +817,48 @@ struct LoraModel : public GGMLRunner { } auto curr_out_diff = ggml_ext_scale(ctx, lx, scale_value, true); - if (out_diff == nullptr) { out_diff = curr_out_diff; } else { - out_diff = ggml_concat(ctx, out_diff, curr_out_diff, 0); + out_diff = ggml_concat(ctx, out_diff, curr_out_diff, is_conv2d ? 2 : 0); + } + + used_tensors.push_back(lora_up_name); + used_tensors.push_back(lora_down_name); + if (lora_mid) { + used_tensors.push_back(lora_mid_name); + } + if (lora_tensors.find(scale_name) != lora_tensors.end()) { + used_tensors.push_back(scale_name); + } + if (lora_tensors.find(alpha_name) != lora_tensors.end()) { + used_tensors.push_back(alpha_name); } index++; } + + if (out_diff == nullptr) + return nullptr; + + int64_t expected_out_dim = is_conv2d ? model_weight->ne[3] : model_weight->ne[1]; + int64_t actual_out_dim = out_diff->ne[is_conv2d ? 2 : 0]; + + if (actual_out_dim != expected_out_dim) { + for (const auto& name : used_tensors) { + skipped_incompatible_lora_tensors.insert(name); + } + if (warned_incompatible_model_tensors.insert(model_tensor_name).second) { + LOG_WARN("skip incompatible LoRA tensors for |%s|: output dim %lld != model dim %lld", + model_tensor_name.c_str(), actual_out_dim, expected_out_dim); + } + return nullptr; + } + + for (const auto& name : used_tensors) { + applied_lora_tensors.insert(name); + } + return out_diff; } From 3ba6422fb8897bf2e8e21a9c1bc469341b05a2a7 Mon Sep 17 00:00:00 2001 From: leejet Date: Sun, 2 Aug 2026 15:46:27 +0800 Subject: [PATCH 2/3] fix: preserve runtime LoRA validation for split QKV --- src/model/adapter/lora.hpp | 53 +++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/src/model/adapter/lora.hpp b/src/model/adapter/lora.hpp index ac34368c5..735eef1d4 100644 --- a/src/model/adapter/lora.hpp +++ b/src/model/adapter/lora.hpp @@ -746,17 +746,59 @@ struct LoraModel : public GGMLRunner { break; } + if (!is_conv2d) { + const int64_t down_in = lora_down->ne[0]; + const int64_t down_out = lora_down->ne[1]; + const int64_t up_in = lora_up->ne[0]; + + bool compatible = down_in == model_weight->ne[0]; + if (lora_mid != nullptr) { + compatible = compatible && + lora_mid->ne[0] == down_out && + up_in == lora_mid->ne[1]; + } else { + compatible = compatible && up_in == down_out; + } + + if (!compatible) { + skipped_incompatible_lora_tensors.insert(lora_down_name); + skipped_incompatible_lora_tensors.insert(lora_up_name); + if (lora_mid != nullptr) { + skipped_incompatible_lora_tensors.insert(lora_mid_name); + } + if (lora_tensors.find(scale_name) != lora_tensors.end()) { + skipped_incompatible_lora_tensors.insert(scale_name); + } else if (lora_tensors.find(alpha_name) != lora_tensors.end()) { + skipped_incompatible_lora_tensors.insert(alpha_name); + } + if (warned_incompatible_model_tensors.insert(model_tensor_name).second) { + LOG_WARN("skip incompatible LoRA tensor |%s|: model input dim = %lld, down shape = [%lld, %lld], up shape = [%lld, %lld]", + model_tensor_name.c_str(), + static_cast(model_weight->ne[0]), + static_cast(down_in), + static_cast(down_out), + static_cast(up_in), + static_cast(lora_up->ne[1])); + } + index++; + continue; + } + } + float scale_value = 1.0f; + std::string scale_tensor_name; int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1]; iter = lora_tensors.find(scale_name); if (iter != lora_tensors.end()) { - scale_value = ggml_ext_backend_tensor_get_f32(iter->second); + scale_value = ggml_ext_backend_tensor_get_f32(iter->second); + scale_tensor_name = scale_name; } else { iter = lora_tensors.find(alpha_name); if (iter != lora_tensors.end()) { float alpha = ggml_ext_backend_tensor_get_f32(iter->second); - scale_value = alpha / rank; + scale_value = alpha / rank; + scale_tensor_name = alpha_name; // LOG_DEBUG("rank %s %ld %.2f %.2f", alpha_name.c_str(), rank, alpha, scale_value); } } @@ -828,11 +870,8 @@ struct LoraModel : public GGMLRunner { if (lora_mid) { used_tensors.push_back(lora_mid_name); } - if (lora_tensors.find(scale_name) != lora_tensors.end()) { - used_tensors.push_back(scale_name); - } - if (lora_tensors.find(alpha_name) != lora_tensors.end()) { - used_tensors.push_back(alpha_name); + if (!scale_tensor_name.empty()) { + used_tensors.push_back(scale_tensor_name); } index++; From 355535df2c5789f0247303e4bb95d0c674d6ffbc Mon Sep 17 00:00:00 2001 From: leejet Date: Sun, 2 Aug 2026 15:47:08 +0800 Subject: [PATCH 3/3] format code --- src/model/adapter/lora.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/model/adapter/lora.hpp b/src/model/adapter/lora.hpp index 735eef1d4..8ddfca24f 100644 --- a/src/model/adapter/lora.hpp +++ b/src/model/adapter/lora.hpp @@ -796,7 +796,7 @@ struct LoraModel : public GGMLRunner { } else { iter = lora_tensors.find(alpha_name); if (iter != lora_tensors.end()) { - float alpha = ggml_ext_backend_tensor_get_f32(iter->second); + float alpha = ggml_ext_backend_tensor_get_f32(iter->second); scale_value = alpha / rank; scale_tensor_name = alpha_name; // LOG_DEBUG("rank %s %ld %.2f %.2f", alpha_name.c_str(), rank, alpha, scale_value);