wmantly/llama.cpp-xrip
1
0 Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

Merge upstream master

Browse Source 
Assisted-by: Codex
This commit is contained in:
xrip committed 2026-10-06 12:03:55 +03:00
2 parents 74facb828e 1a3011cc0c
commit 411bf3c608
584 files changed
+69752 -32248
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/llama-model.cpp
+131 -5
View File
@@ -326,6 +326,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_qwen35(params);
case LLM_ARCH_QWEN35MOE:
return new llama_model_qwen35moe(params);
case LLM_ARCH_CLEF:
return new llama_model_clef(params);
case LLM_ARCH_QWEN4EXP:
return new llama_model_qwen4exp(params);
case LLM_ARCH_MISTRAL3:
@@ -340,6 +342,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_kimi_linear(params);
case LLM_ARCH_KIMI_K3:
return new llama_model_kimi_k3(params);
case LLM_ARCH_GLM5_NEXT:
return new llama_model_glm5_next(params);
case LLM_ARCH_STEP35:
return new llama_model_step35(params);
case LLM_ARCH_SPARK2_5:
@@ -607,10 +611,10 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);
};
// if a model has fused tensors they need to be separated into "segments", see the comments on ggml_backend_meta_split_state struct
auto get_split_segments = [&](int axis, uint32_t il) -> std::vector<std::pair<int64_t, uint32_t>> {
// TODO: clarify why this is necessary specifically for these models
// TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN]
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE ||
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF ||
ud->model->arch == LLM_ARCH_QWEN4EXP) {
// fused full attention layers with Q gate tensors that need n_embd doubled:
@@ -765,7 +769,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
GGML_ASSERT(segments.size() == 1);
// some models have Q gate tensors, for those cases the granularity needs to be doubled:
// TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN]
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE ||
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF ||
ud->model->arch == LLM_ARCH_QWEN4EXP) {
return {std::lcm(2*n_embd_q, blck_size_perf)};
}
@@ -798,7 +802,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) {
// fused full attention layers need Q gate tensors handled like above:
// TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN]
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE ||
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF ||
ud->model->arch == LLM_ARCH_QWEN4EXP) {
return {std::lcm(2*n_embd_q, blck_size_perf), granularity_kv};
}
@@ -1018,6 +1022,7 @@ const char * llm_type_name(llm_type type) {
case LLM_TYPE_685B_A37B: return "685B.A37B";
case LLM_TYPE_744B_A40B: return "744B.A40B";
case LLM_TYPE_2_8T_A50B: return "2.8T.A50B";
case LLM_TYPE_320B_A18B: return "320B.A18B";
case LLM_TYPE_E2B: return "E2B";
case LLM_TYPE_E4B: return "E4B";
default: return "?B";
@@ -1226,6 +1231,56 @@ struct llama_model::impl {
std::vector<float> tensor_split_owned;
};
bool llama_prec_policy::apply(ggml_tensor * res) const {
if (!res || !res->src[0]) {
return false;
}
const auto it = prec_src1.find(res->src[0]);
if (it == prec_src1.end()) {
return false;
}
return ggml_prec_set_src(res, it->second, 1);
}
void llama_prec_policy::load(llama_model_loader & ml, const llama_model & model) {
std::vector<std::string> tensor_names;
if (!ml.get_arr(LLM_KV_GENERAL_TENSOR_EXTRA_NAME, tensor_names, false)) {
return;
}
const gguf_context * ctx = ml.metadata;
const std::string key = ml.llm_kv(LLM_KV_GENERAL_TENSOR_EXTRA_PREC_A4);
const int kid = gguf_find_key(ctx, key.c_str());
if (kid < 0 || gguf_get_kv_type(ctx, kid) != GGUF_TYPE_ARRAY || gguf_get_arr_type(ctx, kid) != GGUF_TYPE_BOOL) {
throw std::runtime_error(format("%s must be a bool array", key.c_str()));
}
const size_t n_values = gguf_get_arr_n(ctx, kid);
if (n_values != tensor_names.size()) {
throw std::runtime_error(format(
"%s tensor/value length mismatch (%zu vs %zu)",
key.c_str(), tensor_names.size(), n_values));
}
// tensors that can not use 4-bit activations, keep src1 at higher precision
const int8_t * values = (const int8_t *) gguf_get_arr_data(ctx, kid);
std::unordered_set<std::string> want;
for (size_t i = 0; i < n_values; ++i) {
if (values[i] == 0) {
want.insert(tensor_names[i]);
}
}
// resolve names to tensor pointers
for (const auto & [name, w] : model.tensors_by_name) {
if (want.count(name)) {
prec_src1.emplace(w, GGML_PREC_Q8);
}
}
}
llama_model::llama_model(const llama_model_params & params) : params(params), pimpl(std::make_unique<impl>()) {
if (params.tensor_split != nullptr) {
// llama_model_params stores tensor_split as a borrowed pointer, but the model
@@ -1277,6 +1332,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_EMBEDDING_LENGTH_OUT, hparams.n_embd_out_impl, false);
ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false);
ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);
ml.get_key(LLM_KV_CLASSIFIER_POOLING_TYPE, hparams.pooling_type_cls, false);
ml.get_key(LLM_KV_BLOCK_COUNT, hparams.n_layer_all);
GGML_ASSERT(hparams.n_layer_all > 0 && hparams.n_layer_all <= LLAMA_MAX_LAYERS);
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);
@@ -1737,8 +1793,15 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
}
}
}
ml.done_getting_tensors();
if (per_layer_tok_embd && ml.lazy.has(per_layer_tok_embd)) {
LLAMA_LOG_INFO("%s: enabling prefetch for '%s'\n", __func__, per_layer_tok_embd->name);
can_prefetch.insert(per_layer_tok_embd);
}
// Tied NVFP4 output is valid when no separate LM-head scale tensors are present.
// If sidecar scales exist, the output weight must be an actual output tensor.
GGML_ASSERT(!(output && tok_embd &&
@@ -1752,6 +1815,9 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
}
}
// per-tensor activation precision policy
prec_policy.load(ml, *this);
ml.init_mappings(true, use_mlock ? &pimpl->mlock_mmaps : nullptr);
pimpl->mappings.reserve(ml.mappings.size());
@@ -1904,7 +1970,11 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
}
ggml_tensor * llama_model_base::create_tensor(llama_model_loader & ml, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {
const buft_list_t * buft_list_layer = tn.bid == -1 ? nullptr : pimpl->dev_layer.at(tn.bid).buft_list;
const buft_list_t * buft_list_layer = nullptr;
if (tn.bid != -1) {
// blocks that are not model layers (e.g. the blocks of a head) go with the output
buft_list_layer = (size_t) tn.bid < pimpl->dev_layer.size() ? pimpl->dev_layer.at(tn.bid).buft_list : pimpl->dev_output.buft_list;
}
return ml.create_tensor(
hparams, &pimpl->cpu_buft_list, pimpl->dev_input.buft_list, pimpl->dev_output.buft_list, buft_list_layer,
tn, ne, flags);
@@ -2312,6 +2382,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
case LLM_ARCH_LLADA:
case LLM_ARCH_LLADA_MOE:
case LLM_ARCH_RND1:
case LLM_ARCH_CLEF:
{
res = nullptr;
} break;
@@ -2390,6 +2461,50 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
nullptr);
}
} break;
case LLM_ARCH_GLM5_NEXT:
{
// KDA layers are recurrent, the DSA layers use a K-only MLA cache plus an indexer cache.
// tThe Nextn block is never attended by the trunk graph
llama_memory_hybrid_idx::layer_filter_cb filter_attn = [&](uint32_t il) {
return il < hparams.n_layer() && !hparams.is_recr(il);
};
llama_memory_hybrid_idx::layer_filter_cb filter_idx = [&](uint32_t il) {
return il < hparams.n_layer() && !hparams.is_recr(il) && hparams.is_indexer_full(il);
};
llama_memory_hybrid_idx::layer_filter_cb filter_recr = [&](uint32_t il) {
return il < hparams.n_layer() && hparams.is_recr(il);
};
// the draft head is a single DSA layer
if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
if (hparams.n_layer_nextn == 0) {
throw std::runtime_error("GLM5-Next MTP requires the NextN block, convert without --no-mtp");
}
filter_attn = [&](uint32_t il) { return il >= hparams.n_layer(); };
filter_idx = [&](uint32_t il) { return il >= hparams.n_layer(); };
filter_recr = [&](uint32_t) { return false; };
}
res = new llama_memory_hybrid_idx(
/* model */ *this,
/* attn_type_k */ params.type_k,
/* attn_type_v */ params.type_v,
/* attn_v_trans */ !cparams.flash_attn,
/* attn_kv_size */ cparams.n_ctx_seq,
/* attn_n_pad */ 1,
/* attn_n_swa */ hparams.n_swa,
/* attn_swa_type */ hparams.swa_type,
/* recurrent_type_r */ GGML_TYPE_F32,
/* recurrent_type_s */ GGML_TYPE_F32,
/* recurrent_rs_size */ std::max((uint32_t) 1, cparams.n_seq_max),
/* n_seq_max */ cparams.n_seq_max,
/* n_rs_seq */ cparams.n_rs_seq,
/* offload */ cparams.offload_kqv,
/* unified */ cparams.kv_unified,
/* filter_attn */ std::move(filter_attn),
/* filter_recr */ std::move(filter_recr),
/* filter_idx */ std::move(filter_idx));
} break;
case LLM_ARCH_HY_V4:
{
if (hparams.indexer_top_k == 0) {
@@ -2608,6 +2723,15 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
return il < hparams.n_layer() && !hparams.is_recr(il);
};
}
// the MTP draft context holds the MTP block alone: its attention and indexer, no recurrent layer
if (arch == LLM_ARCH_QWEN4EXP && params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
filter_attn = [&](uint32_t il) { return il >= hparams.n_layer(); };
filter_recr = [&](uint32_t) { return false; };
if (filter_idx) {
filter_idx = [&](uint32_t il) { return il >= hparams.n_layer(); };
}
}
}
if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
@@ -2945,6 +3069,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_NEMOTRON_H_MOE:
case LLM_ARCH_KIMI_LINEAR:
case LLM_ARCH_KIMI_K3:
case LLM_ARCH_GLM5_NEXT:
return LLAMA_ROPE_TYPE_NONE;
// use what we call a normal RoPE, operating on pairs of consecutive head values
@@ -3090,6 +3215,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_QWEN3VLMOE:
case LLM_ARCH_QWEN35:
case LLM_ARCH_QWEN35MOE:
case LLM_ARCH_CLEF:
case LLM_ARCH_QWEN4EXP:
case LLM_ARCH_QWEN3TTS:
return LLAMA_ROPE_TYPE_IMROPE;