wmantly/llama.cpp-xrip
1
0 Fork 0
Code Issues Pull Requests Actions 33 Packages Projects Releases Wiki Activity
Files
7775ffb7e48fa0c7dcffd5815743b3d2fb3b6761
llama.cpp-xrip/tests/test-llama-archs.cpp
T
Maphist0andSigbjørn Skjæret 7775ffb7e4 model: Add support for SenseNova U1 model (#28919) 
* model : add SenseNova U1 understanding support

Add support for the dense SenseNova U1 understanding branch, including
the U1.5-8B-MoT model.

- Add HF-to-GGUF conversion for NEOChatModel, excluding MoT generation
  tensors and concatenating temporal and spatial Q/K normalization
  weights.
- Register the sensenova_u1 GGUF architecture and custom spatial RoPE
  metadata.
- Implement temporal/spatial Q/K normalization with 4-axis RoPE.
- Keep U1-specific text position handling in the model implementation.
- Add synthetic architecture coverage for sensenova_u1.

Only the dense understanding branch is supported; generation
branches are intentionally rejected during conversion.

* conversion : treat one-expert SenseNova U1 configs as dense

* mtmd : add SenseNova U1 vision support

Add conversion support for the SenseNova U1 vision encoder and implement
its CPU mtmd projector graph.

The graph applies patch embedding, GELU, adjacent-pair 2D RoPE, spatial
downsampling, and dense projection for dynamic-resolution image inputs.

* mtmd : fix SenseNova U1 vision token layout

* model : clean up SenseNova U1 RoPE metadata

Replace the hard-coded spatial RoPE key with shared GGUF and LLM_KV
definitions used by conversion, model loading, and architecture tests.

Clarify why SenseNova U1 retains its custom text-position input: U1 uses
[t, 0, 0, 0], while the generic M-RoPE input uses [t, t, t, 0].

* mtmd : simplify SenseNova U1 projector integration

Map the SenseNova U1 dense projection to the standard mm.0 weight and
bias tensors and reuse the shared projector fields in the loader and graph.

Read vision parameters from the checkpoint configuration and remove the
U1-specific tensor names and model fields while preserving its strided
projection, bias, and row-major token layout.

* mtmd : fix SenseNova U1 image positions and attention

* fix spacing

* mtmd : restore generic test input chunks

* mtmd : restore SenseNova U1 integration after upstream merge

* Update conversion/sensenova_u1.py

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

* Update src/models/sensenova-u1.cpp

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

* model : remove SenseNova U1 BF16 activation casts

Use standard backend precision handling for QKV, FFN, and output
projections instead of casting inputs based on the output tensor type.

Assisted-by: Codex

* Update src/models/sensenova-u1.cpp

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
2026-10-21 21:54:16 +02:00

1337 lines
58 KiB
C++
Raw Blame History

#include "common.h"
#include "log.h"
#include "ggml-backend.h"
#include "ggml.h"
#include "gguf.h"
#include "ggml-cpp.h"
#include "llama.h"
#include "llama-cpp.h"
// TODO: replace with #include "llama-ext.h" in the future
#include "../src/llama-arch.h"
#include "../src/llama-model-saver.h"
#include <cinttypes>
#include <cmath>
#include <cstddef>
#include <cstdio>
#include <cstring>
#include <cstdint>
#include <random>
#include <regex>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
static bool arch_matches(const std::string & filter, llm_arch arch) {
if (filter.empty()) {
return true;
}
return std::regex_search(llm_arch_name(arch), std::regex(filter));
}
// normalized mean squared error = mse(a, b) / mse(a, 0)
static double nmse(const std::vector<float> & a, const std::vector<float> & b) {
GGML_ASSERT(a.size() == b.size());
double mse_a_b = 0.0;
double mse_a_0 = 0.0;
for (size_t i = 0; i < a.size(); i++) {
float a_i = a[i];
float b_i = b[i];
mse_a_b += (a_i - b_i) * (a_i - b_i);
mse_a_0 += a_i * a_i;
}
return mse_a_b / mse_a_0;
}
struct tensor_data_params {
size_t seed;
float stdev;
};
// fills the values of one tensor, the name carries whatever the property needs
typedef void (*tensor_filler)(std::vector<float> & dst, const char * name, const tensor_data_params & params);
// a normal draw keyed on a name, so that tensors which share a draw ask for the same key
static void fill_normal(std::vector<float> & dst, const char * key, const tensor_data_params & params) {
std::mt19937 gen(params.seed ^ std::hash<std::string>{}(key));
std::normal_distribution<float> dis(0.0f, params.stdev);
for (float & v : dst) {
v = dis(gen);
}
}
// Mamba A must be negative (state decay)
static void fill_ssm_a(std::vector<float> & dst, const char * name, const tensor_data_params & params) {
fill_normal(dst, name, params);
for (float & v : dst) {
v = -fabsf(v);
}
}
// a norm preserving residual gate holds alpha^2 + beta^2 == 1, and the conversion resolves both from one raw weight
// the alpha and the beta of a block share that weight, so the key drops the suffix and keeps the block
static void fill_resid_gate(std::vector<float> & dst, const char * name, const tensor_data_params & params) {
const char * suffix = strstr(name, "_resid_");
const bool is_beta = strstr(suffix, "_resid_beta") != nullptr;
fill_normal(dst, std::string(name, suffix - name).c_str(), params);
for (float & v : dst) {
const float beta = 1.0f / (1.0f + expf(-v));
v = is_beta ? beta : sqrtf(1.0f - beta*beta);
}
}
// a random router bias can tie within the noise between the backends, a ramp keeps the top-k selection the same
static void fill_exp_probs_b(std::vector<float> & dst, const char *, const tensor_data_params &) {
for (size_t i = 0; i < dst.size(); i++) {
dst[i] = 2.0f*i;
}
}
// a tensor whose values hold a property of their own, matched on a substring of its name, first match wins
static const struct { const char * match; tensor_filler fill; } TENSOR_FILLERS[] = {
{ "ssm_a", fill_ssm_a },
{ "_resid_", fill_resid_gate },
{ "exp_probs_b", fill_exp_probs_b },
};
static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {
const tensor_data_params & params = *(const tensor_data_params *) userdata;
tensor_filler fill = fill_normal;
for (const auto & filler : TENSOR_FILLERS) {
if (strstr(tensor->name, filler.match) != nullptr) {
fill = filler.fill;
break;
}
}
std::vector<float> tmp(ggml_nelements(tensor));
fill(tmp, tensor->name, params);
if (tensor->type == GGML_TYPE_F32) {
ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor));
} else if (tensor->type == GGML_TYPE_F16) {
std::vector<ggml_fp16_t> tmp16(tmp.size());
for (size_t i = 0; i < tmp.size(); i++) {
tmp16[i] = ggml_fp32_to_fp16(tmp[i]);
}
ggml_backend_tensor_set(tensor, tmp16.data(), 0, ggml_nbytes(tensor));
} else {
GGML_ABORT("fatal error");
}
}
static void usage(char ** argv) {
LOG("Usage: %s [options]\n\n", argv[0]);
LOG("Options:\n");
LOG(" -a, --arch <arch|regex> Run only matching LLM architectures (default: all supported)\n");
LOG(" -s, --seed <seed> Set the random seed for tensor initialization and token generation\n");
LOG(" -d, --stdev <stdev> Set the standard deviation of the tensor initialization distribution (default: 0.1f)\n");
LOG(" -o, --out <dir> Save generated test models to <dir> instead of running backend tests\n");
LOG(" -v <N> Set log verbosity level\n");
LOG(" -b, --backend <backend> Run only on the given backend device\n");
LOG(" -h, --help Show this help message\n\n");
LOG("Examples:\n");
LOG(" %s\n", argv[0]);
LOG(" %s -a qwen35moe\n", argv[0]);
LOG(" %s -a deepseek4 -o tests/test-models/\n", argv[0]);
LOG(" %s -a cohere2moe -v 5\n", argv[0]);
}
static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed){
std::mt19937 gen(seed);
std::uniform_int_distribution<> dis(0, n_vocab - 1);
std::vector<llama_token> ret;
ret.reserve(n_tokens);
for (uint32_t i = 0; i < n_tokens; i++) {
ret.push_back(dis(gen));
}
return ret;
}
// MoE archs that are also tested with the experts in host memory
static bool host_experts_test(const llm_arch arch) {
switch (arch) {
case LLM_ARCH_DEEPSEEK2:
return true;
default:
return false;
}
}
static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
gguf_context_ptr ret(gguf_init_empty());
llama_model_saver ms(arch, ret.get());
const uint32_t n_ctx = 256;
uint32_t n_vocab = 128;
uint32_t n_embd = 256;
uint32_t n_head = 2;
uint32_t n_ff = 384;
uint32_t n_layer = 2;
if (arch == LLM_ARCH_LLAMA4) {
n_layer = 4; // hparams.n_no_rope_layer_step is hard-coded to 4
} else if (arch == LLM_ARCH_MAION_CODER) {
n_head = 4;
n_layer = 4; // dense and MoE layers, each with local and full attention
} else if (arch == LLM_ARCH_GEMMA4) {
n_embd = 128;
n_head = 2;
n_ff = 192;
n_layer = 5; // need at least 5 for swa_pattern (every 5th is full_attention)
} else if (arch == LLM_ARCH_GEMMA3N) {
n_embd = 64;
n_head = 1;
n_ff = 96;
n_layer = 22; // hparams.n_layer_kv_from_start = 20 is hardcoded
} else if (arch == LLM_ARCH_DEEPSEEK4) {
// head size 64 so that GPU flash attention kernels support the model
n_embd = 512;
n_head = 8;
n_ff = 1024;
n_layer = 4;
} else if (arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_LAGUNA) {
n_embd = 160; // exercise per-head tensor split granularity with head size 80
} else if (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE) {
n_head = 4;
} else if (arch == LLM_ARCH_DEEPSEEK2
|| arch == LLM_ARCH_DEEPSEEK32
|| arch == LLM_ARCH_GLM_DSA
|| arch == LLM_ARCH_DOTS3NOTE
|| arch == LLM_ARCH_KIMI_LINEAR
|| arch == LLM_ARCH_BAILINGMOE3
|| arch == LLM_ARCH_KIMI_K3
|| arch == LLM_ARCH_GLM5_NEXT
|| arch == LLM_ARCH_MISTRAL4
|| arch == LLM_ARCH_HY_V4) {
n_embd = 128;
n_head = 1;
n_ff = 192;
} else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {
n_layer = 3;
} else if (arch == LLM_ARCH_CHAMELEON) {
n_vocab = 10240;
} else if (arch == LLM_ARCH_QWEN3TTS) {
//n_vocab = 4096; // must be >= the hard-coded codec head size (3072)
n_vocab = 3072; // TODO: should be 4096, but user code cannot get `n_vocab_out` yet [TAG_LLAMA_N_VOCAB_OUT]
} else if (arch == LLM_ARCH_HRM_TEXT) {
n_layer = 6; // 1 layer per stack x 2 h-cycles x (2 l-cycles + 1) cache slots
}
uint32_t n_head_kv = n_head;
if (arch == LLM_ARCH_QWEN3) {
n_head_kv = 1; // MQA coverage
} else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER ||
arch == LLM_ARCH_KOLIBRI1) {
n_head_kv = 2; // GQA coverage
}
if (arch == LLM_ARCH_BEAM) {
n_head = 4;
n_head_kv = 2;
n_layer = 4;
}
const uint32_t n_embd_head = n_embd / n_head;
ms.add_kv(LLM_KV_GENERAL_ARCHITECTURE, llm_arch_name(arch));
ms.add_kv(LLM_KV_VOCAB_SIZE, n_vocab);
ms.add_kv(LLM_KV_CONTEXT_LENGTH, n_ctx);
ms.add_kv(LLM_KV_EMBEDDING_LENGTH, n_embd);
ms.add_kv(LLM_KV_FEATURES_LENGTH, n_embd);
ms.add_kv(LLM_KV_BLOCK_COUNT, n_layer);
ms.add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, uint32_t(1));
if (arch == LLM_ARCH_K2_HORIZON) {
ms.add_kv(LLM_KV_ROPE_SCALING_YARN_BETA_FAST, 128.0f);
ms.add_kv(LLM_KV_ROPE_SCALING_YARN_BETA_SLOW, 4.0f);
}
if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {
std::vector<uint32_t> n_ff_per_layer;
n_ff_per_layer.reserve(n_layer);
for (uint32_t il = 0; il < n_layer; il++) {
n_ff_per_layer.push_back(il <= 1 ? 0 : n_ff);
}
ms.add_kv(LLM_KV_FEED_FORWARD_LENGTH, n_ff_per_layer);
} else {
ms.add_kv(LLM_KV_FEED_FORWARD_LENGTH, n_ff);
}
ms.add_kv(LLM_KV_USE_PARALLEL_RESIDUAL, false);
ms.add_kv(LLM_KV_LOGIT_SCALE, 1.0f);
ms.add_kv(LLM_KV_TIME_MIX_EXTRA_DIM, uint32_t(64));
ms.add_kv(LLM_KV_TIME_DECAY_EXTRA_DIM, uint32_t(128));
ms.add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, uint32_t(2));
if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE ||
arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_KIMI_LINEAR ||
arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3 || arch == LLM_ARCH_GLM5_NEXT) {
GGML_ASSERT(n_layer >= 2);
std::vector<uint32_t> n_head_per_layer;
n_head_per_layer.reserve(n_layer);
for (uint32_t il = 0; il < n_layer; il++) {
n_head_per_layer.push_back(il == 1 ? 0 : n_head);
}
// GLM5 next KDA heads come from the uniform head count, only head_count_kv is per layer.
if (arch == LLM_ARCH_GLM5_NEXT) {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);
} else {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head_per_layer);
}
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer);
} else {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(1) : n_head_kv);
}
ms.add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, 8.0f);
if (arch == LLM_ARCH_DEEPSEEK4) {
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, n_embd_head);
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, n_embd_head);
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, n_embd_head/2);
} else if (arch == LLM_ARCH_DEEPSEEK2
|| arch == LLM_ARCH_DEEPSEEK32
|| arch == LLM_ARCH_GLM_DSA
|| arch == LLM_ARCH_DOTS3NOTE
|| arch == LLM_ARCH_KIMI_LINEAR
|| arch == LLM_ARCH_BAILINGMOE3
|| arch == LLM_ARCH_KIMI_K3
|| arch == LLM_ARCH_GLM5_NEXT
|| arch == LLM_ARCH_HY_V4
|| arch == LLM_ARCH_MISTRAL4) {
// GLM5 next MLA is nope only, the cache row is the compressed latent alone.
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, arch == LLM_ARCH_GLM5_NEXT ? uint32_t(512) : uint32_t(576));
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, uint32_t(512));
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, arch == LLM_ARCH_GLM5_NEXT ? uint32_t(0) : uint32_t(64));
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA, uint32_t(192));
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, uint32_t(128));
if (arch == LLM_ARCH_DOTS3NOTE) {
// SWA layers reuse the same MLA geometry as the full layers in this fixture
ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK_SWA, uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, uint32_t(576));
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, uint32_t(192));
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, uint32_t(128));
ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f);
// indexer on the full-attention layers (inverse of the swa pattern)
std::vector<uint32_t> indexer_types;
indexer_types.reserve(n_layer);
for (uint32_t il = 0; il < n_layer; il++) {
indexer_types.push_back(il % 2 ? 0 : 1);
}
ms.add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, indexer_types);
}
} else if (arch == LLM_ARCH_MINIMAX_M3) {
// partial rotary: n_rot must not exceed the indexer key length (64)
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, uint32_t(64));
}
if (arch == LLM_ARCH_SENSENOVA_U1) {
ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SPATIAL, 10000.0f);
}
ms.add_kv(LLM_KV_ATTENTION_CLAMP_KQV, 1.0f);
ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_EPS, 1e-5f);
ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, 1e-5f);
ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_EPS, 1e-5f);
ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_GROUPS, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_Q_LORA_RANK, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(64) : uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK, uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, n_ctx/8);
if (arch == LLM_ARCH_MAION_CODER) {
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector<uint32_t>{1, 1, 0, 0});
} else if (arch == LLM_ARCH_BEAM) {
std::vector<uint32_t> pattern;
pattern.reserve(n_layer);
for (uint32_t il = 0; il < n_layer; il++) {
pattern.push_back(il % 2);
}
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);
} else if (arch == LLM_ARCH_GEMMA4) {
ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, n_embd/2);
ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS, uint32_t(0));
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, n_embd_head);
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, n_embd_head);
ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f);
// SWA pattern: every 5th layer is full attention (matches E2B layer_types)
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5));
} else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 ||
arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE ||
arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE) {
std::vector<uint32_t> pattern;
pattern.reserve(n_layer);
for (uint32_t il = 0; il < n_layer; il++) {
pattern.push_back(il % 2);
}
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);
} else {
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(2));
}
if (arch == LLM_ARCH_MUSE_SPARK) {
ms.add_kv(LLM_KV_POST_NORM_EPS, 1e-5f);
ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f);
ms.add_kv(LLM_KV_FINAL_LOGIT_SOFTCAPPING, 30.0f);
ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd/2);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
}
if (arch == LLM_ARCH_KOLIBRI1) {
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, false);
}
// MSA requires one indexer head per GQA (KV) head, unlike the DSA archs where the
// indexer head count is independent of the main attention head count.
if (arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_GLM5_NEXT) {
ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2));
ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);
ms.add_kv(LLM_KV_HYPER_CONNECTION_LOW_RANK, uint32_t(8));
// without this the QSA layers fall back to dense and go uncovered
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>(n_layer, 4));
// has_cell_ext() needs ple_n_heads here: the indexer cache serializes no ext without it
const uint32_t ple_ngram_size = 3;
const uint32_t ple_heads_per_ngram = 2;
const uint32_t ple_n_heads = (ple_ngram_size - 1)*ple_heads_per_ngram;
GGML_ASSERT(n_embd % ple_n_heads == 0);
const uint32_t ple_head_dim = n_embd/ple_n_heads;
std::vector<uint64_t> ple_head_offsets(ple_n_heads);
std::vector<uint64_t> ple_head_vocab_sizes(ple_n_heads, n_vocab);
for (uint32_t h = 0; h < ple_n_heads; h++) {
ple_head_offsets[h] = uint64_t(h)*n_vocab;
}
// the PLE history lives in the recurrent cache, so it must sit on a linear attention layer
ms.add_kv(LLM_KV_PLE_LAYERS, std::vector<uint32_t>({ 0 }));
ms.add_kv(LLM_KV_PLE_NGRAM_SIZE, ple_ngram_size);
ms.add_kv(LLM_KV_PLE_HEADS_PER_NGRAM, ple_heads_per_ngram);
ms.add_kv(LLM_KV_PLE_CONV_KERNEL, uint32_t(4));
ms.add_kv(LLM_KV_PLE_EOS_TOKEN_ID, uint32_t(0));
ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, ple_head_dim);
ms.add_kv(LLM_KV_PLE_LAYER_MULTIPLIERS, std::vector<uint64_t>({ 1, 3, 5 }));
ms.add_kv(LLM_KV_PLE_HEAD_OFFSETS, ple_head_offsets);
ms.add_kv(LLM_KV_PLE_HEAD_VOCAB_SIZES, ple_head_vocab_sizes);
}
// minimax-m3 keeps one indexer head per GQA head; the rest use a fixed 64 to match the fused
ms.add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, arch == LLM_ARCH_MINIMAX_M3 ? n_head : uint32_t(64));
// qwen4exp ropes indexer keys with the main rotary width, so its head can't be < n_rot
ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,
arch == LLM_ARCH_QWEN4EXP ? n_embd_head : uint32_t(128));
// note: using a realistic top-k here makes the results unstable and hard to match between CPU and GPU
// a large value makes things deterministic since all data is selected by the indexer
//ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(131072));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, uint32_t(4));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_KPOOL, uint32_t(4));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_KPOOL_SELECT_TAIL, true);
ms.add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, uint32_t(1));
// mrope sections count rope pairs; Ling 3.0 VL files carry [t, h, w] sections
// summing to n_rot / 2 (n_rot is 64 in this fixture)
if (arch == LLM_ARCH_BAILINGMOE3) {
ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector<uint32_t>({8, 12, 12, 0}));
} else {
ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector<uint32_t>({n_embd_head/4, n_embd_head/4, n_embd_head/4, n_embd_head/4}));
}
if (arch == LLM_ARCH_HY_V4) {
ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);
ms.add_kv(LLM_KV_HYPER_CONNECTION_MAGNITUDE, 2.0f);
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 10.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
// layer 0 must own an indexer, the odd layers share it
std::vector<uint32_t> indexer_types;
indexer_types.reserve(n_layer);
for (uint32_t il = 0; il < n_layer; il++) {
indexer_types.push_back(il % 2 ? 0 : 1);
}
ms.add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, indexer_types);
}
if (arch == LLM_ARCH_DEEPSEEK4) {
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(32));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 0, 4, 128}));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 160000.0f);
ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2));
ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);
ms.add_kv(LLM_KV_HASH_LAYER_COUNT, uint32_t(0));
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 10.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
}
if (arch == LLM_ARCH_HRM_TEXT) {
// 6 cache slots alias 2 physical blocks: 1 low-stack layer + 1 high-stack layer
ms.add_kv(LLM_KV_HRM_LAYERS_PER_STACK, uint32_t(1));
ms.add_kv(LLM_KV_HRM_H_CYCLES, uint32_t(2));
ms.add_kv(LLM_KV_HRM_L_CYCLES, uint32_t(2));
}
if (arch == LLM_ARCH_MAPLE) {
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f);
}
// dummy tokenizer: token ids are derived from fixed-size chunks and detokenized as hex ids
{
std::vector<std::string> tokenizer_list(n_vocab);
std::vector<float> tokenizer_scores(n_vocab, 0.0f);
ms.add_kv(LLM_KV_TOKENIZER_MODEL, "test");
for (uint32_t i = 0; i < n_vocab; i++) {
tokenizer_list[i] = "tok_" + std::to_string(i);
}
ms.add_kv(LLM_KV_TOKENIZER_LIST, tokenizer_list);
ms.add_kv(LLM_KV_TOKENIZER_SCORES, tokenizer_scores);
}
// ms.add_kv(LLM_KV_DENSE_2_FEAT_OUT, n_embd);
// ms.add_kv(LLM_KV_DENSE_3_FEAT_IN, n_embd);
if (moe) {
ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, n_ff);
ms.add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, n_ff / 2); // distinct from n_ff so a saver key-clobber surfaces on reload
ms.add_kv(LLM_KV_EXPERT_LATENT_LENGTH, n_ff);
ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, uint32_t(2));
// with more experts than a ubatch uses, the copy of the used experts in host memory skips some of them
ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(host_experts_test(arch) ? 64 : 2));
ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, uint32_t(1));
uint32_t gating_func = 2;
switch (arch) {
case LLM_ARCH_MELLUM:
gating_func = 1;
break;
case LLM_ARCH_DEEPSEEK4:
gating_func = 4;
break;
default:
break;
}
ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, gating_func); // sigmoid : softmax : sqrtsoftplus
ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERTS_PER_GROUP, uint32_t(1));
if (arch == LLM_ARCH_K2_HORIZON) {
ms.add_kv(LLM_KV_ATTENTION_VALUE_EXPERT_COUNT, uint32_t(2));
ms.add_kv(LLM_KV_ATTENTION_VALUE_EXPERT_USED_COUNT, uint32_t(2));
}
}
if (arch == LLM_ARCH_BEAM) {
ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(8));
ms.add_kv(LLM_KV_EXPERT_GROUP_COUNT, uint32_t(4));
ms.add_kv(LLM_KV_EXPERT_GROUP_USED_COUNT, uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 2.5f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
}
if (arch == LLM_ARCH_MAION_CODER) {
ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd / 2);
ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, std::vector<uint32_t>{n_ff, n_ff / 2, n_ff, n_ff});
}
ms.add_kv(LLM_KV_POSNET_EMBEDDING_LENGTH, n_embd);
ms.add_kv(LLM_KV_POSNET_BLOCK_COUNT, n_layer);
ms.add_kv(LLM_KV_CONVNEXT_EMBEDDING_LENGTH, n_embd);
ms.add_kv(LLM_KV_CONVNEXT_BLOCK_COUNT, n_layer);
ms.add_kv(LLM_KV_XIELU_ALPHA_N, 1.0f);
ms.add_kv(LLM_KV_XIELU_ALPHA_P, 1.0f);
ms.add_kv(LLM_KV_XIELU_BETA, 1.0f);
ms.add_kv(LLM_KV_XIELU_EPS, 1.0e-7f);
ms.add_kv(LLM_KV_SSM_INNER_SIZE, arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP ? 256 : 2*n_embd);
ms.add_kv(LLM_KV_SSM_CONV_KERNEL, uint32_t(4));
ms.add_kv(LLM_KV_SSM_STATE_SIZE, uint32_t(128));
ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head);
ms.add_kv(LLM_KV_SSM_GROUP_COUNT, arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2));
ms.add_kv(LLM_KV_KDA_HEAD_DIM, uint32_t(128));
ms.add_kv(LLM_KV_KDA_SAFE_GATE, true);
ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
if (arch == LLM_ARCH_BAILINGMOE3) {
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector<float>({0.0f, 4.0f}));
ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector<float>({0.0f, 5.0f}));
}
ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head);
ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3));
ms.add_kv(LLM_KV_RESIDUAL_SCALE, 3.5565588200778455f);
ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(12));
ms.add_kv(LLM_KV_ACTIVATION_SITU_BETA, 4.0f);
ms.add_kv(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, 25.0f);
ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
for (uint32_t il = 0; il < n_layer; il++) {
ggml_tensor t;
memset(&t, 0, sizeof(ggml_tensor));
t.type = GGML_TYPE_F16;
ggml_format_name(&t, "conv%" PRIu32 "d.weight", il);
gguf_add_tensor(ms.gguf_ctx, &t);
ggml_format_name(&t, "posnet.%" PRIu32 ".conv1.weight", il);
gguf_add_tensor(ms.gguf_ctx, &t);
ggml_format_name(&t, "posnet.%" PRIu32 ".conv2.weight", il);
gguf_add_tensor(ms.gguf_ctx, &t);
ggml_format_name(&t, "convnext.%" PRIu32 ".dw.weight", il);
gguf_add_tensor(ms.gguf_ctx, &t);
}
return ret;
}
static bool silent_model_load_progress(float /*progress*/, void * /*user_data*/) {
return true;
}
static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const float stdev,
const std::vector<ggml_backend_dev_t> & devs,
const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER, bool encode = false,
const llama_model_tensor_buft_override * tensor_buft_overrides = nullptr, const size_t moe_cache_size = 0) {
GGML_ASSERT((gguf_ctx == nullptr) != (file == nullptr));
llama_model_params model_params = llama_model_default_params();
model_params.progress_callback = silent_model_load_progress;
std::vector<ggml_backend_dev_t> devs_copy = devs;
devs_copy.push_back(nullptr);
model_params.devices = devs_copy.data();
model_params.split_mode = split_mode;
model_params.tensor_buft_overrides = tensor_buft_overrides;
llama_context_params ctx_params = llama_context_default_params();
ctx_params.n_ctx = 0;
ctx_params.n_threads = 4;
ctx_params.n_threads_batch = 4;
if (!encode) {
ctx_params.n_ubatch = 64;
}
if (moe_cache_size > 0) {
// the MoE cache is only used for small ubatches
ctx_params.moe_cache_size = moe_cache_size;
ctx_params.n_ubatch = 2;
}
tensor_data_params tensor_params = { seed, stdev };
llama_model_ptr model(gguf_ctx != nullptr ?
llama_model_init_from_user(gguf_ctx, set_tensor_data, &tensor_params, model_params) :
llama_model_load_from_file_ptr(file, model_params));
if (!model) {
throw std::runtime_error("failed to create llama model");
}
llama_context_ptr lctx(llama_init_from_model(model.get(), ctx_params));
if (!lctx) {
throw std::runtime_error("failed to create llama context");
}
return std::make_pair(std::move(model), std::move(lctx));
}
static std::vector<float> get_logits(
llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens, bool encode = false) {
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
const uint32_t n_ctx = llama_n_ctx(lctx);
const uint32_t n_tokens = tokens.size();
common_batch batch(lctx);
GGML_ASSERT(n_tokens <= n_ctx);
for (uint32_t pos = 0; pos < n_tokens; pos++) {
batch.add(tokens[pos], pos, 0, true);
}
if (encode) {
if (llama_process(lctx, LLAMA_PROCESS_TYPE_ENCODE, batch.get())) {
throw std::runtime_error("failed to encode batch");
}
}
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
throw std::runtime_error("failed to decode batch");
}
std::vector<float> ret;
ret.reserve(n_tokens*n_vocab);
for (uint32_t i = 0; i < n_tokens; i++) {
const float * logits_ith = llama_get_logits_ith(lctx, i);
for (uint32_t j = 0; j < n_vocab; j++) {
ret.push_back(logits_ith[j]);
}
}
return ret;
}
// entries [n/4, n/2) are embd rows, decoded either as token/embd/token chunks or as one mixed batch
// returns the llama_process() error code
static int32_t get_logits_mixed(
llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens, const std::vector<float> & embd, bool mixed,
std::vector<float> & ret) {
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
const uint32_t n_embd = llama_model_n_embd_inp(model);
const uint32_t n_tokens = tokens.size();
const uint32_t i_embd_0 = n_tokens/4;
const uint32_t i_embd_1 = n_tokens/2;
const std::vector<uint32_t> bounds = mixed
? std::vector<uint32_t>{0, n_tokens}
: std::vector<uint32_t>{0, i_embd_0, i_embd_1, n_tokens};
llama_memory_clear(llama_get_memory(lctx), true);
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
ret.clear();
ret.reserve(n_tokens*n_vocab);
for (size_t c = 0; c + 1 < bounds.size(); c++) {
llama_batch_ext_clear(batch.get());
for (uint32_t i = bounds[c]; i < bounds[c + 1]; i++) {
const bool is_embd = i >= i_embd_0 && i < i_embd_1;
const int32_t idx = is_embd
? llama_batch_ext_add_embd(batch.get(), 0, { embd.data() + (size_t) (i - i_embd_0)*n_embd, 1, n_embd })
: llama_batch_ext_add_token(batch.get(), 0, tokens[i]);
GGML_ASSERT(idx >= 0);
const llama_pos pos[4] = { (llama_pos) i, (llama_pos) i, (llama_pos) i, 0 };
llama_batch_ext_set_pos(batch.get(), idx, pos);
llama_batch_ext_set_output_logits(batch.get(), idx, true);
}
const int32_t err = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
if (err != 0) {
return err;
}
for (uint32_t i = 0; i < bounds[c + 1] - bounds[c]; i++) {
const float * logits_ith = llama_get_logits_ith(lctx, i);
ret.insert(ret.end(), logits_ith, logits_ith + n_vocab);
}
}
return 0;
}
static bool check_causal_attn_toggle(
llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
const uint32_t n_past = tokens.size();
const uint32_t n_ubatch = llama_n_ubatch(lctx);
GGML_ASSERT(n_past + n_ubatch/2 + n_ubatch <= llama_n_ctx(lctx));
llama_set_causal_attn(lctx, false);
common_batch batch(lctx);
bool ok = true;
uint32_t pos = n_past;
for (const uint32_t n_tokens : { n_ubatch/2, n_ubatch }) {
batch.clear();
for (uint32_t i = 0; i < n_tokens; i++) {
batch.add(tokens[i], pos++, 0, true);
}
const int32_t rc = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
if (rc != 0) {
LOG_ERR("%s: n_tokens=%u: llama_process returned %d\n", __func__, n_tokens, rc);
ok = false;
break;
}
const float * logits = llama_get_logits_ith(lctx, n_tokens - 1);
if (logits == nullptr) {
LOG_ERR("%s: n_tokens=%u: no logits\n", __func__, n_tokens);
ok = false;
break;
}
for (uint32_t j = 0; j < n_vocab; j++) {
if (std::isnan(logits[j])) {
LOG_ERR("%s: n_tokens=%u: nan logit\n", __func__, n_tokens);
ok = false;
break;
}
}
}
return ok;
}
static bool moe_mandatory(const llm_arch arch) {
switch (arch) {
case LLM_ARCH_LLAMA4:
case LLM_ARCH_MAION_CODER:
case LLM_ARCH_COHERE2MOE:
case LLM_ARCH_GROK:
case LLM_ARCH_QWEN2MOE:
case LLM_ARCH_QWEN3MOE:
case LLM_ARCH_QWEN3NEXT:
case LLM_ARCH_QWEN3VLMOE:
case LLM_ARCH_QWEN35MOE:
case LLM_ARCH_QWEN4EXP:
case LLM_ARCH_PHIMOE:
case LLM_ARCH_DBRX:
case LLM_ARCH_OLMOE:
case LLM_ARCH_ARCTIC:
case LLM_ARCH_DEEPSEEK:
case LLM_ARCH_DEEPSEEK2:
case LLM_ARCH_DEEPSEEK32:
case LLM_ARCH_DOTS3NOTE:
case LLM_ARCH_DEEPSEEK4:
case LLM_ARCH_GLM4_MOE:
case LLM_ARCH_GLM_DSA:
case LLM_ARCH_EXAONE_MOE:
case LLM_ARCH_BAILINGMOE:
case LLM_ARCH_BAILINGMOE2:
case LLM_ARCH_BAILINGMOE3:
case LLM_ARCH_DOTS1:
case LLM_ARCH_AFMOE:
case LLM_ARCH_BEAM:
case LLM_ARCH_ERNIE4_5:
case LLM_ARCH_ERNIE4_5_MOE:
case LLM_ARCH_HUNYUAN_MOE:
case LLM_ARCH_HY_V3:
case LLM_ARCH_HY_V4:
case LLM_ARCH_OPENAI_MOE:
case LLM_ARCH_LFM2MOE:
case LLM_ARCH_SMALLTHINKER:
case LLM_ARCH_LLADA_MOE:
case LLM_ARCH_GROVEMOE:
case LLM_ARCH_MINIMAX_01:
case LLM_ARCH_MINIMAX_M2:
case LLM_ARCH_MINIMAX_M3:
case LLM_ARCH_RND1:
case LLM_ARCH_PADDLEOCR:
case LLM_ARCH_MIMO2:
case LLM_ARCH_KIMI_LINEAR:
case LLM_ARCH_KIMI_K3:
case LLM_ARCH_GLM5_NEXT:
case LLM_ARCH_STEP35:
case LLM_ARCH_MISTRAL4:
case LLM_ARCH_MELLUM:
case LLM_ARCH_LAGUNA:
case LLM_ARCH_MAPLE:
case LLM_ARCH_MUSE_SPARK:
case LLM_ARCH_KOLIBRI1:
return true;
default:
return false;
}
}
static bool moe_implemented(const llm_arch arch) {
if (moe_mandatory(arch)) {
return true;
}
switch (arch) {
case LLM_ARCH_LLAMA:
case LLM_ARCH_REFACT:
case LLM_ARCH_MINICPM:
case LLM_ARCH_GRANITE:
case LLM_ARCH_GRANITE_MOE:
case LLM_ARCH_MISTRAL3:
case LLM_ARCH_LLAMA_EMBED:
case LLM_ARCH_K2_HORIZON:
return true;
default:
return false;
}
}
static bool arch_supported(const llm_arch arch) {
if (arch == LLM_ARCH_CLIP || arch == LLM_ARCH_GPTJ || arch == LLM_ARCH_UNKNOWN) {
return false; // These models don't have usable implementations.
}
if (arch == LLM_ARCH_CHAMELEON) {
return false; // Only half-implemented and to be removed in the future.
}
if (arch == LLM_ARCH_WAVTOKENIZER_DEC) {
return false; // FIXME CUDA backend crashes.
}
if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
return false; // FIXME @ngxson
}
if (arch == LLM_ARCH_GRANITE_SWITCH) {
return false; // FIXME adapter fixture
}
if (arch == LLM_ARCH_LLAMA_EMBED || arch == LLM_ARCH_GEMMA_EMBEDDING || arch == LLM_ARCH_GEMMA_EMBEDDING2 || arch == LLM_ARCH_T5ENCODER) {
return false; // FIXME Embedding (?) models produce inconsistent results.
}
if (arch == LLM_ARCH_RWKV6 || arch == LLM_ARCH_RWKV6QWEN2 || arch == LLM_ARCH_RWKV7 || arch == LLM_ARCH_ARWKV7) {
return false; // FIXME RWKV models hang indefinitely.
}
if (arch == LLM_ARCH_BERT || arch == LLM_ARCH_MODERN_BERT || arch == LLM_ARCH_NOMIC_BERT || arch == LLM_ARCH_NOMIC_BERT_MOE ||
arch == LLM_ARCH_NEO_BERT || arch == LLM_ARCH_JINA_BERT_V2 || arch == LLM_ARCH_JINA_BERT_V3 || arch == LLM_ARCH_EUROBERT) {
return false; // TODO vocab
}
if (arch == LLM_ARCH_PLM) {
return false; // TODO tensor shapes
}
if (arch == LLM_ARCH_CLEF || arch == LLM_ARCH_VLLM_DECISION2) {
return false; // TODO decision head tensors
}
if (arch == LLM_ARCH_DEEPSEEK2OCR) {
return false;
}
// FIXME: these hit scheduler/view-backed-output issues with WebGPU on CI.
#ifdef GGML_USE_WEBGPU
if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_QWEN4EXP ||
arch == LLM_ARCH_HY_V4) {
return false;
}
#endif // GGML_USE_WEBGPU
// FIXME: jamba produces incorrect output (~0.55 NMSE vs CPU) on the HIP
// backend on RDNA3.5 (gfx1151); the SSM kernels need investigation.
#ifdef GGML_USE_HIP
if (arch == LLM_ARCH_JAMBA) {
return false;
}
#endif // GGML_USE_HIP
return true;
}
static int save_models(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity, const std::string & dir) {
struct user_data_t {
struct {
ggml_log_callback callback;
void * user_data;
} log_old;
int verbosity;
user_data_t(int verbosity) : verbosity(verbosity) {
llama_log_get(&log_old.callback, &log_old.user_data);
}
};
user_data_t ud(verbosity);
llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
const user_data_t * ud = (const user_data_t *) user_data;
int verbosity = common_log_get_verbosity(level);
if (verbosity <= ud->verbosity) {
ud->log_old.callback(level, text, ud->log_old.user_data);
}
}, &ud);
for (const llm_arch & arch : llm_arch_all()) {
if (arch == LLM_ARCH_UNKNOWN) {
continue;
}
if (!arch_matches(arch_filter, arch)) {
continue;
}
if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
continue; // FIXME: ISWA KV cache initialization needs more fixture params
}
if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH) {
continue;
}
for (bool moe : {false, true}) {
if (moe && !moe_implemented(arch)) {
continue;
}
if (!moe && moe_mandatory(arch)) {
continue;
}
if (!llama_model_saver_supports_arch(arch) || !arch_supported(arch)) {
LOG_INF("%s: %s model (%s) is unsupported, skipping\n", __func__, llm_arch_name(arch), moe ? "MoE" : "dense");
continue;
}
gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe);
auto model_and_ctx = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, {});
const std::string path = dir + "/" + llm_arch_name(arch) + (moe ? "-moe.gguf" : "-dense.gguf");
LOG_INF("%s: Saving %s model (%s) to %s...\n", __func__, llm_arch_name(arch), moe ? "MoE" : "dense", path.c_str());
llama_model_save_to_file(model_and_ctx.first.get(), path.c_str());
}
}
llama_log_set(ud.log_old.callback, ud.log_old.user_data);
return 0;
}
// the CPU and the GPU of an Apple SoC carry the same description, the name tells them apart (CPU, MTL0)
static std::string get_device_label(ggml_backend_dev_t dev) {
const char * desc = ggml_backend_dev_description(dev);
const char * name = ggml_backend_dev_name(dev);
if (strcmp(desc, name) == 0) {
return desc;
}
return std::string(desc) + " (" + name + ")";
}
static int test_backends(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity, const char * target_backend) {
struct user_data_t {
struct {
ggml_log_callback callback;
void * user_data;
} log_old;
int verbosity;
user_data_t(int verbosity) : verbosity(verbosity) {
llama_log_get(&log_old.callback, &log_old.user_data);
}
};
user_data_t ud(verbosity);
llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
const user_data_t * ud = (const user_data_t *) user_data;
int verbosity = common_log_get_verbosity(level);
if (verbosity <= ud->verbosity) {
ud->log_old.callback(level, text, ud->log_old.user_data);
}
}, &ud);
const std::vector<llama_token> tokens = get_tokens(128, 128, seed);
struct device_config {
std::vector<ggml_backend_dev_t> devs;
std::string label;
llama_split_mode split_mode;
bool host_experts; // keep the experts in host memory, see host_experts_test
size_t moe_cache_size;
device_config(std::vector<ggml_backend_dev_t> devs, std::string name, llama_split_mode split_mode, bool host_experts = false, size_t moe_cache_size = 0)
: devs(std::move(devs)), label(std::move(name)), split_mode(split_mode), host_experts(host_experts), moe_cache_size(moe_cache_size) {}
};
const llama_model_tensor_buft_override host_experts_overrides[] = {
{ LLM_FFN_EXPS_REGEX, ggml_backend_cpu_buffer_type() },
{ nullptr, nullptr },
};
std::vector<device_config> dev_configs;
size_t max_device_label_length = 4;
{
std::vector<ggml_backend_dev_t> devices_meta;
{
const size_t device_count = ggml_backend_dev_count();
for (size_t i = 0; i < device_count; i++) {
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
if (target_backend != nullptr && strcmp(target_backend, ggml_backend_dev_name(dev)) != 0) {
continue;
}
dev_configs.emplace_back(std::vector<ggml_backend_dev_t>{dev}, get_device_label(dev), LLAMA_SPLIT_MODE_LAYER);
max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length());
// cpu-based devices cannot be used in tensor split mode
if (ggml_backend_dev_buffer_type(dev) != ggml_backend_cpu_buffer_type()) {
devices_meta.push_back(dev);
}
}
}
if (target_backend == nullptr) {
dev_configs.emplace_back(devices_meta, "Meta", LLAMA_SPLIT_MODE_TENSOR);
}
// the ops that use the experts are offloaded to the first device and the scheduler copies the used experts
if (!devices_meta.empty()) {
dev_configs.emplace_back(devices_meta, "Host experts", LLAMA_SPLIT_MODE_LAYER, true);
max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length());
}
// the ops that use the host experts run on a GPU and read the experts from a cache
// the cache has only a few slots (4 for 288 KiB experts), so the experts are evicted and uploaded again
if (!devices_meta.empty()) {
const enum ggml_backend_dev_type type = ggml_backend_dev_type(devices_meta[0]);
if (type == GGML_BACKEND_DEVICE_TYPE_GPU || type == GGML_BACKEND_DEVICE_TYPE_IGPU) {
dev_configs.emplace_back(std::vector<ggml_backend_dev_t>{devices_meta[0]}, "MoE cache", LLAMA_SPLIT_MODE_LAYER, true, 1536*1024);
max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length());
// each GPU caches the layers assigned to it
if (devices_meta.size() > 1) {
dev_configs.emplace_back(devices_meta, "MoE cache, layer split", LLAMA_SPLIT_MODE_LAYER, true, 1536*1024);
max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length());
}
}
}
}
size_t max_arch_name_length = 0;
for (const llm_arch & arch : llm_arch_all()) {
max_arch_name_length = std::max(max_arch_name_length, strlen(llm_arch_name(arch)));
}
const std::string template_header = std::string("|%" + std::to_string(max_arch_name_length) + "s|%") + std::to_string(max_device_label_length) + "s|%6s|%15s|%9s|%15s|\n";
const std::string template_row_cfg = std::string("|%" + std::to_string(max_arch_name_length) + "s|%") + std::to_string(max_device_label_length) + "s|%6s|";
const std::string template_row_res = "%15s %10s|%20s|%15s %10s|\n";
bool all_ok = true;
size_t n_tests = 0;
size_t n_failed = 0;
common_log_flush(common_log_main());
LOG(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip", "Mixed batch");
LOG("|");
for (size_t i = 0; i < max_arch_name_length; i++) {
LOG("-");
}
LOG("|");
for (size_t i = 0; i < max_device_label_length; i++) {
LOG("-");
}
LOG("|------|---------------|---------|---------------|\n");
for (const llm_arch & arch : llm_arch_all()) {
if (arch == LLM_ARCH_UNKNOWN) {
continue;
}
if (!arch_matches(arch_filter, arch)) {
continue;
}
if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
continue; // FIXME: ISWA KV cache initialization needs more fixture params
}
if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH) {
continue;
}
const bool encode = arch == LLM_ARCH_T5 || arch == LLM_ARCH_DREAM || arch == LLM_ARCH_LLADA || arch == LLM_ARCH_LLADA_MOE || arch == LLM_ARCH_RND1;
for (bool moe : {false, true}) {
if (moe && !moe_implemented(arch)) {
continue;
}
if (!moe && moe_mandatory(arch)) {
continue;
}
const std::string config_name = moe ? "MoE" : "Dense";
gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe);
if (arch == LLM_ARCH_BAILINGMOE3) {
GGML_ASSERT(gguf_remove_key(gguf_ctx.get(), "bailingmoe3.kda.safe_gate") >= 0);
}
std::pair<llama_model_ptr, llama_context_ptr> model_and_ctx_cpu;
std::vector<float> logits_cpu;
for (device_config & dc : dev_configs) {
if (dc.host_experts && (!moe || !host_experts_test(arch))) {
continue;
}
const llama_model_tensor_buft_override * overrides = dc.host_experts ? host_experts_overrides : nullptr;
// print test config first; should anything fail during model loading or inference, at least we know which test case caused it
LOG(template_row_cfg.c_str(), llm_arch_name(arch), dc.label.c_str(), config_name.c_str());
fflush(stdout);
std::pair<llama_model_ptr, llama_context_ptr> model_and_ctx_dev;
std::vector<float> logits_dev;
std::string status_nmse = "\033[1;33mSKIP\033[0m";
std::string status_roundtrip = "\033[1;33mSKIP\033[0m";
std::string status_mixed = "\033[1;33mSKIP\033[0m";
char nmse_str[12] = {0};
char mixed_str[12] = {0};
bool skip = !arch_supported(arch) || (dc.split_mode == LLAMA_SPLIT_MODE_TENSOR && dc.devs.empty());
bool test_executed = false;
bool test_ok = true;
if (!skip) {
if (logits_cpu.empty()) {
model_and_ctx_cpu = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, {}, LLAMA_SPLIT_MODE_LAYER, encode);
logits_cpu = get_logits(model_and_ctx_cpu.first.get(), model_and_ctx_cpu.second.get(), tokens, encode);
}
if (dc.split_mode != LLAMA_SPLIT_MODE_TENSOR || llm_arch_supports_sm_tensor(arch)) {
test_executed = true;
model_and_ctx_dev = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, dc.devs, dc.split_mode, encode, overrides, dc.moe_cache_size);
logits_dev = get_logits(model_and_ctx_dev.first.get(), model_and_ctx_dev.second.get(), tokens, encode);
const double nmse_val = nmse(logits_cpu, logits_dev);
snprintf(nmse_str, sizeof(nmse_str), "(%.2e)", nmse_val);
status_nmse = "\033[1;32mOK\033[0m";
if (nmse_val > 1e-4) {
test_ok = false;
status_nmse = "\033[1;31mFAIL\033[0m";
}
// chunked decode matches a single batch only with causal attention over a memory
llama_context * lctx_dev = model_and_ctx_dev.second.get();
if (!encode && llama_get_memory(lctx_dev) != nullptr) {
std::vector<float> embd_mixed((size_t) llama_model_n_embd_inp(model_and_ctx_dev.first.get())*tokens.size()/4);
std::mt19937 gen(seed);
std::normal_distribution<float> dis(0.0f, stdev);
for (float & v : embd_mixed) {
v = dis(gen);
}
std::vector<float> logits_mixed;
std::vector<float> logits_chunks;
if (llm_arch_supports_mixed_batch(arch)) {
if (get_logits_mixed(model_and_ctx_dev.first.get(), lctx_dev, tokens, embd_mixed, false, logits_chunks) != 0 ||
get_logits_mixed(model_and_ctx_dev.first.get(), lctx_dev, tokens, embd_mixed, true, logits_mixed) != 0) {
throw std::runtime_error("failed to decode mixed batch");
}
const double nmse_mixed = nmse(logits_chunks, logits_mixed);
snprintf(mixed_str, sizeof(mixed_str), "(%.2e)", nmse_mixed);
status_mixed = "\033[1;32mOK\033[0m";
if (nmse_mixed > 1e-4) {
test_ok = false;
status_mixed = "\033[1;31mFAIL\033[0m";
}
} else {
// must be rejected as an invalid batch, mute the expected error log
ud.verbosity = LOG_LEVEL_OUTPUT;
const int32_t err = get_logits_mixed(model_and_ctx_cpu.first.get(), model_and_ctx_cpu.second.get(), tokens, embd_mixed, true, logits_mixed);
ud.verbosity = verbosity;
if (err != -1) {
test_ok = false;
status_mixed = "\033[1;31mFAIL\033[0m";
}
}
}
// runs after the mixed batch check, as it leaves the context with non-causal attention
if (!encode && !check_causal_attn_toggle(model_and_ctx_dev.first.get(), model_and_ctx_dev.second.get(), tokens)) {
if (test_ok) {
status_nmse = "\033[1;31mFAIL\033[0m (toggle)";
}
test_ok = false;
}
}
FILE * file = tmpfile(); // Can be null on Windows without administrator privileges.
// FIXME: when adding a tensor to a gguf_context a copy is made, this changes the pointer which the meta backend
// in turn uses to map the tensors to their simple equivalents - this is fundamentally incompatible
if (file != nullptr && llama_model_saver_supports_arch(arch) && dc.split_mode != LLAMA_SPLIT_MODE_TENSOR) {
test_executed = true;
GGML_ASSERT(model_and_ctx_dev.first && model_and_ctx_dev.second);
llama_model_saver ms = llama_model_saver(model_and_ctx_dev.first.get());
ms.add_kv_from_model();
ms.add_tensors_from_model();
ms.save(file);
rewind(file);
auto model_and_ctx_roundtrip = get_model_and_ctx(nullptr, file, seed, stdev, dc.devs, dc.split_mode, encode, overrides, dc.moe_cache_size);
const std::vector<float> logits_roundtrip = get_logits(
model_and_ctx_roundtrip.first.get(), model_and_ctx_roundtrip.second.get(), tokens, encode);
status_roundtrip = "\033[1;32mOK\033[0m";
GGML_ASSERT(logits_roundtrip.size() == logits_dev.size());
for (size_t i = 0; i < logits_roundtrip.size(); i++) {
if (logits_roundtrip[i] != logits_dev[i]) {
test_ok = false;
status_roundtrip = "\033[1;31mFAIL\033[0m";
break;
}
}
}
}
if (test_executed) {
n_tests++;
if (!test_ok) {
n_failed++;
all_ok = false;
}
}
// log the results for this test case
LOG(template_row_res.c_str(), status_nmse.c_str(), nmse_str, status_roundtrip.c_str(), status_mixed.c_str(), mixed_str);
}
}
}
if (n_tests == 0) {
LOG("Summary: no tests executed\n");
} else if (n_failed == 0) {
LOG("Summary: all %zu test(s) passed\n", n_tests);
} else {
LOG("Summary: %zu test(s) executed, %zu failed\n", n_tests, n_failed);
}
llama_log_set(ud.log_old.callback, ud.log_old.user_data);
return all_ok ? 0 : 1;
}
int main(int argc, char ** argv) {
// init the logger at max verbosity. filter with a custom callback respecting the user-configure verbosity
common_log_set_verbosity_thold(LOG_LEVEL_DEBUG);
common_init();
llama_backend_init();
std::random_device rd;
std::string arch_filter;
size_t seed = rd();
float stdev = 0.1f;
std::string out;
const char * target_backend = nullptr;
int verbosity = LOG_LEVEL_ERROR;
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
usage(argv);
return 0;
} else if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) {
if (i + 1 < argc) {
const std::string arch_name = argv[++i];
if (llm_arch_from_string(arch_name) != LLM_ARCH_UNKNOWN) {
// exact architecture name
arch_filter = "^" + arch_name + "$";
} else {
try {
std::regex re(arch_name);
arch_filter = arch_name;
} catch (const std::regex_error & err) {
LOG_ERR("%s: invalid architecture regex: %s (%s)\n", __func__, arch_name.c_str(), err.what());
return 1;
}
}
} else {
usage(argv);
return 1;
}
} else if (strcmp(argv[i], "-s") == 0 || strcmp(argv[i], "--seed") == 0) {
if (i + 1 < argc) {
seed = std::stoull(argv[++i]);
} else {
usage(argv);
return 1;
}
} else if (strcmp(argv[i], "-d") == 0 || strcmp(argv[i], "--stdev") == 0) {
if (i + 1 < argc) {
stdev = std::stof(argv[++i]);
} else {
usage(argv);
return 1;
}
} else if (strcmp(argv[i], "-v") == 0) {
if (i + 1 < argc) {
verbosity = std::stoull(argv[++i]);
} else {
usage(argv);
return 1;
}
} else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--out") == 0) {
if (i + 1 < argc) {
out = argv[++i];
} else {
usage(argv);
return 1;
}
} else if (strcmp(argv[i], "-b") == 0 || strcmp(argv[i], "--backend") == 0) {
if (i + 1 < argc) {
const char * backend_name = argv[++i];
ggml_backend_dev_t dev = ggml_backend_dev_by_name(backend_name);
if (dev == nullptr) {
LOG_ERR("%s: unknown backend device: %s\n", __func__, backend_name);
return 1;
}
target_backend = ggml_backend_dev_name(dev);
} else {
usage(argv);
return 1;
}
} else {
LOG_ERR("%s: unknown argument: %s\n", __func__, argv[i]);
usage(argv);
return 1;
}
}
if (stdev <= 0.0f) {
LOG_ERR("%s: stdev must be > 0\n", __func__);
return 1;
}
LOG_INF("%s: using seed %zu, stdev %f\n", __func__, seed, stdev);
try {
if (!out.empty()) {
return save_models(arch_filter, seed, stdev, verbosity, out);
}
return test_backends(arch_filter, seed, stdev, verbosity, target_backend);
} catch (const std::exception & err) {
fprintf(stderr, "encountered runtime error: %s\n", err.what());
return -1;
}
}
Reference in new issue View Git Blame Copy Permalink