wmantly/llama.cpp-xrip
1
0 Fork 0
Code Issues Pull Requests Actions 33 Packages Projects Releases Wiki Activity
Files
bc428e407a52bd3e21b5985cabeef57cdff9bc4a
llama.cpp-xrip/tests/test-llama-archs.cpp
T
Oliver SimonsandSigbjørn Skjæret 68d20b884c Quant: OCP FP8 E4M3 support (#27512) 
* WIP OCP FP8 E4M3 support

* Fix python type-checker

* Materialize input_scales also for FP8 when converting

* Fix conversion of https://huggingface.co/RedHatAI/Qwen3.8-27B-NVFP4

1. Per row FP8 wrongly fell to NVFP4 conversion, as NVFP4 checked on
   dims alone. Also check on dtype for FP8
2. Need to reshape FP8 QKV projections scales in the same way that
   weights are reshaped

* Allow remaining FP8 tensors to be dequantized during conversion

* Allow tensor-strategy for FP8-only compressed-tensor ckpts

* Fix and test gguf-py behavior for FP8_E4M3

* Disallow QKV-fusion for FP8 temporarily

* Fix conversion of fused qkvz & FP8 for Qwen3-Next

* Avoid fusing FP8 for --fuse-gate-up-exps as we may have different ws

* Emit warnings when not fusing FP8 during conversion

* More FP8 conversion fixes

* Add supported for strided copies to/from FP8 in CPU

* remove unneeded scratch buffer from FP8 cpu cpy/dup path

* add explanatory comments and TODOs

* Add dedicated encoder and decoder tests for FP8

* Whitelist ggml_op_get_rows in metal instead of blacklisting

required to fix failing CI, and white-listing should be more
future-proof than adding FP8 to the black-list

https://github.com/ggml-org/llama.cpp/actions/runs/38433916498/job/115422880633?pr=27512

* Fix more conversion bugs

* Defer weight materialization during conversion

* Blacklist fp8 in metal backend instead of white-list

* Adjust indents

* Fix lint + execute tests

* Add fast CPU kernel for F8_E4M3

* Conversion fixes

* Fix remaining gaps

* Enable tied weights with shared scales

* Make mandatory-F32 rules apply to FP8 during conversion

* Fix python type checks

https://github.com/ggml-org/llama.cpp/actions/runs/39765670052/job/119780379595?pr=27512

* Fix kaleidai unit-test failures

https://github.com/ggml-org/llama.cpp/actions/runs/39765670125/job/119780383150

* Relax numeric precision constraints for pre-check

* Leave fast-FP8 CPU on RISC-V for follow-up

We already exclude arm from the fast-path

* Allow output scale tensors to be located in any gguf shard

* Move test_conversion_fp8.py test execution to behavior-check.yaml

* Apply batched suggestions from code review

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

* Move mixed compressed-tensor logic into dequant_fp8 + add unit-tests for it

* Install all requirements also on Windows for behavior-check

Fixes https://github.com/ggml-org/llama.cpp/actions/runs/40535852590/job/122300758223?pr=27512

* Unmap lazy tensors before clean-up in Windows

Should fix https://github.com/ggml-org/llama.cpp/actions/runs/40539797824/job/122314782459?pr=27512

* LF line endings

https://github.com/ggml-org/llama.cpp/actions/runs/40539798168/job/122314791118?pr=27512

* Extract FP8 tests from llama-arch to its own test-fixture

See https://github.com/ggml-org/llama.cpp/pull/27512#discussion_r4400628221

* Update gguf-py/gguf/scripts/gguf_convert_endian.py

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
2026-10-29 18:14:37 +01:00

1502 lines
67 KiB
C++
Raw Blame History

#include "common.h"
#include "log.h"
#include "ggml-backend.h"
#include "ggml.h"
#include "gguf.h"
#include "ggml-cpp.h"
#include "llama.h"
#include "llama-cpp.h"
#include "../src/llama-ext.h"
#include <algorithm>
#include <cinttypes>
#include <cmath>
#include <cstddef>
#include <cstdio>
#include <cstring>
#include <cstdint>
#include <filesystem>
#include <memory>
#include <random>
#include <regex>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
static bool arch_matches(const std::string & filter, llm_arch arch) {
if (filter.empty()) {
return true;
}
return std::regex_search(llama_arch_name(arch), std::regex(filter));
}
// normalized mean squared error = mse(a, b) / mse(a, 0)
static double nmse(const std::vector<float> & a, const std::vector<float> & b) {
GGML_ASSERT(a.size() == b.size());
double mse_a_b = 0.0;
double mse_a_0 = 0.0;
for (size_t i = 0; i < a.size(); i++) {
float a_i = a[i];
float b_i = b[i];
mse_a_b += (a_i - b_i) * (a_i - b_i);
mse_a_0 += a_i * a_i;
}
return mse_a_b / mse_a_0;
}
struct tensor_data_params {
size_t seed;
float stdev;
};
// fills the values of one tensor, the name carries whatever the property needs
typedef void (*tensor_filler)(std::vector<float> & dst, const char * name, const tensor_data_params & params);
// a pool of normal draws, generated once and reused for every model
static std::vector<float> g_normal_pool;
static const size_t NORMAL_POOL_SIZE = 16 * 1024 * 1024;
// fill the pool once; each tensor copies a slice of it
static void init_normal_pool(const size_t seed, const float stdev) {
g_normal_pool.resize(NORMAL_POOL_SIZE);
std::mt19937 gen(seed);
std::normal_distribution<float> dist(0.0f, stdev);
for (float & v : g_normal_pool) {
v = dist(gen);
}
}
// a normal draw keyed on a name, so that tensors which share a draw ask for the same key
static void fill_normal(std::vector<float> & dst, const char * key, const tensor_data_params &) {
const size_t n = dst.size();
const size_t pool_size = g_normal_pool.size();
const size_t start = std::hash<std::string>{}(key) % pool_size;
// copy a cyclic slice of the pool, so a tensor larger than the pool still fills
size_t i = 0;
size_t pos = start;
while (i < n) {
const size_t chunk = std::min(pool_size - pos, n - i);
std::memcpy(dst.data() + i, g_normal_pool.data() + pos, chunk * sizeof(float));
i += chunk;
pos = 0;
}
}
// Mamba A must be negative (state decay)
static void fill_ssm_a(std::vector<float> & dst, const char * name, const tensor_data_params & params) {
fill_normal(dst, name, params);
for (float & v : dst) {
v = -fabsf(v);
}
}
// a norm preserving residual gate holds alpha^2 + beta^2 == 1, and the conversion resolves both from one raw weight
// the alpha and the beta of a block share that weight, so the key drops the suffix and keeps the block
static void fill_resid_gate(std::vector<float> & dst, const char * name, const tensor_data_params & params) {
const char * suffix = strstr(name, "_resid_");
const bool is_beta = strstr(suffix, "_resid_beta") != nullptr;
fill_normal(dst, std::string(name, suffix - name).c_str(), params);
for (float & v : dst) {
const float beta = 1.0f / (1.0f + expf(-v));
v = is_beta ? beta : sqrtf(1.0f - beta*beta);
}
}
// a random router bias can tie within the noise between the backends, a ramp keeps the top-k selection the same
static void fill_exp_probs_b(std::vector<float> & dst, const char *, const tensor_data_params &) {
for (size_t i = 0; i < dst.size(); i++) {
dst[i] = 2.0f*i;
}
}
// a tensor whose values hold a property of their own, matched on a substring of its name, first match wins
static const struct { const char * match; tensor_filler fill; } TENSOR_FILLERS[] = {
{ "ssm_a", fill_ssm_a },
{ "_resid_", fill_resid_gate },
{ "exp_probs_b", fill_exp_probs_b },
};
static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {
const tensor_data_params & params = *(const tensor_data_params *) userdata;
tensor_filler fill = fill_normal;
for (const auto & filler : TENSOR_FILLERS) {
if (strstr(tensor->name, filler.match) != nullptr) {
fill = filler.fill;
break;
}
}
std::vector<float> tmp(ggml_nelements(tensor));
fill(tmp, tensor->name, params);
if (tensor->type == GGML_TYPE_F32) {
ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor));
} else if (tensor->type == GGML_TYPE_F16) {
std::vector<ggml_fp16_t> tmp16(tmp.size());
for (size_t i = 0; i < tmp.size(); i++) {
tmp16[i] = ggml_fp32_to_fp16(tmp[i]);
}
ggml_backend_tensor_set(tensor, tmp16.data(), 0, ggml_nbytes(tensor));
} else {
GGML_ABORT("fatal error");
}
}
static void usage(char ** argv) {
LOG("Usage: %s [options]\n\n", argv[0]);
LOG("Options:\n");
LOG(" -a, --arch <arch|regex> Run only matching LLM architectures (default: all supported)\n");
LOG(" -s, --seed <seed> Set the random seed for tensor initialization and token generation\n");
LOG(" -d, --stdev <stdev> Set the standard deviation of the tensor initialization distribution (default: 0.1f)\n");
LOG(" -o, --out <dir> Save generated test models to <dir> instead of running backend tests\n");
LOG(" -v <N> Set log verbosity level\n");
LOG(" -b, --backend <backend> Run only on the given backend device\n");
LOG(" -h, --help Show this help message\n\n");
LOG("Examples:\n");
LOG(" %s\n", argv[0]);
LOG(" %s -a qwen35moe\n", argv[0]);
LOG(" %s -a deepseek4 -o tests/test-models/\n", argv[0]);
LOG(" %s -a cohere2moe -v 5\n", argv[0]);
}
static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed){
std::mt19937 gen(seed);
std::uniform_int_distribution<> dis(0, n_vocab - 1);
std::vector<llama_token> ret;
ret.reserve(n_tokens);
for (uint32_t i = 0; i < n_tokens; i++) {
ret.push_back(dis(gen));
}
return ret;
}
// MoE archs that are also tested with the experts in host memory
static bool host_experts_test(const llm_arch arch) {
switch (arch) {
case LLM_ARCH_DEEPSEEK2:
return true;
default:
return false;
}
}
// writes arch-resolved GGUF keys into the fixture context, the test-side equivalent of llama_model_saver
struct kv_writer {
struct gguf_context * gguf_ctx;
const llm_arch arch;
kv_writer(enum llm_arch arch, struct gguf_context * gguf_ctx) : gguf_ctx(gguf_ctx), arch(arch) {}
void add_kv(enum llm_kv key, uint32_t value) { gguf_set_val_u32 (gguf_ctx, llama_arch_kv_name(arch, key).c_str(), value); }
void add_kv(enum llm_kv key, int32_t value) { gguf_set_val_i32 (gguf_ctx, llama_arch_kv_name(arch, key).c_str(), value); }
void add_kv(enum llm_kv key, uint64_t value) { gguf_set_val_u64 (gguf_ctx, llama_arch_kv_name(arch, key).c_str(), value); }
void add_kv(enum llm_kv key, float value) { gguf_set_val_f32 (gguf_ctx, llama_arch_kv_name(arch, key).c_str(), value); }
void add_kv(enum llm_kv key, bool value) { gguf_set_val_bool(gguf_ctx, llama_arch_kv_name(arch, key).c_str(), value); }
void add_kv(enum llm_kv key, const char * value) { gguf_set_val_str (gguf_ctx, llama_arch_kv_name(arch, key).c_str(), value); }
template <typename T>
void add_kv(enum llm_kv key, const std::vector<T> & value) {
if (value.empty()) {
return;
}
if (std::is_same<T, uint32_t>::value) {
gguf_set_arr_data(gguf_ctx, llama_arch_kv_name(arch, key).c_str(), GGUF_TYPE_UINT32, value.data(), value.size());
} else if (std::is_same<T, int32_t>::value) {
gguf_set_arr_data(gguf_ctx, llama_arch_kv_name(arch, key).c_str(), GGUF_TYPE_INT32, value.data(), value.size());
} else if (std::is_same<T, uint64_t>::value) {
gguf_set_arr_data(gguf_ctx, llama_arch_kv_name(arch, key).c_str(), GGUF_TYPE_UINT64, value.data(), value.size());
} else if (std::is_same<T, float>::value) {
gguf_set_arr_data(gguf_ctx, llama_arch_kv_name(arch, key).c_str(), GGUF_TYPE_FLOAT32, value.data(), value.size());
} else {
GGML_ABORT("fatal error");
}
}
void add_kv(enum llm_kv key, const std::vector<std::string> & value) {
std::vector<const char *> tmp(value.size());
for (size_t i = 0; i < value.size(); ++i) {
tmp[i] = value[i].c_str();
}
gguf_set_arr_str(gguf_ctx, llama_arch_kv_name(arch, key).c_str(), tmp.data(), tmp.size());
}
};
static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
gguf_context_ptr ret(gguf_init_empty());
kv_writer ms(arch, ret.get());
const uint32_t n_ctx = 256;
uint32_t n_vocab = 128;
uint32_t n_embd = 256;
uint32_t n_head = 2;
uint32_t n_ff = 384;
uint32_t n_layer = 2;
if (arch == LLM_ARCH_LLAMA4) {
n_layer = 4; // hparams.n_no_rope_layer_step is hard-coded to 4
} else if (arch == LLM_ARCH_MAION_CODER) {
n_head = 4;
n_layer = 4; // dense and MoE layers, each with local and full attention
} else if (arch == LLM_ARCH_GEMMA4) {
n_embd = 128;
n_head = 2;
n_ff = 192;
n_layer = 5; // need at least 5 for swa_pattern (every 5th is full_attention)
} else if (arch == LLM_ARCH_GEMMA3N) {
n_embd = 64;
n_head = 1;
n_ff = 96;
n_layer = 22; // hparams.n_layer_kv_from_start = 20 is hardcoded
} else if (arch == LLM_ARCH_LFM3) {
n_layer = 3;
} else if (arch == LLM_ARCH_DEEPSEEK4) {
// head size 64 so that GPU flash attention kernels support the model
n_embd = 512;
n_head = 8;
n_ff = 1024;
n_layer = 4;
} else if (arch == LLM_ARCH_DEEPSEEK41) {
// window-only, Engram, ratio-2 and ratio-1 sources, top-k reuse and a candidate-masked index layer
n_embd = 512;
n_head = 8;
n_ff = 1024;
n_layer = 6;
} else if (arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_LAGUNA) {
n_embd = 160; // exercise per-head tensor split granularity with head size 80
} else if (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE) {
n_head = 4;
} else if (arch == LLM_ARCH_LIMITE) {
n_head = 4;
n_layer = 4; // dense connections on the last two layers
} else if (arch == LLM_ARCH_DEEPSEEK2
|| arch == LLM_ARCH_DEEPSEEK32
|| arch == LLM_ARCH_GLM_DSA
|| arch == LLM_ARCH_DOTS3NOTE
|| arch == LLM_ARCH_KIMI_LINEAR
|| arch == LLM_ARCH_BAILINGMOE3
|| arch == LLM_ARCH_KIMI_K3
|| arch == LLM_ARCH_GLM5_NEXT
|| arch == LLM_ARCH_MISTRAL4
|| arch == LLM_ARCH_MISTRAL_LARGE4
|| arch == LLM_ARCH_HY_V4) {
n_embd = 128;
n_head = 1;
n_ff = 192;
if (arch == LLM_ARCH_MISTRAL_LARGE4) {
n_layer = 4; // cover NoPE, shared indexer and KV sharing layers
}
} else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {
n_layer = 3;
} else if (arch == LLM_ARCH_CHAMELEON) {
n_vocab = 10240;
} else if (arch == LLM_ARCH_QWEN3TTS) {
//n_vocab = 4096; // must be >= the hard-coded codec head size (3072)
n_vocab = 3072; // TODO: should be 4096, but user code cannot get `n_vocab_out` yet [TAG_LLAMA_N_VOCAB_OUT]
} else if (arch == LLM_ARCH_HRM_TEXT) {
n_layer = 6; // 1 layer per stack x 2 h-cycles x (2 l-cycles + 1) cache slots
}
uint32_t n_head_kv = n_head;
if (arch == LLM_ARCH_QWEN3) {
n_head_kv = 1; // MQA coverage
} else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER ||
arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE) {
n_head_kv = 2; // GQA coverage
}
if (arch == LLM_ARCH_BEAM) {
n_head = 4;
n_head_kv = 2;
n_layer = 4;
}
const uint32_t n_embd_head = n_embd / n_head;
ms.add_kv(LLM_KV_GENERAL_ARCHITECTURE, llama_arch_name(arch));
ms.add_kv(LLM_KV_VOCAB_SIZE, n_vocab);
ms.add_kv(LLM_KV_CONTEXT_LENGTH, n_ctx);
ms.add_kv(LLM_KV_EMBEDDING_LENGTH, n_embd);
ms.add_kv(LLM_KV_FEATURES_LENGTH, n_embd);
ms.add_kv(LLM_KV_BLOCK_COUNT, n_layer);
ms.add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, uint32_t(1));
if (arch == LLM_ARCH_K2_HORIZON) {
ms.add_kv(LLM_KV_ROPE_SCALING_YARN_BETA_FAST, 128.0f);
ms.add_kv(LLM_KV_ROPE_SCALING_YARN_BETA_SLOW, 4.0f);
}
if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {
std::vector<uint32_t> n_ff_per_layer;
n_ff_per_layer.reserve(n_layer);
for (uint32_t il = 0; il < n_layer; il++) {
n_ff_per_layer.push_back(il <= 1 ? 0 : n_ff);
}
ms.add_kv(LLM_KV_FEED_FORWARD_LENGTH, n_ff_per_layer);
} else {
ms.add_kv(LLM_KV_FEED_FORWARD_LENGTH, n_ff);
}
ms.add_kv(LLM_KV_USE_PARALLEL_RESIDUAL, false);
ms.add_kv(LLM_KV_LOGIT_SCALE, 1.0f);
ms.add_kv(LLM_KV_TIME_MIX_EXTRA_DIM, uint32_t(64));
ms.add_kv(LLM_KV_TIME_DECAY_EXTRA_DIM, uint32_t(128));
ms.add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, uint32_t(2));
if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE ||
arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_LFM3 ||
arch == LLM_ARCH_KIMI_LINEAR || arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3 || arch == LLM_ARCH_GLM5_NEXT
) {
GGML_ASSERT(n_layer >= 2);
std::vector<uint32_t> n_head_per_layer;
n_head_per_layer.reserve(n_layer);
for (uint32_t il = 0; il < n_layer; il++) {
n_head_per_layer.push_back(il == 1 ? 0 : n_head);
}
// GLM5 next KDA heads come from the uniform head count, only head_count_kv is per layer.
if (arch == LLM_ARCH_GLM5_NEXT) {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);
} else {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head_per_layer);
}
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer);
} else {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) ? uint32_t(1) : n_head_kv);
}
ms.add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, 8.0f);
if (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) {
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, n_embd_head);
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, n_embd_head);
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, n_embd_head/2);
} else if (arch == LLM_ARCH_DEEPSEEK2
|| arch == LLM_ARCH_DEEPSEEK32
|| arch == LLM_ARCH_GLM_DSA
|| arch == LLM_ARCH_DOTS3NOTE
|| arch == LLM_ARCH_KIMI_LINEAR
|| arch == LLM_ARCH_BAILINGMOE3
|| arch == LLM_ARCH_KIMI_K3
|| arch == LLM_ARCH_GLM5_NEXT
|| arch == LLM_ARCH_HY_V4
|| arch == LLM_ARCH_MISTRAL4
|| arch == LLM_ARCH_MISTRAL_LARGE4) {
// GLM5 next MLA is nope only, the cache row is the compressed latent alone.
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, arch == LLM_ARCH_GLM5_NEXT ? uint32_t(512) : uint32_t(576));
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, uint32_t(512));
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, arch == LLM_ARCH_GLM5_NEXT ? uint32_t(0) : uint32_t(64));
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA, uint32_t(192));
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, uint32_t(128));
if (arch == LLM_ARCH_DOTS3NOTE) {
// SWA layers reuse the same MLA geometry as the full layers in this fixture
ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK_SWA, uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, uint32_t(576));
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, uint32_t(192));
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, uint32_t(128));
ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f);
// indexer on the full-attention layers (inverse of the swa pattern)
std::vector<uint32_t> indexer_types;
indexer_types.reserve(n_layer);
for (uint32_t il = 0; il < n_layer; il++) {
indexer_types.push_back(il % 2 ? 0 : 1);
}
ms.add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, indexer_types);
}
} else if (arch == LLM_ARCH_MINIMAX_M3) {
// partial rotary: n_rot must not exceed the indexer key length (64)
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, uint32_t(64));
}
if (arch == LLM_ARCH_SENSENOVA_U1) {
ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SPATIAL, 10000.0f);
}
ms.add_kv(LLM_KV_ATTENTION_CLAMP_KQV, 1.0f);
ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_EPS, 1e-5f);
ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, 1e-5f);
ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_EPS, 1e-5f);
ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_GROUPS, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_Q_LORA_RANK, (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) ? uint32_t(64) : uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK, uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, n_ctx/8);
if (arch == LLM_ARCH_MAION_CODER) {
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector<uint32_t>{1, 1, 0, 0});
} else if (arch == LLM_ARCH_BEAM) {
std::vector<uint32_t> pattern;
pattern.reserve(n_layer);
for (uint32_t il = 0; il < n_layer; il++) {
pattern.push_back(il % 2);
}
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);
} else if (arch == LLM_ARCH_GEMMA4) {
ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, n_embd/2);
ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS, uint32_t(0));
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, n_embd_head);
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, n_embd_head);
ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f);
// SWA pattern: every 5th layer is full attention (matches E2B layer_types)
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5));
} else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 ||
arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE ||
arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_FLINT) {
std::vector<uint32_t> pattern;
pattern.reserve(n_layer);
for (uint32_t il = 0; il < n_layer; il++) {
pattern.push_back(il % 2);
}
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);
if (arch == LLM_ARCH_FLINT) {
ms.add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, pattern);
}
} else if (arch == LLM_ARCH_LFM3) {
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector<uint32_t>({0, 0, 1}));
ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f);
} else {
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(2));
}
if (arch == LLM_ARCH_MUSE_SPARK) {
ms.add_kv(LLM_KV_POST_NORM_EPS, 1e-5f);
ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f);
ms.add_kv(LLM_KV_FINAL_LOGIT_SOFTCAPPING, 30.0f);
ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd/2);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
}
if (arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE) {
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, false);
}
// MSA requires one indexer head per GQA (KV) head, unlike the DSA archs where the
// indexer head count is independent of the main attention head count.
if (arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_GLM5_NEXT) {
ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2));
ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);
ms.add_kv(LLM_KV_HYPER_CONNECTION_LOW_RANK, uint32_t(8));
// without this the QSA layers fall back to dense and go uncovered
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>(n_layer, 4));
// has_cell_ext() needs ple_n_heads here: the indexer cache serializes no ext without it
const uint32_t ple_ngram_size = 3;
const uint32_t ple_heads_per_ngram = 2;
const uint32_t ple_n_heads = (ple_ngram_size - 1)*ple_heads_per_ngram;
GGML_ASSERT(n_embd % ple_n_heads == 0);
const uint32_t ple_head_dim = n_embd/ple_n_heads;
std::vector<uint64_t> ple_head_offsets(ple_n_heads);
std::vector<uint64_t> ple_head_vocab_sizes(ple_n_heads, n_vocab);
for (uint32_t h = 0; h < ple_n_heads; h++) {
ple_head_offsets[h] = uint64_t(h)*n_vocab;
}
// the PLE history lives in the recurrent cache, so it must sit on a linear attention layer
ms.add_kv(LLM_KV_PLE_LAYERS, std::vector<uint32_t>({ 0 }));
ms.add_kv(LLM_KV_PLE_NGRAM_SIZE, ple_ngram_size);
ms.add_kv(LLM_KV_PLE_HEADS_PER_NGRAM, ple_heads_per_ngram);
ms.add_kv(LLM_KV_PLE_CONV_KERNEL, uint32_t(4));
ms.add_kv(LLM_KV_PLE_EOS_TOKEN_ID, uint32_t(0));
ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, ple_head_dim);
ms.add_kv(LLM_KV_PLE_LAYER_MULTIPLIERS, std::vector<uint64_t>({ 1, 3, 5 }));
ms.add_kv(LLM_KV_PLE_HEAD_OFFSETS, ple_head_offsets);
ms.add_kv(LLM_KV_PLE_HEAD_VOCAB_SIZES, ple_head_vocab_sizes);
}
// minimax-m3 keeps one indexer head per GQA head; the rest use a fixed 64 to match the fused
ms.add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, arch == LLM_ARCH_MINIMAX_M3 ? n_head : uint32_t(64));
// qwen4exp ropes indexer keys with the main rotary width, so its head can't be < n_rot
ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,
arch == LLM_ARCH_QWEN4EXP ? n_embd_head : uint32_t(128));
// note: using a realistic top-k here makes the results unstable and hard to match between CPU and GPU
// a large value makes things deterministic since all data is selected by the indexer
//ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(131072));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, uint32_t(4));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_KPOOL, uint32_t(4));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_KPOOL_SELECT_TAIL, true);
ms.add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, uint32_t(1));
// mrope sections count rope pairs; Ling 3.0 VL files carry [t, h, w] sections
// summing to n_rot / 2 (n_rot is 64 in this fixture)
if (arch == LLM_ARCH_BAILINGMOE3) {
ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector<uint32_t>({8, 12, 12, 0}));
} else {
ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector<uint32_t>({n_embd_head/4, n_embd_head/4, n_embd_head/4, n_embd_head/4}));
}
if (arch == LLM_ARCH_HY_V4) {
ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);
ms.add_kv(LLM_KV_HYPER_CONNECTION_MAGNITUDE, 2.0f);
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 10.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
// layer 0 must own an indexer, the odd layers share it
std::vector<uint32_t> indexer_types;
indexer_types.reserve(n_layer);
for (uint32_t il = 0; il < n_layer; il++) {
indexer_types.push_back(il % 2 ? 0 : 1);
}
ms.add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, indexer_types);
}
if (arch == LLM_ARCH_MISTRAL_LARGE4) {
// layer 3 reads the KV cache of layer 2, both are NoPE
ms.add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, std::vector<uint32_t>({1, 1, 0, 0}));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, std::vector<uint32_t>({1, 0, 1, 1}));
ms.add_kv(LLM_KV_ATTENTION_KV_SOURCE_MAP, std::vector<uint32_t>({0, 1, 2, 2}));
ms.add_kv(LLM_KV_FINAL_LOGIT_SOFTCAPPING, 30.0f);
}
if (arch == LLM_ARCH_DEEPSEEK4) {
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(32));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 0, 4, 128}));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 160000.0f);
ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2));
ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);
ms.add_kv(LLM_KV_HASH_LAYER_COUNT, uint32_t(0));
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 10.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
}
if (arch == LLM_ARCH_DEEPSEEK41) {
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(32));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 0, 2, 2, 1, 1}));
ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 160000.0f);
ms.add_kv(LLM_KV_ATTENTION_KV_SOURCE_LAYERS, std::vector<uint32_t>({2, 4}));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_SOURCE_LAYERS, std::vector<uint32_t>({2, 4, 5}));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_SOURCE_LAYER, int32_t(4));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_TOP_K_BLOCKS, uint32_t(4096));
ms.add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE, uint32_t(4));
ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2));
ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);
ms.add_kv(LLM_KV_HASH_LAYER_COUNT, uint32_t(0));
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 10.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
std::vector<int32_t> token_map(n_vocab);
for (uint32_t i = 0; i < n_vocab; ++i) {
token_map[i] = (int32_t) (i % 61);
}
ms.add_kv(LLM_KV_PLE_LAYERS, std::vector<uint32_t>({1, 3}));
ms.add_kv(LLM_KV_PLE_NGRAM_SIZE, uint32_t(3));
ms.add_kv(LLM_KV_PLE_HEADS_PER_NGRAM, uint32_t(2));
ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, uint32_t(64));
ms.add_kv(LLM_KV_PLE_LAYER_MULTIPLIERS, std::vector<uint64_t>({ 76632096046245ull, 4839876093313ull, 35959672319349ull,
67716810739261ull, 51510806800915ull, 30921347202721ull }));
ms.add_kv(LLM_KV_PLE_HEAD_OFFSETS, std::vector<uint64_t>({0, 101, 204, 311, 420, 533, 660, 791}));
ms.add_kv(LLM_KV_PLE_HEAD_VOCAB_SIZES, std::vector<uint64_t>({101, 103, 107, 109, 113, 127, 131, 137}));
ms.add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, uint32_t(2));
ms.add_kv(LLM_KV_ENGRAM_TOKEN_MAP, token_map);
}
if (arch == LLM_ARCH_HRM_TEXT) {
// 6 cache slots alias 2 physical blocks: 1 low-stack layer + 1 high-stack layer
ms.add_kv(LLM_KV_HRM_LAYERS_PER_STACK, uint32_t(1));
ms.add_kv(LLM_KV_HRM_H_CYCLES, uint32_t(2));
ms.add_kv(LLM_KV_HRM_L_CYCLES, uint32_t(2));
}
if (arch == LLM_ARCH_MAPLE) {
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f);
}
if (arch == LLM_ARCH_LIMITE) {
ms.add_kv(LLM_KV_ATTENTION_SCALE, 0.1f);
ms.add_kv(LLM_KV_ATTENTION_XSA_EPS, 1e-4f);
ms.add_kv(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, std::vector<float>({23.0f, 5.0f, 7.5f}));
ms.add_kv(LLM_KV_MUDD_FEED_FORWARD_LENGTH, uint32_t(8));
ms.add_kv(LLM_KV_MUDD_TAP_COUNT, uint32_t(2));
ms.add_kv(LLM_KV_MUDD_TAP_INDICES, std::vector<int32_t>({-1, -1, -1, -1, 0, 2, 1, 3}));
}
// dummy tokenizer: token ids are derived from fixed-size chunks and detokenized as hex ids
{
std::vector<std::string> tokenizer_list(n_vocab);
std::vector<float> tokenizer_scores(n_vocab, 0.0f);
ms.add_kv(LLM_KV_TOKENIZER_MODEL, "test");
for (uint32_t i = 0; i < n_vocab; i++) {
tokenizer_list[i] = "tok_" + std::to_string(i);
}
ms.add_kv(LLM_KV_TOKENIZER_LIST, tokenizer_list);
ms.add_kv(LLM_KV_TOKENIZER_SCORES, tokenizer_scores);
}
// ms.add_kv(LLM_KV_DENSE_2_FEAT_OUT, n_embd);
// ms.add_kv(LLM_KV_DENSE_3_FEAT_IN, n_embd);
if (moe) {
ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, n_ff);
ms.add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, n_ff / 2); // distinct from n_ff so a saver key-clobber surfaces on reload
ms.add_kv(LLM_KV_EXPERT_LATENT_LENGTH, n_ff);
ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, uint32_t(2));
// with more experts than a ubatch uses, the copy of the used experts in host memory skips some of them
ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(host_experts_test(arch) ? 64 : 2));
ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, uint32_t(1));
uint32_t gating_func = 2;
switch (arch) {
case LLM_ARCH_MELLUM:
gating_func = 1;
break;
case LLM_ARCH_DEEPSEEK4:
case LLM_ARCH_DEEPSEEK41:
gating_func = 4;
break;
default:
break;
}
ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, gating_func); // sigmoid : softmax : sqrtsoftplus
ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERTS_PER_GROUP, uint32_t(1));
if (arch == LLM_ARCH_FLINT) {
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
}
if (arch == LLM_ARCH_K2_HORIZON) {
ms.add_kv(LLM_KV_ATTENTION_VALUE_EXPERT_COUNT, uint32_t(2));
ms.add_kv(LLM_KV_ATTENTION_VALUE_EXPERT_USED_COUNT, uint32_t(2));
}
}
if (arch == LLM_ARCH_BEAM) {
ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(8));
ms.add_kv(LLM_KV_EXPERT_GROUP_COUNT, uint32_t(4));
ms.add_kv(LLM_KV_EXPERT_GROUP_USED_COUNT, uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 2.5f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
}
if (arch == LLM_ARCH_MAION_CODER) {
ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd / 2);
ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, std::vector<uint32_t>{n_ff, n_ff / 2, n_ff, n_ff});
}
ms.add_kv(LLM_KV_POSNET_EMBEDDING_LENGTH, n_embd);
ms.add_kv(LLM_KV_POSNET_BLOCK_COUNT, n_layer);
ms.add_kv(LLM_KV_CONVNEXT_EMBEDDING_LENGTH, n_embd);
ms.add_kv(LLM_KV_CONVNEXT_BLOCK_COUNT, n_layer);
ms.add_kv(LLM_KV_XIELU_ALPHA_N, 1.0f);
ms.add_kv(LLM_KV_XIELU_ALPHA_P, 1.0f);
ms.add_kv(LLM_KV_XIELU_BETA, 1.0f);
ms.add_kv(LLM_KV_XIELU_EPS, 1.0e-7f);
ms.add_kv(LLM_KV_SSM_INNER_SIZE, arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP ? 256 : 2*n_embd);
ms.add_kv(LLM_KV_SSM_CONV_KERNEL, uint32_t(4));
ms.add_kv(LLM_KV_SSM_STATE_SIZE, uint32_t(128));
ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head);
ms.add_kv(LLM_KV_SSM_GROUP_COUNT, arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2));
ms.add_kv(LLM_KV_KDA_HEAD_DIM, uint32_t(128));
ms.add_kv(LLM_KV_KDA_SAFE_GATE, true);
ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
if (arch == LLM_ARCH_BAILINGMOE3) {
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector<float>({0.0f, 4.0f}));
ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector<float>({0.0f, 5.0f}));
}
ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head);
ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3));
ms.add_kv(LLM_KV_RESIDUAL_SCALE, 3.5565588200778455f);
ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(12));
ms.add_kv(LLM_KV_ACTIVATION_SITU_BETA, 4.0f);
ms.add_kv(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, 25.0f);
ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
for (uint32_t il = 0; il < n_layer; il++) {
ggml_tensor t;
memset(&t, 0, sizeof(ggml_tensor));
t.type = GGML_TYPE_F16;
ggml_format_name(&t, "conv%" PRIu32 "d.weight", il);
gguf_add_tensor(ms.gguf_ctx, &t);
ggml_format_name(&t, "posnet.%" PRIu32 ".conv1.weight", il);
gguf_add_tensor(ms.gguf_ctx, &t);
ggml_format_name(&t, "posnet.%" PRIu32 ".conv2.weight", il);
gguf_add_tensor(ms.gguf_ctx, &t);
ggml_format_name(&t, "convnext.%" PRIu32 ".dw.weight", il);
gguf_add_tensor(ms.gguf_ctx, &t);
}
return ret;
}
static bool silent_model_load_progress(float /*progress*/, void * /*user_data*/) {
return true;
}
static std::string get_tmp_gguf_path() {
static size_t id = 0;
return (std::filesystem::temp_directory_path() / ("test-llama-archs-" + std::to_string(++id) + ".gguf")).string();
}
static llama_model_ptr get_model(
struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const float stdev,
const std::vector<ggml_backend_dev_t> & devs,
const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER,
const llama_model_tensor_buft_override * tensor_buft_overrides = nullptr) {
GGML_ASSERT((gguf_ctx == nullptr) != (file == nullptr));
llama_model_params model_params = llama_model_default_params();
model_params.progress_callback = silent_model_load_progress;
std::vector<ggml_backend_dev_t> devs_copy = devs;
devs_copy.push_back(nullptr);
model_params.devices = devs_copy.data();
model_params.split_mode = split_mode;
model_params.tensor_buft_overrides = tensor_buft_overrides;
tensor_data_params tensor_params = { seed, stdev };
llama_model_ptr model(gguf_ctx != nullptr ?
llama_model_init_from_user(gguf_ctx, set_tensor_data, &tensor_params, model_params) :
llama_model_load_from_file_ptr(file, model_params));
if (!model) {
throw std::runtime_error("failed to create llama model");
}
return model;
}
static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(
struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const float stdev,
const std::vector<ggml_backend_dev_t> & devs,
const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER, bool encode = false,
const llama_model_tensor_buft_override * tensor_buft_overrides = nullptr, const size_t moe_cache_size = 0) {
llama_model_ptr model = get_model(gguf_ctx, file, seed, stdev, devs, split_mode, tensor_buft_overrides);
llama_context_params ctx_params = llama_context_default_params();
ctx_params.n_ctx = 0;
ctx_params.n_threads = 4;
ctx_params.n_threads_batch = 4;
if (!encode) {
ctx_params.n_ubatch = 64;
}
if (moe_cache_size > 0) {
// the MoE cache is only used for small ubatches
ctx_params.moe_cache_size = moe_cache_size;
ctx_params.n_ubatch = 2;
}
llama_context_ptr lctx(llama_init_from_model(model.get(), ctx_params));
if (!lctx) {
throw std::runtime_error("failed to create llama context");
}
return std::make_pair(std::move(model), std::move(lctx));
}
static std::vector<float> get_logits(
llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens, bool encode = false) {
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
const uint32_t n_ctx = llama_n_ctx(lctx);
const uint32_t n_tokens = tokens.size();
common_batch batch(lctx);
GGML_ASSERT(n_tokens <= n_ctx);
for (uint32_t pos = 0; pos < n_tokens; pos++) {
batch.add(tokens[pos], pos, 0, true);
}
if (encode) {
if (llama_process(lctx, LLAMA_PROCESS_TYPE_ENCODE, batch.get())) {
throw std::runtime_error("failed to encode batch");
}
}
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
throw std::runtime_error("failed to decode batch");
}
std::vector<float> ret;
ret.reserve(n_tokens*n_vocab);
for (uint32_t i = 0; i < n_tokens; i++) {
const float * logits_ith = llama_get_logits_ith(lctx, i);
for (uint32_t j = 0; j < n_vocab; j++) {
ret.push_back(logits_ith[j]);
}
}
return ret;
}
// entries [n/4, n/2) are embd rows, decoded either as token/embd/token chunks or as one mixed batch
// returns the llama_process() error code
static int32_t get_logits_mixed(
llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens, const std::vector<float> & embd, bool mixed,
std::vector<float> & ret) {
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
const uint32_t n_embd = llama_model_n_embd_inp(model);
const uint32_t n_tokens = tokens.size();
const uint32_t i_embd_0 = n_tokens/4;
const uint32_t i_embd_1 = n_tokens/2;
const std::vector<uint32_t> bounds = mixed
? std::vector<uint32_t>{0, n_tokens}
: std::vector<uint32_t>{0, i_embd_0, i_embd_1, n_tokens};
llama_memory_clear(llama_get_memory(lctx), true);
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
ret.clear();
ret.reserve(n_tokens*n_vocab);
for (size_t c = 0; c + 1 < bounds.size(); c++) {
llama_batch_ext_clear(batch.get());
for (uint32_t i = bounds[c]; i < bounds[c + 1]; i++) {
const bool is_embd = i >= i_embd_0 && i < i_embd_1;
const int32_t idx = is_embd
? llama_batch_ext_add_embd(batch.get(), 0, { embd.data() + (size_t) (i - i_embd_0)*n_embd, 1, n_embd })
: llama_batch_ext_add_token(batch.get(), 0, tokens[i]);
GGML_ASSERT(idx >= 0);
const llama_pos pos[4] = { (llama_pos) i, (llama_pos) i, (llama_pos) i, 0 };
llama_batch_ext_set_pos(batch.get(), idx, pos);
llama_batch_ext_set_output_logits(batch.get(), idx, true);
}
const int32_t err = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
if (err != 0) {
return err;
}
for (uint32_t i = 0; i < bounds[c + 1] - bounds[c]; i++) {
const float * logits_ith = llama_get_logits_ith(lctx, i);
ret.insert(ret.end(), logits_ith, logits_ith + n_vocab);
}
}
return 0;
}
static bool check_causal_attn_toggle(
llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
const uint32_t n_past = tokens.size();
const uint32_t n_ubatch = llama_n_ubatch(lctx);
GGML_ASSERT(n_past + n_ubatch/2 + n_ubatch <= llama_n_ctx(lctx));
llama_set_causal_attn(lctx, false);
common_batch batch(lctx);
bool ok = true;
uint32_t pos = n_past;
for (const uint32_t n_tokens : { n_ubatch/2, n_ubatch }) {
batch.clear();
for (uint32_t i = 0; i < n_tokens; i++) {
batch.add(tokens[i], pos++, 0, true);
}
const int32_t rc = llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
if (rc != 0) {
LOG_ERR("%s: n_tokens=%u: llama_process returned %d\n", __func__, n_tokens, rc);
ok = false;
break;
}
const float * logits = llama_get_logits_ith(lctx, n_tokens - 1);
if (logits == nullptr) {
LOG_ERR("%s: n_tokens=%u: no logits\n", __func__, n_tokens);
ok = false;
break;
}
for (uint32_t j = 0; j < n_vocab; j++) {
if (std::isnan(logits[j])) {
LOG_ERR("%s: n_tokens=%u: nan logit\n", __func__, n_tokens);
ok = false;
break;
}
}
}
return ok;
}
static bool moe_mandatory(const llm_arch arch) {
switch (arch) {
case LLM_ARCH_FLINT:
case LLM_ARCH_LLAMA4:
case LLM_ARCH_MAION_CODER:
case LLM_ARCH_COHERE2MOE:
case LLM_ARCH_GROK:
case LLM_ARCH_QWEN2MOE:
case LLM_ARCH_QWEN3MOE:
case LLM_ARCH_QWEN3NEXT:
case LLM_ARCH_QWEN3VLMOE:
case LLM_ARCH_QWEN35MOE:
case LLM_ARCH_QWEN4EXP:
case LLM_ARCH_PHIMOE:
case LLM_ARCH_DBRX:
case LLM_ARCH_OLMOE:
case LLM_ARCH_ARCTIC:
case LLM_ARCH_DEEPSEEK:
case LLM_ARCH_DEEPSEEK2:
case LLM_ARCH_DEEPSEEK32:
case LLM_ARCH_DOTS3NOTE:
case LLM_ARCH_DEEPSEEK4:
case LLM_ARCH_DEEPSEEK41:
case LLM_ARCH_GLM4_MOE:
case LLM_ARCH_GLM_DSA:
case LLM_ARCH_EXAONE_MOE:
case LLM_ARCH_BAILINGMOE:
case LLM_ARCH_BAILINGMOE2:
case LLM_ARCH_BAILINGMOE3:
case LLM_ARCH_DOTS1:
case LLM_ARCH_AFMOE:
case LLM_ARCH_BEAM:
case LLM_ARCH_ERNIE4_5:
case LLM_ARCH_ERNIE4_5_MOE:
case LLM_ARCH_HUNYUAN_MOE:
case LLM_ARCH_HY_V3:
case LLM_ARCH_HY_V4:
case LLM_ARCH_OPENAI_MOE:
case LLM_ARCH_LFM2MOE:
case LLM_ARCH_SMALLTHINKER:
case LLM_ARCH_LLADA_MOE:
case LLM_ARCH_GROVEMOE:
case LLM_ARCH_MINIMAX_01:
case LLM_ARCH_MINIMAX_M2:
case LLM_ARCH_MINIMAX_M3:
case LLM_ARCH_RND1:
case LLM_ARCH_PADDLEOCR:
case LLM_ARCH_MIMO2:
case LLM_ARCH_KIMI_LINEAR:
case LLM_ARCH_KIMI_K3:
case LLM_ARCH_GLM5_NEXT:
case LLM_ARCH_STEP35:
case LLM_ARCH_MISTRAL4:
case LLM_ARCH_MISTRAL_LARGE4:
case LLM_ARCH_MELLUM:
case LLM_ARCH_LAGUNA:
case LLM_ARCH_MAPLE:
case LLM_ARCH_MUSE_SPARK:
case LLM_ARCH_KOLIBRI1:
return true;
default:
return false;
}
}
static bool moe_implemented(const llm_arch arch) {
if (moe_mandatory(arch)) {
return true;
}
switch (arch) {
case LLM_ARCH_LLAMA:
case LLM_ARCH_REFACT:
case LLM_ARCH_MINICPM:
case LLM_ARCH_GRANITE:
case LLM_ARCH_GRANITE_MOE:
case LLM_ARCH_MISTRAL3:
case LLM_ARCH_LLAMA_EMBED:
case LLM_ARCH_K2_HORIZON:
return true;
default:
return false;
}
}
static bool arch_supported(const llm_arch arch) {
if (arch == LLM_ARCH_CLIP || arch == LLM_ARCH_GPTJ || arch == LLM_ARCH_UNKNOWN) {
return false; // These models don't have usable implementations.
}
if (arch == LLM_ARCH_CHAMELEON) {
return false; // Only half-implemented and to be removed in the future.
}
if (arch == LLM_ARCH_WAVTOKENIZER_DEC) {
return false; // FIXME CUDA backend crashes.
}
if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
return false; // FIXME @ngxson
}
if (arch == LLM_ARCH_GRANITE_SWITCH) {
return false; // FIXME adapter fixture
}
if (arch == LLM_ARCH_LLAMA_EMBED || arch == LLM_ARCH_GEMMA_EMBEDDING || arch == LLM_ARCH_GEMMA_EMBEDDING2 || arch == LLM_ARCH_T5ENCODER) {
return false; // FIXME Embedding (?) models produce inconsistent results.
}
if (arch == LLM_ARCH_RWKV6 || arch == LLM_ARCH_RWKV6QWEN2 || arch == LLM_ARCH_RWKV7 || arch == LLM_ARCH_ARWKV7) {
return false; // FIXME RWKV models hang indefinitely.
}
if (arch == LLM_ARCH_BERT || arch == LLM_ARCH_MODERN_BERT || arch == LLM_ARCH_NOMIC_BERT || arch == LLM_ARCH_NOMIC_BERT_MOE ||
arch == LLM_ARCH_NEO_BERT || arch == LLM_ARCH_JINA_BERT_V2 || arch == LLM_ARCH_JINA_BERT_V3 || arch == LLM_ARCH_EUROBERT) {
return false; // TODO vocab
}
if (arch == LLM_ARCH_PLM) {
return false; // TODO tensor shapes
}
if (arch == LLM_ARCH_CLEF || arch == LLM_ARCH_VLLM_DECISION2) {
return false; // TODO decision head tensors
}
if (arch == LLM_ARCH_DEEPSEEK2OCR) {
return false;
}
// FIXME: these hit scheduler/view-backed-output issues with WebGPU on CI.
#ifdef GGML_USE_WEBGPU
if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_QWEN4EXP ||
arch == LLM_ARCH_HY_V4) {
return false;
}
#endif // GGML_USE_WEBGPU
// FIXME: jamba produces incorrect output (~0.55 NMSE vs CPU) on the HIP
// backend on RDNA3.5 (gfx1151); the SSM kernels need investigation.
#ifdef GGML_USE_HIP
if (arch == LLM_ARCH_JAMBA) {
return false;
}
#endif // GGML_USE_HIP
return true;
}
static int save_models(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity, const std::string & dir) {
struct user_data_t {
struct {
ggml_log_callback callback;
void * user_data;
} log_old;
int verbosity;
user_data_t(int verbosity) : verbosity(verbosity) {
llama_log_get(&log_old.callback, &log_old.user_data);
}
};
user_data_t ud(verbosity);
llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
const user_data_t * ud = (const user_data_t *) user_data;
int verbosity = common_log_get_verbosity(level);
if (verbosity <= ud->verbosity) {
ud->log_old.callback(level, text, ud->log_old.user_data);
}
}, &ud);
for (const llm_arch & arch : llama_arch_all()) {
if (arch == LLM_ARCH_UNKNOWN) {
continue;
}
if (!arch_matches(arch_filter, arch)) {
continue;
}
if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
continue; // FIXME: ISWA KV cache initialization needs more fixture params
}
if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH) {
continue;
}
for (bool moe : {false, true}) {
if (moe && !moe_implemented(arch)) {
continue;
}
if (!moe && moe_mandatory(arch)) {
continue;
}
if (!llama_arch_supports_save(arch) || !arch_supported(arch)) {
LOG_INF("%s: %s model (%s) is unsupported, skipping\n", __func__, llama_arch_name(arch), moe ? "MoE" : "dense");
continue;
}
gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe);
llama_model_ptr model = get_model(gguf_ctx.get(), nullptr, seed, stdev, {});
const std::string path = dir + "/" + llama_arch_name(arch) + (moe ? "-moe.gguf" : "-dense.gguf");
LOG_INF("%s: '%s %s' -> %s ...", __func__, llama_arch_name(arch), moe ? "MoE" : "dense", path.c_str());
llama_model_save_to_file(model.get(), path.c_str());
const std::uintmax_t size = std::filesystem::file_size(path);
LOG_CNT(" (%.2f MB)\n", (double) size / (1024.0 * 1024.0));
}
}
llama_log_set(ud.log_old.callback, ud.log_old.user_data);
return 0;
}
// the CPU and the GPU of an Apple SoC carry the same description, the name tells them apart (CPU, MTL0)
static std::string get_device_label(ggml_backend_dev_t dev) {
const char * desc = ggml_backend_dev_description(dev);
const char * name = ggml_backend_dev_name(dev);
if (strcmp(desc, name) == 0) {
return desc;
}
return std::string(desc) + " (" + name + ")";
}
static int test_backends(const std::string & arch_filter, const size_t seed, const float stdev, const int verbosity, const char * target_backend) {
struct user_data_t {
struct {
ggml_log_callback callback;
void * user_data;
} log_old;
int verbosity;
user_data_t(int verbosity) : verbosity(verbosity) {
llama_log_get(&log_old.callback, &log_old.user_data);
}
};
user_data_t ud(verbosity);
llama_log_set([](ggml_log_level level, const char * text, void * user_data) {
const user_data_t * ud = (const user_data_t *) user_data;
int verbosity = common_log_get_verbosity(level);
if (verbosity <= ud->verbosity) {
ud->log_old.callback(level, text, ud->log_old.user_data);
}
}, &ud);
const std::vector<llama_token> tokens = get_tokens(128, 128, seed);
struct device_config {
std::vector<ggml_backend_dev_t> devs;
std::string label;
llama_split_mode split_mode;
bool host_experts; // keep the experts in host memory, see host_experts_test
size_t moe_cache_size;
device_config(std::vector<ggml_backend_dev_t> devs, std::string name, llama_split_mode split_mode, bool host_experts = false, size_t moe_cache_size = 0)
: devs(std::move(devs)), label(std::move(name)), split_mode(split_mode), host_experts(host_experts), moe_cache_size(moe_cache_size) {}
};
const llama_model_tensor_buft_override host_experts_overrides[] = {
{ LLM_FFN_EXPS_REGEX, ggml_backend_cpu_buffer_type() },
{ nullptr, nullptr },
};
std::vector<device_config> dev_configs;
size_t max_device_label_length = 4;
{
std::vector<ggml_backend_dev_t> devices_meta;
{
const size_t device_count = ggml_backend_dev_count();
for (size_t i = 0; i < device_count; i++) {
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
if (target_backend != nullptr && strcmp(target_backend, ggml_backend_dev_name(dev)) != 0) {
continue;
}
dev_configs.emplace_back(std::vector<ggml_backend_dev_t>{dev}, get_device_label(dev), LLAMA_SPLIT_MODE_LAYER);
max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length());
// cpu-based devices cannot be used in tensor split mode
if (ggml_backend_dev_buffer_type(dev) != ggml_backend_cpu_buffer_type()) {
devices_meta.push_back(dev);
}
}
}
if (target_backend == nullptr) {
dev_configs.emplace_back(devices_meta, "Meta", LLAMA_SPLIT_MODE_TENSOR);
}
// the ops that use the experts are offloaded to the first device and the scheduler copies the used experts
if (!devices_meta.empty()) {
dev_configs.emplace_back(devices_meta, "Host experts", LLAMA_SPLIT_MODE_LAYER, true);
max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length());
}
// the ops that use the host experts run on a GPU and read the experts from a cache
// the cache has only a few slots (4 for 288 KiB experts), so the experts are evicted and uploaded again
if (!devices_meta.empty()) {
const enum ggml_backend_dev_type type = ggml_backend_dev_type(devices_meta[0]);
if (type == GGML_BACKEND_DEVICE_TYPE_GPU || type == GGML_BACKEND_DEVICE_TYPE_IGPU) {
dev_configs.emplace_back(std::vector<ggml_backend_dev_t>{devices_meta[0]}, "MoE cache", LLAMA_SPLIT_MODE_LAYER, true, 1536*1024);
max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length());
// each GPU caches the layers assigned to it
if (devices_meta.size() > 1) {
dev_configs.emplace_back(devices_meta, "MoE cache, layer split", LLAMA_SPLIT_MODE_LAYER, true, 1536*1024);
max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length());
}
}
}
}
size_t max_arch_name_length = 0;
for (const llm_arch & arch : llama_arch_all()) {
max_arch_name_length = std::max(max_arch_name_length, strlen(llama_arch_name(arch)));
}
const std::string template_header = std::string("|%" + std::to_string(max_arch_name_length) + "s|%") + std::to_string(max_device_label_length) + "s|%6s|%15s|%9s|%15s|\n";
const std::string template_row_cfg = std::string("|%" + std::to_string(max_arch_name_length) + "s|%") + std::to_string(max_device_label_length) + "s|%6s|";
const std::string template_row_res = "%15s %10s|%20s|%15s %10s|\n";
bool all_ok = true;
size_t n_tests = 0;
size_t n_failed = 0;
common_log_flush(common_log_main());
LOG(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip", "Mixed batch");
LOG("|");
for (size_t i = 0; i < max_arch_name_length; i++) {
LOG("-");
}
LOG("|");
for (size_t i = 0; i < max_device_label_length; i++) {
LOG("-");
}
LOG("|------|---------------|---------|---------------|\n");
for (const llm_arch & arch : llama_arch_all()) {
if (arch == LLM_ARCH_UNKNOWN) {
continue;
}
if (!arch_matches(arch_filter, arch)) {
continue;
}
if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {
continue; // FIXME: ISWA KV cache initialization needs more fixture params
}
if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH) {
continue;
}
const bool encode = arch == LLM_ARCH_T5 || arch == LLM_ARCH_DREAM || arch == LLM_ARCH_LLADA || arch == LLM_ARCH_LLADA_MOE || arch == LLM_ARCH_RND1;
for (bool moe : {false, true}) {
if (moe && !moe_implemented(arch)) {
continue;
}
if (!moe && moe_mandatory(arch)) {
continue;
}
const std::string config_name = moe ? "MoE" : "Dense";
gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe);
if (arch == LLM_ARCH_BAILINGMOE3) {
GGML_ASSERT(gguf_remove_key(gguf_ctx.get(), "bailingmoe3.kda.safe_gate") >= 0);
}
std::pair<llama_model_ptr, llama_context_ptr> model_and_ctx_cpu;
std::vector<float> logits_cpu;
for (device_config & dc : dev_configs) {
if (dc.host_experts && (!moe || !host_experts_test(arch))) {
continue;
}
const llama_model_tensor_buft_override * overrides = dc.host_experts ? host_experts_overrides : nullptr;
// print test config first; should anything fail during model loading or inference, at least we know which test case caused it
LOG(template_row_cfg.c_str(), llama_arch_name(arch), dc.label.c_str(), config_name.c_str());
fflush(stdout);
std::pair<llama_model_ptr, llama_context_ptr> model_and_ctx_dev;
std::vector<float> logits_dev;
std::string status_nmse = "\033[1;33mSKIP\033[0m";
std::string status_roundtrip = "\033[1;33mSKIP\033[0m";
std::string status_mixed = "\033[1;33mSKIP\033[0m";
char nmse_str[12] = {0};
char mixed_str[12] = {0};
bool skip = !arch_supported(arch) || (dc.split_mode == LLAMA_SPLIT_MODE_TENSOR && dc.devs.empty());
bool test_executed = false;
bool test_ok = true;
if (!skip) {
if (logits_cpu.empty()) {
model_and_ctx_cpu = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, {}, LLAMA_SPLIT_MODE_LAYER, encode);
logits_cpu = get_logits(model_and_ctx_cpu.first.get(), model_and_ctx_cpu.second.get(), tokens, encode);
}
if (dc.split_mode != LLAMA_SPLIT_MODE_TENSOR || llama_arch_supports_sm_tensor(arch)) {
test_executed = true;
model_and_ctx_dev = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, stdev, dc.devs, dc.split_mode, encode, overrides, dc.moe_cache_size);
logits_dev = get_logits(model_and_ctx_dev.first.get(), model_and_ctx_dev.second.get(), tokens, encode);
const double nmse_val = nmse(logits_cpu, logits_dev);
snprintf(nmse_str, sizeof(nmse_str), "(%.2e)", nmse_val);
status_nmse = "\033[1;32mOK\033[0m";
if (nmse_val > 1e-4) {
test_ok = false;
status_nmse = "\033[1;31mFAIL\033[0m";
}
// chunked decode matches a single batch only with causal attention over a memory
llama_context * lctx_dev = model_and_ctx_dev.second.get();
if (!encode && llama_get_memory(lctx_dev) != nullptr) {
std::vector<float> embd_mixed((size_t) llama_model_n_embd_inp(model_and_ctx_dev.first.get())*tokens.size()/4);
std::mt19937 gen(seed);
std::normal_distribution<float> dis(0.0f, stdev);
for (float & v : embd_mixed) {
v = dis(gen);
}
std::vector<float> logits_mixed;
std::vector<float> logits_chunks;
if (llama_arch_supports_mixed_batch(arch)) {
if (get_logits_mixed(model_and_ctx_dev.first.get(), lctx_dev, tokens, embd_mixed, false, logits_chunks) != 0 ||
get_logits_mixed(model_and_ctx_dev.first.get(), lctx_dev, tokens, embd_mixed, true, logits_mixed) != 0) {
throw std::runtime_error("failed to decode mixed batch");
}
const double nmse_mixed = nmse(logits_chunks, logits_mixed);
snprintf(mixed_str, sizeof(mixed_str), "(%.2e)", nmse_mixed);
status_mixed = "\033[1;32mOK\033[0m";
if (nmse_mixed > 1e-4) {
test_ok = false;
status_mixed = "\033[1;31mFAIL\033[0m";
}
} else {
// must be rejected as an invalid batch, mute the expected error log
ud.verbosity = LOG_LEVEL_OUTPUT;
const int32_t err = get_logits_mixed(model_and_ctx_cpu.first.get(), model_and_ctx_cpu.second.get(), tokens, embd_mixed, true, logits_mixed);
ud.verbosity = verbosity;
if (err != -1) {
test_ok = false;
status_mixed = "\033[1;31mFAIL\033[0m";
}
}
}
// runs after the mixed batch check, as it leaves the context with non-causal attention
if (!encode && !check_causal_attn_toggle(model_and_ctx_dev.first.get(), model_and_ctx_dev.second.get(), tokens)) {
if (test_ok) {
status_nmse = "\033[1;31mFAIL\033[0m (toggle)";
}
test_ok = false;
}
}
if (llama_arch_supports_save(arch) && dc.split_mode != LLAMA_SPLIT_MODE_TENSOR) {
test_executed = true;
GGML_ASSERT(model_and_ctx_dev.first && model_and_ctx_dev.second);
// FIXME: when adding a tensor to a gguf_context a copy is made, this changes the pointer which the meta backend
// in turn uses to map the tensors to their simple equivalents - this is fundamentally incompatible
const std::string path = get_tmp_gguf_path();
llama_model_save_to_file(model_and_ctx_dev.first.get(), path.c_str());
FILE * file = fopen(path.c_str(), "rb");
GGML_ASSERT(file != nullptr);
{
auto model_and_ctx_roundtrip = get_model_and_ctx(nullptr, file, seed, stdev, dc.devs, dc.split_mode, encode, overrides, dc.moe_cache_size);
const std::vector<float> logits_roundtrip = get_logits(
model_and_ctx_roundtrip.first.get(), model_and_ctx_roundtrip.second.get(), tokens, encode);
status_roundtrip = "\033[1;32mOK\033[0m";
GGML_ASSERT(logits_roundtrip.size() == logits_dev.size());
for (size_t i = 0; i < logits_roundtrip.size(); i++) {
if (logits_roundtrip[i] != logits_dev[i]) {
test_ok = false;
status_roundtrip = "\033[1;31mFAIL\033[0m";
break;
}
}
}
fclose(file);
std::filesystem::remove(path);
}
}
if (test_executed) {
n_tests++;
if (!test_ok) {
n_failed++;
all_ok = false;
}
}
// log the results for this test case
LOG(template_row_res.c_str(), status_nmse.c_str(), nmse_str, status_roundtrip.c_str(), status_mixed.c_str(), mixed_str);
}
}
}
if (n_tests == 0) {
LOG("Summary: no tests executed\n");
} else if (n_failed == 0) {
LOG("Summary: all %zu test(s) passed\n", n_tests);
} else {
LOG("Summary: %zu test(s) executed, %zu failed\n", n_tests, n_failed);
}
llama_log_set(ud.log_old.callback, ud.log_old.user_data);
return all_ok ? 0 : 1;
}
int main(int argc, char ** argv) {
// init the logger at max verbosity. filter with a custom callback respecting the user-configure verbosity
common_log_set_verbosity_thold(LOG_LEVEL_DEBUG);
common_init();
llama_backend_init();
std::random_device rd;
std::string arch_filter;
size_t seed = rd();
float stdev = 0.1f;
std::string out;
const char * target_backend = nullptr;
int verbosity = LOG_LEVEL_ERROR;
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
usage(argv);
return 0;
} else if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) {
if (i + 1 < argc) {
const std::string arch_name = argv[++i];
if (llama_arch_from_string(arch_name) != LLM_ARCH_UNKNOWN) {
// exact architecture name
arch_filter = "^" + arch_name + "$";
} else {
try {
std::regex re(arch_name);
arch_filter = arch_name;
} catch (const std::regex_error & err) {
LOG_ERR("%s: invalid architecture regex: %s (%s)\n", __func__, arch_name.c_str(), err.what());
return 1;
}
}
} else {
usage(argv);
return 1;
}
} else if (strcmp(argv[i], "-s") == 0 || strcmp(argv[i], "--seed") == 0) {
if (i + 1 < argc) {
seed = std::stoull(argv[++i]);
} else {
usage(argv);
return 1;
}
} else if (strcmp(argv[i], "-d") == 0 || strcmp(argv[i], "--stdev") == 0) {
if (i + 1 < argc) {
stdev = std::stof(argv[++i]);
} else {
usage(argv);
return 1;
}
} else if (strcmp(argv[i], "-v") == 0) {
if (i + 1 < argc) {
verbosity = std::stoull(argv[++i]);
} else {
usage(argv);
return 1;
}
} else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--out") == 0) {
if (i + 1 < argc) {
out = argv[++i];
} else {
usage(argv);
return 1;
}
} else if (strcmp(argv[i], "-b") == 0 || strcmp(argv[i], "--backend") == 0) {
if (i + 1 < argc) {
const char * backend_name = argv[++i];
ggml_backend_dev_t dev = ggml_backend_dev_by_name(backend_name);
if (dev == nullptr) {
LOG_ERR("%s: unknown backend device: %s\n", __func__, backend_name);
return 1;
}
target_backend = ggml_backend_dev_name(dev);
} else {
usage(argv);
return 1;
}
} else {
LOG_ERR("%s: unknown argument: %s\n", __func__, argv[i]);
usage(argv);
return 1;
}
}
if (stdev <= 0.0f) {
LOG_ERR("%s: stdev must be > 0\n", __func__);
return 1;
}
LOG_INF("%s: using seed %zu, stdev %f\n", __func__, seed, stdev);
init_normal_pool(seed, stdev);
try {
if (!out.empty()) {
return save_models(arch_filter, seed, stdev, verbosity, out);
}
return test_backends(arch_filter, seed, stdev, verbosity, target_backend);
} catch (const std::exception & err) {
fprintf(stderr, "encountered runtime error: %s\n", err.what());
return -1;
}
}
Reference in new issue View Git Blame Copy Permalink