Merge branch 'ggml-org:master' into master

This commit is contained in:
Ilya Maslennikov authored and GitHub committed 2026-10-07 15:51:44 +03:00
commit bd5ea24694
16 files changed
+507 -210

No files matched your search

+5 -2
View File
@@ -399,8 +399,11 @@ struct common_sampler * common_sampler_init(
// only if user explicitly included adaptive-p sampler
samplers.push_back(llama_sampler_init_adaptive_p(params.adaptive_target, params.adaptive_decay, params.seed));
} else {
// default: sample from distribution
samplers.push_back(llama_sampler_init_dist(params.seed));
// Keep distribution sampling when callers request probabilities.
const bool greedy = params.n_probs == 0 && !params.samplers.empty() &&
((params.samplers.back() == COMMON_SAMPLER_TYPE_TEMPERATURE && params.temp == 0.0f && params.dynatemp_range == 0.0f) ||
(params.samplers.back() == COMMON_SAMPLER_TYPE_TOP_K && params.top_k == 1));
samplers.push_back(greedy ? llama_sampler_init_greedy() : llama_sampler_init_dist(params.seed));
}
} else if (params.mirostat == 1) {
samplers.push_back(llama_sampler_init_temp(params.temp));
+12 -6
View File
@@ -2499,7 +2499,11 @@ class TextModel(ModelBase):
if template is not None:
self.gguf_writer.add_chat_template(template)
def _set_vocab_plamo(self):
def _set_vocab_plamo(
self,
eot_token: str,
normal_tokens: Iterable[str] = (),
):
# PLaMo models use a custom tokenizer with a .jsonl file
tokenizer_jsonl_path = self.dir_model / "tokenizer.jsonl"
tokenizer_config_path = self.dir_model / "tokenizer_config.json"
@@ -2523,27 +2527,30 @@ class TextModel(ModelBase):
tokens = []
scores = []
toktypes = []
normal_tokens = set(normal_tokens)
with open(tokenizer_jsonl_path, "r", encoding="utf-8") as f:
for line_num, line in enumerate(f):
if line.strip():
token_data = json.loads(line)
# Format: [token, score, type, ?, ?, ?, ?]
token = token_data[0].encode("utf-8")
token_str = token_data[0]
token = token_str.encode("utf-8")
score = float(token_data[1])
token_type_str = token_data[2] if len(token_data) > 2 else "NORMAL"
tokens.append(token)
scores.append(score)
if token_type_str == "UNKNOWN":
if token_str in normal_tokens:
toktypes.append(gguf.TokenType.NORMAL)
elif token_type_str == "UNKNOWN":
toktypes.append(gguf.TokenType.UNKNOWN)
elif token_type_str == "CONTROL":
toktypes.append(gguf.TokenType.CONTROL)
elif token_type_str == "BYTE":
toktypes.append(gguf.TokenType.BYTE)
else:
token_str = token_data[0]
if token_str.startswith("<|plamo:") and token_str.endswith("|>"):
toktypes.append(gguf.TokenType.CONTROL)
else:
@@ -2580,8 +2587,7 @@ class TextModel(ModelBase):
token_id = tokens.index(tokenizer_config["unk_token"].encode("utf-8"))
self.gguf_writer.add_unk_token_id(token_id)
# Add <|plamo:op|> as EOT to ensure appropriate end of generation
self.gguf_writer.add_eot_token_id(4)
self.gguf_writer.add_eot_token_id(tokens.index(eot_token.encode("utf-8")))
self.gguf_writer.add_add_space_prefix(False)
+5 -2
View File
@@ -64,7 +64,7 @@ class Plamo2Model(TextModel):
model_arch = gguf.MODEL_ARCH.PLAMO2
def set_vocab(self):
self._set_vocab_plamo()
self._set_vocab_plamo(eot_token="<|plamo:op|>")
def set_gguf_parameters(self):
hparams = self.hparams
@@ -170,7 +170,10 @@ class Plamo3Model(TextModel):
})
def set_vocab(self):
self._set_vocab_plamo()
self._set_vocab_plamo(
eot_token="<|plamo:tag|>",
normal_tokens=("<|plamo:begin_", "<|plamo:end_", ":plamo|>"),
)
tokenizer_config_path = self.dir_model / "tokenizer_config.json"
tokenizer_config = {}
+4 -2
View File
@@ -218,8 +218,10 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
if hparams is None:
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
hparams["text_config"] = {"hidden_size": hparams["talker_config"]["hidden_size"]}
# ECAPA-TDNN has a fixed 4-stage backbone, but MmprojModel.__init__ needs a n_block_keys
hparams["speaker_encoder_config"]["n_layers"] = 4
# ECAPA-TDNN has a fixed 4-stage backbone, but MmprojModel.__init__ needs a n_block_keys.
# The CustomVoice variant ships no speaker encoder, so its config lacks this key entirely.
if "speaker_encoder_config" in hparams:
hparams["speaker_encoder_config"]["n_layers"] = 4
super().__init__(dir_model, *args, hparams=hparams, **kwargs)
self._wav_config_cache = None
+53 -36
View File
@@ -239,6 +239,14 @@ static __global__ void lightning_indexer_kernel_wmma(
// tokens scored per block by the tile kernel
#define LIGHTNING_INDEXER_TILE_TOKENS 8
// heads whose queries the tile kernel stages per pass, MUSA arch 21 caps static shared memory
// at 28 KB and the queries of four heads do not fit there next to the key tile
#if defined(GGML_USE_MUSA) && defined(__MUSA_ARCH__) && __MUSA_ARCH__ < 220
#define LIGHTNING_INDEXER_TILE_HEADS_PER_PASS 2
#else
#define LIGHTNING_INDEXER_TILE_HEADS_PER_PASS 4
#endif
// TODO there is one ugly assumption used in this kernel - that WARP_SIZE is equal to 32
// thanks to that one warp operating on float4 processes whole indexer K/Q vectors
// 32 * 4 = 128 (N_EMBD)
@@ -406,9 +414,11 @@ static __global__ void lightning_indexer_kernel_tile(
constexpr int KEY_LANES = THREADS_PER_BLOCK / TOKENS_PER_BLOCK;
constexpr int KEYS_PER_THREAD = K_VECS_PER_BLOCK / KEY_LANES;
constexpr int N_EMBD_H2 = N_EMBD / 2;
constexpr int HEADS_PER_PASS = N_HEAD < LIGHTNING_INDEXER_TILE_HEADS_PER_PASS ? N_HEAD : LIGHTNING_INDEXER_TILE_HEADS_PER_PASS;
static_assert(THREADS_PER_BLOCK % TOKENS_PER_BLOCK == 0, "threads must cover the token tile");
static_assert(K_VECS_PER_BLOCK % KEY_LANES == 0, "key lanes must cover the key tile");
static_assert(N_HEAD % HEADS_PER_PASS == 0, "head passes must cover the heads");
const int tid = threadIdx.y * WARP_SIZE + threadIdx.x;
const int start_kv = blockIdx.x * K_VECS_PER_BLOCK;
@@ -417,7 +427,7 @@ static __global__ void lightning_indexer_kernel_tile(
// the row padding keeps the keys of consecutive threads in distinct banks
__shared__ half2 k_shared[K_VECS_PER_BLOCK][N_EMBD_H2 + 1];
__shared__ float2 q_shared[N_HEAD][TOKENS_PER_BLOCK][N_EMBD_H2];
__shared__ float2 q_shared[HEADS_PER_PASS][TOKENS_PER_BLOCK][N_EMBD_H2];
__shared__ float w_shared[N_HEAD][TOKENS_PER_BLOCK];
// phase 1 - stage the key tile four elements at a time, rows past n_kv are zero
@@ -451,22 +461,7 @@ static __global__ void lightning_indexer_kernel_tile(
k_shared[r][2*c4 + 1] = hi;
}
// phase 2 - stage the queries and weights of every head, tokens past n_batch are zero
#pragma unroll
for (int i = tid; i < N_HEAD * TOKENS_PER_BLOCK * (N_EMBD / 4); i += THREADS_PER_BLOCK) {
const int h = i / (TOKENS_PER_BLOCK * (N_EMBD / 4));
const int r = i / (N_EMBD / 4) % TOKENS_PER_BLOCK;
const int c4 = i % (N_EMBD / 4);
float4 v = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
if (start_batch + r < n_batch) {
v = *(const float4 *) ((const char *) Q + h*nbq1 + (start_batch + r)*nbq2 + i_stream*nbq3 + c4*sizeof(float4));
}
q_shared[h][r][2*c4 + 0] = make_float2(v.x, v.y);
q_shared[h][r][2*c4 + 1] = make_float2(v.z, v.w);
}
// phase 2 - stage the weights of every head, tokens past n_batch are zero
if (tid < N_HEAD * TOKENS_PER_BLOCK) {
const int h = tid / TOKENS_PER_BLOCK;
@@ -475,33 +470,60 @@ static __global__ void lightning_indexer_kernel_tile(
((const float *) ((const char *) W + (start_batch + r)*nbw1 + i_stream*nbw3))[h] : 0.0f;
}
__syncthreads();
// phase 3 - float products of the widened keys for every head, ReLU, weight
const int kl = tid % KEY_LANES;
const int tl = tid / KEY_LANES;
float qk[N_HEAD][KEYS_PER_THREAD] = { { 0.0f } };
#pragma unroll 8
for (int c = 0; c < N_EMBD_H2; ++c) {
float2 k_val[KEYS_PER_THREAD];
#pragma unroll
for (int j = 0; j < KEYS_PER_THREAD; ++j) {
k_val[j] = __half22float2(k_shared[kl + j*KEY_LANES][c]);
for (int h0 = 0; h0 < N_HEAD; h0 += HEADS_PER_PASS) {
// the previous pass is fully consumed before its queries are replaced
if (h0 > 0) {
__syncthreads();
}
// phase 3 - stage the queries of the heads of this pass, tokens past n_batch are zero
#pragma unroll
for (int h = 0; h < N_HEAD; ++h) {
const float2 q_val = q_shared[h][tl][c];
for (int i = tid; i < HEADS_PER_PASS * TOKENS_PER_BLOCK * (N_EMBD / 4); i += THREADS_PER_BLOCK) {
const int h = i / (TOKENS_PER_BLOCK * (N_EMBD / 4));
const int r = i / (N_EMBD / 4) % TOKENS_PER_BLOCK;
const int c4 = i % (N_EMBD / 4);
float4 v = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
if (start_batch + r < n_batch) {
v = *(const float4 *) ((const char *) Q + (h0 + h)*nbq1 + (start_batch + r)*nbq2 + i_stream*nbq3 + c4*sizeof(float4));
}
q_shared[h][r][2*c4 + 0] = make_float2(v.x, v.y);
q_shared[h][r][2*c4 + 1] = make_float2(v.z, v.w);
}
__syncthreads();
// phase 4 - float products of the widened keys for the heads of this pass
#pragma unroll 8
for (int c = 0; c < N_EMBD_H2; ++c) {
float2 k_val[KEYS_PER_THREAD];
#pragma unroll
for (int j = 0; j < KEYS_PER_THREAD; ++j) {
qk[h][j] = fmaf(k_val[j].x, q_val.x, qk[h][j]);
qk[h][j] = fmaf(k_val[j].y, q_val.y, qk[h][j]);
k_val[j] = __half22float2(k_shared[kl + j*KEY_LANES][c]);
}
#pragma unroll
for (int h = 0; h < HEADS_PER_PASS; ++h) {
const float2 q_val = q_shared[h][tl][c];
#pragma unroll
for (int j = 0; j < KEYS_PER_THREAD; ++j) {
qk[h0 + h][j] = fmaf(k_val[j].x, q_val.x, qk[h0 + h][j]);
qk[h0 + h][j] = fmaf(k_val[j].y, q_val.y, qk[h0 + h][j]);
}
}
}
}
// phase 5 - ReLU, weight, add the mask and write, consecutive threads write consecutive keys
float score[KEYS_PER_THREAD] = { 0.0f };
#pragma unroll
@@ -512,8 +534,6 @@ static __global__ void lightning_indexer_kernel_tile(
}
}
// phase 4 - add the mask and write, consecutive threads write consecutive keys
const int i_batch = start_batch + tl;
if (i_batch >= n_batch) {
return;
@@ -677,8 +697,6 @@ void ggml_cuda_lightning_indexer(ggml_backend_cuda_context & ctx, ggml_tensor *
LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_vec, 128, 32, k, GGML_TYPE_F32)
GGML_ABORT("fatal error");
}
#ifndef GGML_USE_MUSA
// MUSA archs 21 and 22 cap static shared memory at 28 KB, below what the tile kernel stages
} else if (n_embd == 128 && n_head == 4 && n_batch >= LIGHTNING_INDEXER_TILE_TOKENS) {
// too few heads for a wmma tile, the tile kernel shares the keys across the tokens
constexpr int WARPS_PER_BLOCK = 8;
@@ -698,9 +716,8 @@ void ggml_cuda_lightning_indexer(ggml_backend_cuda_context & ctx, ggml_tensor *
LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_tile, 128, 4, k, GGML_TYPE_BF16)
LIGHTNING_INDEXER_CASE(lightning_indexer_kernel_tile, 128, 4, k, GGML_TYPE_F32)
GGML_ABORT("fatal error");
#endif // GGML_USE_MUSA
} else if (n_embd == 128 && n_head == 4) {
// a batch smaller than a token tile, or MUSA, use vector kernel
// a batch smaller than a token tile, use vector kernel
constexpr int K_VECS_PER_WARP = 8;
constexpr int WARPS_PER_BLOCK = 8;
constexpr int K_VECS_PER_BLOCK = K_VECS_PER_WARP * WARPS_PER_BLOCK;
+1 -1
View File
@@ -4544,7 +4544,7 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
break;
}
if (ggml_webgpu_tensor_binding_overlap(ctx->webgpu_global_ctx, src1, src2) &&
src1->type != src2->type && !ggml_is_quantized(src1->type) && !ggml_is_quantized(src2->type)) {
src1->type != src2->type && !(ggml_is_quantized(src1->type) && ggml_is_quantized(src2->type))) {
supports_op = false;
break;
}
+1 -1
View File
@@ -5,7 +5,7 @@ import os
import sys
import subprocess
HTTPLIB_VERSION = "refs/tags/v0.59.0"
HTTPLIB_VERSION = "refs/tags/v0.60.0"
# used by examples/gguf-hash, these repos have no release tag, so we pin a commit
XXHASH_COMMIT = "9f465f1ea932d6ad9a26cd77496311ffa544cd68"
+4
View File
@@ -238,6 +238,10 @@ bool llama_memory_recurrent::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos
// partial rollback via per-token snapshot index (bounded by n_rs_seq)
if (0 < p0 && p0 <= cell.pos && p1 > cell.pos) {
// a cell shared by several sequences cannot move back for only one of them
if (cell.seq_id.size() > 1) {
return false;
}
// the filter kept no layer (e.g. an MTP draft context), so only the position moves back
if (is_empty()) {
cell.pos = p0 - 1;
+6
View File
@@ -1086,6 +1086,12 @@ static void llama_sampler_greedy_backend_apply(
struct ggml_tensor * curl = ggml_argmax(ctx, logits);
ggml_set_name(curl, "greedy_argmax");
if (data->candidates != nullptr) {
struct ggml_tensor * candidates = ggml_reshape_2d(ctx, data->candidates, 1, ggml_nelements(data->candidates));
curl = ggml_get_rows(ctx, candidates, curl);
ggml_set_name(curl, "greedy_sampled_token");
}
data->sampled = curl;
}
+4
View File
@@ -2860,6 +2860,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
|| t.first == "▁<PRE>" // CodeLlama
|| t.first == "<|code_prefix|>" // GLM-4.5
|| t.first == "<|prefix|>" // Falcon-H1-Tiny-Coder
|| t.first == "<|plamo:fim_prefix|>" // PLaMo-3
) {
special_fim_pre_id = t.second;
if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
@@ -2881,6 +2882,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
|| t.first == "▁<SUF>" // CodeLlama
|| t.first == "<|code_suffix|>" // GLM-4.5
|| t.first == "<|suffix|>" // Falcon-H1-Tiny-Coder
|| t.first == "<|plamo:fim_suffix|>" // PLaMo-3
) {
special_fim_suf_id = t.second;
if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
@@ -2902,6 +2904,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
|| t.first == "▁<MID>" // CodeLlama
|| t.first == "<|code_middle|>" // GLM-4.5
|| t.first == "<|middle|>" // Falcon-H1-Tiny-Coder
|| t.first == "<|plamo:fim_middle|>" // PLaMo-3
) {
special_fim_mid_id = t.second;
if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
@@ -2952,6 +2955,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
if (special_fim_sep_id == LLAMA_TOKEN_NULL) {
if (false
|| t.first == "<|file_sep|>" // Qwen
|| t.first == "<|plamo:file_separator|>" // PLaMo-2/3
) {
special_fim_sep_id = t.second;
if ((attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) {
+160 -25
View File
@@ -64,9 +64,13 @@ void llama_model_glm5_next::load_arch_tensors(llama_model_loader & ml) {
const int64_t hc = hparams.dsv4_hc_mult;
const int64_t hc_mix_dim = (2 + hc)*hc;
// the NextN block is loaded but only used by the MTP graph.
// Separated trunk_only/mtp_only handling TODO with DECODER_MTP graph in the MTP follow up
int mtp_flags = 0;
// TODO: consolidate the trunk_only/mtp_only detection and flags shared with glm4-moe and qwen35moe
const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr);
const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";
const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);
const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;
int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;
if (!ml.load_mtp) {
mtp_flags |= TENSOR_SKIP;
}
@@ -79,18 +83,19 @@ void llama_model_glm5_next::load_arch_tensors(llama_model_loader & ml) {
for (int i = 0; i < n_layer_all; ++i) {
auto & layer = layers[i];
const int flags = (i >= n_layer) ? mtp_flags : 0;
const bool is_nextn = i >= n_layer;
const int flags = is_nextn ? mtp_flags : trunk_flags;
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags);
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags);
if (i < n_layer) {
layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc*n_embd, hc_mix_dim}, 0);
layer.hc_attn_base = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE, "weight", i), {hc_mix_dim}, 0);
layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {3}, 0);
layer.hc_ffn_fn = create_tensor(tn(LLM_TENSOR_HC_FFN_FN, "weight", i), {hc*n_embd, hc_mix_dim}, 0);
layer.hc_ffn_base = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE, "weight", i), {hc_mix_dim}, 0);
layer.hc_ffn_scale = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE, "weight", i), {3}, 0);
layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc*n_embd, hc_mix_dim}, flags);
layer.hc_attn_base = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE, "weight", i), {hc_mix_dim}, flags);
layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {3}, flags);
layer.hc_ffn_fn = create_tensor(tn(LLM_TENSOR_HC_FFN_FN, "weight", i), {hc*n_embd, hc_mix_dim}, flags);
layer.hc_ffn_base = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE, "weight", i), {hc_mix_dim}, flags);
layer.hc_ffn_scale = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE, "weight", i), {3}, flags);
}
const int64_t head_dim = hparams.n_embd_head_kda;
@@ -100,25 +105,25 @@ void llama_model_glm5_next::load_arch_tensors(llama_model_loader & ml) {
if (hparams.is_recr(i)) {
auto conv = [&](llm_tensor tid) {
ggml_tensor * t = create_tensor(tn(tid, "weight", i), {d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED);
return t ? t : create_tensor(tn(tid, "weight", i), {d_conv, 1, d_inner}, 0);
return t ? t : create_tensor(tn(tid, "weight", i), {d_conv, 1, d_inner}, flags);
};
layer.ssm_q_conv = conv(LLM_TENSOR_SSM_CONV1D_Q);
layer.ssm_k_conv = conv(LLM_TENSOR_SSM_CONV1D_K);
layer.ssm_v_conv = conv(LLM_TENSOR_SSM_CONV1D_V);
create_tensor_qkv(layer, i, n_embd, d_inner, d_inner, d_inner, 0);
create_tensor_qkv(layer, i, n_embd, d_inner, d_inner, d_inner, flags);
layer.ssm_f_a = create_tensor(tn(LLM_TENSOR_SSM_F_A, "weight", i), {n_embd, head_dim}, 0);
layer.ssm_f_b = create_tensor(tn(LLM_TENSOR_SSM_F_B, "weight", i), {head_dim, d_inner}, 0);
layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, 0);
layer.ssm_f_a = create_tensor(tn(LLM_TENSOR_SSM_F_A, "weight", i), {n_embd, head_dim}, flags);
layer.ssm_f_b = create_tensor(tn(LLM_TENSOR_SSM_F_B, "weight", i), {head_dim, d_inner}, flags);
layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, flags);
layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), {n_head}, 0);
layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner}, 0);
layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), {n_head}, flags);
layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner}, flags);
layer.ssm_g_a = create_tensor(tn(LLM_TENSOR_SSM_G_A, "weight", i), {n_embd, head_dim}, 0);
layer.ssm_g_b = create_tensor(tn(LLM_TENSOR_SSM_G_B, "weight", i), {head_dim, d_inner}, 0);
layer.ssm_o_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {head_dim}, 0);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {d_inner, n_embd}, 0);
layer.ssm_g_a = create_tensor(tn(LLM_TENSOR_SSM_G_A, "weight", i), {n_embd, head_dim}, flags);
layer.ssm_g_b = create_tensor(tn(LLM_TENSOR_SSM_G_B, "weight", i), {head_dim, d_inner}, flags);
layer.ssm_o_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {head_dim}, flags);
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {d_inner, n_embd}, flags);
} else {
const int64_t q_lora_rank = hparams.n_lora_q;
const int64_t kv_lora_rank = hparams.n_lora_kv;
@@ -155,9 +160,9 @@ void llama_model_glm5_next::load_arch_tensors(llama_model_loader & ml) {
}
if (i < (int) hparams.n_layer_dense_lead) {
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, flags);
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, flags);
layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, flags);
} else {
const int64_t n_ff_exp = hparams.n_ff_exp(i);
const int64_t n_expert_shared = hparams.n_expert_shared;
@@ -187,7 +192,7 @@ void llama_model_glm5_next::load_arch_tensors(llama_model_loader & ml) {
std::unique_ptr<llm_graph_context> llama_model_glm5_next::build_arch_graph(const llm_graph_params & params) const {
if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
throw std::runtime_error("GLM5-Next NextN graph not implemented yet");
return std::make_unique<graph_mtp>(*this, params);
}
return std::make_unique<graph>(*this, params);
}
@@ -537,6 +542,136 @@ ggml_tensor * llama_model_glm5_next::graph::build_hc_post(
return out;
}
// construct the graph helpers without building the trunk, so graph_mtp can reuse them
llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_params & params, no_build) :
llm_build_delta_net_base(params), model(model) {
}
llama_model_glm5_next::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) :
graph(model, params, no_build{}) {
GGML_ASSERT(hparams.n_layer_nextn > 0 && "GLM5-Next MTP requires n_layer_nextn > 0");
GGML_ASSERT(hparams.n_layer_nextn == 1 && "GLM5-Next MTP currently supports a single NextN block");
const int il = hparams.n_layer() + cparams.nextn_layer_offset;
GGML_ASSERT(cparams.nextn_layer_offset >= 0 &&
cparams.nextn_layer_offset < (int) hparams.n_layer_nextn &&
"nextn_layer_offset out of range [0, n_layer_nextn)");
const auto & layer = model.layers[il];
GGML_ASSERT(layer.nextn.eh_proj && layer.nextn.enorm && layer.nextn.hnorm &&
"GLM5-Next MTP block is missing - load the model with MTP enabled");
GGML_ASSERT(hparams.n_embd_out() == (uint32_t) n_embd && "GLM5-Next MTP hidden width mismatch");
const auto * mctx_hyb = static_cast<const llama_memory_hybrid_idx_context *>(mctx);
auto * inp_hyb = build_inp_mem_hybrid_k();
auto * inp_attn = inp_hyb->get_attn();
auto * inp_kpool = build_inp_kpool(mctx_hyb);
ggml_build_forward_expand(gf, inp_hyb->get_recr()->s_copy);
ggml_tensor * inp_out_ids = build_inp_out_ids();
auto inp = std::make_unique<llm_graph_input_embd_h>(n_embd);
inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
ggml_set_input(inp->tokens);
inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tokens);
ggml_set_input(inp->embd);
inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tokens);
ggml_set_input(inp->h);
ggml_set_name(inp->h, "mtp_h_input");
ggml_tensor * tok_embd = ggml_get_rows(ctx0,
layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd, inp->tokens);
cb(tok_embd, "mtp_tok_embd", il);
ggml_tensor * h = inp->h;
res->add_input(std::move(inp));
ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il);
cb(e_norm, "mtp_enorm", il);
ggml_tensor * h_norm = build_norm(h, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);
cb(h_norm, "mtp_hnorm", il);
ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, ggml_concat(ctx0, e_norm, h_norm, 0), layer.nextn.eh_proj_s);
cb(cur, "mtp_eh_proj", il);
ggml_tensor * inpSA = cur;
cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "mtp_attn_norm", il);
ggml_tensor * prev_sel = nullptr;
cur = build_dsa_layer(cur, layer, mctx_hyb, inp_attn, inp_kpool, &prev_sel, il);
cb(cur, "mtp_attn_out", il);
// narrow to the output tokens before the position-wise FFN; unmasked nextn embeddings need all rows
if (crop_before_nextn(inp_out_ids)) {
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
}
ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
cb(ffn_inp, "mtp_ffn_inp", il);
cur = build_norm(ffn_inp, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
cb(cur, "mtp_ffn_norm", il);
ggml_tensor * moe_out = build_moe_ffn(cur,
layer.ffn_gate_inp,
layer.ffn_up_exps,
layer.ffn_gate_exps,
layer.ffn_down_exps,
layer.ffn_exp_probs_b,
n_expert, n_expert_used,
LLM_FFN_SILU, hparams.expert_weights_norm,
hparams.expert_weights_scale,
(llama_expert_gating_func_type) hparams.expert_gating_func,
il);
cb(moe_out, "mtp_ffn_moe_out", il);
ggml_tensor * ffn_shexp = build_ffn(cur,
layer.ffn_up_shexp, nullptr, nullptr,
layer.ffn_gate_shexp, nullptr, nullptr,
layer.ffn_down_shexp, nullptr, nullptr,
nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
cb(ffn_shexp, "mtp_ffn_shexp", il);
cur = ggml_add(ctx0, moe_out, ffn_shexp);
cb(cur, "mtp_ffn_out", il);
cur = ggml_add(ctx0, cur, ffn_inp);
cb(cur, "mtp_post_ffn", il);
ggml_tensor * head_norm = layer.nextn.shared_head_norm ? layer.nextn.shared_head_norm : model.output_norm;
GGML_ASSERT(head_norm && "GLM5-Next MTP: missing both nextn.shared_head_norm and output_norm");
cur = build_norm(cur, head_norm, nullptr, LLM_NORM_RMS, -1);
cb(cur, "h_nextn", -1);
res->t_h_nextn = cur;
if (crop_after_nextn(inp_out_ids)) {
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
}
cb(cur, "mtp_shared_head_norm", -1);
ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output;
ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s;
GGML_ASSERT(head_w && "GLM5-Next MTP: missing both nextn.shared_head_head and output");
cur = build_lora_mm(head_w, cur, head_s);
cb(cur, "result_output", -1);
res->t_logits = cur;
ggml_build_forward_expand(gf, cur);
}
llama_model_glm5_next::graph::graph(const llama_model & model, const llm_graph_params & params) :
llm_build_delta_net_base(params), model(model) {
+9
View File
@@ -2747,6 +2747,10 @@ struct llama_model_glm5_next : public llama_model_base {
struct graph : public llm_build_delta_net_base {
graph(const llama_model & model, const llm_graph_params & params);
// build the helpers without the trunk, so graph_mtp can reuse them
struct no_build {};
graph(const llama_model & model, const llm_graph_params & params, no_build);
// collapse the hc streams with per-stream weights
ggml_tensor * build_hc_pre(
ggml_tensor * x,
@@ -2795,6 +2799,11 @@ struct llama_model_glm5_next : public llama_model_base {
};
// the NextN / MTP draft head: one DSA block appended after the trunk
struct graph_mtp : public graph {
graph_mtp(const llama_model & model, const llm_graph_params & params);
};
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
+144 -46
View File
@@ -2,6 +2,7 @@
#include "llama.h"
#include "llama-cpp.h"
#include "common.h"
#include "sampling.h"
#ifdef NDEBUG
#undef NDEBUG
@@ -316,7 +317,7 @@ static llama_sampler * test_single_output_backend_sampler_init(
return llama_sampler_init(&test_single_output_backend_sampler_i, ctx);
}
static void test_backend_greedy_sampling(const test_params & params) {
static void test_greedy(const test_params & params) {
const int seq_id = 0;
struct llama_sampler_chain_params backend_sampler_params = llama_sampler_chain_default_params();
@@ -351,7 +352,95 @@ static void test_backend_greedy_sampling(const test_params & params) {
}
}
static void test_backend_top_k_sampling(const test_params & params) {
static void test_greedy_filtered_common(const test_params & params) {
auto check = [&](common_params_sampling sp, bool greedy) {
common_sampler_ptr sampler(common_sampler_init(params.model.get(), sp));
auto * chain = common_sampler_get(sampler.get());
auto * last = llama_sampler_chain_get(chain, llama_sampler_chain_n(chain) - 1);
GGML_ASSERT(strcmp(llama_sampler_name(last), greedy ? "greedy" : "dist") == 0);
llama_token_data tokens[] = {{0, -2.0f, 0.0f}, {1, -1.0f, 0.0f}, {2, 0.0f, 0.0f}, {3, 1.0f, 0.0f}};
llama_token_data_array candidates = {tokens, 4, -1, false};
llama_sampler_apply(chain, &candidates);
GGML_ASSERT(candidates.selected >= 0);
if (greedy || sp.n_probs > 0) {
GGML_ASSERT(candidates.data[candidates.selected].id == 3);
}
if (sp.n_probs > 0) {
GGML_ASSERT(candidates.data[candidates.selected].p == 1.0f);
}
if (sp.dynatemp_range > 0.0f) {
int positive = 0;
for (size_t i = 0; i < candidates.size; ++i) {
positive += candidates.data[i].p > 0.0f;
}
GGML_ASSERT(positive > 1);
}
};
common_params_sampling sp;
sp.temp = 0.0f;
sp.samplers = {COMMON_SAMPLER_TYPE_TOP_K, COMMON_SAMPLER_TYPE_TEMPERATURE};
for (bool backend : {false, true}) {
sp.backend_sampling = backend;
check(sp, true);
auto probabilities = sp;
probabilities.n_probs = 4;
check(probabilities, false);
auto dynamic = sp;
dynamic.dynatemp_range = 1.0f;
check(dynamic, false);
}
auto grammar = sp;
grammar.grammar = {COMMON_GRAMMAR_TYPE_USER, "root ::= [a-z]+"};
check(grammar, true);
auto budget = sp;
budget.reasoning_budget_start = {1};
budget.reasoning_budget_end = {{2}};
budget.reasoning_budget_tokens = 1;
check(budget, true);
sp.samplers = {COMMON_SAMPLER_TYPE_TEMPERATURE, COMMON_SAMPLER_TYPE_TOP_K};
check(sp, false);
sp.samplers.clear();
check(sp, false);
sp.temp = 0.8f;
sp.samplers = {COMMON_SAMPLER_TYPE_TEMPERATURE, COMMON_SAMPLER_TYPE_TOP_K};
for (bool backend : {false, true}) {
sp.backend_sampling = backend;
sp.top_k = 1;
check(sp, true);
auto probabilities = sp;
probabilities.n_probs = 4;
check(probabilities, false);
sp.top_k = 8;
check(sp, false);
}
}
static void test_greedy_filtered(const test_params & params) {
for (int k : {1, 8}) {
llama_sampler_ptr chain(llama_sampler_chain_init(llama_sampler_chain_default_params()));
llama_sampler_chain_add(chain.get(), llama_sampler_init_top_k(k));
llama_sampler_chain_add(chain.get(), llama_sampler_init_greedy());
std::vector<llama_sampler_seq_config> configs = {{0, chain.get()}};
test_context test_ctx(params, configs);
GGML_ASSERT(test_ctx.decode({{0, "Write a Python function"}}));
for (int step = 0; step < 4; ++step) {
const int idx = test_ctx.idx_for_seq(0);
const auto * logits = llama_get_sampled_logits_ith(test_ctx.ctx.get(), idx);
const auto * ids = llama_get_sampled_candidates_ith(test_ctx.ctx.get(), idx);
GGML_ASSERT(llama_get_sampled_logits_count_ith(test_ctx.ctx.get(), idx) == (uint32_t) k);
GGML_ASSERT(llama_get_sampled_candidates_count_ith(test_ctx.ctx.get(), idx) == (uint32_t) k);
const auto expected = ids[std::max_element(logits, logits + k) - logits];
GGML_ASSERT(llama_get_sampled_token_ith(test_ctx.ctx.get(), idx) == expected);
GGML_ASSERT(test_ctx.decode_token(expected));
}
}
}
static void test_top_k(const test_params & params) {
const int seq_id = 0;
const int32_t k = 8;
struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params();
@@ -393,7 +482,7 @@ static void test_backend_top_k_sampling(const test_params & params) {
printf("backend top-k hybrid sampling test PASSED\n");
}
static void test_backend_temp_sampling(const test_params & params) {
static void test_temp(const test_params & params) {
{
const float temp_0 = 0.8f;
struct llama_sampler_chain_params backend_chain_params_0 = llama_sampler_chain_default_params();
@@ -481,7 +570,7 @@ static void test_backend_temp_sampling(const test_params & params) {
printf("backend temp sampling test PASSED\n");
}
static void test_backend_temp_ext_sampling(const test_params & params) {
static void test_temp_ext(const test_params & params) {
{
int seq_id = 0;
const float temp = 0.8f;
@@ -546,7 +635,7 @@ static void test_backend_temp_ext_sampling(const test_params & params) {
printf("backend temp_ext sampling test PASSED\n");
}
static void test_backend_min_p_sampling(const test_params & params) {
static void test_min_p(const test_params & params) {
const int seq_id = 0;
const float p = 0.1;
struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params();
@@ -598,7 +687,7 @@ static void test_backend_min_p_sampling(const test_params & params) {
printf("min-p sampling test PASSED\n");
}
static void test_backend_top_p_sampling(const test_params & params) {
static void test_top_p(const test_params & params) {
const int seq_id = 0;
const float p = 0.9;
struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params();
@@ -648,7 +737,7 @@ static void test_backend_top_p_sampling(const test_params & params) {
printf("top-p sampling test PASSED\n");
}
static void test_backend_multi_sequence_sampling(const test_params & params) {
static void test_multi_sequence(const test_params & params) {
struct llama_sampler_chain_params chain_params_0 = llama_sampler_chain_default_params();
llama_sampler_ptr sampler_chain_0(llama_sampler_chain_init(chain_params_0));
llama_sampler_chain_add(sampler_chain_0.get(), llama_sampler_init_greedy());
@@ -714,7 +803,7 @@ static void test_backend_multi_sequence_sampling(const test_params & params) {
printf("backend multi-sequence sampling test PASSED\n");
}
static void test_backend_dist_sampling(const test_params & params) {
static void test_dist(const test_params & params) {
const int seq_id = 0;
const int32_t seed = 88;
@@ -742,7 +831,7 @@ static void test_backend_dist_sampling(const test_params & params) {
printf("backend dist sampling test PASSED\n");
}
static void test_backend_dist_sampling_and_cpu(const test_params & params) {
static void test_dist_and_cpu(const test_params & params) {
const int seq_id = 0;
const int32_t seed = 88;
@@ -772,7 +861,7 @@ static void test_backend_dist_sampling_and_cpu(const test_params & params) {
printf("backend dist & cpu sampling test PASSED\n");
}
static void test_backend_logit_bias_sampling(const test_params & params) {
static void test_logit_bias(const test_params & params) {
const auto * model = params.model.get();
const auto * vocab = llama_model_get_vocab(model);
@@ -1093,7 +1182,7 @@ static void compare_penalties_logits(
GGML_ASSERT(stats.n_mismatch == 0);
}
static void test_penalty_parameter_values(const test_params & params) {
static void check_penalty_parameter_values(const test_params & params) {
struct penalty_test_case {
const char * name;
float repeat;
@@ -1292,7 +1381,7 @@ static void compare_masking_penalties_logits(
GGML_ASSERT(stats.n_mismatch == 0);
}
static void test_backend_penalties_sampling(const test_params & params) {
static void test_penalties(const test_params & params) {
printf("Testing backend penalties (repeat + freq + presence)\n");
compare_penalties_logits(params, 64, 1.1f, 0.5f, 0.25f, "Hello Hello world");
@@ -1371,14 +1460,14 @@ static void test_backend_penalties_sampling(const test_params & params) {
}, 64, 1.0f, 0.0f, 0.25f, "Hello", penalties_position::after_filter, true);
printf("Testing backend penalty parameter values\n");
test_penalty_parameter_values(params);
check_penalty_parameter_values(params);
printf("backend penalties sampling test PASSED\n");
}
// This test verifies that it is possible to have two different backend samplers,
// one that uses the backend dist sampler, and another that uses CPU dist sampler.
static void test_backend_mixed_sampling(const test_params & params) {
static void test_mixed(const test_params & params) {
struct llama_sampler_chain_params chain_params_0 = llama_sampler_chain_default_params();
llama_sampler_ptr sampler_chain_0(llama_sampler_chain_init(chain_params_0));
llama_sampler_chain_add(sampler_chain_0.get(), llama_sampler_init_dist(88));
@@ -1428,7 +1517,7 @@ static void test_backend_mixed_sampling(const test_params & params) {
printf("backend mixed sampling test PASSED\n");
}
static void test_backend_set_sampler(const test_params & params) {
static void test_set_sampler(const test_params & params) {
const int seq_id = 0;
const int32_t seed = 88;
@@ -1493,7 +1582,7 @@ static void test_backend_set_sampler(const test_params & params) {
printf("backend set sampler test PASSED\n");
}
static void test_backend_cpu_mixed_batch(const test_params & params) {
static void test_cpu_mixed(const test_params & params) {
// Sequence 0 uses backend sampling
struct llama_sampler_chain_params chain_params_0 = llama_sampler_chain_default_params();
llama_sampler_ptr sampler_chain_0(llama_sampler_chain_init(chain_params_0));
@@ -1581,7 +1670,7 @@ static void test_backend_cpu_mixed_batch(const test_params & params) {
printf("backend-cpu mixed batch test PASSED\n");
}
static void test_backend_multi_output_limit(const test_params & params) {
static void test_multi_output_limit(const test_params & params) {
const llama_seq_id seq_id = 0;
llama_sampler_ptr chain(llama_sampler_chain_init(llama_sampler_chain_default_params()));
@@ -1594,15 +1683,15 @@ static void test_backend_multi_output_limit(const test_params & params) {
batch.add(llama_vocab_bos(test_ctx.vocab), i, seq_id, true);
}
printf(">>> test_backend_multi_output_limit expected error start:\n");
printf(">>> test_multi_output_limit expected error start:\n");
const int ret = llama_process(test_ctx.ctx.get(), LLAMA_PROCESS_TYPE_DECODE, batch.get());
GGML_ASSERT(ret != 0 && "llama_decode should reject outputs above the per-sequence limit");
printf("<<< test_backend_multi_output_limit expected error end.\n");
printf("<<< test_multi_output_limit expected error end.\n");
printf("backend multi-output limit test PASSED\n");
}
static void test_backend_multi_sequence_multi_output_dist(const test_params & params) {
static void test_multi_output_multi_sequence_dist(const test_params & params) {
const llama_vocab * vocab = llama_model_get_vocab(params.model.get());
const int32_t n_vocab = llama_vocab_n_tokens(vocab);
const uint32_t seeds[] = { 88, 1337 };
@@ -1697,7 +1786,7 @@ static void test_backend_multi_sequence_multi_output_dist(const test_params & pa
printf("backend multi-sequence multi-output dist test PASSED\n");
}
static void test_backend_multi_output_dist_transaction(const test_params & params) {
static void test_multi_output_dist_transaction(const test_params & params) {
const llama_seq_id seq_id = 0;
const uint32_t seed = 95;
const llama_vocab * vocab = llama_model_get_vocab(params.model.get());
@@ -1762,7 +1851,7 @@ static void test_backend_multi_output_dist_transaction(const test_params & param
printf("backend multi-output dist transaction test PASSED\n");
}
static void test_backend_multi_output_sampling_chain(const test_params & params) {
static void test_multi_output_sampling_chain(const test_params & params) {
const llama_seq_id seq_id = 0;
const uint32_t seed = 88;
const float p = 0.9f;
@@ -1913,7 +2002,7 @@ static void test_backend_multi_output_sampling_chain(const test_params & params)
printf("backend multi-output sampling chain test PASSED\n");
}
static void test_backend_multi_output_cpu_suffix(const test_params & params) {
static void test_multi_output_cpu_suffix(const test_params & params) {
const llama_seq_id seq_id = 0;
const int32_t k = 8;
const llama_vocab * vocab = llama_model_get_vocab(params.model.get());
@@ -1977,28 +2066,35 @@ struct backend_test_case {
bool enabled_by_default;
};
// note: test names are "test_<suffix>" and match the function implementing them
static const backend_test_case BACKEND_TESTS[] = {
{ "greedy", test_backend_greedy_sampling, true },
{ "logit_bias", test_backend_logit_bias_sampling, true },
{ "penalties", test_backend_penalties_sampling, true },
{ "temp", test_backend_temp_sampling, true },
{ "temp_ext", test_backend_temp_ext_sampling, true },
{ "top_k", test_backend_top_k_sampling, true },
{ "multi_sequence", test_backend_multi_sequence_sampling, true },
{ "dist", test_backend_dist_sampling, true },
{ "dist_and_cpu", test_backend_dist_sampling_and_cpu, true },
{ "set_sampler", test_backend_set_sampler, true },
{ "multi_output_limit", test_backend_multi_output_limit, true },
{ "multi_sequence_multi_output_dist", test_backend_multi_sequence_multi_output_dist, true },
{ "multi_output_dist_transaction", test_backend_multi_output_dist_transaction, true },
{ "multi_output_sampling_chain", test_backend_multi_output_sampling_chain, true },
{ "multi_output_cpu", test_backend_multi_output_cpu_suffix, true },
{ "mixed", test_backend_mixed_sampling, true },
{ "min_p", test_backend_min_p_sampling, true },
{ "cpu_mixed", test_backend_cpu_mixed_batch, true },
{ "top_p", test_backend_top_p_sampling, true },
// single sampler
{ "test_greedy", test_greedy, true },
{ "test_greedy_filtered", test_greedy_filtered, true },
{ "test_greedy_filtered_common", test_greedy_filtered_common, true },
{ "test_temp", test_temp, true },
{ "test_temp_ext", test_temp_ext, true },
{ "test_top_k", test_top_k, true },
{ "test_top_p", test_top_p, true },
{ "test_min_p", test_min_p, true },
{ "test_logit_bias", test_logit_bias, true },
{ "test_penalties", test_penalties, true },
// multiple sequences
{ "test_multi_sequence", test_multi_sequence, true },
{ "test_dist", test_dist, true },
{ "test_dist_and_cpu", test_dist_and_cpu, true },
{ "test_mixed", test_mixed, true },
{ "test_cpu_mixed", test_cpu_mixed, true },
{ "test_set_sampler", test_set_sampler, true },
// multiple outputs per sequence
{ "test_multi_output_limit", test_multi_output_limit, true },
{ "test_multi_output_multi_sequence_dist", test_multi_output_multi_sequence_dist, true },
{ "test_multi_output_dist_transaction", test_multi_output_dist_transaction, true },
{ "test_multi_output_sampling_chain", test_multi_output_sampling_chain, true },
{ "test_multi_output_cpu_suffix", test_multi_output_cpu_suffix, true },
};
// TODO: add usage, examples
static test_args parse_cli(int argc, char ** argv) {
test_args out;
@@ -2082,10 +2178,12 @@ static std::vector<const backend_test_case *> collect_tests_to_run(const std::st
}
#ifdef GGML_USE_HIP
// TODO: remove this when https://github.com/ggml-org/llama.cpp/pull/26592 is merged
if (test.name == "penalties" || test.name == "set_sampler" ||
test.name == "mixed" || test.name == "top_p" ||
test.name == "multi_output_sampling_chain" ||
test.name == "multi_output_cpu") {
if (test.name == "test_penalties" ||
test.name == "test_set_sampler" ||
test.name == "test_mixed" ||
test.name == "test_top_p" ||
test.name == "test_multi_output_sampling_chain" ||
test.name == "test_multi_output_cpu_suffix") {
fprintf(stderr, "Skipping test '%s' on HIP backend (no backend TOP_K support)\n", test.name.c_str());
continue;
}
+1
View File
@@ -10,6 +10,7 @@
#include <chrono>
#include <climits>
#include <clocale>
#include <cmath>
#include <cstring>
#include <fstream>
+95 -86
View File
@@ -169,7 +169,8 @@ void SSEClient::stop() {
}
bool SSEClient::parse_sse_line(const std::string &line, SSEMessage &msg,
int &retry_ms, bool &has_data) {
int &retry_ms, bool &has_data,
bool &has_id) {
// Blank line signals end of event
if (line.empty()) { return true; }
@@ -199,6 +200,7 @@ bool SSEClient::parse_sse_line(const std::string &line, SSEMessage &msg,
} else if (field == "id") {
// Empty id is valid (clears the last event ID)
msg.id = value;
has_id = true;
} else if (field == "retry") {
// Parse retry interval in milliseconds
// Per the SSE spec, a value that is not all ASCII digits is ignored.
@@ -269,7 +271,8 @@ void SSEClient::run_event_loop() {
// Event receiving loop
std::string buffer;
SSEMessage current_msg;
bool has_data = false;
auto has_data = false;
auto has_id = false;
while (running_.load() && result.next()) {
buffer.append(result.data(), result.size());
@@ -288,13 +291,13 @@ void SSEClient::run_event_loop() {
if (!line.empty() && line.back() == '\r') { line.pop_back(); }
// Parse the line and check if event is complete
auto event_complete =
parse_sse_line(line, current_msg, reconnect_interval_ms_, has_data);
auto event_complete = parse_sse_line(
line, current_msg, reconnect_interval_ms_, has_data, has_id);
if (event_complete) {
// Update last_event_id for reconnection, even for an event that
// has no data
if (!current_msg.id.empty()) { last_event_id_ = current_msg.id; }
// has no data. An empty id clears it.
if (has_id) { last_event_id_ = current_msg.id; }
// An event without a data field is not dispatched
if (has_data) { dispatch_event(current_msg); }
@@ -302,6 +305,7 @@ void SSEClient::run_event_loop() {
// Reset the message for the next event either way
current_msg.clear();
has_data = false;
has_id = false;
}
}
@@ -3550,9 +3554,11 @@ struct WebSocketUpgradeResponse {
std::string selected_subprotocol;
};
bool read_websocket_upgrade_response(Stream &strm,
const std::string &expected_accept,
WebSocketUpgradeResponse &upgrade) {
bool
read_websocket_upgrade_response(Stream &strm,
const std::string &expected_accept,
const std::string &offered_subprotocols,
WebSocketUpgradeResponse &upgrade) {
// Read status line
const auto bufsiz = 2048;
char buf[bufsiz];
@@ -3609,6 +3615,22 @@ bool read_websocket_upgrade_response(Stream &strm,
upgrade.selected_subprotocol = proto_it->second;
}
// Verify the subprotocol is one the client offered (RFC 6455 4.1)
if (!upgrade.selected_subprotocol.empty()) {
auto was_offered = false;
split(offered_subprotocols.data(),
offered_subprotocols.data() + offered_subprotocols.size(), ',',
[&](const char *b, const char *e) {
if (std::string(b, e) == upgrade.selected_subprotocol) {
was_offered = true;
}
});
if (!was_offered) {
upgrade.error = Error::WebSocketHandshake;
return false;
}
}
return true;
}
@@ -4254,6 +4276,9 @@ bool redirect(T &cli, Request &req, Response &res,
new_req.method = "GET";
new_req.body.clear();
new_req.headers.clear();
new_req.content_length_ = 0;
new_req.content_provider_ = nullptr;
new_req.is_chunked_content_provider_ = false;
}
Response new_res;
@@ -5603,7 +5628,10 @@ bool perform_websocket_handshake(Stream &strm, Request &req,
// Verify 101 response and Sec-WebSocket-Accept header
auto expected_accept = websocket_accept_key(client_key);
return read_websocket_upgrade_response(strm, expected_accept, upgrade);
auto offered_subprotocols =
get_combined_header_value(req.headers, "Sec-WebSocket-Protocol");
return read_websocket_upgrade_response(strm, expected_accept,
offered_subprotocols, upgrade);
}
bool is_ip_address(const std::string &host) {
@@ -9946,6 +9974,12 @@ Server::process_request(Stream &strm, const std::string &remote_addr,
protocols.emplace_back(b, e);
});
selected_subprotocol = entry.sub_protocol_selector(protocols);
// Ignore a selection the client did not offer (RFC 6455 4.2.2)
if (std::find(protocols.begin(), protocols.end(),
selected_subprotocol) == protocols.end()) {
selected_subprotocol.clear();
}
}
}
@@ -11016,7 +11050,7 @@ bool ClientImpl::redirect(Request &req, Response &res, Error &error) {
if (next_host.empty()) { next_host = host_; }
if (next_path.empty()) { next_path = "/"; }
auto path = decode_path_component(next_path) + next_query;
auto path = std::move(next_path) + next_query;
// Same host redirect - use current client
if (next_scheme == scheme && next_host == host_ && next_port == port_) {
@@ -11271,9 +11305,9 @@ bool ClientImpl::write_request(Stream &strm, Request &req,
// Write request line and headers
if (detail::write_request_line(bstrm, req.method, path_with_query) < 0) {
// A rejected method (not a token, e.g. carrying CR/LF) or target (e.g.
// CR/LF smuggled in via a decoded redirect Location under
// set_path_encode(false)) must fail the request cleanly instead of
// emitting a request-line-less, header-injecting request.
// CR/LF in a caller-supplied path under set_path_encode(false)) must
// fail the request cleanly instead of emitting a request-line-less,
// header-injecting request.
error = Error::Write;
rejected_locally = true;
output_error_log(error, &req);
@@ -16304,29 +16338,24 @@ bool verify_hostname(cert_t cert, const char *hostname) {
auto ip_len = impl::parse_ip_address(host_str, ip_bytes);
auto is_ip = ip_len > 0;
// Check Subject Alternative Names (SAN)
// In Mbed TLS 3.x, subject_alt_names contains raw values without ASN.1 tags
// - DNS names: raw string bytes
// - IP addresses: raw IP bytes (4 for IPv4, 16 for IPv6)
// Check Subject Alternative Names (SAN). Mbed TLS keeps the GeneralName type
// in buf.tag and the raw value in buf.p / buf.len.
const mbedtls_x509_sequence *san = &mcert->subject_alt_names;
while (san != nullptr && san->buf.p != nullptr && san->buf.len > 0) {
const unsigned char *p = san->buf.p;
size_t len = san->buf.len;
auto san_type = san->buf.tag & MBEDTLS_ASN1_TAG_VALUE_MASK;
if (is_ip) {
// For an IP host, only a matching iPAddress SAN of the same family
// (4 bytes for IPv4, 16 bytes for IPv6) may authenticate it.
if (len == ip_len && memcmp(p, ip_bytes, ip_len) == 0) { return true; }
} else {
// Check if this SAN is a DNS name (printable ASCII string)
bool is_dns = len > 0;
for (size_t i = 0; i < len && is_dns; i++) {
if (p[i] < 32 || p[i] > 126) { is_dns = false; }
}
if (is_dns) {
std::string san_name(reinterpret_cast<const char *>(p), len);
if (detail::match_hostname(san_name, host_str)) { return true; }
if (san_type == MBEDTLS_X509_SAN_IP_ADDRESS && len == ip_len &&
memcmp(p, ip_bytes, ip_len) == 0) {
return true;
}
} else if (san_type == MBEDTLS_X509_SAN_DNS_NAME) {
std::string san_name(reinterpret_cast<const char *>(p), len);
if (detail::match_hostname(san_name, host_str)) { return true; }
}
san = san->next;
}
@@ -16405,65 +16434,45 @@ bool get_cert_sans(cert_t cert, std::vector<SanEntry> &sans) {
const mbedtls_x509_sequence *cur = &x509->subject_alt_names;
while (cur != nullptr) {
if (cur->buf.len > 0) {
// Mbed TLS stores SAN as ASN.1 sequences
// The tag byte indicates the type
const unsigned char *p = cur->buf.p;
size_t len = cur->buf.len;
size_t value_len = cur->buf.len;
// First byte is the tag
unsigned char tag = *p;
p++;
len--;
// Parse length (simple single-byte length assumed)
if (len > 0 && *p < 0x80) {
size_t value_len = *p;
p++;
len--;
if (value_len <= len) {
SanEntry entry;
// ASN.1 context tags for GeneralName
switch (tag & 0x1F) {
case 2: // dNSName
entry.type = SanType::DNS;
entry.value =
std::string(reinterpret_cast<const char *>(p), value_len);
break;
case 7: // iPAddress
entry.type = SanType::IP;
if (value_len == 4) {
// IPv4
char buf[16];
snprintf(buf, sizeof(buf), "%d.%d.%d.%d", p[0], p[1], p[2], p[3]);
entry.value = buf;
} else if (value_len == 16) {
// IPv6
char buf[64];
snprintf(buf, sizeof(buf),
"%02x%02x:%02x%02x:%02x%02x:%02x%02x:"
"%02x%02x:%02x%02x:%02x%02x:%02x%02x",
p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], p[8],
p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
entry.value = buf;
}
break;
case 1: // rfc822Name (email)
entry.type = SanType::EMAIL;
entry.value =
std::string(reinterpret_cast<const char *>(p), value_len);
break;
case 6: // uniformResourceIdentifier
entry.type = SanType::URI;
entry.value =
std::string(reinterpret_cast<const char *>(p), value_len);
break;
default: entry.type = SanType::OTHER; break;
}
if (!entry.value.empty()) { sans.push_back(std::move(entry)); }
SanEntry entry;
switch (cur->buf.tag & MBEDTLS_ASN1_TAG_VALUE_MASK) {
case MBEDTLS_X509_SAN_DNS_NAME:
entry.type = SanType::DNS;
entry.value = std::string(reinterpret_cast<const char *>(p), value_len);
break;
case MBEDTLS_X509_SAN_IP_ADDRESS:
entry.type = SanType::IP;
if (value_len == 4) {
// IPv4
char buf[16];
snprintf(buf, sizeof(buf), "%d.%d.%d.%d", p[0], p[1], p[2], p[3]);
entry.value = buf;
} else if (value_len == 16) {
// IPv6
char buf[64];
snprintf(buf, sizeof(buf),
"%02x%02x:%02x%02x:%02x%02x:%02x%02x:"
"%02x%02x:%02x%02x:%02x%02x:%02x%02x",
p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], p[8], p[9],
p[10], p[11], p[12], p[13], p[14], p[15]);
entry.value = buf;
}
break;
case MBEDTLS_X509_SAN_RFC822_NAME:
entry.type = SanType::EMAIL;
entry.value = std::string(reinterpret_cast<const char *>(p), value_len);
break;
case MBEDTLS_X509_SAN_UNIFORM_RESOURCE_IDENTIFIER:
entry.type = SanType::URI;
entry.value = std::string(reinterpret_cast<const char *>(p), value_len);
break;
default: entry.type = SanType::OTHER; break;
}
if (!entry.value.empty()) { sans.push_back(std::move(entry)); }
}
cur = cur->next;
}
@@ -18105,9 +18114,9 @@ void WebSocket::start_heartbeat() {
if (ping_interval_sec_ == 0) { return; }
ping_thread_ = std::thread([this]() {
std::unique_lock<std::mutex> lock(ping_mutex_);
while (!closed_) {
ping_cv_.wait_for(lock, std::chrono::seconds(ping_interval_sec_));
if (closed_) { break; }
// The predicate keeps a spurious wakeup from sending a ping early
while (!ping_cv_.wait_for(lock, std::chrono::seconds(ping_interval_sec_),
[this]() { return closed_.load(); })) {
// If the peer has failed to respond to the previous pings, give up.
// RFC 6455 does not define a pong-timeout mechanism; this is an
// opt-in liveness check controlled by max_missed_pongs_.
+3 -3
View File
@@ -8,8 +8,8 @@
#ifndef CPPHTTPLIB_HTTPLIB_H
#define CPPHTTPLIB_HTTPLIB_H
#define CPPHTTPLIB_VERSION "0.59.0"
#define CPPHTTPLIB_VERSION_NUM "0x003b00"
#define CPPHTTPLIB_VERSION "0.60.0"
#define CPPHTTPLIB_VERSION_NUM "0x003c00"
#ifdef _WIN32
#if defined(_WIN32_WINNT) && _WIN32_WINNT < 0x0A00
@@ -4377,7 +4377,7 @@ public:
private:
bool parse_sse_line(const std::string &line, SSEMessage &msg, int &retry_ms,
bool &has_data);
bool &has_data, bool &has_id);
void run_event_loop();
void dispatch_event(const SSEMessage &msg);
bool should_reconnect(int count) const;