diff --git a/common/common.cpp b/common/common.cpp index ca85f66d2..221f0897c 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1188,6 +1188,8 @@ static const std::map COMMON_DECISION_TYPE_NA { COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"}, { COMMON_DECISION_TYPE_H2O_LIGHTNING, "h2o-lightning" }, { COMMON_DECISION_TYPE_MINICPM_OCR_JEV, "openbmb-minicpm-ocr-jev" }, + { COMMON_DECISION_TYPE_LFM3_D1, "lfm3-d1" }, + { COMMON_DECISION_TYPE_LFM3_D1_OMNI, "lfm3-d1-omni" }, // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname" }; diff --git a/common/common.h b/common/common.h index acdbd66ec..cd89a27e9 100644 --- a/common/common.h +++ b/common/common.h @@ -981,6 +981,8 @@ enum common_decision_type { COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch COMMON_DECISION_TYPE_H2O_LIGHTNING, // same as openjev, the labels are read after an answer prefill COMMON_DECISION_TYPE_MINICPM_OCR_JEV, // one score per class, the pooled embeddings output of the last token + COMMON_DECISION_TYPE_LFM3_D1, // same as lfm2-d1, non-causal, label codes A..Z then AA, AB.. + COMMON_DECISION_TYPE_LFM3_D1_OMNI, // same as lfm3-d1 COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported }; diff --git a/conversion/__init__.py b/conversion/__init__.py index c2a5f0b5e..4e4fae608 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -172,6 +172,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "D1OmniModel": "lfm2", "Lfm2Model": "lfm2", "Lfm2MoeForCausalLM": "lfm2", + "Lfm3D1Model": "lfm3", "Lfm3ForCausalLM": "lfm3", "Lfm3OmniForConditionalGeneration": "lfm3", "Lfm3VlForConditionalGeneration": "lfm3", diff --git a/conversion/lfm2.py b/conversion/lfm2.py index 7f4f792e7..61b7a0b2b 100644 --- a/conversion/lfm2.py +++ b/conversion/lfm2.py @@ -77,7 +77,9 @@ def _is_d1_checkpoint(dir_model: Path) -> bool: if not (dir_model / "config.json").is_file(): return False with open(dir_model / "config.json", encoding="utf-8") as f: - return json.load(f).get("auto_map", {}).get("AutoModel", "").endswith(".D1Model") + config = json.load(f) + # d1 on LFM3 is converted by conversion/lfm3.py + return config.get("auto_map", {}).get("AutoModel", "").endswith(".D1Model") and not config.get("model_type", "").startswith("lfm3") @ModelBase.register_hparams_loader(_is_d1_checkpoint) diff --git a/conversion/lfm3.py b/conversion/lfm3.py index 6f80efabc..e8b6d98b0 100644 --- a/conversion/lfm3.py +++ b/conversion/lfm3.py @@ -11,8 +11,8 @@ import torch if TYPE_CHECKING: from torch import Tensor -from .base import MmprojModel, ModelBase, TextModel, gguf -from .lfm2 import lfm2_feed_forward_length +from .base import MmprojModel, ModelBase, TextModel, gguf, logger +from .lfm2 import D1Model, lfm2_feed_forward_length @ModelBase.register("Lfm3ForCausalLM", "Lfm3VlForConditionalGeneration", "Lfm3OmniForConditionalGeneration") @@ -119,6 +119,63 @@ class LFM3Model(TextModel): yield from super().modify_tensors(data_torch, name, bid) +# d1 on LFM3-VL / LFM3-Omni: the stock weights, the d1 prompt read at its last token +_D1_TYPES = {"lfm3_vl": gguf.DecisionType.LFM3_D1, "lfm3_omni": gguf.DecisionType.LFM3_D1_OMNI} + + +def _is_lfm3_d1_checkpoint(dir_model: Path) -> bool: + if not (dir_model / "config.json").is_file(): + return False + with open(dir_model / "config.json", encoding="utf-8") as f: + config = json.load(f) + return config.get("architectures") == ["D1Model"] and config.get("model_type") in _D1_TYPES + + +@ModelBase.register_hparams_loader(_is_lfm3_d1_checkpoint) +def _load_lfm3_d1_hparams(dir_model: Path) -> dict[str, Any]: + logger.info("gguf: detected LFM3 d1 checkpoint") + hparams = ModelBase.load_hparams(dir_model, False, guess=False) + hparams["d1_family"] = hparams["model_type"] + # the mmproj is the base model's, the text model writes the decision + hparams["architectures"] = ["Lfm3VlForConditionalGeneration" if hparams["model_type"] == "lfm3_vl" else "Lfm3OmniForConditionalGeneration"] + hparams["text_config"]["architectures"] = ["Lfm3D1Model"] + return hparams + + +@ModelBase.register("Lfm3D1Model") +@ModelBase.example("LiquidAI/d1-2B", "LiquidAI/d1-omni-2B") +class LFM3D1Model(LFM3Model): + model_arch = gguf.MODEL_ARCH.LFM3 + + # how the checkpoint serves, hard-coded in the server: assert the checkpoint records the same + _served = {"d1_codes": 2, "d1_attention": "set"} + + def set_vocab(self): + # the vocab without the speech task templates of LFM3-Audio/Omni, d1 only decides + TextModel.set_vocab(self) + # the d1 prompt of LFM2, the LFM3 template writes an empty thought before the answer + self.gguf_writer.add_chat_template([{"name": "systemone", "template": D1Model._systemone_template() + ""}]) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + for key, value in self._served.items(): + if self.hparams.get(key) != value: + raise ValueError(f"unsupported d1 checkpoint: {key} = {self.hparams.get(key)!r}, expected {value!r}") + + # a lone question reads the whole prompt both ways on the full attention layers + self.gguf_writer.add_causal_attention(False) + self.gguf_writer.add_decision_type(_D1_TYPES[self.hparams["d1_family"]]) + + # the option answers share one temperature + with open(self.dir_model / "calibration.json", encoding="utf-8") as f: + calibration = json.load(f) + if calibration.get("temperatures"): + raise ValueError("unsupported d1 calibration: per-bucket temperatures") + for name in ("noul", "choice", "score"): + self.gguf_writer.add_decision_temperature(name, calibration["default"]) + + class LFM3VLVisionMixin(MmprojModel): """LFM3-VL vision tower and projector, shared with LFM3-Omni.""" diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 51474c29e..e28fdf605 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -6544,6 +6544,8 @@ class DecisionType: VLLM_D3 = "vllm-d3" # same as pplx-decider, non-causal attention LFM2_D1 = "lfm2-d1" # same as openjev, the labels depend on the question type LFM2_D1_OMNI = "lfm2-d1-omni" # same head as laya on a bidirectional LFM2 trunk, other prompt layout + LFM3_D1 = "lfm3-d1" # same as lfm2-d1 on LFM3-VL, full attention layers read the prompt both ways + LFM3_D1_OMNI = "lfm3-d1-omni" # same as lfm3-d1 on LFM3-Omni GRANITE_SWITCH = "granite-switch" # same as openjev, run by the decision aLoRA of granite switch H2O_LIGHTNING = "h2o-lightning" # same as openjev, labels read after an answer prefill MINICPM_OCR_JEV = "openbmb-minicpm-ocr-jev" # class scores of a value head read at the last token, one fixed task diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 455f5198e..33ad7845f 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1614,8 +1614,9 @@ int llama_context::encode(const llama_batch_ext & batch_inp) { // eagle3/DFlash: features as encoder input, and non-draft paths fall back to model's input dim const int64_t n_vocab = model.vocab.n_tokens(); - // note: during encode, we always output all tokens and skip position continuity checks (output_all=true) - if (!balloc->init(batch_inp, model.vocab, true)) { + // note: during encode, we skip position continuity checks and output all tokens + // a decoder without memory (see decode()) outputs the marked tokens, unless it extracts embeddings + if (!balloc->init(batch_inp, model.vocab, memory || cparams.embeddings)) { LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__); return -1; } @@ -1644,17 +1645,20 @@ int llama_context::encode(const llama_batch_ext & batch_inp) { n_queued_tokens += n_tokens; + const uint32_t n_outputs_all = balloc->get_n_outputs(); + // reserve output buffer - if (output_reserve(n_tokens) < n_tokens) { - LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_tokens); + if (output_reserve(n_outputs_all) < n_outputs_all) { + LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_outputs_all); return -2; }; - for (uint32_t i = 0; i < n_tokens; ++i) { - output_ids[i] = i; + const auto & out_ids = balloc->get_out_ids(); + for (uint32_t i = 0; i < n_outputs_all; ++i) { + output_ids[out_ids[i]] = i; } - n_outputs = n_tokens; + n_outputs = n_outputs_all; const auto causal_attn_org = cparams.causal_attn; @@ -1687,7 +1691,7 @@ int llama_context::encode(const llama_batch_ext & batch_inp) { GGML_ASSERT(backend_res != nullptr); GGML_ASSERT(logits.data != nullptr); - ggml_backend_tensor_get_async(backend_res, t_logits, logits.data, 0, n_tokens*n_vocab*sizeof(float)); + ggml_backend_tensor_get_async(backend_res, t_logits, logits.data, 0, n_outputs*n_vocab*sizeof(float)); } // extract embeddings @@ -1702,8 +1706,8 @@ int llama_context::encode(const llama_batch_ext & batch_inp) { GGML_ASSERT(embd.data != nullptr); const uint32_t n_embd_out = hparams.n_embd_out(); - GGML_ASSERT(n_tokens*n_embd_out <= (int64_t) embd.size); - ggml_backend_tensor_get_async(backend_embd, t_embd, embd.data, 0, n_tokens*n_embd_out*sizeof(float)); + GGML_ASSERT(n_outputs*n_embd_out <= (int64_t) embd.size); + ggml_backend_tensor_get_async(backend_embd, t_embd, embd.data, 0, n_outputs*n_embd_out*sizeof(float)); } break; case LLAMA_POOLING_TYPE_MEAN: case LLAMA_POOLING_TYPE_CLS: @@ -1751,8 +1755,8 @@ int llama_context::encode(const llama_batch_ext & batch_inp) { GGML_ASSERT(backend_h != nullptr); const uint32_t n_embd = hparams.n_embd_out(); - GGML_ASSERT(n_tokens*n_embd <= (int64_t) embd_nextn.size); - ggml_backend_tensor_get_async(backend_h, t_h_nextn, embd_nextn.data, 0, n_tokens*n_embd*sizeof(float)); + GGML_ASSERT(n_outputs*n_embd <= (int64_t) embd_nextn.size); + ggml_backend_tensor_get_async(backend_h, t_h_nextn, embd_nextn.data, 0, n_outputs*n_embd*sizeof(float)); } // TODO: hacky solution diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index a2d1c7def..cffd083d3 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -479,7 +479,7 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) { const int64_t n_kv = self_kq_mask->ne[0]; const int64_t n_tokens = ubatch->n_tokens; - const auto fill_mask = [&](auto * data, int64_t ne, int n_swa, llama_swa_type swa_type) { + const auto fill_mask = [&](auto * data, int64_t ne, int n_swa, llama_swa_type swa_type, bool causal) { using T = std::remove_reference_t; std::fill(data, data + ne, llama_cast(-INFINITY)); @@ -499,7 +499,7 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) { } // mask future tokens - if (cparams.causal_attn && p0 > p1) { + if (causal && p0 > p1) { continue; } @@ -517,21 +517,25 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) { } }; + // see llama_non_causal_type + const bool causal = cparams.causal_attn; + const bool causal_swa = cparams.causal_attn || hparams.non_causal_type == LLAMA_NON_CAUSAL_TYPE_DENSE_ONLY; + GGML_ASSERT(self_kq_mask); GGML_ASSERT(ggml_backend_buffer_is_host(self_kq_mask->buffer)); if (self_kq_mask->type == GGML_TYPE_F16) { - fill_mask((ggml_fp16_t *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE); + fill_mask((ggml_fp16_t *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE, causal); } else { - fill_mask((float *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE); + fill_mask((float *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE, causal); } if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { GGML_ASSERT(self_kq_mask_swa); GGML_ASSERT(ggml_backend_buffer_is_host(self_kq_mask_swa->buffer)); if (self_kq_mask_swa->type == GGML_TYPE_F16) { - fill_mask((ggml_fp16_t *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type); + fill_mask((ggml_fp16_t *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type, causal_swa); } else { - fill_mask((float *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type); + fill_mask((float *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type, causal_swa); } } } diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 7abb372ce..2c464ad30 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -34,6 +34,7 @@ enum llama_non_causal_type { LLAMA_NON_CAUSAL_TYPE_ALL = 0, // all layers non-causal, SWA still applied (gemma 3, qwen-vl, ...) LLAMA_NON_CAUSAL_TYPE_SWA_ONLY = 1, // SWA layers non-causal, dense layers stay causal (gemma 4) LLAMA_NON_CAUSAL_TYPE_SWA_FULL = 2, // all layers non-causal, SWA not applied between tokens of the current ubatch (deepseek 4) + LLAMA_NON_CAUSAL_TYPE_DENSE_ONLY = 3, // dense layers non-causal, SWA layers stay causal (lfm3 d1) }; // M-RoPE: which input position slot feeds each RoPE section diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index e108f0423..9dde1a9b7 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -1749,6 +1749,9 @@ void llama_kv_cache::set_input_kq_mask(ggml_tensor * dst, const llama_ubatch * u if (!causal_attn && hparams.non_causal_type == LLAMA_NON_CAUSAL_TYPE_SWA_ONLY) { causal_attn = swa_type == LLAMA_SWA_TYPE_NONE; } + if (!causal_attn && hparams.non_causal_type == LLAMA_NON_CAUSAL_TYPE_DENSE_ONLY) { + causal_attn = swa_type != LLAMA_SWA_TYPE_NONE; + } //const int64_t t_start = ggml_time_us(); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index c7b2fcc93..81af8903f 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2801,10 +2801,13 @@ ggml_tensor * llama_model::get_rope_factors(const llama_cparams & cparams, int i llama_memory_i * llama_model::create_memory(const llama_memory_params & params, const llama_cparams & cparams) const { llama_memory_i * res; - // the non-causal LFM2 decision graph reads the whole prompt in one batch, nothing is kept + // the non-causal LFM2 and LFM3 decision graphs read the whole prompt in one batch, nothing is kept if (arch == LLM_ARCH_LFM2 && !hparams.causal_attn && hparams.n_layer_decision > 0) { return nullptr; } + if (arch == LLM_ARCH_LFM3 && !hparams.causal_attn) { + return nullptr; + } switch (arch) { // Models that need specific instantiation should be handled in the diff --git a/src/models/lfm-base.cpp b/src/models/lfm-base.cpp index 5e722eca8..f5ff7d178 100644 --- a/src/models/lfm-base.cpp +++ b/src/models/lfm-base.cpp @@ -84,6 +84,33 @@ ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_model & model, return y; } +ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_model & model, + ggml_tensor * cur, + int il) { + const int64_t d_conv = hparams.n_shortconv_l_cache - 1; + + auto * bcx = build_lora_mm(model.layers[il].shortconv.in_proj, cur); + cb(bcx, "model.layers.{}.conv.in_proj", il); + + const int64_t chunk_size = bcx->ne[0] / 3; + auto * b = ggml_view_2d(ctx0, bcx, chunk_size, bcx->ne[1], bcx->nb[1], 0 * chunk_size * ggml_element_size(bcx)); + auto * c = ggml_view_2d(ctx0, bcx, chunk_size, bcx->ne[1], bcx->nb[1], 1 * chunk_size * ggml_element_size(bcx)); + auto * x = ggml_view_2d(ctx0, bcx, chunk_size, bcx->ne[1], bcx->nb[1], 2 * chunk_size * ggml_element_size(bcx)); + + // causal from a zero state + auto * bx = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_mul(ctx0, b, x))); + bx = ggml_pad_ext(ctx0, bx, d_conv, 0, 0, 0, 0, 0, 0, 0); + + auto * conv_out = ggml_ssm_conv(ctx0, bx, model.layers[il].shortconv.conv); + cb(conv_out, "model.layers.{}.conv.conv", il); + + auto * y = ggml_mul(ctx0, c, ggml_reshape_2d(ctx0, conv_out, chunk_size, n_tokens)); + y = build_lora_mm(model.layers[il].shortconv.out_proj, y); + cb(y, "model.layers.{}.conv.out_proj", il); + + return y; +} + ggml_tensor * llm_build_lfm_base::build_lfm_dense_ffn(const llama_model & model, ggml_tensor * cur, int il) { diff --git a/src/models/lfm3.cpp b/src/models/lfm3.cpp index 77eabc467..174ce9cf7 100644 --- a/src/models/lfm3.cpp +++ b/src/models/lfm3.cpp @@ -19,6 +19,11 @@ void llama_model_lfm3::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa); + // d1: full attention layers read the prompt both ways, sliding window layers stay causal + if (!hparams.causal_attn) { + hparams.non_causal_type = LLAMA_NON_CAUSAL_TYPE_DENSE_ONLY; + } + switch (hparams.n_ff()) { case 7680: type = LLM_TYPE_1_7B; break; default: type = LLM_TYPE_UNKNOWN; @@ -84,15 +89,19 @@ void llama_model_lfm3::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_lfm3::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); + if (!hparams.causal_attn) { + return std::make_unique>(*this, params); + } + return std::make_unique>(*this, params); } -llama_model_lfm3::graph::graph(const llama_model & model, const llm_graph_params & params) : +template +llama_model_lfm3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_build_lfm_base(params) { - auto build_attn_block = [&model, this](ggml_tensor * cur_in, - ggml_tensor * inp_pos, - llm_graph_input_attn_kv_iswa * inp_attn, - int il) -> ggml_tensor * { + auto build_attn_block = [&model, this](ggml_tensor * cur_in, + ggml_tensor * inp_pos, + auto * inp_attn, + int il) -> ggml_tensor * { const int64_t n_embd_head = hparams.n_embd_head_k(il); const int64_t n_head_il = hparams.n_head(il); const int64_t n_head_kv = hparams.n_head_kv(il); @@ -140,7 +149,9 @@ llama_model_lfm3::graph::graph(const llama_model & model, const llm_graph_params ggml_build_forward_expand(gf, cur); - auto * inp_hybrid = build_inp_mem_hybrid_iswa(); + // decision: no memory, the whole prompt is read in one batch + auto * inp_hybrid = decision ? nullptr : build_inp_mem_hybrid_iswa(); + auto * inp_attn = decision ? build_attn_inp_no_cache() : nullptr; ggml_tensor * inp_pos = build_inp_pos(); ggml_tensor * inp_out_ids = build_inp_out_ids(); @@ -152,8 +163,14 @@ llama_model_lfm3::graph::graph(const llama_model & model, const llm_graph_params cb(cur, "model.layers.{}.operator_norm", il); if (hparams.is_recr(il)) { - const auto * mctx_recr = static_cast(mctx)->get_recr(); - cur = build_lfm_shortconv(model, mctx_recr, inp_hybrid->get_recr(), cur, il); + if constexpr (decision) { + cur = build_lfm_shortconv(model, cur, il); + } else { + const auto * mctx_recr = static_cast(mctx)->get_recr(); + cur = build_lfm_shortconv(model, mctx_recr, inp_hybrid->get_recr(), cur, il); + } + } else if constexpr (decision) { + cur = build_attn_block(cur, inp_pos, inp_attn, il); } else { cur = build_attn_block(cur, inp_pos, inp_hybrid->get_attn(), il); } @@ -195,3 +212,6 @@ llama_model_lfm3::graph::graph(const llama_model & model, const llm_graph_params res->t_logits = cur; } + +template struct llama_model_lfm3::graph; +template struct llama_model_lfm3::graph; diff --git a/src/models/models.h b/src/models/models.h index 5cd69b59f..6c938c458 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -111,6 +111,11 @@ struct llm_build_lfm_base : public llm_graph_context { ggml_tensor * cur, int il); + // no recurrent state (decision graphs): causal over the ubatch from a zero state + ggml_tensor * build_lfm_shortconv(const llama_model & model, + ggml_tensor * cur, + int il); + ggml_tensor * build_lfm_dense_ffn(const llama_model & model, ggml_tensor * cur, int il); @@ -2438,6 +2443,8 @@ struct llama_model_lfm3 : public llama_model_base { void load_arch_hparams(llama_model_loader & ml) override; void load_arch_tensors(llama_model_loader & ml) override; + // decision: d1, the prompt read in one batch, no memory + template struct graph : public llm_build_lfm_base { graph(const llama_model & model, const llm_graph_params & params); }; diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index ade5668bb..311eb4f12 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -220,6 +220,8 @@ std::vector server_model_output_modalities(common_decision_type dec case COMMON_DECISION_TYPE_LFM2_D1_OMNI: case COMMON_DECISION_TYPE_H2O_LIGHTNING: case COMMON_DECISION_TYPE_MINICPM_OCR_JEV: + case COMMON_DECISION_TYPE_LFM3_D1: + case COMMON_DECISION_TYPE_LFM3_D1_OMNI: out = {"decisions"}; break; case COMMON_DECISION_TYPE_GRANITE_SWITCH: diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 3f8645706..3ca27762c 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -529,6 +529,12 @@ struct server_slot { return false; } + // lfm3 d1 has no memory, its prompt is read as one flat ubatch and the short convs would mix the sequences + // TODO: batch them along the sequence dimension, this needs ubatches of sequences with unequal lengths + if (task->decision.no_batching || other_slot.task->decision.no_batching) { + return false; + } + return task->type == other_slot.task->type && inp_embd.size() == other_slot.inp_embd.size() && are_lora_equal(lora, other_slot.lora); @@ -3955,7 +3961,8 @@ private: } // TODO: support memory-less logits computation - if (slot.task->need_logits() && !llama_get_memory(ctx_tgt)) { + // a decision reads its logits from the one batch of its prompt + if (slot.task->need_logits() && !llama_get_memory(ctx_tgt) && slot.task->type != SERVER_TASK_TYPE_DECISION) { send_error(slot, "the current context does not support logits computation. skipping", ERROR_TYPE_SERVER); slot.release(); return; diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp index 06234cd17..ba70c7b01 100644 --- a/tools/server/server-decision.cpp +++ b/tools/server/server-decision.cpp @@ -162,9 +162,13 @@ void server_decision_context::init(const llama_model * model) { noul_criteria_order = true; strict_payload = true; canonical_legend = true; - } else if (model_type == COMMON_DECISION_TYPE_LFM2_D1) { + } else if (model_type == COMMON_DECISION_TYPE_LFM2_D1 || model_type == COMMON_DECISION_TYPE_LFM3_D1 || model_type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) { n_options_max = 255; noul_true_first = true; + if (model_type != COMMON_DECISION_TYPE_LFM2_D1) { + // the texts of the special tokens are broken in the input + init_quoted_spellings(LLAMA_TOKEN_ATTR_CONTROL, 2); + } } else if (model_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) { token_marker = llama_vocab_mask(vocab); if (token_marker == LLAMA_TOKEN_NULL) { @@ -186,22 +190,7 @@ void server_decision_context::init(const llama_model * model) { throw std::runtime_error("decision model has no labels"); } // the texts of the control and user-defined tokens are quoted in the input, so that they stay text - for (llama_token id = 0; id < llama_vocab_n_tokens(vocab); id++) { - if (llama_vocab_get_attr(vocab, id) & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) { - const std::string text = llama_vocab_get_text(vocab, id); - if (!text.empty()) { - quoted_spellings.push_back(text); - } - } - } - std::stable_sort(quoted_spellings.begin(), quoted_spellings.end(), [](const std::string & a, const std::string & b) { - return a.size() > b.size(); - }); - for (const auto & spelling : quoted_spellings) { - if (quoted_first.find(spelling[0]) == std::string::npos) { - quoted_first += spelling[0]; - } - } + init_quoted_spellings(LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED, 1); n_options_max = labels.size(); noul_true_first = true; } else if (model_type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV) { @@ -221,17 +210,42 @@ void server_decision_context::init(const llama_model * model) { SRV_INF("decision model type: %s\n", type_name.c_str()); } +void server_decision_context::init_quoted_spellings(uint32_t attrs, size_t min_size) { + for (llama_token id = 0; id < llama_vocab_n_tokens(vocab); id++) { + if (llama_vocab_get_attr(vocab, id) & attrs) { + const std::string text = llama_vocab_get_text(vocab, id); + if (text.size() >= min_size) { + quoted_spellings.push_back(text); + } + } + } + std::stable_sort(quoted_spellings.begin(), quoted_spellings.end(), [](const std::string & a, const std::string & b) { + return a.size() > b.size(); + }); + for (const auto & spelling : quoted_spellings) { + if (quoted_first.find(spelling[0]) == std::string::npos) { + quoted_first += spelling[0]; + } + } +} + // // request parsing // +// lfm3-d1-omni: a spoken question, its instructions are an input_audio part +static bool decision_is_spoken(const json & instructions) { + return instructions.is_object() && json_value(instructions, "type", std::string()) == "input_audio"; +} + static bool decision_is_payload(const json & val) { return val.is_string() || val.is_object() || val.is_array(); } std::vector server_decision_context::parse_questions(const json & body) const { - // lfm2-d1, lfm2-d1-omni and vllm-d3 accept a null state, for example to ask about images only - if (!body.contains("state") || (body.at("state").is_null() && type != COMMON_DECISION_TYPE_LFM2_D1 && type != COMMON_DECISION_TYPE_LFM2_D1_OMNI && type != COMMON_DECISION_TYPE_VLLM_D3)) { + // lfm2-d1, lfm2-d1-omni, lfm3-d1 and vllm-d3 accept a null state, for example to ask about images only + const bool null_state = type == COMMON_DECISION_TYPE_LFM2_D1 || type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || type == COMMON_DECISION_TYPE_VLLM_D3 || type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI; + if (!body.contains("state") || (body.at("state").is_null() && !null_state)) { throw std::invalid_argument("\"state\" must be provided"); } if (strict_payload && !decision_is_payload(body.at("state"))) { @@ -264,6 +278,9 @@ std::vector server_decision_context::parse_questions(c if (strict_payload && (!decision_is_payload(instructions) || instructions == "")) { throw err("\"instructions\" must be non-empty text, an object or an array"); } + if (type == COMMON_DECISION_TYPE_LFM3_D1 && decision_is_spoken(instructions)) { + throw err("spoken instructions need a model with audio input"); + } server_decision_question question; question.id = id; @@ -581,31 +598,65 @@ static std::string decision_kev_text(const json & val) { return server_decision_escape_special_tokens(decision_kev_render(val)); } -// lfm2-d1-omni: special tokens written in the input must not be parsed as such, in keys too (d1-omni prompt.py: escape) -static json decision_d1omni_escape(const json & val, int depth = 0) { +// lfm2-d1-omni, lfm3-d1: special tokens written in the input must not be parsed as such, in keys too +template +static json decision_escape(const json & val, const F & escape, int depth = 0) { if (depth >= RECURSIVE_LIMIT) { throw std::invalid_argument("JSON is nested more than " + std::to_string(RECURSIVE_LIMIT) + " levels deep"); } if (val.is_string()) { - return server_decision_escape_special_tokens(val.get()); + return escape(val.get()); } if (val.is_array()) { json out = json::array(); for (const auto & item : val) { - out.push_back(decision_d1omni_escape(item, depth + 1)); + out.push_back(decision_escape(item, escape, depth + 1)); } return out; } if (val.is_object()) { json out = json::object(); for (const auto & [key, item] : val.items()) { - out[server_decision_escape_special_tokens(key)] = decision_d1omni_escape(item, depth + 1); + out[escape(key)] = decision_escape(item, escape, depth + 1); } return out; } return val; } +std::string server_decision_context::d1_break(const std::string & text) const { + static const std::string zwsp = "\xE2\x80\x8B"; + std::string cur = text; + // again for a match that overlapped the one broken before it + for (bool changed = true; changed;) { + changed = false; + std::string out; + for (size_t i = 0; i < cur.size();) { + const std::string * hit = nullptr; + if (quoted_first.find(cur[i]) != std::string::npos) { + for (const auto & spelling : quoted_spellings) { + if (cur.compare(i, spelling.size(), spelling) == 0) { + hit = &spelling; + break; + } + } + } + if (hit == nullptr) { + out += cur[i++]; + continue; + } + // after the first character, its UTF-8 bytes + const unsigned char c = (*hit)[0]; + const size_t n = c < 0x80 ? 1 : c < 0xE0 ? 2 : c < 0xF0 ? 3 : 4; + out += hit->substr(0, n) + zwsp + hit->substr(n); + i += hit->size(); + changed = true; + } + cur = std::move(out); + } + return cur; +} + // // h2o-lightning: the texts are rendered the way the reference shim renders them with Python (json.dumps, str.lstrip) // @@ -955,6 +1006,9 @@ void server_decision_context::d1_labels(const server_decision_question & questio codes.push_back(question.options[i].key); } else if (n_options <= 26) { codes.push_back(std::string(1, 'A' + i)); + } else if (type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) { + // A..Z, then AA, AB.. + codes.push_back(i < 26 ? std::string(1, 'A' + i) : std::string{(char) ('A' + (i - 26) / 26), (char) ('A' + (i - 26) % 26)}); } else { codes.push_back(string_format("%02zu", i)); } @@ -1012,7 +1066,7 @@ json server_decision_context::render_options(const server_decision_question & qu std::vector d1_texts; std::vector d1_groups; - if (type == COMMON_DECISION_TYPE_LFM2_D1) { + if (type == COMMON_DECISION_TYPE_LFM2_D1 || type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) { d1_labels(question, d1_texts, d1_groups); } @@ -1092,8 +1146,17 @@ std::string server_decision_context::render( inp = decision_replace_text(inp, text_marker, " "); } + // lfm3-d1-omni: a spoken question is heard where its text would be, after the media of the state + const bool is_spoken = decision_is_spoken(question.instructions); + if (is_spoken) { + inp["instructions"] = ""; + } + if (is_lfm3_d1()) { + inp = decision_escape(inp, [this](const std::string & text) { return d1_break(text); }); + } + if (type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) { - inp = decision_replace_text(decision_d1omni_escape(inp), D1OMNI_MARKER, "< 0) { + if (n_images > 0 || is_spoken) { inp = decision_replace_text(inp, get_media_marker(), " "); for (size_t i = 0; i < n_images; i++) { images.push_back(get_media_marker()); } } inp["images"] = images; + if (is_spoken) { + inp["instructions"] = get_media_marker(); + } jinja::context ctx(tmpl->source()); jinja::global_from_json(ctx, inp, false); @@ -1145,12 +1211,18 @@ void server_decision_context::fill_task( return; } + std::vector media = files; + if (decision_is_spoken(question.instructions)) { + const json audio = json_value(question.instructions, "input_audio", json::object()); + decision_load_audio(audio.contains("data") ? audio.at("data") : json(), media); + } + if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_GRANITE_SWITCH || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3 || type == COMMON_DECISION_TYPE_H2O_LIGHTNING) { // lev reads the ratings of a noul question at its first labels, not at the digits task.decision.labels.assign(labels.begin(), labels.begin() + n_outputs(question)); } - if (type == COMMON_DECISION_TYPE_LFM2_D1) { + if (type == COMMON_DECISION_TYPE_LFM2_D1 || type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) { std::vector texts; std::vector groups; d1_labels(question, texts, groups); @@ -1158,11 +1230,13 @@ void server_decision_context::fill_task( task.decision.labels.insert(task.decision.labels.end(), group.begin(), group.end()); task.decision.label_groups.push_back(group.size()); } + // the short convs of lfm3 d1 have no memory to separate the sequences of a batch + task.decision.no_batching = is_lfm3_d1(); } if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3 || type == COMMON_DECISION_TYPE_LFM2_D1 || - type == COMMON_DECISION_TYPE_H2O_LIGHTNING) { - if (!files.empty()) { - task.tokens = process_mtmd_prompt(mctx, prompt, files, init_opt); + type == COMMON_DECISION_TYPE_H2O_LIGHTNING || type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) { + if (!media.empty()) { + task.tokens = process_mtmd_prompt(mctx, prompt, media, init_opt); return; } } @@ -1556,6 +1630,10 @@ double server_decision_context::get_decision_confidence(server_decision_question if (type == COMMON_DECISION_TYPE_VLLM_DECISION2 || type == COMMON_DECISION_TYPE_VLLM_D3) { return decision_confidence_entropy(probs); } + // lfm3-d1: the probability of the answer (d1 api.py) + if (is_lfm3_d1()) { + return *std::max_element(probs.begin(), probs.end()); + } switch (question_type) { case SERVER_DECISION_QUESTION_CHOICE: return decision_confidence_choice(probs); case SERVER_DECISION_QUESTION_SCORE: return decision_confidence_score(probs); @@ -1644,6 +1722,16 @@ json server_decision_context::format_answer(const server_decision_question & que const json & description = question.options[i].description; legend[question.options[i].key] = canonical_legend && !description.is_string() ? json(decision_sort_keys(description).dump()) : description; } + // lfm3-d1: the median level, the first whose cumulative probability reaches one half (d1 api.py) + if (is_lfm3_d1()) { + double mass = 0.0; + size_t level = n - 1; + for (size_t i = 0; i < n && mass < 0.5; i++) { + mass += probs[i]; + level = i; + } + expected = (double) level; + } answer["score"] = expected; answer["legend"] = legend; answer["probabilities"] = probabilities; diff --git a/tools/server/server-decision.h b/tools/server/server-decision.h index 45933ffa1..82af98ca1 100644 --- a/tools/server/server-decision.h +++ b/tools/server/server-decision.h @@ -59,6 +59,10 @@ struct server_decision_context { return type == COMMON_DECISION_TYPE_CLEF; } + bool is_lfm3_d1() const { + return type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI; + } + // true if the prompt of the model has a place for images bool can_use_images() const { switch (type) { @@ -70,6 +74,8 @@ struct server_decision_context { case COMMON_DECISION_TYPE_LFM2_D1_OMNI: case COMMON_DECISION_TYPE_H2O_LIGHTNING: case COMMON_DECISION_TYPE_MINICPM_OCR_JEV: + case COMMON_DECISION_TYPE_LFM3_D1: + case COMMON_DECISION_TYPE_LFM3_D1_OMNI: return true; default: return false; @@ -164,6 +170,10 @@ private: json render_h2o(const json & state, const server_decision_question & question) const; std::string h2o_cut_state(const std::string & text) const; std::string h2o_quote(const std::string & text) const; + // the texts of the tokens with these attributes, quoted_spellings and quoted_first + void init_quoted_spellings(uint32_t attrs, size_t min_size); + // LFM3_D1: a zero-width space after the first character of each special token text (d1 prompt.py: plain) + std::string d1_break(const std::string & text) const; void fill_task_laya(llama_tokens & tokens, const server_decision_question & question, server_task & task) const; void fill_task_d1omni( const json & state, diff --git a/tools/server/server-task.h b/tools/server/server-task.h index daefeb457..ab128f3bd 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -196,6 +196,8 @@ struct server_task { int32_t column = 0; // if set, embeddings is [q | k], and the output is instead the scaled dot product of q[pointer] and k[marker] int32_t pointer = -1; + // if set, the prompt must be the only sequence of its batch + bool no_batching = false; // if set, the outputs are the pooled embeddings of the sequence, one score per class bool pooled = false;