model : add LiquidAI/d1-2B and LiquidAI/d1-omni-2B (#31275)

* model : add LiquidAI/d1-2B and LiquidAI/d1-omni-2B

Assisted-by: Claude Opus 5.5

* convert : keep the speech task templates out of the d1 GGUFs

Assisted-by: Claude Opus 5.5

* small refactoring: no_batching

---------

Co-authored-by: Xuan Son Nguyen <son@huggingface.co>

common/common.cpp

@@ -1188,6 +1188,8 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
     { COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"},
     { COMMON_DECISION_TYPE_H2O_LIGHTNING,  "h2o-lightning" },
     { COMMON_DECISION_TYPE_MINICPM_OCR_JEV, "openbmb-minicpm-ocr-jev" },
+    { COMMON_DECISION_TYPE_LFM3_D1,        "lfm3-d1"       },
+    { COMMON_DECISION_TYPE_LFM3_D1_OMNI,   "lfm3-d1-omni"  },
     // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
 };
 

common/common.h

@@ -981,6 +981,8 @@ enum common_decision_type {
     COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch
     COMMON_DECISION_TYPE_H2O_LIGHTNING, // same as openjev, the labels are read after an answer prefill
     COMMON_DECISION_TYPE_MINICPM_OCR_JEV, // one score per class, the pooled embeddings output of the last token
+    COMMON_DECISION_TYPE_LFM3_D1,       // same as lfm2-d1, non-causal, label codes A..Z then AA, AB..
+    COMMON_DECISION_TYPE_LFM3_D1_OMNI,  // same as lfm3-d1
     COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
 };
 

conversion/__init__.py

@@ -172,6 +172,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "D1OmniModel": "lfm2",
     "Lfm2Model": "lfm2",
     "Lfm2MoeForCausalLM": "lfm2",
+    "Lfm3D1Model": "lfm3",
     "Lfm3ForCausalLM": "lfm3",
     "Lfm3OmniForConditionalGeneration": "lfm3",
     "Lfm3VlForConditionalGeneration": "lfm3",

conversion/lfm2.py

@@ -77,7 +77,9 @@ def _is_d1_checkpoint(dir_model: Path) -> bool:
     if not (dir_model / "config.json").is_file():
         return False
     with open(dir_model / "config.json", encoding="utf-8") as f:
-        return json.load(f).get("auto_map", {}).get("AutoModel", "").endswith(".D1Model")
+        config = json.load(f)
+    # d1 on LFM3 is converted by conversion/lfm3.py
+    return config.get("auto_map", {}).get("AutoModel", "").endswith(".D1Model") and not config.get("model_type", "").startswith("lfm3")
 
 
 @ModelBase.register_hparams_loader(_is_d1_checkpoint)

conversion/lfm3.py

@@ -11,8 +11,8 @@ import torch
 if TYPE_CHECKING:
     from torch import Tensor
 
-from .base import MmprojModel, ModelBase, TextModel, gguf
-from .lfm2 import lfm2_feed_forward_length
+from .base import MmprojModel, ModelBase, TextModel, gguf, logger
+from .lfm2 import D1Model, lfm2_feed_forward_length
 
 
 @ModelBase.register("Lfm3ForCausalLM", "Lfm3VlForConditionalGeneration", "Lfm3OmniForConditionalGeneration")
@@ -119,6 +119,63 @@ class LFM3Model(TextModel):
         yield from super().modify_tensors(data_torch, name, bid)
 
 
+# d1 on LFM3-VL / LFM3-Omni: the stock weights, the d1 prompt read at its last token
+_D1_TYPES = {"lfm3_vl": gguf.DecisionType.LFM3_D1, "lfm3_omni": gguf.DecisionType.LFM3_D1_OMNI}
+
+
+def _is_lfm3_d1_checkpoint(dir_model: Path) -> bool:
+    if not (dir_model / "config.json").is_file():
+        return False
+    with open(dir_model / "config.json", encoding="utf-8") as f:
+        config = json.load(f)
+    return config.get("architectures") == ["D1Model"] and config.get("model_type") in _D1_TYPES
+
+
+@ModelBase.register_hparams_loader(_is_lfm3_d1_checkpoint)
+def _load_lfm3_d1_hparams(dir_model: Path) -> dict[str, Any]:
+    logger.info("gguf: detected LFM3 d1 checkpoint")
+    hparams = ModelBase.load_hparams(dir_model, False, guess=False)
+    hparams["d1_family"] = hparams["model_type"]
+    # the mmproj is the base model's, the text model writes the decision
+    hparams["architectures"] = ["Lfm3VlForConditionalGeneration" if hparams["model_type"] == "lfm3_vl" else "Lfm3OmniForConditionalGeneration"]
+    hparams["text_config"]["architectures"] = ["Lfm3D1Model"]
+    return hparams
+
+
+@ModelBase.register("Lfm3D1Model")
+@ModelBase.example("LiquidAI/d1-2B", "LiquidAI/d1-omni-2B")
+class LFM3D1Model(LFM3Model):
+    model_arch = gguf.MODEL_ARCH.LFM3
+
+    # how the checkpoint serves, hard-coded in the server: assert the checkpoint records the same
+    _served = {"d1_codes": 2, "d1_attention": "set"}
+
+    def set_vocab(self):
+        # the vocab without the speech task templates of LFM3-Audio/Omni, d1 only decides
+        TextModel.set_vocab(self)
+        # the d1 prompt of LFM2, the LFM3 template writes an empty thought before the answer
+        self.gguf_writer.add_chat_template([{"name": "systemone", "template": D1Model._systemone_template() + "<think></think>"}])
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+
+        for key, value in self._served.items():
+            if self.hparams.get(key) != value:
+                raise ValueError(f"unsupported d1 checkpoint: {key} = {self.hparams.get(key)!r}, expected {value!r}")
+
+        # a lone question reads the whole prompt both ways on the full attention layers
+        self.gguf_writer.add_causal_attention(False)
+        self.gguf_writer.add_decision_type(_D1_TYPES[self.hparams["d1_family"]])
+
+        # the option answers share one temperature
+        with open(self.dir_model / "calibration.json", encoding="utf-8") as f:
+            calibration = json.load(f)
+        if calibration.get("temperatures"):
+            raise ValueError("unsupported d1 calibration: per-bucket temperatures")
+        for name in ("noul", "choice", "score"):
+            self.gguf_writer.add_decision_temperature(name, calibration["default"])
+
+
 class LFM3VLVisionMixin(MmprojModel):
     """LFM3-VL vision tower and projector, shared with LFM3-Omni."""
 

gguf-py/gguf/constants.py

@@ -6544,6 +6544,8 @@ class DecisionType:
     VLLM_D3 = "vllm-d3"  # same as pplx-decider, non-causal attention
     LFM2_D1 = "lfm2-d1"  # same as openjev, the labels depend on the question type
     LFM2_D1_OMNI = "lfm2-d1-omni"  # same head as laya on a bidirectional LFM2 trunk, other prompt layout
+    LFM3_D1 = "lfm3-d1"  # same as lfm2-d1 on LFM3-VL, full attention layers read the prompt both ways
+    LFM3_D1_OMNI = "lfm3-d1-omni"  # same as lfm3-d1 on LFM3-Omni
     GRANITE_SWITCH = "granite-switch"  # same as openjev, run by the decision aLoRA of granite switch
     H2O_LIGHTNING = "h2o-lightning"  # same as openjev, labels read after an answer prefill
     MINICPM_OCR_JEV = "openbmb-minicpm-ocr-jev"  # class scores of a value head read at the last token, one fixed task

src/llama-context.cpp

@@ -1614,8 +1614,9 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
     // eagle3/DFlash: features as encoder input, and non-draft paths fall back to model's input dim
     const int64_t n_vocab = model.vocab.n_tokens();
 
-    // note: during encode, we always output all tokens and skip position continuity checks (output_all=true)
-    if (!balloc->init(batch_inp, model.vocab, true)) {
+    // note: during encode, we skip position continuity checks and output all tokens
+    //       a decoder without memory (see decode()) outputs the marked tokens, unless it extracts embeddings
+    if (!balloc->init(batch_inp, model.vocab, memory || cparams.embeddings)) {
         LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__);
         return -1;
     }
@@ -1644,17 +1645,20 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
 
     n_queued_tokens += n_tokens;
 
+    const uint32_t n_outputs_all = balloc->get_n_outputs();
+
     // reserve output buffer
-    if (output_reserve(n_tokens) < n_tokens) {
-        LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_tokens);
+    if (output_reserve(n_outputs_all) < n_outputs_all) {
+        LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_outputs_all);
         return -2;
     };
 
-    for (uint32_t i = 0; i < n_tokens; ++i) {
-        output_ids[i] = i;
+    const auto & out_ids = balloc->get_out_ids();
+    for (uint32_t i = 0; i < n_outputs_all; ++i) {
+        output_ids[out_ids[i]] = i;
     }
 
-    n_outputs = n_tokens;
+    n_outputs = n_outputs_all;
 
     const auto causal_attn_org = cparams.causal_attn;
 
@@ -1687,7 +1691,7 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
         GGML_ASSERT(backend_res != nullptr);
         GGML_ASSERT(logits.data != nullptr);
 
-        ggml_backend_tensor_get_async(backend_res, t_logits, logits.data, 0, n_tokens*n_vocab*sizeof(float));
+        ggml_backend_tensor_get_async(backend_res, t_logits, logits.data, 0, n_outputs*n_vocab*sizeof(float));
     }
 
     // extract embeddings
@@ -1702,8 +1706,8 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
                     GGML_ASSERT(embd.data != nullptr);
                     const uint32_t n_embd_out = hparams.n_embd_out();
 
-                    GGML_ASSERT(n_tokens*n_embd_out <= (int64_t) embd.size);
-                    ggml_backend_tensor_get_async(backend_embd, t_embd, embd.data, 0, n_tokens*n_embd_out*sizeof(float));
+                    GGML_ASSERT(n_outputs*n_embd_out <= (int64_t) embd.size);
+                    ggml_backend_tensor_get_async(backend_embd, t_embd, embd.data, 0, n_outputs*n_embd_out*sizeof(float));
                 } break;
             case LLAMA_POOLING_TYPE_MEAN:
             case LLAMA_POOLING_TYPE_CLS:
@@ -1751,8 +1755,8 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
         GGML_ASSERT(backend_h != nullptr);
 
         const uint32_t n_embd = hparams.n_embd_out();
-        GGML_ASSERT(n_tokens*n_embd <= (int64_t) embd_nextn.size);
-        ggml_backend_tensor_get_async(backend_h, t_h_nextn, embd_nextn.data, 0, n_tokens*n_embd*sizeof(float));
+        GGML_ASSERT(n_outputs*n_embd <= (int64_t) embd_nextn.size);
+        ggml_backend_tensor_get_async(backend_h, t_h_nextn, embd_nextn.data, 0, n_outputs*n_embd*sizeof(float));
     }
 
     // TODO: hacky solution

src/llama-graph.cpp

@@ -479,7 +479,7 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
     const int64_t n_kv     = self_kq_mask->ne[0];
     const int64_t n_tokens = ubatch->n_tokens;
 
-    const auto fill_mask = [&](auto * data, int64_t ne, int n_swa, llama_swa_type swa_type) {
+    const auto fill_mask = [&](auto * data, int64_t ne, int n_swa, llama_swa_type swa_type, bool causal) {
         using T = std::remove_reference_t<decltype(*data)>;
         std::fill(data, data + ne, llama_cast<T>(-INFINITY));
 
@@ -499,7 +499,7 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
                 }
 
                 // mask future tokens
-                if (cparams.causal_attn && p0 > p1) {
+                if (causal && p0 > p1) {
                     continue;
                 }
 
@@ -517,21 +517,25 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
         }
     };
 
+    // see llama_non_causal_type
+    const bool causal     = cparams.causal_attn;
+    const bool causal_swa = cparams.causal_attn || hparams.non_causal_type == LLAMA_NON_CAUSAL_TYPE_DENSE_ONLY;
+
     GGML_ASSERT(self_kq_mask);
     GGML_ASSERT(ggml_backend_buffer_is_host(self_kq_mask->buffer));
     if (self_kq_mask->type == GGML_TYPE_F16) {
-        fill_mask((ggml_fp16_t *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE);
+        fill_mask((ggml_fp16_t *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE, causal);
     } else {
-        fill_mask((float       *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE);
+        fill_mask((float       *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE, causal);
     }
 
     if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
         GGML_ASSERT(self_kq_mask_swa);
         GGML_ASSERT(ggml_backend_buffer_is_host(self_kq_mask_swa->buffer));
         if (self_kq_mask_swa->type == GGML_TYPE_F16) {
-            fill_mask((ggml_fp16_t *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type);
+            fill_mask((ggml_fp16_t *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type, causal_swa);
         } else {
-            fill_mask((float       *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type);
+            fill_mask((float       *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type, causal_swa);
         }
     }
 }

src/llama-hparams.h

@@ -34,6 +34,7 @@ enum llama_non_causal_type {
     LLAMA_NON_CAUSAL_TYPE_ALL      = 0, // all layers non-causal, SWA still applied (gemma 3, qwen-vl, ...)
     LLAMA_NON_CAUSAL_TYPE_SWA_ONLY = 1, // SWA layers non-causal, dense layers stay causal (gemma 4)
     LLAMA_NON_CAUSAL_TYPE_SWA_FULL = 2, // all layers non-causal, SWA not applied between tokens of the current ubatch (deepseek 4)
+    LLAMA_NON_CAUSAL_TYPE_DENSE_ONLY = 3, // dense layers non-causal, SWA layers stay causal (lfm3 d1)
 };
 
 // M-RoPE: which input position slot feeds each RoPE section

src/llama-kv-cache.cpp

@@ -1749,6 +1749,9 @@ void llama_kv_cache::set_input_kq_mask(ggml_tensor * dst, const llama_ubatch * u
     if (!causal_attn && hparams.non_causal_type == LLAMA_NON_CAUSAL_TYPE_SWA_ONLY) {
         causal_attn = swa_type == LLAMA_SWA_TYPE_NONE;
     }
+    if (!causal_attn && hparams.non_causal_type == LLAMA_NON_CAUSAL_TYPE_DENSE_ONLY) {
+        causal_attn = swa_type != LLAMA_SWA_TYPE_NONE;
+    }
 
     //const int64_t t_start = ggml_time_us();
 

src/llama-model.cpp

@@ -2801,10 +2801,13 @@ ggml_tensor * llama_model::get_rope_factors(const llama_cparams & cparams, int i
 llama_memory_i * llama_model::create_memory(const llama_memory_params & params, const llama_cparams & cparams) const {
     llama_memory_i * res;
 
-    // the non-causal LFM2 decision graph reads the whole prompt in one batch, nothing is kept
+    // the non-causal LFM2 and LFM3 decision graphs read the whole prompt in one batch, nothing is kept
     if (arch == LLM_ARCH_LFM2 && !hparams.causal_attn && hparams.n_layer_decision > 0) {
         return nullptr;
     }
+    if (arch == LLM_ARCH_LFM3 && !hparams.causal_attn) {
+        return nullptr;
+    }
 
     switch (arch) {
         // Models that need specific instantiation should be handled in the

src/models/lfm-base.cpp

@@ -84,6 +84,33 @@ ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_model & model,
     return y;
 }
 
+ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_model & model,
+                                                      ggml_tensor * cur,
+                                                      int il) {
+    const int64_t d_conv = hparams.n_shortconv_l_cache - 1;
+
+    auto * bcx = build_lora_mm(model.layers[il].shortconv.in_proj, cur);
+    cb(bcx, "model.layers.{}.conv.in_proj", il);
+
+    const int64_t chunk_size = bcx->ne[0] / 3;
+    auto * b = ggml_view_2d(ctx0, bcx, chunk_size, bcx->ne[1], bcx->nb[1], 0 * chunk_size * ggml_element_size(bcx));
+    auto * c = ggml_view_2d(ctx0, bcx, chunk_size, bcx->ne[1], bcx->nb[1], 1 * chunk_size * ggml_element_size(bcx));
+    auto * x = ggml_view_2d(ctx0, bcx, chunk_size, bcx->ne[1], bcx->nb[1], 2 * chunk_size * ggml_element_size(bcx));
+
+    // causal from a zero state
+    auto * bx = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_mul(ctx0, b, x)));
+    bx = ggml_pad_ext(ctx0, bx, d_conv, 0, 0, 0, 0, 0, 0, 0);
+
+    auto * conv_out = ggml_ssm_conv(ctx0, bx, model.layers[il].shortconv.conv);
+    cb(conv_out, "model.layers.{}.conv.conv", il);
+
+    auto * y = ggml_mul(ctx0, c, ggml_reshape_2d(ctx0, conv_out, chunk_size, n_tokens));
+    y = build_lora_mm(model.layers[il].shortconv.out_proj, y);
+    cb(y, "model.layers.{}.conv.out_proj", il);
+
+    return y;
+}
+
 ggml_tensor * llm_build_lfm_base::build_lfm_dense_ffn(const llama_model & model,
                                                       ggml_tensor * cur,
                                                       int il) {

src/models/lfm3.cpp

@@ -19,6 +19,11 @@ void llama_model_lfm3::load_arch_hparams(llama_model_loader & ml) {
 
     ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa);
 
+    // d1: full attention layers read the prompt both ways, sliding window layers stay causal
+    if (!hparams.causal_attn) {
+        hparams.non_causal_type = LLAMA_NON_CAUSAL_TYPE_DENSE_ONLY;
+    }
+
     switch (hparams.n_ff()) {
         case 7680: type = LLM_TYPE_1_7B; break;
         default:   type = LLM_TYPE_UNKNOWN;
@@ -84,15 +89,19 @@ void llama_model_lfm3::load_arch_tensors(llama_model_loader &) {
 }
 
 std::unique_ptr<llm_graph_context> llama_model_lfm3::build_arch_graph(const llm_graph_params & params) const {
-    return std::make_unique<graph>(*this, params);
+    if (!hparams.causal_attn) {
+        return std::make_unique<graph<true>>(*this, params);
+    }
+    return std::make_unique<graph<false>>(*this, params);
 }
 
-llama_model_lfm3::graph::graph(const llama_model & model, const llm_graph_params & params) :
+template <bool decision>
+llama_model_lfm3::graph<decision>::graph(const llama_model & model, const llm_graph_params & params) :
     llm_build_lfm_base(params) {
-    auto build_attn_block = [&model, this](ggml_tensor *                  cur_in,
-                                           ggml_tensor *                  inp_pos,
-                                           llm_graph_input_attn_kv_iswa * inp_attn,
-                                           int                            il) -> ggml_tensor * {
+    auto build_attn_block = [&model, this](ggml_tensor * cur_in,
+                                           ggml_tensor * inp_pos,
+                                           auto *        inp_attn,
+                                           int           il) -> ggml_tensor * {
         const int64_t n_embd_head = hparams.n_embd_head_k(il);
         const int64_t n_head_il   = hparams.n_head(il);
         const int64_t n_head_kv   = hparams.n_head_kv(il);
@@ -140,7 +149,9 @@ llama_model_lfm3::graph::graph(const llama_model & model, const llm_graph_params
 
     ggml_build_forward_expand(gf, cur);
 
-    auto * inp_hybrid = build_inp_mem_hybrid_iswa();
+    // decision: no memory, the whole prompt is read in one batch
+    auto * inp_hybrid = decision ? nullptr : build_inp_mem_hybrid_iswa();
+    auto * inp_attn   = decision ? build_attn_inp_no_cache() : nullptr;
 
     ggml_tensor * inp_pos     = build_inp_pos();
     ggml_tensor * inp_out_ids = build_inp_out_ids();
@@ -152,8 +163,14 @@ llama_model_lfm3::graph::graph(const llama_model & model, const llm_graph_params
         cb(cur, "model.layers.{}.operator_norm", il);
 
         if (hparams.is_recr(il)) {
-            const auto * mctx_recr = static_cast<const llama_memory_hybrid_iswa_context *>(mctx)->get_recr();
-            cur = build_lfm_shortconv(model, mctx_recr, inp_hybrid->get_recr(), cur, il);
+            if constexpr (decision) {
+                cur = build_lfm_shortconv(model, cur, il);
+            } else {
+                const auto * mctx_recr = static_cast<const llama_memory_hybrid_iswa_context *>(mctx)->get_recr();
+                cur = build_lfm_shortconv(model, mctx_recr, inp_hybrid->get_recr(), cur, il);
+            }
+        } else if constexpr (decision) {
+            cur = build_attn_block(cur, inp_pos, inp_attn, il);
         } else {
             cur = build_attn_block(cur, inp_pos, inp_hybrid->get_attn(), il);
         }
@@ -195,3 +212,6 @@ llama_model_lfm3::graph::graph(const llama_model & model, const llm_graph_params
 
     res->t_logits = cur;
 }
+
+template struct llama_model_lfm3::graph<false>;
+template struct llama_model_lfm3::graph<true>;

src/models/models.h

@@ -111,6 +111,11 @@ struct llm_build_lfm_base : public llm_graph_context {
                                       ggml_tensor * cur,
                                       int il);
 
+    // no recurrent state (decision graphs): causal over the ubatch from a zero state
+    ggml_tensor * build_lfm_shortconv(const llama_model & model,
+                                      ggml_tensor * cur,
+                                      int il);
+
     ggml_tensor * build_lfm_dense_ffn(const llama_model & model,
                                       ggml_tensor * cur,
                                       int il);
@@ -2438,6 +2443,8 @@ struct llama_model_lfm3 : public llama_model_base {
     void load_arch_hparams(llama_model_loader & ml) override;
     void load_arch_tensors(llama_model_loader & ml) override;
 
+    // decision: d1, the prompt read in one batch, no memory
+    template <bool decision>
     struct graph : public llm_build_lfm_base {
         graph(const llama_model & model, const llm_graph_params & params);
     };

tools/server/server-common.cpp

@@ -220,6 +220,8 @@ std::vector<std::string> server_model_output_modalities(common_decision_type dec
         case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
         case COMMON_DECISION_TYPE_H2O_LIGHTNING:
         case COMMON_DECISION_TYPE_MINICPM_OCR_JEV:
+        case COMMON_DECISION_TYPE_LFM3_D1:
+        case COMMON_DECISION_TYPE_LFM3_D1_OMNI:
             out = {"decisions"};
             break;
         case COMMON_DECISION_TYPE_GRANITE_SWITCH:

tools/server/server-context.cpp

@@ -529,6 +529,12 @@ struct server_slot {
             return false;
         }
 
+        // lfm3 d1 has no memory, its prompt is read as one flat ubatch and the short convs would mix the sequences
+        // TODO: batch them along the sequence dimension, this needs ubatches of sequences with unequal lengths
+        if (task->decision.no_batching || other_slot.task->decision.no_batching) {
+            return false;
+        }
+
         return task->type == other_slot.task->type
             && inp_embd.size() == other_slot.inp_embd.size()
             && are_lora_equal(lora, other_slot.lora);
@@ -3955,7 +3961,8 @@ private:
                         }
 
                         // TODO: support memory-less logits computation
-                        if (slot.task->need_logits() && !llama_get_memory(ctx_tgt)) {
+                        // a decision reads its logits from the one batch of its prompt
+                        if (slot.task->need_logits() && !llama_get_memory(ctx_tgt) && slot.task->type != SERVER_TASK_TYPE_DECISION) {
                             send_error(slot, "the current context does not support logits computation. skipping", ERROR_TYPE_SERVER);
                             slot.release();
                             return;

tools/server/server-decision.cpp

@@ -162,9 +162,13 @@ void server_decision_context::init(const llama_model * model) {
         noul_criteria_order = true;
         strict_payload      = true;
         canonical_legend    = true;
-    } else if (model_type == COMMON_DECISION_TYPE_LFM2_D1) {
+    } else if (model_type == COMMON_DECISION_TYPE_LFM2_D1 || model_type == COMMON_DECISION_TYPE_LFM3_D1 || model_type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) {
         n_options_max   = 255;
         noul_true_first = true;
+        if (model_type != COMMON_DECISION_TYPE_LFM2_D1) {
+            // the texts of the special tokens are broken in the input
+            init_quoted_spellings(LLAMA_TOKEN_ATTR_CONTROL, 2);
+        }
     } else if (model_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {
         token_marker = llama_vocab_mask(vocab);
         if (token_marker == LLAMA_TOKEN_NULL) {
@@ -186,22 +190,7 @@ void server_decision_context::init(const llama_model * model) {
             throw std::runtime_error("decision model has no labels");
         }
         // the texts of the control and user-defined tokens are quoted in the input, so that they stay text
-        for (llama_token id = 0; id < llama_vocab_n_tokens(vocab); id++) {
-            if (llama_vocab_get_attr(vocab, id) & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) {
-                const std::string text = llama_vocab_get_text(vocab, id);
-                if (!text.empty()) {
-                    quoted_spellings.push_back(text);
-                }
-            }
-        }
-        std::stable_sort(quoted_spellings.begin(), quoted_spellings.end(), [](const std::string & a, const std::string & b) {
-            return a.size() > b.size();
-        });
-        for (const auto & spelling : quoted_spellings) {
-            if (quoted_first.find(spelling[0]) == std::string::npos) {
-                quoted_first += spelling[0];
-            }
-        }
+        init_quoted_spellings(LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED, 1);
         n_options_max   = labels.size();
         noul_true_first = true;
     } else if (model_type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV) {
@@ -221,17 +210,42 @@ void server_decision_context::init(const llama_model * model) {
     SRV_INF("decision model type: %s\n", type_name.c_str());
 }
 
+void server_decision_context::init_quoted_spellings(uint32_t attrs, size_t min_size) {
+    for (llama_token id = 0; id < llama_vocab_n_tokens(vocab); id++) {
+        if (llama_vocab_get_attr(vocab, id) & attrs) {
+            const std::string text = llama_vocab_get_text(vocab, id);
+            if (text.size() >= min_size) {
+                quoted_spellings.push_back(text);
+            }
+        }
+    }
+    std::stable_sort(quoted_spellings.begin(), quoted_spellings.end(), [](const std::string & a, const std::string & b) {
+        return a.size() > b.size();
+    });
+    for (const auto & spelling : quoted_spellings) {
+        if (quoted_first.find(spelling[0]) == std::string::npos) {
+            quoted_first += spelling[0];
+        }
+    }
+}
+
 //
 // request parsing
 //
 
+// lfm3-d1-omni: a spoken question, its instructions are an input_audio part
+static bool decision_is_spoken(const json & instructions) {
+    return instructions.is_object() && json_value(instructions, "type", std::string()) == "input_audio";
+}
+
 static bool decision_is_payload(const json & val) {
     return val.is_string() || val.is_object() || val.is_array();
 }
 
 std::vector<server_decision_question> server_decision_context::parse_questions(const json & body) const {
-    // lfm2-d1, lfm2-d1-omni and vllm-d3 accept a null state, for example to ask about images only
-    if (!body.contains("state") || (body.at("state").is_null() && type != COMMON_DECISION_TYPE_LFM2_D1 && type != COMMON_DECISION_TYPE_LFM2_D1_OMNI && type != COMMON_DECISION_TYPE_VLLM_D3)) {
+    // lfm2-d1, lfm2-d1-omni, lfm3-d1 and vllm-d3 accept a null state, for example to ask about images only
+    const bool null_state = type == COMMON_DECISION_TYPE_LFM2_D1 || type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || type == COMMON_DECISION_TYPE_VLLM_D3 || type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI;
+    if (!body.contains("state") || (body.at("state").is_null() && !null_state)) {
         throw std::invalid_argument("\"state\" must be provided");
     }
     if (strict_payload && !decision_is_payload(body.at("state"))) {
@@ -264,6 +278,9 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
         if (strict_payload && (!decision_is_payload(instructions) || instructions == "")) {
             throw err("\"instructions\" must be non-empty text, an object or an array");
         }
+        if (type == COMMON_DECISION_TYPE_LFM3_D1 && decision_is_spoken(instructions)) {
+            throw err("spoken instructions need a model with audio input");
+        }
 
         server_decision_question question;
         question.id           = id;
@@ -581,31 +598,65 @@ static std::string decision_kev_text(const json & val) {
     return server_decision_escape_special_tokens(decision_kev_render(val));
 }
 
-// lfm2-d1-omni: special tokens written in the input must not be parsed as such, in keys too (d1-omni prompt.py: escape)
-static json decision_d1omni_escape(const json & val, int depth = 0) {
+// lfm2-d1-omni, lfm3-d1: special tokens written in the input must not be parsed as such, in keys too
+template <typename F>
+static json decision_escape(const json & val, const F & escape, int depth = 0) {
     if (depth >= RECURSIVE_LIMIT) {
         throw std::invalid_argument("JSON is nested more than " + std::to_string(RECURSIVE_LIMIT) + " levels deep");
     }
     if (val.is_string()) {
-        return server_decision_escape_special_tokens(val.get<std::string>());
+        return escape(val.get<std::string>());
     }
     if (val.is_array()) {
         json out = json::array();
         for (const auto & item : val) {
-            out.push_back(decision_d1omni_escape(item, depth + 1));
+            out.push_back(decision_escape(item, escape, depth + 1));
         }
         return out;
     }
     if (val.is_object()) {
         json out = json::object();
         for (const auto & [key, item] : val.items()) {
-            out[server_decision_escape_special_tokens(key)] = decision_d1omni_escape(item, depth + 1);
+            out[escape(key)] = decision_escape(item, escape, depth + 1);
         }
         return out;
     }
     return val;
 }
 
+std::string server_decision_context::d1_break(const std::string & text) const {
+    static const std::string zwsp = "\xE2\x80\x8B";
+    std::string cur = text;
+    // again for a match that overlapped the one broken before it
+    for (bool changed = true; changed;) {
+        changed = false;
+        std::string out;
+        for (size_t i = 0; i < cur.size();) {
+            const std::string * hit = nullptr;
+            if (quoted_first.find(cur[i]) != std::string::npos) {
+                for (const auto & spelling : quoted_spellings) {
+                    if (cur.compare(i, spelling.size(), spelling) == 0) {
+                        hit = &spelling;
+                        break;
+                    }
+                }
+            }
+            if (hit == nullptr) {
+                out += cur[i++];
+                continue;
+            }
+            // after the first character, its UTF-8 bytes
+            const unsigned char c = (*hit)[0];
+            const size_t n = c < 0x80 ? 1 : c < 0xE0 ? 2 : c < 0xF0 ? 3 : 4;
+            out += hit->substr(0, n) + zwsp + hit->substr(n);
+            i += hit->size();
+            changed = true;
+        }
+        cur = std::move(out);
+    }
+    return cur;
+}
+
 //
 // h2o-lightning: the texts are rendered the way the reference shim renders them with Python (json.dumps, str.lstrip)
 //
@@ -955,6 +1006,9 @@ void server_decision_context::d1_labels(const server_decision_question & questio
             codes.push_back(question.options[i].key);
         } else if (n_options <= 26) {
             codes.push_back(std::string(1, 'A' + i));
+        } else if (type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) {
+            // A..Z, then AA, AB..
+            codes.push_back(i < 26 ? std::string(1, 'A' + i) : std::string{(char) ('A' + (i - 26) / 26), (char) ('A' + (i - 26) % 26)});
         } else {
             codes.push_back(string_format("%02zu", i));
         }
@@ -1012,7 +1066,7 @@ json server_decision_context::render_options(const server_decision_question & qu
 
     std::vector<std::string>  d1_texts;
     std::vector<llama_tokens> d1_groups;
-    if (type == COMMON_DECISION_TYPE_LFM2_D1) {
+    if (type == COMMON_DECISION_TYPE_LFM2_D1 || type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) {
         d1_labels(question, d1_texts, d1_groups);
     }
 
@@ -1092,8 +1146,17 @@ std::string server_decision_context::render(
         inp = decision_replace_text(inp, text_marker, " ");
     }
 
+    // lfm3-d1-omni: a spoken question is heard where its text would be, after the media of the state
+    const bool is_spoken = decision_is_spoken(question.instructions);
+    if (is_spoken) {
+        inp["instructions"] = "";
+    }
+    if (is_lfm3_d1()) {
+        inp = decision_escape(inp, [this](const std::string & text) { return d1_break(text); });
+    }
+
     if (type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {
-        inp = decision_replace_text(decision_d1omni_escape(inp), D1OMNI_MARKER, "<<d1omni ");
+        inp = decision_replace_text(decision_escape(inp, server_decision_escape_special_tokens), D1OMNI_MARKER, "<<d1omni ");
         inp["audio"]         = is_audio;
         inp["sep"]           = D1OMNI_SEP;
         inp["mark_state"]    = D1OMNI_MARK_STATE;
@@ -1103,13 +1166,16 @@ std::string server_decision_context::render(
 
     // the template puts one media marker per image
     json images = json::array();
-    if (n_images > 0) {
+    if (n_images > 0 || is_spoken) {
         inp = decision_replace_text(inp, get_media_marker(), " ");
         for (size_t i = 0; i < n_images; i++) {
             images.push_back(get_media_marker());
         }
     }
     inp["images"] = images;
+    if (is_spoken) {
+        inp["instructions"] = get_media_marker();
+    }
 
     jinja::context ctx(tmpl->source());
     jinja::global_from_json(ctx, inp, false);
@@ -1145,12 +1211,18 @@ void server_decision_context::fill_task(
         return;
     }
 
+    std::vector<raw_buffer> media = files;
+    if (decision_is_spoken(question.instructions)) {
+        const json audio = json_value(question.instructions, "input_audio", json::object());
+        decision_load_audio(audio.contains("data") ? audio.at("data") : json(), media);
+    }
+
     if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_GRANITE_SWITCH || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3 ||
         type == COMMON_DECISION_TYPE_H2O_LIGHTNING) {
         // lev reads the ratings of a noul question at its first labels, not at the digits
         task.decision.labels.assign(labels.begin(), labels.begin() + n_outputs(question));
     }
-    if (type == COMMON_DECISION_TYPE_LFM2_D1) {
+    if (type == COMMON_DECISION_TYPE_LFM2_D1 || type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) {
         std::vector<std::string>  texts;
         std::vector<llama_tokens> groups;
         d1_labels(question, texts, groups);
@@ -1158,11 +1230,13 @@ void server_decision_context::fill_task(
             task.decision.labels.insert(task.decision.labels.end(), group.begin(), group.end());
             task.decision.label_groups.push_back(group.size());
         }
+        // the short convs of lfm3 d1 have no memory to separate the sequences of a batch
+        task.decision.no_batching = is_lfm3_d1();
     }
     if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3 || type == COMMON_DECISION_TYPE_LFM2_D1 ||
-        type == COMMON_DECISION_TYPE_H2O_LIGHTNING) {
-        if (!files.empty()) {
-            task.tokens = process_mtmd_prompt(mctx, prompt, files, init_opt);
+        type == COMMON_DECISION_TYPE_H2O_LIGHTNING || type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) {
+        if (!media.empty()) {
+            task.tokens = process_mtmd_prompt(mctx, prompt, media, init_opt);
             return;
         }
     }
@@ -1556,6 +1630,10 @@ double server_decision_context::get_decision_confidence(server_decision_question
     if (type == COMMON_DECISION_TYPE_VLLM_DECISION2 || type == COMMON_DECISION_TYPE_VLLM_D3) {
         return decision_confidence_entropy(probs);
     }
+    // lfm3-d1: the probability of the answer (d1 api.py)
+    if (is_lfm3_d1()) {
+        return *std::max_element(probs.begin(), probs.end());
+    }
     switch (question_type) {
         case SERVER_DECISION_QUESTION_CHOICE: return decision_confidence_choice(probs);
         case SERVER_DECISION_QUESTION_SCORE:  return decision_confidence_score(probs);
@@ -1644,6 +1722,16 @@ json server_decision_context::format_answer(const server_decision_question & que
             const json & description = question.options[i].description;
             legend[question.options[i].key] = canonical_legend && !description.is_string() ? json(decision_sort_keys(description).dump()) : description;
         }
+        // lfm3-d1: the median level, the first whose cumulative probability reaches one half (d1 api.py)
+        if (is_lfm3_d1()) {
+            double mass = 0.0;
+            size_t level = n - 1;
+            for (size_t i = 0; i < n && mass < 0.5; i++) {
+                mass += probs[i];
+                level = i;
+            }
+            expected = (double) level;
+        }
         answer["score"]         = expected;
         answer["legend"]        = legend;
         answer["probabilities"] = probabilities;

tools/server/server-decision.h

@@ -59,6 +59,10 @@ struct server_decision_context {
         return type == COMMON_DECISION_TYPE_CLEF;
     }
 
+    bool is_lfm3_d1() const {
+        return type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI;
+    }
+
     // true if the prompt of the model has a place for images
     bool can_use_images() const {
         switch (type) {
@@ -70,6 +74,8 @@ struct server_decision_context {
             case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
             case COMMON_DECISION_TYPE_H2O_LIGHTNING:
             case COMMON_DECISION_TYPE_MINICPM_OCR_JEV:
+            case COMMON_DECISION_TYPE_LFM3_D1:
+            case COMMON_DECISION_TYPE_LFM3_D1_OMNI:
                 return true;
             default:
                 return false;
@@ -164,6 +170,10 @@ private:
     json render_h2o(const json & state, const server_decision_question & question) const;
     std::string h2o_cut_state(const std::string & text) const;
     std::string h2o_quote(const std::string & text) const;
+    // the texts of the tokens with these attributes, quoted_spellings and quoted_first
+    void init_quoted_spellings(uint32_t attrs, size_t min_size);
+    // LFM3_D1: a zero-width space after the first character of each special token text (d1 prompt.py: plain)
+    std::string d1_break(const std::string & text) const;
     void fill_task_laya(llama_tokens & tokens, const server_decision_question & question, server_task & task) const;
     void fill_task_d1omni(
             const json & state,

tools/server/server-task.h

@@ -196,6 +196,8 @@ struct server_task {
         int32_t                  column = 0;
         // if set, embeddings is [q | k], and the output is instead the scaled dot product of q[pointer] and k[marker]
         int32_t                  pointer = -1;
+        // if set, the prompt must be the only sequence of its batch
+        bool                     no_batching = false;
 
         // if set, the outputs are the pooled embeddings of the sequence, one score per class
         bool                     pooled = false;