diff --git a/common/common.cpp b/common/common.cpp index 9a71a33ff..9d2578424 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1170,8 +1170,10 @@ static const std::map COMMON_DECISION_TYPE_NA { COMMON_DECISION_TYPE_LAYA, "laya" }, { COMMON_DECISION_TYPE_CLEF, "clef" }, { COMMON_DECISION_TYPE_PPLX_DECIDER, "pplx-decider" }, + { COMMON_DECISION_TYPE_VLLM_DECISION2, "vllm-decision2" }, { COMMON_DECISION_TYPE_LFM2_D1, "lfm2-d1" }, { COMMON_DECISION_TYPE_LFM2_D1_OMNI, "lfm2-d1-omni" }, + // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname" }; static common_decision_type common_decision_type_from_string(const std::string & str) { @@ -1289,7 +1291,7 @@ common_init_result::common_init_result(common_params & params, bool model_only) // TODO: maybe improve this in the future const auto decision_type = common_get_decision_type(model); if (decision_type == COMMON_DECISION_TYPE_LAYA || decision_type == COMMON_DECISION_TYPE_KEV || decision_type == COMMON_DECISION_TYPE_CLEF || - decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) { + decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || decision_type == COMMON_DECISION_TYPE_VLLM_DECISION2) { params.embedding = true; params.pooling_type = LLAMA_POOLING_TYPE_NONE; diff --git a/common/common.h b/common/common.h index d89f803af..9513a04fa 100644 --- a/common/common.h +++ b/common/common.h @@ -963,6 +963,7 @@ enum common_decision_type { COMMON_DECISION_TYPE_LAYA, // score of one marker token per option, read from the embeddings output COMMON_DECISION_TYPE_CLEF, // all questions in one prompt, score of option i read from the embeddings output at row i COMMON_DECISION_TYPE_PPLX_DECIDER, // same as openjev, label codes of 1 or 2 letters + COMMON_DECISION_TYPE_VLLM_DECISION2, // score of each option read from the embeddings output, the options of a question are marked in the batch COMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question type COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported diff --git a/conversion/__init__.py b/conversion/__init__.py index 2a62dbac9..c2a8fb92c 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -291,6 +291,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "UltravoxModel": "ultravox", "UnlimitedOCRForCausalLM": "deepseek", "VLlama3ForCausalLM": "llama", + "VllmDecision2Model": "vllm_decision2", "VoxtralForConditionalGeneration": "llama", "WavTokenizerDec": "wavtokenizer", "XLMRobertaForSequenceClassification": "bert", diff --git a/conversion/vllm_decision2.py b/conversion/vllm_decision2.py new file mode 100644 index 000000000..3e7085f14 --- /dev/null +++ b/conversion/vllm_decision2.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import json + +from pathlib import Path +from typing import Any, Callable, Iterator, TYPE_CHECKING + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, gguf, logger +from .qwen import Qwen3Model + + +def _is_vllm_decision2_checkpoint(dir_model: Path) -> bool: + if not (dir_model / "config.json").is_file(): + return False + with open(dir_model / "config.json", encoding="utf-8") as f: + return json.load(f).get("decision_format") == "vllm-sr-decision" + + +@ModelBase.register_hparams_loader(_is_vllm_decision2_checkpoint) +def _load_vllm_decision2_hparams(dir_model: Path) -> dict[str, Any]: + logger.info("gguf: detected vllm-sr Decision 2.0 checkpoint") + with open(dir_model / "config.json", encoding="utf-8") as f: + package = json.load(f) + hparams = ModelBase.load_hparams((dir_model / package["backbone"]["config"]).parent, False, guess=False) + hparams["architectures"] = ["VllmDecision2Model"] + with open(dir_model / package["model_config"], encoding="utf-8") as f: + hparams["decision"] = json.load(f) + with open(dir_model / "score_bias.json", encoding="utf-8") as f: + hparams["decision"]["score_bias"] = json.load(f)["offsets"] + return hparams + + +@ModelBase.register("VllmDecision2Model") +@ModelBase.example("vllm-sr/Decision-2.0-Kai-0.6B") +class VllmDecision2Model(Qwen3Model): + model_arch = gguf.MODEL_ARCH.VLLM_DECISION2 + + # torch.nn.LayerNorm default, used by the head + _HEAD_NORM_EPS = 1e-5 + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + head = self.hparams["decision"] + if head["architecture"] != "qwen3-text-endpoints-global-query-shared-bilinear-mlp" or head["head_variant"] != "shared": + raise ValueError(f"unsupported decision head: {head['architecture']}") + + def index_tensors(self, *args, **kwargs) -> dict[str, Callable[[], Tensor]]: + # the backbone weights are in their own directory + dir_model, self.dir_model = self.dir_model, self.dir_model / "backbone" + try: + return super().index_tensors(*args, **kwargs) + finally: + self.dir_model = dir_model + + def get_tensors(self) -> Iterator[tuple[str, Tensor]]: + yield from super().get_tensors() + from safetensors.torch import load_file + for name, data in load_file(self.dir_model / "decision_head.safetensors").items(): + yield "decision_head." + name, data + + def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: + # the head is computed in F32 + if new_name.startswith("decision."): + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + def set_vocab(self): + super().set_vocab() + self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}]) + + @classmethod + def _systemone_template(cls) -> str: + # prompt follows decision2/_vendor/dev2model/decision_model.py (segments) of the model repo + # the pieces of the prompt are tokenized one by one, the server gives the text that separates them (sep) + # and the text that starts the span of a question or of an option (mark_question, mark_option) + # the head reads the last token of each option and the last token of the question + # the start of an option is a piece of its own, so that the spans of two options are not joined + # the keys of JSON objects are given in sorted order + def text(value: str) -> str: + return "{{ " + json.dumps(value) + " }}" + + def render(name: str) -> str: + return "{{ " + name + " if " + name + " is string else " + name + " | tojson(separators=[',', ':']) }}" + + option = ( + "{% set d = o.description %}" + "{% if q.type == 'noul' and d is none %}{% set d = 'Yes' if o.key == 'true' else 'No' %}{% endif %}" + "{{ {'description': d, 'key': o.key} | tojson(separators=[',', ':']) }}" + ) + return ( + "{% for q in questions %}" + + text("Context:\n") + render("state") + + text("\n\nTask type: ") + "{{ q.type }}" + text("\nQuestion:\n") + render("q.instructions") + text("\nOptions:") + + "{% for o in q.options %}{{ sep }}" + text("\n") + "{% endfor %}" + + "{{ sep }}{{ mark_question }}" + text("\n\nSelect the single option best supported by the context and instructions.\nDecision:") + + "{% endfor %}" + ) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + self.gguf_writer.add_decision_type(gguf.DecisionType.VLLM_DECISION2) + self.gguf_writer.add_layer_norm_eps(self._HEAD_NORM_EPS) + for offsets in self.hparams["decision"]["score_bias"].values(): + self.gguf_writer.add_decision_score_bias(offsets) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 5adb40205..a569afc96 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -332,6 +332,7 @@ class Keys: HEAD_COUNT = "{arch}.decision.head_count" MAX_HEAD_TOKENS = "{arch}.decision.max_head_tokens" TEMPERATURE = "{arch}.decision.temperature.{name}" # name: "" or "." + SCORE_BIAS = "{arch}.decision.score_bias.{n}.{i}" # added to the logit of level i of a score question with n levels class Tokenizer: MODEL = "tokenizer.ggml.model" @@ -559,6 +560,7 @@ class MODEL_ARCH(IntEnum): QWEN35 = auto() QWEN35MOE = auto() CLEF = auto() + VLLM_DECISION2 = auto() QWEN4EXP = auto() PHI2 = auto() PHI3 = auto() @@ -923,6 +925,11 @@ class MODEL_TENSOR(IntEnum): DECISION_SCALES = auto() DECISION_SCORER = auto() DECISION_SCORER_OUT = auto() + DECISION_QUERY_NORM = auto() + DECISION_KEY = auto() + DECISION_QUERY = auto() + DECISION_OPTION_MLP = auto() + DECISION_QUERY_MLP = auto() CONV1D = auto() CONVNEXT_DW = auto() CONVNEXT_NORM = auto() @@ -1343,6 +1350,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.QWEN35: "qwen35", MODEL_ARCH.QWEN35MOE: "qwen35moe", MODEL_ARCH.CLEF: "clef", + MODEL_ARCH.VLLM_DECISION2: "vllm-decision2", MODEL_ARCH.QWEN4EXP: "qwen4exp", MODEL_ARCH.PHI2: "phi2", MODEL_ARCH.PHI3: "phi3", @@ -1706,6 +1714,11 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.DECISION_SCALES: "decision.scales", MODEL_TENSOR.DECISION_SCORER: "decision.scorer", MODEL_TENSOR.DECISION_SCORER_OUT: "decision.scorer_out", + MODEL_TENSOR.DECISION_QUERY_NORM: "decision.query_norm", + MODEL_TENSOR.DECISION_KEY: "decision.key", + MODEL_TENSOR.DECISION_QUERY: "decision.query", + MODEL_TENSOR.DECISION_OPTION_MLP: "decision.option_mlp", + MODEL_TENSOR.DECISION_QUERY_MLP: "decision.query_mlp", MODEL_TENSOR.CONV1D: "conv1d", MODEL_TENSOR.CONVNEXT_DW: "convnext.{bid}.dw", MODEL_TENSOR.CONVNEXT_NORM: "convnext.{bid}.norm", @@ -3034,6 +3047,29 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.DECISION_SCORER, MODEL_TENSOR.DECISION_SCORER_OUT, ], + MODEL_ARCH.VLLM_DECISION2: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_Q_NORM, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_K_NORM, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + # decision head + MODEL_TENSOR.DECISION_OPTION_NORM, + MODEL_TENSOR.DECISION_QUERY_NORM, + MODEL_TENSOR.DECISION_KEY, + MODEL_TENSOR.DECISION_QUERY, + MODEL_TENSOR.DECISION_OPTION_MLP, + MODEL_TENSOR.DECISION_QUERY_MLP, + MODEL_TENSOR.DECISION_SCORER_OUT, + ], MODEL_ARCH.QWEN35MOE: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, @@ -6100,6 +6136,7 @@ class DecisionType: NIMBLE = "nimble" # same as openjev, the prompt lists all the questions of the request CLEF = "clef" # joint head over all questions, one score per option PPLX_DECIDER = "pplx-decider" # same as openjev, label codes of 1 or 2 letters + VLLM_DECISION2 = "vllm-decision2" # head over the end token of each option and the last token, one score per option LFM2_D1 = "lfm2-d1" # same as openjev, the labels depend on the question type LFM2_D1_OMNI = "lfm2-d1-omni" # same head as laya on a bidirectional LFM2 trunk, other prompt layout diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 86d4d6e2f..cda59a628 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1365,6 +1365,10 @@ class GGUFWriter: def add_decision_temperature(self, name: str, value: float) -> None: self.add_float32(Keys.Decision.TEMPERATURE.format(arch=self.arch, name=name), value) + def add_decision_score_bias(self, values: Sequence[float]) -> None: + for i, value in enumerate(values): + self.add_float32(Keys.Decision.SCORE_BIAS.format(arch=self.arch, n=len(values), i=i), value) + # for vision models def add_clip_has_vision_encoder(self, value: bool) -> None: diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 566b7348a..6445784a3 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -1328,7 +1328,8 @@ class TensorNameMap: ), MODEL_TENSOR.DECISION_OPTION_NORM: ( - "joint_head.option_norm", # clef + "joint_head.option_norm", # clef + "decision_head.candidate_norm", # vllm-decision2 ), MODEL_TENSOR.DECISION_SCORER: ( @@ -1337,6 +1338,27 @@ class TensorNameMap: MODEL_TENSOR.DECISION_SCORER_OUT: ( "joint_head.residual_scorer.3", # clef + "decision_head.scalar", # vllm-decision2 + ), + + MODEL_TENSOR.DECISION_QUERY_NORM: ( + "decision_head.query_norm", # vllm-decision2 + ), + + MODEL_TENSOR.DECISION_KEY: ( + "decision_head.key", # vllm-decision2 + ), + + MODEL_TENSOR.DECISION_QUERY: ( + "decision_head.query", # vllm-decision2 + ), + + MODEL_TENSOR.DECISION_OPTION_MLP: ( + "decision_head.candidate_mlp", # vllm-decision2 + ), + + MODEL_TENSOR.DECISION_QUERY_MLP: ( + "decision_head.query_mlp", # vllm-decision2 ), MODEL_TENSOR.ENC_ATTN_NORM: ( diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 88eab7b4d..0dc4cc961 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -41,6 +41,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_QWEN35, "qwen35" }, { LLM_ARCH_QWEN35MOE, "qwen35moe" }, { LLM_ARCH_CLEF, "clef" }, + { LLM_ARCH_VLLM_DECISION2, "vllm-decision2" }, { LLM_ARCH_QWEN4EXP, "qwen4exp" }, { LLM_ARCH_PHI2, "phi2" }, { LLM_ARCH_PHI3, "phi3" }, @@ -524,6 +525,11 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_DECISION_SCALES, "decision.scales" }, { LLM_TENSOR_DECISION_SCORER, "decision.scorer" }, { LLM_TENSOR_DECISION_SCORER_OUT, "decision.scorer_out" }, + { LLM_TENSOR_DECISION_QUERY_NORM, "decision.query_norm" }, + { LLM_TENSOR_DECISION_KEY, "decision.key" }, + { LLM_TENSOR_DECISION_QUERY, "decision.query" }, + { LLM_TENSOR_DECISION_OPTION_MLP, "decision.option_mlp" }, + { LLM_TENSOR_DECISION_QUERY_MLP, "decision.query_mlp" }, { LLM_TENSOR_ENC_OUTPUT_NORM, "enc.output_norm" }, { LLM_TENSOR_FFN_GATE_INP_SHEXP, "blk.%d.ffn_gate_inp_shexp" }, { LLM_TENSOR_SSM_A_NOSCAN, "blk.%d.ssm_a" }, @@ -789,6 +795,11 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_DECISION_SCALES, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, {LLM_TENSOR_DECISION_SCORER, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, {LLM_TENSOR_DECISION_SCORER_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DECISION_QUERY_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, + {LLM_TENSOR_DECISION_KEY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DECISION_QUERY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DECISION_OPTION_MLP, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DECISION_QUERY_MLP, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, {LLM_TENSOR_DENSE_2_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output {LLM_TENSOR_DENSE_3_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output {LLM_TENSOR_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, diff --git a/src/llama-arch.h b/src/llama-arch.h index 2d8ad6b07..8f8cc27ed 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -46,6 +46,7 @@ enum llm_arch { LLM_ARCH_QWEN35, LLM_ARCH_QWEN35MOE, LLM_ARCH_CLEF, + LLM_ARCH_VLLM_DECISION2, LLM_ARCH_QWEN4EXP, LLM_ARCH_PHI2, LLM_ARCH_PHI3, @@ -695,6 +696,11 @@ enum llm_tensor { LLM_TENSOR_DECISION_SCALES, LLM_TENSOR_DECISION_SCORER, LLM_TENSOR_DECISION_SCORER_OUT, + LLM_TENSOR_DECISION_QUERY_NORM, + LLM_TENSOR_DECISION_KEY, + LLM_TENSOR_DECISION_QUERY, + LLM_TENSOR_DECISION_OPTION_MLP, + LLM_TENSOR_DECISION_QUERY_MLP, LLM_TENSOR_CONV1D, LLM_TENSOR_CONVNEXT_DW, LLM_TENSOR_CONVNEXT_NORM, diff --git a/src/llama-ext.h b/src/llama-ext.h index fcc78f47e..faf4beb69 100644 --- a/src/llama-ext.h +++ b/src/llama-ext.h @@ -100,7 +100,7 @@ LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value // chain multiple trained NextN heads. Default 0 (first head). LLAMA_API void llama_set_nextn_layer_offset(struct llama_context * ctx, int32_t offset); -// Marks the entries that a joint decision head (clef) reads, the default is 0 +// Marks the entries that a decision head (clef, vllm-decision2) reads, the default is 0 // See https://github.com/ggml-org/llama.cpp/pull/29831 for details // A run of entries with the same value is one span, spans must be separated by entries with value 0 // An option belongs to the last question before it diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 1c7cbd900..47e36d374 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -21,6 +21,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) { case LLM_ARCH_APERTUS: case LLM_ARCH_STEP35: case LLM_ARCH_CLEF: // the head tensors are not saved + case LLM_ARCH_VLLM_DECISION2: return false; default: return true; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index b3e88322b..6872589fa 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -330,6 +330,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_qwen35moe(params); case LLM_ARCH_CLEF: return new llama_model_clef(params); + case LLM_ARCH_VLLM_DECISION2: + return new llama_model_vllm_decision2(params); case LLM_ARCH_QWEN4EXP: return new llama_model_qwen4exp(params); case LLM_ARCH_MISTRAL3: @@ -3534,6 +3536,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_DREAM: case LLM_ARCH_QWEN2MOE: case LLM_ARCH_QWEN3: + case LLM_ARCH_VLLM_DECISION2: case LLM_ARCH_QWEN3MOE: case LLM_ARCH_LLADA_MOE: case LLM_ARCH_RND1: diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 821b4798c..9c6066796 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -346,6 +346,11 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param quantize &= name.find("attn_v_b.weight") == std::string::npos; } + // the decision head of vllm-decision2 is small and computed in F32 + if (arch == LLM_ARCH_VLLM_DECISION2) { + quantize &= name.compare(0, 9, "decision.") != 0; + } + // do not quantize RWKV's small yet 2D weights quantize &= name.find("time_mix_first.weight") == std::string::npos; quantize &= name.find("time_mix_w0.weight") == std::string::npos; diff --git a/src/models/models.h b/src/models/models.h index d6ddf9d16..a55983cfb 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -583,6 +583,44 @@ struct llama_model_qwen3 : public llama_model_base { struct graph : public llm_graph_context { graph(const llama_model & model, const llm_graph_params & params); + protected: + // the helpers alone, a derived graph builds its own body + struct no_build {}; + graph(const llm_graph_params & params, no_build) : llm_graph_context(params) {} + + // the layers and the final norm: [n_embd, n_outputs] + ggml_tensor * build_backbone(const llama_model & model); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + +// Qwen3 with the decision head of vllm-sr Decision 2.0: one score per option +struct llama_model_vllm_decision2 : public llama_model_qwen3 { + llama_model_vllm_decision2(const struct llama_model_params & params) : llama_model_qwen3(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + ggml_tensor * option_norm = nullptr; + ggml_tensor * option_norm_b = nullptr; + ggml_tensor * query_norm = nullptr; + ggml_tensor * query_norm_b = nullptr; + ggml_tensor * key = nullptr; + ggml_tensor * query = nullptr; + ggml_tensor * option_mlp = nullptr; + ggml_tensor * option_mlp_b = nullptr; + ggml_tensor * query_mlp = nullptr; + ggml_tensor * scorer_out = nullptr; + + class input_decision; + + struct graph : public llama_model_qwen3::graph { + graph(const llama_model & model, const llm_graph_params & params); + private: + // hidden: [n_embd, n_outputs], returns one score per option: [1, n_options] + ggml_tensor * build_head(ggml_tensor * hidden, input_decision * inp); + + const llama_model_vllm_decision2 & model; }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; diff --git a/src/models/qwen3.cpp b/src/models/qwen3.cpp index f4b2a2aeb..a565e3e68 100644 --- a/src/models/qwen3.cpp +++ b/src/models/qwen3.cpp @@ -51,6 +51,18 @@ std::unique_ptr llama_model_qwen3::build_arch_graph(const llm } llama_model_qwen3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { + ggml_tensor * cur = build_backbone(model); + + // lm_head + cur = build_lora_mm(model.output, cur, model.output_s); + + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} + +ggml_tensor * llama_model_qwen3::graph::build_backbone(const llama_model & model) { const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); @@ -149,11 +161,5 @@ llama_model_qwen3::graph::graph(const llama_model & model, const llm_graph_param cb(cur, "result_norm", -1); res->t_embd = cur; - // lm_head - cur = build_lora_mm(model.output, cur, model.output_s); - - cb(cur, "result_output", -1); - res->t_logits = cur; - - ggml_build_forward_expand(gf, cur); + return cur; } diff --git a/src/models/vllm-decision2.cpp b/src/models/vllm-decision2.cpp new file mode 100644 index 000000000..bbac808de --- /dev/null +++ b/src/models/vllm-decision2.cpp @@ -0,0 +1,179 @@ +#include "models.h" + +#include "llama-ext.h" + +#include + +void llama_model_vllm_decision2::load_arch_hparams(llama_model_loader & ml) { + llama_model_qwen3::load_arch_hparams(ml); + + // used by the head + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); + + // the output is one score per token, see llama_batch_ext_set_decision_order() + hparams.n_embd_out_impl = 1; +} + +void llama_model_vllm_decision2::load_arch_tensors(llama_model_loader & ml) { + llama_model_qwen3::load_arch_tensors(ml); + + LLAMA_LOAD_LOCALS; + + const auto * w_key = ml.get_weight(tn(LLM_TENSOR_DECISION_KEY, "weight").str().c_str()); + if (w_key == nullptr) { + throw std::runtime_error("the decision head is missing"); + } + const int64_t n_embd_h = w_key->tensor->ne[1]; + + option_norm = create_tensor(tn(LLM_TENSOR_DECISION_OPTION_NORM, "weight"), {n_embd}, 0); + option_norm_b = create_tensor(tn(LLM_TENSOR_DECISION_OPTION_NORM, "bias"), {n_embd}, 0); + query_norm = create_tensor(tn(LLM_TENSOR_DECISION_QUERY_NORM, "weight"), {n_embd}, 0); + query_norm_b = create_tensor(tn(LLM_TENSOR_DECISION_QUERY_NORM, "bias"), {n_embd}, 0); + + key = create_tensor(tn(LLM_TENSOR_DECISION_KEY, "weight"), {n_embd, n_embd_h}, 0); + query = create_tensor(tn(LLM_TENSOR_DECISION_QUERY, "weight"), {n_embd, n_embd_h}, 0); + option_mlp = create_tensor(tn(LLM_TENSOR_DECISION_OPTION_MLP, "weight"), {n_embd, n_embd_h}, 0); + option_mlp_b = create_tensor(tn(LLM_TENSOR_DECISION_OPTION_MLP, "bias"), {n_embd_h}, 0); + query_mlp = create_tensor(tn(LLM_TENSOR_DECISION_QUERY_MLP, "weight"), {n_embd, n_embd_h}, 0); + scorer_out = create_tensor(tn(LLM_TENSOR_DECISION_SCORER_OUT, "weight"), {n_embd_h, 1}, 0); +} + +std::unique_ptr llama_model_vllm_decision2::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +// output rows read by the head: the last token of each option span and the last token of the question span +struct decision2_rows { + std::vector options; + int32_t query = 0; + bool valid = false; +}; + +// see llama_batch_ext_set_decision_order() +// if the batch has no usable order, returns one option at row 0 +static decision2_rows decision2_get_rows(const llama_ubatch & ubatch) { + decision2_rows res; + + const int32_t n_tokens = ubatch.n_tokens; + bool ok = ubatch.decision_order != nullptr && ubatch.n_seqs_unq == 1; + + // output row of each token, -1 if the token has no output + std::vector rows(n_tokens, -1); + for (int32_t i = 0, n = 0; i < n_tokens; i++) { + if (ubatch.output[i]) { + rows[i] = n++; + } + } + + int32_t n_questions = 0; + for (int32_t i = 0; ok && i < n_tokens;) { + const int32_t order = ubatch.decision_order[i]; + int32_t end = i + 1; + while (end < n_tokens && ubatch.decision_order[end] == order) { + end++; + } + switch (order) { + case LLAMA_DECISION_ORDER_NONE: + break; + case LLAMA_DECISION_ORDER_QUESTION_NOUL: + case LLAMA_DECISION_ORDER_QUESTION_CHOICE: + case LLAMA_DECISION_ORDER_QUESTION_SCORE: + res.query = rows[end - 1]; + n_questions++; + break; + case LLAMA_DECISION_ORDER_OPTION: + res.options.push_back(rows[end - 1]); + break; + default: + ok = false; + } + ok = ok && (order == LLAMA_DECISION_ORDER_NONE || rows[end - 1] >= 0); + i = end; + } + + res.valid = ok && n_questions == 1 && !res.options.empty(); + if (!res.valid) { + res.options.assign(1, 0); + res.query = 0; + } + return res; +} + +class llama_model_vllm_decision2::input_decision : public llm_graph_input_i { +public: + input_decision(const llama_ubatch & ubatch) : n_options(decision2_get_rows(ubatch).options.size()) {} + + void set_input(const llama_ubatch * ubatch) override { + const auto rows = decision2_get_rows(*ubatch); + GGML_ASSERT(rows.options.size() == n_options); + + // the scores are NaN if the batch has a decision order that cannot be used + const float status_data = rows.valid || ubatch->decision_order == nullptr ? 0.0f : NAN; + + ggml_backend_tensor_set(options, rows.options.data(), 0, ggml_nbytes(options)); + ggml_backend_tensor_set(query, &rows.query, 0, ggml_nbytes(query)); + ggml_backend_tensor_set(status, &status_data, 0, ggml_nbytes(status)); + } + + bool can_reuse(const llm_graph_params & params) override { + // the values are computed again in set_input(), only the shapes must match + return decision2_get_rows(params.ubatch).options.size() == n_options; + } + + ggml_tensor * options = nullptr; // I32 [n_options] + ggml_tensor * query = nullptr; // I32 [1] + ggml_tensor * status = nullptr; // F32 [1], added to the scores: 0, or NaN on invalid input + + const size_t n_options; +}; + +llama_model_vllm_decision2::graph::graph(const llama_model & model_base, const llm_graph_params & params) : + llama_model_qwen3::graph(params, no_build()), model(static_cast(model_base)) { + auto inp_decision_ptr = std::make_unique(ubatch); + auto * inp_decision = inp_decision_ptr.get(); + res->add_input(std::move(inp_decision_ptr)); + + ggml_tensor * cur = build_backbone(model); + + // the head is always evaluated, so that the graph has the same nodes for every batch + cur = build_head(cur, inp_decision); + + // row i of the output is the score of option i + cur = ggml_pad(ctx0, cur, 0, n_outputs - cur->ne[1], 0, 0); + cur = ggml_add(ctx0, cur, inp_decision->status); + cb(cur, "result_decision", -1); + + res->t_embd = cur; + + ggml_build_forward_expand(gf, cur); +} + +ggml_tensor * llama_model_vllm_decision2::graph::build_head(ggml_tensor * hidden, input_decision * inp) { + inp->options = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, inp->n_options); + inp->query = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); + inp->status = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1); + for (ggml_tensor * t : { inp->options, inp->query, inp->status }) { + ggml_set_input(t); + } + + ggml_tensor * options = ggml_get_rows(ctx0, hidden, inp->options); // [n_embd, n_options] + ggml_tensor * query = ggml_get_rows(ctx0, hidden, inp->query); // [n_embd, 1] + + options = build_norm(options, model.option_norm, model.option_norm_b, LLM_NORM, -1); + query = build_norm(query, model.query_norm, model.query_norm_b, LLM_NORM, -1); + + // scaled dot product of the option key and the query + ggml_tensor * k = ggml_mul_mat(ctx0, model.key, options); // [n_embd_h, n_options] + ggml_tensor * q = ggml_mul_mat(ctx0, model.query, query); // [n_embd_h, 1] + ggml_tensor * bilinear = ggml_sum_rows(ctx0, ggml_mul(ctx0, k, q)); + bilinear = ggml_scale(ctx0, bilinear, 1.0f / sqrtf((float) k->ne[0])); // [1, n_options] + + ggml_tensor * mlp = ggml_add(ctx0, ggml_mul_mat(ctx0, model.option_mlp, options), model.option_mlp_b); + mlp = ggml_add(ctx0, mlp, ggml_mul_mat(ctx0, model.query_mlp, query)); + mlp = ggml_gelu_erf(ctx0, mlp); + ggml_tensor * nonlinear = ggml_mul_mat(ctx0, model.scorer_out, mlp); // [1, n_options] + + ggml_tensor * scores = ggml_add(ctx0, bilinear, nonlinear); + cb(scores, "decision_scores", -1); + return scores; +} diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 78128ebf0..1e8e915b1 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -757,7 +757,7 @@ static bool arch_supported(const llm_arch arch) { if (arch == LLM_ARCH_PLM) { return false; // TODO tensor shapes } - if (arch == LLM_ARCH_CLEF) { + if (arch == LLM_ARCH_CLEF || arch == LLM_ARCH_VLLM_DECISION2) { return false; // TODO decision head tensors } if (arch == LLM_ARCH_DEEPSEEK2OCR) { diff --git a/tools/server/README.md b/tools/server/README.md index 0c6650412..c7cf50d5a 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1718,9 +1718,9 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly. -The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for laya, clef, pplx-decider, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with. +The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for laya, clef, pplx-decider, vllm-decision2, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with. -For laya, clef and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. A server that runs clef only serves this endpoint, text generation is not available. +For laya, clef and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. *Image input:* diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 5f77d43d0..9f43e7a2c 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -156,6 +156,7 @@ std::vector server_model_output_modalities(common_decision_type dec case COMMON_DECISION_TYPE_LAYA: case COMMON_DECISION_TYPE_CLEF: case COMMON_DECISION_TYPE_PPLX_DECIDER: + case COMMON_DECISION_TYPE_VLLM_DECISION2: case COMMON_DECISION_TYPE_LFM2_D1: case COMMON_DECISION_TYPE_LFM2_D1_OMNI: return {"decisions"}; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 2a25b75d8..3014f52cf 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3604,6 +3604,11 @@ private: n_past = std::min(n_past, slot.task->n_tokens_shared); } + // the outputs of a decision are read from the batch, so its first position must be evaluated + if (n_decision_first >= 0) { + n_past = std::min(n_past, n_decision_first); + } + // if there is an alora invoked, don't cache after the invocation start if (slot.alora_invocation_start > 0) { SLT_DBG(slot, "only caching to alora invocation start (n_past = %d, alora_invocation_start = %d)\n", n_past, slot.alora_invocation_start); diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp index 11ad66a19..6b7d2c16e 100644 --- a/tools/server/server-decision.cpp +++ b/tools/server/server-decision.cpp @@ -66,6 +66,28 @@ void server_decision_context::init(const llama_model * model) { temperatures[key + prefix_temp.size()] = temp; } + // "." -> offset + const std::string prefix_bias = prefix + "score_bias."; + for (int32_t i = 0; i < llama_model_meta_count(model); i++) { + char key[256]; + char val[64]; + if (llama_model_meta_key_by_index(model, i, key, sizeof(key)) < 0 || !string_starts_with(key, prefix_bias)) { + continue; + } + if (llama_model_meta_val_str_by_index(model, i, val, sizeof(val)) < 0) { + continue; + } + const auto parts = string_split(key + prefix_bias.size(), '.'); + const size_t n = parts.size() == 2 ? std::strtoul(parts[0].c_str(), nullptr, 10) : 0; + const size_t level = parts.size() == 2 ? std::strtoul(parts[1].c_str(), nullptr, 10) : 0; + if (n < 2 || n > 255 || level >= n) { + throw std::runtime_error(string_format("invalid decision score bias: %s", key)); + } + auto & bias = score_bias[n]; + bias.resize(n, 0.0f); + bias[level] = std::strtof(val, nullptr); + } + if (model_type == COMMON_DECISION_TYPE_OPENJEV) { // one letter per option, each must be a single token const std::string letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; @@ -123,6 +145,10 @@ void server_decision_context::init(const llama_model * model) { n_options_max = 255; noul_true_first = true; choice_sorted = true; + } else if (model_type == COMMON_DECISION_TYPE_VLLM_DECISION2) { + n_options_max = 255; + noul_criteria_order = true; + strict_payload = true; } else if (model_type == COMMON_DECISION_TYPE_LFM2_D1) { n_options_max = 255; noul_true_first = true; @@ -144,11 +170,18 @@ void server_decision_context::init(const llama_model * model) { // request parsing // +static bool decision_is_payload(const json & val) { + return val.is_string() || val.is_object() || val.is_array(); +} + std::vector server_decision_context::parse_questions(const json & body) const { // lfm2-d1 and lfm2-d1-omni accept a null state, for example to ask about images only if (!body.contains("state") || (body.at("state").is_null() && type != COMMON_DECISION_TYPE_LFM2_D1 && type != COMMON_DECISION_TYPE_LFM2_D1_OMNI)) { throw std::invalid_argument("\"state\" must be provided"); } + if (strict_payload && !decision_is_payload(body.at("state"))) { + throw std::invalid_argument("\"state\" must be text, an object or an array"); + } if (!body.contains("questions") || !body.at("questions").is_object() || body.at("questions").empty()) { throw std::invalid_argument("\"questions\" must be a non-empty object"); } @@ -164,6 +197,9 @@ std::vector server_decision_context::parse_questions(c if (!q.contains("instructions") || q.at("instructions").is_null()) { throw err("\"instructions\" must be provided"); } + if (strict_payload && (!decision_is_payload(q.at("instructions")) || q.at("instructions") == "")) { + throw err("\"instructions\" must be non-empty text, an object or an array"); + } server_decision_question question; question.id = id; @@ -178,8 +214,14 @@ std::vector server_decision_context::parse_questions(c throw err("\"criteria\" must be a non-empty object"); } for (const auto & [key, description] : criteria.items()) { + if (strict_payload && (key.empty() || !(description.is_null() || decision_is_payload(description)))) { + throw err("option keys must be non-empty, descriptions must be null, text, an object or an array"); + } question.options.push_back({key, description}); } + if (strict_payload && question.options.size() < 2) { + throw err("\"criteria\" must have at least 2 options"); + } if (choice_sorted) { std::sort(question.options.begin(), question.options.end(), [](const auto & a, const auto & b) { return a.key < b.key; @@ -191,6 +233,9 @@ std::vector server_decision_context::parse_questions(c throw err("\"criteria\" must be an array of 2 to 10 levels"); } for (size_t i = 0; i < criteria.size(); i++) { + if (strict_payload && !decision_is_payload(criteria.at(i))) { + throw err("level descriptions must be text, an object or an array"); + } question.options.push_back({std::to_string(i), criteria.at(i)}); } } else if (type_name == "noul") { @@ -198,6 +243,13 @@ std::vector server_decision_context::parse_questions(c if (!criteria.is_null() && !criteria.is_object()) { throw err("\"criteria\" must be an object"); } + if (strict_payload && criteria.is_object()) { + for (const auto & [key, description] : criteria.items()) { + if ((key != "false" && key != "true") || !decision_is_payload(description)) { + throw err("\"criteria\" may only describe \"false\" and \"true\" with text, an object or an array"); + } + } + } for (const char * key : {"false", "true"}) { json description; if (criteria.is_object() && criteria.contains(key)) { @@ -209,7 +261,7 @@ std::vector server_decision_context::parse_questions(c } question.options.push_back({key, description}); } - if (noul_true_first) { + if (noul_true_first || (noul_criteria_order && criteria.is_object() && criteria.size() == 2 && criteria.begin().key() == "true")) { std::swap(question.options[0], question.options[1]); } } else { @@ -658,6 +710,11 @@ void server_decision_context::fill_task( mtmd_context * mctx, const mtmd_helper_init_opt & init_opt, server_task & task) const { + // each question has its own prompt, with the spans that the head reads + if (type == COMMON_DECISION_TYPE_VLLM_DECISION2) { + fill_task_joint(state, { question }, files, mctx, init_opt, task); + return; + } if (type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) { fill_task_d1omni(state, questions, question, variant, files, mctx, init_opt, task); return; @@ -1056,6 +1113,32 @@ static double decision_confidence_score(const std::vector & probs) { return std::max(0.0, 1.0 - dist / dist_uniform); } +// VLLM_DECISION2: normalized entropy, as in the reference implementation +static double decision_confidence_entropy(const std::vector & probs) { + if (probs.size() < 2) { + return 1.0; + } + double entropy = 0.0; + for (const double p : probs) { + if (p > 0.0) { + entropy -= p * std::log(p); + } + } + return std::max(0.0, std::min(1.0, 1.0 - entropy / std::log((double) probs.size()))); +} + +double server_decision_context::get_decision_confidence(server_decision_question_type question_type, const std::vector & probs) const { + if (type == COMMON_DECISION_TYPE_VLLM_DECISION2) { + return decision_confidence_entropy(probs); + } + switch (question_type) { + case SERVER_DECISION_QUESTION_CHOICE: return decision_confidence_choice(probs); + case SERVER_DECISION_QUESTION_SCORE: return decision_confidence_score(probs); + case SERVER_DECISION_QUESTION_NOUL: break; + } + throw std::runtime_error("no confidence for this question type"); +} + json server_decision_context::format_answer(const server_decision_question & question, const std::vector> & scores, bool has_media) const { const size_t n = n_outputs(question); if (scores.size() != n_variants(question)) { @@ -1075,11 +1158,17 @@ json server_decision_context::format_answer(const server_decision_question & que if (std::any_of(s.begin(), s.end(), [](float v) { return std::isnan(v); })) { throw std::runtime_error("the model could not evaluate the decision"); } - const float score_max = *std::max_element(s.begin(), s.end()); + std::vector logits(s.begin(), s.end()); + if (question.type == SERVER_DECISION_QUESTION_SCORE && score_bias.count(n) > 0) { + for (size_t i = 0; i < n; i++) { + logits[i] += score_bias.at(n)[i]; + } + } + const double logit_max = *std::max_element(logits.begin(), logits.end()); std::vector p(n); double sum = 0.0; for (size_t i = 0; i < n; i++) { - p[i] = std::exp((double) (s[i] - score_max) / temperature); + p[i] = std::exp((logits[i] - logit_max) / temperature); sum += p[i]; } for (size_t i = 0; i < n; i++) { @@ -1116,18 +1205,20 @@ json server_decision_context::format_answer(const server_decision_question & que const size_t best = std::max_element(probs.begin(), probs.end()) - probs.begin(); answer["choice"] = question.options[best].key; answer["probabilities"] = probabilities; - answer["confidence"] = decision_confidence_choice(probs); + answer["confidence"] = get_decision_confidence(question.type, probs); } else { double expected = 0.0; json legend = json::object(); for (size_t i = 0; i < n; i++) { expected += i * probs[i]; - legend[question.options[i].key] = question.options[i].description; + const json & description = question.options[i].description; + // vllm-decision2 returns structured descriptions as canonical JSON text + legend[question.options[i].key] = strict_payload && !description.is_string() ? json(decision_sort_keys(description).dump()) : description; } answer["score"] = expected; answer["legend"] = legend; answer["probabilities"] = probabilities; - answer["confidence"] = decision_confidence_score(probs); + answer["confidence"] = get_decision_confidence(question.type, probs); } return answer; } diff --git a/tools/server/server-decision.h b/tools/server/server-decision.h index 5c7e44f37..3e81098cb 100644 --- a/tools/server/server-decision.h +++ b/tools/server/server-decision.h @@ -43,6 +43,7 @@ struct server_decision_context { case COMMON_DECISION_TYPE_KEV: case COMMON_DECISION_TYPE_NIMBLE: case COMMON_DECISION_TYPE_PPLX_DECIDER: + case COMMON_DECISION_TYPE_VLLM_DECISION2: case COMMON_DECISION_TYPE_LFM2_D1: return true; default: @@ -110,9 +111,12 @@ private: std::shared_ptr tmpl; // the "systemone" template std::map temperatures; // "" or "." + std::map> score_bias; // added to the scores of a score question, by number of levels size_t n_options_max = 0; bool noul_true_first = false; // noul options are [true, false] instead of [false, true] + bool noul_criteria_order = false; // noul options are in the order of the criteria keys, if given bool choice_sorted = false; // choice options are in the order of their keys + bool strict_payload = false; // state, instructions and descriptions must be text, an object or an array // OPENJEV, LEV, NIMBLE, PPLX_DECIDER std::vector labels; @@ -148,6 +152,7 @@ private: server_task & task) const; float get_temperature(const server_decision_question & question) const; + double get_decision_confidence(server_decision_question_type type, const std::vector & probs) const; }; // group the tasks so that the common prefix of their prompts is evaluated only once diff --git a/tools/server/server-task.h b/tools/server/server-task.h index b8d6a3b67..50926ec3c 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -191,6 +191,11 @@ struct server_task { for (const int32_t marker : markers) { pos = pos < 0 ? marker : std::min(pos, marker); } + for (size_t i = 0; i < order.size(); i++) { + if (order[i] != 0) { + return pos < 0 ? (int32_t) i : std::min(pos, (int32_t) i); + } + } return pos; }