diff --git a/conversion/granite.py b/conversion/granite.py index d2ef7c836..4c6c5a0e4 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -296,19 +296,20 @@ class GraniteSwitchModel(GraniteMoeModel): def set_vocab(self): super().set_vocab() if self._has_decision_adapter(): - self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}]) + systemone_template = self._systemone_template() + logger.info("System One Template: %s", systemone_template) + self.gguf_writer.add_chat_template([{"name": "systemone", "template": systemone_template}]) def _systemone_template(self) -> str: # prompt of the decision aLoRA: a state turn, then a question turn that opens with the control token # (it replaces the first token of "### DECISION"), the answer is the label token that comes next description = jinja_str_or_json("o.description") option = ( - "{% if type != 'noul' %}{{ letters[loop.index0] }}: {% if o.description %}" + description + "{% endif %}" + "{% if type != 'noul' %}{{ o.label }}: {% if o.description %}" + description + "{% endif %}" "{% elif o.key == 'true' %}A: {% if o.description %}" + description + "{% else %}The statement is true.{% endif %}" "{% else %}B: {% if o.description %}" + description + "{% else %}The statement is false.{% endif %}{% endif %}" ) return ( - "{% set letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' %}" "<|start_of_role|>user<|end_of_role|>State:\n" + jinja_str_or_json("state") + "<|end_of_text|>\n" "<|start_of_role|>user<|end_of_role|><|decision|> DECISION\nQuestion:\n" + jinja_str_or_json("instructions") + "{% if type == 'score' %} Rate along the ordered levels below (lowest first).{% endif %}" diff --git a/tools/server/README.md b/tools/server/README.md index e4a0c418b..c42336c7f 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1723,7 +1723,7 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly. -The number of options of a `choice` question is limited by the model, for example: 52 for openjev and granite-switch, 255 for laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with. +The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for granite-switch, laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with. For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`. diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp index 3905c81b3..be9bc8ef4 100644 --- a/tools/server/server-decision.cpp +++ b/tools/server/server-decision.cpp @@ -87,7 +87,7 @@ void server_decision_context::init(const llama_model * model) { bias[level] = std::strtof(val, nullptr); } - if (model_type == COMMON_DECISION_TYPE_OPENJEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) { + if (model_type == COMMON_DECISION_TYPE_OPENJEV) { // one letter per option, each must be a single token const std::string letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; for (const char c : letters) { @@ -99,7 +99,7 @@ void server_decision_context::init(const llama_model * model) { } n_options_max = labels.size(); noul_true_first = true; - } else if (model_type == COMMON_DECISION_TYPE_LEV || model_type == COMMON_DECISION_TYPE_NIMBLE || model_type == COMMON_DECISION_TYPE_PPLX_DECIDER || model_type == COMMON_DECISION_TYPE_VLLM_D3) { + } else if (model_type == COMMON_DECISION_TYPE_LEV || model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH || model_type == COMMON_DECISION_TYPE_NIMBLE || model_type == COMMON_DECISION_TYPE_PPLX_DECIDER || model_type == COMMON_DECISION_TYPE_VLLM_D3) { // label codes are A..Z then AA..ZZ, only the ones that are a single token are used std::vector codes; for (char a = 'A'; a <= 'Z'; a++) { @@ -121,6 +121,9 @@ void server_decision_context::init(const llama_model * model) { if (model_type == COMMON_DECISION_TYPE_VLLM_D3) { canonical_legend = true; } + if (model_type == COMMON_DECISION_TYPE_GRANITE_SWITCH) { + noul_true_first = true; + } } else if (model_type == COMMON_DECISION_TYPE_KEV) { // the hidden state of an option is read at the token that ends it const auto toks = common_tokenize(vocab, "<|box_end|>", false, true);