From 7481354a174714a3447ea0f8723c1dd1b3cbf8e2 Mon Sep 17 00:00:00 2001 From: Toki Nasin <141258697+tokinasin@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:34:16 +0900 Subject: [PATCH] convert : fix token configuration for PLaMo-3 (#29843) * convert : Fix token configuration for PLaMo-3 The reasoning and tool calling tags in PLaMo-3 consist of three tokens each. For example, for reasoning: * reasoning start: `<|plamo:begin_`, `think`, `:plamo|>` * reasoning end: `<|plamo:end_`, `think`, `:plamo|>` Registering `<|plamo:begin_`, `<|plamo:end_`, and `:plamo|>` as `USER_DEFINED` so that they are parsed correctly. Also PLaMo-3 models use <|plamo:tag|> as EOT, while PLaMo-2 models use <|plamo:op|>. Take EOT token as a parameter and look it up so that PLaMo-2 and PLaMo-3 can use their appropriate ones. * use NORMAL instead of USER_DEFINED --- conversion/base.py | 18 ++++++++++++------ conversion/plamo.py | 7 +++++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/conversion/base.py b/conversion/base.py index 3b6fe04c2..306a9f0f2 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -2499,7 +2499,11 @@ class TextModel(ModelBase): if template is not None: self.gguf_writer.add_chat_template(template) - def _set_vocab_plamo(self): + def _set_vocab_plamo( + self, + eot_token: str, + normal_tokens: Iterable[str] = (), + ): # PLaMo models use a custom tokenizer with a .jsonl file tokenizer_jsonl_path = self.dir_model / "tokenizer.jsonl" tokenizer_config_path = self.dir_model / "tokenizer_config.json" @@ -2523,27 +2527,30 @@ class TextModel(ModelBase): tokens = [] scores = [] toktypes = [] + normal_tokens = set(normal_tokens) with open(tokenizer_jsonl_path, "r", encoding="utf-8") as f: for line_num, line in enumerate(f): if line.strip(): token_data = json.loads(line) # Format: [token, score, type, ?, ?, ?, ?] - token = token_data[0].encode("utf-8") + token_str = token_data[0] + token = token_str.encode("utf-8") score = float(token_data[1]) token_type_str = token_data[2] if len(token_data) > 2 else "NORMAL" tokens.append(token) scores.append(score) - if token_type_str == "UNKNOWN": + if token_str in normal_tokens: + toktypes.append(gguf.TokenType.NORMAL) + elif token_type_str == "UNKNOWN": toktypes.append(gguf.TokenType.UNKNOWN) elif token_type_str == "CONTROL": toktypes.append(gguf.TokenType.CONTROL) elif token_type_str == "BYTE": toktypes.append(gguf.TokenType.BYTE) else: - token_str = token_data[0] if token_str.startswith("<|plamo:") and token_str.endswith("|>"): toktypes.append(gguf.TokenType.CONTROL) else: @@ -2580,8 +2587,7 @@ class TextModel(ModelBase): token_id = tokens.index(tokenizer_config["unk_token"].encode("utf-8")) self.gguf_writer.add_unk_token_id(token_id) - # Add <|plamo:op|> as EOT to ensure appropriate end of generation - self.gguf_writer.add_eot_token_id(4) + self.gguf_writer.add_eot_token_id(tokens.index(eot_token.encode("utf-8"))) self.gguf_writer.add_add_space_prefix(False) diff --git a/conversion/plamo.py b/conversion/plamo.py index 52055e4a1..a843e059a 100644 --- a/conversion/plamo.py +++ b/conversion/plamo.py @@ -64,7 +64,7 @@ class Plamo2Model(TextModel): model_arch = gguf.MODEL_ARCH.PLAMO2 def set_vocab(self): - self._set_vocab_plamo() + self._set_vocab_plamo(eot_token="<|plamo:op|>") def set_gguf_parameters(self): hparams = self.hparams @@ -170,7 +170,10 @@ class Plamo3Model(TextModel): }) def set_vocab(self): - self._set_vocab_plamo() + self._set_vocab_plamo( + eot_token="<|plamo:tag|>", + normal_tokens=("<|plamo:begin_", "<|plamo:end_", ":plamo|>"), + ) tokenizer_config_path = self.dir_model / "tokenizer_config.json" tokenizer_config = {}