diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 51a16ef52..4482f7f33 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -371,6 +371,7 @@ class Keys: DNA_AUTO_TAGS = "tokenizer.ggml.dna_auto_tags" ADD_PREFIX = "tokenizer.ggml.add_space_prefix" REMOVE_EXTRA_WS = "tokenizer.ggml.remove_extra_whitespaces" + CLEAN_SPACES = "tokenizer.ggml.clean_spaces" PRECOMPILED_CHARSMAP = "tokenizer.ggml.precompiled_charsmap" SUPPRESS_TOKENS = "tokenizer.ggml.suppress_tokens" HF_JSON = "tokenizer.huggingface.json" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index d00392211..2faaefa59 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1326,6 +1326,9 @@ class GGUFWriter: def add_remove_extra_whitespaces(self, value: bool) -> None: self.add_bool(Keys.Tokenizer.REMOVE_EXTRA_WS, value) + def add_clean_spaces(self, value: bool) -> None: + self.add_bool(Keys.Tokenizer.CLEAN_SPACES, value) + def add_precompiled_charsmap(self, charsmap: bytes) -> None: self.add_array(Keys.Tokenizer.PRECOMPILED_CHARSMAP, charsmap) diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index 78e90eddc..6383178ea 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -55,6 +55,7 @@ class SpecialVocab: normalizer_nfc: bool | None normalizer_lowercase: bool | None normalizer_strip_accents: bool | None + clean_spaces: bool | None def __init__( self, path: str | os.PathLike[str], load_merges: bool = False, @@ -70,6 +71,7 @@ class SpecialVocab: self.normalizer_nfc = None self.normalizer_lowercase = None self.normalizer_strip_accents = None + self.clean_spaces = None if special_token_types is not None: self.special_token_types = special_token_types else: @@ -120,6 +122,10 @@ class SpecialVocab: if not quiet: logger.info(f'Setting normalizer_strip_accents to {self.normalizer_strip_accents}') gw.add_normalizer_strip_accents(self.normalizer_strip_accents) + if self.clean_spaces is not None: + if not quiet: + logger.info(f'Setting clean_spaces to {self.clean_spaces}') + gw.add_clean_spaces(self.clean_spaces) def _load(self, path: Path) -> None: self._try_load_from_tokenizer_json(path) @@ -345,6 +351,10 @@ class SpecialVocab: self.chat_template = chat_template else: logger.warning(f'Bad type for chat_template field in {tokenizer_config_file!r} - ignoring') + # decode clean-up of spaces before punctuation (transformers clean_up_tokenization_spaces) + clean_spaces = tokenizer_config.get('clean_up_tokenization_spaces') + if isinstance(clean_spaces, bool): + self.clean_spaces = clean_spaces for typ in self.special_token_types: add_entry = tokenizer_config.get(f'add_{typ}_token') if isinstance(add_entry, bool): diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 62b012c5c..d79b18dec 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -437,6 +437,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_TOKENIZER_DNA_AUTO_TAGS, "tokenizer.ggml.dna_auto_tags" }, { LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" }, { LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" }, + { LLM_KV_TOKENIZER_CLEAN_SPACES, "tokenizer.ggml.clean_spaces" }, { LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" }, { LLM_KV_TOKENIZER_HF_JSON, "tokenizer.huggingface.json" }, { LLM_KV_TOKENIZER_RWKV, "tokenizer.rwkv.world" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 6713db6b5..12d8fe392 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -400,6 +400,7 @@ enum llm_kv { LLM_KV_TOKENIZER_DNA_AUTO_TAGS, LLM_KV_TOKENIZER_ADD_PREFIX, LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, + LLM_KV_TOKENIZER_CLEAN_SPACES, LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, LLM_KV_TOKENIZER_HF_JSON, LLM_KV_TOKENIZER_RWKV, diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index d620fcd9c..2135b82b1 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -424,6 +424,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_TOKENIZER_ADD_SEP, vocab.get_add_sep()); add_kv(LLM_KV_TOKENIZER_ADD_PREFIX, vocab.get_add_space_prefix()); add_kv(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, vocab.get_remove_extra_whitespaces()); + add_kv(LLM_KV_TOKENIZER_CLEAN_SPACES, vocab.get_clean_spaces()); add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, vocab.get_precompiled_charsmap()); // add_kv(LLM_KV_TOKENIZER_HF_JSON, ???); // add_kv(LLM_KV_TOKENIZER_RWKV, ???); diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index d25c4ceec..80a5fcdff 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -2432,6 +2432,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } else if (tokenizer_pre == "minicpm5") { pre_type = LLAMA_VOCAB_PRE_TYPE_MINICPM5; ignore_merges = true; + clean_spaces = false; } else if ( tokenizer_pre == "llama3" || tokenizer_pre == "llama-v3" || @@ -2439,11 +2440,16 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { tokenizer_pre == "falcon3" || tokenizer_pre == "falcon-h1" || tokenizer_pre == "midm-2.0" || - tokenizer_pre == "lfm2" || tokenizer_pre == "jina-v5-nano") { pre_type = LLAMA_VOCAB_PRE_TYPE_LLAMA3; ignore_merges = true; add_bos = true; + } else if ( + tokenizer_pre == "lfm2") { + pre_type = LLAMA_VOCAB_PRE_TYPE_LLAMA3; + ignore_merges = true; + add_bos = true; + clean_spaces = false; } else if ( tokenizer_pre == "deepseek-llm") { pre_type = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM; @@ -2494,6 +2500,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { tokenizer_pre == "granite-embed-multi-311m") { pre_type = LLAMA_VOCAB_PRE_TYPE_GEMMA4; escape_whitespaces = true; + clean_spaces = false; } else if ( tokenizer_pre == "mmbert") { pre_type = LLAMA_VOCAB_PRE_TYPE_MMBERT; @@ -2540,6 +2547,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } else if ( tokenizer_pre == "dbrx") { pre_type = LLAMA_VOCAB_PRE_TYPE_DBRX; + clean_spaces = false; } else if ( tokenizer_pre == "smaug-bpe") { pre_type = LLAMA_VOCAB_PRE_TYPE_SMAUG; @@ -2551,12 +2559,14 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { tokenizer_pre == "chatglm-bpe") { pre_type = LLAMA_VOCAB_PRE_TYPE_CHATGLM4; special_bos_id = LLAMA_TOKEN_NULL; + clean_spaces = false; } else if ( tokenizer_pre == "glm4" || tokenizer_pre == "glm5") { pre_type = LLAMA_VOCAB_PRE_TYPE_CHATGLM4; special_bos_id = LLAMA_TOKEN_NULL; ignore_merges = true; + clean_spaces = false; } else if ( tokenizer_pre == "viking") { pre_type = LLAMA_VOCAB_PRE_TYPE_VIKING; @@ -2590,6 +2600,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } else if ( tokenizer_pre == "exaone4") { pre_type = LLAMA_VOCAB_PRE_TYPE_GPT2; + clean_spaces = false; } else if ( tokenizer_pre == "exaone-moe") { pre_type = LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE; @@ -2690,6 +2701,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } else if ( tokenizer_pre == "mellum2") { pre_type = LLAMA_VOCAB_PRE_TYPE_MELLUM2; + clean_spaces = false; } else if ( tokenizer_pre == "k2-horizon") { pre_type = LLAMA_VOCAB_PRE_TYPE_K2_HORIZON; @@ -2726,6 +2738,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { ml.get_key(LLM_KV_TOKENIZER_ADD_PREFIX, add_space_prefix, false); ml.get_key(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, remove_extra_whitespaces, false); + ml.get_key(LLM_KV_TOKENIZER_CLEAN_SPACES, clean_spaces, false); } const int token_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_LIST).c_str()); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 24e6ba609..815cc15a3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -142,7 +142,7 @@ llama_test(test-tokenizer-0 NAME test-tokenizer-0-command-r ARGS ${PROJE llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-coder ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-coder.gguf) llama_test(test-tokenizer-0 NAME test-tokenizer-0-deepseek-llm ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-deepseek-llm.gguf) llama_test(test-tokenizer-0 NAME test-tokenizer-0-falcon ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-falcon.gguf) -llama_test(test-tokenizer-0 NAME test-tokenizer-0-gemma-4 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gemma-4.gguf) +llama_test(test-tokenizer-0 NAME test-tokenizer-0-gemma-4 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gemma-4.gguf --check-decode) llama_test(test-tokenizer-0 NAME test-tokenizer-0-gpt-2 ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-gpt-2.gguf) llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-bpe ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-bpe.gguf) llama_test(test-tokenizer-0 NAME test-tokenizer-0-llama-spm ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-spm.gguf) diff --git a/tests/test-tokenizer-0.cpp b/tests/test-tokenizer-0.cpp index 37f8312c4..5253a46a6 100644 --- a/tests/test-tokenizer-0.cpp +++ b/tests/test-tokenizer-0.cpp @@ -125,7 +125,7 @@ static llama_tests read_tests(const std::string & fname_inp, const std::string & int main(int argc, char **argv) { if (argc < 2) { - fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]); + fprintf(stderr, "Usage: %s vocab-file [text-file] [--check-decode]\n", argv[0]); return 1; } @@ -135,8 +135,13 @@ int main(int argc, char **argv) { const std::string fname_out = fname + ".out"; std::string fname_text; - if (argc > 2) { - fname_text = argv[2]; + bool check_decode = false; // also check that the expected tokens detokenize to the input text + for (int i = 2; i < argc; i++) { + if (std::string(argv[i]) == "--check-decode") { + check_decode = true; + } else { + fname_text = argv[i]; + } } fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str()); @@ -244,6 +249,14 @@ int main(int argc, char **argv) { success = false; } + + if (check_decode) { + const std::string dec = common_detokenize(ctx, test_kv.second); + if (dec != test_kv.first) { + fprintf(stderr, "%s : failed decode: '%s' detokenized to '%s'\n", __func__, test_kv.first.c_str(), dec.c_str()); + success = false; + } + } } }); }