unicode : add opt-in NFC normalization for Unigram tokenizers (#30390)

* unicode : add opt-in NFC normalization for Unigram tokenizers

Note: used help from Codex, Sol 6.1 Medium

* Fixing initial bot comments + linter error (using existing out instead of print)

* removed test_nfc_tokenizer

gguf-py/gguf/constants.py

@@ -364,6 +364,7 @@ class Keys:
         CHAT_TEMPLATE_N      = "tokenizer.chat_template.{name}"
         CHAT_TEMPLATES       = "tokenizer.chat_templates"
         # Normalizer constants
+        NORMALIZER_NFC           = "tokenizer.ggml.normalizer.nfc"
         NORMALIZER_LOWERCASE     = "tokenizer.ggml.normalizer.lowercase"
         NORMALIZER_STRIP_ACCENTS = "tokenizer.ggml.normalizer.strip_accents"
         # FIM/Infill special tokens constants

gguf-py/gguf/gguf_writer.py

@@ -1329,6 +1329,9 @@ class GGUFWriter:
     def add_suppress_tokens(self, tokens: Sequence[int]) -> None:
         self.add_array(Keys.Tokenizer.SUPPRESS_TOKENS, tokens)
 
+    def add_normalizer_nfc(self, value: bool) -> None:
+        self.add_bool(Keys.Tokenizer.NORMALIZER_NFC, value)
+
     def add_normalizer_lowercase(self, value: bool) -> None:
         self.add_bool(Keys.Tokenizer.NORMALIZER_LOWERCASE, value)
 

scripts/gen-unicode-data.py

@@ -1,10 +1,50 @@
 from __future__ import annotations
 
 import array
+import sys
 import unicodedata
+
 import requests
 
 
+def out(line=""):
+    print(line, end='\n')  # noqa
+
+
+def normalization_tables():
+    out(f"// Canonical normalization tables: Unicode {unicodedata.unidata_version}.")
+    out("const std::unordered_map<uint32_t, uint8_t> unicode_combining_class = {")
+    for cpt in range(0x110000):
+        if value := unicodedata.combining(chr(cpt)):
+            out(f"{{0x{cpt:06X}, {value}}},")
+    out("};\n")
+    out("const std::unordered_map<uint32_t, std::pair<uint32_t, uint32_t>> unicode_decomposition = {")
+    compositions = []
+    for cpt in range(0x110000):
+        if 0xAC00 <= cpt < 0xAC00 + 11172:
+            continue
+        raw = unicodedata.decomposition(chr(cpt))
+        if not raw or raw.startswith("<"):
+            continue
+        parts = [int(part, 16) for part in raw.split()]
+        assert 1 <= len(parts) <= 2
+        first, second = parts[0], parts[1] if len(parts) == 2 else 0
+        out(f"{{0x{cpt:06X}, {{0x{first:06X}, 0x{second:06X}}}}},")
+        if second and unicodedata.normalize("NFC", chr(first) + chr(second)) == chr(cpt):
+            compositions.append((first, second, cpt))
+    out("};\n")
+    out("const std::unordered_map<uint64_t, uint32_t> unicode_composition = {")
+    for first, second, cpt in compositions:
+        out(f"{{0x{((first << 32) | second):016X}ULL, 0x{cpt:06X}}},")
+    out("};")
+
+
+# Generate only canonical tables, without downloading the UCD data.
+if sys.argv[1:] == ["--normalization-only"]:
+    normalization_tables()
+    sys.exit(0)
+
+
 MAX_CODEPOINTS = 0x110000
 
 UNICODE_DATA_URL = "https://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt"
@@ -155,10 +195,6 @@ for codepoint, norm in table_nfd:
 # Generate 'unicode-data.cpp':
 #   python ./scripts//gen-unicode-data.py > unicode-data.cpp
 
-def out(line=""):
-    print(line, end='\n')  # noqa
-
-
 out("""\
 // generated with scripts/gen-unicode-data.py
 
@@ -194,3 +230,5 @@ out("const std::vector<range_nfd> unicode_ranges_nfd = {  // start, last, nfd")
 for triple in ranges_nfd:
     out("{0x%06X, 0x%06X, 0x%06X}," % triple)
 out("};\n")
+
+normalization_tables()

src/llama-arch.cpp

@@ -421,6 +421,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
     { LLM_KV_TOKENIZER_HF_JSON,                  "tokenizer.huggingface.json"              },
     { LLM_KV_TOKENIZER_RWKV,                     "tokenizer.rwkv.world"                    },
     { LLM_KV_TOKENIZER_CHAT_TEMPLATE,            "tokenizer.chat_template"                 },
+    { LLM_KV_TOKENIZER_NORMALIZER_NFC,           "tokenizer.ggml.normalizer.nfc"           },
     { LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE,     "tokenizer.ggml.normalizer.lowercase"     },
     { LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, "tokenizer.ggml.normalizer.strip_accents" },
     { LLM_KV_TOKENIZER_FIM_PRE_ID,               "tokenizer.ggml.fim_pre_token_id"         },

src/llama-arch.h

@@ -394,6 +394,7 @@ enum llm_kv {
     LLM_KV_TOKENIZER_HF_JSON,
     LLM_KV_TOKENIZER_RWKV,
     LLM_KV_TOKENIZER_CHAT_TEMPLATE,
+    LLM_KV_TOKENIZER_NORMALIZER_NFC,
     LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE,
     LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS,
     LLM_KV_TOKENIZER_FIM_PRE_ID,

src/llama-vocab.cpp

@@ -1021,7 +1021,20 @@ struct llm_tokenizer_ugm_session {
 
         // normalize the input first
         std::string normalized;
-        normalize(text, &normalized);
+        if (vocab.get_normalizer_opts().nfc) {
+            try {
+                std::string canonical;
+                for (const uint32_t cpt : unicode_cpts_normalize_nfc(unicode_cpts_from_utf8(text))) {
+                    canonical += unicode_cpt_to_utf8(cpt);
+                }
+                normalize(canonical, &normalized);
+            } catch (const std::invalid_argument &) {
+                // Use the existing UGM normalization path for malformed input.
+                normalize(text, &normalized);
+            }
+        } else {
+            normalize(text, &normalized);
+        }
         size_t input_len = normalized.size();
         if (input_len == 0) {
             return;
@@ -1950,7 +1963,7 @@ struct llama_vocab::impl {
     bool escape_whitespaces         = true;
     bool treat_whitespace_as_suffix = false;
 
-    // BertNormalizer options
+    // Tokenizer normalizer options
     llama_vocab::normalizer_options normalizer_opts;
 
     std::unordered_map<std::string, llama_token> token_to_id;
@@ -2778,7 +2791,8 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
             }
         }
 
-        // BertNormalizer options
+        // Tokenizer normalizer options
+        ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_NFC, normalizer_opts.nfc, false);
         ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_LOWERCASE,     normalizer_opts.lowercase,     false);
         normalizer_opts.strip_accents = normalizer_opts.lowercase;
         ml.get_key(LLM_KV_TOKENIZER_NORMALIZER_STRIP_ACCENTS, normalizer_opts.strip_accents, false);

src/llama-vocab.h

@@ -83,6 +83,7 @@ struct llama_vocab {
     };
 
     struct normalizer_options {
+        bool nfc           = false;
         bool lowercase     = true;
         bool strip_accents = true;
         // TODO: clean_text, handle_chinese_chars

src/unicode-data.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <cstdint>
+#include <utility>
 #include <vector>
 #include <unordered_map>
 #include <unordered_set>
@@ -11,6 +12,10 @@ struct range_nfd {
     uint32_t nfd;
 };
 
+extern const std::unordered_map<uint32_t, uint8_t> unicode_combining_class;
+extern const std::unordered_map<uint32_t, std::pair<uint32_t, uint32_t>> unicode_decomposition;
+extern const std::unordered_map<uint64_t, uint32_t> unicode_composition;
+
 static const uint32_t MAX_CODEPOINTS = 0x110000;
 
 extern const std::initializer_list<std::pair<uint32_t, uint16_t>> unicode_ranges_flags;

src/unicode.cpp

@@ -1279,6 +1279,92 @@ std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & c
     return result;
 }
 
+static uint8_t unicode_canonical_class(uint32_t cpt) {
+    const auto it = unicode_combining_class.find(cpt);
+    return it == unicode_combining_class.end() ? 0 : it->second;
+}
+
+static void unicode_canonical_decompose(uint32_t cpt, std::vector<uint32_t> & result, std::vector<uint32_t> & pending) {
+    pending.push_back(cpt);
+    while (!pending.empty()) {
+        cpt = pending.back();
+        pending.pop_back();
+        // Hangul syllables use algorithmic decomposition (UAX #15).
+        if (cpt >= 0xAC00 && cpt < 0xAC00 + 11172) {
+            const uint32_t index = cpt - 0xAC00;
+            result.push_back(0x1100 + index / 588);
+            result.push_back(0x1161 + (index % 588) / 28);
+            if (index % 28) {
+                result.push_back(0x11A7 + index % 28);
+            }
+            continue;
+        }
+        const auto it = unicode_decomposition.find(cpt);
+        if (it == unicode_decomposition.end()) {
+            result.push_back(cpt);
+            continue;
+        }
+        if (it->second.second) {
+            pending.push_back(it->second.second);
+        }
+        pending.push_back(it->second.first);
+    }
+}
+
+static uint32_t unicode_canonical_compose(uint32_t first, uint32_t second) {
+    if (first >= 0x1100 && first < 0x1100 + 19 && second >= 0x1161 && second < 0x1161 + 21) {
+        return 0xAC00 + ((first - 0x1100) * 21 + second - 0x1161) * 28;
+    }
+    if (first >= 0xAC00 && first < 0xAC00 + 11172 && (first - 0xAC00) % 28 == 0 && second > 0x11A7 && second < 0x11A7 + 28) {
+        return first + second - 0x11A7;
+    }
+    const auto it = unicode_composition.find((uint64_t(first) << 32) | second);
+    return it == unicode_composition.end() ? 0 : it->second;
+}
+
+std::vector<uint32_t> unicode_cpts_normalize_nfc(const std::vector<uint32_t> & cpts) {
+    std::vector<uint32_t> decomposed;
+    decomposed.reserve(cpts.size());
+    std::vector<uint32_t> pending;
+    for (const uint32_t cpt : cpts) {
+        unicode_canonical_decompose(cpt, decomposed, pending);
+    }
+
+    // Sort each run of combining marks; equal classes retain their order.
+    for (size_t i = 0; i < decomposed.size();) {
+        if (unicode_canonical_class(decomposed[i]) == 0) {
+            ++i;
+        }
+        const size_t begin = i;
+        while (i < decomposed.size() && unicode_canonical_class(decomposed[i]) != 0) {
+            ++i;
+        }
+        std::stable_sort(decomposed.begin() + begin, decomposed.begin() + i, [](uint32_t a, uint32_t b) {
+            return unicode_canonical_class(a) < unicode_canonical_class(b);
+        });
+    }
+
+    std::vector<uint32_t> result;
+    result.reserve(decomposed.size());
+    size_t starter = 0;
+    uint8_t previous_class = 0;
+    for (const uint32_t cpt : decomposed) {
+        const uint8_t current_class = unicode_canonical_class(cpt);
+        const uint32_t composite = !result.empty() && (previous_class == 0 || previous_class < current_class)
+            ? unicode_canonical_compose(result[starter], cpt) : 0;
+        if (composite) {
+            result[starter] = composite;
+        } else {
+            if (current_class == 0) {
+                starter = result.size();
+            }
+            result.push_back(cpt);
+            previous_class = current_class;
+        }
+    }
+    return result;
+}
+
 std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8) {
     std::vector<uint32_t> result;
     result.reserve(utf8.size());

src/unicode.h

@@ -97,6 +97,7 @@ uint32_t    unicode_cpt_from_utf8(const std::string & utf8, size_t & offset);
 std::vector<uint32_t> unicode_cpts_from_utf8(const std::string & utf8);
 
 std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts);
+std::vector<uint32_t> unicode_cpts_normalize_nfc(const std::vector<uint32_t> & cpts);
 
 unicode_cpt_flags unicode_cpt_flags_from_cpt (uint32_t cpt);
 unicode_cpt_flags unicode_cpt_flags_from_utf8(const std::string & utf8);

tests/test-unicode.cpp

@@ -2,9 +2,31 @@
 
 #include <cstdio>
 #include <string>
+#include <utility>
 #include <vector>
 
 int main() {
+    const std::vector<std::pair<std::vector<uint32_t>, std::vector<uint32_t>>> normalization = {
+        {{}, {}},
+        {{0x65, 0x301}, {0xE9}},
+        {{0xE9}, {0xE9}},
+        {{0x212B}, {0xC5}},
+        {{0x61, 0x315, 0x300}, {0xE0, 0x315}},
+        {{0x61, 0x305, 0x301}, {0x61, 0x305, 0x301}},
+        {{0x301, 0x327}, {0x327, 0x301}},
+        {{0x1100, 0x1161, 0x11A8}, {0xAC01}},
+        {{0xAC01}, {0xAC01}},
+        {{0xFB01, 0xFF21}, {0xFB01, 0xFF21}},
+        {{0x958}, {0x915, 0x93C}},
+        {{0x1D15E}, {0x1D157, 0x1D165}},
+    };
+    for (const auto & test : normalization) {
+        if (unicode_cpts_normalize_nfc(test.first) != test.second || unicode_cpts_normalize_nfc(test.second) != test.second) {
+            fprintf(stderr, "unexpected NFC normalization\n");
+            return 1;
+        }
+    }
+
     const std::vector<std::string> regex_exprs = {
         "[~][A-Za-z]+| ?[\\p{S}]+|\\s+",
     };