From 683bbc0dce0295dd3f82c0244841e0c07ee1a239 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?=
 <sigbjorn.skjaeret@huggingface.co>
Date: Wed, 14 Oct 2026 20:08:09 +0200
Subject: [PATCH] gguf-py : detect nfc normalization (#30540)

---
 gguf-py/gguf/vocab.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py
index 31036bbe5..78e90eddc 100644
--- a/gguf-py/gguf/vocab.py
+++ b/gguf-py/gguf/vocab.py
@@ -52,6 +52,7 @@ class SpecialVocab:
     add_special_token: dict[str, bool]
     special_token_ids: dict[str, int]
     chat_template: str | Sequence[Mapping[str, str]] | None
+    normalizer_nfc: bool | None
     normalizer_lowercase: bool | None
     normalizer_strip_accents: bool | None
 
@@ -66,6 +67,7 @@ class SpecialVocab:
         self.load_merges = load_merges
         self.merges = []
         self.chat_template = None
+        self.normalizer_nfc = None
         self.normalizer_lowercase = None
         self.normalizer_strip_accents = None
         if special_token_types is not None:
@@ -106,6 +108,10 @@ class SpecialVocab:
             if not quiet:
                 logger.info(f'Setting chat_template to {self.chat_template}')
             gw.add_chat_template(self.chat_template)
+        if self.normalizer_nfc is not None:
+            if not quiet:
+                logger.info(f'Setting normalizer_nfc to {self.normalizer_nfc}')
+            gw.add_normalizer_nfc(self.normalizer_nfc)
         if self.normalizer_lowercase is not None:
             if not quiet:
                 logger.info(f'Setting normalizer_lowercase to {self.normalizer_lowercase}')
@@ -169,6 +175,8 @@ class SpecialVocab:
         normalizer_type = normalizer.get('type')
         if normalizer_type == 'Lowercase':
             self.normalizer_lowercase = True
+        elif normalizer_type == 'NFC':
+            self.normalizer_nfc = True
         elif normalizer_type == 'StripAccents':
             self.normalizer_strip_accents = True
         elif normalizer_type == 'BertNormalizer':