gguf-py : detect nfc normalization (#30540)

gguf-py/gguf/vocab.py

@@ -52,6 +52,7 @@ class SpecialVocab:
     add_special_token: dict[str, bool]
     special_token_ids: dict[str, int]
     chat_template: str | Sequence[Mapping[str, str]] | None
+    normalizer_nfc: bool | None
     normalizer_lowercase: bool | None
     normalizer_strip_accents: bool | None
 
@@ -66,6 +67,7 @@ class SpecialVocab:
         self.load_merges = load_merges
         self.merges = []
         self.chat_template = None
+        self.normalizer_nfc = None
         self.normalizer_lowercase = None
         self.normalizer_strip_accents = None
         if special_token_types is not None:
@@ -106,6 +108,10 @@ class SpecialVocab:
             if not quiet:
                 logger.info(f'Setting chat_template to {self.chat_template}')
             gw.add_chat_template(self.chat_template)
+        if self.normalizer_nfc is not None:
+            if not quiet:
+                logger.info(f'Setting normalizer_nfc to {self.normalizer_nfc}')
+            gw.add_normalizer_nfc(self.normalizer_nfc)
         if self.normalizer_lowercase is not None:
             if not quiet:
                 logger.info(f'Setting normalizer_lowercase to {self.normalizer_lowercase}')
@@ -169,6 +175,8 @@ class SpecialVocab:
         normalizer_type = normalizer.get('type')
         if normalizer_type == 'Lowercase':
             self.normalizer_lowercase = True
+        elif normalizer_type == 'NFC':
+            self.normalizer_nfc = True
         elif normalizer_type == 'StripAccents':
             self.normalizer_strip_accents = True
         elif normalizer_type == 'BertNormalizer':