From e197cef4554d466e0becac20e453a355daba8e5d Mon Sep 17 00:00:00 2001
From: tkran2 <157059462+tkran2@users.noreply.github.com>
Date: Thu, 22 Oct 2026 02:06:47 -0500
Subject: [PATCH] vocab : use Tekken pre-tokenizer for pixtral rather than the
 Llama 3 one (#30960)

---
 src/llama-vocab.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index 05c0f96a2..d25c4ceec 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -2438,7 +2438,6 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                     tokenizer_pre == "llama-bpe"||
                     tokenizer_pre == "falcon3"  ||
                     tokenizer_pre == "falcon-h1" ||
-                    tokenizer_pre == "pixtral"  ||
                     tokenizer_pre == "midm-2.0" ||
                     tokenizer_pre == "lfm2"     ||
                     tokenizer_pre == "jina-v5-nano") {
@@ -2566,7 +2565,8 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
                 tokenizer_pre == "jais") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_JAIS;
             } else if (
-                tokenizer_pre == "tekken") {
+                tokenizer_pre == "tekken" ||
+                tokenizer_pre == "pixtral") {
                 pre_type = LLAMA_VOCAB_PRE_TYPE_TEKKEN;
                 clean_spaces = false;
                 ignore_merges = true;