From 4c3c5810f87efe177949f1b85ae509d38a2a311f Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Mon, 19 Oct 2026 23:01:40 +0200 Subject: [PATCH] hybrid dna tokenizer fixes (#30825) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * vocab : support Carbon-14B HybridDNA tokenizer Carbon-14B auto-tags untagged DNA runs and uses N-padded tail tokens (tail=pad_n). Also strip whitespace inside tags and handle a stray like the HF tokenizer. * convert : fix special tokens and merges for older Carbon models Carbon-500M/3B/8B don't register the Qwen added tokens and ship no tokenizer.json, so take the token types, merges and special ids from the base tokenizer. * Update conversion/base.py Co-authored-by: Sigbjørn Skjæret * Update src/llama-vocab.cpp Co-authored-by: Sigbjørn Skjæret * Update src/llama-vocab.cpp Co-authored-by: Sigbjørn Skjæret * Update src/llama-vocab.cpp Co-authored-by: Sigbjørn Skjæret * Update conversion/base.py Co-authored-by: Sigbjørn Skjæret * Update conversion/base.py Co-authored-by: Sigbjørn Skjæret * Update conversion/base.py Co-authored-by: Sigbjørn Skjæret * address review comments * Update src/llama-vocab.cpp Co-authored-by: Sigbjørn Skjæret --------- Co-authored-by: Sigbjørn Skjæret --- conversion/base.py | 44 ++++++-- conversion/llama.py | 5 +- conversion/qwen.py | 4 + gguf-py/gguf/constants.py | 1 + gguf-py/gguf/gguf_writer.py | 3 + src/llama-arch.cpp | 1 + src/llama-arch.h | 1 + src/llama-vocab.cpp | 211 +++++++++++++++++++++++++++++------- src/llama-vocab.h | 1 + 9 files changed, 222 insertions(+), 49 deletions(-) diff --git a/conversion/base.py b/conversion/base.py index 14a52f01d..dec1d4322 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -10,6 +10,7 @@ import json import os import re import sys +import tempfile from enum import IntEnum from pathlib import Path from hashlib import sha256 @@ -2169,20 +2170,34 @@ class TextModel(ModelBase): special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) special_vocab.add_to_gguf(self.gguf_writer) + def _is_hybriddna(self) -> bool: + tokenizer_config_file = self.dir_model / "tokenizer_config.json" + if not tokenizer_config_file.is_file(): + return False + with open(tokenizer_config_file, "r", encoding="utf-8") as f: + return json.load(f).get("tokenizer_class") == "HybridDNATokenizer" + def _set_vocab_hybriddna(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) - vocab_size = self.hparams.get("vocab_size", len(tokenizer.vocab)) # ty: ignore[unresolved-attribute] - assert max(tokenizer.vocab.values()) < vocab_size # ty: ignore[unresolved-attribute] + vocab = tokenizer.get_vocab() + vocab_size = self.hparams.get("vocab_size", len(vocab)) + assert max(vocab.values()) < vocab_size - reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in tokenizer.vocab.items()} # ty: ignore[unresolved-attribute] - # k-mers can share text with a base-vocab BPE token (e.g. CCCCCC) and get - # dropped by get_vocab(); a reserved marker suffix (U+E000) keeps each - # k-mer's own id (llama.cpp strips it on detokenization) - for kmer in tokenizer.kmers: # ty: ignore[unresolved-attribute] + reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab.items()} + # k-mers can share text with a base-vocab BPE token (e.g. CCCCCC) and get dropped by get_vocab() + # a reserved marker suffix (U+E000) keeps each k-mer's own id (llama.cpp strips it on detokenization) + # tail="pad_n" (Carbon-14B) adds N-padded tail tokens ("CT" -> "CTNNNN") that get the same marker + # llama.cpp detects them to pad with N and strips the padding on detokenization, tail="char" is unsupported + tail = getattr(tokenizer, "tail", "pad_a") + if tokenizer.k != 6 or tail not in ("pad_a", "pad_n"): + raise NotImplementedError(f"HybridDNATokenizer with k={tokenizer.k}, tail={tail!r}") + for kmer in [*tokenizer.kmers, *getattr(tokenizer, "npad_tokens", [])]: reverse_vocab[tokenizer.dna_token_to_id[kmer]] = kmer + "\ue000" # ty: ignore[unresolved-attribute] - added_vocab = tokenizer.get_added_vocab() # ty: ignore[unresolved-attribute] - added_tokens_decoder = tokenizer.added_tokens_decoder # ty: ignore[unresolved-attribute] + # older HybridDNATokenizer versions (Carbon-500M/3B/8B) do not register the base tokenizer's added tokens (<|im_start|>, , ...) as their own + # but they are still encoded as special tokens: take their types from the base tokenizer + base_tokenizer = getattr(tokenizer, "_base_tokenizer", tokenizer) + added_tokens_decoder = {**base_tokenizer.added_tokens_decoder, **tokenizer.added_tokens_decoder} # ty: ignore[unresolved-attribute] tokens: list[str] = [] toktypes: list[int] = [] @@ -2192,7 +2207,7 @@ class TextModel(ModelBase): toktypes.append(gguf.TokenType.UNUSED) else: token: str = reverse_vocab[i] - if token in added_vocab: + if i in added_tokens_decoder: if added_tokens_decoder[i].special or self.does_token_look_special(token): toktypes.append(gguf.TokenType.CONTROL) else: @@ -2204,10 +2219,19 @@ class TextModel(ModelBase): tokpre = self.get_vocab_base_pre(tokenizer) self.gguf_writer.add_tokenizer_model("hybriddna") self.gguf_writer.add_tokenizer_pre(tokpre) + self.gguf_writer.add_dna_auto_tags(getattr(tokenizer, "auto_dna_tags", False)) self.gguf_writer.add_token_list(tokens) self.gguf_writer.add_token_types(toktypes) special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) + if not special_vocab.merges: + # Carbon-500M/3B/8B ship only tokenizer.py, no tokenizer.json + # take the merges and special token ids from the base BPE tokenizer, keeping the model's own chat template (if any) + with tempfile.TemporaryDirectory() as tmp: + base_tokenizer.save_pretrained(tmp) + base_special_vocab = gguf.SpecialVocab(Path(tmp), load_merges=True) + base_special_vocab.chat_template = special_vocab.chat_template + special_vocab = base_special_vocab special_vocab.add_to_gguf(self.gguf_writer) def _set_vocab_qwen(self): diff --git a/conversion/llama.py b/conversion/llama.py index 41d8c2309..b04f54d41 100644 --- a/conversion/llama.py +++ b/conversion/llama.py @@ -129,8 +129,9 @@ class LlamaModel(TextModel): tokenizer_config_json = json.load(f) if (add_prefix_space := tokenizer_config_json.get("add_prefix_space")) is not None: self.gguf_writer.add_add_space_prefix(add_prefix_space) - if tokenizer_config_json.get("tokenizer_class") == "HybridDNATokenizer": - return self._set_vocab_hybriddna() + + if self._is_hybriddna(): + return self._set_vocab_hybriddna() try: self._set_vocab_sentencepiece() diff --git a/conversion/qwen.py b/conversion/qwen.py index 0c0d0c1ed..e1013dd53 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -59,6 +59,10 @@ class Qwen2Model(TextModel): model_arch = gguf.MODEL_ARCH.QWEN2 def set_vocab(self): + # HuggingFaceBio/Carbon-14B: Qwen3 with the HybridDNATokenizer (Qwen3 BPE plus DNA 6-mers) + if self._is_hybriddna(): + self._set_vocab_hybriddna() + return try: self._set_vocab_sentencepiece() except FileNotFoundError: diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index e098efae4..8c19e22da 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -366,6 +366,7 @@ class Keys: ADD_BOS = "tokenizer.ggml.add_bos_token" ADD_EOS = "tokenizer.ggml.add_eos_token" ADD_SEP = "tokenizer.ggml.add_sep_token" + DNA_AUTO_TAGS = "tokenizer.ggml.dna_auto_tags" ADD_PREFIX = "tokenizer.ggml.add_space_prefix" REMOVE_EXTRA_WS = "tokenizer.ggml.remove_extra_whitespaces" PRECOMPILED_CHARSMAP = "tokenizer.ggml.precompiled_charsmap" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 93ec95eb6..7becd74af 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1314,6 +1314,9 @@ class GGUFWriter: def add_add_sep_token(self, value: bool) -> None: self.add_bool(Keys.Tokenizer.ADD_SEP, value) + def add_dna_auto_tags(self, value: bool) -> None: + self.add_bool(Keys.Tokenizer.DNA_AUTO_TAGS, value) + def add_add_space_prefix(self, value: bool) -> None: self.add_bool(Keys.Tokenizer.ADD_PREFIX, value) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index ffa35bdea..961011a2c 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -430,6 +430,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_TOKENIZER_ADD_BOS, "tokenizer.ggml.add_bos_token" }, { LLM_KV_TOKENIZER_ADD_EOS, "tokenizer.ggml.add_eos_token" }, { LLM_KV_TOKENIZER_ADD_SEP, "tokenizer.ggml.add_sep_token" }, + { LLM_KV_TOKENIZER_DNA_AUTO_TAGS, "tokenizer.ggml.dna_auto_tags" }, { LLM_KV_TOKENIZER_ADD_PREFIX, "tokenizer.ggml.add_space_prefix" }, { LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, "tokenizer.ggml.remove_extra_whitespaces" }, { LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, "tokenizer.ggml.precompiled_charsmap" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 14ce29ab3..3ea86c601 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -394,6 +394,7 @@ enum llm_kv { LLM_KV_TOKENIZER_ADD_BOS, LLM_KV_TOKENIZER_ADD_EOS, LLM_KV_TOKENIZER_ADD_SEP, + LLM_KV_TOKENIZER_DNA_AUTO_TAGS, LLM_KV_TOKENIZER_ADD_PREFIX, LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 05d2527dc..a237a4b04 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -1776,16 +1776,15 @@ private: // text at load static const std::string dna_kmer_marker = "\xee\x80\x80"; +// HybridDNATokenizer (HuggingFaceBio/Carbon-*): text inside ... is cut into fixed 6-mers, everything else goes through the base BPE +// Mirrors HybridDNATokenizer.encode, with optional tagging of untagged DNA (auto_dna_tags, Carbon-14B) and tail="pad_n" vocab, whose dedicated tokens pad a trailing partial k-mer with N instead of A ("CT" -> "CTNNNN") struct llm_tokenizer_hybriddna_session : llm_tokenizer_bpe_session { llm_tokenizer_hybriddna_session(const llama_vocab & vocab, const llm_tokenizer_bpe & tokenizer) : llm_tokenizer_bpe_session{vocab, tokenizer}, vocab{vocab} {} void tokenize(const std::string & text, std::vector & output) override { - static const std::string open_tag = ""; - static const std::string close_tag = ""; - - const auto dna_begin_id = vocab.text_to_token(open_tag); - const auto dna_end_id = vocab.text_to_token(close_tag); - const auto dna_oov_id = vocab.text_to_token(""); + dna_begin_id = vocab.text_to_token(open_tag); + dna_end_id = vocab.text_to_token(close_tag); + dna_oov_id = vocab.text_to_token(""); // Fall back to plain BPE if the DNA pieces aren't in the vocab. if (dna_begin_id == LLAMA_TOKEN_NULL || dna_end_id == LLAMA_TOKEN_NULL || dna_oov_id == LLAMA_TOKEN_NULL) { @@ -1793,63 +1792,187 @@ struct llm_tokenizer_hybriddna_session : llm_tokenizer_bpe_session { return; } - const size_t k = 6; - size_t pos = 0; + pad_char = vocab.text_to_token("A" + std::string(k - 1, 'N') + dna_kmer_marker) != LLAMA_TOKEN_NULL ? 'N' : 'A'; + // split on explicit tags as HybridDNATokenizer._split_by_dna_tags does; only the text outside them is auto-tagged + size_t pos = 0; while (pos < text.size()) { const size_t start = text.find(open_tag, pos); - if (start == std::string::npos) { - if (pos < text.size()) { - llm_tokenizer_bpe_session::tokenize(text.substr(pos), output); - } + const size_t end = text.find(close_tag, pos); + + if (start == std::string::npos && end == std::string::npos) { + tokenize_text(text.substr(pos), output); break; } - if (start > pos) { - llm_tokenizer_bpe_session::tokenize(text.substr(pos, start - pos), output); + if (start == std::string::npos || end < start) { + // as in _split_by_dna_tags, text before a without a matching is DNA (no begin token) + tokenize_dna(text.substr(pos, end - pos), false, true, output); + pos = end + close_tag.size(); + continue; } - output.push_back(dna_begin_id); + + tokenize_text(text.substr(pos, start - pos), output); const size_t content_start = start + open_tag.size(); - const size_t end = text.find(close_tag, content_start); - const size_t content_end = (end == std::string::npos) ? text.size() : end; - - emit_dna_kmers(text.substr(content_start, content_end - content_start), k, dna_oov_id, output); - if (end == std::string::npos) { + // an unterminated runs to the end of the text (no end token) + tokenize_dna(text.substr(content_start), true, false, output); break; } - output.push_back(dna_end_id); + tokenize_dna(text.substr(content_start, end - content_start), true, true, output); pos = end + close_tag.size(); } } private: - void emit_dna_kmers(const std::string & raw, size_t k, llama_token oov_id, std::vector & output) { - std::string seq = raw; - for (char & c : seq) { - if (c >= 'a' && c <= 'z') { - c = char(c - 32); + static bool is_letter(char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); } + static bool is_acgt (char c) { return c == 'A' || c == 'C' || c == 'G' || c == 'T'; } + static char to_upper (char c) { return c >= 'a' && c <= 'z' ? char(c - 32) : c; } + + // IUPAC nucleotide codes, either case (HybridDNATokenizer.IUPAC_RUN_RE) + static bool is_iupac(char c) { + c = to_upper(c); + return std::string("ACGTRYSWKMBDHVN").find(c) != std::string::npos; + } + + // HybridDNATokenizer._normalize_spaced_dna: join displays like "ACGTA CGTAC ..." when they hold at least twelve bases + // (at least six chunks of 1-5 uppercase ACGT separated by spaces or tabs, not touching a letter) + static std::string normalize_spaced_dna(const std::string & text) { + const auto is_blank = [](char c) { return c == ' ' || c == '\t'; }; + std::string out; + size_t p = 0; + while (p < text.size()) { + if (is_acgt(text[p]) && (p == 0 || !is_letter(text[p - 1]))) { + size_t match_end = std::string::npos; + size_t chunk = p; + for (int n_chunks = 1; ; n_chunks++) { + size_t chunk_end = chunk; + while (chunk_end < text.size() && chunk_end - chunk < 5 && is_acgt(text[chunk_end])) { + chunk_end++; + } + if (chunk_end == chunk) { + break; + } + if (n_chunks >= 6 && (chunk_end == text.size() || !is_letter(text[chunk_end]))) { + match_end = chunk_end; // the regex is greedy, so the last valid end wins + } + size_t next = chunk_end; + while (next < text.size() && is_blank(text[next])) { + next++; + } + if (next == chunk_end) { + break; + } + chunk = next; + } + if (match_end != std::string::npos) { + std::string seq; + for (size_t i = p; i < match_end; i++) { + if (!is_blank(text[i])) { + seq += text[i]; + } + } + out += seq.size() >= 12 ? seq : text.substr(p, match_end - p); + p = match_end; + continue; + } } + out += text[p++]; + } + return out; + } + + // text outside explicit tags: with auto_dna_tags, tag DNA runs as HybridDNATokenizer._auto_tag_text_segment does + // (pure ACGT runs of at least 6 bases, or runs of at least 12 with at most 20% IUPAC ambiguity codes) + void tokenize_text(const std::string & raw, std::vector & output) { + auto bpe = [&](const std::string & piece) { + if (!piece.empty()) { + llm_tokenizer_bpe_session::tokenize(piece, output); + } + }; + if (!vocab.get_dna_auto_tags()) { + bpe(raw); + return; } - // k-mers carry the reserved marker suffix; a non-ACGT k-mer simply - // isn't in the vocab and falls back to - auto kmer_token = [&](const std::string & kmer) { + const std::string text = normalize_spaced_dna(raw); + size_t prev = 0; + size_t i = 0; + while (i < text.size()) { + if (!is_iupac(text[i])) { + i++; + continue; + } + size_t j = i; + size_t n_ambiguous = 0; + for (; j < text.size() && is_iupac(text[j]); j++) { + n_ambiguous += !is_acgt(to_upper(text[j])); + } + const size_t len = j - i; + if ((n_ambiguous == 0 && len >= 6) || (n_ambiguous > 0 && len >= 12 && 5 * n_ambiguous <= len)) { + bpe(text.substr(prev, i - prev)); + tokenize_dna(text.substr(i, len), true, true, output); + prev = j; + } + i = j; + } + bpe(text.substr(prev)); + } + + // HybridDNATokenizer._process_dna_sequence: strip, uppercase, then one token per k bases + // a k-mer with anything but ACGT is , and a trailing partial k-mer is right-padded with pad_char + void tokenize_dna(const std::string & content, bool has_start, bool has_end, std::vector & output) { + if (has_start) { + output.push_back(dna_begin_id); + } + + auto cpts = unicode_cpts_from_utf8(content); + size_t first = 0; + size_t last = cpts.size(); + while (first < last && unicode_cpt_flags_from_cpt(cpts[first]).is_whitespace) { + first++; + } + while (last > first && unicode_cpt_flags_from_cpt(cpts[last - 1]).is_whitespace) { + last--; + } + + // k-mers carry the reserved marker suffix, so they never resolve to a base-vocab token + auto kmer_token = [&](size_t from, size_t to) { + std::string kmer; + for (size_t i = from; i < to; i++) { + const uint32_t c = cpts[i] < 128 ? (uint32_t) to_upper((char) cpts[i]) : cpts[i]; + if (c > 127 || !is_acgt((char) c)) { + return dna_oov_id; + } + kmer += (char) c; + } + kmer.append(k - kmer.size(), pad_char); const auto tok = vocab.text_to_token(kmer + dna_kmer_marker); - return tok != LLAMA_TOKEN_NULL ? tok : oov_id; + return tok != LLAMA_TOKEN_NULL ? tok : dna_oov_id; }; - size_t i = 0; - for (; i + k <= seq.size(); i += k) { - output.push_back(kmer_token(seq.substr(i, k))); + size_t i = first; + for (; i + k <= last; i += k) { + output.push_back(kmer_token(i, i + k)); } - if (i < seq.size()) { - std::string kmer = seq.substr(i); - kmer.append(k - kmer.size(), 'A'); - output.push_back(kmer_token(kmer)); + if (i < last) { + output.push_back(kmer_token(i, last)); + } + + if (has_end) { + output.push_back(dna_end_id); } } + const std::string open_tag = ""; + const std::string close_tag = ""; + const size_t k = 6; + + llama_token dna_begin_id = LLAMA_TOKEN_NULL; + llama_token dna_end_id = LLAMA_TOKEN_NULL; + llama_token dna_oov_id = LLAMA_TOKEN_NULL; + char pad_char = 'A'; + const llama_vocab & vocab; }; @@ -1957,6 +2080,7 @@ struct llama_vocab::impl { bool add_bos = false; bool add_eos = false; bool add_sep = false; + bool dna_auto_tags = false; // hybriddna: tag untagged DNA runs before tokenizing bool ignore_merges = false; bool clean_spaces = false; // clean_up_tokenization_spaces bool remove_extra_whitespaces = false; @@ -2689,6 +2813,12 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { if (text.size() > dna_kmer_marker.size() && text.compare(text.size() - dna_kmer_marker.size(), dna_kmer_marker.size(), dna_kmer_marker) == 0) { text.erase(text.size() - dna_kmer_marker.size()); + // tail="pad_n": "CTNNNN" detokenizes to its real bases "CT" + // full k-mers are ACGT only, so only N-padded tail tokens end in N + const size_t last_base = text.find_last_not_of('N'); + if (last_base != std::string::npos) { + text.erase(last_base + 1); + } } } } @@ -2781,6 +2911,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { if (ml.get_key(LLM_KV_TOKENIZER_ADD_SEP, temp, false)) { add_sep = temp; } + if (ml.get_key(LLM_KV_TOKENIZER_DNA_AUTO_TAGS, temp, false)) { + dna_auto_tags = temp; + } // workaround for Gemma 4 // ref: https://github.com/ggml-org/llama.cpp/pull/21500 @@ -4312,6 +4445,10 @@ bool llama_vocab::get_add_sep() const { return pimpl->add_sep; } +bool llama_vocab::get_dna_auto_tags() const { + return pimpl->dna_auto_tags; +} + bool llama_vocab::get_ignore_merges() const { return pimpl->ignore_merges; } diff --git a/src/llama-vocab.h b/src/llama-vocab.h index 898a23ae6..415acd22b 100644 --- a/src/llama-vocab.h +++ b/src/llama-vocab.h @@ -149,6 +149,7 @@ struct llama_vocab { bool get_add_bos () const; bool get_add_eos () const; bool get_add_sep () const; + bool get_dna_auto_tags () const; bool get_ignore_merges () const; bool get_clean_spaces () const; bool get_remove_extra_whitespaces () const;