diff --git a/conversion/__init__.py b/conversion/__init__.py index 5f252b490..35e73b41c 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -286,6 +286,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "SeedOssForCausalLM": "olmo", "SmallThinkerForCausalLM": "smallthinker", "SmolLM3ForCausalLM": "llama", + "SopranoTextModel": "soprano", "Spark2_5ForCausalLM": "spark2_5", "SolarOpenForCausalLM": "glm", "StableLMEpochForCausalLM": "stablelm", @@ -380,6 +381,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "Qwen3ASRForConditionalGeneration": "qwen3vl", "Qwen3OmniMoeForConditionalGeneration": "qwen3vl", "PocketTTSModel": "pockettts", + "SopranoModel": "soprano", "Qwen3TTSForConditionalGeneration": "qwen3tts", "Qwen3VLForConditionalGeneration": "qwen3vl", "Qwen3VLMoeForConditionalGeneration": "qwen3vl", diff --git a/conversion/soprano.py b/conversion/soprano.py new file mode 100644 index 000000000..cf4c959c6 --- /dev/null +++ b/conversion/soprano.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import torch + +from .base import ModelBase, MmprojModel, gguf, logger +from .qwen import Qwen3Model + + +def _is_soprano_checkpoint(dir_model: Path) -> bool: + if not (dir_model / "decoder.pth").is_file() or not (dir_model / "config.json").is_file(): + return False + with open(dir_model / "config.json", encoding="utf-8") as f: + config = json.load(f) + if config.get("model_type") != "qwen3" or config.get("bos_token_id") != 3 or config.get("eos_token_id") != 3: + return False + if not (dir_model / "tokenizer.json").is_file(): + return False + with open(dir_model / "tokenizer.json", encoding="utf-8") as f: + added = {t["id"]: t["content"] for t in json.load(f).get("added_tokens", [])} + return [added.get(i) for i in range(4)] == ["[UNK]", "[TEXT]", "[START]", "[STOP]"] + + +@ModelBase.register_hparams_loader(_is_soprano_checkpoint) +def _load_soprano_hparams(dir_model: Path) -> dict[str, Any]: + logger.info("gguf: detected Soprano checkpoint") + hparams = ModelBase.load_hparams(dir_model, False, guess=False) + # the text model stays Qwen3, the mmproj holds the decoder + hparams["text_config"] = {"architectures": ["SopranoTextModel"]} + hparams["architectures"] = ["SopranoModel"] + return hparams + + +@ModelBase.register("SopranoTextModel") +class SopranoTextModel(Qwen3Model): + model_arch = gguf.MODEL_ARCH.QWEN3 + + def get_vocab_base_pre(self, tokenizer) -> str: + return "soprano" + + def set_vocab(self): + super().set_vocab() + self.gguf_writer.add_unk_token_id(0) + + +@ModelBase.register("SopranoModel") +@ModelBase.example("ekwek/Soprano-80M", "ekwek/Soprano-1.1-80M") +class SopranoModel(MmprojModel): + has_vision_encoder = False + has_audio_encoder = False + + def _load_decoder(self) -> dict[str, torch.Tensor]: + state = torch.load(self.dir_model / "decoder.pth", map_location="cpu", mmap=True, weights_only=True) + return state.get("state_dict", state) + + def get_audio_config(self): + state = self._load_decoder() + n_embd, n_inp, _ = state["decoder.embed.weight"].shape + n_ff = state["decoder.convnext.0.pwconv1.weight"].shape[0] + n_layer = sum(1 for name in state if name.endswith(".pwconv1.weight")) + if n_inp != self.n_embd_text: + raise ValueError(f"Soprano decoder input dim {n_inp} does not match the backbone dim {self.n_embd_text}") + return {"num_hidden_layers": n_layer, "hidden_size": n_embd, "intermediate_size": n_ff} + + def set_gguf_parameters(self): + self.gguf_writer.add_file_type(self.ftype) + self.gguf_writer.add_clip_has_gen_audio_encoder(True) + self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.SOPRANO) + self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text) + self.gguf_writer.add_gen_audio_embedding_length(self.hparams["hidden_size"]) + self.gguf_writer.add_gen_audio_feed_forward_length(self.hparams["intermediate_size"]) + self.gguf_writer.add_gen_audio_block_count(self.hparams["num_hidden_layers"]) + self.gguf_writer.add_gen_audio_head_count(1) + self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-6) + + def get_tensors(self): + yield from self._load_decoder().items() + + def tensor_force_quant(self, name, new_name, bid, n_dims): + if "dwconv.weight" in new_name or (name == "decoder.embed.weight" and n_dims == 3): + return gguf.GGMLQuantizationType.F16 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + def modify_tensors(self, data_torch, name, bid): + if name == "head.istft.window": + # the ISTFT in mtmd builds its own window + if not torch.equal(data_torch.float(), torch.hann_window(data_torch.shape[0])): + raise ValueError("Soprano requires a periodic Hann window") + return + if name == "decoder.embed.weight" and data_torch.shape[-1] == 1: + data_torch = data_torch.squeeze(-1) + + T = gguf.MODEL_TENSOR + if name.startswith("decoder.convnext."): + _, _, blk, key, *suffix = name.split(".") + block_map = { + "dwconv": T.A_GEN_WAV_UP_DWCONV, + "norm": T.A_GEN_WAV_UP_NORM, + "pwconv1": T.A_GEN_WAV_UP_PW1, + "pwconv2": T.A_GEN_WAV_UP_PW2, + "gamma": T.A_GEN_WAV_UP_GAMMA, + } + yield self.format_tensor_name(block_map[key], int(blk), suffix="".join("." + s for s in suffix)), data_torch + return + + tensor_map = { + "decoder.embed": T.A_GEN_WAV_INPUT, + "decoder.norm": T.A_GEN_WAV_NORM, + "decoder.final_layer_norm": T.A_GEN_WAV_OUTPUT_NORM, + "head.out": T.A_GEN_WAV_OUTPUT, + } + key, suffix = name.rsplit(".", 1) + yield self.format_tensor_name(tensor_map[key], suffix="." + suffix), data_torch diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index b8cfa3a85..f941106f2 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -1272,6 +1272,11 @@ class MODEL_TENSOR(IntEnum): A_GEN_WAV_SEANET_RES_CONV1 = auto() A_GEN_WAV_SEANET_RES_CONV2 = auto() A_GEN_WAV_SEANET_SCALE_CONV = auto() # strided upsample convtr + # soprano: Vocos decoder, the ConvNeXt blocks reuse A_GEN_WAV_UP_* + A_GEN_WAV_INPUT = auto() + A_GEN_WAV_NORM = auto() + A_GEN_WAV_OUTPUT_NORM = auto() + A_GEN_WAV_OUTPUT = auto() A_MMPROJ = auto() A_MMPROJ_FC = auto() A_MM_NORM_PRE = auto() @@ -2084,6 +2089,10 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV1: "a.gen.wav.seanet.blk.{bid}.res_conv1", MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV2: "a.gen.wav.seanet.blk.{bid}.res_conv2", MODEL_TENSOR.A_GEN_WAV_SEANET_SCALE_CONV: "a.gen.wav.seanet.blk.{bid}.scale_conv", + MODEL_TENSOR.A_GEN_WAV_INPUT: "a.gen.wav.input", + MODEL_TENSOR.A_GEN_WAV_NORM: "a.gen.wav.norm", + MODEL_TENSOR.A_GEN_WAV_OUTPUT_NORM: "a.gen.wav.output_norm", + MODEL_TENSOR.A_GEN_WAV_OUTPUT: "a.gen.wav.output", MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}", MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc", MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre", @@ -2462,6 +2471,10 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV1, MODEL_TENSOR.A_GEN_WAV_SEANET_RES_CONV2, MODEL_TENSOR.A_GEN_WAV_SEANET_SCALE_CONV, + MODEL_TENSOR.A_GEN_WAV_INPUT, + MODEL_TENSOR.A_GEN_WAV_NORM, + MODEL_TENSOR.A_GEN_WAV_OUTPUT_NORM, + MODEL_TENSOR.A_GEN_WAV_OUTPUT, MODEL_TENSOR.A_ENC_CONV_NORM_MEAN, MODEL_TENSOR.A_ENC_CONV_NORM_VAR, MODEL_TENSOR.A_ENC_MEL_FILTERS, @@ -6447,6 +6460,7 @@ class VisionProjectorType: QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor POCKETTTS_SPKENC = "pockettts_spkenc" # audio: mimi encoder as voice-prompt encoder POCKETTTS_GEN = "pockettts_gen" # audio generation: flow-matching decoder + mimi decoder + SOPRANO = "soprano" # audio generation: Vocos decoder HUNYUANVL = "hunyuanvl" PARAKEET = "parakeet" # audio MINIMAXM3 = "minimax_m3" diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index f72fbd221..05334e1fc 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -572,6 +572,10 @@ struct llm_tokenizer_bpe : llm_tokenizer { "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?(?:\\p{L}|\\p{M}|\\u200C|\\u200D)+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", }; break; + case LLAMA_VOCAB_PRE_TYPE_SOPRANO: + regex_exprs = { "\\p{N}", "\\s+|[\\p{L}\\p{N}_]+|[^\\p{L}\\p{N}_\\s]+" }; + byte_encode = false; + break; case LLAMA_VOCAB_PRE_TYPE_WHITESPACE: // whitespace pre-tokenizer (jinaai/jina-embeddings-v2-base-zh) regex_exprs = { @@ -647,6 +651,22 @@ struct llm_tokenizer_bpe_session { replace_all(normalized_text, "[" + zwnj, "["); replace_all(normalized_text, "<" + zwnj, "<"); word_collection = unicode_regex_split(normalized_text, tokenizer.regex_exprs, tokenizer.byte_encode); + } else if (tok_pre == LLAMA_VOCAB_PRE_TYPE_SOPRANO) { + std::string normalized_text; + bool space = false; + for (uint32_t cpt : unicode_cpts_from_utf8(text)) { + if (unicode_cpt_flags_from_cpt(cpt).is_whitespace) { + if (!space) { + normalized_text += ' '; + } + space = true; + } else { + // Unicode full lowercase expands LATIN CAPITAL LETTER I WITH DOT ABOVE. + normalized_text += cpt == 0x0130 ? "i\xcc\x87" : unicode_cpt_to_utf8(unicode_tolower(cpt)); + space = false; + } + } + word_collection = unicode_regex_split(normalized_text, tokenizer.regex_exprs, tokenizer.byte_encode); } else { word_collection = unicode_regex_split(text, tokenizer.regex_exprs, tokenizer.byte_encode); } @@ -769,7 +789,9 @@ struct llm_tokenizer_bpe_session { const std::string str = std::string(symbol.text, symbol.n); const auto token = vocab.text_to_token(str); - if (token == LLAMA_TOKEN_NULL) { + if (token == LLAMA_TOKEN_NULL && tok_pre == LLAMA_VOCAB_PRE_TYPE_SOPRANO) { + output.push_back(vocab.token_unk()); + } else if (token == LLAMA_TOKEN_NULL) { for (auto j = str.begin(); j != str.end(); ++j) { llama_token token_multibyte = LLAMA_TOKEN_NULL; if (tokenizer.byte_encode) { @@ -2730,6 +2752,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { pre_type = LLAMA_VOCAB_PRE_TYPE_NEMOTRON_3_5_EMBD; ignore_merges = true; add_bos = true; + } else if ( + tokenizer_pre == "soprano") { + pre_type = LLAMA_VOCAB_PRE_TYPE_SOPRANO; } else { throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str())); } @@ -3283,6 +3308,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } } + if (pre_type == LLAMA_VOCAB_PRE_TYPE_SOPRANO) { + special_eog_ids.insert(special_eos_id); + } + // sanity checks if (special_eos_id != LLAMA_TOKEN_NULL && special_eog_ids.count(special_eos_id) == 0) { special_eog_ids.insert(special_eos_id); @@ -4096,6 +4125,9 @@ int32_t llama_vocab::impl::token_to_piece(llama_token token, char * buf, int32_t return _try_copy(token_text.data(), token_text.size()); } if (attr & LLAMA_TOKEN_ATTR_NORMAL) { + if (pre_type == LLAMA_VOCAB_PRE_TYPE_SOPRANO) { + return _try_copy(token_text.data(), token_text.size()); + } if (escape_whitespaces) { // SPM-style BPE: tokens contain ▁ for spaces std::string result = token_text; diff --git a/src/llama-vocab.h b/src/llama-vocab.h index c6f3ac06d..14a90fbdb 100644 --- a/src/llama-vocab.h +++ b/src/llama-vocab.h @@ -73,6 +73,7 @@ enum llama_vocab_pre_type { LLAMA_VOCAB_PRE_TYPE_FLINT = 62, LLAMA_VOCAB_PRE_TYPE_DOMYN_SMALL = 63, LLAMA_VOCAB_PRE_TYPE_NEMOTRON_3_5_EMBD = 64, + LLAMA_VOCAB_PRE_TYPE_SOPRANO = 65, }; // HF Metaspace pre-tokenizer prepend_scheme diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index bd7fb7356..8ca4ae3a4 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -67,6 +67,7 @@ add_library(mtmd models/pockettts-seanet.cpp models/pockettts-spkenc.cpp models/pockettts-gen.cpp + models/soprano.cpp models/step3vl.cpp models/siglip.cpp models/whisper-enc.cpp diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 5583a7c57..4ffb7056a 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -300,6 +300,12 @@ #define TN_A_GEN_WAV_SEANET_RES_CONV2 "a.gen.wav.seanet.blk.%d.res_conv2.%s" #define TN_A_GEN_WAV_SEANET_SCALE_CONV "a.gen.wav.seanet.blk.%d.scale_conv.%s" +// soprano: Vocos decoder, the ConvNeXt blocks use TN_A_GEN_WAV_UP_* +#define TN_A_GEN_WAV_INPUT "a.gen.wav.input.%s" +#define TN_A_GEN_WAV_NORM "a.gen.wav.norm.%s" +#define TN_A_GEN_WAV_OUTPUT_NORM "a.gen.wav.output_norm.%s" +#define TN_A_GEN_WAV_OUTPUT "a.gen.wav.output.%s" + // cogvlm #define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s" #define TN_MM_H_TO_4H "mm.up.%s" @@ -547,6 +553,7 @@ enum projector_type { PROJECTOR_TYPE_QWEN3TTS_GEN, PROJECTOR_TYPE_POCKETTTS_SPKENC, PROJECTOR_TYPE_POCKETTTS_GEN, + PROJECTOR_TYPE_SOPRANO, PROJECTOR_TYPE_MUSE_GLIMMER, PROJECTOR_TYPE_SENSENOVA_U1, PROJECTOR_TYPE_COHERE2V, @@ -623,6 +630,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_QWEN3TTS_GEN, "qwen3tts_gen"}, { PROJECTOR_TYPE_POCKETTTS_SPKENC, "pockettts_spkenc"}, { PROJECTOR_TYPE_POCKETTTS_GEN, "pockettts_gen"}, + { PROJECTOR_TYPE_SOPRANO, "soprano"}, { PROJECTOR_TYPE_MUSE_GLIMMER, "muse-glimmer"}, { PROJECTOR_TYPE_SENSENOVA_U1, "sensenova_u1"}, { PROJECTOR_TYPE_COHERE2V, "cohere2v"}, diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 5caa93b42..350c8fc9d 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -579,6 +579,18 @@ struct clip_code2wav { ggml_tensor * dac_post_conv_b = nullptr; }; +struct clip_vocos { + ggml_tensor * input_w = nullptr; + ggml_tensor * input_b = nullptr; + ggml_tensor * norm_w = nullptr; + ggml_tensor * norm_b = nullptr; + ggml_tensor * output_norm_w = nullptr; + ggml_tensor * output_norm_b = nullptr; + ggml_tensor * output_w = nullptr; + ggml_tensor * output_b = nullptr; + std::vector blocks; +}; + struct clip_model { clip_modality modality = CLIP_MODALITY_VISION; projector_type proj_type = PROJECTOR_TYPE_MLP; @@ -845,6 +857,9 @@ struct clip_model { ggml_tensor * gen_upsample_w = nullptr; // depthwise convtr, frame rate -> encoder frame rate std::vector gen_tfm_layers; // mimi decoder_transformer + // soprano: Vocos decoder, hidden states -> STFT coefficients + clip_vocos vocos; + // cogvlm ggml_tensor * mm_post_fc_norm_w = nullptr; ggml_tensor * mm_post_fc_norm_b = nullptr; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 5737400df..950d8fde0 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1168,6 +1168,11 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const const float top_p = params ? params->top_p : 1.0f; builder = std::make_unique(ctx, img, gen_process, top_k, top_p); } break; + case PROJECTOR_TYPE_SOPRANO: + { + const int n_frames = params && params->feats ? (int) (params->feats->size() / ctx->model.hparams.projection_dim) : 2; + builder = std::make_unique(ctx, img, n_frames); + } break; case PROJECTOR_TYPE_YOUTUVL: { builder = std::make_unique(ctx, img); @@ -2073,6 +2078,12 @@ struct clip_model_loader { hparams.flow_n_step = 1; hparams.gen_eos_threshold = -4.0f; } break; + case PROJECTOR_TYPE_SOPRANO: + { + // SopranoDecoder defaults, n_fft comes from the head tensor + hparams.audio_sample_rate = 32000; + hparams.audio_hop_len = 512; + } break; case PROJECTOR_TYPE_PADDLEOCR: { hparams.n_merge = 2; @@ -2452,6 +2463,7 @@ struct clip_model_loader { model.proj_type != PROJECTOR_TYPE_GEMMA3NV && model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC && model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN && + model.proj_type != PROJECTOR_TYPE_SOPRANO && model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO); // layers @@ -3391,6 +3403,31 @@ struct clip_model_loader { c2w.dac_post_conv_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_CONV, "bias")); } } break; + case PROJECTOR_TYPE_SOPRANO: + { + auto & v = model.vocos; + v.input_w = get_tensor(string_format(TN_A_GEN_WAV_INPUT, "weight")); + v.input_b = get_tensor(string_format(TN_A_GEN_WAV_INPUT, "bias")); + v.norm_w = get_tensor(string_format(TN_A_GEN_WAV_NORM, "weight")); + v.norm_b = get_tensor(string_format(TN_A_GEN_WAV_NORM, "bias")); + v.output_norm_w = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT_NORM, "weight")); + v.output_norm_b = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT_NORM, "bias")); + v.output_w = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT, "weight")); + v.output_b = get_tensor(string_format(TN_A_GEN_WAV_OUTPUT, "bias")); + v.blocks.resize(hparams.n_layer); + for (int il = 0; il < hparams.n_layer; ++il) { + auto & b = v.blocks[il]; + b.dwconv_w = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "weight")); + b.dwconv_b = get_tensor(string_format(TN_A_GEN_WAV_UP_DWCONV, il, "bias")); + b.norm_w = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "weight")); + b.norm_b = get_tensor(string_format(TN_A_GEN_WAV_UP_NORM, il, "bias")); + b.pw1_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "weight")); + b.pw1_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW1, il, "bias")); + b.pw2_w = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "weight")); + b.pw2_b = get_tensor(string_format(TN_A_GEN_WAV_UP_PW2, il, "bias")); + b.gamma = get_tensor(string_format(TN_A_GEN_WAV_UP_GAMMA, il)); + } + } break; case PROJECTOR_TYPE_VOXTRAL: case PROJECTOR_TYPE_VOXTRAL_RT: { @@ -5679,6 +5716,10 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { set_input_f32("inp_noise", noise); } } break; + case PROJECTOR_TYPE_SOPRANO: + { + set_input_f32("inp_feats", *params->feats); + } break; case PROJECTOR_TYPE_GEMMA4V: case PROJECTOR_TYPE_GEMMA4UV: { @@ -6329,6 +6370,11 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { out_audio.resize(ggml_nelements(audio)); ggml_backend_tensor_get(audio, out_audio.data(), 0, ggml_nbytes(audio)); + if (model.proj_type == PROJECTOR_TYPE_SOPRANO) { + std::vector spectrum = std::move(out_audio); + clip_graph_soprano::decode_spectrum(model, spectrum, out_audio); + } + // drop the tail audio that comes from the code-0 rear padding const int64_t n_codes = params->codes ? model.gen_code_head_w->ne[2] + 1 : 0; const int64_t n_frames_w = hparams.wav_tfm_swa; @@ -6536,6 +6582,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.spk_proj_w->ne[1]; case PROJECTOR_TYPE_POCKETTTS_GEN: return ctx->model.gen_input_lin_w->ne[1]; + case PROJECTOR_TYPE_SOPRANO: + return ctx->model.hparams.projection_dim; case PROJECTOR_TYPE_PARAKEET: return ctx->model.mm_1_w->ne[1]; default: diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 55e525cdc..8ae2a5b25 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -420,6 +420,15 @@ std::vector list_c2w_state_slots(const clip_hparams & hparams, c // same, for the streaming mimi decoder (pocket-tts GEN_WAV) std::vector list_pockettts_state_slots(const clip_hparams & hparams, const clip_model & model); +struct clip_graph_soprano : clip_graph { + clip_graph_soprano(clip_ctx * ctx, const clip_image_f32 & img, int n_frames) : clip_graph(ctx, img), n_frames(n_frames) {} + ggml_cgraph * build() override; + static void decode_spectrum(const clip_model & model, const std::vector & spectrum, std::vector & pcm); + // SopranoDecoder default + static constexpr int upscale = 4; + int n_frames; +}; + struct clip_graph_kimik25 : clip_graph { clip_graph_kimik25(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/models/soprano.cpp b/tools/mtmd/models/soprano.cpp new file mode 100644 index 000000000..bd16c6912 --- /dev/null +++ b/tools/mtmd/models/soprano.cpp @@ -0,0 +1,78 @@ +#include "models.h" +#include "../mtmd-audio.h" + +// Soprano Vocos: normalized Qwen3 hidden states -> STFT coefficients. +ggml_cgraph * clip_graph_soprano::build() { + const auto & v = model.vocos; + const int n_inp = hparams.projection_dim; + const int n_up = upscale * (n_frames - 1) + 1; + ggml_tensor * cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_inp, n_frames); + ggml_set_name(cur, "inp_feats"); + ggml_set_input(cur); + + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cur = ggml_interpolate(ctx0, cur, n_up, n_inp, 1, 1, GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS); + if (ggml_n_dims(v.input_w) == 3) { + const int k = v.input_w->ne[0]; + cur = ggml_conv_1d(ctx0, v.input_w, cur, 1, k / 2, 1); + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + } else { + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cur = build_mm(v.input_w, cur); + } + cur = ggml_add(ctx0, cur, v.input_b); + cur = build_norm(cur, v.norm_w, v.norm_b, NORM_TYPE_NORMAL, eps, -1); + + for (size_t il = 0; il < v.blocks.size(); ++il) { + const auto & b = v.blocks[il]; + const int k = b.dwconv_w->ne[0]; + ggml_tensor * residual = cur; + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cur = ggml_conv_1d_dw(ctx0, b.dwconv_w, cur, 1, k / 2, 1); + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cur = ggml_add(ctx0, cur, b.dwconv_b); + cur = build_norm(cur, b.norm_w, b.norm_b, NORM_TYPE_NORMAL, eps, il); + cur = ggml_add(ctx0, build_mm(b.pw1_w, cur), b.pw1_b); + cur = ggml_gelu_erf(ctx0, cur); + cur = ggml_add(ctx0, build_mm(b.pw2_w, cur), b.pw2_b); + cur = ggml_add(ctx0, residual, ggml_mul(ctx0, cur, b.gamma)); + cb(cur, "vocos_block", il); + } + cur = build_norm(cur, v.output_norm_w, v.output_norm_b, NORM_TYPE_NORMAL, eps, -1); + cur = ggml_add(ctx0, build_mm(v.output_w, cur), v.output_b); + cb(cur, "vocos_head", -1); + + // the head holds n_fft/2+1 log-magnitudes followed by n_fft/2+1 phases + const int n_bins = v.output_w->ne[1] / 2; + ggml_tensor * mag = ggml_view_2d(ctx0, cur, n_bins, n_up, cur->nb[1], 0); + ggml_tensor * phase = ggml_view_2d(ctx0, cur, n_bins, n_up, cur->nb[1], n_bins * sizeof(float)); + mag = ggml_clamp(ctx0, ggml_exp(ctx0, ggml_cont(ctx0, mag)), 0.0f, 100.0f); + phase = ggml_cont(ctx0, phase); + ggml_tensor * re = ggml_mul(ctx0, mag, ggml_cos(ctx0, phase)); + ggml_tensor * im = ggml_mul(ctx0, mag, ggml_sin(ctx0, phase)); + re = ggml_reshape_3d(ctx0, re, 1, n_bins, n_up); + im = ggml_reshape_3d(ctx0, im, 1, n_bins, n_up); + cur = ggml_concat(ctx0, re, im, 0); + ggml_set_name(cur, "out_audio"); + ggml_set_output(cur); + ggml_build_forward_expand(gf, cur); + return gf; +} + +void clip_graph_soprano::decode_spectrum(const clip_model & model, const std::vector & spectrum, std::vector & pcm) { + const int n_frame_vals = model.vocos.output_w->ne[1]; + const int n_fft = n_frame_vals - 2; + const int hop_length = model.hparams.audio_hop_len; + const size_t n_frames = spectrum.size() / n_frame_vals; + mtmd_audio_streaming_istft istft(n_fft, hop_length); + pcm.clear(); + for (size_t i = 0; i < n_frames; ++i) { + auto frame = istft.process_frame(spectrum.data() + i * n_frame_vals); + pcm.insert(pcm.end(), frame.begin(), frame.end()); + } + auto tail = istft.flush(); + pcm.insert(pcm.end(), tail.begin(), tail.end()); + // the shared ISTFT removes (n_fft - hop) / 2 samples, torch.istft(center=True) removes n_fft / 2 + pcm.erase(pcm.begin(), pcm.begin() + hop_length / 2); + pcm.resize((n_frames - 1) * hop_length); +} diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index d8299ba44..44bee60bd 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -1076,12 +1076,199 @@ private: bool wav_header_sent = false; }; +// Soprano uses the hidden state that predicts each non-EOS token, including the prompt's last state. +class soprano_gen_audio_pipeline : public mtmd_gen_audio_pipeline { +public: + using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; + + void reset() override { + prompt.clear(); + features.clear(); + h_next.clear(); + audio.clear(); + wav.clear(); + pos = 0; + prompt_pos = 0; + decoded = false; + sent = false; + } + + int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { + reset(); + if (inp->speaker_ref || !inp->prompt || inp->prompt_len == 0 || inp->prompt_len > INT32_MAX || + inp->seq_id < 0 || (uint32_t) inp->seq_id >= llama_n_seq_max(lctx)) { + LOG_ERR("mtmd_helper_gen_audio: soprano requires text, without a speaker reference\n"); + return 1; + } + seq_id = inp->seq_id; + out_type = inp->out_type; + stream = inp->stream; + const llama_token text = find_special_token(vocab, "[TEXT]"); + const llama_token start = find_special_token(vocab, "[START]"); + const llama_token stop = find_special_token(vocab, "[STOP]"); + if (text == LLAMA_TOKEN_NULL || start == LLAMA_TOKEN_NULL || stop == LLAMA_TOKEN_NULL) { + LOG_ERR("mtmd_helper_gen_audio: incompatible soprano vocabulary\n"); + return 1; + } + const int required = llama_tokenize(vocab, inp->prompt, inp->prompt_len, nullptr, 0, false, false); + if (required >= 0 || (uint32_t) (-required + 3) >= llama_n_ctx_seq(lctx)) { + LOG_ERR("mtmd_helper_gen_audio: soprano prompt does not fit in the context\n"); + return 1; + } + const int n = -required; + prompt.resize(n + 3); + prompt[0] = stop; + prompt[1] = text; + if (llama_tokenize(vocab, inp->prompt, inp->prompt_len, prompt.data() + 2, n, false, false) != n) { + return 1; + } + prompt.back() = start; + if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) { + LOG_ERR("mtmd_helper_gen_audio: cannot reset soprano sequence\n"); + return 1; + } + if (!init_gen_state(inp->seed)) { + return 1; + } + return 0; + } + + int32_t step_prompt(int32_t n_batch) override { + if (n_batch <= 0 || prompt.empty()) { + return -1; + } + const int n = std::min(n_batch, (int) prompt.size() - prompt_pos); + if (n == 0) { + return 0; + } + if (!decode(prompt.data() + prompt_pos, n)) { + return -1; + } + prompt_pos += n; + return (int) prompt.size() - prompt_pos; + } + + int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { + *h_state_out = nullptr; + *out_stop = false; + if (sampled < 0 || sampled >= llama_vocab_n_tokens(vocab) || !h_state_in || prompt_pos != (int) prompt.size() || prompt.empty()) { + return 1; + } + if (llama_vocab_is_eog(vocab, sampled) || (uint32_t) pos >= llama_n_ctx_seq(lctx)) { + *out_stop = true; + return 0; + } + features.insert(features.end(), h_state_in, h_state_in + n_embd); + if (!decode(&sampled, 1)) { + features.resize(features.size() - n_embd); + return 1; + } + const float * h = llama_get_embeddings_ith(lctx, -1); + if (!h) { + return 1; + } + h_next.assign(h, h + n_embd); + *h_state_out = h_next.data(); + return 0; + } + + int32_t get_output(int32_t * sample_rate, const char ** data, size_t * data_len, int64_t * n_samples) override { + *sample_rate = info.sample_rate; + *data_len = 0; + if (n_samples) { *n_samples = 0; } + if (stream) { + // the decoder needs the whole utterance, so the audio comes out in one chunk after flush() + if (!decoded || sent) { + return 0; + } + sent = true; + } else if (!decode_audio()) { + return 1; + } + if (n_samples) { *n_samples = audio.size(); } + if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { + *data = (const char *) audio.data(); + *data_len = audio.size() * sizeof(float); + } else { + wav.clear(); + if (!write_wav16(wav, audio, info.sample_rate)) { + return 1; + } + *data = wav.data(); + *data_len = wav.size(); + } + return 0; + } + + // the whole utterance is decoded at once + int32_t flush() override { + return decode_audio() ? 0 : 1; + } + +private: + bool decode_audio() { + if (decoded) { + return true; + } + if (features.size() < 2 * (size_t) n_embd) { + LOG_ERR("mtmd_helper_gen_audio: soprano needs at least two generated frames\n"); + return false; + } + mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; + inp.feats = features.data(); + inp.n_feats = features.size(); + mtmd_gen_out out{}; + if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { + return false; + } + audio.assign(out.audio, out.audio + out.n_samples); + decoded = true; + return true; + } + + bool decode(llama_token * tokens, int n) { + llama_batch batch = llama_batch_init(n, 0, 1); + batch.n_tokens = n; + for (int i = 0; i < n; ++i) { + batch.token[i] = tokens[i]; + batch.pos[i] = pos + i; + batch.n_seq_id[i] = 1; + batch.seq_id[i][0] = seq_id; + batch.logits[i] = i == n - 1; + } + const int ret = llama_decode(lctx, batch); + llama_batch_free(batch); + if (ret != 0) { + LOG_ERR("mtmd_helper_gen_audio: soprano backbone decode failed\n"); + return false; + } + pos += n; + return true; + } + + llama_seq_id seq_id = 0; + llama_pos pos = 0; + int prompt_pos = 0; + std::vector prompt; + std::vector features; + std::vector h_next; + std::vector audio; + std::vector wav; + mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; + bool stream = false; + bool decoded = false; + bool sent = false; +}; + static std::unique_ptr make_pipeline(llama_context * lctx, mtmd_context * mctx) { switch (mtmd_gen_audio_get_info(mctx).type) { case MTMD_GEN_AUDIO_TYPE_QWEN3TTS: return std::unique_ptr(new qwen3tts_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_POCKETTTS: return std::unique_ptr(new pockettts_gen_audio_pipeline(lctx, mctx)); + case MTMD_GEN_AUDIO_TYPE_SOPRANO: + return std::unique_ptr(new soprano_gen_audio_pipeline(lctx, mctx)); default: return nullptr; } diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 7c825c0dc..e18f639de 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -624,7 +624,9 @@ struct mtmd_context { ctx_v = res.ctx_v; ctx_a = res.ctx_a; ctx_gen_a = res.ctx_gen_a; - if (!ctx_v && !ctx_a) { + // soprano has no speaker encoder, its mmproj only holds the decoder + const bool gen_only = ctx_gen_a && clip_get_projector_type(ctx_gen_a) == PROJECTOR_TYPE_SOPRANO; + if (!ctx_v && !ctx_a && !gen_only) { throw std::runtime_error(string_format("Failed to load CLIP model from %s\n", mmproj_fname)); } @@ -641,7 +643,8 @@ struct mtmd_context { // since we already validate n_embd of vision and audio mmproj, // we can safely assume that they are the same - int n_embd_clip = clip_n_mmproj_embd(ctx_v ? ctx_v : ctx_a); + // a gen-only mmproj is validated below + int n_embd_clip = ctx_v || ctx_a ? clip_n_mmproj_embd(ctx_v ? ctx_v : ctx_a) : n_embd_text; if (n_embd_text > 0 && n_embd_text != n_embd_clip) { throw std::runtime_error(string_format( "mismatch between text model (n_embd = %d) and mmproj (n_embd = %d)\n" @@ -2082,6 +2085,10 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { info.type = MTMD_GEN_AUDIO_TYPE_POCKETTTS; info.sample_rate = 24000; break; + case PROJECTOR_TYPE_SOPRANO: + info.type = MTMD_GEN_AUDIO_TYPE_SOPRANO; + info.sample_rate = 32000; + break; default: info.type = MTMD_GEN_AUDIO_TYPE_NONE; break; @@ -2109,6 +2116,9 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { inp.top_p = 1.0f; inp.temp = 0.7f; break; + case PROJECTOR_TYPE_SOPRANO: + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; + break; default: break; } @@ -2170,6 +2180,14 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * *out = {}; + if (clip_get_projector_type(ctx_clip) == PROJECTOR_TYPE_SOPRANO) { + const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip); + if (inp->type != MTMD_GEN_PROCESS_TYPE_GEN_WAV || !inp->feats || inp->n_feats < 2 * n_embd || inp->n_feats % n_embd != 0 || inp->codes) { + LOG_ERR("%s: soprano requires at least 2 frames of features, without codes\n", __func__); + return 1; + } + } + if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE) { const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip); @@ -3046,6 +3064,9 @@ mtmd_memory_usage mtmd_get_memory_usage(const char * mmproj_fname, if (ctx->ctx_a) { merge(ctx->ctx_a); } + if (ctx->ctx_gen_a) { + merge(ctx->ctx_gen_a); + } mtmd_memory_usage res; res.backend_mem_usage = std::move(total_mem); res.image_max_tokens = ctx->ctx_v ? clip_get_image_max_tokens(ctx->ctx_v) : -1; diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index 5af73a0f4..0a6bf4c4c 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -427,6 +427,7 @@ enum mtmd_gen_audio_type { MTMD_GEN_AUDIO_TYPE_NONE, // not supported MTMD_GEN_AUDIO_TYPE_QWEN3TTS, MTMD_GEN_AUDIO_TYPE_POCKETTTS, + MTMD_GEN_AUDIO_TYPE_SOPRANO, }; struct mtmd_gen_audio_info { diff --git a/tools/tts/README.md b/tools/tts/README.md index 55b99932c..41397b82e 100644 --- a/tools/tts/README.md +++ b/tools/tts/README.md @@ -59,3 +59,28 @@ The [upstream repository](https://huggingface.co/kyutai/pocket-tts) holds one co python convert_hf_to_gguf.py path/to/pocket-tts/languages/english --outfile pocket-tts.gguf python convert_hf_to_gguf.py path/to/pocket-tts/languages/english --mmproj --outfile mmproj-pocket-tts.gguf ``` + +## Soprano + +Available params: +- `--tts-speaker-file` is not supported; the model uses a fixed voice +- Note: `lang` is not used. Supply normalized English text + +Example usage: + +```sh +llama-tts -m soprano.gguf \ + -mm mmproj-soprano.gguf \ + -p "Hello world" \ + --temp 0 \ + --output out.wav +``` + +**Note for GGUF conversion:** + +Both [Soprano-80M](https://huggingface.co/ekwek/Soprano-80M) and [Soprano-1.1-80M](https://huggingface.co/ekwek/Soprano-1.1-80M) are supported. Convert both files from the same model directory, including `decoder.pth`: + +```sh +python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --outtype f32 --outfile soprano.gguf +python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --mmproj --outtype f16 --outfile mmproj-soprano.gguf +```