From d4f1b18b3a1522eec48decbe9ff7dc66eb60a9f4 Mon Sep 17 00:00:00 2001 From: Tarek Dakhran Date: Wed, 28 Oct 2026 12:23:06 +0100 Subject: [PATCH] mtmd: enable video input support for LFM3 model family (#31283) * mtmd : add LFM3-VL and LFM3-Omni video input Co-authored-by: Xuan Son Nguyen Assisted-by: Claude Opus 5.5 * mtmd : reserve the soundtrack command to avoid a GCC 11 warning --------- Co-authored-by: Xuan Son Nguyen --- conversion/lfm3.py | 19 +++ tools/mtmd/clip.cpp | 1 + tools/mtmd/models/lfm3vl.cpp | 15 +- tools/mtmd/mtmd-helper.cpp | 283 ++++++++++++++++++++++++++++--- tools/mtmd/mtmd-helper.h | 1 + tools/mtmd/mtmd-image.cpp | 49 +++++- tools/mtmd/mtmd-image.h | 12 +- tools/mtmd/mtmd.cpp | 186 +++++++++++++++++++- tools/mtmd/mtmd.h | 27 ++- tools/server/README.md | 1 + tools/server/server-decision.cpp | 12 +- 11 files changed, 560 insertions(+), 46 deletions(-) diff --git a/conversion/lfm3.py b/conversion/lfm3.py index e8b6d98b0..435edd3de 100644 --- a/conversion/lfm3.py +++ b/conversion/lfm3.py @@ -203,6 +203,24 @@ class LFM3VLVisionMixin(MmprojModel): if resize_algo is None: raise ValueError(f"unsupported resample: {resample}") self.gguf_writer.add_vision_image_resize_algo(resize_algo) + self._check_video_config() + + # mtmd_get_video_ts_style and the lfm3vl video frame sizing hardcode these, conversion fails on a mismatch + _video = {"fps": 2.0, "min_frames": 4, "temporal_patch_size": 2, "size": {"shortest_edge": 4096, "longest_edge": 25165824}} + _video_max_frames = 768 + + def _check_video_config(self): + import json + video = self.preprocessor_config.get("video_processor") + path = self.dir_model / "video_preprocessor_config.json" + if video is None and path.is_file(): + with open(path, encoding="utf-8") as f: + video = json.load(f) + if video is None: + return + for k, v in {**self._video, "max_frames": self._video_max_frames}.items(): + if video.get(k) != v: + raise ValueError(f"unsupported video {k}={video.get(k)!r}, expected {v!r}") def tensor_force_quant(self, name, new_name, bid, n_dims): if "patch_embd" in new_name: @@ -562,6 +580,7 @@ class LFM3OmniMmprojModel(LFM3VLVisionMixin, LFM3AudioMmprojModel): has_audio_encoder = True _audio_sampling = {"audio_temperature": 0.4, "audio_repetition_penalty": 1.2, "audio_repetition_window": 64} + _video_max_frames = 384 # the audio side is the LFM3-Audio model under the HF-native names _audio_renames = ( diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 26aa05e25..1f6498efd 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -7099,6 +7099,7 @@ int clip_model_n_temporal_merge(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_QWEN3VL: + case PROJECTOR_TYPE_LFM3VL: case PROJECTOR_TYPE_NEMOTRON_OMNI: case PROJECTOR_TYPE_LING3VL: return 2; diff --git a/tools/mtmd/models/lfm3vl.cpp b/tools/mtmd/models/lfm3vl.cpp index e8e455b11..e37390976 100644 --- a/tools/mtmd/models/lfm3vl.cpp +++ b/tools/mtmd/models/lfm3vl.cpp @@ -4,7 +4,7 @@ ggml_cgraph * clip_graph_lfm3vl::build() { GGML_ASSERT(model.patch_embeddings_0 != nullptr); GGML_ASSERT(model.lfm3vl_embedding_norm_w != nullptr); GGML_ASSERT(model.class_embedding == nullptr); - GGML_ASSERT(n_batch == 1); + GGML_ASSERT(n_batch == 1 || n_batch == 2); const int batch_size = 1; const int n_pos = n_patches; @@ -15,12 +15,13 @@ ggml_cgraph * clip_graph_lfm3vl::build() { // 2D rope: pairs (j, j+n_dims); first d_head/4 pairs rotate by y, next d_head/4 by x int mrope_sections[4] = {d_head/4, d_head/4, 0, 0}; - ggml_tensor * inp_raw = build_inp_raw(); - - // one patchifier kernel per temporal slice: a still image fills both slices, so both kernels apply to it - ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1); - if (model.patch_embeddings_1) { - inp = ggml_add(ctx0, inp, ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1)); + // one patchifier kernel per temporal slice: a still image fills both slices, video frames are merged in pairs + ggml_tensor * inp; + if (model.patch_embeddings_1 == nullptr) { + GGML_ASSERT(n_batch == 1); + inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, build_inp_raw(), patch_size, patch_size, 0, 0, 1, 1); + } else { + inp = build_inp_temporal_merge(); } inp = ggml_reshape_2d(ctx0, inp, n_pos, n_embd); inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index 28f1efdca..29812219c 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -16,6 +16,7 @@ #include "hash/hash.h" #include +#include #include #include @@ -444,7 +445,8 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(const mtmd_context * *out_text = text; // heap-allocated by read_next; freed automatically by mtmd return ret; }); - return {result, video_ctx}; + mtmd_bitmap_set_is_video(result, true); + return {result, video_ctx}; } #else if (!result) { @@ -600,32 +602,63 @@ struct mtmd_helper_video { int32_t timestamp_interval_ms = 5000; // emit a timestamp text every N ms (0 = disabled) float next_timestamp_ms = 0.0f; // next elapsed-ms threshold at which to emit + // apply mtmd_get_video_config(); false for still images decoded via ffmpeg + bool use_video_config = false; + mtmd_video_template tmpl = MTMD_VIDEO_TEMPLATE_DEFAULT; + + int32_t target_frames = 0; // exact frame count to emit (0 = no limit) + int32_t out_width = 0; // frame size from mtmd_get_video_config (0 = source size) + int32_t out_height = 0; + + // source frames to emit, as picked by the reference processors (empty = use the fps filter) + std::vector frame_indices; + size_t next_index = 0; + int32_t src_frame = 0; // source frames read from ffmpeg so far + double src_fps = 0.0; + std::vector frame_buf; - std::string pending_text; // text queued to be returned before the next frame - bool start_emitted = false; + std::string pending_text; // text queued to be returned before the next frame + mtmd_bitmap * pending_frame = nullptr; // frame queued to be returned after its group timestamp + + // soundtrack: after each group of frames comes the audio from its first frame to the next group's + double video_start_s = 0.0; + int32_t n_src_frames = 0; + std::vector soundtrack; + mtmd_bitmap * pending_audio = nullptr; + bool start_emitted = false; bool is_buf_input() const { return !input_buf.empty(); } + // number of frames per timestamp group (0 = timestamp every timestamp_interval_ms) + int32_t ts_per_n_frames() const { + return tmpl == MTMD_VIDEO_TEMPLATE_LFM3 || tmpl == MTMD_VIDEO_TEMPLATE_LFM3_OMNI ? 2 : 0; + } + bool probe(float fps_target_arg) { - const char * input_arg = is_buf_input() ? "pipe:0" : path.c_str(); - const char * cmd[] = { + // cache: makes stdin seekable; without it ffprobe reports width/height 0 + // for mp4 files whose moov atom sits at the end + const char * input_arg = is_buf_input() ? "cache:pipe:0" : path.c_str(); + std::vector cmd = { ffprobe_bin.c_str(), "-v", "quiet", - "-show_entries", "stream=width,height,r_frame_rate,nb_frames,duration", + "-show_entries", "stream=width,height,r_frame_rate,nb_frames,duration,start_time", "-select_streams", "v:0", "-of", "default=noprint_wrappers=1", - input_arg, - nullptr, }; + if (is_buf_input()) { + // remove the 64KB read-ahead limit of cache:, or else ffprobe cannot reach a moov atom at end of file + cmd.insert(cmd.end(), {"-read_ahead_limit", "-1"}); + } + cmd.insert(cmd.end(), {input_arg, nullptr}); LOG_DBG("%s: launching:", __func__); for (size_t i = 0; cmd[i]; i++) { LOG_DBG(" %s", cmd[i]); } LOG_DBG("\n"); subprocess_handle probe_sp; - if (subprocess_create(cmd, + if (subprocess_create(cmd.data(), subprocess_option_search_user_path | subprocess_option_inherit_environment, &probe_sp.proc) != 0) { LOG_ERR("%s: failed to launch ffprobe\n", __func__); @@ -665,6 +698,8 @@ struct mtmd_helper_video { n_frames_orig = atoi(val); } else if (strcmp(key, "duration") == 0 && strcmp(val, "N/A") != 0) { duration = (float)atof(val); + } else if (strcmp(key, "start_time") == 0 && strcmp(val, "N/A") != 0) { + video_start_s = atof(val); } } @@ -678,17 +713,118 @@ struct mtmd_helper_video { duration = (float)n_frames_orig / orig_fps; } - fps_target = fps_target_arg > 0.0f ? fps_target_arg : orig_fps; + mtmd_video_config cfg{MTMD_VIDEO_TEMPLATE_DEFAULT, 0.0f, 0, 0, 0}; + if (use_video_config) { + cfg = mtmd_get_video_config(mctx, width, height, n_frames_orig, orig_fps); + tmpl = cfg.tmpl; + } + + // the model's recommended sampling rate wins over the generic default + fps_target = cfg.fps > 0.0f ? cfg.fps + : fps_target_arg > 0.0f ? fps_target_arg : orig_fps; + + if (cfg.n_frames > 0 && duration > 0.0f) { + const int32_t n = cfg.n_frames; + target_frames = n; + fps_target = (float) n / duration; + + // same frames as the reference processors: round(linspace(0, n_frames - 1, n)) + const double step = n > 1 ? (double) (n_frames_orig - 1) / (n - 1) : 0.0; + frame_indices.resize(n); + for (int32_t i = 0; i < n; i++) { + frame_indices[i] = (int32_t) std::nearbyint(i * step); + } + src_fps = orig_fps; + n_src_frames = n_frames_orig; + + if (cfg.frame_width > 0 && cfg.frame_height > 0 && + (cfg.frame_width != (int32_t) width || cfg.frame_height != (int32_t) height)) { + out_width = cfg.frame_width; + out_height = cfg.frame_height; + } + } + + if (out_width > 0 && out_height > 0) { + width = (uint32_t) out_width; + height = (uint32_t) out_height; + } + info.width = width; info.height = height; info.fps = fps_target; - LOG_DBG("%s: %ux%u fps=%.2f duration=%.2fs n_frames=%d\n", - __func__, width, height, fps_target, duration, info.n_frames); - info.n_frames = duration > 0.0f ? (int32_t)(duration * fps_target + 0.5f) : -1; + LOG_DBG("%s: %ux%u fps=%.2f duration=%.2fs n_frames=%d target_frames=%d\n", + __func__, width, height, fps_target, duration, n_frames_orig, target_frames); + info.n_frames = target_frames > 0 ? target_frames + : duration > 0.0f ? (int32_t)(duration * fps_target + 0.5f) : -1; frame_buf.resize((size_t)width * height * 3); return true; } + // the soundtrack over the span of the video stream: from its first frame for n_frames / fps seconds, mono at the + // rate of the audio encoder; empty if the file has no audio + void load_soundtrack() { + const int rate = mtmd_get_audio_sample_rate(mctx); + if (tmpl != MTMD_VIDEO_TEMPLATE_LFM3_OMNI || frame_indices.empty() || rate <= 0) { + return; + } + const std::string rate_str = std::to_string(rate); + std::vector cmd = {ffmpeg_bin.c_str(), "-nostdin"}; + cmd.reserve(20); // GCC 11 -Wstringop-overflow false positive on the inserts below otherwise + if (is_buf_input()) { + cmd.insert(cmd.end(), {"-read_ahead_limit", "-1"}); + } + cmd.insert(cmd.end(), {"-i", is_buf_input() ? "cache:pipe:0" : path.c_str(), "-map", "0:a:0", "-vn", "-ac", "1", + "-ar", rate_str.c_str(), "-f", "f32le", "pipe:1", "-loglevel", "error", nullptr}); + subprocess_handle audio_sp; + if (subprocess_create(cmd.data(), subprocess_option_search_user_path | subprocess_option_inherit_environment, + &audio_sp.proc) != 0) { + LOG_WRN("%s: failed to launch ffmpeg for the soundtrack\n", __func__); + return; + } + audio_sp.created = true; + audio_sp.alive = true; + if (is_buf_input()) { + audio_sp.start_feeder(input_buf); + } + std::vector pcm; + float block[4096]; + size_t n; + while ((n = fread(block, sizeof(float), 4096, audio_sp.stdout_pipe())) > 0) { + pcm.insert(pcm.end(), block, block + n); + } + audio_sp.stop(); + + const size_t start = (size_t) std::max(0.0, std::nearbyint(video_start_s * rate)); + const size_t len = (size_t) std::nearbyint((double) n_src_frames / src_fps * rate); + if (start >= pcm.size()) { + LOG_WRN("%s: the video has no soundtrack\n", __func__); + return; + } + soundtrack.assign(pcm.begin() + start, pcm.begin() + std::min(pcm.size(), start + len)); + } + + // the soundtrack of the group of frames that ends with frame idx, nullptr if the soundtrack has ended + mtmd_bitmap * soundtrack_slice(int32_t idx) const { + const int32_t ts_per_n_frames = this->ts_per_n_frames(); + const int32_t group = idx / ts_per_n_frames; + const int32_t n_groups = ((int32_t) frame_indices.size() + ts_per_n_frames - 1) / ts_per_n_frames; + const int rate = mtmd_get_audio_sample_rate(mctx); + auto group_start = [&](int32_t g) { + const double t = frame_indices[g * ts_per_n_frames] / src_fps; + return std::min(soundtrack.size(), (size_t) std::nearbyint(t * rate)); + }; + const size_t begin = group == 0 ? 0 : group_start(group); + const size_t end = group == n_groups - 1 ? soundtrack.size() : group_start(group + 1); + if (begin >= end) { + return nullptr; + } + mtmd_bitmap * slice = mtmd_bitmap_init_from_audio(end - begin, soundtrack.data() + begin); + if (!id.empty()) { + mtmd_bitmap_set_id(slice, (id + "+audio" + std::to_string(group)).c_str()); + } + return slice; + } + bool start_ffmpeg(float seek_seconds) { char seek_buf[64]; char fps_buf[64]; @@ -721,10 +857,35 @@ struct mtmd_helper_video { cmd.push_back(seek_buf); } - if (fps_target > 0.0f) { - snprintf(fps_buf, sizeof(fps_buf), "fps=%.6f", fps_target); + std::string vf; + if (!frame_indices.empty()) { + // frames are picked by index in read_next_frame(); B=0 C=0.5 bicubic on RGB is closest to Pillow + if (out_width > 0 && out_height > 0) { + char scale_buf[160]; + snprintf(scale_buf, sizeof(scale_buf), + "format=rgb24,scale=%d:%d:flags=bicubic+accurate_rnd+full_chroma_int:param0=0:param1=0.5", + out_width, out_height); + vf = scale_buf; + } + } else if (fps_target > 0.0f) { + if (out_width > 0 && out_height > 0) { + snprintf(fps_buf, sizeof(fps_buf), "fps=%.6f,scale=%d:%d", fps_target, out_width, out_height); + } else { + snprintf(fps_buf, sizeof(fps_buf), "fps=%.6f", fps_target); + } + vf = fps_buf; + } + if (!vf.empty()) { cmd.push_back("-vf"); - cmd.push_back(fps_buf); + cmd.push_back(vf.c_str()); + } + + char frames_buf[16]; + const int32_t n_out = !frame_indices.empty() ? frame_indices.back() + 1 : target_frames; + if (n_out > 0) { + snprintf(frames_buf, sizeof(frames_buf), "%d", n_out); + cmd.push_back("-frames:v"); + cmd.push_back(frames_buf); } cmd.push_back("-f"); @@ -763,9 +924,8 @@ struct mtmd_helper_video { sp.stop(); } - mtmd_bitmap * read_next_frame() { - if (!sp.alive) return nullptr; - + // read one raw frame from ffmpeg into frame_buf + bool read_raw_frame() { FILE * fp = sp.stdout_pipe(); const size_t frame_size = (size_t)info.width * info.height * 3; LOG_DBG("%s: reading frame %d, expecting %zu bytes (%ux%u)\n", @@ -779,14 +939,37 @@ struct mtmd_helper_video { LOG_DBG("%s: fread returned 0 after %zu/%zu bytes (ferror=%d)\n", __func__, total_read, frame_size, ferror(fp)); sp.alive = false; - return nullptr; + return false; } total_read += n; } + return true; + } + + mtmd_bitmap * read_next_frame() { + if (!sp.alive) return nullptr; + + if (!frame_indices.empty()) { + if (next_index >= frame_indices.size()) { + return nullptr; + } + // skip to the wanted source frame; a repeated index reuses the frame in frame_buf + const int32_t want = frame_indices[next_index]; + while (src_frame <= want) { + if (!read_raw_frame()) { + return nullptr; + } + src_frame++; + } + next_index++; + } else if (!read_raw_frame()) { + return nullptr; + } LOG_DBG("%s: frame %d read OK\n", __func__, current_frame); mtmd_bitmap * frame = mtmd_bitmap_init(info.width, info.height, frame_buf.data()); mtmd_bitmap_set_mergeable(frame, true); + mtmd_bitmap_set_is_video(frame, true); if (!id.empty()) { // each frame gets a unique id in the form of {hash}+{frame}, so that it can be identified in cache std::string frame_id = id + "+" + std::to_string(current_frame); @@ -806,6 +989,18 @@ struct mtmd_helper_video { return *out_text ? 0 : -2; } + if (pending_frame) { + *out_bitmap = pending_frame; + pending_frame = nullptr; + return 0; + } + + if (pending_audio) { + *out_bitmap = pending_audio; + pending_audio = nullptr; + return 0; + } + LOG_DBG("%s: proc_alive=%d start_emitted=%d current_frame=%d\n", __func__, (int)sp.alive, (int)start_emitted, current_frame); @@ -815,14 +1010,46 @@ struct mtmd_helper_video { if (!start_emitted) { start_emitted = true; - if (!prompt_start.empty()) { + // model templates have no prompt prefix, the video begin/end tokens are added by mtmd_tokenize + if (tmpl == MTMD_VIDEO_TEMPLATE_DEFAULT && !prompt_start.empty()) { *out_text = mtmd_strdup(prompt_start); return *out_text ? 0 : -2; } } mtmd_bitmap * frame = read_next_frame(); - if (!frame) return -1; + if (!frame) { + return -1; + } + + const int32_t ts_per_n_frames = this->ts_per_n_frames(); + if (ts_per_n_frames > 0) { + // emit the group timestamp before the group's first frame; the value is the + // midpoint time of the group, matching HF processors that stamp each temporal group + const int32_t idx = current_frame - 1; // read_next_frame() already incremented + // the soundtrack of a group follows its last frame + const bool group_ends = idx % ts_per_n_frames == ts_per_n_frames - 1 || idx + 1 == (int32_t) frame_indices.size(); + if (!soundtrack.empty() && !frame_indices.empty() && group_ends) { + pending_audio = soundtrack_slice(idx); + } + if (idx % ts_per_n_frames == 0) { + char ts_buf[64]; + double ts_s = ((double)idx + 0.5*(double)(ts_per_n_frames - 1)) / info.fps; + if (!frame_indices.empty()) { + // stamp from the source frames; a short last group repeats its last frame + const size_t first = idx; + const size_t last = std::min(first + ts_per_n_frames - 1, frame_indices.size() - 1); + ts_s = (frame_indices[first] / src_fps + frame_indices[last] / src_fps) / 2.0; + } + snprintf(ts_buf, sizeof(ts_buf), "<%.1f seconds>", ts_s); + pending_frame = frame; + *out_text = mtmd_strdup(ts_buf); + return *out_text ? 0 : -2; + } + *out_bitmap = frame; + return 0; + } + *out_bitmap = frame; if (timestamp_interval_ms > 0) { @@ -861,6 +1088,7 @@ mtmd_helper_video_init_params mtmd_helper_video_init_params_default() { /* fps_target */ 4.0f, /* ffmpeg_bin_dir */ nullptr, /* timestamp_interval_ms */ 5000, + /* no_soundtrack */ false, }; } @@ -916,6 +1144,7 @@ static mtmd_bitmap * decode_webp_with_ffmpeg(const mtmd_context * mctx, const un mtmd_bitmap * frame = vctx.read_next_frame(); if (frame) { mtmd_bitmap_set_mergeable(frame, false); + mtmd_bitmap_set_is_video(frame, false); } return frame; } @@ -933,12 +1162,16 @@ mtmd_helper_video * mtmd_helper_video_init( ctx->ffmpeg_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffmpeg"); ctx->ffprobe_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffprobe"); ctx->timestamp_interval_ms = params.timestamp_interval_ms; + ctx->use_video_config = true; if (!ctx->probe(params.fps_target)) { LOG_ERR("%s: ffprobe failed for '%s' (is ffprobe in PATH?)\n", __func__, path); delete ctx; return nullptr; } + if (!params.no_soundtrack) { + ctx->load_soundtrack(); + } if (!ctx->start_ffmpeg(0.0f)) { LOG_ERR("%s: failed to start ffmpeg for '%s' (is ffmpeg in PATH?)\n", __func__, path); @@ -968,12 +1201,16 @@ mtmd_helper_video * mtmd_helper_video_init_from_buf( ctx->ffmpeg_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffmpeg"); ctx->ffprobe_bin = video_resolve_bin(params.ffmpeg_bin_dir, "ffprobe"); ctx->timestamp_interval_ms = params.timestamp_interval_ms; + ctx->use_video_config = true; if (!ctx->probe(params.fps_target)) { LOG_ERR("%s: ffprobe failed on buffer (is ffprobe in PATH?)\n", __func__); delete ctx; return nullptr; } + if (!params.no_soundtrack) { + ctx->load_soundtrack(); + } if (!ctx->start_ffmpeg(0.0f)) { LOG_ERR("%s: failed to start ffmpeg on buffer (is ffmpeg in PATH?)\n", __func__); @@ -996,6 +1233,8 @@ void mtmd_helper_video_free(mtmd_helper_video * ctx) { #ifdef MTMD_VIDEO if (!ctx) return; ctx->stop_ffmpeg(); + mtmd_bitmap_free(ctx->pending_frame); + mtmd_bitmap_free(ctx->pending_audio); delete ctx; #else GGML_UNUSED(ctx); diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index c9f7ad75d..99ae1738d 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -27,6 +27,7 @@ struct mtmd_helper_video_init_params { float fps_target; // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms + bool no_soundtrack; // frames only, also for models that hear the soundtrack of a video (LFM3-Omni) // TODO @ngxson : allow "placeholder" bitmap output for counting tokens }; diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index ffa483b25..2750bc3fc 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -823,17 +823,26 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_i // // rounds before the pixel checks (an edge may round to 0) and computes in double, as transformers does -clip_image_size mtmd_image_preprocessor_lfm3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels) { - const double h = height; - const double w = width; +clip_image_size mtmd_image_preprocessor_lfm3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels, + int n_frames, int temporal_factor) { + double h = height; + double w = width; + // the video variant first upscales edges below factor + if (temporal_factor > 1 && (h < factor || w < factor)) { + const double scale = std::max(factor / h, factor / w); + h = static_cast(h * scale); + w = static_cast(w * scale); + } + + const int64_t t_bar = static_cast(std::nearbyint(static_cast(n_frames) / temporal_factor)) * temporal_factor; int h_bar = static_cast(std::nearbyint(h / factor)) * factor; int w_bar = static_cast(std::nearbyint(w / factor)) * factor; - if (static_cast(h_bar) * w_bar > max_pixels) { - const double beta = std::sqrt(h * w / max_pixels); + if (t_bar * h_bar * w_bar > max_pixels) { + const double beta = std::sqrt(n_frames * h * w / max_pixels); h_bar = static_cast(std::floor(h / beta / factor)) * factor; w_bar = static_cast(std::floor(w / beta / factor)) * factor; - } else if (static_cast(h_bar) * w_bar < min_pixels) { - const double beta = std::sqrt(min_pixels / (h * w)); + } else if (t_bar * h_bar * w_bar < min_pixels) { + const double beta = std::sqrt(min_pixels / (n_frames * h * w)); h_bar = static_cast(std::ceil(h * beta / factor)) * factor; w_bar = static_cast(std::ceil(w * beta / factor)) * factor; } @@ -846,6 +855,32 @@ clip_image_size mtmd_image_preprocessor_lfm3vl::target_size(const clip_image_siz hparams.image_min_pixels, hparams.image_max_pixels); } +bool mtmd_image_preprocessor_lfm3vl::get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const { + // Lfm3VlVideoProcessor size: shortest_edge 4096, longest_edge 25165824, temporal_patch_size 2 + out = smart_resize(width, height, hparams.patch_size * hparams.n_merge, 4096, 25165824, n_frames, 2); + return true; +} + +mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::preprocess_video_frame(const clip_image_u8 & img) const { + const int f = hparams.patch_size * hparams.n_merge; + const auto size = img.get_size(); + return resize_to(img, { + std::max(f, static_cast(std::nearbyint(static_cast(size.width) / f)) * f), + std::max(f, static_cast(std::nearbyint(static_cast(size.height) / f)) * f), + }); +} + +mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::resize_to(const clip_image_u8 & img, const clip_image_size & size) const { + clip_image_u8 resized_image; + img_tool::resize(img, resized_image, size, + hparams.image_resize_algo, + hparams.image_resize_pad, + hparams.image_pad_color); + mtmd_image_preproc_out output; + output.append(hparams, resized_image, true); + return output; +} + // // mtmd_image_preprocessor_glm5v // diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index 0e97a370a..07feae4b2 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -35,6 +35,10 @@ struct mtmd_image_preprocessor { virtual ~mtmd_image_preprocessor() = default; virtual mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const = 0; + // frames of a video, which the video helper may have already sized + virtual mtmd_image_preproc_out preprocess_video_frame(const clip_image_u8 & img) const { return preprocess(img); } + // frame size of a video with n_frames sampled frames, as the reference video processor; false if none + virtual bool get_video_frame_size(int /*n_frames*/, int /*width*/, int /*height*/, clip_image_size & /*out*/) const { return false; } }; /** @@ -142,8 +146,14 @@ struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor { struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_dyn_size { mtmd_image_preprocessor_lfm3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_dyn_size(ctx) {} clip_image_size target_size(const clip_image_size & original_size) const override; + // the video helper sizes the frames (get_video_frame_size), they are only snapped to the grid + mtmd_image_preproc_out preprocess_video_frame(const clip_image_u8 & img) const override; + bool get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const override; - static clip_image_size smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels); + mtmd_image_preproc_out resize_to(const clip_image_u8 & img, const clip_image_size & size) const; + // with temporal_factor > 1 it is the Qwen3-VL video variant over n_frames + static clip_image_size smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels, + int n_frames = 1, int temporal_factor = 1); }; // GLM 5.3 flash, similar to dyn_size, but each edge is aligned up to patch_size*n_merge, and max budget is met by a search over the height with the width scaled proportionally diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 7e0da513a..410881b51 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -25,11 +25,12 @@ #include #include #include +#include #include #include // remember to bump this if the serialization format changes -#define MTMD_SERIALIZATION_VERSION 3 +#define MTMD_SERIALIZATION_VERSION 4 // oldest compat version that can be loaded #define MTMD_SERIALIZATION_VERSION_MIN 2 @@ -174,6 +175,7 @@ struct mtmd_bitmap { std::string id; // optional user-defined id, for ex: can be set to image hash, useful for KV cache tracking bool is_audio = false; // true if the bitmap is audio bool mergeable = false; // [QWEN_VIDEO] set only on frames of the same video + bool is_video = false; // see mtmd_bitmap_set_is_video() // lazy-loaded bitmap mtmd_bitmap_lazy_callback lazy_callback = nullptr; @@ -209,6 +211,10 @@ struct mtmd_bitmap { return data.size(); } + bool is_video_frame() const { + return mergeable && is_video; + } + bool can_merge_with(const mtmd_bitmap & other) const { // [QWEN_VIDEO] can (temporal) merge if both are images with same size return mergeable && other.mergeable @@ -326,14 +332,26 @@ struct mtmd_image_tokens { }; using mtmd_image_tokens_ptr = std::unique_ptr; +// an audio track split across chunks (e.g. the soundtrack of a video): it is encoded once, as one clip, and every +// chunk takes its rows of it +struct mtmd_audio_track { + clip_image_f32_batch batch_f32; // the mel of the whole track + uint32_t n_tokens = 0; + std::vector embd; // filled by the first encode +}; + struct mtmd_audio_tokens { uint32_t n_tokens = 0; // number of tokens clip_image_f32_batch batch_f32; // preprocessed image patches std::string id; // optional user-defined ID, useful for KV cache tracking + // a slice of a track: the rows [row_begin, row_begin + n_tokens) of it, batch_f32 is unused + std::shared_ptr track; + uint32_t row_begin = 0; + // true if one of entries in batch_f32 is a placeholder bool is_placeholder() const { - for (const auto & entry : batch_f32.entries) { + for (const auto & entry : (track ? track->batch_f32 : batch_f32).entries) { if (entry.is_placeholder()) { return true; } @@ -345,7 +363,9 @@ struct mtmd_audio_tokens { return mtmd_audio_tokens{ n_tokens, batch_f32.clone(), - id + id, + track, + row_begin, }; } @@ -354,11 +374,25 @@ struct mtmd_audio_tokens { ser.write(n_tokens); ser.write(id); batch_f32.serialize(ser); + // a slice keeps its own copy of the track + ser.write(track != nullptr); + if (track) { + ser.write(row_begin); + ser.write(track->n_tokens); + track->batch_f32.serialize(ser); + } } void deserialize(mtmd_serialization & ser) { n_tokens = ser.read(); id = ser.read(); batch_f32.deserialize(ser); + track.reset(); + if (ser.version >= 4 && ser.read()) { + row_begin = ser.read(); + track = std::make_shared(); + track->n_tokens = ser.read(); + track->batch_f32.deserialize(ser); + } } }; using mtmd_audio_tokens_ptr = std::unique_ptr; @@ -538,6 +572,8 @@ struct mtmd_context { std::string img_end; std::string aud_beg; std::string aud_end; + std::string vid_beg; // wraps a whole video (lazy bitmap with is_video) + std::string vid_end; // for llava-uhd style models, we need special tokens in-between slices // minicpmv calls them "slices", llama 4 calls them "tiles" @@ -777,6 +813,8 @@ struct mtmd_context { // <|vision_start|> ... (image embeddings) ... <|vision_end|> img_beg = "<|vision_start|>"; img_end = "<|vision_end|>"; + vid_beg = img_beg; + vid_end = img_end; image_preproc = std::make_unique(ctx_v); } break; case PROJECTOR_TYPE_YOUTUVL: @@ -1158,6 +1196,11 @@ struct mtmd_context { return ctx_a ? clip_get_projector_type(ctx_a) : PROJECTOR_TYPE_UNKNOWN; } + // [LFM3_OMNI] the audio of a video is its soundtrack, see MTMD_VIDEO_TEMPLATE_LFM3_OMNI + bool video_has_soundtrack() const { + return proj_type_v() == PROJECTOR_TYPE_LFM3VL && proj_type_a() == PROJECTOR_TYPE_LFM3A; + } + int64_t n_embd_out() const { if (ctx_v) { return clip_n_mmproj_embd(ctx_v); @@ -1256,6 +1299,16 @@ struct mtmd_tokenizer { std::vector bm_from_lazy; // TODO @ngxson : refactor, free bm_from_lazy progressively std::vector text_from_lazy; + // [LFM3_OMNI] the audio pieces of a video are joined back and encoded as one clip, each piece takes its rows + struct soundtrack { + std::vector pieces; + std::shared_ptr track; // built on the first piece + size_t n_samples = 0; // samples of the pieces added so far + uint32_t n_assigned = 0; // rows of the pieces added so far + }; + std::vector soundtracks; + std::map soundtrack_of; // piece -> index in soundtracks + mtmd_input_chunks cur; uint32_t n_images_added = 0; // 0-based index assigned to the next image chunk @@ -1329,6 +1382,11 @@ struct mtmd_tokenizer { for (auto & p : parts) { if (p.bitmap != nullptr && p.bitmap->lazy_callback) { LOG_DBG("%s: expanding lazy bitmap\n", __func__); + const bool is_video = p.bitmap->is_video; + if (is_video && !ctx->vid_beg.empty()) { + expanded.push_back({ctx->vid_beg, nullptr, true}); + } + soundtrack st; for (size_t i = 0;; i++) { char * out_str = nullptr; mtmd_bitmap * out_bm = nullptr; @@ -1344,6 +1402,9 @@ struct mtmd_tokenizer { if (out_bm) { auto & ptr = bm_from_lazy.emplace_back(out_bm); // remember to free it later expanded.push_back({"", ptr.ptr.get()}); + if (is_video && out_bm->is_audio && ctx->video_has_soundtrack()) { + st.pieces.push_back(out_bm); + } LOG_DBG("%s: lazy callback returned bitmap with dimensions %d x %d\n", __func__, out_bm->nx, out_bm->ny); } else if (out_str) { auto & ptr = text_from_lazy.emplace_back(out_str); // remember to free it later @@ -1352,12 +1413,21 @@ struct mtmd_tokenizer { } } else if (res == -1) { // EOF: lazy part removes itself (not added to expanded) + if (is_video && !ctx->vid_end.empty()) { + expanded.push_back({ctx->vid_end, nullptr, true}); + } break; } else if (res == -2) { // error throw std::runtime_error(string_format("lazy callback returned error")); } } + if (!st.pieces.empty()) { + for (const auto * piece : st.pieces) { + soundtrack_of[piece] = soundtracks.size(); + } + soundtracks.push_back(std::move(st)); + } } else { expanded.push_back(std::move(p)); } @@ -1501,7 +1571,9 @@ struct mtmd_tokenizer { img_u8.cpy_buf(bmp->get_ro_buf()); // preprocess image - mtmd_image_preproc_out tmp_preproc_out = ctx->image_preproc->preprocess(img_u8); + mtmd_image_preproc_out tmp_preproc_out = bmp->is_video_frame() + ? ctx->image_preproc->preprocess_video_frame(img_u8) + : ctx->image_preproc->preprocess(img_u8); // move entries and grid dimensions to the "global" preproc_out for (auto & entry : tmp_preproc_out.entries) { @@ -1667,7 +1739,7 @@ struct mtmd_tokenizer { clip_image_f32_batch batch_f32; batch_f32.is_audio = false; - batch_f32.is_video = bitmaps[0]->mergeable; + batch_f32.is_video = bitmaps[0]->is_video_frame(); batch_f32.entries = std::move(preproc_out.entries); // do NOT use preproc_out from this point on, it's moved @@ -1705,6 +1777,10 @@ struct mtmd_tokenizer { return 2; } + if (auto it = soundtrack_of.find(bitmap); it != soundtrack_of.end()) { + return add_soundtrack_piece(soundtracks[it->second], *bitmap); + } + if (bitmap->nx == 0) { LOG_ERR("%s: error: empty audio data\n", __func__); return 2; @@ -1793,6 +1869,58 @@ struct mtmd_tokenizer { return 0; } + // a piece of a soundtrack goes in without the audio markers, it lies inside the span of its video + int32_t add_soundtrack_piece(soundtrack & st, const mtmd_bitmap & bitmap) { + if (!st.track) { + std::vector samples; + for (const auto * piece : st.pieces) { + if (piece->is_placeholder()) { + samples.resize(samples.size() + piece->nx, 0.0f); + } else { + const float * data = (const float *) piece->get_ro_buf().data(); + samples.insert(samples.end(), data, data + piece->nx); + } + } + std::vector mels; + if (!ctx->audio_preproc || !ctx->audio_preproc->preprocess(samples.data(), samples.size(), mels) || mels.size() != 1) { + LOG_ERR("%s: unable to preprocess the soundtrack\n", __func__); + return 2; + } + clip_image_f32 mel_f32; + mel_f32.set_size({(int) mels[0].n_len, (int) mels[0].n_mel}, mels[0].data.empty(), /* is_audio */ true); + mel_f32.cpy_buf(mels[0].data); + st.track = std::make_shared(); + st.track->n_tokens = clip_n_output_tokens(ctx->ctx_a, &mel_f32); + st.track->batch_f32.is_audio = true; + st.track->batch_f32.entries.push_back(std::move(mel_f32)); + } + + // rows of [begin, end): round(sample / samples per row), the last piece takes the rest of the track + const double samples_per_row = 8.0 * clip_get_hparams(ctx->ctx_a)->audio_hop_len; // one row per 8 mel frames + const uint32_t n_tokens = st.track->n_tokens; + st.n_samples += bitmap.nx; + const uint32_t begin = st.n_assigned; + const uint32_t end = &bitmap == st.pieces.back() ? n_tokens + : std::clamp((uint32_t) std::nearbyint(st.n_samples / samples_per_row), begin, n_tokens); + st.n_assigned = end; + if (end == begin) { + return 0; + } + + mtmd_audio_tokens_ptr audio_tokens(new mtmd_audio_tokens); + audio_tokens->n_tokens = end - begin; + audio_tokens->track = st.track; + audio_tokens->row_begin = begin; + audio_tokens->id = bitmap.id; + cur.entries.emplace_back(mtmd_input_chunk{ + MTMD_INPUT_CHUNK_TYPE_AUDIO, + {}, // text tokens + nullptr, // image tokens + std::move(audio_tokens), + }); + return 0; + } + // MiniCPM-V 4.7: the overview and all slices go in one chunk, clip appends the separators after each tile: // [ov] [S00] [S01] \n [S10] [S11] void add_canvas_chunk(mtmd_image_preproc_out && preproc_out, const std::string & id) { @@ -2049,6 +2177,20 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk return 1; } int n_mmproj_embd = ctx->n_embd_out(); + const auto & track = chunk->tokens_audio->track; + if (track) { + if (track->embd.empty()) { + track->embd.resize((size_t) track->n_tokens * n_mmproj_embd); + if (!clip_image_batch_encode(ctx->ctx_a, ctx->n_threads, &track->batch_f32, track->embd)) { + track->embd.clear(); + return 1; + } + } + const size_t off = (size_t) chunk->tokens_audio->row_begin * n_mmproj_embd; + out_embd.assign(track->embd.begin() + off, + track->embd.begin() + off + (size_t) chunk->tokens_audio->n_tokens * n_mmproj_embd); + return 0; + } out_embd.resize((size_t)chunk->tokens_audio->n_tokens * n_mmproj_embd); bool ok = clip_image_batch_encode( ctx->ctx_a, @@ -2579,6 +2721,36 @@ int mtmd_get_audio_sample_rate(const mtmd_context * ctx) { return clip_get_hparams(ctx->ctx_a)->audio_sample_rate; } +mtmd_video_config mtmd_get_video_config(const mtmd_context * ctx, + int32_t src_width, int32_t src_height, + int32_t src_n_frames, float src_fps) { + mtmd_video_config cfg{MTMD_VIDEO_TEMPLATE_DEFAULT, 0.0f, 0, 0, 0}; + if (!ctx->ctx_v || clip_get_projector_type(ctx->ctx_v) != PROJECTOR_TYPE_LFM3VL) { + return cfg; + } + + // matches Lfm3VlVideoProcessor: 2 fps sampling with a [4, 768] frame clamp + // LFM3-Omni samples at most 384 frames and hears the soundtrack + const bool omni = ctx->video_has_soundtrack(); + cfg.tmpl = omni ? MTMD_VIDEO_TEMPLATE_LFM3_OMNI : MTMD_VIDEO_TEMPLATE_LFM3; + cfg.fps = 2.0f; + if (src_n_frames <= 0 || src_fps <= 0.0f) { + return cfg; + } + + int32_t n = (int32_t)((double) src_n_frames / src_fps * cfg.fps); + n = std::clamp(n, 4, omni ? 384 : 768); + n = std::min(n, src_n_frames); + cfg.n_frames = n; + + clip_image_size size; + if (ctx->image_preproc && ctx->image_preproc->get_video_frame_size(n, src_width, src_height, size)) { + cfg.frame_width = size.width; + cfg.frame_height = size.height; + } + return cfg; +} + const char * mtmd_get_marker(const mtmd_context * ctx) { return ctx->media_marker.c_str(); } @@ -2648,6 +2820,10 @@ void mtmd_bitmap_set_mergeable(mtmd_bitmap * bitmap, bool mergeable) { bitmap->mergeable = mergeable; } +void mtmd_bitmap_set_is_video(mtmd_bitmap * bitmap, bool is_video) { + bitmap->is_video = is_video; +} + mtmd_bitmap * mtmd_bitmap_init_lazy(const mtmd_context * ctx, const char * id, void * user_data, diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index f552dddf9..3e03142a4 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -58,6 +58,13 @@ enum mtmd_input_chunk_type { MTMD_INPUT_CHUNK_TYPE_COUNT, // for validation }; +// how the frames and timestamps of a video are laid out in the prompt +enum mtmd_video_template { + MTMD_VIDEO_TEMPLATE_DEFAULT, // "Video:" prefix, then a "[XmY.YYs]" timestamp every N ms + MTMD_VIDEO_TEMPLATE_LFM3, // "" before every group of 2 frames + MTMD_VIDEO_TEMPLATE_LFM3_OMNI, // same as LFM3, then the soundtrack of each group, as an audio bitmap after its frames +}; + // opaque types struct mtmd_context; struct mtmd_bitmap; @@ -156,6 +163,20 @@ MTMD_API bool mtmd_support_audio(const mtmd_context * ctx); // return -1 if audio is not supported MTMD_API int mtmd_get_audio_sample_rate(const mtmd_context * ctx); +// recommended video input for the current model, given the source video +// fields set to 0 mean "no preference, use the caller's default" +struct mtmd_video_config { + enum mtmd_video_template tmpl; + float fps; // frame sampling rate + int32_t n_frames; // number of frames to sample + int32_t frame_width; // size to resize the frames to + int32_t frame_height; +}; +// src_n_frames and src_fps can be <= 0 if unknown +MTMD_API struct mtmd_video_config mtmd_get_video_config(const mtmd_context * ctx, + int32_t src_width, int32_t src_height, + int32_t src_n_frames, float src_fps); + // get the current marker string MTMD_API const char * mtmd_get_marker(const mtmd_context * ctx); @@ -166,7 +187,7 @@ MTMD_API const char * mtmd_get_marker(const mtmd_context * ctx); // the data is in RGBRGBRGB... format // note: some video-capable models (i.e. qwen-vl) can merge consecutive bitmaps // into one chunk; mtmd_tokenize() handles this, but remember to set -// mtmd_bitmap_set_mergeable(true) for every frame +// mtmd_bitmap_set_mergeable(true) and mtmd_bitmap_set_is_video(true) for every frame // if bitmap is audio: // length of data must be n_samples * sizeof(float) // the data is in float format (PCM F32) @@ -189,6 +210,10 @@ MTMD_API const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap); MTMD_API void mtmd_bitmap_set_id(mtmd_bitmap * bitmap, const char * id); // if true, this bitmap can be merged (temporal merge) with an adjacent mergeable bitmap by certain video input models MTMD_API void mtmd_bitmap_set_mergeable(mtmd_bitmap * bitmap, bool mergeable); +// on a frame: video frames must be both mergeable and is_video +// on a lazy bitmap: the expanded chunks are one video, wrapped by the model's video begin/end tokens +// with MTMD_VIDEO_TEMPLATE_LFM3_OMNI, its audio bitmaps are the pieces of one soundtrack, in order +MTMD_API void mtmd_bitmap_set_is_video(mtmd_bitmap * bitmap, bool is_video); // mtmd_bitmap lazy // diff --git a/tools/server/README.md b/tools/server/README.md index ccb27b49d..6188a084b 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1465,6 +1465,7 @@ For multimodal input (typed content, `messages[i].content[j]`): - If `type == "input_video"`: - Either `input_video.data` or `input_video.url` can be specified, can be a remote URL, raw base64 or path to local file - Accepts formats supported by `ffmpeg` + - Models that hear their video (LFM3-Omni) also get its soundtrack, the audio of each group of frames follows them - Note: for local file, make sure to set `--media-path`. File path must be prefixed by `file://` *Examples:* diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp index ba70c7b01..8b9911f64 100644 --- a/tools/server/server-decision.cpp +++ b/tools/server/server-decision.cpp @@ -388,10 +388,10 @@ std::vector server_decision_context::parse_questions(c static const size_t DECISION_MAX_IMAGES = 8; -// any media, mtmd tells an audio clip from an image by its content +// any media, mtmd tells an audio clip or a video from an image by its content static void decision_load_image(const json & url, std::vector & files) { if (!url.is_string() || !string_starts_with(url.get(), "data:")) { - throw std::invalid_argument("images must be data URLs (data:image/...;base64,... or data:audio/...;base64,...)"); + throw std::invalid_argument("media must be data URLs (data:image/..., data:audio/... or data:video/...;base64,...)"); } if (files.size() >= DECISION_MAX_IMAGES) { throw std::invalid_argument(string_format("too many images, the maximum is %zu", DECISION_MAX_IMAGES)); @@ -457,6 +457,9 @@ json server_decision_context::parse_state(const json & body, std::vector