mtmd: enable video input support for LFM3 model family (#31283)

* mtmd : add LFM3-VL and LFM3-Omni video input

Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
Assisted-by: Claude Opus 5.5

* mtmd : reserve the soundtrack command to avoid a GCC 11 warning

---------

Co-authored-by: Xuan Son Nguyen <son@huggingface.co>

conversion/lfm3.py

@@ -203,6 +203,24 @@ class LFM3VLVisionMixin(MmprojModel):
         if resize_algo is None:
             raise ValueError(f"unsupported resample: {resample}")
         self.gguf_writer.add_vision_image_resize_algo(resize_algo)
+        self._check_video_config()
+
+    # mtmd_get_video_ts_style and the lfm3vl video frame sizing hardcode these, conversion fails on a mismatch
+    _video = {"fps": 2.0, "min_frames": 4, "temporal_patch_size": 2, "size": {"shortest_edge": 4096, "longest_edge": 25165824}}
+    _video_max_frames = 768
+
+    def _check_video_config(self):
+        import json
+        video = self.preprocessor_config.get("video_processor")
+        path = self.dir_model / "video_preprocessor_config.json"
+        if video is None and path.is_file():
+            with open(path, encoding="utf-8") as f:
+                video = json.load(f)
+        if video is None:
+            return
+        for k, v in {**self._video, "max_frames": self._video_max_frames}.items():
+            if video.get(k) != v:
+                raise ValueError(f"unsupported video {k}={video.get(k)!r}, expected {v!r}")
 
     def tensor_force_quant(self, name, new_name, bid, n_dims):
         if "patch_embd" in new_name:
@@ -562,6 +580,7 @@ class LFM3OmniMmprojModel(LFM3VLVisionMixin, LFM3AudioMmprojModel):
     has_audio_encoder = True
 
     _audio_sampling = {"audio_temperature": 0.4, "audio_repetition_penalty": 1.2, "audio_repetition_window": 64}
+    _video_max_frames = 384
 
     # the audio side is the LFM3-Audio model under the HF-native names
     _audio_renames = (

tools/mtmd/clip.cpp

@@ -7099,6 +7099,7 @@ int clip_model_n_temporal_merge(const struct clip_ctx * ctx) {
         case PROJECTOR_TYPE_QWEN2VL:
         case PROJECTOR_TYPE_QWEN25VL:
         case PROJECTOR_TYPE_QWEN3VL:
+        case PROJECTOR_TYPE_LFM3VL:
         case PROJECTOR_TYPE_NEMOTRON_OMNI:
         case PROJECTOR_TYPE_LING3VL:
             return 2;

tools/mtmd/models/lfm3vl.cpp

@@ -4,7 +4,7 @@ ggml_cgraph * clip_graph_lfm3vl::build() {
     GGML_ASSERT(model.patch_embeddings_0 != nullptr);
     GGML_ASSERT(model.lfm3vl_embedding_norm_w != nullptr);
     GGML_ASSERT(model.class_embedding == nullptr);
-    GGML_ASSERT(n_batch == 1);
+    GGML_ASSERT(n_batch == 1 || n_batch == 2);
 
     const int batch_size       = 1;
     const int n_pos            = n_patches;
@@ -15,12 +15,13 @@ ggml_cgraph * clip_graph_lfm3vl::build() {
     // 2D rope: pairs (j, j+n_dims); first d_head/4 pairs rotate by y, next d_head/4 by x
     int mrope_sections[4] = {d_head/4, d_head/4, 0, 0};
 
-    ggml_tensor * inp_raw = build_inp_raw();
-
-    // one patchifier kernel per temporal slice: a still image fills both slices, so both kernels apply to it
-    ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
-    if (model.patch_embeddings_1) {
-        inp = ggml_add(ctx0, inp, ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
+    // one patchifier kernel per temporal slice: a still image fills both slices, video frames are merged in pairs
+    ggml_tensor * inp;
+    if (model.patch_embeddings_1 == nullptr) {
+        GGML_ASSERT(n_batch == 1);
+        inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, build_inp_raw(), patch_size, patch_size, 0, 0, 1, 1);
+    } else {
+        inp = build_inp_temporal_merge();
     }
     inp = ggml_reshape_2d(ctx0, inp, n_pos, n_embd);
     inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));

tools/mtmd/mtmd-helper.cpp

@@ -16,6 +16,7 @@
 #include "hash/hash.h"
 
 #include <algorithm>
+#include <cmath>
 #include <cinttypes>
 #include <vector>
 
@@ -444,7 +445,8 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(const mtmd_context *
                 *out_text = text; // heap-allocated by read_next; freed automatically by mtmd
                 return ret;
             });
-         return {result, video_ctx};
+        mtmd_bitmap_set_is_video(result, true);
+        return {result, video_ctx};
     }
 #else
     if (!result) {
@@ -600,32 +602,63 @@ struct mtmd_helper_video {
     int32_t     timestamp_interval_ms = 5000; // emit a timestamp text every N ms (0 = disabled)
     float       next_timestamp_ms     = 0.0f; // next elapsed-ms threshold at which to emit
 
+    // apply mtmd_get_video_config(); false for still images decoded via ffmpeg
+    bool                use_video_config = false;
+    mtmd_video_template tmpl = MTMD_VIDEO_TEMPLATE_DEFAULT;
+
+    int32_t     target_frames = 0;   // exact frame count to emit (0 = no limit)
+    int32_t     out_width     = 0;   // frame size from mtmd_get_video_config (0 = source size)
+    int32_t     out_height    = 0;
+
+    // source frames to emit, as picked by the reference processors (empty = use the fps filter)
+    std::vector<int32_t> frame_indices;
+    size_t      next_index = 0;
+    int32_t     src_frame  = 0;      // source frames read from ffmpeg so far
+    double      src_fps    = 0.0;
+
     std::vector<uint8_t> frame_buf;
-    std::string pending_text; // text queued to be returned before the next frame
-    bool        start_emitted = false;
+    std::string   pending_text;          // text queued to be returned before the next frame
+    mtmd_bitmap * pending_frame = nullptr; // frame queued to be returned after its group timestamp
+
+    // soundtrack: after each group of frames comes the audio from its first frame to the next group's
+    double             video_start_s = 0.0;
+    int32_t            n_src_frames  = 0;
+    std::vector<float> soundtrack;
+    mtmd_bitmap *      pending_audio = nullptr;
+    bool          start_emitted = false;
 
     bool is_buf_input() const {
         return !input_buf.empty();
     }
 
+    // number of frames per timestamp group (0 = timestamp every timestamp_interval_ms)
+    int32_t ts_per_n_frames() const {
+        return tmpl == MTMD_VIDEO_TEMPLATE_LFM3 || tmpl == MTMD_VIDEO_TEMPLATE_LFM3_OMNI ? 2 : 0;
+    }
+
     bool probe(float fps_target_arg) {
-        const char * input_arg = is_buf_input() ? "pipe:0" : path.c_str();
-        const char * cmd[] = {
+        // cache: makes stdin seekable; without it ffprobe reports width/height 0
+        // for mp4 files whose moov atom sits at the end
+        const char * input_arg = is_buf_input() ? "cache:pipe:0" : path.c_str();
+        std::vector<const char *> cmd = {
             ffprobe_bin.c_str(),
             "-v", "quiet",
-            "-show_entries", "stream=width,height,r_frame_rate,nb_frames,duration",
+            "-show_entries", "stream=width,height,r_frame_rate,nb_frames,duration,start_time",
             "-select_streams", "v:0",
             "-of", "default=noprint_wrappers=1",
-            input_arg,
-            nullptr,
         };
+        if (is_buf_input()) {
+            // remove the 64KB read-ahead limit of cache:, or else ffprobe cannot reach a moov atom at end of file
+            cmd.insert(cmd.end(), {"-read_ahead_limit", "-1"});
+        }
+        cmd.insert(cmd.end(), {input_arg, nullptr});
 
         LOG_DBG("%s: launching:", __func__);
         for (size_t i = 0; cmd[i]; i++) { LOG_DBG(" %s", cmd[i]); }
         LOG_DBG("\n");
 
         subprocess_handle probe_sp;
-        if (subprocess_create(cmd,
+        if (subprocess_create(cmd.data(),
                 subprocess_option_search_user_path | subprocess_option_inherit_environment,
                 &probe_sp.proc) != 0) {
             LOG_ERR("%s: failed to launch ffprobe\n", __func__);
@@ -665,6 +698,8 @@ struct mtmd_helper_video {
                 n_frames_orig = atoi(val);
             } else if (strcmp(key, "duration") == 0 && strcmp(val, "N/A") != 0) {
                 duration = (float)atof(val);
+            } else if (strcmp(key, "start_time") == 0 && strcmp(val, "N/A") != 0) {
+                video_start_s = atof(val);
             }
         }
 
@@ -678,17 +713,118 @@ struct mtmd_helper_video {
             duration = (float)n_frames_orig / orig_fps;
         }
 
-        fps_target = fps_target_arg > 0.0f ? fps_target_arg : orig_fps;
+        mtmd_video_config cfg{MTMD_VIDEO_TEMPLATE_DEFAULT, 0.0f, 0, 0, 0};
+        if (use_video_config) {
+            cfg  = mtmd_get_video_config(mctx, width, height, n_frames_orig, orig_fps);
+            tmpl = cfg.tmpl;
+        }
+
+        // the model's recommended sampling rate wins over the generic default
+        fps_target = cfg.fps > 0.0f ? cfg.fps
+                   : fps_target_arg > 0.0f ? fps_target_arg : orig_fps;
+
+        if (cfg.n_frames > 0 && duration > 0.0f) {
+            const int32_t n = cfg.n_frames;
+            target_frames = n;
+            fps_target    = (float) n / duration;
+
+            // same frames as the reference processors: round(linspace(0, n_frames - 1, n))
+            const double step = n > 1 ? (double) (n_frames_orig - 1) / (n - 1) : 0.0;
+            frame_indices.resize(n);
+            for (int32_t i = 0; i < n; i++) {
+                frame_indices[i] = (int32_t) std::nearbyint(i * step);
+            }
+            src_fps      = orig_fps;
+            n_src_frames = n_frames_orig;
+
+            if (cfg.frame_width > 0 && cfg.frame_height > 0 &&
+                    (cfg.frame_width != (int32_t) width || cfg.frame_height != (int32_t) height)) {
+                out_width  = cfg.frame_width;
+                out_height = cfg.frame_height;
+            }
+        }
+
+        if (out_width > 0 && out_height > 0) {
+            width  = (uint32_t) out_width;
+            height = (uint32_t) out_height;
+        }
+
         info.width    = width;
         info.height   = height;
         info.fps      = fps_target;
-        LOG_DBG("%s: %ux%u fps=%.2f duration=%.2fs n_frames=%d\n",
-                __func__, width, height, fps_target, duration, info.n_frames);
-        info.n_frames = duration > 0.0f ? (int32_t)(duration * fps_target + 0.5f) : -1;
+        LOG_DBG("%s: %ux%u fps=%.2f duration=%.2fs n_frames=%d target_frames=%d\n",
+                __func__, width, height, fps_target, duration, n_frames_orig, target_frames);
+        info.n_frames = target_frames > 0 ? target_frames
+                      : duration > 0.0f ? (int32_t)(duration * fps_target + 0.5f) : -1;
         frame_buf.resize((size_t)width * height * 3);
         return true;
     }
 
+    // the soundtrack over the span of the video stream: from its first frame for n_frames / fps seconds, mono at the
+    // rate of the audio encoder; empty if the file has no audio
+    void load_soundtrack() {
+        const int rate = mtmd_get_audio_sample_rate(mctx);
+        if (tmpl != MTMD_VIDEO_TEMPLATE_LFM3_OMNI || frame_indices.empty() || rate <= 0) {
+            return;
+        }
+        const std::string rate_str = std::to_string(rate);
+        std::vector<const char *> cmd = {ffmpeg_bin.c_str(), "-nostdin"};
+        cmd.reserve(20); // GCC 11 -Wstringop-overflow false positive on the inserts below otherwise
+        if (is_buf_input()) {
+            cmd.insert(cmd.end(), {"-read_ahead_limit", "-1"});
+        }
+        cmd.insert(cmd.end(), {"-i", is_buf_input() ? "cache:pipe:0" : path.c_str(), "-map", "0:a:0", "-vn", "-ac", "1",
+                               "-ar", rate_str.c_str(), "-f", "f32le", "pipe:1", "-loglevel", "error", nullptr});
+        subprocess_handle audio_sp;
+        if (subprocess_create(cmd.data(), subprocess_option_search_user_path | subprocess_option_inherit_environment,
+                              &audio_sp.proc) != 0) {
+            LOG_WRN("%s: failed to launch ffmpeg for the soundtrack\n", __func__);
+            return;
+        }
+        audio_sp.created = true;
+        audio_sp.alive   = true;
+        if (is_buf_input()) {
+            audio_sp.start_feeder(input_buf);
+        }
+        std::vector<float> pcm;
+        float block[4096];
+        size_t n;
+        while ((n = fread(block, sizeof(float), 4096, audio_sp.stdout_pipe())) > 0) {
+            pcm.insert(pcm.end(), block, block + n);
+        }
+        audio_sp.stop();
+
+        const size_t start = (size_t) std::max(0.0, std::nearbyint(video_start_s * rate));
+        const size_t len   = (size_t) std::nearbyint((double) n_src_frames / src_fps * rate);
+        if (start >= pcm.size()) {
+            LOG_WRN("%s: the video has no soundtrack\n", __func__);
+            return;
+        }
+        soundtrack.assign(pcm.begin() + start, pcm.begin() + std::min(pcm.size(), start + len));
+    }
+
+    // the soundtrack of the group of frames that ends with frame idx, nullptr if the soundtrack has ended
+    mtmd_bitmap * soundtrack_slice(int32_t idx) const {
+        const int32_t ts_per_n_frames = this->ts_per_n_frames();
+        const int32_t group    = idx / ts_per_n_frames;
+        const int32_t n_groups = ((int32_t) frame_indices.size() + ts_per_n_frames - 1) / ts_per_n_frames;
+        const int     rate     = mtmd_get_audio_sample_rate(mctx);
+        auto group_start = [&](int32_t g) {
+            const double t = frame_indices[g * ts_per_n_frames] / src_fps;
+            return std::min(soundtrack.size(), (size_t) std::nearbyint(t * rate));
+        };
+        const size_t begin = group == 0 ? 0 : group_start(group);
+        const size_t end   = group == n_groups - 1 ? soundtrack.size() : group_start(group + 1);
+        if (begin >= end) {
+            return nullptr;
+        }
+        mtmd_bitmap * slice = mtmd_bitmap_init_from_audio(end - begin, soundtrack.data() + begin);
+        if (!id.empty()) {
+            mtmd_bitmap_set_id(slice, (id + "+audio" + std::to_string(group)).c_str());
+        }
+        return slice;
+    }
+
     bool start_ffmpeg(float seek_seconds) {
         char seek_buf[64];
         char fps_buf[64];
@@ -721,10 +857,35 @@ struct mtmd_helper_video {
             cmd.push_back(seek_buf);
         }
 
-        if (fps_target > 0.0f) {
-            snprintf(fps_buf, sizeof(fps_buf), "fps=%.6f", fps_target);
+        std::string vf;
+        if (!frame_indices.empty()) {
+            // frames are picked by index in read_next_frame(); B=0 C=0.5 bicubic on RGB is closest to Pillow
+            if (out_width > 0 && out_height > 0) {
+                char scale_buf[160];
+                snprintf(scale_buf, sizeof(scale_buf),
+                        "format=rgb24,scale=%d:%d:flags=bicubic+accurate_rnd+full_chroma_int:param0=0:param1=0.5",
+                        out_width, out_height);
+                vf = scale_buf;
+            }
+        } else if (fps_target > 0.0f) {
+            if (out_width > 0 && out_height > 0) {
+                snprintf(fps_buf, sizeof(fps_buf), "fps=%.6f,scale=%d:%d", fps_target, out_width, out_height);
+            } else {
+                snprintf(fps_buf, sizeof(fps_buf), "fps=%.6f", fps_target);
+            }
+            vf = fps_buf;
+        }
+        if (!vf.empty()) {
             cmd.push_back("-vf");
-            cmd.push_back(fps_buf);
+            cmd.push_back(vf.c_str());
+        }
+
+        char frames_buf[16];
+        const int32_t n_out = !frame_indices.empty() ? frame_indices.back() + 1 : target_frames;
+        if (n_out > 0) {
+            snprintf(frames_buf, sizeof(frames_buf), "%d", n_out);
+            cmd.push_back("-frames:v");
+            cmd.push_back(frames_buf);
         }
 
         cmd.push_back("-f");
@@ -763,9 +924,8 @@ struct mtmd_helper_video {
         sp.stop();
     }
 
-    mtmd_bitmap * read_next_frame() {
-        if (!sp.alive) return nullptr;
-
+    // read one raw frame from ffmpeg into frame_buf
+    bool read_raw_frame() {
         FILE * fp = sp.stdout_pipe();
         const size_t frame_size = (size_t)info.width * info.height * 3;
         LOG_DBG("%s: reading frame %d, expecting %zu bytes (%ux%u)\n",
@@ -779,14 +939,37 @@ struct mtmd_helper_video {
                 LOG_DBG("%s: fread returned 0 after %zu/%zu bytes (ferror=%d)\n",
                         __func__, total_read, frame_size, ferror(fp));
                 sp.alive = false;
-                return nullptr;
+                return false;
             }
             total_read += n;
         }
+        return true;
+    }
+
+    mtmd_bitmap * read_next_frame() {
+        if (!sp.alive) return nullptr;
+
+        if (!frame_indices.empty()) {
+            if (next_index >= frame_indices.size()) {
+                return nullptr;
+            }
+            // skip to the wanted source frame; a repeated index reuses the frame in frame_buf
+            const int32_t want = frame_indices[next_index];
+            while (src_frame <= want) {
+                if (!read_raw_frame()) {
+                    return nullptr;
+                }
+                src_frame++;
+            }
+            next_index++;
+        } else if (!read_raw_frame()) {
+            return nullptr;
+        }
 
         LOG_DBG("%s: frame %d read OK\n", __func__, current_frame);
         mtmd_bitmap * frame = mtmd_bitmap_init(info.width, info.height, frame_buf.data());
         mtmd_bitmap_set_mergeable(frame, true);
+        mtmd_bitmap_set_is_video(frame, true);
         if (!id.empty()) {
             // each frame gets a unique id in the form of {hash}+{frame}, so that it can be identified in cache
             std::string frame_id = id + "+" + std::to_string(current_frame);
@@ -806,6 +989,18 @@ struct mtmd_helper_video {
             return *out_text ? 0 : -2;
         }
 
+        if (pending_frame) {
+            *out_bitmap  = pending_frame;
+            pending_frame = nullptr;
+            return 0;
+        }
+
+        if (pending_audio) {
+            *out_bitmap   = pending_audio;
+            pending_audio = nullptr;
+            return 0;
+        }
+
         LOG_DBG("%s: proc_alive=%d start_emitted=%d current_frame=%d\n",
                 __func__, (int)sp.alive, (int)start_emitted, current_frame);
 
@@ -815,14 +1010,46 @@ struct mtmd_helper_video {
 
         if (!start_emitted) {
             start_emitted = true;
-            if (!prompt_start.empty()) {
+            // model templates have no prompt prefix, the video begin/end tokens are added by mtmd_tokenize
+            if (tmpl == MTMD_VIDEO_TEMPLATE_DEFAULT && !prompt_start.empty()) {
                 *out_text = mtmd_strdup(prompt_start);
                 return *out_text ? 0 : -2;
             }
         }
 
         mtmd_bitmap * frame = read_next_frame();
-        if (!frame) return -1;
+        if (!frame) {
+            return -1;
+        }
+
+        const int32_t ts_per_n_frames = this->ts_per_n_frames();
+        if (ts_per_n_frames > 0) {
+            // emit the group timestamp before the group's first frame; the value is the
+            // midpoint time of the group, matching HF processors that stamp each temporal group
+            const int32_t idx = current_frame - 1; // read_next_frame() already incremented
+            // the soundtrack of a group follows its last frame
+            const bool group_ends = idx % ts_per_n_frames == ts_per_n_frames - 1 || idx + 1 == (int32_t) frame_indices.size();
+            if (!soundtrack.empty() && !frame_indices.empty() && group_ends) {
+                pending_audio = soundtrack_slice(idx);
+            }
+            if (idx % ts_per_n_frames == 0) {
+                char ts_buf[64];
+                double ts_s = ((double)idx + 0.5*(double)(ts_per_n_frames - 1)) / info.fps;
+                if (!frame_indices.empty()) {
+                    // stamp from the source frames; a short last group repeats its last frame
+                    const size_t first = idx;
+                    const size_t last  = std::min(first + ts_per_n_frames - 1, frame_indices.size() - 1);
+                    ts_s = (frame_indices[first] / src_fps + frame_indices[last] / src_fps) / 2.0;
+                }
+                snprintf(ts_buf, sizeof(ts_buf), "<%.1f seconds>", ts_s);
+                pending_frame = frame;
+                *out_text = mtmd_strdup(ts_buf);
+                return *out_text ? 0 : -2;
+            }
+            *out_bitmap = frame;
+            return 0;
+        }
+
         *out_bitmap = frame;
 
         if (timestamp_interval_ms > 0) {
@@ -861,6 +1088,7 @@ mtmd_helper_video_init_params mtmd_helper_video_init_params_default() {
         /* fps_target             */ 4.0f,
         /* ffmpeg_bin_dir         */ nullptr,
         /* timestamp_interval_ms  */ 5000,
+        /* no_soundtrack          */ false,
     };
 }
 
@@ -916,6 +1144,7 @@ static mtmd_bitmap * decode_webp_with_ffmpeg(const mtmd_context * mctx, const un
     mtmd_bitmap * frame = vctx.read_next_frame();
     if (frame) {
         mtmd_bitmap_set_mergeable(frame, false);
+        mtmd_bitmap_set_is_video(frame, false);
     }
     return frame;
 }
@@ -933,12 +1162,16 @@ mtmd_helper_video * mtmd_helper_video_init(
     ctx->ffmpeg_bin           = video_resolve_bin(params.ffmpeg_bin_dir, "ffmpeg");
     ctx->ffprobe_bin          = video_resolve_bin(params.ffmpeg_bin_dir, "ffprobe");
     ctx->timestamp_interval_ms = params.timestamp_interval_ms;
+    ctx->use_video_config      = true;
 
     if (!ctx->probe(params.fps_target)) {
         LOG_ERR("%s: ffprobe failed for '%s' (is ffprobe in PATH?)\n", __func__, path);
         delete ctx;
         return nullptr;
     }
+    if (!params.no_soundtrack) {
+        ctx->load_soundtrack();
+    }
 
     if (!ctx->start_ffmpeg(0.0f)) {
         LOG_ERR("%s: failed to start ffmpeg for '%s' (is ffmpeg in PATH?)\n", __func__, path);
@@ -968,12 +1201,16 @@ mtmd_helper_video * mtmd_helper_video_init_from_buf(
     ctx->ffmpeg_bin            = video_resolve_bin(params.ffmpeg_bin_dir, "ffmpeg");
     ctx->ffprobe_bin           = video_resolve_bin(params.ffmpeg_bin_dir, "ffprobe");
     ctx->timestamp_interval_ms = params.timestamp_interval_ms;
+    ctx->use_video_config      = true;
 
     if (!ctx->probe(params.fps_target)) {
         LOG_ERR("%s: ffprobe failed on buffer (is ffprobe in PATH?)\n", __func__);
         delete ctx;
         return nullptr;
     }
+    if (!params.no_soundtrack) {
+        ctx->load_soundtrack();
+    }
 
     if (!ctx->start_ffmpeg(0.0f)) {
         LOG_ERR("%s: failed to start ffmpeg on buffer (is ffmpeg in PATH?)\n", __func__);
@@ -996,6 +1233,8 @@ void mtmd_helper_video_free(mtmd_helper_video * ctx) {
 #ifdef MTMD_VIDEO
     if (!ctx) return;
     ctx->stop_ffmpeg();
+    mtmd_bitmap_free(ctx->pending_frame);
+    mtmd_bitmap_free(ctx->pending_audio);
     delete ctx;
 #else
     GGML_UNUSED(ctx);

tools/mtmd/mtmd-helper.h

@@ -27,6 +27,7 @@ struct mtmd_helper_video_init_params {
     float fps_target;            // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f
     const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH
     int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms
+    bool no_soundtrack;          // frames only, also for models that hear the soundtrack of a video (LFM3-Omni)
     // TODO @ngxson : allow "placeholder" bitmap output for counting tokens
 };
 

tools/mtmd/mtmd-image.cpp

@@ -823,17 +823,26 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_i
 //
 
 // rounds before the pixel checks (an edge may round to 0) and computes in double, as transformers does
-clip_image_size mtmd_image_preprocessor_lfm3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels) {
-    const double h = height;
-    const double w = width;
+clip_image_size mtmd_image_preprocessor_lfm3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels,
+                                                             int n_frames, int temporal_factor) {
+    double h = height;
+    double w = width;
+    // the video variant first upscales edges below factor
+    if (temporal_factor > 1 && (h < factor || w < factor)) {
+        const double scale = std::max(factor / h, factor / w);
+        h = static_cast<int>(h * scale);
+        w = static_cast<int>(w * scale);
+    }
+
+    const int64_t t_bar = static_cast<int64_t>(std::nearbyint(static_cast<double>(n_frames) / temporal_factor)) * temporal_factor;
     int h_bar = static_cast<int>(std::nearbyint(h / factor)) * factor;
     int w_bar = static_cast<int>(std::nearbyint(w / factor)) * factor;
-    if (static_cast<int64_t>(h_bar) * w_bar > max_pixels) {
-        const double beta = std::sqrt(h * w / max_pixels);
+    if (t_bar * h_bar * w_bar > max_pixels) {
+        const double beta = std::sqrt(n_frames * h * w / max_pixels);
         h_bar = static_cast<int>(std::floor(h / beta / factor)) * factor;
         w_bar = static_cast<int>(std::floor(w / beta / factor)) * factor;
-    } else if (static_cast<int64_t>(h_bar) * w_bar < min_pixels) {
-        const double beta = std::sqrt(min_pixels / (h * w));
+    } else if (t_bar * h_bar * w_bar < min_pixels) {
+        const double beta = std::sqrt(min_pixels / (n_frames * h * w));
         h_bar = static_cast<int>(std::ceil(h * beta / factor)) * factor;
         w_bar = static_cast<int>(std::ceil(w * beta / factor)) * factor;
     }
@@ -846,6 +855,32 @@ clip_image_size mtmd_image_preprocessor_lfm3vl::target_size(const clip_image_siz
                         hparams.image_min_pixels, hparams.image_max_pixels);
 }
 
+bool mtmd_image_preprocessor_lfm3vl::get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const {
+    // Lfm3VlVideoProcessor size: shortest_edge 4096, longest_edge 25165824, temporal_patch_size 2
+    out = smart_resize(width, height, hparams.patch_size * hparams.n_merge, 4096, 25165824, n_frames, 2);
+    return true;
+}
+
+mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::preprocess_video_frame(const clip_image_u8 & img) const {
+    const int f = hparams.patch_size * hparams.n_merge;
+    const auto size = img.get_size();
+    return resize_to(img, {
+        std::max(f, static_cast<int>(std::nearbyint(static_cast<double>(size.width)  / f)) * f),
+        std::max(f, static_cast<int>(std::nearbyint(static_cast<double>(size.height) / f)) * f),
+    });
+}
+
+mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::resize_to(const clip_image_u8 & img, const clip_image_size & size) const {
+    clip_image_u8 resized_image;
+    img_tool::resize(img, resized_image, size,
+                        hparams.image_resize_algo,
+                        hparams.image_resize_pad,
+                        hparams.image_pad_color);
+    mtmd_image_preproc_out output;
+    output.append(hparams, resized_image, true);
+    return output;
+}
+
 //
 // mtmd_image_preprocessor_glm5v
 //

tools/mtmd/mtmd-image.h

@@ -35,6 +35,10 @@ struct mtmd_image_preprocessor {
 
     virtual ~mtmd_image_preprocessor() = default;
     virtual mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const = 0;
+    // frames of a video, which the video helper may have already sized
+    virtual mtmd_image_preproc_out preprocess_video_frame(const clip_image_u8 & img) const { return preprocess(img); }
+    // frame size of a video with n_frames sampled frames, as the reference video processor; false if none
+    virtual bool get_video_frame_size(int /*n_frames*/, int /*width*/, int /*height*/, clip_image_size & /*out*/) const { return false; }
 };
 
 /**
@@ -142,8 +146,14 @@ struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor {
 struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_dyn_size {
     mtmd_image_preprocessor_lfm3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_dyn_size(ctx) {}
     clip_image_size target_size(const clip_image_size & original_size) const override;
+    // the video helper sizes the frames (get_video_frame_size), they are only snapped to the grid
+    mtmd_image_preproc_out preprocess_video_frame(const clip_image_u8 & img) const override;
+    bool get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const override;
 
-    static clip_image_size smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels);
+    mtmd_image_preproc_out resize_to(const clip_image_u8 & img, const clip_image_size & size) const;
+    // with temporal_factor > 1 it is the Qwen3-VL video variant over n_frames
+    static clip_image_size smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels,
+                                        int n_frames = 1, int temporal_factor = 1);
 };
 
 // GLM 5.3 flash, similar to dyn_size, but each edge is aligned up to patch_size*n_merge, and max budget is met by a search over the height with the width scaled proportionally

tools/mtmd/mtmd.cpp

@@ -25,11 +25,12 @@
 #include <cstdlib>
 #include <cstring>
 #include <climits>
+#include <map>
 #include <type_traits>
 #include <vector>
 
 // remember to bump this if the serialization format changes
-#define MTMD_SERIALIZATION_VERSION 3
+#define MTMD_SERIALIZATION_VERSION 4
 
 // oldest compat version that can be loaded
 #define MTMD_SERIALIZATION_VERSION_MIN 2
@@ -174,6 +175,7 @@ struct mtmd_bitmap {
     std::string id; // optional user-defined id, for ex: can be set to image hash, useful for KV cache tracking
     bool is_audio = false; // true if the bitmap is audio
     bool mergeable = false; // [QWEN_VIDEO] set only on frames of the same video
+    bool is_video  = false; // see mtmd_bitmap_set_is_video()
 
     // lazy-loaded bitmap
     mtmd_bitmap_lazy_callback lazy_callback = nullptr;
@@ -209,6 +211,10 @@ struct mtmd_bitmap {
         return data.size();
     }
 
+    bool is_video_frame() const {
+        return mergeable && is_video;
+    }
+
     bool can_merge_with(const mtmd_bitmap & other) const {
         // [QWEN_VIDEO] can (temporal) merge if both are images with same size
         return mergeable && other.mergeable
@@ -326,14 +332,26 @@ struct mtmd_image_tokens {
 };
 using mtmd_image_tokens_ptr = std::unique_ptr<mtmd_image_tokens>;
 
+// an audio track split across chunks (e.g. the soundtrack of a video): it is encoded once, as one clip, and every
+// chunk takes its rows of it
+struct mtmd_audio_track {
+    clip_image_f32_batch batch_f32; // the mel of the whole track
+    uint32_t n_tokens = 0;
+    std::vector<float> embd;        // filled by the first encode
+};
+
 struct mtmd_audio_tokens {
     uint32_t n_tokens = 0; // number of tokens
     clip_image_f32_batch batch_f32; // preprocessed image patches
     std::string id; // optional user-defined ID, useful for KV cache tracking
 
+    // a slice of a track: the rows [row_begin, row_begin + n_tokens) of it, batch_f32 is unused
+    std::shared_ptr<mtmd_audio_track> track;
+    uint32_t row_begin = 0;
+
     // true if one of entries in batch_f32 is a placeholder
     bool is_placeholder() const {
-        for (const auto & entry : batch_f32.entries) {
+        for (const auto & entry : (track ? track->batch_f32 : batch_f32).entries) {
             if (entry.is_placeholder()) {
                 return true;
             }
@@ -345,7 +363,9 @@ struct mtmd_audio_tokens {
         return mtmd_audio_tokens{
             n_tokens,
             batch_f32.clone(),
-            id
+            id,
+            track,
+            row_begin,
         };
     }
 
@@ -354,11 +374,25 @@ struct mtmd_audio_tokens {
         ser.write(n_tokens);
         ser.write(id);
         batch_f32.serialize(ser);
+        // a slice keeps its own copy of the track
+        ser.write(track != nullptr);
+        if (track) {
+            ser.write(row_begin);
+            ser.write(track->n_tokens);
+            track->batch_f32.serialize(ser);
+        }
     }
     void deserialize(mtmd_serialization & ser) {
         n_tokens = ser.read<uint32_t>();
         id = ser.read<std::string>();
         batch_f32.deserialize(ser);
+        track.reset();
+        if (ser.version >= 4 && ser.read<bool>()) {
+            row_begin = ser.read<uint32_t>();
+            track = std::make_shared<mtmd_audio_track>();
+            track->n_tokens = ser.read<uint32_t>();
+            track->batch_f32.deserialize(ser);
+        }
     }
 };
 using mtmd_audio_tokens_ptr = std::unique_ptr<mtmd_audio_tokens>;
@@ -538,6 +572,8 @@ struct mtmd_context {
     std::string img_end;
     std::string aud_beg;
     std::string aud_end;
+    std::string vid_beg; // wraps a whole video (lazy bitmap with is_video)
+    std::string vid_end;
 
     // for llava-uhd style models, we need special tokens in-between slices
     // minicpmv calls them "slices", llama 4 calls them "tiles"
@@ -777,6 +813,8 @@ struct mtmd_context {
                     // <|vision_start|> ... (image embeddings) ... <|vision_end|>
                     img_beg = "<|vision_start|>";
                     img_end = "<|vision_end|>";
+                    vid_beg = img_beg;
+                    vid_end = img_end;
                     image_preproc = std::make_unique<mtmd_image_preprocessor_lfm3vl>(ctx_v);
                 } break;
             case PROJECTOR_TYPE_YOUTUVL:
@@ -1158,6 +1196,11 @@ struct mtmd_context {
         return ctx_a ? clip_get_projector_type(ctx_a) : PROJECTOR_TYPE_UNKNOWN;
     }
 
+    // [LFM3_OMNI] the audio of a video is its soundtrack, see MTMD_VIDEO_TEMPLATE_LFM3_OMNI
+    bool video_has_soundtrack() const {
+        return proj_type_v() == PROJECTOR_TYPE_LFM3VL && proj_type_a() == PROJECTOR_TYPE_LFM3A;
+    }
+
     int64_t n_embd_out() const {
         if (ctx_v) {
             return clip_n_mmproj_embd(ctx_v);
@@ -1256,6 +1299,16 @@ struct mtmd_tokenizer {
     std::vector<mtmd::bitmap> bm_from_lazy; // TODO @ngxson : refactor, free bm_from_lazy progressively
     std::vector<const char *> text_from_lazy;
 
+    // [LFM3_OMNI] the audio pieces of a video are joined back and encoded as one clip, each piece takes its rows
+    struct soundtrack {
+        std::vector<const mtmd_bitmap *> pieces;
+        std::shared_ptr<mtmd_audio_track> track; // built on the first piece
+        size_t   n_samples  = 0; // samples of the pieces added so far
+        uint32_t n_assigned = 0; // rows of the pieces added so far
+    };
+    std::vector<soundtrack> soundtracks;
+    std::map<const mtmd_bitmap *, size_t> soundtrack_of; // piece -> index in soundtracks
+
     mtmd_input_chunks cur;
     uint32_t n_images_added = 0; // 0-based index assigned to the next image chunk
 
@@ -1329,6 +1382,11 @@ struct mtmd_tokenizer {
         for (auto & p : parts) {
             if (p.bitmap != nullptr && p.bitmap->lazy_callback) {
                 LOG_DBG("%s: expanding lazy bitmap\n", __func__);
+                const bool is_video = p.bitmap->is_video;
+                if (is_video && !ctx->vid_beg.empty()) {
+                    expanded.push_back({ctx->vid_beg, nullptr, true});
+                }
+                soundtrack st;
                 for (size_t i = 0;; i++) {
                     char * out_str = nullptr;
                     mtmd_bitmap * out_bm = nullptr;
@@ -1344,6 +1402,9 @@ struct mtmd_tokenizer {
                         if (out_bm) {
                             auto & ptr = bm_from_lazy.emplace_back(out_bm); // remember to free it later
                             expanded.push_back({"", ptr.ptr.get()});
+                            if (is_video && out_bm->is_audio && ctx->video_has_soundtrack()) {
+                                st.pieces.push_back(out_bm);
+                            }
                             LOG_DBG("%s: lazy callback returned bitmap with dimensions %d x %d\n", __func__, out_bm->nx, out_bm->ny);
                         } else if (out_str) {
                             auto & ptr = text_from_lazy.emplace_back(out_str); // remember to free it later
@@ -1352,12 +1413,21 @@ struct mtmd_tokenizer {
                         }
                     } else if (res == -1) {
                         // EOF: lazy part removes itself (not added to expanded)
+                        if (is_video && !ctx->vid_end.empty()) {
+                            expanded.push_back({ctx->vid_end, nullptr, true});
+                        }
                         break;
                     } else if (res == -2) {
                         // error
                         throw std::runtime_error(string_format("lazy callback returned error"));
                     }
                 }
+                if (!st.pieces.empty()) {
+                    for (const auto * piece : st.pieces) {
+                        soundtrack_of[piece] = soundtracks.size();
+                    }
+                    soundtracks.push_back(std::move(st));
+                }
             } else {
                 expanded.push_back(std::move(p));
             }
@@ -1501,7 +1571,9 @@ struct mtmd_tokenizer {
                 img_u8.cpy_buf(bmp->get_ro_buf());
 
                 // preprocess image
-                mtmd_image_preproc_out tmp_preproc_out = ctx->image_preproc->preprocess(img_u8);
+                mtmd_image_preproc_out tmp_preproc_out = bmp->is_video_frame()
+                    ? ctx->image_preproc->preprocess_video_frame(img_u8)
+                    : ctx->image_preproc->preprocess(img_u8);
 
                 // move entries and grid dimensions to the "global" preproc_out
                 for (auto & entry : tmp_preproc_out.entries) {
@@ -1667,7 +1739,7 @@ struct mtmd_tokenizer {
 
                 clip_image_f32_batch batch_f32;
                 batch_f32.is_audio = false;
-                batch_f32.is_video = bitmaps[0]->mergeable;
+                batch_f32.is_video = bitmaps[0]->is_video_frame();
                 batch_f32.entries = std::move(preproc_out.entries);
                 // do NOT use preproc_out from this point on, it's moved
 
@@ -1705,6 +1777,10 @@ struct mtmd_tokenizer {
                 return 2;
             }
 
+            if (auto it = soundtrack_of.find(bitmap); it != soundtrack_of.end()) {
+                return add_soundtrack_piece(soundtracks[it->second], *bitmap);
+            }
+
             if (bitmap->nx == 0) {
                 LOG_ERR("%s: error: empty audio data\n", __func__);
                 return 2;
@@ -1793,6 +1869,58 @@ struct mtmd_tokenizer {
         return 0;
     }
 
+    // a piece of a soundtrack goes in without the audio markers, it lies inside the span of its video
+    int32_t add_soundtrack_piece(soundtrack & st, const mtmd_bitmap & bitmap) {
+        if (!st.track) {
+            std::vector<float> samples;
+            for (const auto * piece : st.pieces) {
+                if (piece->is_placeholder()) {
+                    samples.resize(samples.size() + piece->nx, 0.0f);
+                } else {
+                    const float * data = (const float *) piece->get_ro_buf().data();
+                    samples.insert(samples.end(), data, data + piece->nx);
+                }
+            }
+            std::vector<mtmd_audio_mel> mels;
+            if (!ctx->audio_preproc || !ctx->audio_preproc->preprocess(samples.data(), samples.size(), mels) || mels.size() != 1) {
+                LOG_ERR("%s: unable to preprocess the soundtrack\n", __func__);
+                return 2;
+            }
+            clip_image_f32 mel_f32;
+            mel_f32.set_size({(int) mels[0].n_len, (int) mels[0].n_mel}, mels[0].data.empty(), /* is_audio */ true);
+            mel_f32.cpy_buf(mels[0].data);
+            st.track = std::make_shared<mtmd_audio_track>();
+            st.track->n_tokens = clip_n_output_tokens(ctx->ctx_a, &mel_f32);
+            st.track->batch_f32.is_audio = true;
+            st.track->batch_f32.entries.push_back(std::move(mel_f32));
+        }
+
+        // rows of [begin, end): round(sample / samples per row), the last piece takes the rest of the track
+        const double samples_per_row = 8.0 * clip_get_hparams(ctx->ctx_a)->audio_hop_len; // one row per 8 mel frames
+        const uint32_t n_tokens = st.track->n_tokens;
+        st.n_samples += bitmap.nx;
+        const uint32_t begin = st.n_assigned;
+        const uint32_t end   = &bitmap == st.pieces.back() ? n_tokens
+                             : std::clamp((uint32_t) std::nearbyint(st.n_samples / samples_per_row), begin, n_tokens);
+        st.n_assigned = end;
+        if (end == begin) {
+            return 0;
+        }
+
+        mtmd_audio_tokens_ptr audio_tokens(new mtmd_audio_tokens);
+        audio_tokens->n_tokens  = end - begin;
+        audio_tokens->track     = st.track;
+        audio_tokens->row_begin = begin;
+        audio_tokens->id        = bitmap.id;
+        cur.entries.emplace_back(mtmd_input_chunk{
+            MTMD_INPUT_CHUNK_TYPE_AUDIO,
+            {}, // text tokens
+            nullptr, // image tokens
+            std::move(audio_tokens),
+        });
+        return 0;
+    }
+
     // MiniCPM-V 4.7: the overview and all slices go in one chunk, clip appends the separators after each tile:
     //   [ov] </image><slice> [S00] </slice><slice> [S01] </slice>\n<slice> [S10] </slice><slice> [S11] </slice>
     void add_canvas_chunk(mtmd_image_preproc_out && preproc_out, const std::string & id) {
@@ -2049,6 +2177,20 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk
             return 1;
         }
         int n_mmproj_embd = ctx->n_embd_out();
+        const auto & track = chunk->tokens_audio->track;
+        if (track) {
+            if (track->embd.empty()) {
+                track->embd.resize((size_t) track->n_tokens * n_mmproj_embd);
+                if (!clip_image_batch_encode(ctx->ctx_a, ctx->n_threads, &track->batch_f32, track->embd)) {
+                    track->embd.clear();
+                    return 1;
+                }
+            }
+            const size_t off = (size_t) chunk->tokens_audio->row_begin * n_mmproj_embd;
+            out_embd.assign(track->embd.begin() + off,
+                            track->embd.begin() + off + (size_t) chunk->tokens_audio->n_tokens * n_mmproj_embd);
+            return 0;
+        }
         out_embd.resize((size_t)chunk->tokens_audio->n_tokens * n_mmproj_embd);
         bool ok = clip_image_batch_encode(
             ctx->ctx_a,
@@ -2579,6 +2721,36 @@ int mtmd_get_audio_sample_rate(const mtmd_context * ctx) {
     return clip_get_hparams(ctx->ctx_a)->audio_sample_rate;
 }
 
+mtmd_video_config mtmd_get_video_config(const mtmd_context * ctx,
+                                        int32_t src_width, int32_t src_height,
+                                        int32_t src_n_frames, float src_fps) {
+    mtmd_video_config cfg{MTMD_VIDEO_TEMPLATE_DEFAULT, 0.0f, 0, 0, 0};
+    if (!ctx->ctx_v || clip_get_projector_type(ctx->ctx_v) != PROJECTOR_TYPE_LFM3VL) {
+        return cfg;
+    }
+
+    // matches Lfm3VlVideoProcessor: 2 fps sampling with a [4, 768] frame clamp
+    // LFM3-Omni samples at most 384 frames and hears the soundtrack
+    const bool omni = ctx->video_has_soundtrack();
+    cfg.tmpl = omni ? MTMD_VIDEO_TEMPLATE_LFM3_OMNI : MTMD_VIDEO_TEMPLATE_LFM3;
+    cfg.fps  = 2.0f;
+    if (src_n_frames <= 0 || src_fps <= 0.0f) {
+        return cfg;
+    }
+
+    int32_t n = (int32_t)((double) src_n_frames / src_fps * cfg.fps);
+    n = std::clamp(n, 4, omni ? 384 : 768);
+    n = std::min(n, src_n_frames);
+    cfg.n_frames = n;
+
+    clip_image_size size;
+    if (ctx->image_preproc && ctx->image_preproc->get_video_frame_size(n, src_width, src_height, size)) {
+        cfg.frame_width  = size.width;
+        cfg.frame_height = size.height;
+    }
+    return cfg;
+}
+
 const char * mtmd_get_marker(const mtmd_context * ctx) {
     return ctx->media_marker.c_str();
 }
@@ -2648,6 +2820,10 @@ void mtmd_bitmap_set_mergeable(mtmd_bitmap * bitmap, bool mergeable) {
     bitmap->mergeable = mergeable;
 }
 
+void mtmd_bitmap_set_is_video(mtmd_bitmap * bitmap, bool is_video) {
+    bitmap->is_video = is_video;
+}
+
 mtmd_bitmap * mtmd_bitmap_init_lazy(const mtmd_context * ctx,
                                     const char * id,
                                     void * user_data,

tools/mtmd/mtmd.h

@@ -58,6 +58,13 @@ enum mtmd_input_chunk_type {
     MTMD_INPUT_CHUNK_TYPE_COUNT, // for validation
 };
 
+// how the frames and timestamps of a video are laid out in the prompt
+enum mtmd_video_template {
+    MTMD_VIDEO_TEMPLATE_DEFAULT,   // "Video:" prefix, then a "[XmY.YYs]" timestamp every N ms
+    MTMD_VIDEO_TEMPLATE_LFM3,      // "<X.X seconds>" before every group of 2 frames
+    MTMD_VIDEO_TEMPLATE_LFM3_OMNI, // same as LFM3, then the soundtrack of each group, as an audio bitmap after its frames
+};
+
 // opaque types
 struct mtmd_context;
 struct mtmd_bitmap;
@@ -156,6 +163,20 @@ MTMD_API bool mtmd_support_audio(const mtmd_context * ctx);
 // return -1 if audio is not supported
 MTMD_API int mtmd_get_audio_sample_rate(const mtmd_context * ctx);
 
+// recommended video input for the current model, given the source video
+// fields set to 0 mean "no preference, use the caller's default"
+struct mtmd_video_config {
+    enum mtmd_video_template tmpl;
+    float   fps;          // frame sampling rate
+    int32_t n_frames;     // number of frames to sample
+    int32_t frame_width;  // size to resize the frames to
+    int32_t frame_height;
+};
+// src_n_frames and src_fps can be <= 0 if unknown
+MTMD_API struct mtmd_video_config mtmd_get_video_config(const mtmd_context * ctx,
+                                                        int32_t src_width, int32_t src_height,
+                                                        int32_t src_n_frames, float src_fps);
+
 // get the current marker string
 MTMD_API const char * mtmd_get_marker(const mtmd_context * ctx);
 
@@ -166,7 +187,7 @@ MTMD_API const char * mtmd_get_marker(const mtmd_context * ctx);
 //     the data is in RGBRGBRGB... format
 //     note: some video-capable models (i.e. qwen-vl) can merge consecutive bitmaps
 //           into one chunk; mtmd_tokenize() handles this, but remember to set
-//           mtmd_bitmap_set_mergeable(true) for every frame
+//           mtmd_bitmap_set_mergeable(true) and mtmd_bitmap_set_is_video(true) for every frame
 // if bitmap is audio:
 //     length of data must be n_samples * sizeof(float)
 //     the data is in float format (PCM F32)
@@ -189,6 +210,10 @@ MTMD_API const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap);
 MTMD_API void         mtmd_bitmap_set_id(mtmd_bitmap * bitmap, const char * id);
 // if true, this bitmap can be merged (temporal merge) with an adjacent mergeable bitmap by certain video input models
 MTMD_API void         mtmd_bitmap_set_mergeable(mtmd_bitmap * bitmap, bool mergeable);
+// on a frame: video frames must be both mergeable and is_video
+// on a lazy bitmap: the expanded chunks are one video, wrapped by the model's video begin/end tokens
+//                   with MTMD_VIDEO_TEMPLATE_LFM3_OMNI, its audio bitmaps are the pieces of one soundtrack, in order
+MTMD_API void         mtmd_bitmap_set_is_video(mtmd_bitmap * bitmap, bool is_video);
 
 // mtmd_bitmap lazy
 //

tools/server/README.md

@@ -1465,6 +1465,7 @@ For multimodal input (typed content, `messages[i].content[j]`):
 - If `type == "input_video"`:
     - Either `input_video.data` or `input_video.url` can be specified, can be a remote URL, raw base64 or path to local file
     - Accepts formats supported by `ffmpeg`
+    - Models that hear their video (LFM3-Omni) also get its soundtrack, the audio of each group of frames follows them
 - Note: for local file, make sure to set `--media-path`. File path must be prefixed by `file://`
 
 *Examples:*

tools/server/server-decision.cpp

@@ -388,10 +388,10 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
 
 static const size_t DECISION_MAX_IMAGES = 8;
 
-// any media, mtmd tells an audio clip from an image by its content
+// any media, mtmd tells an audio clip or a video from an image by its content
 static void decision_load_image(const json & url, std::vector<raw_buffer> & files) {
     if (!url.is_string() || !string_starts_with(url.get<std::string>(), "data:")) {
-        throw std::invalid_argument("images must be data URLs (data:image/...;base64,... or data:audio/...;base64,...)");
+        throw std::invalid_argument("media must be data URLs (data:image/..., data:audio/... or data:video/...;base64,...)");
     }
     if (files.size() >= DECISION_MAX_IMAGES) {
         throw std::invalid_argument(string_format("too many images, the maximum is %zu", DECISION_MAX_IMAGES));
@@ -457,6 +457,9 @@ json server_decision_context::parse_state(const json & body, std::vector<raw_buf
             } else if (part.is_object() && json_value(part, "type", std::string()) == "input_audio" && part.contains("input_audio")) {
                 const json input_audio = json_value(part, "input_audio", json::object());
                 decision_load_audio(input_audio.contains("data") ? input_audio.at("data") : json_value(input_audio, "url", json()), files);
+            } else if (is_lfm3_d1() && part.is_object() && json_value(part, "type", std::string()) == "video_url" && part.contains("video_url")) {
+                const json & video_url = part.at("video_url");
+                decision_load_image(video_url.is_object() && video_url.contains("url") ? video_url.at("url") : video_url, files);
             } else {
                 content.push_back(part);
             }
@@ -1236,7 +1239,10 @@ void server_decision_context::fill_task(
     if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3 || type == COMMON_DECISION_TYPE_LFM2_D1 ||
         type == COMMON_DECISION_TYPE_H2O_LIGHTNING || type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) {
         if (!media.empty()) {
-            task.tokens = process_mtmd_prompt(mctx, prompt, media, init_opt);
+            // the d1 reference sees the frames of a video, not its soundtrack
+            mtmd_helper_init_opt opt = init_opt;
+            opt.video_params.no_soundtrack = is_lfm3_d1();
+            task.tokens = process_mtmd_prompt(mctx, prompt, media, opt);
             return;
         }
     }