model: Add MiniCPM-OCR-Jev (#30980)
* model : add MiniCPM-OCR-Jev support The model is a Qwen3.5-VL trunk with a value head read at the last token: it scores an OCR text against an image rendered from it, it does not generate. Add the openbmb-minicpm-ocr-jev decision type, served by POST /v1/systemone. The class scores are the pooled output of the sequence, read with last pooling, so the conversion turns score.weight into cls.output and sets embedding_length_out. The checkpoint has no LM head, so text generation is rejected instead of using the output weights duplicated from the token embeddings. The task is fixed, so its prompt is entirely in the systemone template, the same way h2o-lightning keeps its default there. The processor of the checkpoint allows up to 16384 merged tokens per image, more than the Qwen-VL default of mtmd, so the mmproj keeps the image budget of the preprocessor config. The checkpoint declares MiniCPMOCRForSequenceClassification in its config.json, as other models do. Signed-off-by: tc-mb <tianchi_cai@icloud.com> * fix code Signed-off-by: tc-mb <tianchi_cai@icloud.com> * use gguf meta classifier_output_labels * remove redudant code changes --------- Signed-off-by: tc-mb <tianchi_cai@icloud.com> Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
This commit is contained in:
tc-mbandXuan Son Nguyen authored and 
GitHub committed
1 parent
276378515c
commit
7eed6b7685
13 files changed
+270
-10
No files matched your search
@@ -4,6 +4,8 @@
|
||||
#include "../tools/server/server-decision.h"
|
||||
#include "../tools/server/server-schema.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstdlib>
|
||||
#include <regex>
|
||||
#include <string>
|
||||
@@ -466,6 +468,116 @@ static void test_rerank_template(testing & t) {
|
||||
}
|
||||
|
||||
//
|
||||
// server_decision_context::format_answer, 2 class scores (minicpm-ocr-jev)
|
||||
//
|
||||
|
||||
static void test_decision_class_scores(testing & t) {
|
||||
server_decision_context ctx;
|
||||
ctx.type = COMMON_DECISION_TYPE_MINICPM_OCR_JEV;
|
||||
|
||||
const server_decision_question choice = {"q", SERVER_DECISION_QUESTION_CHOICE, json(), {{"0", json()}, {"1", json()}}};
|
||||
const server_decision_question noul = {"q", SERVER_DECISION_QUESTION_NOUL, json(), {{"false", json()}, {"true", json()}}};
|
||||
|
||||
const std::vector<std::pair<float, float>> cases = {
|
||||
{0.0f, 0.0f}, {-1.5f, 2.25f}, {3.0f, -0.5f}, {80.0f, -80.0f}, {1000.0f, 1001.0f},
|
||||
};
|
||||
for (const auto & [z0, z1] : cases) {
|
||||
const double p1 = 1.0 / (1.0 + std::exp((double) z0 - (double) z1));
|
||||
const json a = ctx.format_answer(choice, {{z0, z1}});
|
||||
const double ap0 = a.at("probabilities").at("0").get<double>();
|
||||
const double ap1 = a.at("probabilities").at("1").get<double>();
|
||||
t.assert_true("p1", std::fabs(ap1 - p1) <= 1e-6);
|
||||
t.assert_true("sum", std::fabs(ap0 + ap1 - 1.0) <= 1e-9);
|
||||
// a tie picks the first class
|
||||
t.assert_equal("choice", std::string(z1 > z0 ? "1" : "0"), a.at("choice").get<std::string>());
|
||||
t.assert_true("confidence", std::fabs(a.at("confidence").get<double>() - (2.0 * std::max(ap0, ap1) - 1.0)) <= 1e-9);
|
||||
|
||||
const json n = ctx.format_answer(noul, {{z0, z1}});
|
||||
t.assert_true("noul is p1", n.at("noul").get<double>() == ap1);
|
||||
}
|
||||
|
||||
for (const float bad : {std::nanf(""), INFINITY}) {
|
||||
bool thrown = false;
|
||||
try {
|
||||
ctx.format_answer(choice, {{bad, 0.0f}});
|
||||
} catch (const std::exception &) {
|
||||
thrown = true;
|
||||
}
|
||||
t.assert_true("nan or +inf", thrown);
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// server_decision_context::parse_questions / parse_state, the request checks of minicpm-ocr-jev
|
||||
//
|
||||
|
||||
static void test_decision_ocr_jev_validation(testing & t) {
|
||||
server_decision_context ctx;
|
||||
ctx.type = COMMON_DECISION_TYPE_MINICPM_OCR_JEV;
|
||||
|
||||
// only base64-decoded, the image itself is not parsed here
|
||||
const std::string image = "data:image/png;base64,AAAA";
|
||||
|
||||
const json choice = {
|
||||
{"type", "choice"},
|
||||
{"criteria", json{{"0", nullptr}, {"1", nullptr}}},
|
||||
};
|
||||
const auto questions_of = [](const json & question) {
|
||||
return json{{"state", "some OCR text"}, {"questions", json{{"q", question}}}};
|
||||
};
|
||||
const auto files_of = [&](size_t n) {
|
||||
json files = json::array();
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
files.push_back(image);
|
||||
}
|
||||
return json{{"state", "some OCR text"}, {"files", files}};
|
||||
};
|
||||
|
||||
const auto reject_questions = [&](const char * name, const json & body, const std::string & expected) {
|
||||
std::string error;
|
||||
try {
|
||||
ctx.parse_questions(body);
|
||||
} catch (const std::invalid_argument & e) {
|
||||
error = e.what();
|
||||
}
|
||||
t.assert_true(std::string(name) + ": " + error, error.find(expected) != std::string::npos);
|
||||
};
|
||||
const auto reject_state = [&](const char * name, const json & body, const std::string & expected) {
|
||||
std::vector<raw_buffer> files;
|
||||
std::string error;
|
||||
try {
|
||||
ctx.parse_state(body, files);
|
||||
} catch (const std::invalid_argument & e) {
|
||||
error = e.what();
|
||||
}
|
||||
t.assert_true(std::string(name) + ": " + error, error.find(expected) != std::string::npos);
|
||||
};
|
||||
|
||||
reject_state("state is text", {{"state", 42}}, "\"state\" must be the OCR text");
|
||||
|
||||
// one fixed question, no instructions
|
||||
reject_questions("one question", {{"state", "x"}, {"questions", json{{"a", choice}, {"b", choice}}}},
|
||||
"answers exactly one question");
|
||||
reject_questions("no instructions",
|
||||
questions_of(json{{"type", "choice"}, {"instructions", "judge it"}, {"criteria", json{{"0", nullptr}, {"1", nullptr}}}}),
|
||||
"\"instructions\" is not used by this model");
|
||||
|
||||
// the classes are fixed, so are the criteria
|
||||
reject_questions("criteria are the classes",
|
||||
questions_of(json{{"type", "choice"}, {"criteria", json{{"yes", nullptr}, {"no", nullptr}}}}),
|
||||
"\"criteria\" must be {\"0\": null, \"1\": null}");
|
||||
reject_questions("noul without criteria",
|
||||
questions_of(json{{"type", "noul"}, {"criteria", json{{"false", "no"}, {"true", "yes"}}}}),
|
||||
"\"criteria\" is not supported for a noul question");
|
||||
reject_questions("choice or noul only",
|
||||
questions_of(json{{"type", "score"}, {"criteria", json::array({"low", "high"})}}),
|
||||
"only choice and noul questions are supported");
|
||||
|
||||
// the prompt has the original image, then the one rendered from the OCR text
|
||||
reject_state("two images", files_of(1), "this model needs exactly 2 images");
|
||||
reject_state("two images, three given", files_of(3), "this model needs exactly 2 images");
|
||||
}
|
||||
|
||||
// server_decision_context: the recursive functions stop at RECURSIVE_LIMIT
|
||||
//
|
||||
|
||||
@@ -573,6 +685,8 @@ int main(int argc, char ** argv) {
|
||||
t.test("server_pipe", test_server_pipe);
|
||||
t.test("decision_escape", test_decision_escape);
|
||||
t.test("rerank_template", test_rerank_template);
|
||||
t.test("decision_class_scores", test_decision_class_scores);
|
||||
t.test("decision_ocr_jev_validation", test_decision_ocr_jev_validation);
|
||||
t.test("decision_depth", test_decision_depth);
|
||||
t.test("glob_match", test_glob_match);
|
||||
t.test("dry_sequence_breakers", test_dry_sequence_breakers);
|
||||
|
||||
Reference in new issue
Block a user