Pascal
cf24317bea
* common: bound control vector layer indices by the model layer count A direction tensor layer index is parsed from the tensor name with no upper bound, and the buffer size and write offset are computed in int. A large index overflows both, so the accumulation writes before the start of the buffer. Pass the model layer count to the loader, reject indices above it like the existing zero index check, and compute the sizes in size_t. * mtmd: fill M-RoPE positions by whole merge tiles The position loops of the Qwen-VL, GLM, PaddleOCR and Qwen2.5-VL encoders hardcode a 2x2 merge inside a stride of n_merge, so a patch grid that is not a multiple of the merge writes past the positions buffer. Iterate over whole merge tiles with n_merge inner steps, which bounds the writes by the patch count and gives the same positions on aligned grids. PaddleOCR also requires a merge aligned image at graph build, as the Qwen-VL family already does.
56 lines
1.9 KiB
C++
56 lines
1.9 KiB
C++
#include "models.h"
|
|
|
|
ggml_cgraph * clip_graph_paddleocr::build() {
|
|
GGML_ASSERT(img.nx() % (patch_size * model.hparams.n_merge) == 0);
|
|
GGML_ASSERT(img.ny() % (patch_size * model.hparams.n_merge) == 0);
|
|
|
|
const int n_pos = n_patches;
|
|
const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position
|
|
|
|
int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};
|
|
|
|
ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);
|
|
ggml_set_name(positions, "positions");
|
|
ggml_set_input(positions);
|
|
|
|
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
|
|
return ggml_rope_multi(
|
|
ctx0, cur, positions, nullptr,
|
|
d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
|
|
32768, 10000, 1, 0, 1, 32, 1);
|
|
};
|
|
|
|
ggml_tensor * learned_pos_embd = resize_position_embeddings();
|
|
ggml_tensor * inp = build_inp();
|
|
ggml_tensor * cur = build_vit(
|
|
inp, n_patches,
|
|
NORM_TYPE_NORMAL,
|
|
hparams.ffn_op,
|
|
learned_pos_embd,
|
|
add_pos);
|
|
|
|
cb(cur, "vit_out", -1);
|
|
|
|
{
|
|
// mlp_AR paddleocr projector
|
|
float proj_norm_eps = 1e-5;
|
|
cur = build_norm(cur,
|
|
model.mm_input_norm_w, model.mm_input_norm_b,
|
|
NORM_TYPE_NORMAL, proj_norm_eps, -1);
|
|
|
|
const int scale_factor = model.hparams.n_merge;
|
|
cur = build_patch_merge_permute(cur, scale_factor);
|
|
cur = build_ffn(cur,
|
|
model.mm_1_w, model.mm_1_b,
|
|
nullptr, nullptr,
|
|
model.mm_2_w, model.mm_2_b,
|
|
hparams.ffn_op, -1);
|
|
cb(cur, "mlp_out", -1);
|
|
}
|
|
|
|
// build the graph
|
|
ggml_build_forward_expand(gf, cur);
|
|
|
|
return gf;
|
|
}
|