llama: support both embd + raw tokens in batch (#29622)
* llama: support both embd + raw tokens in batch * add to test-llama-archs * also check case llm_arch_supports_mixed_batch = false * constant graph topology * have dedicated input for mixed case * rm set_tensor_backend * is_embd --> type * consolidate m-rope pos handling into one place * nits
This commit is contained in:
1 parent
2ca15f5404
commit
0bb496dbd3
29 files changed
+506
-99
No files matched your search
@@ -604,6 +604,7 @@ llama_model_dflash::graph<false>::graph(const llama_model & model, const llm_gra
|
||||
};
|
||||
|
||||
// KV cache injection
|
||||
ASSERT_EMBD_OR_TOKEN(ubatch);
|
||||
if (ubatch.embd) {
|
||||
auto inp = std::make_unique<llm_graph_input_embd>(n_embd_inp);
|
||||
|
||||
@@ -870,6 +871,7 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
|
||||
llm_graph_input_attn_k_iswa * inp_attn = build_attn_inp_k_iswa();
|
||||
|
||||
// KV cache injection: fused target features from the encoder
|
||||
ASSERT_EMBD_OR_TOKEN(ubatch);
|
||||
if (ubatch.embd) {
|
||||
auto inp = std::make_unique<llm_graph_input_embd>(n_embd_inp);
|
||||
|
||||
|
||||
Reference in new issue
Block a user