From fa3645feb5f47cbb30df5e6d031c22543d43d240 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 25 Oct 2026 11:56:04 +0200 Subject: [PATCH] llama : respect no_alloc in all memory caches (#31087) * llama : respect no_alloc in memory caches Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB) * llama : TODO: move no_alloc out of llama_hparams Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB) * llama : TODO: move no_alloc out of llama_hparams Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB) --- src/llama-hparams.h | 2 +- src/llama-kv-cache-dsv4.cpp | 23 ++++++++++++++++++----- src/llama-kv-cache-dsv4.h | 2 ++ src/llama-memory-recurrent.cpp | 21 ++++++++++++++++++--- 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 2b92f5d60..a3bf9e688 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -61,7 +61,7 @@ struct llama_hparams { // for example: n_embd_out() vs n_embd_out_impl bool vocab_only; - bool no_alloc; + bool no_alloc; // TODO: no_alloc is not a model hyperparameter; move it out of llama_hparams bool rope_finetuned; bool use_par_res; bool swin_norm; diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp index e479dc564..c18d913da 100644 --- a/src/llama-kv-cache-dsv4.cpp +++ b/src/llama-kv-cache-dsv4.cpp @@ -946,13 +946,12 @@ llama_dsv4_comp_state::llama_dsv4_comp_state( uint32_t n_rs_seq, const char * name, const llama_memory_i::layer_filter_cb & filter) : + hparams(model.hparams), ratio(ratio), state_size(state_size), n_embd_state(n_embd_state), n_stream(unified ? 1 : n_seq_max), n_rs_seq(n_rs_seq) { - const llama_hparams & hparams = model.hparams; - struct ggml_backend_buft_comparator { bool operator()(const ggml_backend_buffer_type_t & lhs, const ggml_backend_buffer_type_t & rhs) const { return strcmp(ggml_backend_buft_name(lhs), ggml_backend_buft_name(rhs)) < 0; @@ -1027,7 +1026,15 @@ llama_dsv4_comp_state::llama_dsv4_comp_state( } for (auto & [buft, ctx] : ctx_map) { - ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); + ggml_backend_buffer_t buf; + if (hparams.no_alloc) { + buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer + for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) { + t->buffer = buf; // set dummy buffer for DSV4 state so that the backend scheduler won't try to allocate it + } + } else { + buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer + } if (!buf) { throw std::runtime_error("failed to allocate buffer for DSV4 compressor state"); } @@ -1119,9 +1126,15 @@ uint32_t llama_dsv4_comp_state::get_n_rows() const { std::map llama_dsv4_comp_state::memory_breakdown() const { std::map ret; - for (const auto & [_, buf] : ctxs_bufs) { + for (const auto & [ctx, buf] : ctxs_bufs) { ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get()); - ret[buft] += ggml_backend_buffer_get_size(buf.get()); + + if (hparams.no_alloc) { + GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr); + ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft); + } else { + ret[buft] += ggml_backend_buffer_get_size(buf.get()); + } } return ret; } diff --git a/src/llama-kv-cache-dsv4.h b/src/llama-kv-cache-dsv4.h index 0368ea59f..99ac73ddb 100644 --- a/src/llama-kv-cache-dsv4.h +++ b/src/llama-kv-cache-dsv4.h @@ -61,6 +61,8 @@ private: std::vector score_stream; }; + const llama_hparams & hparams; + const uint32_t ratio; const uint32_t state_size; const uint32_t n_embd_state; diff --git a/src/llama-memory-recurrent.cpp b/src/llama-memory-recurrent.cpp index 2893d819e..d00630d4c 100644 --- a/src/llama-memory-recurrent.cpp +++ b/src/llama-memory-recurrent.cpp @@ -116,7 +116,15 @@ llama_memory_recurrent::llama_memory_recurrent( // allocate tensors and initialize the buffers to avoid NaNs in the padding for (auto & [buft, ctx] : ctx_map) { - ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); + ggml_backend_buffer_t buf; + if (hparams.no_alloc) { + buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer + for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) { + t->buffer = buf; // set dummy buffer for RS cache so that the backend scheduler won't try to allocate it + } + } else { + buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer + } if (!buf) { throw std::runtime_error("failed to allocate buffer for rs cache"); } @@ -435,8 +443,15 @@ void llama_memory_recurrent::set_rs_idx(llama_seq_id seq_id, uint32_t idx) { std::map llama_memory_recurrent::memory_breakdown() const { std::map ret; - for (const auto & [_, buf] : ctxs_bufs) { - ret[ggml_backend_buffer_get_type(buf.get())] += ggml_backend_buffer_get_size(buf.get()); + for (const auto & [ctx, buf] : ctxs_bufs) { + ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get()); + + if (hparams.no_alloc) { + GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr); + ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft); + } else { + ret[buft] += ggml_backend_buffer_get_size(buf.get()); + } } return ret; }