llama : move no_alloc out of llama_hparams (#31141)

* llama : move no_alloc out of llama_hparams

Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (2x DGX Spark (RDMA, TP=2))

* cont : clean-up

* cont : clean-up

Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (2x DGX Spark (RDMA, TP=2))

* tests : add a no_alloc case to test-llama-context

Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (2x DGX Spark (RDMA, TP=2))
This commit is contained in:
Georgi Gerganov authored and GitHub committed 2026-10-26 16:21:25 +02:00
1 parent 4b851fbee8
commit a4699400e3
30 files changed
+266 -179

No files matched your search

+9 -8
View File
@@ -489,6 +489,7 @@ llama_context::llama_context(
/*.swa_full =*/ params.swa_full,
/*.ctx_type =*/ cparams.ctx_type,
/*.mem_other =*/ llama_get_memory(cparams.ctx_other),
/*.no_alloc =*/ model.no_alloc(),
};
memory.reset(model.create_memory(params_mem, cparams));
@@ -559,7 +560,7 @@ llama_context::llama_context(
}
if (cparams.moe_cache_size > 0) {
moe_cache = std::make_unique<llama_moe_cache>(model, backend_ptrs, backend_buft, cparams.moe_cache_size);
moe_cache = std::make_unique<llama_moe_cache>(model, model.no_alloc(), backend_ptrs, backend_buft, cparams.moe_cache_size);
}
sched_reserve();
@@ -587,7 +588,7 @@ llama_context::~llama_context() {
synchronize();
// when training, ggml_opt allocates extra buffers through the scheduler, so the sizes no longer match the expectation
if (!model.hparams.no_alloc && !opt_ctx) {
if (!model.no_alloc() && !opt_ctx) {
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
ggml_backend_t backend = backend_ptrs[i];
ggml_backend_buffer_type_t buft = backend_buft[i];
@@ -788,7 +789,7 @@ void llama_context::sched_reserve() {
// reserve pp (prompt processing) graph first so that buffers are only allocated once
{
auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(),
model.hparams.no_alloc, model.hparams.no_alloc ? backend_buf_exp_size.data() : nullptr);
model.no_alloc(), model.no_alloc() ? backend_buf_exp_size.data() : nullptr);
if (!gf) {
if (cparams.pipeline_parallel) {
LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__);
@@ -815,7 +816,7 @@ void llama_context::sched_reserve() {
n_splits_tg = n_splits_pp;
n_nodes_tg = n_nodes_pp;
} else {
auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.hparams.no_alloc);
auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.no_alloc());
if (!gf) {
throw std::runtime_error("failed to allocate compute tg buffers");
}
@@ -837,10 +838,10 @@ void llama_context::sched_reserve() {
// [TAG_RESERVE_DIAG_DECAY]
// the `inp_diag_decay` tensor size scales with `n_seq_tokens^2` which
// makes `n_seqs == 1` use more memory for the compute graph compared to `n_seqs > 1`
gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.hparams.no_alloc);
gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.no_alloc());
break;
default:
gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc);
gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.no_alloc());
};
if (!gf) {
@@ -851,7 +852,7 @@ void llama_context::sched_reserve() {
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
ggml_backend_t backend = backend_ptrs[i];
ggml_backend_buffer_type_t buft = backend_buft[i];
if (!model.hparams.no_alloc) {
if (!model.no_alloc()) {
backend_buf_exp_size[i] = ggml_backend_sched_get_buffer_size(sched.get(), backend);
}
if (backend_buf_exp_size[i] > 1) {
@@ -3730,7 +3731,7 @@ llama_memory_breakdown llama_context::memory_breakdown() const {
ret[buft].context += size;
}
}
if (model.hparams.no_alloc) {
if (model.no_alloc()) {
for (size_t i = 0; i < backends.size(); ++i) {
ggml_backend_t backend = backends[i].get();
ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend);