wmantly/llama.cpp-xrip
1
0 Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

llama : move no_alloc out of llama_hparams (#31141)

Browse Source 
* llama : move no_alloc out of llama_hparams

Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (2x DGX Spark (RDMA, TP=2))

* cont : clean-up

* cont : clean-up

Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (2x DGX Spark (RDMA, TP=2))

* tests : add a no_alloc case to test-llama-context

Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (2x DGX Spark (RDMA, TP=2))
This commit is contained in:
Georgi Gerganov authored and GitHub committed 2026-10-26 16:21:25 +02:00
1 parent 4b851fbee8
commit a4699400e3
30 files changed
+266 -179
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/llama-context.cpp
+9 -8
View File
@@ -489,6 +489,7 @@ llama_context::llama_context(
/*.swa_full =*/ params.swa_full,
/*.ctx_type =*/ cparams.ctx_type,
/*.mem_other =*/ llama_get_memory(cparams.ctx_other),
/*.no_alloc =*/ model.no_alloc(),
};
memory.reset(model.create_memory(params_mem, cparams));
@@ -559,7 +560,7 @@ llama_context::llama_context(
}
if (cparams.moe_cache_size > 0) {
moe_cache = std::make_unique<llama_moe_cache>(model, backend_ptrs, backend_buft, cparams.moe_cache_size);
moe_cache = std::make_unique<llama_moe_cache>(model, model.no_alloc(), backend_ptrs, backend_buft, cparams.moe_cache_size);
}
sched_reserve();
@@ -587,7 +588,7 @@ llama_context::~llama_context() {
synchronize();
// when training, ggml_opt allocates extra buffers through the scheduler, so the sizes no longer match the expectation
if (!model.hparams.no_alloc && !opt_ctx) {
if (!model.no_alloc() && !opt_ctx) {
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
ggml_backend_t backend = backend_ptrs[i];
ggml_backend_buffer_type_t buft = backend_buft[i];
@@ -788,7 +789,7 @@ void llama_context::sched_reserve() {
// reserve pp (prompt processing) graph first so that buffers are only allocated once
{
auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(),
model.hparams.no_alloc, model.hparams.no_alloc ? backend_buf_exp_size.data() : nullptr);
model.no_alloc(), model.no_alloc() ? backend_buf_exp_size.data() : nullptr);
if (!gf) {
if (cparams.pipeline_parallel) {
LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__);
@@ -815,7 +816,7 @@ void llama_context::sched_reserve() {
n_splits_tg = n_splits_pp;
n_nodes_tg = n_nodes_pp;
} else {
auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.hparams.no_alloc);
auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.no_alloc());
if (!gf) {
throw std::runtime_error("failed to allocate compute tg buffers");
}
@@ -837,10 +838,10 @@ void llama_context::sched_reserve() {
// [TAG_RESERVE_DIAG_DECAY]
// the `inp_diag_decay` tensor size scales with `n_seq_tokens^2` which
// makes `n_seqs == 1` use more memory for the compute graph compared to `n_seqs > 1`
gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.hparams.no_alloc);
gf = graph_reserve(n_tokens, 1, n_outputs_pp, mctx.get(), model.no_alloc());
break;
default:
gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc);
gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.no_alloc());
};
if (!gf) {
@@ -851,7 +852,7 @@ void llama_context::sched_reserve() {
for (size_t i = 0; i < backend_ptrs.size(); ++i) {
ggml_backend_t backend = backend_ptrs[i];
ggml_backend_buffer_type_t buft = backend_buft[i];
if (!model.hparams.no_alloc) {
if (!model.no_alloc()) {
backend_buf_exp_size[i] = ggml_backend_sched_get_buffer_size(sched.get(), backend);
}
if (backend_buf_exp_size[i] > 1) {
@@ -3730,7 +3731,7 @@ llama_memory_breakdown llama_context::memory_breakdown() const {
ret[buft].context += size;
}
}
if (model.hparams.no_alloc) {
if (model.no_alloc()) {
for (size_t i = 0; i < backends.size(); ++i) {
ggml_backend_t backend = backends[i].get();
ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend);
src/llama-hparams.h
-1
View File
@@ -61,7 +61,6 @@ struct llama_hparams {
// for example: n_embd_out() vs n_embd_out_impl
bool vocab_only;
bool no_alloc; // TODO: no_alloc is not a model hyperparameter; move it out of llama_hparams
bool rope_finetuned;
bool use_par_res;
bool swin_norm;
src/llama-kv-cache-dsa-iswa.cpp
+3 -2
View File
@@ -13,6 +13,7 @@
llama_kv_cache_dsa_iswa::llama_kv_cache_dsa_iswa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -63,14 +64,14 @@ llama_kv_cache_dsa_iswa::llama_kv_cache_dsa_iswa(
LLAMA_LOG_INFO("%s: creating DSA KV cache, size = %u cells\n", __func__, size_dsa);
kv_dsa = std::make_unique<llama_kv_cache_dsa>(
model, type_k, type_v,
model, no_alloc, type_k, type_v,
v_trans, offload, unified, size_dsa, n_seq_max, n_pad,
0, LLAMA_SWA_TYPE_NONE, filter_dsa, filter_lid, reuse);
LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa);
kv_swa = std::make_unique<llama_kv_cache>(
model, hparams, type_k, type_v,
model, no_alloc, hparams, type_k, type_v,
v_trans, offload, unified, size_swa, n_seq_max, n_pad,
hparams.n_swa, hparams.swa_type, nullptr, filter_swa, reuse, nullptr);
}
src/llama-kv-cache-dsa-iswa.h
+1
View File
@@ -14,6 +14,7 @@ class llama_kv_cache_dsa_iswa : public llama_memory_i {
public:
llama_kv_cache_dsa_iswa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
src/llama-kv-cache-dsa.cpp
+3 -2
View File
@@ -13,6 +13,7 @@
llama_kv_cache_dsa::llama_kv_cache_dsa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -31,7 +32,7 @@ llama_kv_cache_dsa::llama_kv_cache_dsa(
LLAMA_LOG_INFO("%s: creating main KV cache, size = %u cells\n", __func__, kv_size);
kv_mla = std::make_unique<llama_kv_cache>(
model, model.hparams, type_k, type_v,
model, no_alloc, model.hparams, type_k, type_v,
v_trans, offload, unified, kv_size, n_seq_max, n_pad,
n_swa, swa_type, nullptr, filter_mla, reuse, nullptr);
@@ -48,7 +49,7 @@ llama_kv_cache_dsa::llama_kv_cache_dsa(
LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size);
kv_lid = std::make_unique<llama_kv_cache>(
model, hparams_lid, type_k, type_v,
model, no_alloc, hparams_lid, type_k, type_v,
v_trans, offload, unified, kv_size, n_seq_max, n_pad,
n_swa, swa_type, nullptr, filter_lid, reuse, nullptr);
}
src/llama-kv-cache-dsa.h
+1
View File
@@ -16,6 +16,7 @@ class llama_kv_cache_dsa : public llama_memory_i {
public:
llama_kv_cache_dsa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
src/llama-kv-cache-dsv4.cpp
+23 -45
View File
@@ -931,21 +931,24 @@ static void dsv4_make_k_only(llama_hparams & hparams) {
llama_dsv4_comp_state::llama_dsv4_comp_state(
const llama_model & model,
bool offload,
bool unified,
uint32_t n_seq_max,
uint32_t ratio,
uint32_t state_size,
uint32_t n_embd_state,
uint32_t n_rs_seq,
const char * name,
bool no_alloc,
bool offload,
bool unified,
uint32_t n_seq_max,
uint32_t ratio,
uint32_t state_size,
uint32_t n_embd_state,
uint32_t n_rs_seq,
const char * name,
const llama_memory_i::layer_filter_cb & filter) :
hparams(model.hparams),
ratio(ratio),
state_size(state_size),
n_embd_state(n_embd_state),
n_stream(unified ? 1 : n_seq_max),
n_rs_seq(n_rs_seq) {
n_rs_seq(n_rs_seq),
ctxs_bufs(no_alloc) {
const llama_hparams & hparams = model.hparams;
struct ggml_backend_buft_comparator {
bool operator()(const ggml_backend_buffer_type_t & lhs, const ggml_backend_buffer_type_t & rhs) const {
return strcmp(ggml_backend_buft_name(lhs), ggml_backend_buft_name(rhs)) < 0;
@@ -1020,25 +1023,10 @@ llama_dsv4_comp_state::llama_dsv4_comp_state(
}
for (auto & [buft, ctx] : ctx_map) {
ggml_backend_buffer_t buf;
if (hparams.no_alloc) {
buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer
for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) {
t->buffer = buf; // set dummy buffer for DSV4 state so that the backend scheduler won't try to allocate it
}
} else {
buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer
}
if (!buf) {
throw std::runtime_error("failed to allocate buffer for DSV4 compressor state");
}
ggml_backend_buffer_clear(buf, 0);
ggml_backend_buffer_t buf = ctxs_bufs.emplace(std::move(ctx), buft, "DSV4 compressor state");
LLAMA_LOG_INFO("%s: %10s DSV4 %s state buffer size = %8.2f MiB\n",
__func__, ggml_backend_buffer_name(buf), name, ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
ctxs_bufs.emplace_back(std::move(ctx), buf);
}
LLAMA_LOG_INFO("%s: %s ratio = %u, state = %u x %u, streams = %u, rs_seq = %u, layers = %zu, size = %7.2f MiB\n",
@@ -1119,18 +1107,7 @@ uint32_t llama_dsv4_comp_state::get_n_rows() const {
}
std::map<ggml_backend_buffer_type_t, size_t> llama_dsv4_comp_state::memory_breakdown() const {
std::map<ggml_backend_buffer_type_t, size_t> ret;
for (const auto & [ctx, buf] : ctxs_bufs) {
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
if (hparams.no_alloc) {
GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr);
ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
} else {
ret[buft] += ggml_backend_buffer_get_size(buf.get());
}
}
return ret;
return ctxs_bufs.memory_breakdown();
}
void llama_dsv4_comp_state::state_write(
@@ -1265,6 +1242,7 @@ size_t llama_dsv4_comp_state::total_size() const {
llama_kv_cache_dsv4::llama_kv_cache_dsv4(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -1309,7 +1287,7 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
dsv4_make_k_only(hparams_raw);
kv_raw = std::make_unique<llama_kv_cache_iswa>(
model, hparams_raw, type_k, type_v,
model, no_alloc, hparams_raw, type_k, type_v,
v_trans, offload, swa_full, unified_raw, kv_size, n_seq_max, n_ubatch, n_pad,
nullptr, filter_raw, reuse, nullptr);
@@ -1413,7 +1391,7 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
__func__, dsv4_comp_size(kv_size, ratio_csa));
kv_csa = std::make_unique<llama_kv_cache>(
model, hparams_csa, type_k, type_v,
model, no_alloc, hparams_csa, type_k, type_v,
v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max, n_pad,
0, LLAMA_SWA_TYPE_NONE, nullptr, filter_csa, nullptr, nullptr);
@@ -1421,7 +1399,7 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
__func__, dsv4_comp_size(kv_size, ratio_hca));
kv_hca = std::make_unique<llama_kv_cache>(
model, hparams_hca, type_k, type_v,
model, no_alloc, hparams_hca, type_k, type_v,
v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_hca), 256u), n_seq_max, n_pad,
0, LLAMA_SWA_TYPE_NONE, nullptr, filter_hca, nullptr, nullptr);
@@ -1429,7 +1407,7 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
__func__, dsv4_comp_size(kv_size, ratio_csa));
kv_lid = std::make_unique<llama_kv_cache>(
model, hparams_lid, type_k, type_v,
model, no_alloc, hparams_lid, type_k, type_v,
v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max, n_pad,
0, LLAMA_SWA_TYPE_NONE, nullptr, filter_lid, nullptr, nullptr);
@@ -1439,19 +1417,19 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
const uint32_t csa_state_embd = is_v41 ? model.hparams.n_embd_head_k() : 2*model.hparams.n_embd_head_k();
csa_state = std::make_unique<llama_dsv4_comp_state>(
model, offload, unified_compressed, n_seq_max, ratio_csa, csa_state_size,
model, no_alloc, offload, unified_compressed, n_seq_max, ratio_csa, csa_state_size,
csa_state_embd, n_rs_seq, "csa", filter_csa_state);
LLAMA_LOG_INFO("%s: creating DSV4 HCA compressor state\n", __func__);
hca_state = std::make_unique<llama_dsv4_comp_state>(
model, offload, unified_compressed, n_seq_max, ratio_hca, ratio_hca,
model, no_alloc, offload, unified_compressed, n_seq_max, ratio_hca, ratio_hca,
model.hparams.n_embd_head_k(), n_rs_seq, "hca", filter_hca_state);
LLAMA_LOG_INFO("%s: creating DSV4 lightning-indexer compressor state\n", __func__);
lid_state = std::make_unique<llama_dsv4_comp_state>(
model, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO,
model, no_alloc, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO,
2*model.hparams.indexer_head_size, n_rs_seq, "lid", filter_lid);
// DSV4 attention reads compressed-K / compressor-state rows that the current
src/llama-kv-cache-dsv4.h
+11 -11
View File
@@ -16,14 +16,15 @@ public:
llama_dsv4_comp_state(
const llama_model & model,
bool offload,
bool unified,
uint32_t n_seq_max,
uint32_t ratio,
uint32_t state_size,
uint32_t n_embd_state,
uint32_t n_rs_seq,
const char * name,
bool no_alloc,
bool offload,
bool unified,
uint32_t n_seq_max,
uint32_t ratio,
uint32_t state_size,
uint32_t n_embd_state,
uint32_t n_rs_seq,
const char * name,
const llama_memory_i::layer_filter_cb & filter);
void clear(llama_seq_id seq_id, bool data);
@@ -61,15 +62,13 @@ private:
std::vector<ggml_tensor *> score_stream;
};
const llama_hparams & hparams;
const uint32_t ratio;
const uint32_t state_size;
const uint32_t n_embd_state;
const uint32_t n_stream;
const uint32_t n_rs_seq;
std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;
llama_memory_bufs ctxs_bufs;
std::vector<layer> layers;
@@ -92,6 +91,7 @@ class llama_kv_cache_dsv4 : public llama_memory_i {
public:
llama_kv_cache_dsv4(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
src/llama-kv-cache-iswa.cpp
+6 -4
View File
@@ -13,6 +13,7 @@
llama_kv_cache_iswa::llama_kv_cache_iswa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -27,13 +28,14 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
const layer_filter_cb & filter,
const layer_reuse_cb & reuse,
const layer_share_cb & share) :
llama_kv_cache_iswa(model, model.hparams, type_k, type_v, v_trans, offload, swa_full, unified,
llama_kv_cache_iswa(model, no_alloc, model.hparams, type_k, type_v, v_trans, offload, swa_full, unified,
kv_size, n_seq_max, n_ubatch, n_pad, mem_other, filter, reuse, share) {
}
llama_kv_cache_iswa::llama_kv_cache_iswa(
const llama_model & model,
const llama_hparams & hparams,
bool no_alloc,
const llama_hparams & hparams,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -93,14 +95,14 @@ llama_kv_cache_iswa::llama_kv_cache_iswa(
}
kv_base = std::make_unique<llama_kv_cache>(
model, hparams, type_k, type_v,
model, no_alloc, hparams, type_k, type_v,
v_trans, offload, unified, size_base, n_seq_max, n_pad,
0, LLAMA_SWA_TYPE_NONE, mem_other_base, filter_base, reuse, share);
LLAMA_LOG_INFO("%s: creating SWA KV cache, size = %u cells\n", __func__, size_swa);
kv_swa = std::make_unique<llama_kv_cache>(
model, hparams, type_k, type_v,
model, no_alloc, hparams, type_k, type_v,
v_trans, offload, unified, size_swa, n_seq_max, n_pad,
hparams.n_swa, hparams.swa_type, mem_other_swa, filter_swa, reuse, share);
}
src/llama-kv-cache-iswa.h
+3 -1
View File
@@ -15,6 +15,7 @@ class llama_kv_cache_iswa : public llama_memory_i {
public:
llama_kv_cache_iswa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -32,7 +33,8 @@ public:
llama_kv_cache_iswa(
const llama_model & model,
const llama_hparams & hparams,
bool no_alloc,
const llama_hparams & hparams,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
src/llama-kv-cache-msa.cpp
+3 -2
View File
@@ -12,6 +12,7 @@
llama_kv_cache_msa::llama_kv_cache_msa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -32,7 +33,7 @@ llama_kv_cache_msa::llama_kv_cache_msa(
LLAMA_LOG_INFO("%s: creating main KV cache, size = %u cells\n", __func__, kv_size);
kv_base = std::make_unique<llama_kv_cache>(
model, model.hparams, type_k, type_v,
model, no_alloc, model.hparams, type_k, type_v,
v_trans, offload, unified, kv_size, n_seq_max, n_pad,
n_swa, swa_type, nullptr, filter, reuse, nullptr);
@@ -44,7 +45,7 @@ llama_kv_cache_msa::llama_kv_cache_msa(
LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size);
kv_idx = std::make_unique<llama_kv_cache>(
model, hparams_idx, type_k, type_v,
model, no_alloc, hparams_idx, type_k, type_v,
v_trans, offload, unified, kv_size, n_seq_max, n_pad,
n_swa, swa_type, nullptr, filter_idx, reuse, nullptr);
}
src/llama-kv-cache-msa.h
+1
View File
@@ -15,6 +15,7 @@ class llama_kv_cache_msa : public llama_memory_i {
public:
llama_kv_cache_msa(
const llama_model & model,
bool no_alloc,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
src/llama-kv-cache.cpp
+7 -32
View File
@@ -64,7 +64,8 @@ static void ggml_gen_hadamard(ggml_tensor * tensor) {
llama_kv_cache::llama_kv_cache(
const llama_model & model,
const llama_hparams & hparams,
bool no_alloc,
const llama_hparams & hparams,
ggml_type type_k,
ggml_type type_v,
bool v_trans,
@@ -79,9 +80,10 @@ llama_kv_cache::llama_kv_cache(
const layer_filter_cb & filter,
const layer_reuse_cb & reuse,
const layer_share_cb & share,
const char * name_tag) :
const char * name_tag) :
model(model), hparams(hparams), v_trans(v_trans),
n_seq_max(n_seq_max), n_stream(unified ? 1 : n_seq_max), n_pad(n_pad), n_swa(n_swa), swa_type(swa_type),
ctxs_bufs(no_alloc),
other(static_cast<llama_kv_cache *>(mem_other)),
v_cells_impl(other ? other->v_cells_impl : std::make_shared<llama_kv_cells_vec>()),
v_cells(*v_cells_impl) {
@@ -273,25 +275,11 @@ llama_kv_cache::llama_kv_cache(
}
}
// allocate tensors and initialize the buffers to avoid NaNs in the padding
// allocate the tensors and initialize the buffers to avoid NaNs in the padding
for (auto & [buft, ctx] : ctx_map) {
ggml_backend_buffer_t buf;
if (hparams.no_alloc) {
buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer
for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) {
t->buffer = buf; // set dummy buffer for KV cache so that the backend scheduler won't try to allocate it
}
} else {
buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer
}
if (!buf) {
throw std::runtime_error("failed to allocate buffer for kv cache");
}
ggml_backend_buffer_t buf = ctxs_bufs.emplace(std::move(ctx), buft, "kv cache");
LLAMA_LOG_INFO("%s: %10s KV buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
ggml_backend_buffer_clear(buf, 0);
ctxs_bufs.emplace_back(std::move(ctx), buf);
}
{
@@ -705,20 +693,7 @@ llama_pos llama_kv_cache::seq_pos_max(llama_seq_id seq_id) const {
}
std::map<ggml_backend_buffer_type_t, size_t> llama_kv_cache::memory_breakdown() const {
std::map<ggml_backend_buffer_type_t, size_t> ret;
for (const auto & [ctx, buf] : ctxs_bufs) {
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
if (hparams.no_alloc) {
GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr);
ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
} else {
// GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) != nullptr); // multi_buffer does not have a defined base
ret[buft] += ggml_backend_buffer_get_size(buf.get());
}
}
return ret;
return ctxs_bufs.memory_breakdown();
}
llama_memory_context_ptr llama_kv_cache::init_batch(
src/llama-kv-cache.h
+2 -1
View File
@@ -98,6 +98,7 @@ public:
// likely through `struct llama_memory_params`
llama_kv_cache(
const llama_model & model,
bool no_alloc,
const llama_hparams & hparams,
ggml_type type_k,
ggml_type type_v,
@@ -296,7 +297,7 @@ private:
const llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE;
// ggml contexts for the KV cache along with the allocated backend buffers:
std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;
llama_memory_bufs ctxs_bufs;
// the current index from where we start searching for a free slot in the ring buffer of KV cells (see find_slot())
// note: this is not part of the KV state and it's only used to speed-up the find_slot() method
src/llama-memory-hybrid-idx.cpp
+4 -6
View File
@@ -1,20 +1,16 @@
#include "llama-memory-hybrid-idx.h"
#include <algorithm>
#include <cmath>
#include <type_traits>
#include "llama-impl.h"
#include "llama-batch.h"
#include "llama-io.h"
#include "llama-model.h"
#include <algorithm>
#include <cassert>
#include <cmath>
#include <iterator>
#include <stdexcept>
#include <type_traits>
//
// llama_memory_hybrid_idx
@@ -22,6 +18,7 @@
llama_memory_hybrid_idx::llama_memory_hybrid_idx(
const llama_model & model,
bool no_alloc,
/* attn */
ggml_type type_k,
ggml_type type_v,
@@ -45,6 +42,7 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx(
const layer_filter_cb & filter_idx) :
llama_memory_hybrid(
model,
no_alloc,
type_k, type_v, v_trans, kv_size, n_pad, n_swa, swa_type,
type_r, type_s, rs_size,
n_seq_max, n_rs_seq, offload, unified,
@@ -68,7 +66,7 @@ llama_memory_hybrid_idx::llama_memory_hybrid_idx(
LLAMA_LOG_INFO("%s: creating indexer KV cache, size = %u cells\n", __func__, kv_size);
return new llama_kv_cache(
model, hparams_idx, type_k, type_v, v_trans, offload, unified,
model, no_alloc, hparams_idx, type_k, type_v, v_trans, offload, unified,
kv_size, n_seq_max, n_pad, n_swa, swa_type,
nullptr, filter_idx, nullptr, nullptr, "idx_");
}()) {}
src/llama-memory-hybrid-idx.h
+1
View File
@@ -18,6 +18,7 @@ class llama_memory_hybrid_idx : public llama_memory_hybrid {
public:
llama_memory_hybrid_idx(
const llama_model & model,
bool no_alloc,
/* attn */
ggml_type type_k,
ggml_type type_v,
src/llama-memory-hybrid-iswa.cpp
+3
View File
@@ -10,6 +10,7 @@
llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
const llama_model & model,
bool no_alloc,
/* attn */
ggml_type type_k,
ggml_type type_v,
@@ -33,6 +34,7 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
hparams(model.hparams),
mem_attn(new llama_kv_cache_iswa(
model,
no_alloc,
type_k,
type_v,
v_trans,
@@ -52,6 +54,7 @@ llama_memory_hybrid_iswa::llama_memory_hybrid_iswa(
)),
mem_recr(new llama_memory_recurrent(
model,
no_alloc,
type_r,
type_s,
offload,
src/llama-memory-hybrid-iswa.h
+1
View File
@@ -20,6 +20,7 @@ class llama_memory_hybrid_iswa : public llama_memory_i {
public:
llama_memory_hybrid_iswa(
const llama_model & model,
bool no_alloc,
/* attn */
ggml_type type_k,
ggml_type type_v,
src/llama-memory-hybrid.cpp
+3
View File
@@ -10,6 +10,7 @@
llama_memory_hybrid::llama_memory_hybrid(
const llama_model & model,
bool no_alloc,
/* attn */
ggml_type type_k,
ggml_type type_v,
@@ -33,6 +34,7 @@ llama_memory_hybrid::llama_memory_hybrid(
hparams(model.hparams),
mem_attn(new llama_kv_cache(
model,
no_alloc,
model.hparams,
type_k,
type_v,
@@ -53,6 +55,7 @@ llama_memory_hybrid::llama_memory_hybrid(
)),
mem_recr(new llama_memory_recurrent(
model,
no_alloc,
type_r,
type_s,
offload,
src/llama-memory-hybrid.h
+1
View File
@@ -20,6 +20,7 @@ class llama_memory_hybrid : public llama_memory_i {
public:
llama_memory_hybrid(
const llama_model & model,
bool no_alloc,
/* attn */
ggml_type type_k,
ggml_type type_v,
src/llama-memory-recurrent.cpp
+6 -28
View File
@@ -19,13 +19,14 @@
llama_memory_recurrent::llama_memory_recurrent(
const llama_model & model,
bool no_alloc,
ggml_type type_r,
ggml_type type_s,
bool offload,
uint32_t mem_size,
uint32_t n_seq_max,
uint32_t n_rs_seq,
const layer_filter_cb & filter) : hparams(model.hparams), n_seq_max(n_seq_max) {
const layer_filter_cb & filter) : hparams(model.hparams), n_seq_max(n_seq_max), ctxs_bufs(no_alloc) {
const int32_t n_layer = hparams.n_layer();
head = 0;
@@ -114,23 +115,11 @@ llama_memory_recurrent::llama_memory_recurrent(
}
}
// allocate tensors and initialize the buffers to avoid NaNs in the padding
// allocate the tensors and initialize the buffers to avoid NaNs in the padding
for (auto & [buft, ctx] : ctx_map) {
ggml_backend_buffer_t buf;
if (hparams.no_alloc) {
buf = ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0); // dummy buffer
for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != nullptr; t = ggml_get_next_tensor(ctx.get(), t)) {
t->buffer = buf; // set dummy buffer for RS cache so that the backend scheduler won't try to allocate it
}
} else {
buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); // real buffer
}
if (!buf) {
throw std::runtime_error("failed to allocate buffer for rs cache");
}
ggml_backend_buffer_clear(buf, 0);
ggml_backend_buffer_t buf = ctxs_bufs.emplace(std::move(ctx), buft, "rs cache");
LLAMA_LOG_INFO("%s: %10s RS buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
ctxs_bufs.emplace_back(std::move(ctx), buf);
}
if (is_empty()) {
@@ -442,18 +431,7 @@ void llama_memory_recurrent::set_rs_idx(llama_seq_id seq_id, uint32_t idx) {
}
std::map<ggml_backend_buffer_type_t, size_t> llama_memory_recurrent::memory_breakdown() const {
std::map<ggml_backend_buffer_type_t, size_t> ret;
for (const auto & [ctx, buf] : ctxs_bufs) {
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
if (hparams.no_alloc) {
GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr);
ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
} else {
ret[buft] += ggml_backend_buffer_get_size(buf.get());
}
}
return ret;
return ctxs_bufs.memory_breakdown();
}
llama_memory_context_ptr llama_memory_recurrent::init_batch(llama_batch_allocr & balloc, uint32_t n_ubatch, bool embd_all) {
src/llama-memory-recurrent.h
+3 -2
View File
@@ -18,6 +18,7 @@ class llama_memory_recurrent : public llama_memory_i {
public:
llama_memory_recurrent(
const llama_model & model,
bool no_alloc,
ggml_type type_r,
ggml_type type_s,
bool offload,
@@ -120,8 +121,8 @@ private:
const uint32_t n_seq_max = 1;
// ggml contexts for the KV cache along with the allocated backend buffers:
std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> ctxs_bufs;
// ggml contexts for the RS cache along with the allocated backend buffers:
llama_memory_bufs ctxs_bufs;
// true if no layers - can happen if the layer filter removes all layers
bool is_empty() const;
src/llama-memory.cpp
+52
View File
@@ -4,6 +4,8 @@
#include "llama-io.h"
#include <set>
#include <stdexcept>
#include <string>
#include <vector>
llama_memory_status llama_memory_status_combine(llama_memory_status s0, llama_memory_status s1) {
@@ -64,6 +66,52 @@ bool llama_memory_status_is_fail(llama_memory_status status) {
return false;
}
//
// llama_memory_bufs
//
ggml_backend_buffer_ptr llama_memory_bufs::alloc_buf(ggml_context * ctx, ggml_backend_buffer_type_t buft, bool no_alloc, const char * name) {
ggml_backend_buffer_ptr buf;
if (no_alloc) {
buf.reset(ggml_backend_buft_alloc_buffer(buft, /*size =*/ 0)); // dummy buffer
for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {
t->buffer = buf.get(); // set dummy buffer so that the backend scheduler won't try to allocate the tensors
}
} else {
buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx, buft)); // real buffer
}
if (!buf) {
throw std::runtime_error(std::string("failed to allocate buffer for ") + name);
}
return buf;
}
ggml_backend_buffer_t llama_memory_bufs::emplace(ggml_context_ptr ctx, ggml_backend_buffer_type_t buft, const char * name) {
ggml_backend_buffer_ptr buf = alloc_buf(ctx.get(), buft, no_alloc, name);
// initialize the buffer to avoid NaNs in the padding
ggml_backend_buffer_clear(buf.get(), 0);
bufs.emplace_back(std::move(ctx), std::move(buf));
return bufs.back().second.get();
}
std::map<ggml_backend_buffer_type_t, size_t> llama_memory_bufs::memory_breakdown() const {
std::map<ggml_backend_buffer_type_t, size_t> ret;
for (const auto & [ctx, buf] : bufs) {
ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(buf.get());
if (no_alloc) {
GGML_ASSERT(ggml_backend_buffer_get_base(buf.get()) == nullptr);
ret[buft] += ggml_backend_alloc_ctx_tensors_from_buft_size(ctx.get(), buft);
} else {
ret[buft] += ggml_backend_buffer_get_size(buf.get());
}
}
return ret;
}
// collects the buffers of the tensors that a state write visits, without keeping the data
struct llama_memory_buffers_writer : public llama_io_write_i {
std::set<ggml_backend_buffer_t> buffers;
@@ -84,6 +132,10 @@ struct llama_memory_buffers_writer : public llama_io_write_i {
};
std::vector<ggml_backend_buffer_t> llama_memory_get_buffers(const llama_memory_t mem) {
if (mem == nullptr) {
return {};
}
llama_memory_buffers_writer writer;
mem->state_write(writer);
src/llama-memory.h
+31
View File
@@ -6,6 +6,7 @@
#include <map>
#include <memory>
#include <functional>
#include <vector>
struct llama_ubatch;
@@ -25,6 +26,36 @@ struct llama_memory_params {
llama_context_type ctx_type;
llama_memory_t mem_other;
// only simulate the memory allocations
bool no_alloc;
};
// the tensor contexts and backend buffers of a memory instance
// with no_alloc, dummy buffers are attached instead of allocating the tensor data, so the memory use can still be measured
struct llama_memory_bufs {
explicit llama_memory_bufs(bool no_alloc) : no_alloc(no_alloc) {}
// allocate the tensors of ctx on buft, or attach a dummy buffer if no_alloc is set
// returns the buffer, for logging
ggml_backend_buffer_t emplace(ggml_context_ptr ctx, ggml_backend_buffer_type_t buft, const char * name);
// the memory used by the tensors: the buffer sizes, or the simulated sizes if no_alloc is set
std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const;
// allocate a single ctx buffer - for the memory instances that keep their own buffer layout
static ggml_backend_buffer_ptr alloc_buf(ggml_context * ctx, ggml_backend_buffer_type_t buft, bool no_alloc, const char * name);
bool empty() const { return bufs.empty(); }
auto begin() { return bufs.begin(); }
auto end() { return bufs.end(); }
auto begin() const { return bufs.begin(); }
auto end() const { return bufs.end(); }
std::vector<std::pair<ggml_context_ptr, ggml_backend_buffer_ptr>> bufs;
const bool no_alloc;
};
enum llama_memory_status {
src/llama-model.cpp
+24 -2
View File
@@ -2448,10 +2448,14 @@ llama_split_mode llama_model::split_mode() const {
return params.split_mode;
}
bool llama_model::no_alloc() const {
return params.no_alloc;
}
std::map<ggml_backend_buffer_type_t, size_t> llama_model::memory_breakdown() const {
std::map<ggml_backend_buffer_type_t, size_t> ret;
for (const auto & [ctx, bufs] : pimpl->ctxs_bufs) {
if (hparams.no_alloc) {
if (no_alloc()) {
GGML_ASSERT(bufs.size() == 1);
ggml_backend_buffer_t buf = bufs[0].get();
GGML_ASSERT(ggml_backend_buffer_get_base(buf) == nullptr);
@@ -2506,7 +2510,7 @@ void llama_model::print_info() const {
// hparams
LLAMA_LOG_INFO("%s: arch = %s\n", __func__, arch_name().c_str());
LLAMA_LOG_INFO("%s: vocab_only = %d\n", __func__, hparams.vocab_only);
LLAMA_LOG_INFO("%s: no_alloc = %d\n", __func__, hparams.no_alloc);
LLAMA_LOG_INFO("%s: no_alloc = %d\n", __func__, no_alloc());
if (!hparams.vocab_only) {
LLAMA_LOG_INFO("%s: n_ctx_train = %u\n", __func__, hparams.n_ctx_train);
@@ -2830,6 +2834,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache_msa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -2856,6 +2861,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache(
*this,
params.no_alloc,
hparams,
params.type_k,
params.type_v,
@@ -2882,6 +2888,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache_dsa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -2923,6 +2930,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_memory_hybrid_idx(
/* model */ *this,
/* no_alloc */ params.no_alloc,
/* attn_type_k */ params.type_k,
/* attn_type_v */ params.type_v,
/* attn_v_trans */ !cparams.flash_attn,
@@ -2947,6 +2955,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
// full-attention checkpoint: no indexer, so no indexer key cache
res = new llama_kv_cache(
*this,
params.no_alloc,
hparams,
params.type_k,
params.type_v,
@@ -2968,6 +2977,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache_dsa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -2994,6 +3004,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache(
*this,
params.no_alloc,
hparams,
params.type_k,
params.type_v,
@@ -3019,6 +3030,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache_dsa_iswa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -3046,6 +3058,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache_iswa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -3063,6 +3076,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
} else {
res = new llama_kv_cache_dsv4(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -3086,6 +3100,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache_iswa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -3120,6 +3135,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
if (llm_arch_is_recurrent(arch)) {
res = new llama_memory_recurrent(
*this,
params.no_alloc,
GGML_TYPE_F32,
GGML_TYPE_F32,
cparams.offload_kqv,
@@ -3175,6 +3191,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
// Use hybrid-iswa for hybrid models with SWA
res = new llama_memory_hybrid_iswa(
/* model */ *this,
/* no_alloc */ params.no_alloc,
/* attn_type_k */ params.type_k,
/* attn_type_v */ params.type_v,
/* attn_v_trans */ !cparams.flash_attn,
@@ -3195,6 +3212,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
// sparse attention over a per-token indexer cache, in its own memory type
res = new llama_memory_hybrid_idx(
/* model */ *this,
/* no_alloc */ params.no_alloc,
/* attn_type_k */ params.type_k,
/* attn_type_v */ params.type_v,
/* attn_v_trans */ !cparams.flash_attn,
@@ -3215,6 +3233,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
} else {
res = new llama_memory_hybrid(
/* model */ *this,
/* no_alloc */ params.no_alloc,
/* attn_type_k */ params.type_k,
/* attn_type_v */ params.type_v,
/* attn_v_trans */ !cparams.flash_attn,
@@ -3281,6 +3300,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache_iswa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -3298,6 +3318,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
} else {
res = new llama_kv_cache_iswa(
*this,
params.no_alloc,
params.type_k,
params.type_v,
!cparams.flash_attn,
@@ -3318,6 +3339,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
res = new llama_kv_cache(
*this,
params.no_alloc,
hparams,
params.type_k,
params.type_v,
src/llama-model.h
+1
View File
@@ -886,6 +886,7 @@ struct llama_model {
uint32_t n_gpu_layers() const;
llama_split_mode split_mode() const;
bool no_alloc() const;
std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const;
src/llama-moe-cache.cpp
+13 -30
View File
@@ -1,6 +1,7 @@
#include "llama-moe-cache.h"
#include "llama-impl.h"
#include "llama-memory.h"
#include "llama-model.h"
#include "ggml-cpp.h"
@@ -252,7 +253,7 @@ struct llama_moe_cache::impl {
// views used by copy_experts
ggml_context_ptr ctx_views;
impl(const llama_model & model, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size) :
impl(const llama_model & model, bool no_alloc, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size) :
n_expert_used(model.hparams.n_expert_used_max()), layers(model.layers.size()) {
for (size_t i = 0; i < backends.size(); ++i) {
const auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backends[i]));
@@ -418,37 +419,19 @@ struct llama_moe_cache::impl {
devices[g.id].buf_size += alloc_size(g, g.n_slots);
}
if (model.hparams.no_alloc) {
// only used to measure the memory use, see llama_context::memory_breakdown
for (device & d : devices) {
if (!d.ctx) {
continue;
}
d.buf.reset(ggml_backend_buft_alloc_buffer(d.buft, 0));
for (ggml_tensor * t = ggml_get_first_tensor(d.ctx.get()); t != nullptr; t = ggml_get_next_tensor(d.ctx.get(), t)) {
t->buffer = d.buf.get();
}
// with no_alloc, dummy buffers are attached so the memory use can still be measured, see llama_context::memory_breakdown
for (device & d : devices) {
if (!d.ctx) {
continue;
}
buf_host.reset(ggml_backend_buft_alloc_buffer(buft_host, 0));
for (ggml_tensor * t = ggml_get_first_tensor(ctx_host.get()); t != nullptr; t = ggml_get_next_tensor(ctx_host.get(), t)) {
t->buffer = buf_host.get();
}
} else {
for (device & d : devices) {
if (!d.ctx) {
continue;
}
d.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(d.ctx.get(), d.buft));
if (!d.buf) {
throw std::runtime_error("failed to allocate the MoE cache buffers");
}
d.buf = llama_memory_bufs::alloc_buf(d.ctx.get(), d.buft, no_alloc, "MoE cache");
if (!no_alloc) {
ggml_backend_buffer_clear(d.buf.get(), 0);
d.buf_size = ggml_backend_buffer_get_size(d.buf.get());
}
buf_host.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_host.get(), buft_host));
if (!buf_host) {
throw std::runtime_error("failed to allocate the MoE cache buffers");
}
}
buf_host = llama_memory_bufs::alloc_buf(ctx_host.get(), buft_host, no_alloc, "MoE cache slot maps");
if (!no_alloc) {
ggml_backend_buffer_clear(buf_host.get(), 0xff); // all slots are -1
buf_host_size = ggml_backend_buffer_get_size(buf_host.get());
@@ -633,8 +616,8 @@ struct llama_moe_cache::impl {
}
};
llama_moe_cache::llama_moe_cache(const llama_model & model, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size) :
pimpl(new impl(model, backends, bufts, size)) {
llama_moe_cache::llama_moe_cache(const llama_model & model, bool no_alloc, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size) :
pimpl(new impl(model, no_alloc, backends, bufts, size)) {
}
llama_moe_cache::~llama_moe_cache() = default;
src/llama-moe-cache.h
+1 -1
View File
@@ -13,7 +13,7 @@ struct llama_model;
class llama_moe_cache {
public:
// backends are all the backends of the context, each GPU gets its own cache of the given size for the layers assigned to it
llama_moe_cache(const llama_model & model, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size);
llama_moe_cache(const llama_model & model, bool no_alloc, const std::vector<ggml_backend_t> & backends, const std::vector<ggml_backend_buffer_type_t> & bufts, size_t size);
~llama_moe_cache();
// the device that caches layer il
src/llama.cpp
-1
View File
@@ -342,7 +342,6 @@ static std::pair<int, llama_model *> llama_model_load(struct gguf_context * meta
model->t_start_us = tm.t_start_us;
model->hparams.vocab_only = params.vocab_only;
model->hparams.no_alloc = params.no_alloc;
try {
model->load_hparams(ml);
tests/llama-context/cases-kv.cpp
+49
View File
@@ -718,6 +718,54 @@ struct case_state_restore_fragmented : ctx_test {
}
};
//
// no_alloc
//
// - load a second instance of the model with no_alloc, so the tensor data stays on disk
// - create a context from it, which reserves the compute graphs over the dummy buffers
// - the memory use must still be reported, and nothing real may be allocated for the memory
struct case_noalloc : ctx_test {
const char * name() const override { return "noalloc"; }
const char * description() const override {
return "with no_alloc the memory use is simulated: non-zero breakdown, null buffer bases";
}
case_result run(model_run & mr) const override {
const std::string & path = mr.params.model.path;
CTX_REQUIRE(!path.empty(), "the model path is not known");
llama_model_params mparams = llama_model_default_params();
mparams.no_alloc = true;
mparams.load_mode = LLAMA_LOAD_MODE_NONE;
llama_model_ptr model(llama_model_load_from_file(path.c_str(), mparams));
CTX_REQUIRE(model, "failed to load the model with no_alloc");
auto ctx = init_ctx(model.get(), common_context_params_to_llama(mr.params), 0);
CTX_REQUIRE(ctx, "failed to create the context with no_alloc");
size_t n_ctx_mem = 0;
for (const auto & [_, data] : llama_get_memory_breakdown(ctx.get())) {
n_ctx_mem += data.context;
}
// models without a memory module (BERT, diffusion) have a null memory
if (llama_get_memory(ctx.get()) != nullptr) {
CTX_REQUIRE(n_ctx_mem > 0, "the simulated memory breakdown is zero");
} else {
CTX_REQUIRE(n_ctx_mem == 0, "a model without memory reported context memory");
}
for (ggml_backend_buffer_t buf : llama_memory_get_buffers(llama_get_memory(ctx.get()))) {
CTX_REQUIRE(ggml_backend_buffer_get_base(buf) == nullptr, "a memory buffer was really allocated");
}
return case_result::pass();
}
};
std::vector<ctx_test_ptr> make_kv_cases() {
std::vector<ctx_test_ptr> cases;
@@ -733,6 +781,7 @@ std::vector<ctx_test_ptr> make_kv_cases() {
cases.push_back(std::make_unique<case_state_restore_failure>());
cases.push_back(std::make_unique<case_state_rotation>());
cases.push_back(std::make_unique<case_state_restore_fragmented>());
cases.push_back(std::make_unique<case_noalloc>());
return cases;
}