wmantly/llama.cpp-xrip
1
0 Fork 0
Code Issues Pull Requests Actions 32 Packages Projects Releases Wiki Activity

common : read ngram cache parts by reference instead of copying (#30061)

Browse Source
This commit is contained in:
hayder authored and GitHub committed 2026-10-16 10:06:32 +03:00
1 parent 0f4376268f
commit 13b8048920
1 file changed
+7 -9
Download Patch File Download Diff File Expand all files Collapse all files
common/ngram-cache.cpp
+7 -9
View File
@@ -68,7 +68,7 @@ static llama_token try_draft(common_ngram_cache & nc_static, const common_ngram
if (part_static_it == nc_static.end()) {
return LLAMA_TOKEN_NULL;
}
const common_ngram_cache_part part_static = part_static_it->second;
const common_ngram_cache_part & part_static = part_static_it->second;
int max_count_static = 0;
int sum_count_static = 0;
@@ -96,7 +96,7 @@ static llama_token try_draft(common_ngram_cache & nc_static, const common_ngram
// Try to draft a token from primary cache (context/dynamic), validate with static cache:
static llama_token try_draft(
common_ngram_cache & nc_primary, const std::vector<common_ngram> & ngrams_primary, common_ngram_cache_part & part_static,
common_ngram_cache & nc_primary, const std::vector<common_ngram> & ngrams_primary, const common_ngram_cache_part & part_static,
const int * min_sample_size, const int * min_percent) {
llama_token drafted_token = LLAMA_TOKEN_NULL;
@@ -108,7 +108,7 @@ static llama_token try_draft(
if (part_primary_it == nc_primary.end()) {
continue;
}
const common_ngram_cache_part part_primary = part_primary_it->second;
const common_ngram_cache_part & part_primary = part_primary_it->second;
int max_count_primary = 0;
int max_count_static = 0;
@@ -118,7 +118,7 @@ static llama_token try_draft(
for (std::pair<llama_token, int> token_count_primary : part_primary) {
const llama_token token = token_count_primary.first;
common_ngram_cache_part::iterator token_count_static_it = part_static.find(token);
common_ngram_cache_part::const_iterator token_count_static_it = part_static.find(token);
const int32_t count_primary = token_count_primary.second;
const int32_t count_static = token_count_static_it != part_static.end() ? 100*token_count_static_it->second : 1;
@@ -162,11 +162,9 @@ void common_ngram_cache_draft(
for (int j = ngram_start_static; j < ngram_start_static + LLAMA_NGRAM_STATIC; ++j) {
ngram_static.tokens[j-ngram_start_static] = get_token(inp, draft, j);
}
common_ngram_cache::iterator part_static_it = nc_static.find(ngram_static);
common_ngram_cache_part part_static;
if (part_static_it != nc_static.end()) {
part_static = part_static_it->second;
}
static const common_ngram_cache_part part_static_empty;
common_ngram_cache::const_iterator part_static_it = nc_static.find(ngram_static);
const common_ngram_cache_part & part_static = part_static_it != nc_static.end() ? part_static_it->second : part_static_empty;
// cd = context + dynamic
std::vector<common_ngram> ngrams_cd;