Georgi Gerganov
a17e94d229
* tests : consolidate the context state tests into test-llama-context test-save-load-state, test-recurrent-state-rollback and test-state-restore-fragmented check the same object from three places: the KV cache and the recurrent state of a llama_context, saved, loaded, copied, rolled back. Each had its own main, its own loop over the models, its own copy of the token and the comparison helpers, and a run of all three loaded the same models several times. They are one binary now, and the case is the unit of the test: tests/llama-context/test-case.h the interface a case implements tests/llama-context/cases-kv.cpp the cases of the KV cache tests/llama-context/cases-rs.cpp the cases of the recurrent state tests/llama-context/test-util.cpp tokens, generation, comparison tests/llama-context/test-runner.cpp the models of a case, on threads tests/llama-context/test-log.cpp the log file of a model A run is a table: a row per model, a column per case, a cell holds the verdict of that pair. The models load on several threads (-j, --threads, min(cores, 4) by default), the table is printed when it is complete and the log of a model that failed is printed after it, in the order of the rows. A case returns a verdict and the reason for it, and the runner is the only one who reports - the test does not use tests/testing.h, a run over 130 models is a matrix of verdicts and not a tree of subtests. Staging no longer travels through the environment: llama_model_set_attn_rot replaces the LLAMA_ATTN_ROT_DISABLE variable, which a model loaded in one thread used to change for the models the other threads were loading, and llama_memory_get_buffers is added for the cases that copy the buffers of a memory. common_log_set_stdo keeps the console quiet while the table is being built. ggml_aligned_malloc allocates with posix_memalign when the thread sanitizer is on. On macOS it allocates with the Mach VM, which the sanitizer does not intercept, so it never sees a page range change hands and reports the first write into recycled memory as a race - 30 of them in this test, and every one of them false. Assisted-by: pi:llama.cpp/Qwen3.8-Flash-Next * cont : run single-threaded by default * cont : try to build the tests on all platforms * cont : trim down some body fat * cont : fix * cont : fix the fix
118 lines
3.7 KiB
C++
118 lines
3.7 KiB
C++
#include "test-runner.h"
|
|
|
|
#include "test-log.h"
|
|
|
|
#include <algorithm>
|
|
#include <cstdio>
|
|
#include <cstring>
|
|
#include <regex>
|
|
#include <utility>
|
|
|
|
std::vector<const ctx_test *> select_cases(const std::string & filter) {
|
|
std::vector<const ctx_test *> selected;
|
|
|
|
if (filter.empty()) {
|
|
selected.reserve(test_cases().size());
|
|
for (const auto & tc : test_cases()) {
|
|
selected.push_back(tc.get());
|
|
}
|
|
|
|
return selected;
|
|
}
|
|
|
|
const std::regex re(filter);
|
|
|
|
for (const auto & tc : test_cases()) {
|
|
if (std::regex_match(tc->name(), re)) {
|
|
selected.push_back(tc.get());
|
|
}
|
|
}
|
|
|
|
return selected;
|
|
}
|
|
|
|
std::vector<case_run> run_cases(model_run & mr, const std::vector<const ctx_test *> & cases, bool report) {
|
|
// the state cases keep the historical default of a unified KV cache when running a single
|
|
// sequence, the recurrent cases use whatever was passed on the command line
|
|
const bool unified_base = mr.params.kv_unified;
|
|
const bool unified_state = mr.params.n_parallel == 1 ? true : unified_base;
|
|
|
|
size_t name_width = 4; // "case"
|
|
for (const ctx_test * tc : cases) {
|
|
name_width = std::max(name_width, strlen(tc->name()));
|
|
}
|
|
|
|
std::vector<case_run> runs;
|
|
runs.reserve(cases.size());
|
|
|
|
for (const ctx_test * tc : cases) {
|
|
mr.params.kv_unified = tc->group() == TEST_GROUP_STATE ? unified_state : unified_base;
|
|
|
|
case_run run;
|
|
run.name = tc->name();
|
|
|
|
// the log of a model is read on its own, mark where each case begins
|
|
TLOG_INF("%s\n", run.name.c_str());
|
|
|
|
// a case that throws is a case that failed, and the run goes on with the next one
|
|
try {
|
|
if (const std::string why = tc->skip_reason(mr); !why.empty()) {
|
|
run.status = test_status::SKIP;
|
|
run.reason = why;
|
|
} else if (tc->needs_baseline() && !mr.have_baseline) {
|
|
run.status = test_status::SKIP;
|
|
run.reason = "no baseline";
|
|
} else if (!tc->uses_fill()) {
|
|
mr.fill = 0;
|
|
|
|
const case_result res = tc->run(mr);
|
|
|
|
run.status = res.status;
|
|
run.reason = res.reason;
|
|
} else {
|
|
// the recurrent cases are also run against a cache pre-filled with a known
|
|
// pattern, so that a restore reading cells it never wrote cannot pass
|
|
|
|
for (const uint8_t fill : { 0, 0x3e }) {
|
|
TLOG_INF("(fill 0x%02x)\n", fill);
|
|
|
|
mr.fill = fill;
|
|
|
|
const case_result res = tc->run(mr);
|
|
|
|
run.status = merge_status(run.status, res.status);
|
|
if (run.reason.empty()) {
|
|
run.reason = res.reason;
|
|
}
|
|
if (res.status == test_status::FAIL) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
} catch (...) {
|
|
run.status = test_status::FAIL;
|
|
run.reason = "unhandled exception";
|
|
}
|
|
|
|
if (unified_state) {
|
|
mr.params.kv_unified = unified_base;
|
|
}
|
|
|
|
// the verdict is one word, what it means goes to the log of the model
|
|
if (!run.reason.empty()) {
|
|
TLOG_INF("%s %s: %s\n", run.name.c_str(),
|
|
run.status == test_status::FAIL ? "failed" : "skipped", run.reason.c_str());
|
|
}
|
|
|
|
if (report) {
|
|
fprintf(stderr, "%-*s %s%s%s\n", (int) name_width, run.name.c_str(),
|
|
test_status_str(run.status),
|
|
run.reason.empty() ? "" : " ", run.reason.c_str());
|
|
}
|
|
|
|
runs.push_back(std::move(run));
|
|
}
|
|
|
|
return runs;
|
|
}
|