diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index c1683d9ef..c20167b90 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2736,6 +2736,87 @@ private: return n_read; } + // the live draft state follows the checkpoints: the full draft context and the speculative state + // the llama state payload holds the target context only + static constexpr uint32_t SLOT_DRAFT_MAGIC = 0x54464453; // "SDFT" + static constexpr uint32_t SLOT_DRAFT_VERSION = 1; + + // returns false if the draft state could not be written completely + bool save_slot_draft(const std::string & filepath, const server_slot & slot, size_t & n_written) const { + n_written = 0; + std::vector data_dft; + std::vector data_spec; + if (ctx_dft != nullptr) { + data_dft.resize(llama_state_seq_get_size_ext(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_NONE)); + llama_state_seq_get_data_ext(ctx_dft, data_dft.data(), data_dft.size(), slot.id, LLAMA_STATE_SEQ_FLAGS_NONE); + } + common_speculative_get_state(spec.get(), slot.id, data_spec); + if (data_dft.empty() && data_spec.empty()) { + return true; + } + std::ofstream ofs(std::filesystem::u8path(filepath), std::ios::binary | std::ios::app); + if (!ofs) { + SLT_WRN(slot, "failed to append the draft state to '%s'\n", filepath.c_str()); + return false; + } + const uint32_t magic = SLOT_DRAFT_MAGIC; + const uint32_t version = SLOT_DRAFT_VERSION; + ckpt_write(ofs, &magic, sizeof(magic), n_written); + ckpt_write(ofs, &version, sizeof(version), n_written); + ckpt_write_buf(ofs, data_dft, n_written); + ckpt_write_buf(ofs, data_spec, n_written); + ofs.flush(); + if (!ofs) { + SLT_WRN(slot, "failed to append the draft state to '%s' - the draft state is incomplete\n", filepath.c_str()); + return false; + } + SLT_INF(slot, "appended the draft state (%.3f MiB) to '%s'\n", (float) n_written / 1024 / 1024, filepath.c_str()); + return true; + } + + // returns the number of bytes consumed, 0 if there is no usable draft state + // the draft starts empty unless the file holds a draft state that fits the draft context + size_t load_slot_draft(const std::string & filepath, size_t offset, server_slot & slot) const { + if (ctx_dft != nullptr) { + llama_memory_seq_rm(llama_get_memory(ctx_dft), slot.id, -1, -1); + } + std::ifstream ifs(std::filesystem::u8path(filepath), std::ios::binary | std::ios::ate); + const size_t file_size = ifs ? (size_t) ifs.tellg() : 0; + if (!ifs || file_size < offset || !ifs.seekg(offset)) { + return 0; + } + const size_t n_avail = file_size - offset; + size_t n_read = 0; + uint32_t magic = 0; + uint32_t version = 0; + if (!ckpt_read(ifs, &magic, sizeof(magic), n_read) || magic != SLOT_DRAFT_MAGIC) { + return 0; + } + std::vector data_dft; + std::vector data_spec; + if (!ckpt_read(ifs, &version, sizeof(version), n_read) || version != SLOT_DRAFT_VERSION || + !ckpt_read_buf(ifs, data_dft, n_avail, n_read) || + !ckpt_read_buf(ifs, data_spec, n_avail, n_read)) { + SLT_WRN(slot, "invalid draft state in '%s' - ignored\n", filepath.c_str()); + return 0; + } + if (!data_dft.empty()) { + if (ctx_dft == nullptr || + llama_state_seq_set_data_ext(ctx_dft, data_dft.data(), data_dft.size(), slot.id, LLAMA_STATE_SEQ_FLAGS_NONE) != data_dft.size()) { + if (ctx_dft != nullptr) { + llama_memory_seq_rm(llama_get_memory(ctx_dft), slot.id, -1, -1); + } + SLT_WRN(slot, "draft state in '%s' does not match the draft context - dropped\n", filepath.c_str()); + return n_read; + } + } + if (!data_spec.empty()) { + common_speculative_set_state(spec.get(), slot.id, data_spec); + } + SLT_INF(slot, "restored the draft state (%.3f MiB) from '%s'\n", (float) n_read / 1024 / 1024, filepath.c_str()); + return n_read; + } + // returns false to decline the task, it is offered again after the decode is done bool process_single_task(server_task && task, bool is_yielding) { // while yielding, an encode / decode is running and only reading the server state is safe @@ -2951,6 +3032,12 @@ private: break; } + size_t nwrite_dft = 0; + if (!save_slot_draft(filepath, *slot, nwrite_dft)) { + send_error(task, "Unable to save slot: incomplete draft state", ERROR_TYPE_SERVER); + break; + } + const int64_t t_end = ggml_time_us(); const double t_save_ms = (t_end - t_start) / 1000.0; @@ -2960,7 +3047,7 @@ private: res->filename = filename; res->is_save = true; res->n_tokens = slot->prompt.tokens.size(); - res->n_bytes = nwrite + nwrite_ckpt; + res->n_bytes = nwrite + nwrite_ckpt + nwrite_dft; res->t_ms = t_save_ms; queue_results.send(std::move(res)); } break; @@ -3018,6 +3105,7 @@ private: // nread is the end offset of the llama state payload within the file const size_t nread_ckpt = load_slot_checkpoints(filepath, nread, *slot); + const size_t nread_dft = load_slot_draft(filepath, nread + nread_ckpt, *slot); const int64_t t_end = ggml_time_us(); const double t_restore_ms = (t_end - t_start) / 1000.0; @@ -3028,7 +3116,7 @@ private: res->filename = filename; res->is_save = false; res->n_tokens = slot->prompt.tokens.size(); - res->n_bytes = nread + nread_ckpt; + res->n_bytes = nread + nread_ckpt + nread_dft; res->t_ms = t_restore_ms; queue_results.send(std::move(res)); } break; diff --git a/tools/server/tests/unit/test_slot_save.py b/tools/server/tests/unit/test_slot_save.py index 33bcede93..793f1a0a7 100644 --- a/tools/server/tests/unit/test_slot_save.py +++ b/tools/server/tests/unit/test_slot_save.py @@ -132,6 +132,55 @@ def test_slot_restore_legacy_token_list(): assert res.body["timings"]["prompt_n"] == 6 # only the different part is processed +# the slot file carries the live draft state: a restored slot drafts like a slot that was never saved +def test_slot_restore_draft_state(): + global server + server.model_draft_hf_repo = server.model_hf_repo + server.spec_type = "draft-simple" + server.start() + + prompt = "What is the capital of France?" + turn = {} + for id_slot in [0, 1]: + res = server.make_request("POST", "/completion", data={ + "prompt": prompt, + "n_predict": 16, + "id_slot": id_slot, + "cache_prompt": True, + }) + assert res.status_code == 200 + turn[id_slot] = res.body["content"] + assert turn[0] == turn[1] + + res = server.make_request("POST", "/slots/0?action=save", data={ + "filename": "draft_slot0.bin", + }) + assert res.status_code == 200 + + res = server.make_request("POST", "/slots/0?action=erase") + assert res.status_code == 200 + + res = server.make_request("POST", "/slots/0?action=restore", data={ + "filename": "draft_slot0.bin", + }) + assert res.status_code == 200 + + timings = {} + for id_slot in [0, 1]: + res = server.make_request("POST", "/completion", data={ + "prompt": prompt + turn[id_slot] + " And", + "n_predict": 32, + "id_slot": id_slot, + "cache_prompt": True, + }) + assert res.status_code == 200 + timings[id_slot] = res.body["timings"] + + assert timings[0]["prompt_n"] == timings[1]["prompt_n"] + assert timings[0]["draft_n_accepted"] > 0 + assert timings[0]["draft_n"] == timings[1]["draft_n"] + assert timings[0]["draft_n_accepted"] == timings[1]["draft_n_accepted"] + def test_slot_erase(): global server