diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp index 9e3b6d640..1112738af 100644 --- a/common/chat-auto-parser-generator.cpp +++ b/common/chat-auto-parser-generator.cpp @@ -61,8 +61,7 @@ common_chat_params peg_generator::generate_parser(const common_chat_template & data.prompt += data.generation_prompt; } - auto parser = autoparser.build_parser(inputs, parser_generation_prompt); - data.parser = parser.save(); + data.parser = autoparser.build_parser(inputs, parser_generation_prompt); // Build grammar if tools are present bool has_tools = @@ -78,7 +77,7 @@ common_chat_params peg_generator::generate_parser(const common_chat_template & if (include_grammar) { data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); // Set grammar triggers based on tool section markers (fall back to per-call markers) diff --git a/common/chat.cpp b/common/chat.cpp index c8f156a70..87f1e99a2 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -9,6 +9,7 @@ #include "json.h" #include "log.h" #include "parsers/parsers.h" +#include "sampling.h" #include "jinja/value.h" #include "jinja/runtime.h" @@ -112,38 +113,6 @@ const char * common_chat_role_to_string(common_chat_role role) { return ""; } -json common_chat_msg_delimiters::to_json() const { - json result = json::array(); - for (const auto & d : delimiters) { - result.push_back({ - { "role", common_chat_role_to_string(d.role) }, - { "delimiter", d.delimiter }, - }); - } - return result; -} - -common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimiters) { - common_chat_msg_delimiters result; - - if (!delimiters.is_array()) { - return result; - } - - result.delimiters.reserve(delimiters.size()); - for (const auto & d : delimiters) { - if (!d.is_object()) { - continue; - } - result.delimiters.push_back({ - common_chat_role_from_string(d.value("role", std::string())), - d.value("delimiter", std::string()), - }); - } - - return result; -} - void common_chat_msg_delimiters::tokenize(const llama_vocab * vocab) { for (auto & d : delimiters) { d.tokens = common_tokenize(vocab, d.delimiter, false, true); @@ -620,8 +589,11 @@ std::vector common_chat_tools_parse_oaicompat(const json & too } common_chat_continuation common_chat_continuation_parse(const common_json & value) { - if (value.is_boolean() && value.get()) { - return COMMON_CHAT_CONTINUATION_AUTO; + if (value.is_null()) { + return COMMON_CHAT_CONTINUATION_NONE; + } + if (value.is_boolean()) { + return value.get() ? COMMON_CHAT_CONTINUATION_AUTO : COMMON_CHAT_CONTINUATION_NONE; } if (value.is_string()) { auto value_str = value.get(); @@ -632,7 +604,7 @@ common_chat_continuation common_chat_continuation_parse(const common_json & valu return COMMON_CHAT_CONTINUATION_CONTENT; } } - return COMMON_CHAT_CONTINUATION_NONE; + throw std::invalid_argument("Invalid continue_final_message: expected a boolean, \"content\" or \"reasoning_content\""); } bool common_chat_verify_template(const std::string & tmpl, bool use_jinja) { @@ -1087,41 +1059,55 @@ static json common_chat_extra_context() { return ctx; } -std::optional common_chat_try_specialized_template( - const common_chat_template & tmpl, - const std::string & src, - autoparser::generation_params & params) { +static common_chat_params common_chat_params_init_lfm2_tokens(const common_chat_template & tmpl, const autoparser::generation_params & inputs) { + return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ true); +} + +static common_chat_params common_chat_params_init_lfm2_5(const common_chat_template & tmpl, const autoparser::generation_params & inputs) { + return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ false); +} + +// Older gemma4 templates need their tool responses rewritten before rendering +static common_chat_params common_chat_params_init_gemma4_legacy(const common_chat_template & tmpl, const autoparser::generation_params & inputs) { + auto adjusted = inputs; + workaround::convert_tool_responses_gemma4(adjusted.messages); + return common_chat_params_init_gemma4(tmpl, adjusted); +} + +// Pick the dedicated handler for a template from its source, or null for the autoparser. +// Order matters: the first match wins, and later checks assume the earlier ones did not match. +static common_chat_params_init_fn common_chat_template_detect_params_init(const std::string & src) { // Ministral/Mistral Large 3 - uses special reasoning structure fixes, can't use autoparser // Note: Mistral Small 3.2 uses [CALL_ID] which Ministral doesn't have, so we can distinguish them if (src.find("[SYSTEM_PROMPT]") != std::string::npos && src.find("[TOOL_CALLS]") != std::string::npos && src.find("[ARGS]") != std::string::npos && src.find("[CALL_ID]") == std::string::npos) { LOG_DBG("Using specialized template: Ministral/Magistral Large 3\n"); - return common_chat_params_init_ministral_3(tmpl, params); + return common_chat_params_init_ministral_3; } // LLM-jp-4.1 - GPT-OSS dialect (spaces after special tokens, <|end|>-separated parallel calls) if (src.find("chat_format=llm-jp-harmony-v1") != std::string::npos) { LOG_DBG("Using specialized template: LLM-jp Harmony v1\n"); - return common_chat_params_init_llm_jp_harmony(tmpl, params); + return common_chat_params_init_llm_jp_harmony; } // GPT-OSS - has unique channel-based structure that needs dedicated handler if (src.find("<|channel|>") != std::string::npos) { LOG_DBG("Using specialized template: GPT-OSS\n"); - return common_chat_params_init_gpt_oss(tmpl, params); + return common_chat_params_init_gpt_oss; } // Muse Glimmer format using " to=" recipients and <|eom|>/<|eot|> message terminators. if (src.find("") != std::string::npos && src.find("<|eom|>") != std::string::npos) { LOG_DBG("Using specialized template: Muse Glimmer\n"); - return common_chat_params_init_muse_glimmer(tmpl, params); + return common_chat_params_init_muse_glimmer; } // Functionary v3.2 - uses recipient-based format with >>>recipient\n{content} // Detection: template has ">>>all" for content and ">>>" prefix for tool calls if (src.find(">>>all") != std::string::npos && src.find(">>>${recipient}") != std::string::npos) { LOG_DBG("Using specialized template: Functionary v3.2\n"); - return common_chat_params_init_functionary_v3_2(tmpl, params); + return common_chat_params_init_functionary_v3_2; } // Kimi K2 Thinking - uses unique tool call ID format: functions.: @@ -1129,14 +1115,14 @@ std::optional common_chat_try_specialized_template( if (src.find("<|tool_calls_section_begin|>") != std::string::npos && src.find("<|tool_call_begin|>") != std::string::npos) { LOG_DBG("Using specialized template: Kimi K2 Thinking\n"); - return common_chat_params_init_kimi_k2(tmpl, params); + return common_chat_params_init_kimi_k2; } // Kimi K3 - the <|open|>/<|close|>/<|end_of_msg|> markers are unique to it if (src.find("<|open|>") != std::string::npos && src.find("<|close|>") != std::string::npos && src.find("<|end_of_msg|>") != std::string::npos) { LOG_DBG("Using specialized template: Kimi K3\n"); - return common_chat_params_init_kimi_k3(tmpl, params); + return common_chat_params_init_kimi_k3; } // K2 Horizon - <|ifm|im_start|> turns, reasoning picked by reasoning_effort and @@ -1144,7 +1130,7 @@ std::optional common_chat_try_specialized_template( if (src.find("<|ifm|im_start|>") != std::string::npos && src.find("") != std::string::npos) { LOG_DBG("Using specialized template: K2 Horizon\n"); - return common_chat_params_init_k2_horizon(tmpl, params); + return common_chat_params_init_k2_horizon; } // Ling 3.0 / Bailing V3 - X sections with / tagged @@ -1152,7 +1138,7 @@ std::optional common_chat_try_specialized_template( if (src.find("ASSISTANT") != std::string::npos && src.find("") != std::string::npos) { LOG_DBG("Using specialized template: Ling 3.0 (Bailing V3)\n"); - return common_chat_params_init_ling3(tmpl, params); + return common_chat_params_init_ling3; } // Cohere2 MoE / North Code - marker-wrapped format with <|START_TEXT|> content and @@ -1161,19 +1147,19 @@ std::optional common_chat_try_specialized_template( if (src.find("<|START_TEXT|>") != std::string::npos && src.find("<|START_ACTION|>") != std::string::npos) { LOG_DBG("Using specialized template: Cohere2 MoE\n"); - return common_chat_params_init_cohere2moe(tmpl, params); + return common_chat_params_init_cohere2moe; } if (is_lfm2_template(src)) { LOG_DBG("Using specialized template: LFM2\n"); - return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ true); + return common_chat_params_init_lfm2_tokens; } // LFM2.5 format detection: template uses plain "List of tools: [...]" with no special tokens if (src.find("List of tools: [") != std::string::npos && src.find("<|tool_list_start|>") == std::string::npos) { LOG_DBG("Using specialized template: LFM2.5\n"); - return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ false); + return common_chat_params_init_lfm2_5; } // GigaChatV3 format detection @@ -1181,7 +1167,7 @@ std::optional common_chat_try_specialized_template( src.find("<|message_sep|>") != std::string::npos && src.find("<|function_call|>") == std::string::npos) { LOG_DBG("Using specialized template: GigaChatV3\n"); - return common_chat_params_init_gigachat_v3(tmpl, params); + return common_chat_params_init_gigachat_v3; } // MiniMax-M3: the namespace token "]<]minimax[>[" collides with the autoparser's @@ -1190,7 +1176,7 @@ std::optional common_chat_try_specialized_template( src.find("") != std::string::npos && src.find(" common_chat_try_specialized_template( (src.find("function_calls") != std::string::npos || src.find("tool_calls") != std::string::npos)) { LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n"); - return common_chat_params_init_deepseek_v3_2(tmpl, params); + return common_chat_params_init_deepseek_v3_2; } // Gemma4 format detection if (src.find("'<|tool_call>call:'") != std::string::npos) { + LOG_DBG("Using specialized template: Gemma4\n"); if (src.find("{#- OpenAI Chat Completions:") == std::string::npos) { - // apply workarounds if using the older gemma4 templates LOG_WRN("%s: detected an outdated gemma4 chat template, applying compatibility workarounds. " "Consider updating to the official template.\n", __func__); - workaround::convert_tool_responses_gemma4(params.messages); + return common_chat_params_init_gemma4_legacy; } - return common_chat_params_init_gemma4(tmpl, params); + return common_chat_params_init_gemma4; } // MiniCPM5 - XML tool calls with ... @@ -1220,14 +1206,14 @@ std::optional common_chat_try_specialized_template( src.find(" common_chat_try_specialized_template( // Exclude models that don't use \n between tags src.find("''") == std::string::npos) { LOG_DBG("Using specialized template: Qwen3-Coder\n"); - return common_chat_params_init_qwen3_coder(tmpl, params); + return common_chat_params_init_qwen3_coder; } - return std::nullopt; + return nullptr; +} + +common_chat_template::common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token) { + jinja::lexer lexer; + auto lexer_res = lexer.tokenize(src); + this->prog = jinja::parse_from_tokens(lexer_res); + + this->src = lexer_res.source; + this->bos_tok = bos_token; + this->eos_tok = eos_token; + + this->caps = jinja::caps_get(prog); + // LOG_INF("%s: caps:\n%s\n", __func__, this->caps.to_string().c_str()); + + this->params_init = common_chat_template_detect_params_init(this->src); + if (this->params_init) { + return; + } + + // The analysis depends only on the template, so run it once here instead of on every apply. + // A failure is kept for apply to report, so a bad template still loads like it did before. + try { + analysis = std::make_unique(); + analysis->analyze_template(*this); + } catch (const std::exception & e) { + analysis.reset(); + analysis_error = e.what(); + } +} + +common_chat_template::~common_chat_template() = default; +common_chat_template::common_chat_template(common_chat_template &&) = default; +common_chat_template & common_chat_template::operator=(common_chat_template &&) = default; + +std::optional common_chat_try_specialized_template( + const common_chat_template & tmpl, + const autoparser::generation_params & params) { + if (!tmpl.params_init) { + return std::nullopt; + } + return tmpl.params_init(tmpl, params); } static common_chat_params common_chat_templates_apply_jinja(const struct common_chat_templates * tmpls, @@ -1342,21 +1369,23 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ data.prompt = common_chat_template_direct_apply_impl(tmpl, params_copy); data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, params); data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - auto parser = build_chat_peg_parser([&data](common_chat_peg_builder &p) { + data.parser = build_chat_peg_parser([&data](common_chat_peg_builder &p) { return p.literal(data.generation_prompt) << p.content(p.rest()); }); - data.parser = parser.save(); return data; } - if (auto result = common_chat_try_specialized_template(tmpl, src, params)) { + if (auto result = common_chat_try_specialized_template(tmpl, params)) { return *result; } + if (!tmpl.analysis) { + throw std::invalid_argument("Unable to generate parser for this template. Automatic parser generation failed: " + tmpl.analysis_error); + } + try { LOG_DBG("%s: using differential autoparser\n", __func__); - struct autoparser::autoparser autoparser; - autoparser.analyze_template(tmpl); + const auto & autoparser = *tmpl.analysis; auto auto_params = autoparser::peg_generator::generate_parser(tmpl, params, autoparser); common_chat_msg_delimiters delimiters; @@ -1377,8 +1406,7 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ auto_params.thinking_end_tags = {std::move(end_tag)}; } } - common_peg_arena arena; - arena.load(auto_params.parser); + const auto & arena = auto_params.parser; LOG_DBG("%s: generated parser:\n%s\n\nparser generation prompt: %s\n", __func__, arena.dump(arena.root()).c_str(), auto_params.generation_prompt.c_str()); return auto_params; } catch (const std::exception & e) { @@ -1525,9 +1553,10 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars const common_chat_input & input, bool is_partial, const common_chat_parser_params & params) { - const common_peg_arena & parser = src_parser.empty() ? - build_chat_peg_parser([](common_chat_peg_builder & p) { return p.content(p.rest()) + p.end(); }) : - src_parser; + // both branches must be lvalues, a temporary here would copy the arena on every call + static const common_peg_arena content_only = + build_chat_peg_parser([](common_chat_peg_builder & p) { return p.content(p.rest()) + p.end(); }); + const common_peg_arena & parser = src_parser.empty() ? content_only : src_parser; if (src_parser.empty()) { LOG_DBG("No parser definition detected, assuming pure content parser."); @@ -1598,6 +1627,84 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars return msg; } +common_chat_session::common_chat_session(const common_chat_templates * tmpls, + const llama_vocab * vocab, + const common_chat_templates_inputs & inputs, + const common_chat_session_params & params) { + auto applied = common_chat_templates_apply(tmpls, inputs); + + templated = true; + prompt_text = std::move(applied.prompt); + result.role = "assistant"; + + grammar_text = std::move(applied.grammar); + grammar_lazy = applied.grammar_lazy; + stops = std::move(applied.additional_stops); + generation_prompt_text = applied.generation_prompt; + thinking_start = std::move(applied.thinking_start_tag); + thinking_ends = std::move(applied.thinking_end_tags); + + parser_params.format = applied.format; + parser_params.generation_prompt = vocab ? common_chat_input_tokenize(vocab, applied.generation_prompt) + : common_chat_input(applied.generation_prompt); + parser_params.debug = params.debug; + parser_params.parser = std::move(applied.parser); + + delimiters = std::move(applied.message_delimiters); + + if (vocab) { + common_params_sampling resolved; + resolved.grammar_lazy = applied.grammar_lazy; + common_sampling_add_preserved_tokens(resolved, vocab, applied.preserved_tokens); + common_sampling_add_grammar_triggers(resolved, vocab, std::move(applied.grammar_triggers)); + preserved_tokens = std::move(resolved.preserved_tokens); + grammar_triggers = std::move(resolved.grammar_triggers); + + delimiters.tokenize(vocab); + } else { + grammar_triggers = std::move(applied.grammar_triggers); + } + + if (inputs.continue_final_message != COMMON_CHAT_CONTINUATION_NONE && !params.echo) { + // start from the prefill so it is not emitted as part of the first delta + result = common_chat_parse(input, true, parser_params); + } +} + +void common_chat_session::apply_sampling(common_params_sampling & sampling) const { + if (!templated) { + return; + } + if (!grammar_text.empty()) { + sampling.grammar = {COMMON_GRAMMAR_TYPE_TOOL_CALLS, grammar_text}; + } + sampling.grammar_lazy = grammar_lazy; + sampling.generation_prompt = generation_prompt_text; + sampling.preserved_tokens.insert(preserved_tokens.begin(), preserved_tokens.end()); + sampling.grammar_triggers.insert(sampling.grammar_triggers.end(), grammar_triggers.begin(), grammar_triggers.end()); +} + +const common_chat_msg & common_chat_session::feed(const common_chat_input & chunk) { + GGML_ASSERT(!finished && "feed() after finish()"); + input.append(chunk); + auto msg = common_chat_parse(input, true, parser_params); + if (!msg.empty()) { + result = std::move(msg); + } + return result; +} + +const common_chat_msg & common_chat_session::finish(const common_chat_input & chunk) { + GGML_ASSERT(!finished && "finish() called twice"); + finished = true; + input.append(chunk); + auto msg = common_chat_parse(input, false, parser_params); + if (!msg.empty()) { + result = std::move(msg); + } + return result; +} + std::map common_chat_templates_get_caps(const common_chat_templates * chat_templates) { GGML_ASSERT(chat_templates != nullptr); GGML_ASSERT(chat_templates->template_default != nullptr); diff --git a/common/chat.h b/common/chat.h index 4509a04cc..775334924 100644 --- a/common/chat.h +++ b/common/chat.h @@ -22,8 +22,16 @@ struct common_chat_templates; namespace autoparser { struct generation_params; +struct autoparser; } // namespace autoparser +struct common_chat_params; +struct common_chat_template; + +// Builds the prompt and parser for a template that has a dedicated handler (see common/parsers) +using common_chat_params_init_fn = common_chat_params (*)(const common_chat_template & tmpl, + const autoparser::generation_params & inputs); + struct common_chat_tool_call { std::string name; std::string arguments; @@ -54,19 +62,20 @@ struct common_chat_template { std::string eos_tok; std::string src; chat_template_caps caps; + // Dedicated handler picked once from the source, null when the differential autoparser is used + common_chat_params_init_fn params_init = nullptr; - common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token) { - jinja::lexer lexer; - auto lexer_res = lexer.tokenize(src); - this->prog = jinja::parse_from_tokens(lexer_res); + // Differential analysis, run once here when there is no dedicated handler. Null when there + // is one, or when the analysis failed, in which case analysis_error says why. + std::unique_ptr analysis; + std::string analysis_error; - this->src = lexer_res.source; - this->bos_tok = bos_token; - this->eos_tok = eos_token; + common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token); - this->caps = jinja::caps_get(prog); - // LOG_INF("%s: caps:\n%s\n", __func__, this->caps.to_string().c_str()); - } + // autoparser is incomplete here, so these are defined where it is complete + ~common_chat_template(); + common_chat_template(common_chat_template &&); + common_chat_template & operator=(common_chat_template &&); const std::string & source() const { return src; } const std::string & bos_token() const { return bos_tok; } @@ -209,8 +218,6 @@ struct common_chat_msg_delimiters { // split tokens into message spans. skips maps a start index to a length of a region to jump over without matching common_chat_msg_spans split(const llama_tokens & tokens, const std::map & skips = {}) const; - - common_json to_json() const; }; struct common_chat_tool { @@ -278,7 +285,7 @@ struct common_chat_params { std::vector grammar_triggers; std::vector preserved_tokens; std::vector additional_stops; - std::string parser; + common_peg_arena parser; common_chat_msg_delimiters message_delimiters; }; @@ -310,16 +317,10 @@ common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const st // per-message parsing syntax // should be derived from common_chat_params struct common_chat_parser_params { - common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY; - common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool parse_reasoning" - // Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode) - bool reasoning_in_content = false; - common_chat_input generation_prompt; - bool parse_tool_calls = true; - bool is_continuation = false; - bool echo = false; // Include assistant prefilled msg in output - bool debug = false; // Enable debug output for PEG parser - common_peg_arena parser = {}; + common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY; + common_chat_input generation_prompt; + bool debug = false; // Enable debug output for PEG parser + common_peg_arena parser = {}; common_chat_parser_params() = default; common_chat_parser_params(const common_chat_params & chat_params) { format = chat_params.format; @@ -365,6 +366,60 @@ const char * common_chat_format_name(common_chat_format format); common_chat_msg common_chat_parse(const common_chat_input & input, bool is_partial, const common_chat_parser_params & params); common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const common_chat_input & input, bool is_partial, const common_chat_parser_params & params); +struct common_chat_session_params { + bool echo = false; // include the assistant prefill in the output when continuing a message + bool debug = false; // enable debug output for the PEG parser +}; + +class common_chat_session { + public: + common_chat_session() { result.role = "assistant"; } + + common_chat_session(const common_chat_templates * tmpls, + const llama_vocab * vocab, + const common_chat_templates_inputs & inputs, + const common_chat_session_params & params = {}); + + const std::string & prompt() const { return prompt_text; } + common_chat_format format() const { return parser_params.format; } + const common_chat_msg & msg() const { return result; } + const common_peg_arena & parser() const { return parser_params.parser; } + + const std::string & grammar() const { return grammar_text; } + const std::string & generation_prompt() const { return generation_prompt_text; } + const std::string & thinking_start_tag() const { return thinking_start; } + const std::vector & thinking_end_tags() const { return thinking_ends; } + const std::vector & additional_stops() const { return stops; } + + const common_chat_msg_delimiters & message_delimiters() const { return delimiters; } + + void apply_sampling(common_params_sampling & sampling) const; + + bool has_template() const { return templated; } + + const common_chat_msg & feed(const common_chat_input & chunk); + + const common_chat_msg & finish(const common_chat_input & chunk = {}); + + private: + std::string prompt_text; + std::string grammar_text; + bool grammar_lazy = false; + std::vector grammar_triggers; + std::set preserved_tokens; + std::vector stops; + std::string generation_prompt_text; + std::string thinking_start; + std::vector thinking_ends; + + common_chat_parser_params parser_params; + common_chat_msg_delimiters delimiters; + common_chat_input input; + common_chat_msg result; + bool templated = false; + bool finished = false; +}; + // used by arg and server const char * common_reasoning_format_name(common_reasoning_format format); common_reasoning_format common_reasoning_format_from_name(const std::string & format); @@ -401,8 +456,7 @@ std::string common_chat_template_generation_prompt( std::optional common_chat_try_specialized_template( const common_chat_template & tmpl, - const std::string & src, - autoparser::generation_params & params); + const autoparser::generation_params & params); // specialized per-task preset @@ -412,5 +466,3 @@ struct common_chat_prompt_preset { }; common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates); - -common_chat_msg_delimiters common_chat_msg_delimiters_parse(const common_json & delimiters); diff --git a/common/parsers/cohere2moe.cpp b/common/parsers/cohere2moe.cpp index 59595368d..dde2da929 100644 --- a/common/parsers/cohere2moe.cpp +++ b/common/parsers/cohere2moe.cpp @@ -77,7 +77,7 @@ common_chat_params common_chat_params_init_cohere2moe(const common_chat_template data.prompt += data.generation_prompt; } - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto generation_prompt = p.literal(GEN_PREFIX); auto end = p.end(); @@ -124,12 +124,10 @@ common_chat_params common_chat_params_init_cohere2moe(const common_chat_template return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end; }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); data.grammar_triggers = { diff --git a/common/parsers/deepseek.cpp b/common/parsers/deepseek.cpp index 88122eaab..61b2b66c6 100644 --- a/common/parsers/deepseek.cpp +++ b/common/parsers/deepseek.cpp @@ -145,7 +145,7 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ bool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto generation_prompt = p.literal(GEN_PROMPT); auto end = p.end(); @@ -256,12 +256,10 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ generation_prompt + reasoning + content_before_tools + tool_calls + end; }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = has_tools && !require_tools; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); data.grammar_triggers = { diff --git a/common/parsers/functionary-v3-2.cpp b/common/parsers/functionary-v3-2.cpp index 56f7643f2..3a1eff3d5 100644 --- a/common/parsers/functionary-v3-2.cpp +++ b/common/parsers/functionary-v3-2.cpp @@ -21,7 +21,7 @@ common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_te data.prompt += data.generation_prompt; } - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { // Functionary v3.2 format: // - Normal content: >>>all\n{content} // - Tool calls: >>>function_name\n{json_args} @@ -76,13 +76,11 @@ common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_te return generation_prompt + ret; }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); // Grammar trigger for when the model starts outputting a tool call diff --git a/common/parsers/gemma4.cpp b/common/parsers/gemma4.cpp index 2223089da..5ba264889 100644 --- a/common/parsers/gemma4.cpp +++ b/common/parsers/gemma4.cpp @@ -198,7 +198,7 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template & auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto start = p.rule("start", p.optional(p.literal("<|turn>model\n"))); if (extract_reasoning) { @@ -290,12 +290,10 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template & return start + p.one_or_more(message); }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); data.grammar_triggers = { diff --git a/common/parsers/gigachat-v3.cpp b/common/parsers/gigachat-v3.cpp index 949942547..0d20f83f7 100644 --- a/common/parsers/gigachat-v3.cpp +++ b/common/parsers/gigachat-v3.cpp @@ -25,7 +25,7 @@ common_chat_params common_chat_params_init_gigachat_v3( auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; const auto *tool_call_start_prefix = "<|message_sep|>\n\nfunction call<|role_sep|>\n"; - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto ret = p.eps(); if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { // Build a choice of all available tools @@ -60,13 +60,11 @@ common_chat_params common_chat_params_init_gigachat_v3( return p.literal("assistant<|role_sep|>\n") + ret; }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); data.grammar_triggers = { diff --git a/common/parsers/gpt-oss.cpp b/common/parsers/gpt-oss.cpp index ed50524a9..b91c61d2a 100644 --- a/common/parsers/gpt-oss.cpp +++ b/common/parsers/gpt-oss.cpp @@ -45,8 +45,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & data.thinking_start_tag = "<|channel|>analysis<|message|>"; data.thinking_end_tags = {"<|end|>"}; - // These special tokens are required to parse properly, so we include them - // even if parse_tool_calls is false. + // These special tokens are required to parse properly data.preserved_tokens = { "<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>", }; @@ -68,7 +67,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto start = p.rule("start", p.literal("<|start|>assistant")); auto end = p.rule("end", p.literal("<|end|>")); auto content = p.rule("message-content", p.until("<|end|>")); @@ -138,12 +137,10 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & return p.zero_or_more(start + any) + start + (final_msg | unsolicited); }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); data.grammar_triggers = { diff --git a/common/parsers/k2-horizon.cpp b/common/parsers/k2-horizon.cpp index bd64f6890..b73734fbd 100644 --- a/common/parsers/k2-horizon.cpp +++ b/common/parsers/k2-horizon.cpp @@ -77,7 +77,7 @@ common_chat_params common_chat_params_init_k2_horizon(const common_chat_template data.prompt += data.generation_prompt; } - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto generation_prompt = p.literal(GEN_PREFIX); auto think_end = p.choice(); @@ -174,12 +174,10 @@ common_chat_params common_chat_params_init_k2_horizon(const common_chat_template return generation_prompt + (reasoning << content << tool_calls); }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = !(has_response_format || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED); data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); if (data.grammar_lazy) { diff --git a/common/parsers/kimi-k2.cpp b/common/parsers/kimi-k2.cpp index 9992968a5..5e9d49ef9 100644 --- a/common/parsers/kimi-k2.cpp +++ b/common/parsers/kimi-k2.cpp @@ -48,7 +48,7 @@ common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & data.prompt += data.generation_prompt; } - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { // Kimi K2 Thinking format: // - Reasoning: {reasoning} // - Content: text after reasoning @@ -111,12 +111,10 @@ common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & return generation_prompt + reasoning + content_before_tools + tool_calls + end; }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); data.grammar_triggers = { diff --git a/common/parsers/kimi-k3.cpp b/common/parsers/kimi-k3.cpp index cf153f6b5..cee1c1671 100644 --- a/common/parsers/kimi-k3.cpp +++ b/common/parsers/kimi-k3.cpp @@ -66,7 +66,7 @@ common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & data.prompt += data.generation_prompt; } - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto end = p.end(); auto start = p.optional(p.literal(MSG_START)); @@ -151,12 +151,10 @@ common_chat_params common_chat_params_init_kimi_k3(const common_chat_template & return start + reasoning + response + tools + trailer + end; }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); data.grammar_triggers = { diff --git a/common/parsers/lfm2.cpp b/common/parsers/lfm2.cpp index 280788509..865929860 100644 --- a/common/parsers/lfm2.cpp +++ b/common/parsers/lfm2.cpp @@ -64,7 +64,7 @@ common_chat_params common_chat_params_init_lfm2(const common_chat_template & data.prompt += data.generation_prompt; } - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto generation_prompt = p.literal(GEN_PROMPT); auto end = p.end(); @@ -93,12 +93,10 @@ common_chat_params common_chat_params_init_lfm2(const common_chat_template & return generation_prompt + reasoning + content + tool_calls + end; }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); data.grammar_triggers = { diff --git a/common/parsers/ling3.cpp b/common/parsers/ling3.cpp index 67d44d47a..5d8deb9af 100644 --- a/common/parsers/ling3.cpp +++ b/common/parsers/ling3.cpp @@ -80,7 +80,7 @@ common_chat_params common_chat_params_init_ling3(const common_chat_template & auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto end = p.end(); // the effective parse input is generation_prompt + model output, so the @@ -185,12 +185,10 @@ common_chat_params common_chat_params_init_ling3(const common_chat_template & return opener + reasoning + content + tools + tail + end; }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = !has_response_format && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); data.grammar_triggers = { diff --git a/common/parsers/llm-jp-harmony.cpp b/common/parsers/llm-jp-harmony.cpp index cafb82c12..aebceb126 100644 --- a/common/parsers/llm-jp-harmony.cpp +++ b/common/parsers/llm-jp-harmony.cpp @@ -48,8 +48,7 @@ common_chat_params common_chat_params_init_llm_jp_harmony(const common_chat_temp data.thinking_start_tag = "<|channel|>analysis<|message|>"; data.thinking_end_tags = {"<|end|>"}; - // These special tokens are required to parse properly, so we include them - // even if parse_tool_calls is false. + // These special tokens are required to parse properly data.preserved_tokens = { "<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>", }; @@ -71,7 +70,7 @@ common_chat_params common_chat_params_init_llm_jp_harmony(const common_chat_temp auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { // tokenizer space after special tokens; not p.space() since GBNF `space` allows one space only auto sp = p.chars("[ ]", 0, -1); auto channel_tag = p.literal("<|channel|>") + sp; @@ -144,12 +143,10 @@ common_chat_params common_chat_params_init_llm_jp_harmony(const common_chat_temp return p.zero_or_more(start + any) + start + final_msg; }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); data.grammar_triggers = { diff --git a/common/parsers/minicpm5.cpp b/common/parsers/minicpm5.cpp index 06f4cc8c8..45fad952b 100644 --- a/common/parsers/minicpm5.cpp +++ b/common/parsers/minicpm5.cpp @@ -46,7 +46,7 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template & data.prompt += data.generation_prompt; } - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto generation_prompt = p.literal("<|im_start|>assistant\n"); auto reasoning = p.eps(); @@ -113,12 +113,10 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template & return generation_prompt + reasoning + p.content(p.rest()) + p.end(); }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); data.grammar_triggers = { diff --git a/common/parsers/minimax-m3.cpp b/common/parsers/minimax-m3.cpp index c38ab3e23..65a023959 100644 --- a/common/parsers/minimax-m3.cpp +++ b/common/parsers/minimax-m3.cpp @@ -56,7 +56,7 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template data.prompt += data.generation_prompt; } - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START); auto end = p.end(); @@ -213,12 +213,10 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_template return generation_prompt + reasoning + content_before_tools + tool_calls + end; }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); data.grammar_triggers = { diff --git a/common/parsers/ministral3.cpp b/common/parsers/ministral3.cpp index 9cc5248b5..83089d711 100644 --- a/common/parsers/ministral3.cpp +++ b/common/parsers/ministral3.cpp @@ -72,7 +72,7 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templat data.prompt += data.generation_prompt; } - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto generation_prompt = p.eps(); auto reasoning = extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps(); @@ -108,13 +108,11 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templat return generation_prompt + (reasoning << p.content(p.rest())); }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); data.grammar_triggers = { diff --git a/common/parsers/muse-glimmer.cpp b/common/parsers/muse-glimmer.cpp index 6c7303701..f75af1cee 100644 --- a/common/parsers/muse-glimmer.cpp +++ b/common/parsers/muse-glimmer.cpp @@ -48,7 +48,7 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa // Constrained grammar whenever tools are offered or a response format is requested. auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto start = p.rule("start", p.literal("<|start|>assistant")); if (!extract_reasoning && !include_grammar) { @@ -131,12 +131,10 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa return p.zero_or_more(start + analysis) + start + final_msg; }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); data.grammar_triggers = { { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, diff --git a/common/parsers/qwen3-coder.cpp b/common/parsers/qwen3-coder.cpp index 323b3da31..006ec6f98 100644 --- a/common/parsers/qwen3-coder.cpp +++ b/common/parsers/qwen3-coder.cpp @@ -71,7 +71,7 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat }); } - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto generation_prompt = p.literal(GEN_PREFIX); auto reasoning = p.eps(); @@ -174,13 +174,11 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat return generation_prompt + (reasoning << p.content(p.rest())); }); - data.parser = parser.save(); - if (include_grammar) { data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; data.grammar = build_grammar([&](const common_grammar_builder & builder) { - parser.build_grammar(builder, data.grammar_lazy); + data.parser.build_grammar(builder, data.grammar_lazy); }); if (data.grammar_lazy) { diff --git a/common/parsers/translate-gemma.cpp b/common/parsers/translate-gemma.cpp index c52778064..45b740e90 100644 --- a/common/parsers/translate-gemma.cpp +++ b/common/parsers/translate-gemma.cpp @@ -54,10 +54,9 @@ common_chat_params common_chat_params_init_translate_gemma( data.prompt += data.generation_prompt; } - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { return p.literal(data.generation_prompt) << p.content(p.rest()); }); - data.parser = parser.save(); return data; } diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 75a908a28..8033f7e81 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -1814,309 +1814,6 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo } } -static common_json serialize_parser_variant(const common_peg_parser_variant & variant) { - using json = common_json; - - return std::visit([](const auto & p) -> json { - using T = std::decay_t; - - if constexpr (std::is_same_v) { - return json{{"type", "epsilon"}}; - } else if constexpr (std::is_same_v) { - return json{{"type", "start"}}; - } else if constexpr (std::is_same_v) { - return json{{"type", "end"}}; - } else if constexpr (std::is_same_v) { - return json{{"type", "literal"}, {"literal", p.literal}}; - } else if constexpr (std::is_same_v) { - return json{{"type", "sequence"}, {"children", p.children}}; - } else if constexpr (std::is_same_v) { - return json{{"type", "choice"}, {"children", p.children}}; - } else if constexpr (std::is_same_v) { - return json{ - {"type", "repetition"}, - {"child", p.child}, - {"min_count", p.min_count}, - {"max_count", p.max_count} - }; - } else if constexpr (std::is_same_v) { - return json{{"type", "and"}, {"child", p.child}}; - } else if constexpr (std::is_same_v) { - return json{{"type", "not"}, {"child", p.child}}; - } else if constexpr (std::is_same_v) { - return json{{"type", "any"}}; - } else if constexpr (std::is_same_v) { - return json{{"type", "space"}}; - } else if constexpr (std::is_same_v) { - json ranges = json::array(); - for (const auto & range : p.ranges) { - ranges.push_back({{"start", range.start}, {"end", range.end}}); - } - return json{ - {"type", "chars"}, - {"pattern", p.pattern}, - {"ranges", ranges}, - {"negated", p.negated}, - {"min_count", p.min_count}, - {"max_count", p.max_count} - }; - } else if constexpr (std::is_same_v) { - return json{{"type", "string"}, {"delimiter", std::string(1, p.delimiter)}}; - } else if constexpr (std::is_same_v) { - return json{{"type", "until"}, {"delimiters", p.delimiters}}; - } else if constexpr (std::is_same_v) { - return json{ - {"type", "schema"}, - {"child", p.child}, - {"name", p.name}, - {"raw", p.raw} - }; - } else if constexpr (std::is_same_v) { - return json{ - {"type", "rule"}, - {"name", p.name}, - {"child", p.child}, - {"trigger", p.trigger} - }; - } else if constexpr (std::is_same_v) { - return json{{"type", "ref"}, {"name", p.name}}; - } else if constexpr (std::is_same_v) { - return json{{"type", "atomic"}, {"child", p.child}}; - } else if constexpr (std::is_same_v) { - return json{ - {"type", "tag"}, - {"child", p.child}, - {"tag", p.tag} - }; - } else if constexpr (std::is_same_v) { - return json{{"type", "gbnf"}, {"child", p.child}, {"grammar", p.grammar}}; - } else if constexpr (std::is_same_v) { - return json{{"type", "ac"}, {"child", p.child}, {"delimiters", p.delimiters}}; - } - }, variant); -} - -common_json common_peg_arena::to_json() const { - auto parsers = common_json::array(); - for (const auto & parser : parsers_) { - parsers.push_back(serialize_parser_variant(parser)); - } - return common_json{ - {"parsers", parsers}, - {"rules", rules_}, - {"root", root_} - }; -} - -static common_peg_parser_variant deserialize_parser_variant(const common_json & j) { - if (!j.contains("type") || !j["type"].is_string()) { - throw std::runtime_error("Parser variant JSON missing or invalid 'type' field"); - } - - std::string type = j["type"]; - - if (type == "epsilon") { - return common_peg_epsilon_parser{}; - } - if (type == "start") { - return common_peg_start_parser{}; - } - if (type == "end") { - return common_peg_end_parser{}; - } - if (type == "literal") { - if (!j.contains("literal") || !j["literal"].is_string()) { - throw std::runtime_error("literal parser missing or invalid 'literal' field"); - } - return common_peg_literal_parser{j["literal"]}; - } - if (type == "sequence") { - if (!j.contains("children") || !j["children"].is_array()) { - throw std::runtime_error("sequence parser missing or invalid 'children' field"); - } - return common_peg_sequence_parser{j["children"].get>()}; - } - if (type == "choice") { - if (!j.contains("children") || !j["children"].is_array()) { - throw std::runtime_error("choice parser missing or invalid 'children' field"); - } - return common_peg_choice_parser{j["children"].get>()}; - } - if (type == "repetition") { - if (!j.contains("child") || !j.contains("min_count") || !j.contains("max_count")) { - throw std::runtime_error("repetition parser missing required fields"); - } - return common_peg_repetition_parser{ - j["child"].get(), - j["min_count"].get(), - j["max_count"].get() - }; - } - if (type == "and") { - if (!j.contains("child")) { - throw std::runtime_error("and parser missing 'child' field"); - } - return common_peg_and_parser{j["child"].get()}; - } - if (type == "not") { - if (!j.contains("child")) { - throw std::runtime_error("not parser missing 'child' field"); - } - return common_peg_not_parser{j["child"].get()}; - } - if (type == "any") { - return common_peg_any_parser{}; - } - if (type == "space") { - return common_peg_space_parser{}; - } - if (type == "chars") { - if (!j.contains("pattern") || !j.contains("ranges") || !j.contains("negated") || - !j.contains("min_count") || !j.contains("max_count")) { - throw std::runtime_error("chars parser missing required fields"); - } - common_peg_chars_parser parser; - parser.pattern = j["pattern"]; - parser.negated = j["negated"].get(); - parser.min_count = j["min_count"].get(); - parser.max_count = j["max_count"].get(); - for (const auto & range_json : j["ranges"]) { - if (!range_json.contains("start") || !range_json.contains("end")) { - throw std::runtime_error("char_range missing 'start' or 'end' field"); - } - parser.ranges.push_back({ - range_json["start"].get(), - range_json["end"].get() - }); - } - return parser; - } - if (type == "string") { - if (!j.contains("delimiter")) { - throw std::runtime_error("string parser missing delimiter field."); - } - std::string delimiter = j["delimiter"]; - if (delimiter.empty()) { - throw std::runtime_error("string parser delimiter is empty."); - } - return common_peg_string_parser{delimiter[0]}; - } - if (type == "until") { - if (!j.contains("delimiters") || !j["delimiters"].is_array()) { - throw std::runtime_error("until parser missing or invalid 'delimiters' field"); - } - return common_peg_until_parser{j["delimiters"].get>()}; - } - if (type == "schema") { - if (!j.contains("child") || !j.contains("name") || !j.contains("raw")) { - throw std::runtime_error("schema parser missing required fields"); - } - common_peg_schema_parser parser; - parser.child = j["child"].get(); - parser.name = j["name"]; - parser.raw = j["raw"].get(); - return parser; - } - if (type == "rule") { - if (!j.contains("name") || !j.contains("child") || !j.contains("trigger")) { - throw std::runtime_error("rule parser missing required fields"); - } - return common_peg_rule_parser{ - j["name"].get(), - j["child"].get(), - j["trigger"].get() - }; - } - if (type == "ref") { - if (!j.contains("name") || !j["name"].is_string()) { - throw std::runtime_error("ref parser missing or invalid 'name' field"); - } - return common_peg_ref_parser{j["name"]}; - } - if (type == "atomic") { - if (!j.contains("child")) { - throw std::runtime_error("tag parser missing required fields"); - } - return common_peg_atomic_parser{ - j["child"].get(), - }; - } - if (type == "tag") { - if (!j.contains("child") || !j.contains("tag")) { - throw std::runtime_error("tag parser missing required fields"); - } - return common_peg_tag_parser{ - j["child"].get(), - j["tag"].get(), - }; - } - - if (type == "gbnf") { - if (!j.contains("child") || !j.contains("grammar")) { - throw std::runtime_error("gbnf parser missing required fields"); - } - return common_peg_gbnf_parser{ - j["child"].get(), - j["grammar"].get(), - }; - } - - if (type == "ac") { - if (!j.contains("child") || !j.contains("delimiters") || !j["delimiters"].is_array() || j["delimiters"].empty()) { - throw std::runtime_error("ac parser requires 'child' and a non-empty 'delimiters' array"); - } - return common_peg_ac_parser{ - j["child"].get(), - j["delimiters"].get>(), - }; - } - - throw std::runtime_error("Unknown parser type: " + type); -} - -common_peg_arena common_peg_arena::from_json(const common_json & j) { - if (!j.contains("parsers") || !j["parsers"].is_array()) { - throw std::runtime_error("JSON missing or invalid 'parsers' array"); - } - if (!j.contains("rules") || !j["rules"].is_object()) { - throw std::runtime_error("JSON missing or invalid 'rules' object"); - } - if (!j.contains("root")) { - throw std::runtime_error("JSON missing 'root' field"); - } - - common_peg_arena arena; - - const auto & parsers_json = j["parsers"]; - arena.parsers_.reserve(parsers_json.size()); - for (const auto & parser_json : parsers_json) { - arena.parsers_.push_back(deserialize_parser_variant(parser_json)); - } - - arena.rules_ = j["rules"].get>(); - - for (const auto & [name, id] : arena.rules_) { - if (id >= arena.parsers_.size()) { - throw std::runtime_error("Rule '" + name + "' references invalid parser ID: " + std::to_string(id)); - } - } - - arena.root_ = j["root"].get(); - if (arena.root_ != COMMON_PEG_INVALID_PARSER_ID && arena.root_ >= arena.parsers_.size()) { - throw std::runtime_error("Root references invalid parser ID: " + std::to_string(arena.root_)); - } - - return arena; -} - -std::string common_peg_arena::save() const { - return to_json().dump(); -} - -void common_peg_arena::load(const std::string & data) { - *this = from_json(common_json::parse(data)); -} - common_peg_arena build_peg_parser(const std::function & fn) { common_peg_parser_builder builder; builder.set_root(fn(builder)); diff --git a/common/peg-parser.h b/common/peg-parser.h index 816d007b0..9255d6eb9 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -357,12 +357,6 @@ class common_peg_arena { std::string dump(common_peg_parser_id id) const; - common_json to_json() const; - static common_peg_arena from_json(const common_json & j); - - std::string save() const; - void load(const std::string & data); - friend class common_peg_parser_builder; private: diff --git a/common/sampling.cpp b/common/sampling.cpp index 6cc8872f2..9e16693fb 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -1050,3 +1050,41 @@ std::vector common_sampler_types_from_chars(const std::stri return samplers; } + +void common_sampling_add_preserved_tokens(common_params_sampling & sampling, const llama_vocab * vocab, const std::vector & tokens) { + GGML_ASSERT(vocab != nullptr); + for (const auto & t : tokens) { + auto ids = common_tokenize(vocab, t, false, true); + if (ids.size() == 1) { + sampling.preserved_tokens.insert(ids[0]); + } + } +} + +void common_sampling_add_grammar_triggers(common_params_sampling & sampling, const llama_vocab * vocab, std::vector triggers) { + GGML_ASSERT(vocab != nullptr); + for (auto & trigger : triggers) { + if (trigger.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) { + const auto & word = trigger.value; + auto ids = common_tokenize(vocab, word, false, true); + if (ids.size() == 1) { + auto token = ids[0]; + if (std::find(sampling.preserved_tokens.begin(), sampling.preserved_tokens.end(), (llama_token) token) == sampling.preserved_tokens.end()) { + throw std::runtime_error("Grammar trigger word should be marked as preserved token: " + word); + } + common_grammar_trigger token_trigger; + token_trigger.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN; + token_trigger.value = word; + token_trigger.token = token; + sampling.grammar_triggers.push_back(std::move(token_trigger)); + } else { + sampling.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word}); + } + } else { + sampling.grammar_triggers.push_back(std::move(trigger)); + } + } + if (sampling.grammar_lazy && sampling.grammar_triggers.empty()) { + throw std::runtime_error("Error: no triggers set for lazy grammar!"); + } +} diff --git a/common/sampling.h b/common/sampling.h index 7ebae3df8..f19bd27c4 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -118,6 +118,12 @@ std::string common_sampler_type_to_str(enum common_sampler_type cnstr); std::vector common_sampler_types_from_names(const std::vector & names); std::vector common_sampler_types_from_chars(const std::string & chars); +// add the strings that are a single token in the vocab to the preserved tokens +void common_sampling_add_preserved_tokens(common_params_sampling & sampling, const llama_vocab * vocab, const std::vector & tokens); + +// add grammar triggers, a trigger word that is a single token becomes a token trigger and must be a preserved token +void common_sampling_add_grammar_triggers(common_params_sampling & sampling, const llama_vocab * vocab, std::vector triggers); + llama_sampler * llama_sampler_init_llg(const llama_vocab * vocab, const char * grammar_kind, const char * grammar_data); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c44189ceb..5f9bd037f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -245,7 +245,6 @@ llama_build_and_test( peg-parser/test-basic.cpp peg-parser/test-gbnf-generation.cpp peg-parser/test-json-parser.cpp - peg-parser/test-json-serialization.cpp peg-parser/test-python-dict-parser.cpp peg-parser/test-unicode.cpp peg-parser/tests.h diff --git a/tests/peg-parser/test-json-serialization.cpp b/tests/peg-parser/test-json-serialization.cpp deleted file mode 100644 index da63a23bf..000000000 --- a/tests/peg-parser/test-json-serialization.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "tests.h" - -void test_json_serialization(testing &t) { - auto original = build_peg_parser([](common_peg_parser_builder & p) { - return "" + p.json() + ""; - }); - - auto json_serialized = original.to_json().dump(); - - t.test("compare before/after", [&](testing &t) { - auto deserialized = common_peg_arena::from_json(common_json::parse(json_serialized)); - - // Test complex JSON - std::string input = R"({"name": "test", "values": [1, 2, 3], "nested": {"a": true}})"; - common_peg_parse_context ctx1(input); - common_peg_parse_context ctx2(input); - - auto result1 = original.parse(ctx1); - auto result2 = deserialized.parse(ctx2); - - t.assert_equal("both_succeed", result1.success(), result2.success()); - t.assert_equal("same_end_pos", result1.end, result2.end); - }); - - t.bench("deserialize", [&]() { - auto deserialized = common_peg_arena::from_json(common_json::parse(json_serialized)); - }, 100); -} diff --git a/tests/peg-parser/tests.h b/tests/peg-parser/tests.h index 00e81815b..287da16d1 100644 --- a/tests/peg-parser/tests.h +++ b/tests/peg-parser/tests.h @@ -21,5 +21,4 @@ void test_basic(testing &t); void test_json_parser(testing &t); void test_gbnf_generation(testing &t); void test_unicode(testing &t); -void test_json_serialization(testing &t); void test_python_dict_parser(testing &t); diff --git a/tests/test-chat-auto-parser.cpp b/tests/test-chat-auto-parser.cpp index 5aa948251..1c641b329 100644 --- a/tests/test-chat-auto-parser.cpp +++ b/tests/test-chat-auto-parser.cpp @@ -450,7 +450,7 @@ static int debug_single_template(const debug_options & opts) { generation_params params = prepare_debug_params(opts, tools); common_chat_params parser_data; if (std::optional spec_tmpl = - common_chat_try_specialized_template(chat_template, template_source, params)) { + common_chat_try_specialized_template(chat_template, params)) { LOG_ERR("\n"); LOG_ERR("This template uses a specialized parser, analysis results will not be available.\n"); parser_data = *spec_tmpl; @@ -484,8 +484,7 @@ static int debug_single_template(const debug_options & opts) { if (!std::empty(parser_data.parser)) { LOG_ERR("\n=== Generated Parser ===\n"); - common_peg_arena arena; - arena.load(parser_data.parser); + const common_peg_arena & arena = parser_data.parser; LOG_ERR("%s\n", arena.dump(arena.root()).c_str()); LOG_ERR("\n=== Generated Grammar ===\n"); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 41e4948c4..09f593692 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -1090,26 +1090,6 @@ struct peg_test_case { std::vector expect_rules; }; -struct make_peg_parser { - common_chat_params params_; - common_peg_arena arena_; - bool detailed_debug_; - - make_peg_parser(common_chat_templates * tmpls, - const common_chat_templates_inputs & inputs, - bool detailed_debug = false) { - detailed_debug_ = detailed_debug; - params_ = common_chat_templates_apply(tmpls, inputs); - arena_.load(params_.parser); - } - - common_chat_msg parse(const std::string & msg, bool is_partial) const { - common_chat_parser_params parser_params(params_); - parser_params.debug = detailed_debug_; - return common_chat_peg_parse(arena_, common_chat_input(msg), is_partial, parser_params); - } -}; - // Global template filter for --template flag static std::string g_template_filter; @@ -1161,14 +1141,19 @@ static void test_peg_parser(common_chat_templates * tmpls, tc.expect.role = "assistant"; } - auto parser = make_peg_parser(tmpls, tc.params, detailed_debug); + common_chat_session_params session_params; + session_params.debug = detailed_debug; + common_chat_session session(tmpls, nullptr, tc.params, session_params); + const auto & parser = session.parser(); + common_params_sampling sampling; + session.apply_sampling(sampling); if (detailed_debug) { - LOG_DBG("Using parser: \n%s\n", parser.arena_.dump(parser.arena_.root()).c_str()); - LOG_DBG("Generation prompt: '%s'\n", parser.params_.generation_prompt.c_str()); + LOG_DBG("Using parser: \n%s\n", parser.dump(parser.root()).c_str()); + LOG_DBG("Generation prompt: '%s'\n", session.generation_prompt().c_str()); } for (const auto & rule : tc.expect_rules) { - if (!parser.arena_.has_rule(rule)) { + if (!parser.has_rule(rule)) { LOG_ERR("Missing rule: %s\n", rule.c_str()); common_log_flush(common_log_main()); throw std::runtime_error("Test failed"); @@ -1179,12 +1164,14 @@ static void test_peg_parser(common_chat_templates * tmpls, common_chat_msg msg_prev; msg_accum.role = msg_prev.role = "assistant"; + size_t fed = 0; for (size_t i = 1; i <= tc.input.size(); ++i) { auto is_partial = i < tc.input.size() || tc.is_partial; // Use UTF-8 safe truncation to avoid corrupting multi-byte characters size_t safe_len = utf8_truncate_safe_len(std::string_view(tc.input).substr(0, i)); - std::string prefix = tc.input.substr(0, safe_len); - common_chat_msg msg_current = parser.parse(prefix, is_partial); + common_chat_input chunk(tc.input.substr(fed, safe_len - fed)); + fed = safe_len; + const common_chat_msg & msg_current = is_partial ? session.feed(chunk) : session.finish(chunk); for (const auto & diff : common_chat_msg_diff::compute_diffs(msg_prev, msg_current)) { if (!diff.reasoning_content_delta.empty()) { @@ -1222,30 +1209,33 @@ static void test_peg_parser(common_chat_templates * tmpls, } if (!tc.is_partial) { - assert_msg_equals(tc.expect, parser.parse(tc.input, false), true); + if (tc.input.empty()) { + session.finish(); + } + assert_msg_equals(tc.expect, session.msg(), true); } assert_msg_equals(tc.expect, msg_accum, true); // A response format must be enforced by an eager grammar if (!tc.params.json_schema.empty()) { - if (parser.params_.grammar.empty()) { + if (session.grammar().empty()) { throw std::runtime_error("json_schema is set but no grammar was produced"); } - if (parser.params_.grammar_lazy) { + if (sampling.grammar_lazy) { throw std::runtime_error("json_schema is set but the grammar is lazy"); } } // Test grammar if present in params - if (!parser.params_.grammar.empty()) { - auto grammar = build_grammar(parser.params_.grammar); + if (!session.grammar().empty()) { + auto grammar = build_grammar(session.grammar()); if (!grammar) { - throw std::runtime_error("Failed to build grammar: " + parser.params_.grammar); + throw std::runtime_error("Failed to build grammar: " + session.grammar()); } // In production, grammar triggers match against the full generated text // including the generation prompt. All positions are in full_input coordinates. - const auto & gen_prompt = parser.params_.generation_prompt; + const auto & gen_prompt = sampling.generation_prompt; std::string full_input = gen_prompt + tc.input; // Determine whether the reasoning-budget sampler path applies: tool-call grammar @@ -1253,9 +1243,9 @@ static void test_peg_parser(common_chat_templates * tmpls, // budget sampler inhibits grammar application while inside thinking blocks — // triggers inside ... are suppressed. bool use_reasoning_budget_path = false; - if (parser.params_.grammar_lazy && !parser.params_.thinking_end_tags.empty()) { + if (sampling.grammar_lazy && !session.thinking_end_tags().empty()) { use_reasoning_budget_path = true; - for (const auto & trigger : parser.params_.grammar_triggers) { + for (const auto & trigger : sampling.grammar_triggers) { if (trigger.type != COMMON_GRAMMAR_TRIGGER_TYPE_WORD) { use_reasoning_budget_path = false; break; @@ -1270,8 +1260,8 @@ static void test_peg_parser(common_chat_templates * tmpls, // Reasoning-budget path: simulate thinking-aware trigger detection. // Walk through full_input tracking thinking state; only match triggers // when outside thinking blocks. - const auto & think_start = parser.params_.thinking_start_tag; - const auto & think_ends = parser.params_.thinking_end_tags; + const auto & think_start = session.thinking_start_tag(); + const auto & think_ends = session.thinking_end_tags(); bool in_thinking = false; for (size_t i = 0; i < full_input.size(); ++i) { @@ -1292,7 +1282,7 @@ static void test_peg_parser(common_chat_templates * tmpls, continue; } // Outside thinking — check if any trigger word starts here - for (const auto & trigger : parser.params_.grammar_triggers) { + for (const auto & trigger : sampling.grammar_triggers) { if (full_input.compare(i, trigger.value.size(), trigger.value) == 0) { if (earliest_trigger_pos == std::string::npos || i < earliest_trigger_pos) { earliest_trigger_pos = i; @@ -1314,7 +1304,7 @@ static void test_peg_parser(common_chat_templates * tmpls, if (!use_reasoning_budget_path) { // Legacy path: find triggers without thinking-awareness - for (const auto & trigger : parser.params_.grammar_triggers) { + for (const auto & trigger : sampling.grammar_triggers) { size_t pos = std::string::npos; std::smatch match; switch (trigger.type) { @@ -1370,16 +1360,16 @@ static void test_peg_parser(common_chat_templates * tmpls, // If the test expects tool calls and the grammar is lazy, the trigger must fire. // Otherwise the grammar would never activate in production and tool calls wouldn't // be constrained. A silent skip here would hide broken triggers. - if (parser.params_.grammar_lazy && !tc.expect.tool_calls.empty() && !tc.is_partial + if (sampling.grammar_lazy && !tc.expect.tool_calls.empty() && !tc.is_partial && earliest_trigger_pos == std::string::npos) { std::string trigger_desc; - for (const auto & trigger : parser.params_.grammar_triggers) { + for (const auto & trigger : sampling.grammar_triggers) { trigger_desc += "\n [type=" + std::to_string(trigger.type) + "] " + trigger.value; } throw std::runtime_error( "Grammar trigger did not fire, but test expects tool calls (lazy grammar).\n" ">>> Input: " + full_input + "\n" - ">>> Triggers (" + std::to_string(parser.params_.grammar_triggers.size()) + "):" + trigger_desc); + ">>> Triggers (" + std::to_string(sampling.grammar_triggers.size()) + "):" + trigger_desc); } // Determine the constrained portion of input to test against grammar. @@ -1392,7 +1382,7 @@ static void test_peg_parser(common_chat_templates * tmpls, auto constrain_from = std::max(earliest_trigger_pos, gen_prompt.size()); constrained = full_input.substr(constrain_from); grammar_triggered = true; - } else if (!parser.params_.grammar_lazy) { + } else if (!sampling.grammar_lazy) { // For non-lazy grammars, the entire input should match grammar_triggered = true; } @@ -1411,7 +1401,7 @@ static void test_peg_parser(common_chat_templates * tmpls, std::to_string(result.matched_codepoints) + " codepoints): " + (result.matched_prefix.size() > 100 ? result.matched_prefix.substr(0, 100) + "..." : result.matched_prefix) + "\n\n>>> Expected next: " + result.expected_description + - "\n\n>>> Grammar: " + parser.params_.grammar; + "\n\n>>> Grammar: " + session.grammar(); } else { error_msg = "Grammar match failed:\n\n" @@ -1422,7 +1412,7 @@ static void test_peg_parser(common_chat_templates * tmpls, (result.matched_prefix.size() > 100 ? result.matched_prefix.substr(0, 100) + "..." : result.matched_prefix) + "\n\n>>> Failing character: " + result.failing_char + "\n\n>>> Expected: " + result.expected_description + - "\n\n>>> Grammar: " + parser.params_.grammar; + "\n\n>>> Grammar: " + session.grammar(); } throw std::runtime_error(error_msg); } @@ -1437,7 +1427,7 @@ static void test_peg_parser(common_chat_templates * tmpls, // Start from tc.expect but copy tool call arguments from the actual parser // output, which preserves original JSON formatting (e.g. {"arg1":1} vs {"arg1": 1}). auto reconstruction_msg = tc.expect; - auto parsed_msg = parser.parse(tc.input, false); + const auto & parsed_msg = session.msg(); for (size_t i = 0; i < reconstruction_msg.tool_calls.size() && i < parsed_msg.tool_calls.size(); i++) { reconstruction_msg.tool_calls[i].arguments = parsed_msg.tool_calls[i].arguments; } @@ -1446,7 +1436,7 @@ static void test_peg_parser(common_chat_templates * tmpls, reconstruction_inputs.add_generation_prompt = false; auto reconstruction_params = common_chat_templates_apply(tmpls, reconstruction_inputs); - std::string expected_text = parser.params_.prompt + tc.input; + std::string expected_text = session.prompt() + tc.input; bool match = reconstruction_params.prompt == expected_text || (reconstruction_params.prompt.size() > expected_text.size() && reconstruction_params.prompt.compare(0, expected_text.size(), expected_text) == 0); @@ -4644,8 +4634,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) { inputs.messages = { msg }; auto params = common_chat_templates_apply(tmpls.get(), inputs); - common_peg_arena arena; - arena.load(params.parser); + const common_peg_arena & arena = params.parser; common_chat_parser_params pp(params); // generation_prompt is non-empty for thinking models, so result.end @@ -7791,6 +7780,77 @@ static void test_deepseek_v4_tool_result_ordering() { } } +static void test_chat_session() { + LOG_DBG("%s\n", __func__); + auto tmpls = read_templates("models/templates/Qwen3.5-4B.jinja"); + + common_chat_templates_inputs inputs; + inputs.messages = { message_user }; + inputs.tools = { special_function_tool }; + inputs.reasoning_format = COMMON_REASONING_FORMAT_AUTO; + + const std::string output = + "I'm\nthinking\n\n\n" + "\n" + "\n" + "\n1\n\n" + "\n" + ""; + + // the session renders the prompt and parses the output fed to it in small chunks + { + common_chat_session session(tmpls.get(), nullptr, inputs); + + assert_contains(session.prompt(), "\"name\": \"special_function\""); + assert_contains(session.prompt(), "<|im_start|>user\nHey there!<|im_end|>\n<|im_start|>assistant\n\n"); + assert_equals(false, session.grammar().empty()); + assert_equals(std::string("<|im_start|>assistant\n\n"), session.generation_prompt()); + + const std::string thinking = "I'm\nthinking\n\n\n"; + for (size_t i = 0; i < thinking.size(); i += 3) { + session.feed(common_chat_input(thinking.substr(i, 3))); + } + assert_msg_equals(simple_assist_msg("", "I'm\nthinking\n"), session.msg()); + + for (size_t i = thinking.size(); i < output.size(); i += 3) { + session.feed(common_chat_input(output.substr(i, 3))); + } + assert_msg_equals(simple_assist_msg("", "I'm\nthinking\n", "special_function", "{\"arg1\":1}"), session.finish()); + } + + // a copy does not see what is fed to the original + { + common_chat_session a(tmpls.get(), nullptr, inputs); + common_chat_session b = a; + a.feed(common_chat_input(output)); + assert_equals(true, b.msg().empty()); + b.feed(common_chat_input("I'm\nthinking\n\n\nHello")); + assert_equals(std::string("Hello"), b.msg().content); + assert_equals(std::string("special_function"), a.finish().tool_calls.at(0).name); + } + + // a continued message starts from the prefill, unless it is echoed + { + common_chat_templates_inputs cont; + cont.messages = { message_user, message_assist_prefill_content }; + cont.add_generation_prompt = false; + cont.continue_final_message = COMMON_CHAT_CONTINUATION_CONTENT; + cont.reasoning_format = COMMON_REASONING_FORMAT_AUTO; + + common_chat_session session(tmpls.get(), nullptr, cont); + assert_equals(std::string("<|im_start|>assistant\n\nI'm thinking\n\n\nHello, "), + session.generation_prompt()); + assert_msg_equals(simple_assist_msg("Hello, ", "I'm thinking\n"), session.msg()); + session.feed(common_chat_input("world!")); + assert_equals(std::string("Hello, world!"), session.msg().content); + + common_chat_session_params echo; + echo.echo = true; + common_chat_session echoed(tmpls.get(), nullptr, cont, echo); + assert_equals(true, echoed.msg().empty()); + } +} + static void test_reasoning_budget_tokens_per_request() { LOG_DBG("%s\n", __func__); // Use Qwen3 template which has ... reasoning markers. @@ -7811,7 +7871,8 @@ static void test_reasoning_budget_tokens_per_request() { {"reasoning_budget_tokens", 0}, }; std::vector out_files; - auto llama_params = oaicompat_chat_params_parse(body, opt, out_files); + common_chat_session out_session; + auto llama_params = oaicompat_chat_params_parse(nullptr, body, opt, out_files, out_session); // The per-request value must win over the server default (-1). if (!llama_params.contains("reasoning_budget_tokens")) { @@ -7844,7 +7905,8 @@ static void test_reasoning_budget_message_per_request() { {"reasoning_budget_message", per_request_message}, }; std::vector out_files; - auto llama_params = oaicompat_chat_params_parse(body, opt, out_files); + common_chat_session out_session; + auto llama_params = oaicompat_chat_params_parse(nullptr, body, opt, out_files, out_session); // The per-request value must win over the server default. if (!llama_params.contains("reasoning_budget_message")) { @@ -8035,6 +8097,7 @@ int main(int argc, char ** argv) { test_deepseek_v4_tool_result_ordering(); test_template_generation_prompt(); test_reasoning_effort_caps(); + test_chat_session(); test_reasoning_budget_tokens_per_request(); test_reasoning_budget_message_per_request(); test_template_output_peg_parsers(detailed_debug); diff --git a/tests/test-peg-parser.cpp b/tests/test-peg-parser.cpp index 7d22d7761..863bfe8a0 100644 --- a/tests/test-peg-parser.cpp +++ b/tests/test-peg-parser.cpp @@ -19,7 +19,6 @@ int main(int argc, char *argv[]) { t.test("unicode", test_unicode); t.test("json", test_json_parser); t.test("gbnf", test_gbnf_generation); - t.test("serialization", test_json_serialization); t.test("python-dict", test_python_dict_parser); return t.summary(); diff --git a/tools/server/README.md b/tools/server/README.md index 41dca6152..8fe260a32 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1353,10 +1353,6 @@ The `response_format` parameter supports both plain JSON output (e.g. `{"type": `reasoning_control`: Arms realtime reasoning control for this completion so it can be ended early via `/v1/chat/completions/control`. Defaults to `false`. -`generation_prompt`: The generation prompt that was prefilled in by the template. Prepended to model output before parsing. - -`parse_tool_calls`: Whether to parse the generated tool call. - `parallel_tool_calls` : Whether to enable parallel/multiple tool calls (only supported on some models, verification is based on jinja template). For multimodal input (typed content, `messages[i].content[j]`): diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index d364ba898..5f77d43d0 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1266,9 +1266,11 @@ server_tokens tokenize_oai_content_array(const llama_vocab * vocab, mtmd_context // used by /chat/completions endpoint json oaicompat_chat_params_parse( + const llama_vocab * vocab, json & body, /* openai api json semantics */ const server_chat_params & opt, - std::vector & out_files) + std::vector & out_files, + common_chat_session & out_session) { json llama_params; @@ -1393,7 +1395,6 @@ json oaicompat_chat_params_parse( if (body.contains("grammar")) { throw std::invalid_argument("Cannot use custom grammar constraints with tools."); } - llama_params["parse_tool_calls"] = true; } // merge the template args provided from command line with the args provided in the user request @@ -1427,31 +1428,11 @@ json oaicompat_chat_params_parse( inputs.force_pure_content = opt.force_pure_content; // Apply chat template to the list of messages - auto chat_params = common_chat_templates_apply(opt.tmpls.get(), inputs); + common_chat_session_params session_params; + session_params.echo = json_value(body, "echo", false); + out_session = common_chat_session(opt.tmpls.get(), vocab, inputs, session_params); - llama_params["chat_format"] = static_cast(chat_params.format); - llama_params["prompt"] = chat_params.prompt; - if (!chat_params.grammar.empty()) { - llama_params["grammar"] = chat_params.grammar; - llama_params["grammar_type"] = std::string("tool_calls"); - } - llama_params["grammar_lazy"] = chat_params.grammar_lazy; - auto grammar_triggers = json::array(); - for (const auto & trigger : chat_params.grammar_triggers) { - server_grammar_trigger ct(trigger); - grammar_triggers.push_back(ct.to_json()); - } - llama_params["grammar_triggers"] = grammar_triggers; - llama_params["preserved_tokens"] = chat_params.preserved_tokens; - llama_params["generation_prompt"] = chat_params.generation_prompt; - for (const auto & stop : chat_params.additional_stops) { - llama_params["stop"].push_back(stop); - } - if (!chat_params.parser.empty()) { - llama_params["chat_parser"] = chat_params.parser; - } - - llama_params["message_delimiters"] = chat_params.message_delimiters.to_json(); + llama_params["prompt"] = out_session.prompt(); // Reasoning budget: pass parameters through to sampling layer { @@ -1461,10 +1442,10 @@ json oaicompat_chat_params_parse( reasoning_budget = opt.reasoning_budget; } - if (!chat_params.thinking_end_tags.empty()) { + if (!out_session.thinking_end_tags().empty()) { llama_params["reasoning_budget_tokens"] = reasoning_budget; - llama_params["reasoning_budget_start_tag"] = chat_params.thinking_start_tag; - llama_params["reasoning_budget_end_tags"] = chat_params.thinking_end_tags; + llama_params["reasoning_budget_start_tag"] = out_session.thinking_start_tag(); + llama_params["reasoning_budget_end_tags"] = out_session.thinking_end_tags(); llama_params["reasoning_budget_message"] = json_value(body, "reasoning_budget_message", opt.reasoning_budget_message); llama_params["reasoning_control"] = json_value(body, "reasoning_control", false); } @@ -1491,6 +1472,15 @@ json oaicompat_chat_params_parse( } } + // the session owns these, the server applies them with server_task::apply_chat_session() + for (const char * key : { "grammar_lazy", "grammar_triggers", "preserved_tokens" }) { + llama_params.erase(key); + } + if (!out_session.grammar().empty()) { + llama_params.erase("grammar"); + llama_params.erase("json_schema"); + } + return llama_params; } diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 6165d871c..5bd2ed57a 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -355,9 +355,11 @@ json oaicompat_completion_params_parse(const json & body); // used by /chat/completions endpoint json oaicompat_chat_params_parse( + const llama_vocab * vocab, json & body, /* openai api json semantics */ const server_chat_params & opt, - std::vector & out_files); + std::vector & out_files, + common_chat_session & out_session); // used by /embeddings endpoint, content has the same format as a chat message content array server_tokens tokenize_oai_content_array( diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 1e3021d4a..9a9c0c90d 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -4767,7 +4767,8 @@ std::unique_ptr server_routes::handle_completions_impl( server_task_type type, const json & data, const std::vector & files, - task_response_type res_type) { + task_response_type res_type, + const common_chat_session & chat_session) { GGML_ASSERT(type == SERVER_TASK_TYPE_COMPLETION || type == SERVER_TASK_TYPE_INFILL); auto res = create_response(); @@ -4809,11 +4810,6 @@ std::unique_ptr server_routes::handle_completions_impl( // tasks.reserve(inputs.size()); // TODO: this is inaccurate due to child tasks - // message delimiters for checkpointing - json delims = json_value(data, "message_delimiters", json::array()); - auto delimiters = common_chat_msg_delimiters_parse(delims); - delimiters.tokenize(ctx_server.vocab); - for (size_t i = 0; i < inputs.size(); i++) { server_task task = server_task(type); @@ -4826,7 +4822,7 @@ std::unique_ptr server_routes::handle_completions_impl( meta->logit_bias_eog, data); - task.params.message_spans = task.tokens.find_message_spans(delimiters); + task.apply_chat_session(chat_session); task.id_slot = json_value(data, "id_slot", -1); sse_ping_interval = task.params.sse_ping_interval; @@ -4847,7 +4843,7 @@ std::unique_ptr server_routes::handle_completions_impl( tasks.push_back(std::move(task)); } - rd.post_tasks(std::move(tasks)); + rd.post_tasks(std::move(tasks), chat_session); } catch (const std::exception & e) { res->error(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST)); return res; @@ -5444,16 +5440,20 @@ void server_routes::init_routes() { auto res = create_response(); std::vector files; json body = json::parse(req.body); + common_chat_session session; json body_parsed = oaicompat_chat_params_parse( + ctx_server.vocab, body, meta->chat_params, - files); + files, + session); return handle_completions_impl( req, SERVER_TASK_TYPE_COMPLETION, body_parsed, files, - TASK_RESPONSE_TYPE_OAI_CHAT); + TASK_RESPONSE_TYPE_OAI_CHAT, + session); }; this->post_chat_completions_tok = [this](const server_http_req & req) { @@ -5503,16 +5503,20 @@ void server_routes::init_routes() { json body = server_chat_convert_responses_to_chatcmpl(json::parse(req.body)); SRV_DBG("%s\n", "Request converted: OpenAI Responses -> OpenAI Chat Completions"); SRV_DBG("converted request: %s\n", body.dump().c_str()); + common_chat_session session; json body_parsed = oaicompat_chat_params_parse( + ctx_server.vocab, body, meta->chat_params, - files); + files, + session); return handle_completions_impl( req, SERVER_TASK_TYPE_COMPLETION, body_parsed, files, - TASK_RESPONSE_TYPE_OAI_RESP); + TASK_RESPONSE_TYPE_OAI_RESP, + session); }; this->post_responses_tok_oai = [this](const server_http_req & req) { @@ -5535,16 +5539,20 @@ void server_routes::init_routes() { files); SRV_DBG("%s\n", "Request converted: OpenAI Transcriptions -> OpenAI Chat Completions"); SRV_DBG("converted request: %s\n", body.dump().c_str()); + common_chat_session session; json body_parsed = oaicompat_chat_params_parse( + ctx_server.vocab, body, meta->chat_params, - files); + files, + session); return handle_completions_impl( req, SERVER_TASK_TYPE_COMPLETION, body_parsed, files, - TASK_RESPONSE_TYPE_OAI_ASR); + TASK_RESPONSE_TYPE_OAI_ASR, + session); }; this->post_anthropic_messages = [this](const server_http_req & req) { @@ -5553,16 +5561,20 @@ void server_routes::init_routes() { json body = server_chat_convert_anthropic_to_oai(json::parse(req.body)); SRV_DBG("%s\n", "Request converted: Anthropic -> OpenAI Chat Completions"); SRV_DBG("converted request: %s\n", body.dump().c_str()); + common_chat_session session; json body_parsed = oaicompat_chat_params_parse( + ctx_server.vocab, body, meta->chat_params, - files); + files, + session); return handle_completions_impl( req, SERVER_TASK_TYPE_COMPLETION, body_parsed, files, - TASK_RESPONSE_TYPE_ANTHROPIC); + TASK_RESPONSE_TYPE_ANTHROPIC, + session); }; this->post_anthropic_count_tokens = [this](const server_http_req & req) { @@ -5573,11 +5585,14 @@ void server_routes::init_routes() { this->post_apply_template = [this](const server_http_req & req) { auto res = create_response(); std::vector files; // dummy, unused + common_chat_session session; // dummy, unused json body = json::parse(req.body); json data = oaicompat_chat_params_parse( + ctx_server.vocab, body, meta->chat_params, - files); + files, + session); res->ok({{ "prompt", std::move(data.at("prompt")) }}); return res; }; @@ -6129,10 +6144,13 @@ std::unique_ptr server_routes::handle_count_tokens(const s return res; } + common_chat_session session; // dummy, unused json body_parsed = oaicompat_chat_params_parse( + ctx_server.vocab, body, meta->chat_params, - files); + files, + session); json prompt = body_parsed.at("prompt"); // SRV_DBG("prompt = %s\n", prompt.dump().c_str()); diff --git a/tools/server/server-context.h b/tools/server/server-context.h index 7fe003531..ed23f7d4e 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -166,7 +166,8 @@ private: server_task_type type, const json & data, const std::vector & files, - task_response_type res_type); + task_response_type res_type, + const common_chat_session & chat_session = {}); std::unique_ptr handle_slots_save(const server_http_req & req, int id_slot); std::unique_ptr handle_slots_restore(const server_http_req & req, int id_slot); std::unique_ptr handle_slots_erase(const server_http_req &, int id_slot); diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cpp index 78169e9a5..3352c3cb5 100644 --- a/tools/server/server-queue.cpp +++ b/tools/server/server-queue.cpp @@ -517,23 +517,23 @@ void server_response_reader::post_task(server_task && task, bool front) { GGML_ASSERT(!task.is_parent() && "not supported, use post_tasks() instead"); task.index = 0; id_tasks.insert(task.id); - states.push_back(task.create_state()); + states.push_back(task_result_state()); queue_results.add_waiting_task_id(task.id); queue_tasks.post(std::move(task), front); } -void server_response_reader::post_tasks(std::vector && tasks, bool front) { +void server_response_reader::post_tasks(std::vector && tasks, const common_chat_session & session, bool front) { GGML_ASSERT(id_tasks.empty() && "post_tasks() can only be called once per reader"); id_tasks = server_task::get_list_id(tasks); states.reserve(tasks.size()); size_t index = 0; for (auto & task : tasks) { task.index = index++; - states.push_back(task.create_state()); + states.push_back(task_result_state(session)); // for child tasks for (auto & child_task : task.child_tasks) { child_task.index = index++; - states.push_back(child_task.create_state()); + states.push_back(task_result_state(session)); } } GGML_ASSERT(states.size() == id_tasks.size()); diff --git a/tools/server/server-queue.h b/tools/server/server-queue.h index e17733a74..692255e4f 100644 --- a/tools/server/server-queue.h +++ b/tools/server/server-queue.h @@ -225,7 +225,7 @@ struct server_response_reader { // if front = true, the task will be posted to the front of the queue (high priority) void post_task(server_task && task, bool front = false); - void post_tasks(std::vector && tasks, bool front = false); + void post_tasks(std::vector && tasks, const common_chat_session & session = {}, bool front = false); bool has_next() const; // return nullptr if should_stop() is true before receiving a result diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp index c7d0bc4de..54e126aa0 100644 --- a/tools/server/server-schema.cpp +++ b/tools/server/server-schema.cpp @@ -291,54 +291,23 @@ std::vector> make_llama_cmpl_schema(const common_params & // Chat parser params // - // TODO: change this to string field instead - add((new field_json("chat_format")) - ->set_desc("Chat format used internally by the server") - ->set_handler([&](field_eval_context & ctx, const json & data) { - ctx.params.chat_parser_params.format = static_cast(data.at("chat_format").get()); - SRV_TRC("chat format: %s\n", common_chat_format_name(ctx.params.chat_parser_params.format)); - })); - add((new field_str("reasoning_format")) ->set_desc("Reasoning format for chain-of-thought models") ->set_handler([&](field_eval_context & ctx, const json & data) { - auto reasoning_format = common_reasoning_format_from_name(data.at("reasoning_format").get()); - ctx.params.chat_parser_params.reasoning_format = reasoning_format; - ctx.params.chat_parser_params.reasoning_in_content = ctx.params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY); - })); - - add((new field_str("generation_prompt")) - ->set_desc("Generation prompt appended to the chat template output") - ->set_handler([&](field_eval_context & ctx, const json & data) { - std::string s = data.at("generation_prompt").get(); - ctx.params.sampling.generation_prompt = s; - - if (ctx.vocab == nullptr) { - ctx.params.chat_parser_params.generation_prompt = common_chat_input(s); - return; - } - - ctx.params.chat_parser_params.generation_prompt = common_chat_input_tokenize(ctx.vocab, s); - })); - - add((new field_bool("parse_tool_calls", params.chat_parser_params.parse_tool_calls)) - ->set_desc("Whether to parse tool calls from the generated output")); - - add((new field_str("chat_parser")) - ->set_desc("Chat parser configuration string") - ->set_handler([&](field_eval_context & ctx, const json & data) { - ctx.params.chat_parser_params.parser.load(data.at("chat_parser").get()); + ctx.params.reasoning_format = common_reasoning_format_from_name(data.at("reasoning_format").get()); })); add((new field_json("continue_final_message")) ->set_desc("Whether to continue the final message of the chat template") - ->set_handler([&](field_eval_context & ctx, const json & data) { - auto continuation = common_chat_continuation_parse(data.at("continue_final_message")); - ctx.params.chat_parser_params.is_continuation = continuation != COMMON_CHAT_CONTINUATION_NONE; + ->set_handler([&](field_eval_context &, const json & data) { + common_chat_continuation_parse(data.at("continue_final_message")); })); - add((new field_bool("echo", params.chat_parser_params.echo)) - ->set_desc("Whether to echo the input tokens in the output")); + add((new field_json("echo")) + ->set_desc("Whether to include the continued assistant message in the output") + ->set_handler([&](field_eval_context &, const json & data) { + data.at("echo").get(); + })); // // Token-level fields (require vocab) @@ -347,44 +316,17 @@ std::vector> make_llama_cmpl_schema(const common_params & add((new field_json("preserved_tokens")) ->set_desc("List of token strings that must not be split during tokenization") ->set_handler([&](field_eval_context & ctx, const json & data) { - GGML_ASSERT(ctx.vocab != nullptr); - for (const auto & t : data.at("preserved_tokens")) { - auto ids = common_tokenize(ctx.vocab, t.get(), false, true); - if (ids.size() == 1) { - ctx.params.sampling.preserved_tokens.insert(ids[0]); - } - } + common_sampling_add_preserved_tokens(ctx.params.sampling, ctx.vocab, data.at("preserved_tokens").get>()); })); add((new field_json("grammar_triggers")) ->set_desc("List of strings or patterns that trigger grammar-constrained generation") ->set_handler([&](field_eval_context & ctx, const json & data) { - GGML_ASSERT(ctx.vocab != nullptr); + std::vector triggers; for (const auto & t : data.at("grammar_triggers")) { - server_grammar_trigger ct(t); - if (ct.value.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) { - const auto & word = ct.value.value; - auto ids = common_tokenize(ctx.vocab, word, false, true); - if (ids.size() == 1) { - auto token = ids[0]; - if (std::find(ctx.params.sampling.preserved_tokens.begin(), ctx.params.sampling.preserved_tokens.end(), (llama_token) token) == ctx.params.sampling.preserved_tokens.end()) { - throw std::runtime_error("Grammar trigger word should be marked as preserved token: " + word); - } - common_grammar_trigger trigger; - trigger.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN; - trigger.value = word; - trigger.token = token; - ctx.params.sampling.grammar_triggers.push_back(std::move(trigger)); - } else { - ctx.params.sampling.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word}); - } - } else { - ctx.params.sampling.grammar_triggers.emplace_back(std::move(ct.value)); - } - } - if (ctx.params.sampling.grammar_lazy && ctx.params.sampling.grammar_triggers.empty()) { - throw std::runtime_error("Error: no triggers set for lazy grammar!"); + triggers.push_back(server_grammar_trigger(t).value); } + common_sampling_add_grammar_triggers(ctx.params.sampling, ctx.vocab, std::move(triggers)); })); add((new field_bool("reasoning_control", params.sampling.reasoning_control)) @@ -542,7 +484,7 @@ task_params eval_llama_cmpl_schema( // enabling this will output extra debug information in the HTTP responses from the server params.verbose = params_base.verbosity > 9; - params.chat_parser_params.reasoning_format = params_base.reasoning_format; + params.reasoning_format = params_base.reasoning_format; // create context and schema field_eval_context ctx(params); @@ -556,13 +498,6 @@ task_params eval_llama_cmpl_schema( f->eval(ctx, data); } - // post-processing - { - // if "reasoning_format" is not provided, its handler will not be called, we will need to handle it here - auto reasoning_format = params.chat_parser_params.reasoning_format; - params.chat_parser_params.reasoning_in_content = params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY); - } - // debugging { auto budget = params.sampling.reasoning_budget_tokens; diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 21c1fe4e1..3f30da2cd 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -73,10 +73,10 @@ json task_params::to_json(bool only_metrics) const { {"stream", stream}, {"n_probs", sampling.n_probs}, {"min_keep", sampling.min_keep}, - {"chat_format", common_chat_format_name(chat_parser_params.format)}, - {"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)}, - {"reasoning_in_content", chat_parser_params.reasoning_in_content}, - {"generation_prompt", chat_parser_params.generation_prompt.text}, + {"chat_format", common_chat_format_name(chat_format)}, + {"reasoning_format", common_reasoning_format_name(reasoning_format)}, + {"reasoning_in_content", stream && reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY}, + {"generation_prompt", sampling.generation_prompt}, {"samplers", samplers}, {"speculative.types", common_speculative_type_name_str(speculative.types)}, {"timings_per_token", timings_per_token}, @@ -132,10 +132,10 @@ json task_params::to_json(bool only_metrics) const { {"grammar_lazy", sampling.grammar_lazy}, {"grammar_triggers", grammar_triggers}, {"preserved_tokens", sampling.preserved_tokens}, - {"chat_format", common_chat_format_name(chat_parser_params.format)}, - {"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)}, - {"reasoning_in_content", chat_parser_params.reasoning_in_content}, - {"generation_prompt", chat_parser_params.generation_prompt.text}, + {"chat_format", common_chat_format_name(chat_format)}, + {"reasoning_format", common_reasoning_format_name(reasoning_format)}, + {"reasoning_in_content", stream && reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY}, + {"generation_prompt", sampling.generation_prompt}, {"samplers", samplers}, {"speculative.types", common_speculative_type_name_str(speculative.types)}, {"timings_per_token", timings_per_token}, @@ -148,15 +148,12 @@ json task_params::to_json(bool only_metrics) const { // // task_result_state // -task_result_state::task_result_state(const common_chat_parser_params & chat_parser_params) - : chat_parser_params(chat_parser_params) +task_result_state::task_result_state(common_chat_session session) + : chat_session(std::move(session)) + , chat_msg(chat_session.msg()) , oai_resp_id("resp_" + random_string()) , oai_resp_reasoning_id("rs_" + random_string()) , oai_resp_message_id("msg_" + random_string()) { - if (chat_parser_params.is_continuation && !chat_parser_params.echo) { - // initialize chat_msg to avoid emitting a delta containing the assistant prefill - chat_msg = common_chat_parse(generated_input, true, chat_parser_params); - } } common_chat_msg task_result_state::update_chat_msg( @@ -164,13 +161,8 @@ common_chat_msg task_result_state::update_chat_msg( bool is_partial, std::vector & diffs, bool filter_tool_calls) { - generated_input.append(added); auto msg_prv_copy = chat_msg; - //SRV_DBG("Parsing chat message: %s\n", generated_input.text.c_str()); - auto new_msg = common_chat_parse( - generated_input, - is_partial, - chat_parser_params); + auto new_msg = is_partial ? chat_session.feed(added) : chat_session.finish(added); if (!new_msg.empty()) { new_msg.set_tool_call_ids(generated_tool_call_ids, gen_tool_call_id); chat_msg = new_msg; diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 0cbc5f353..b8d6a3b67 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -89,8 +89,9 @@ struct task_params { std::string control_action; std::string control_cmpl_id; - // per-request parameters for chat parsing - common_chat_parser_params chat_parser_params; + // reported in generation_settings, parsing itself is owned by the chat session + common_chat_format chat_format = COMMON_CHAT_FORMAT_CONTENT_ONLY; + common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // message spans for checkpointing common_chat_msg_spans message_spans; @@ -106,9 +107,8 @@ struct task_params { struct task_result_state { // tracking diffs for partial tool calls std::vector diffs; - common_chat_parser_params chat_parser_params; + common_chat_session chat_session; // owns all parsing for this generation common_chat_msg chat_msg; - common_chat_input generated_input; // append new chunks of generated text here std::vector generated_tool_call_ids; std::unordered_set sent_tool_call_names; @@ -124,7 +124,7 @@ struct task_result_state { const std::string oai_resp_message_id; std::string oai_resp_fc_id; // function call ID for current args delta - task_result_state(const common_chat_parser_params & chat_parser_params); + task_result_state(common_chat_session session = {}); // parse partial tool calls and update the internal state common_chat_msg update_chat_msg( @@ -258,6 +258,17 @@ struct server_task { return ids; } + void apply_chat_session(const common_chat_session & session) { + if (!session.has_template()) { + return; + } + + session.apply_sampling(params.sampling); + params.chat_format = session.format(); + params.antiprompt.insert(params.antiprompt.end(), session.additional_stops().begin(), session.additional_stops().end()); + params.message_spans = tokens.find_message_spans(session.message_delimiters()); + } + void add_child(int id_parent, int id_child) { server_task copy; @@ -277,12 +288,6 @@ struct server_task { child_tasks.push_back(std::move(copy)); } - // the task will be moved into queue, then onto slots - // however, the state must be kept by caller (e.g., HTTP thread) - task_result_state create_state() const { - return task_result_state(params.chat_parser_params); - } - bool is_parent() const { return child_tasks.size() > 0; }