diff --git a/common/chat.cpp b/common/chat.cpp index 227621761..46b98bdbe 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -190,10 +190,17 @@ json common_chat_msg::to_json_oaicompat(bool concat_typed_text) const { } else { auto & parts = jmsg["content"] = json::array(); for (const auto & part : content_parts) { - parts.push_back({ - {"type", part.type}, - {"text", part.text}, - }); + if (part.type == "tool_reference") { + parts.push_back({ + {"type", part.type}, + {"name", part.name}, + }); + } else { + parts.push_back({ + {"type", part.type}, + {"text", part.text}, + }); + } } } } else { @@ -370,12 +377,16 @@ std::vector common_chat_msgs_parse_oaicompat(const json & messa throw std::invalid_argument("Missing content part type: " + part.dump()); } const auto & type = part.at("type"); - if (type != "text" && type != "media_marker") { - throw std::invalid_argument("Unsupported content part type: " + type.dump()); - } common_chat_msg_content_part msg_part; msg_part.type = type; - msg_part.text = part.at("text"); + if (type == "text" || type == "media_marker") { + msg_part.text = part.at("text"); + } else if (type == "tool_reference") { + // names a request tool, the template renders its definition + msg_part.name = part.at("name"); + } else { + throw std::invalid_argument("Unsupported content part type: " + type.dump()); + } msg.content_parts.push_back(msg_part); } } else if (!content.is_null()) { @@ -610,13 +621,17 @@ json common_chat_tools_to_json_oaicompat(const std::vector & t auto result = json::array(); for (const auto & tool : tools) { + json function { + { "name", tool.name }, + { "description", tool.description }, + { "parameters", json::parse(tool.parameters) }, + }; + if (tool.defer_loading) { + function["defer_loading"] = true; + } result.push_back({ { "type", "function" }, - { "function", { - { "name", tool.name }, - { "description", tool.description }, - { "parameters", json::parse(tool.parameters) }, - }}, + { "function", function }, }); } return result; @@ -653,10 +668,19 @@ std::vector common_chat_tools_parse_oaicompat(const json & too } const auto & function = tool.at("function"); + + // "defer_loading" sits on the function, or on the tool next to "type"; the function wins + const json & defer = function.contains("defer_loading") ? function.at("defer_loading") + : tool.value("defer_loading", json(false)); + if (!defer.is_boolean()) { + throw std::invalid_argument("Expected 'defer_loading' to be a boolean, got " + defer.dump()); + } + result.push_back({ /* .name = */ function.at("name"), /* .description = */ function.value("description", ""), /* .parameters = */ function.value("parameters", json::object()).dump(), + /* .defer_loading = */ defer.get(), }); } } diff --git a/common/chat.h b/common/chat.h index 29aaf8b68..647e56cc4 100644 --- a/common/chat.h +++ b/common/chat.h @@ -45,6 +45,7 @@ struct common_chat_tool_call { struct common_chat_msg_content_part { std::string type; std::string text; + std::string name; // tool_reference: name of a tool from the request tools // TODO @ngxson : no known chat templates support reasoning_content in content parts yet // this can be useful for models with interleaved thinking (like Kimi-K2) @@ -52,7 +53,7 @@ struct common_chat_msg_content_part { // std::string reasoning_content; bool operator==(const common_chat_msg_content_part & other) const { - return type == other.type && text == other.text; + return type == other.type && text == other.text && name == other.name; } }; @@ -226,6 +227,9 @@ struct common_chat_tool { std::string name; std::string description; std::string parameters; + + // withheld from the system prompt, a tool_reference reveals it later in the conversation + bool defer_loading = false; }; enum common_chat_tool_choice { diff --git a/common/jinja/caps.cpp b/common/jinja/caps.cpp index 6ff17d10b..ac295abd3 100644 --- a/common/jinja/caps.cpp +++ b/common/jinja/caps.cpp @@ -184,6 +184,10 @@ caps caps_get(jinja::program & prog) { { {"role", "user"}, {"content", json::array({ + { + {"type", "text"}, + {"text", content_marker} + } })} } }); diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 9340c7d7d..41a8d4cfe 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -986,8 +986,8 @@ const common_chat_msg message_user_parts{ /* .content = */ "", /* .content_parts = */ { - { "text", "Hey" }, - { "text", "there" }, + { "text", "Hey", "" }, + { "text", "there", "" }, }, /* .tool_calls = */ { }, @@ -3302,9 +3302,9 @@ static void test_template_output_peg_parsers(bool detailed_debug) { common_chat_msg message_parts; message_parts.role = "user"; message_parts.content_parts = { - { /* .type = */ "text", /* .text = */ "First part.\n\n" }, - { /* .type = */ "media_marker", /* .text = */ "<__media__>" }, - { /* .type = */ "text", /* .text = */ "Second part.\n\n" }, + { /* .type = */ "text", /* .text = */ "First part.\n\n", /* .name = */ "" }, + { /* .type = */ "media_marker", /* .text = */ "<__media__>", /* .name = */ "" }, + { /* .type = */ "text", /* .text = */ "Second part.\n\n", /* .name = */ "" }, }; common_chat_templates_inputs inputs; @@ -5628,6 +5628,44 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .tools({ special_function_tool }) .expect(message_assist_call_thoughts) .run(); + + // a deferred tool is left out of , a tool_reference renders it, and its schema types the call + const auto deferred_tools = common_chat_tools_parse_oaicompat(json::parse(R"([ + {"type": "function", "function": {"name": "tool_search", "description": "Search for tools", + "parameters": {"type": "object", "properties": {"query": {"type": "string"}}}}}, + {"type": "function", "function": {"name": "buy", "description": "Buy an item", "defer_loading": true, + "parameters": {"type": "object", "properties": {"id": {"type": "string"}, "qty": {"type": "integer"}}}}} + ])")); + { + auto tmpls = read_templates("models/templates/LFM3.jinja"); + common_chat_templates_inputs inputs; + inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(R"([ + {"role": "user", "content": "Buy the blue one"}, + {"role": "assistant", "tool_calls": [{"id": "c0", "type": "function", + "function": {"name": "tool_search", "arguments": "{\"query\": \"buy\"}"}}]}, + {"role": "tool", "tool_call_id": "c0", "name": "tool_search", "content": [{"type": "tool_reference", "name": "buy"}]} + ])")); + inputs.tools = deferred_tools; + inputs.add_generation_prompt = true; + const auto prompt = common_chat_templates_apply(tmpls.get(), inputs).prompt.str(); + assert_contains(prompt, "\n\n{\"name\": \"buy\""); + assert_equals(true, prompt.find("\"name\": \"buy\"") > prompt.find("")); + assert_not_contains(prompt, "defer_loading"); + assert_not_contains(prompt, "tool_reference"); + } + + tst.test( + "\n" + "\n" + "\nblue\n\n" + "\n2\n\n" + "\n" + "") + .enable_thinking(false) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools(deferred_tools) + .expect_tool_calls({ { "buy", R"({"id": "blue", "qty": 2})", {} } }) + .run(); } // Thinking cases only apply to LFM2.5-8B-A1B, the one LFM2 template that emits diff --git a/tests/test-jinja.cpp b/tests/test-jinja.cpp index 2a0117284..444d57fbe 100644 --- a/tests/test-jinja.cpp +++ b/tests/test-jinja.cpp @@ -2550,6 +2550,23 @@ static void test_caps(testing & t) { t.assert_true("does not support string content", !caps.supports_string_content); t.assert_true("supports typed content", caps.supports_typed_content); }); + + t.test("string and typed content, behind a truthiness check", [](testing & t) { + // an empty content array is falsy, the parts are only read when there are some (LFM3 case) + auto caps = get_caps( + "{% for message in messages %}" + "{% if message['content'] %}" + "{% if message['content'] is string %}" + "{{ message['content'] }}" + "{% else %}" + "{% for part in message['content'] %}{{ part['text'] }}{% endfor %}" + "{% endif %}" + "{% endif %}" + "{% endfor %}" + ); + t.assert_true("supports string content", caps.supports_string_content); + t.assert_true("supports typed content", caps.supports_typed_content); + }); } static void test_string_parts(testing & t) { diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index 1c2326f33..b0245786c 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -464,7 +464,7 @@ json server_chat_convert_anthropic_to_oai(const json & body) { // Single-pass: build both text and content_parts, decide format at the end std::string result_text; json content_parts = json::array(); - bool has_images = false; + bool keep_parts = false; // images or tool references for (const auto & c : result_content) { std::string c_type = json_value(c, "type", std::string()); @@ -476,7 +476,7 @@ json server_chat_convert_anthropic_to_oai(const json & body) { {"text", text} }); } else if (c_type == "image") { - has_images = true; + keep_parts = true; json source = json_value(c, "source", json::object()); std::string source_type = json_value(source, "type", std::string()); if (source_type == "base64") { @@ -493,10 +493,17 @@ json server_chat_convert_anthropic_to_oai(const json & body) { {"image_url", {{"url", json_value(source, "url", std::string())}}} }); } + } else if (c_type == "tool_reference") { + // tool search result naming a deferred tool + keep_parts = true; + content_parts.push_back({ + {"type", "tool_reference"}, + {"name", json_value(c, "tool_name", std::string())} + }); } } - if (!has_images) { + if (!keep_parts) { // Text-only: collapse to a plain string for maximum compatibility tool_results.push_back({ {"role", "tool"}, @@ -504,7 +511,7 @@ json server_chat_convert_anthropic_to_oai(const json & body) { {"content", result_text} }); } else { - // Mixed or image-only: use array content parts (OpenAI multimodal tool format) + // images or tool references: keep array content parts tool_results.push_back({ {"role", "tool"}, {"tool_call_id", tool_use_id}, @@ -559,6 +566,9 @@ json server_chat_convert_anthropic_to_oai(const json & body) { {"parameters", tool.contains("input_schema") ? tool.at("input_schema") : json::object()} }} }); + if (tool.contains("defer_loading")) { + oai_tools.back()["defer_loading"] = tool.at("defer_loading"); + } } oai_body["tools"] = oai_tools; } diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index c3550de95..f0ebf513e 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -1316,7 +1316,7 @@ static void oaicompat_content_load_media(json & content, const server_chat_param p.erase("input_video"); p.erase("video_url"); - } else if (type != "text") { + } else if (type != "text" && type != "tool_reference") { throw std::invalid_argument("unsupported content[].type"); } } diff --git a/tools/server/tests/unit/test_compat_anthropic.py b/tools/server/tests/unit/test_compat_anthropic.py index d292f83cf..13d077445 100644 --- a/tools/server/tests/unit/test_compat_anthropic.py +++ b/tools/server/tests/unit/test_compat_anthropic.py @@ -498,6 +498,54 @@ def test_anthropic_tool_result_error(): assert res.body["type"] == "message" +def test_anthropic_tool_reference(): + """Anthropic defer_loading and tool_reference convert to their OpenAI forms (behaviour: test_tool_call.py)""" + server.jinja = True + server.chat_template_file = '../../../models/templates/LFM3.jinja' + server.start() + + search = {"name": "tool_search", "description": "Search for available tools by keyword", + "parameters": {"type": "object", "properties": {"query": {"type": "string"}}}} + purchase = {"name": "markItemPurchased", "description": "Mark an item as purchased", + "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}} + + anthropic = server.make_request("POST", "/v1/messages/count_tokens", data={ + "model": "test", + "tools": [ + {"name": t["name"], "description": t["description"], "input_schema": t["parameters"], **extra} + for t, extra in ((search, {}), (purchase, {"defer_loading": True})) + ], + "messages": [ + {"role": "user", "content": "Mark item abc as purchased"}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "call_1", "name": "tool_search", "input": {"query": "purchase"}}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "call_1", + "content": [{"type": "tool_reference", "tool_name": "markItemPurchased"}]}, + ]}, + ], + }) + oai = server.make_request("POST", "/chat/completions/input_tokens", data={ + "tools": [ + {"type": "function", "function": search}, + {"type": "function", "function": {**purchase, "defer_loading": True}}, + ], + "messages": [ + {"role": "user", "content": "Mark item abc as purchased"}, + {"role": "assistant", "content": None, "tool_calls": [ + {"id": "call_1", "type": "function", + "function": {"name": "tool_search", "arguments": "{\"query\": \"purchase\"}"}}, + ]}, + {"role": "tool", "tool_call_id": "call_1", + "content": [{"type": "tool_reference", "name": "markItemPurchased"}]}, + ], + }) + assert anthropic.status_code == 200, anthropic.body + assert oai.status_code == 200, oai.body + assert anthropic.body["input_tokens"] == oai.body["input_tokens"] + + def test_anthropic_tool_streaming(): """Test streaming with tool use""" server.jinja = True diff --git a/tools/server/tests/unit/test_tool_call.py b/tools/server/tests/unit/test_tool_call.py index 87c4ad166..80a1cae62 100755 --- a/tools/server/tests/unit/test_tool_call.py +++ b/tools/server/tests/unit/test_tool_call.py @@ -642,3 +642,54 @@ def do_test_hello_world(server: ServerProcess, **kwargs): code = actual_arguments["code"] assert isinstance(code, str), f"Expected code to be a string, got {type(code)}: {json.dumps(code)}" assert re.match(r'''print\(("[Hh]ello,? [Ww]orld!?"|'[Hh]ello,? [Ww]orld!?')\)''', re.sub(r'#.*\n?', '', code)), f'Expected hello world, got {code}' + + +SEARCH_TOOL = { + "type": "function", + "function": { + "name": "tool_search", + "description": "Search for available tools by keyword", + "parameters": {"type": "object", "properties": {"query": {"type": "string"}}}, + }, +} +PURCHASE_FUNCTION = { + "name": "markItemPurchased", + "description": "Mark an item as purchased", + "parameters": {"type": "object", "properties": {"id": {"type": "string"}, "qty": {"type": "integer"}}}, +} + +PURCHASE_TOOL = {"type": "function", "function": PURCHASE_FUNCTION} + + +def count_deferred_tokens(tools, tool_content): + res = server.make_request("POST", "/chat/completions/input_tokens", data={ + "tools": tools, + "messages": [ + {"role": "user", "content": "Mark item abc as purchased"}, + {"role": "assistant", "content": None, "tool_calls": [ + {"id": "call_1", "type": "function", + "function": {"name": "tool_search", "arguments": "{\"query\": \"purchase\"}"}}, + ]}, + {"role": "tool", "tool_call_id": "call_1", "content": tool_content}, + ], + }) + assert res.status_code == 200, res.body + return res.body["input_tokens"] + + +def test_deferred_tool_reference(): + """A defer_loading tool is withheld from the system prompt, a tool_reference reveals its definition""" + server.jinja = True + server.chat_template_file = '../../../models/templates/LFM3.jinja' + server.start() + + deferred = {"type": "function", "function": {**PURCHASE_FUNCTION, "defer_loading": True}} + tools = [SEARCH_TOOL, deferred] + + # withheld from the system prompt + assert count_deferred_tokens(tools, "none") == count_deferred_tokens([SEARCH_TOOL], "none") + + # the reference renders like the definition sent inline + by_reference = count_deferred_tokens(tools, [{"type": "tool_reference", "name": "markItemPurchased"}]) + inline = count_deferred_tokens(tools, "\n" + json.dumps(PURCHASE_FUNCTION) + "\n") + assert by_reference == inline