common : support deferred tools and tool_reference content parts (#31256)

* common : support deferred tools and tool_reference content parts

Assisted-by: Claude Opus 5.5

* common : credit the deferred tools co-author

Co-authored-by: Aldehir Rojas <765227+aldehir@users.noreply.github.com>

Assisted-by: Claude Opus 5.5

* tests : fix the type check of the deferred tools test

Assisted-by: Claude Opus 5.5

common/chat.cpp

@@ -190,10 +190,17 @@ json common_chat_msg::to_json_oaicompat(bool concat_typed_text) const {
         } else {
             auto & parts = jmsg["content"] = json::array();
             for (const auto & part : content_parts) {
-                parts.push_back({
-                    {"type", part.type},
-                    {"text", part.text},
-                });
+                if (part.type == "tool_reference") {
+                    parts.push_back({
+                        {"type", part.type},
+                        {"name", part.name},
+                    });
+                } else {
+                    parts.push_back({
+                        {"type", part.type},
+                        {"text", part.text},
+                    });
+                }
             }
         }
     } else {
@@ -370,12 +377,16 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
                             throw std::invalid_argument("Missing content part type: " + part.dump());
                         }
                         const auto & type = part.at("type");
-                        if (type != "text" && type != "media_marker") {
-                            throw std::invalid_argument("Unsupported content part type: " + type.dump());
-                        }
                         common_chat_msg_content_part msg_part;
                         msg_part.type = type;
-                        msg_part.text = part.at("text");
+                        if (type == "text" || type == "media_marker") {
+                            msg_part.text = part.at("text");
+                        } else if (type == "tool_reference") {
+                            // names a request tool, the template renders its definition
+                            msg_part.name = part.at("name");
+                        } else {
+                            throw std::invalid_argument("Unsupported content part type: " + type.dump());
+                        }
                         msg.content_parts.push_back(msg_part);
                     }
                 } else if (!content.is_null()) {
@@ -610,13 +621,17 @@ json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & t
 
     auto result = json::array();
     for (const auto & tool : tools) {
+        json function {
+            { "name",        tool.name },
+            { "description", tool.description },
+            { "parameters",  json::parse(tool.parameters) },
+        };
+        if (tool.defer_loading) {
+            function["defer_loading"] = true;
+        }
         result.push_back({
             { "type",     "function" },
-            { "function", {
-                { "name", tool.name },
-                { "description", tool.description },
-                { "parameters", json::parse(tool.parameters) },
-            }},
+            { "function", function },
         });
     }
     return result;
@@ -653,10 +668,19 @@ std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & too
                 }
 
                 const auto & function = tool.at("function");
+
+                // "defer_loading" sits on the function, or on the tool next to "type"; the function wins
+                const json & defer = function.contains("defer_loading") ? function.at("defer_loading")
+                                                                        : tool.value("defer_loading", json(false));
+                if (!defer.is_boolean()) {
+                    throw std::invalid_argument("Expected 'defer_loading' to be a boolean, got " + defer.dump());
+                }
+
                 result.push_back({
                     /* .name = */ function.at("name"),
                     /* .description = */ function.value("description", ""),
                     /* .parameters = */ function.value("parameters", json::object()).dump(),
+                    /* .defer_loading = */ defer.get<bool>(),
                 });
             }
         }

common/chat.h

@@ -45,6 +45,7 @@ struct common_chat_tool_call {
 struct common_chat_msg_content_part {
     std::string type;
     std::string text;
+    std::string name; // tool_reference: name of a tool from the request tools
 
     // TODO @ngxson : no known chat templates support reasoning_content in content parts yet
     //                this can be useful for models with interleaved thinking (like Kimi-K2)
@@ -52,7 +53,7 @@ struct common_chat_msg_content_part {
     // std::string reasoning_content;
 
     bool operator==(const common_chat_msg_content_part & other) const {
-        return type == other.type && text == other.text;
+        return type == other.type && text == other.text && name == other.name;
     }
 };
 
@@ -226,6 +227,9 @@ struct common_chat_tool {
     std::string name;
     std::string description;
     std::string parameters;
+
+    // withheld from the system prompt, a tool_reference reveals it later in the conversation
+    bool defer_loading = false;
 };
 
 enum common_chat_tool_choice {

common/jinja/caps.cpp

@@ -184,6 +184,10 @@ caps caps_get(jinja::program & prog) {
                     {
                         {"role", "user"},
                         {"content", json::array({
+                            {
+                                {"type", "text"},
+                                {"text", content_marker}
+                            }
                         })}
                     }
                 });

tests/test-chat.cpp

@@ -986,8 +986,8 @@ const common_chat_msg message_user_parts{
     /* .content = */ "",
     /* .content_parts = */
     {
-     { "text", "Hey" },
-     { "text", "there" },
+     { "text", "Hey", "" },
+     { "text", "there", "" },
      },
     /* .tool_calls = */
     {                 },
@@ -3302,9 +3302,9 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
                 common_chat_msg message_parts;
                 message_parts.role          = "user";
                 message_parts.content_parts = {
-                    { /* .type = */ "text", /* .text = */ "First part.\n\n" },
-                    { /* .type = */ "media_marker", /* .text = */ "<__media__>" },
-                    { /* .type = */ "text", /* .text = */ "Second part.\n\n" },
+                    { /* .type = */ "text", /* .text = */ "First part.\n\n", /* .name = */ "" },
+                    { /* .type = */ "media_marker", /* .text = */ "<__media__>", /* .name = */ "" },
+                    { /* .type = */ "text", /* .text = */ "Second part.\n\n", /* .name = */ "" },
                 };
 
                 common_chat_templates_inputs inputs;
@@ -5628,6 +5628,44 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
             .tools({ special_function_tool })
             .expect(message_assist_call_thoughts)
             .run();
+
+        // a deferred tool is left out of <tools>, a tool_reference renders it, and its schema types the call
+        const auto deferred_tools = common_chat_tools_parse_oaicompat(json::parse(R"([
+            {"type": "function", "function": {"name": "tool_search", "description": "Search for tools",
+             "parameters": {"type": "object", "properties": {"query": {"type": "string"}}}}},
+            {"type": "function", "function": {"name": "buy", "description": "Buy an item", "defer_loading": true,
+             "parameters": {"type": "object", "properties": {"id": {"type": "string"}, "qty": {"type": "integer"}}}}}
+        ])"));
+        {
+            auto tmpls = read_templates("models/templates/LFM3.jinja");
+            common_chat_templates_inputs inputs;
+            inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(R"([
+                {"role": "user", "content": "Buy the blue one"},
+                {"role": "assistant", "tool_calls": [{"id": "c0", "type": "function",
+                 "function": {"name": "tool_search", "arguments": "{\"query\": \"buy\"}"}}]},
+                {"role": "tool", "tool_call_id": "c0", "name": "tool_search", "content": [{"type": "tool_reference", "name": "buy"}]}
+            ])"));
+            inputs.tools                 = deferred_tools;
+            inputs.add_generation_prompt = true;
+            const auto prompt = common_chat_templates_apply(tmpls.get(), inputs).prompt.str();
+            assert_contains(prompt, "<tool_response>\n<additional_tools>\n{\"name\": \"buy\"");
+            assert_equals(true, prompt.find("\"name\": \"buy\"") > prompt.find("</tools>"));
+            assert_not_contains(prompt, "defer_loading");
+            assert_not_contains(prompt, "tool_reference");
+        }
+
+        tst.test(
+               "<tool_call>\n"
+               "<function=buy>\n"
+               "<parameter=id>\nblue\n</parameter>\n"
+               "<parameter=qty>\n2\n</parameter>\n"
+               "</function>\n"
+               "</tool_call>")
+            .enable_thinking(false)
+            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+            .tools(deferred_tools)
+            .expect_tool_calls({ { "buy", R"({"id": "blue", "qty": 2})", {} } })
+            .run();
     }
 
     // Thinking cases only apply to LFM2.5-8B-A1B, the one LFM2 template that emits <think>

tests/test-jinja.cpp

@@ -2550,6 +2550,23 @@ static void test_caps(testing & t) {
         t.assert_true("does not support string content", !caps.supports_string_content);
         t.assert_true("supports typed content", caps.supports_typed_content);
     });
+
+    t.test("string and typed content, behind a truthiness check", [](testing & t) {
+        // an empty content array is falsy, the parts are only read when there are some (LFM3 case)
+        auto caps = get_caps(
+            "{% for message in messages %}"
+            "{% if message['content'] %}"
+            "{% if message['content'] is string %}"
+            "{{ message['content'] }}"
+            "{% else %}"
+            "{% for part in message['content'] %}{{ part['text'] }}{% endfor %}"
+            "{% endif %}"
+            "{% endif %}"
+            "{% endfor %}"
+        );
+        t.assert_true("supports string content", caps.supports_string_content);
+        t.assert_true("supports typed content", caps.supports_typed_content);
+    });
 }
 
 static void test_string_parts(testing & t) {

tools/server/server-chat.cpp

@@ -464,7 +464,7 @@ json server_chat_convert_anthropic_to_oai(const json & body) {
                         // Single-pass: build both text and content_parts, decide format at the end
                         std::string result_text;
                         json content_parts = json::array();
-                        bool has_images = false;
+                        bool keep_parts = false; // images or tool references
 
                         for (const auto & c : result_content) {
                             std::string c_type = json_value(c, "type", std::string());
@@ -476,7 +476,7 @@ json server_chat_convert_anthropic_to_oai(const json & body) {
                                     {"text", text}
                                 });
                             } else if (c_type == "image") {
-                                has_images = true;
+                                keep_parts = true;
                                 json source = json_value(c, "source", json::object());
                                 std::string source_type = json_value(source, "type", std::string());
                                 if (source_type == "base64") {
@@ -493,10 +493,17 @@ json server_chat_convert_anthropic_to_oai(const json & body) {
                                         {"image_url", {{"url", json_value(source, "url", std::string())}}}
                                     });
                                 }
+                            } else if (c_type == "tool_reference") {
+                                // tool search result naming a deferred tool
+                                keep_parts = true;
+                                content_parts.push_back({
+                                    {"type", "tool_reference"},
+                                    {"name", json_value(c, "tool_name", std::string())}
+                                });
                             }
                         }
 
-                        if (!has_images) {
+                        if (!keep_parts) {
                             // Text-only: collapse to a plain string for maximum compatibility
                             tool_results.push_back({
                                 {"role", "tool"},
@@ -504,7 +511,7 @@ json server_chat_convert_anthropic_to_oai(const json & body) {
                                 {"content", result_text}
                             });
                         } else {
-                            // Mixed or image-only: use array content parts (OpenAI multimodal tool format)
+                            // images or tool references: keep array content parts
                             tool_results.push_back({
                                 {"role", "tool"},
                                 {"tool_call_id", tool_use_id},
@@ -559,6 +566,9 @@ json server_chat_convert_anthropic_to_oai(const json & body) {
                         {"parameters", tool.contains("input_schema") ? tool.at("input_schema") : json::object()}
                     }}
                 });
+                if (tool.contains("defer_loading")) {
+                    oai_tools.back()["defer_loading"] = tool.at("defer_loading");
+                }
             }
             oai_body["tools"] = oai_tools;
         }

tools/server/server-common.cpp

@@ -1316,7 +1316,7 @@ static void oaicompat_content_load_media(json & content, const server_chat_param
             p.erase("input_video");
             p.erase("video_url");
 
-        } else if (type != "text") {
+        } else if (type != "text" && type != "tool_reference") {
             throw std::invalid_argument("unsupported content[].type");
         }
     }

tools/server/tests/unit/test_compat_anthropic.py

@@ -498,6 +498,54 @@ def test_anthropic_tool_result_error():
     assert res.body["type"] == "message"
 
 
+def test_anthropic_tool_reference():
+    """Anthropic defer_loading and tool_reference convert to their OpenAI forms (behaviour: test_tool_call.py)"""
+    server.jinja = True
+    server.chat_template_file = '../../../models/templates/LFM3.jinja'
+    server.start()
+
+    search = {"name": "tool_search", "description": "Search for available tools by keyword",
+              "parameters": {"type": "object", "properties": {"query": {"type": "string"}}}}
+    purchase = {"name": "markItemPurchased", "description": "Mark an item as purchased",
+                "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}}
+
+    anthropic = server.make_request("POST", "/v1/messages/count_tokens", data={
+        "model": "test",
+        "tools": [
+            {"name": t["name"], "description": t["description"], "input_schema": t["parameters"], **extra}
+            for t, extra in ((search, {}), (purchase, {"defer_loading": True}))
+        ],
+        "messages": [
+            {"role": "user", "content": "Mark item abc as purchased"},
+            {"role": "assistant", "content": [
+                {"type": "tool_use", "id": "call_1", "name": "tool_search", "input": {"query": "purchase"}},
+            ]},
+            {"role": "user", "content": [
+                {"type": "tool_result", "tool_use_id": "call_1",
+                 "content": [{"type": "tool_reference", "tool_name": "markItemPurchased"}]},
+            ]},
+        ],
+    })
+    oai = server.make_request("POST", "/chat/completions/input_tokens", data={
+        "tools": [
+            {"type": "function", "function": search},
+            {"type": "function", "function": {**purchase, "defer_loading": True}},
+        ],
+        "messages": [
+            {"role": "user", "content": "Mark item abc as purchased"},
+            {"role": "assistant", "content": None, "tool_calls": [
+                {"id": "call_1", "type": "function",
+                 "function": {"name": "tool_search", "arguments": "{\"query\": \"purchase\"}"}},
+            ]},
+            {"role": "tool", "tool_call_id": "call_1",
+             "content": [{"type": "tool_reference", "name": "markItemPurchased"}]},
+        ],
+    })
+    assert anthropic.status_code == 200, anthropic.body
+    assert oai.status_code == 200, oai.body
+    assert anthropic.body["input_tokens"] == oai.body["input_tokens"]
+
+
 def test_anthropic_tool_streaming():
     """Test streaming with tool use"""
     server.jinja = True

tools/server/tests/unit/test_tool_call.py

@@ -642,3 +642,54 @@ def do_test_hello_world(server: ServerProcess, **kwargs):
     code = actual_arguments["code"]
     assert isinstance(code, str), f"Expected code to be a string, got {type(code)}: {json.dumps(code)}"
     assert re.match(r'''print\(("[Hh]ello,? [Ww]orld!?"|'[Hh]ello,? [Ww]orld!?')\)''', re.sub(r'#.*\n?', '', code)), f'Expected hello world, got {code}'
+
+
+SEARCH_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "tool_search",
+        "description": "Search for available tools by keyword",
+        "parameters": {"type": "object", "properties": {"query": {"type": "string"}}},
+    },
+}
+PURCHASE_FUNCTION = {
+    "name": "markItemPurchased",
+    "description": "Mark an item as purchased",
+    "parameters": {"type": "object", "properties": {"id": {"type": "string"}, "qty": {"type": "integer"}}},
+}
+
+PURCHASE_TOOL = {"type": "function", "function": PURCHASE_FUNCTION}
+
+
+def count_deferred_tokens(tools, tool_content):
+    res = server.make_request("POST", "/chat/completions/input_tokens", data={
+        "tools": tools,
+        "messages": [
+            {"role": "user", "content": "Mark item abc as purchased"},
+            {"role": "assistant", "content": None, "tool_calls": [
+                {"id": "call_1", "type": "function",
+                 "function": {"name": "tool_search", "arguments": "{\"query\": \"purchase\"}"}},
+            ]},
+            {"role": "tool", "tool_call_id": "call_1", "content": tool_content},
+        ],
+    })
+    assert res.status_code == 200, res.body
+    return res.body["input_tokens"]
+
+
+def test_deferred_tool_reference():
+    """A defer_loading tool is withheld from the system prompt, a tool_reference reveals its definition"""
+    server.jinja = True
+    server.chat_template_file = '../../../models/templates/LFM3.jinja'
+    server.start()
+
+    deferred = {"type": "function", "function": {**PURCHASE_FUNCTION, "defer_loading": True}}
+    tools = [SEARCH_TOOL, deferred]
+
+    # withheld from the system prompt
+    assert count_deferred_tokens(tools, "none") == count_deferred_tokens([SEARCH_TOOL], "none")
+
+    # the reference renders like the definition sent inline
+    by_reference = count_deferred_tokens(tools, [{"type": "tool_reference", "name": "markItemPurchased"}])
+    inline = count_deferred_tokens(tools, "<additional_tools>\n" + json.dumps(PURCHASE_FUNCTION) + "\n</additional_tools>")
+    assert by_reference == inline