diff --git a/tools/server/README.md b/tools/server/README.md index 6188a084b..66a34ff84 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1654,6 +1654,8 @@ See [examples/realtime_client.py](../../examples/realtime_client.py) for a clien See [OpenAI Responses API documentation](https://platform.openai.com/docs/api-reference/responses). +`extra_params`: llama.cpp parameters that the Responses API has no field for, such as `seed`, `min_p` or `repeat_penalty`, as an object (e.g. `{"seed": 42, "min_p": 0.05}`). Its keys are merged into the converted request and behave as on `/v1/chat/completions`. + *Examples:* You can use either Python `openai` library with appropriate checkpoints: @@ -1684,7 +1686,8 @@ curl http://localhost:9931/v1/responses \ -d '{ "model": "gpt-4.1", "instructions": "You are ChatGPT, an AI assistant. Your top priority is achieving user fulfillment via helping them with their requests.", -"input": "Write a limerick about python exceptions" +"input": "Write a limerick about python exceptions", +"extra_params": {"seed": 42, "min_p": 0.05} }' ``` diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index aa4947182..a7431800b 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -311,6 +311,16 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) { chatcmpl_body.erase("reasoning"); } + // llama.cpp params (e.g. seed, min_p) go under extra_params and are merged into the converted body + if (response_body.contains("extra_params")) { + chatcmpl_body.erase("extra_params"); + if (response_body.at("extra_params").is_object()) { + for (const auto & [key, value] : response_body.at("extra_params").items()) { + chatcmpl_body[key] = value; + } + } + } + return chatcmpl_body; }