diff --git a/common/tool-call.cpp b/common/tool-call.cpp index cb9ee2ecf4124..ca25b803804fb 100644 --- a/common/tool-call.cpp +++ b/common/tool-call.cpp @@ -39,6 +39,8 @@ static bool parse_json(std::string::const_iterator & it, const std::string::cons std::size_t position; bool found_error; + json_error_locator() : position(0), found_error(false) {} + bool parse_error(std::size_t position, const std::string & last_token, const json::exception & ex) override { // LOG_WARNING("JSON error (Expected)", {{"position", position}, {"last_token", last_token}, {"error", ex.what()}}); this->position = position - 1; @@ -67,7 +69,7 @@ static bool parse_json(std::string::const_iterator & it, const std::string::cons } else { temptative_end = end; } - std::string json_sub {it, it + err_loc.position}; + std::string json_sub {it, temptative_end}; // LOG_WARNING("Parsing json", {{"json_sub", json_sub}}); try { out = json::parse(json_sub); @@ -155,9 +157,7 @@ static llama_tool_calls parse_llama_3_1_tool_calls(const json & tools, const std return {input, {}}; } -static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::string& input) { - static std::regex function_regex(R"()"); - static std::regex close_regex(R"()"); +static llama_tool_calls parse_functionary_tool_calls(const std::string& input, const std::regex & function_regex, const std::regex & close_regex) { std::smatch match; llama_tool_calls result; @@ -190,22 +190,16 @@ static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::str return result; } +static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::string& input) { + static std::regex function_regex(R"()"); + static std::regex close_regex(R"()"); + return parse_functionary_tool_calls(input, function_regex, close_regex); +} + static llama_tool_calls parse_functionary_v3_tool_calls(const std::string& input) { - static std::regex python_tag_regex(R"(>>>(\w+)\n((?!>>>)[\s\S\n]*))"); - std::smatch match; - llama_tool_calls result; - std::string content; - std::string in = input; - while (std::regex_search(in, match, python_tag_regex)) { - content += match.prefix().str(); - result.tool_calls.push_back({ - match[1].str(), - (json {{"code", match[2].str()}}).dump(), - }); - in = match.suffix().str(); - } - result.content = content + in; - return result; + static std::regex function_regex(R"(>>>(\w+)\n)"); + static std::regex close_regex(R"($|\n(?=>>>))"); + return parse_functionary_tool_calls(input, function_regex, close_regex); } llama_tool_calls parse_tool_calls(const json & tools, const std::string & chat_template, const std::string& input) { diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py index 480b85c23c0c6..04e2d2875e7bf 100644 --- a/examples/server/tests/features/steps/steps.py +++ b/examples/server/tests/features/steps/steps.py @@ -166,7 +166,7 @@ def step_use_jinja(context): context.use_jinja = True -@step('chat template file {file}') +@step('a chat template file {file}') def step_use_jinja(context, file): context.chat_template_file = file diff --git a/examples/server/tests/features/tool_call.feature b/examples/server/tests/features/tool_call.feature index 43edc651e9b06..81c427bdb2224 100644 --- a/examples/server/tests/features/tool_call.feature +++ b/examples/server/tests/features/tool_call.feature @@ -15,34 +15,36 @@ Feature: llama.cpp server And 64 server max tokens to predict And prometheus compatible metrics exposed And jinja templates are enabled - And chat template file ../../../tests/chat/templates/meta-llama-Meta-Llama-3.1-8B-Instruct.jinja - Then the server is starting - Then the server is healthy - - Scenario: Health - Then the server is ready - And all slots are idle + @wip Scenario Outline: OAI Compatibility w/ required tool - Given a model test + Given a chat template file ../../../tests/chat/templates/.jinja + And the server is starting + And the server is healthy + And a model test And max tokens to predict And a user prompt write a hello world in python And a tool choice And tools - Given an OAI compatible chat completions request with no api error + And an OAI compatible chat completions request with no api error Then tool is called with arguments Examples: Prompts - | n | tool_name | tool_arguments | tool_choice | tools | - | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | - | 16 | ipython | {"code": "it and "} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | + | template_name | n | tool_name | tool_arguments | tool_choice | tools | + | meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | + | meta-llama-Meta-Llama-3.1-8B-Instruct | 16 | ipython | {"code": "it and "} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | + | meetkai-functionary-medium-v3.2 | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | + | meetkai-functionary-medium-v3.2 | 64 | ipython | {"code": "Yes,"} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | Scenario: OAI Compatibility w/ no tool - Given a model test + Given a chat template file ../../../tests/chat/templates/meta-llama-Meta-Llama-3.1-8B-Instruct.jinja + And the server is starting + And the server is healthy + And a model test And 16 max tokens to predict And a user prompt write a hello world in python And a tool choice And tools [] - Given an OAI compatible chat completions request with no api error + And an OAI compatible chat completions request with no api error Then no tool is called diff --git a/tests/test-tool-call.cpp b/tests/test-tool-call.cpp index 24ef8a589d093..b43aca0670c9b 100644 --- a/tests/test-tool-call.cpp +++ b/tests/test-tool-call.cpp @@ -74,7 +74,7 @@ int main() { std::string functionary_v3_like_tmpl = "Functionary 3.2 template should have <|start_header_id|> and then some >>>all inside it"; test_parse_tool_call(tools, functionary_v3_like_tmpl, - ">>>ipython\nprint('Hello, world!')", + ">>>ipython\n{\"code\": \"print('Hello, world!')\"}", "", json {{ {"function", { @@ -84,6 +84,15 @@ int main() { }).dump()} }} }}); + test_parse_tool_call(tools, functionary_v3_like_tmpl, + ">>>test\n{ } \n ", + "", + json {{ + {"function", { + {"name", "test"}, + {"arguments", "{}"} + }} + }}); std::string functionary_v3_llama_3_1_like_tmpl = "Functionary 3.2 template for llama 3.1 should have <|start_header_id|> and then some {...} inside it"; test_parse_tool_call(tools, functionary_v3_llama_3_1_like_tmpl,