diff --git a/common/tool-call.cpp b/common/tool-call.cpp
index cb9ee2ecf4124..ca25b803804fb 100644
--- a/common/tool-call.cpp
+++ b/common/tool-call.cpp
@@ -39,6 +39,8 @@ static bool parse_json(std::string::const_iterator & it, const std::string::cons
std::size_t position;
bool found_error;
+ json_error_locator() : position(0), found_error(false) {}
+
bool parse_error(std::size_t position, const std::string & last_token, const json::exception & ex) override {
// LOG_WARNING("JSON error (Expected)", {{"position", position}, {"last_token", last_token}, {"error", ex.what()}});
this->position = position - 1;
@@ -67,7 +69,7 @@ static bool parse_json(std::string::const_iterator & it, const std::string::cons
} else {
temptative_end = end;
}
- std::string json_sub {it, it + err_loc.position};
+ std::string json_sub {it, temptative_end};
// LOG_WARNING("Parsing json", {{"json_sub", json_sub}});
try {
out = json::parse(json_sub);
@@ -155,9 +157,7 @@ static llama_tool_calls parse_llama_3_1_tool_calls(const json & tools, const std
return {input, {}};
}
-static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::string& input) {
- static std::regex function_regex(R"()");
- static std::regex close_regex(R"()");
+static llama_tool_calls parse_functionary_tool_calls(const std::string& input, const std::regex & function_regex, const std::regex & close_regex) {
std::smatch match;
llama_tool_calls result;
@@ -190,22 +190,16 @@ static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::str
return result;
}
+static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::string& input) {
+ static std::regex function_regex(R"()");
+ static std::regex close_regex(R"()");
+ return parse_functionary_tool_calls(input, function_regex, close_regex);
+}
+
static llama_tool_calls parse_functionary_v3_tool_calls(const std::string& input) {
- static std::regex python_tag_regex(R"(>>>(\w+)\n((?!>>>)[\s\S\n]*))");
- std::smatch match;
- llama_tool_calls result;
- std::string content;
- std::string in = input;
- while (std::regex_search(in, match, python_tag_regex)) {
- content += match.prefix().str();
- result.tool_calls.push_back({
- match[1].str(),
- (json {{"code", match[2].str()}}).dump(),
- });
- in = match.suffix().str();
- }
- result.content = content + in;
- return result;
+ static std::regex function_regex(R"(>>>(\w+)\n)");
+ static std::regex close_regex(R"($|\n(?=>>>))");
+ return parse_functionary_tool_calls(input, function_regex, close_regex);
}
llama_tool_calls parse_tool_calls(const json & tools, const std::string & chat_template, const std::string& input) {
diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py
index 480b85c23c0c6..04e2d2875e7bf 100644
--- a/examples/server/tests/features/steps/steps.py
+++ b/examples/server/tests/features/steps/steps.py
@@ -166,7 +166,7 @@ def step_use_jinja(context):
context.use_jinja = True
-@step('chat template file {file}')
+@step('a chat template file {file}')
def step_use_jinja(context, file):
context.chat_template_file = file
diff --git a/examples/server/tests/features/tool_call.feature b/examples/server/tests/features/tool_call.feature
index 43edc651e9b06..81c427bdb2224 100644
--- a/examples/server/tests/features/tool_call.feature
+++ b/examples/server/tests/features/tool_call.feature
@@ -15,34 +15,36 @@ Feature: llama.cpp server
And 64 server max tokens to predict
And prometheus compatible metrics exposed
And jinja templates are enabled
- And chat template file ../../../tests/chat/templates/meta-llama-Meta-Llama-3.1-8B-Instruct.jinja
- Then the server is starting
- Then the server is healthy
-
- Scenario: Health
- Then the server is ready
- And all slots are idle
+ @wip
Scenario Outline: OAI Compatibility w/ required tool
- Given a model test
+ Given a chat template file ../../../tests/chat/templates/.jinja
+ And the server is starting
+ And the server is healthy
+ And a model test
And max tokens to predict
And a user prompt write a hello world in python
And a tool choice
And tools
- Given an OAI compatible chat completions request with no api error
+ And an OAI compatible chat completions request with no api error
Then tool is called with arguments
Examples: Prompts
- | n | tool_name | tool_arguments | tool_choice | tools |
- | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
- | 16 | ipython | {"code": "it and "} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
+ | template_name | n | tool_name | tool_arguments | tool_choice | tools |
+ | meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
+ | meta-llama-Meta-Llama-3.1-8B-Instruct | 16 | ipython | {"code": "it and "} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
+ | meetkai-functionary-medium-v3.2 | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
+ | meetkai-functionary-medium-v3.2 | 64 | ipython | {"code": "Yes,"} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
Scenario: OAI Compatibility w/ no tool
- Given a model test
+ Given a chat template file ../../../tests/chat/templates/meta-llama-Meta-Llama-3.1-8B-Instruct.jinja
+ And the server is starting
+ And the server is healthy
+ And a model test
And 16 max tokens to predict
And a user prompt write a hello world in python
And a tool choice
And tools []
- Given an OAI compatible chat completions request with no api error
+ And an OAI compatible chat completions request with no api error
Then no tool is called
diff --git a/tests/test-tool-call.cpp b/tests/test-tool-call.cpp
index 24ef8a589d093..b43aca0670c9b 100644
--- a/tests/test-tool-call.cpp
+++ b/tests/test-tool-call.cpp
@@ -74,7 +74,7 @@ int main() {
std::string functionary_v3_like_tmpl = "Functionary 3.2 template should have <|start_header_id|> and then some >>>all inside it";
test_parse_tool_call(tools, functionary_v3_like_tmpl,
- ">>>ipython\nprint('Hello, world!')",
+ ">>>ipython\n{\"code\": \"print('Hello, world!')\"}",
"",
json {{
{"function", {
@@ -84,6 +84,15 @@ int main() {
}).dump()}
}}
}});
+ test_parse_tool_call(tools, functionary_v3_like_tmpl,
+ ">>>test\n{ } \n ",
+ "",
+ json {{
+ {"function", {
+ {"name", "test"},
+ {"arguments", "{}"}
+ }}
+ }});
std::string functionary_v3_llama_3_1_like_tmpl = "Functionary 3.2 template for llama 3.1 should have <|start_header_id|> and then some {...} inside it";
test_parse_tool_call(tools, functionary_v3_llama_3_1_like_tmpl,