tool-call: fix/test functionary v3

ggerganov · Sep 26, 2024 · 595e11c · 595e11c
1 parent c124ab4
commit 595e11c
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 35 deletions.
diff --git a/common/tool-call.cpp b/common/tool-call.cpp
@@ -39,6 +39,8 @@ static bool parse_json(std::string::const_iterator & it, const std::string::cons
         std::size_t position;
         bool found_error;
 
+        json_error_locator() : position(0), found_error(false) {}
+
         bool parse_error(std::size_t position, const std::string & last_token, const json::exception & ex) override {
             // LOG_WARNING("JSON error (Expected)", {{"position", position}, {"last_token", last_token}, {"error", ex.what()}});
             this->position = position - 1;
@@ -67,7 +69,7 @@ static bool parse_json(std::string::const_iterator & it, const std::string::cons
     } else {
         temptative_end = end;
     }
-    std::string json_sub {it, it + err_loc.position};
+    std::string json_sub {it, temptative_end};
     // LOG_WARNING("Parsing json", {{"json_sub", json_sub}});
     try {
         out = json::parse(json_sub);
@@ -155,9 +157,7 @@ static llama_tool_calls parse_llama_3_1_tool_calls(const json & tools, const std
     return {input, {}};
 }
 
-static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::string& input) {
-    static std::regex function_regex(R"(<function=(\w+)>)");
-    static std::regex close_regex(R"(</function>)");
+static llama_tool_calls parse_functionary_tool_calls(const std::string& input, const std::regex & function_regex, const std::regex & close_regex) {
     std::smatch match;
 
     llama_tool_calls result;
@@ -190,22 +190,16 @@ static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::str
     return result;
 }
 
+static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::string& input) {
+    static std::regex function_regex(R"(<function=(\w+)>)");
+    static std::regex close_regex(R"(</function>)");
+    return parse_functionary_tool_calls(input, function_regex, close_regex);
+}
+
 static llama_tool_calls parse_functionary_v3_tool_calls(const std::string& input) {
-    static std::regex python_tag_regex(R"(>>>(\w+)\n((?!>>>)[\s\S\n]*))");
-    std::smatch match;
-    llama_tool_calls result;
-    std::string content;
-    std::string in = input;
-    while (std::regex_search(in, match, python_tag_regex)) {
-        content += match.prefix().str();
-        result.tool_calls.push_back({
-            match[1].str(),
-            (json {{"code", match[2].str()}}).dump(),
-        });
-        in = match.suffix().str();
-    }
-    result.content = content + in;
-    return result;
+    static std::regex function_regex(R"(>>>(\w+)\n)");
+    static std::regex close_regex(R"($|\n(?=>>>))");
+    return parse_functionary_tool_calls(input, function_regex, close_regex);
 }
 
 llama_tool_calls parse_tool_calls(const json & tools, const std::string & chat_template, const std::string& input) {

diff --git a/examples/server/tests/features/steps/steps.py b/examples/server/tests/features/steps/steps.py
@@ -166,7 +166,7 @@ def step_use_jinja(context):
     context.use_jinja = True
 
 
-@step('chat template file {file}')
+@step('a chat template file {file}')
 def step_use_jinja(context, file):
     context.chat_template_file = file
 

diff --git a/examples/server/tests/features/tool_call.feature b/examples/server/tests/features/tool_call.feature
@@ -15,34 +15,36 @@ Feature: llama.cpp server
     And   64 server max tokens to predict
     And   prometheus compatible metrics exposed
     And   jinja templates are enabled
-    And   chat template file ../../../tests/chat/templates/meta-llama-Meta-Llama-3.1-8B-Instruct.jinja
-    Then  the server is starting
-    Then  the server is healthy
-
-  Scenario: Health
-    Then the server is ready
-    And  all slots are idle
 
+  @wip
   Scenario Outline: OAI Compatibility w/ required tool
-    Given a model test
+    Given a chat template file ../../../tests/chat/templates/<template_name>.jinja
+    And   the server is starting
+    And   the server is healthy
+    And   a model test
     And   <n> max tokens to predict
     And   a user prompt write a hello world in python
     And   a tool choice <tool_choice>
     And   tools <tools>
-    Given an OAI compatible chat completions request with no api error
+    And   an OAI compatible chat completions request with no api error
     Then  tool <tool_name> is called with arguments <tool_arguments>
 
     Examples: Prompts
-      | n   | tool_name | tool_arguments       | tool_choice | tools |
-      | 64  | test      | {}                   | required    | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
-      | 16  | ipython   | {"code": "it and "}  | required    | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
+      | template_name                         | n   | tool_name | tool_arguments       | tool_choice | tools |
+      | meta-llama-Meta-Llama-3.1-8B-Instruct | 64  | test      | {}                   | required    | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
+      | meta-llama-Meta-Llama-3.1-8B-Instruct | 16  | ipython   | {"code": "it and "}  | required    | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
+      | meetkai-functionary-medium-v3.2       | 64  | test      | {}                   | required    | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
+      | meetkai-functionary-medium-v3.2       | 64  | ipython   | {"code": "Yes,"}     | required    | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
 
   Scenario: OAI Compatibility w/ no tool
-    Given a model test
+    Given a chat template file ../../../tests/chat/templates/meta-llama-Meta-Llama-3.1-8B-Instruct.jinja
+    And   the server is starting
+    And   the server is healthy
+    And   a model test
     And   16 max tokens to predict
     And   a user prompt write a hello world in python
     And   a tool choice <tool_choice>
     And   tools []
-    Given an OAI compatible chat completions request with no api error
+    And   an OAI compatible chat completions request with no api error
     Then  no tool is called
 
diff --git a/tests/test-tool-call.cpp b/tests/test-tool-call.cpp
@@ -74,7 +74,7 @@ int main() {
 
     std::string functionary_v3_like_tmpl = "Functionary 3.2 template should have <|start_header_id|> and then some >>>all inside it";
     test_parse_tool_call(tools, functionary_v3_like_tmpl,
-      ">>>ipython\nprint('Hello, world!')",
+      ">>>ipython\n{\"code\": \"print('Hello, world!')\"}",
       "",
       json {{
         {"function", {
@@ -84,6 +84,15 @@ int main() {
           }).dump()}
         }}
       }});
+    test_parse_tool_call(tools, functionary_v3_like_tmpl,
+      ">>>test\n{ } \n ",
+      "",
+      json {{
+        {"function", {
+          {"name", "test"},
+          {"arguments", "{}"}
+        }}
+      }});
 
     std::string functionary_v3_llama_3_1_like_tmpl = "Functionary 3.2 template for llama 3.1 should have <|start_header_id|> and then some <function=foo>{...}</function> inside it";
     test_parse_tool_call(tools, functionary_v3_llama_3_1_like_tmpl,