From 53c1df54ddea80e233acc9a4716132cf1ad204cb Mon Sep 17 00:00:00 2001 From: Christopher Creutzig <89011131+ccreutzi@users.noreply.github.com> Date: Mon, 5 Aug 2024 12:12:45 +0200 Subject: [PATCH] Avoid bogus access to `json.choices.delta.content` The streaming interface sometimes responds with something like `json.choices = {"index":0,"delta":{},"logprobs":[],"finish_reason":"length"}`. Stop blindly assuming `json.choices.delta.content` exists. --- +llms/+stream/responseStreamer.m | 3 ++- tests/topenAIChat.m | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/+llms/+stream/responseStreamer.m b/+llms/+stream/responseStreamer.m index b13048d..d3b60b1 100644 --- a/+llms/+stream/responseStreamer.m +++ b/+llms/+stream/responseStreamer.m @@ -84,7 +84,8 @@ end this.StreamFun(''); this.ResponseText = txt; - else + elseif isfield(json.choices,"delta") && ... + isfield(json.choices.delta,"content") txt = json.choices.delta.content; this.StreamFun(txt); this.ResponseText = [this.ResponseText txt]; diff --git a/tests/topenAIChat.m b/tests/topenAIChat.m index b156b18..ad3f69e 100644 --- a/tests/topenAIChat.m +++ b/tests/topenAIChat.m @@ -97,6 +97,14 @@ function invalidInputsConstructor(testCase, InvalidConstructorInput) testCase.verifyError(@()openAIChat(InvalidConstructorInput.Input{:}), InvalidConstructorInput.Error); end + function generateWithStreamFunAndMaxNumTokens(testCase) + sf = @(x) fprintf("%s",x); + chat = openAIChat(StreamFun=sf); + result = generate(chat,"Why is a raven like a writing desk?",MaxNumTokens=5); + testCase.verifyClass(result,"string"); + testCase.verifyLessThan(strlength(result), 100); + end + function generateWithToolsAndStreamFunc(testCase) import matlab.unittest.constraints.HasField