Merge pull request #52 from matlab-deep-learning/openAI-updates

gpt-4o-mini does handle images
matlab-deep-learning · Jul 22, 2024 · d42e09d · d42e09d
2 parents d74ec2b + 29cf265
commit d42e09d
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 3 deletions.
diff --git a/+llms/+openai/validateMessageSupported.m b/+llms/+openai/validateMessageSupported.m
@@ -5,7 +5,9 @@
 
     % only certain models support image generation
     if iscell(message.content) && any(cellfun(@(x) isfield(x,"image_url"), message.content))
-        if ~ismember(model,["gpt-4-turbo","gpt-4-turbo-2024-04-09","gpt-4o","gpt-4o-2024-05-13"]) 
+        if ~ismember(model,["gpt-4-turbo","gpt-4-turbo-2024-04-09",...
+            "gpt-4o-mini","gpt-4o-mini-2024-07-18",...
+            "gpt-4o","gpt-4o-2024-05-13"]) 
          error("llms:invalidContentTypeForModel", ...
                llms.utils.errorMessageCatalog.getMessage("llms:invalidContentTypeForModel", "Image content", model));
         end

diff --git a/examples/DescribeImagesUsingChatGPT.mlx b/examples/DescribeImagesUsingChatGPT.mlx
diff --git a/tests/topenAIChat.m b/tests/topenAIChat.m
@@ -173,6 +173,16 @@ function generateWithToolsAndStreamFunc(testCase)
             testCase.verifyThat(data,HasField("explanation"));
         end
 
+        function generateWithImages(testCase)
+            chat = openAIChat;
+            image_path = "peppers.png";
+            emptyMessages = messageHistory;
+            messages = addUserMessageWithImages(emptyMessages,"What is in the image?",image_path);
+
+            text = generate(chat,messages);
+            testCase.verifyThat(text,matlab.unittest.constraints.ContainsSubstring("pepper"));
+        end
+
         function invalidInputsGenerate(testCase, InvalidGenerateInput)
             f = openAIFunction("validfunction");
             chat = openAIChat(Tools=f, APIKey="this-is-not-a-real-key");
@@ -189,8 +199,8 @@ function assignValueToProperty(property, value)
             testCase.verifyError(@()assignValueToProperty(InvalidValuesSetters.Property,InvalidValuesSetters.Value), InvalidValuesSetters.Error);
         end
 
-        function invalidGenerateInputforModel(testCase)
-            chat = openAIChat(APIKey="this-is-not-a-real-key");
+        function gpt35TurboErrorsForImages(testCase)
+            chat = openAIChat(APIKey="this-is-not-a-real-key",Model="gpt-3.5-turbo");
             image_path = "peppers.png";
             emptyMessages = messageHistory;
             inValidMessages = addUserMessageWithImages(emptyMessages,"What is in the image?",image_path);