doc: openAIChat

+llms/+internal/callAzureChatAPI.m

@@ -64,7 +64,7 @@
 
 parameters = buildParametersCall(messages, functions, nvp);
 
-[response, streamedText] = llms.internal.sendRequest(parameters,nvp.APIKey, URL, nvp.TimeOut, nvp.StreamFun);
+[response, streamedText] = llms.internal.sendRequestWrapper(parameters,nvp.APIKey, URL, nvp.TimeOut, nvp.StreamFun);
 
 % If call errors, "choices" will not be part of response.Body.Data, instead
 % we get response.Body.Data.error

+llms/+internal/callOllamaChatAPI.m

@@ -53,7 +53,7 @@
 
 parameters = buildParametersCall(model, messages, nvp);
 
-[response, streamedText] = llms.internal.sendRequest(parameters,[],URL,nvp.TimeOut,nvp.StreamFun);
+[response, streamedText] = llms.internal.sendRequestWrapper(parameters,[],URL,nvp.TimeOut,nvp.StreamFun);
 
 % If call errors, "choices" will not be part of response.Body.Data, instead
 % we get response.Body.Data.error

+llms/+internal/callOpenAIChatAPI.m

@@ -62,7 +62,7 @@
 
 parameters = buildParametersCall(messages, functions, nvp);
 
-[response, streamedText] = llms.internal.sendRequest(parameters,nvp.APIKey, END_POINT, nvp.TimeOut, nvp.StreamFun);
+[response, streamedText] = llms.internal.sendRequestWrapper(parameters,nvp.APIKey, END_POINT, nvp.TimeOut, nvp.StreamFun);
 
 % If call errors, "choices" will not be part of response.Body.Data, instead
 % we get response.Body.Data.error

+llms/+internal/sendRequestWrapper.m

@@ -0,0 +1,5 @@
+function [response, streamedText] = sendRequestWrapper(varargin)
+% This function is undocumented and will change in a future release
+
+% A wrapper around sendRequest to have a test seam
+[response, streamedText] = llms.internal.sendRequest(varargin{:});

+llms/+internal/textGenerator.m

@@ -28,4 +28,10 @@
     properties (Access=protected)
         StreamFun
     end
+
+    methods
+        function hObj = set.StopSequences(hObj,value)
+            hObj.StopSequences = string(value);
+        end
+    end
 end

+llms/+stream/responseStreamer.m

@@ -84,7 +84,8 @@
                                 end
                                 this.StreamFun('');
                                 this.ResponseText = txt;
-                            else
+                            elseif isfield(json.choices,"delta") && ...
+                                    isfield(json.choices.delta,"content")
                                 txt = json.choices.delta.content;
                                 this.StreamFun(txt);
                                 this.ResponseText = [this.ResponseText txt];

+llms/+utils/errorMessageCatalog.m

@@ -41,22 +41,22 @@
 catalog("llms:mustBeAssistantWithIdAndFunction") = "Field 'tool_call' must be a struct with fields 'id' and 'function'.";
 catalog("llms:mustBeAssistantWithNameAndArguments") = "Field 'function' must be a struct with fields 'name' and 'arguments'.";
 catalog("llms:assistantMustHaveTextNameAndArguments") = "Fields 'name' and 'arguments' must be text with one or more characters.";
-catalog("llms:mustBeValidIndex") = "Index exceeds the number of array elements. Index must be less than or equal to ({1}).";
-catalog("llms:stopSequencesMustHaveMax4Elements") = "Number of elements must not be larger than 4.";
+catalog("llms:mustBeValidIndex") = "Index exceeds the number of array elements. Index must be less than or equal to {1}.";
+catalog("llms:stopSequencesMustHaveMax4Elements") = "Number of stop sequences must be less than or equal to 4.";
 catalog("llms:endpointMustBeSpecified") = "Unable to find endpoint. Either set environment variable AZURE_OPENAI_ENDPOINT or specify name-value argument ""Endpoint"".";
 catalog("llms:deploymentMustBeSpecified") = "Unable to find deployment name. Either set environment variable AZURE_OPENAI_DEPLOYMENT or specify name-value argument ""Deployment"".";
 catalog("llms:keyMustBeSpecified") = "Unable to find API key. Either set environment variable {1} or specify name-value argument ""APIKey"".";
-catalog("llms:mustHaveMessages") = "Value must contain at least one message in Messages.";
+catalog("llms:mustHaveMessages") = "Message history must not be empty.";
 catalog("llms:mustSetFunctionsForCall") = "When no functions are defined, ToolChoice must not be specified.";
-catalog("llms:mustBeMessagesOrTxt") = "Messages must be text with one or more characters or a messageHistory object.";
-catalog("llms:invalidOptionAndValueForModel") = "'{1}' with value '{2}' is not supported for ModelName '{3}'";
-catalog("llms:invalidOptionForModel") = "{1} is not supported for ModelName '{2}'";
-catalog("llms:invalidContentTypeForModel") = "{1} is not supported for ModelName '{2}'";
-catalog("llms:functionNotAvailableForModel") = "This function is not supported for ModelName '{1}'";
-catalog("llms:promptLimitCharacter") = "Prompt must have a maximum length of {1} characters for ModelName '{2}'";
-catalog("llms:pngExpected") = "Argument must be a PNG image.";
+catalog("llms:mustBeMessagesOrTxt") = "Message must be nonempty string, character array, cell array of character vectors, or messageHistory object.";
+catalog("llms:invalidOptionAndValueForModel") = "'{1}' with value '{2}' is not supported for model ""{3}"".";
+catalog("llms:invalidOptionForModel") = "Invalid argument name {1} for model ""{2}"".";
+catalog("llms:invalidContentTypeForModel") = "{1} is not supported for model ""{2}"".";
+catalog("llms:functionNotAvailableForModel") = "Image editing is not supported for model ""{1}"".";
+catalog("llms:promptLimitCharacter") = "Prompt must contain at most {1} characters for model ""{2}"".";
+catalog("llms:pngExpected") = "Image must be a PNG file (*.png).";
 catalog("llms:warningJsonInstruction") = "When using JSON mode, you must also prompt the model to produce JSON yourself via a system or user message.";
-catalog("llms:apiReturnedError") = "Server error: ""{1}""";
+catalog("llms:apiReturnedError") = "Server returned error indicating: ""{1}""";
 catalog("llms:dimensionsMustBeSmallerThan") = "Dimensions must be less than or equal to {1}.";
 catalog("llms:stream:responseStreamer:InvalidInput") = "Input does not have the expected json format, got ""{1}"".";
 end

+llms/+utils/mustBeValidStop.m

@@ -5,6 +5,7 @@ function mustBeValidStop(value)
     if ~isempty(value)
         mustBeVector(value);
         mustBeNonzeroLengthText(value);
+        value = string(value);
         % This restriction is set by the OpenAI API
         if numel(value)>4
             error("llms:stopSequencesMustHaveMax4Elements", llms.utils.errorMessageCatalog.getMessage("llms:stopSequencesMustHaveMax4Elements"));

.githooks/pre-commit

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+cd $(git rev-parse --show-toplevel)
+pwd
+
+# For all commits of mlx files, create corresponding Markdown (md) files.
+# If the mlx files are in .../mlx-scripts/*.mlx, the corresponding
+# md files will go into .../*.md.
+#
+# This script assumes that the mlx files as currently in the file system
+# are what is being committed, instead of doing a lot of extra work to
+# get them from the stage area.
+#
+# Note that this script will not remove media files. If an mlx has
+# fewer plots at some point in the future, there will be file system
+# cruft. Which doesn't hurt the md display in GitHub or elswehere.
+changedMlxFiles=`git diff --cached --name-only --diff-filter=d '*.mlx'`
+
+if [ -n "$changedMlxFiles" ]; then
+    # Keep the line break here, we replace end-of-line with "' '" to get the quotes right
+    matlab -batch "for file = {'${changedMlxFiles//
+/' '}'}, export(file{1},replace(erase(file{1},'mlx-scripts'),'.mlx','.md')); end"
+    tmp=${changedMlxFiles//mlx-scripts\//}
+    mdFiles=${tmp//.mlx/.md}
+    for file in $mdFiles; do
+        if [ -d ${file%.md}_media ]; then
+            git add ${file%.md}_media/
+        fi
+        perl -pi -e "\$cnt++ if /^#/; " \
+            -e "\$_ .= \"\nTo run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/$(basename $file .md).mlx](mlx-scripts/$(basename $file .md).mlx) \n\" if /^#/ && \$cnt==1;" \
+            $file
+    done
+    git add $mdFiles
+fi

.github/CODEOWNERS

@@ -0,0 +1,4 @@
+# Code owners, to get auto-filled reviewer lists
+
+# To start with, we just assume everyone in the core team is included on all reviews
+* @adulai @ccreutzi @debymf @MiriamScharnke @vpapanasta

.github/workflows/ci.yml

@@ -30,6 +30,7 @@ jobs:
       - name: Pull models
         run: |
             ollama pull mistral
+            ollama pull moondream
             OLLAMA_HOST=127.0.0.1:11435 ollama pull qwen2:0.5b
       - name: Set up MATLAB
         uses: matlab-actions/setup-matlab@v2

.gitignore

@@ -1,10 +1,12 @@
 *.env
 *.asv
 *.mat
+!tests/recordings/*.mat
 startup.m
 papers_to_read.csv
 data/*
 examples/data/*
+examples/mlx-scripts/data/*
 ._*
 .nfs*
 .DS_Store

DEVELOPMENT.md

@@ -0,0 +1,11 @@
+# Notes for Developers
+
+Nothing in this file should be required knowledge to use the repository. These are notes for people actually making changes that are going to be submitted and incorporated into the main branch.
+
+## Git Hooks
+
+After checkout, link or (on Windows) copy the files from `.githooks` into the local `.git/hooks` folder:
+
+```
+(cd .git/hooks/; ln -s ../../.githooks/pre-commit .)
+```

README.md

@@ -1,8 +1,8 @@
-# Large Language Models (LLMs) with MATLAB®
+# Large Language Models (LLMs) with MATLAB
 
 [![Open in MATLAB Online](https://www.mathworks.com/images/responsive/global/open-in-matlab-online.svg)](https://matlab.mathworks.com/open/github/v1?repo=matlab-deep-learning/llms-with-matlab) [![View Large Language Models (LLMs) with MATLAB on File Exchange](https://www.mathworks.com/matlabcentral/images/matlab-file-exchange.svg)](https://www.mathworks.com/matlabcentral/fileexchange/163796-large-language-models-llms-with-matlab) 
 
-This repository contains code to connect MATLAB to the [OpenAI™ Chat Completions API](https://platform.openai.com/docs/guides/text-generation/chat-completions-api) (which powers ChatGPT™), OpenAI Images API (which powers DALL·E™), [Azure® OpenAI Service](https://learn.microsoft.com/en-us/azure/ai-services/openai/), and both local and nonlocal [Ollama™](https://ollama.com/) models. This allows you to leverage the natural language processing capabilities of large language models directly within your MATLAB environment.
+This repository contains code to connect MATLAB® to the [OpenAI® Chat Completions API](https://platform.openai.com/docs/guides/text-generation/chat-completions-api) (which powers ChatGPT™), OpenAI Images API (which powers DALL·E™), [Azure® OpenAI Service](https://learn.microsoft.com/en-us/azure/ai-services/openai/), and both local and nonlocal [Ollama™](https://ollama.com/) models. This allows you to leverage the natural language processing capabilities of large language models directly within your MATLAB environment.
 
 ## Requirements
 
@@ -52,16 +52,16 @@ To use this repository with a local installation of MATLAB, first clone the repo
 ## Examples
 To learn how to use this in your workflows, see [Examples](/examples/).
 
-- [ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.mlx](/examples/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.mlx): Learn to implement a simple chat that stream the response. 
-- [SummarizeLargeDocumentsUsingChatGPTandMATLAB.mlx](/examples/SummarizeLargeDocumentsUsingChatGPTandMATLAB.mlx): Learn to create concise summaries of long texts with ChatGPT. (Requires Text Analytics Toolbox™)
-- [CreateSimpleChatBot.mlx](/examples/CreateSimpleChatBot.mlx): Build a conversational chatbot capable of handling various dialogue scenarios using ChatGPT. (Requires Text Analytics Toolbox)
-- [AnalyzeScientificPapersUsingFunctionCalls.mlx](/examples/AnalyzeScientificPapersUsingFunctionCalls.mlx): Learn how to create agents capable of executing MATLAB functions. 
-- [AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mlx](/examples/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mlx): Learn how to take advantage of parallel function calling. 
-- [RetrievalAugmentedGenerationUsingChatGPTandMATLAB.mlx](/examples/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.mlx): Learn about retrieval augmented generation with a simple use case. (Requires Text Analytics Toolbox™)
-- [DescribeImagesUsingChatGPT.mlx](/examples/DescribeImagesUsingChatGPT.mlx): Learn how to use GPT-4 Turbo with Vision to understand the content of an image. 
-- [AnalyzeSentimentinTextUsingChatGPTinJSONMode.mlx](/examples/AnalyzeSentimentinTextUsingChatGPTinJSONMode.mlx): Learn how to use JSON mode in chat completions
-- [UsingDALLEToEditImages.mlx](/examples/UsingDALLEToEditImages.mlx): Learn how to generate images
-- [UsingDALLEToGenerateImages.mlx](/examples/UsingDALLEToGenerateImages.mlx): Create variations of images and editimages. 
+- [ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.md](/examples/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.md): Learn to implement a simple chat that stream the response. 
+- [SummarizeLargeDocumentsUsingChatGPTandMATLAB.md](/examples/SummarizeLargeDocumentsUsingChatGPTandMATLAB.md): Learn to create concise summaries of long texts with ChatGPT. (Requires Text Analytics Toolbox™)
+- [CreateSimpleChatBot.md](/examples/CreateSimpleChatBot.md): Build a conversational chatbot capable of handling various dialogue scenarios using ChatGPT. (Requires Text Analytics Toolbox)
+- [AnalyzeScientificPapersUsingFunctionCalls.md](/examples/AnalyzeScientificPapersUsingFunctionCalls.md): Learn how to create agents capable of executing MATLAB functions. 
+- [AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.md](/examples/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.md): Learn how to take advantage of parallel function calling. 
+- [RetrievalAugmentedGenerationUsingChatGPTandMATLAB.md](/examples/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.md): Learn about retrieval augmented generation with a simple use case. (Requires Text Analytics Toolbox™)
+- [DescribeImagesUsingChatGPT.md](/examples/DescribeImagesUsingChatGPT.md): Learn how to use GPT-4 Turbo with Vision to understand the content of an image. 
+- [AnalyzeSentimentinTextUsingChatGPTinJSONMode.md](/examples/AnalyzeSentimentinTextUsingChatGPTinJSONMode.md): Learn how to use JSON mode in chat completions
+- [UsingDALLEToEditImages.md](/examples/UsingDALLEToEditImages.md): Learn how to generate images
+- [UsingDALLEToGenerateImages.md](/examples/UsingDALLEToGenerateImages.md): Create variations of images and editimages. 
 
 ## License
 

azureChat.m

@@ -191,7 +191,7 @@
             if isstring(messages) && isscalar(messages)
                 messagesStruct = {struct("role", "user", "content", messages)};
             else
-                messagesStruct = messages.Messages;
+                messagesStruct = this.encodeImages(messages.Messages);
             end
 
             if ~isempty(this.SystemPrompt)
@@ -251,6 +251,40 @@ function mustBeValidFunctionCall(this, functionCall)
             end
 
         end
+
+        function messageStruct = encodeImages(~, messageStruct)
+            for k=1:numel(messageStruct)
+                if isfield(messageStruct{k},"images")
+                    images = messageStruct{k}.images;
+                    detail = messageStruct{k}.image_detail;
+                    messageStruct{k} = rmfield(messageStruct{k},["images","image_detail"]);
+                    messageStruct{k}.content = ...
+                        {struct("type","text","text",messageStruct{k}.content)};
+                    for img = images(:).'
+                        if startsWith(img,("https://"|"http://"))
+                            s = struct( ...
+                                "type","image_url", ...
+                                "image_url",struct("url",img));
+                        else
+                            [~,~,ext] = fileparts(img);
+                            MIMEType = "data:image/" + erase(ext,".") + ";base64,";
+                            % Base64 encode the image using the given MIME type
+                            fid = fopen(img);
+                            im = fread(fid,'*uint8');
+                            fclose(fid);
+                            b64 = matlab.net.base64encode(im);
+                            s = struct( ...
+                                "type","image_url", ...
+                                "image_url",struct("url",MIMEType + b64));
+                        end
+
+                        s.image_url.detail = detail;
+
+                        messageStruct{k}.content{end+1} = s;
+                    end
+                end
+            end
+        end
     end
 end
 

doc/Azure.md

@@ -1,6 +1,6 @@
-# Connecting to Azure® OpenAI Service
+# Connecting to Azure OpenAI Service
 
-This repository contains code to connect MATLAB to the [Azure® OpenAI Service](https://learn.microsoft.com/en-us/azure/ai-services/openai/).
+This repository contains code to connect MATLAB to the [Azure® OpenAI® Service](https://learn.microsoft.com/en-us/azure/ai-services/openai/).
 
 To use Azure OpenAI Services, you need to create a model deployment on your Azure account and obtain one of the keys for it. You are responsible for any fees Azure may charge for the use of their APIs. You should be familiar with the limitations and risks associated with using this technology, and you agree that you shall be solely responsible for full compliance with any terms that may apply to your use of the Azure APIs.
 
@@ -31,7 +31,7 @@ loadenv(".env")
 
 ## Establishing a connection to Chat Completions API using Azure
 
-To connect MATLAB to Chat Completions API via Azure, you will have to create an `azureChat` object. See [the Azure documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/chatgpt-quickstart) for details on the setup required and where to find your key, endpoint, and deployment name. As explained above, the endpoint, deployment, and key should be in the environment variables `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_DEPLOYMENYT`, and `AZURE_OPENAI_API_KEY`, or provided as `Endpoint=…`, `Deployment=…`, and `APIKey=…` in the `azureChat` call below.
+To connect MATLAB® to Chat Completions API via Azure, you will have to create an `azureChat` object. See [the Azure documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/chatgpt-quickstart) for details on the setup required and where to find your key, endpoint, and deployment name. As explained above, the endpoint, deployment, and key should be in the environment variables `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_DEPLOYMENYT`, and `AZURE_OPENAI_API_KEY`, or provided as `Endpoint=…`, `Deployment=…`, and `APIKey=…` in the `azureChat` call below.
 
 In order to create the chat assistant, use the `azureChat` function, optionally providing a system prompt:
 ```matlab
@@ -115,6 +115,19 @@ txt = generate(chat,"What is Model-Based Design and how is it related to Digital
 % Should stream the response token by token
 ```
 
+## Understanding the content of an image
+
+You can use gpt-4o, gpt-4o-mini, or gpt-4-turbo to experiment with image understanding. 
+```matlab
+chat = azureChat("You are an AI assistant.",Deployment="gpt-4o");
+image_path = "peppers.png";
+messages = messageHistory;
+messages = addUserMessageWithImages(messages,"What is in the image?",image_path);
+[txt,response] = generate(chat,messages,MaxNumTokens=4096);
+txt
+% outputs a description of the image
+```
+
 ## Calling MATLAB functions with the API
 
 Optionally, `Tools=functions` can be used to provide function specifications to the API. The purpose of this is to enable models to generate function arguments which adhere to the provided specifications.