diff --git a/+llms/+internal/callAzureChatAPI.m b/+llms/+internal/callAzureChatAPI.m index bb73053..c053b4a 100644 --- a/+llms/+internal/callAzureChatAPI.m +++ b/+llms/+internal/callAzureChatAPI.m @@ -64,7 +64,7 @@ parameters = buildParametersCall(messages, functions, nvp); -[response, streamedText] = llms.internal.sendRequest(parameters,nvp.APIKey, URL, nvp.TimeOut, nvp.StreamFun); +[response, streamedText] = llms.internal.sendRequestWrapper(parameters,nvp.APIKey, URL, nvp.TimeOut, nvp.StreamFun); % If call errors, "choices" will not be part of response.Body.Data, instead % we get response.Body.Data.error diff --git a/+llms/+internal/callOllamaChatAPI.m b/+llms/+internal/callOllamaChatAPI.m index 4596231..0bad15f 100644 --- a/+llms/+internal/callOllamaChatAPI.m +++ b/+llms/+internal/callOllamaChatAPI.m @@ -53,7 +53,7 @@ parameters = buildParametersCall(model, messages, nvp); -[response, streamedText] = llms.internal.sendRequest(parameters,[],URL,nvp.TimeOut,nvp.StreamFun); +[response, streamedText] = llms.internal.sendRequestWrapper(parameters,[],URL,nvp.TimeOut,nvp.StreamFun); % If call errors, "choices" will not be part of response.Body.Data, instead % we get response.Body.Data.error diff --git a/+llms/+internal/callOpenAIChatAPI.m b/+llms/+internal/callOpenAIChatAPI.m index 8d58fd4..742ce50 100644 --- a/+llms/+internal/callOpenAIChatAPI.m +++ b/+llms/+internal/callOpenAIChatAPI.m @@ -62,7 +62,7 @@ parameters = buildParametersCall(messages, functions, nvp); -[response, streamedText] = llms.internal.sendRequest(parameters,nvp.APIKey, END_POINT, nvp.TimeOut, nvp.StreamFun); +[response, streamedText] = llms.internal.sendRequestWrapper(parameters,nvp.APIKey, END_POINT, nvp.TimeOut, nvp.StreamFun); % If call errors, "choices" will not be part of response.Body.Data, instead % we get response.Body.Data.error diff --git a/+llms/+internal/sendRequestWrapper.m b/+llms/+internal/sendRequestWrapper.m new file mode 100644 index 0000000..18160ce --- /dev/null +++ b/+llms/+internal/sendRequestWrapper.m @@ -0,0 +1,5 @@ +function [response, streamedText] = sendRequestWrapper(varargin) +% This function is undocumented and will change in a future release + +% A wrapper around sendRequest to have a test seam +[response, streamedText] = llms.internal.sendRequest(varargin{:}); diff --git a/+llms/+internal/textGenerator.m b/+llms/+internal/textGenerator.m index f6cb167..204e516 100644 --- a/+llms/+internal/textGenerator.m +++ b/+llms/+internal/textGenerator.m @@ -28,4 +28,10 @@ properties (Access=protected) StreamFun end + + methods + function hObj = set.StopSequences(hObj,value) + hObj.StopSequences = string(value); + end + end end diff --git a/+llms/+stream/responseStreamer.m b/+llms/+stream/responseStreamer.m index b13048d..d3b60b1 100644 --- a/+llms/+stream/responseStreamer.m +++ b/+llms/+stream/responseStreamer.m @@ -84,7 +84,8 @@ end this.StreamFun(''); this.ResponseText = txt; - else + elseif isfield(json.choices,"delta") && ... + isfield(json.choices.delta,"content") txt = json.choices.delta.content; this.StreamFun(txt); this.ResponseText = [this.ResponseText txt]; diff --git a/+llms/+utils/errorMessageCatalog.m b/+llms/+utils/errorMessageCatalog.m index 36fef5e..5a9be78 100644 --- a/+llms/+utils/errorMessageCatalog.m +++ b/+llms/+utils/errorMessageCatalog.m @@ -41,22 +41,22 @@ catalog("llms:mustBeAssistantWithIdAndFunction") = "Field 'tool_call' must be a struct with fields 'id' and 'function'."; catalog("llms:mustBeAssistantWithNameAndArguments") = "Field 'function' must be a struct with fields 'name' and 'arguments'."; catalog("llms:assistantMustHaveTextNameAndArguments") = "Fields 'name' and 'arguments' must be text with one or more characters."; -catalog("llms:mustBeValidIndex") = "Index exceeds the number of array elements. Index must be less than or equal to ({1})."; -catalog("llms:stopSequencesMustHaveMax4Elements") = "Number of elements must not be larger than 4."; +catalog("llms:mustBeValidIndex") = "Index exceeds the number of array elements. Index must be less than or equal to {1}."; +catalog("llms:stopSequencesMustHaveMax4Elements") = "Number of stop sequences must be less than or equal to 4."; catalog("llms:endpointMustBeSpecified") = "Unable to find endpoint. Either set environment variable AZURE_OPENAI_ENDPOINT or specify name-value argument ""Endpoint""."; catalog("llms:deploymentMustBeSpecified") = "Unable to find deployment name. Either set environment variable AZURE_OPENAI_DEPLOYMENT or specify name-value argument ""Deployment""."; catalog("llms:keyMustBeSpecified") = "Unable to find API key. Either set environment variable {1} or specify name-value argument ""APIKey""."; -catalog("llms:mustHaveMessages") = "Value must contain at least one message in Messages."; +catalog("llms:mustHaveMessages") = "Message history must not be empty."; catalog("llms:mustSetFunctionsForCall") = "When no functions are defined, ToolChoice must not be specified."; -catalog("llms:mustBeMessagesOrTxt") = "Messages must be text with one or more characters or a messageHistory object."; -catalog("llms:invalidOptionAndValueForModel") = "'{1}' with value '{2}' is not supported for ModelName '{3}'"; -catalog("llms:invalidOptionForModel") = "{1} is not supported for ModelName '{2}'"; -catalog("llms:invalidContentTypeForModel") = "{1} is not supported for ModelName '{2}'"; -catalog("llms:functionNotAvailableForModel") = "This function is not supported for ModelName '{1}'"; -catalog("llms:promptLimitCharacter") = "Prompt must have a maximum length of {1} characters for ModelName '{2}'"; -catalog("llms:pngExpected") = "Argument must be a PNG image."; +catalog("llms:mustBeMessagesOrTxt") = "Message must be nonempty string, character array, cell array of character vectors, or messageHistory object."; +catalog("llms:invalidOptionAndValueForModel") = "'{1}' with value '{2}' is not supported for model ""{3}""."; +catalog("llms:invalidOptionForModel") = "Invalid argument name {1} for model ""{2}""."; +catalog("llms:invalidContentTypeForModel") = "{1} is not supported for model ""{2}""."; +catalog("llms:functionNotAvailableForModel") = "Image editing is not supported for model ""{1}""."; +catalog("llms:promptLimitCharacter") = "Prompt must contain at most {1} characters for model ""{2}""."; +catalog("llms:pngExpected") = "Image must be a PNG file (*.png)."; catalog("llms:warningJsonInstruction") = "When using JSON mode, you must also prompt the model to produce JSON yourself via a system or user message."; -catalog("llms:apiReturnedError") = "Server error: ""{1}"""; +catalog("llms:apiReturnedError") = "Server returned error indicating: ""{1}"""; catalog("llms:dimensionsMustBeSmallerThan") = "Dimensions must be less than or equal to {1}."; catalog("llms:stream:responseStreamer:InvalidInput") = "Input does not have the expected json format, got ""{1}""."; end diff --git a/+llms/+utils/mustBeValidStop.m b/+llms/+utils/mustBeValidStop.m index f3862c7..7301a2c 100644 --- a/+llms/+utils/mustBeValidStop.m +++ b/+llms/+utils/mustBeValidStop.m @@ -5,6 +5,7 @@ function mustBeValidStop(value) if ~isempty(value) mustBeVector(value); mustBeNonzeroLengthText(value); + value = string(value); % This restriction is set by the OpenAI API if numel(value)>4 error("llms:stopSequencesMustHaveMax4Elements", llms.utils.errorMessageCatalog.getMessage("llms:stopSequencesMustHaveMax4Elements")); diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..4420e9a --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,34 @@ +#!/bin/bash + +cd $(git rev-parse --show-toplevel) +pwd + +# For all commits of mlx files, create corresponding Markdown (md) files. +# If the mlx files are in .../mlx-scripts/*.mlx, the corresponding +# md files will go into .../*.md. +# +# This script assumes that the mlx files as currently in the file system +# are what is being committed, instead of doing a lot of extra work to +# get them from the stage area. +# +# Note that this script will not remove media files. If an mlx has +# fewer plots at some point in the future, there will be file system +# cruft. Which doesn't hurt the md display in GitHub or elswehere. +changedMlxFiles=`git diff --cached --name-only --diff-filter=d '*.mlx'` + +if [ -n "$changedMlxFiles" ]; then + # Keep the line break here, we replace end-of-line with "' '" to get the quotes right + matlab -batch "for file = {'${changedMlxFiles// +/' '}'}, export(file{1},replace(erase(file{1},'mlx-scripts'),'.mlx','.md')); end" + tmp=${changedMlxFiles//mlx-scripts\//} + mdFiles=${tmp//.mlx/.md} + for file in $mdFiles; do + if [ -d ${file%.md}_media ]; then + git add ${file%.md}_media/ + fi + perl -pi -e "\$cnt++ if /^#/; " \ + -e "\$_ .= \"\nTo run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/$(basename $file .md).mlx](mlx-scripts/$(basename $file .md).mlx) \n\" if /^#/ && \$cnt==1;" \ + $file + done + git add $mdFiles +fi diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..cf5aa55 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,4 @@ +# Code owners, to get auto-filled reviewer lists + +# To start with, we just assume everyone in the core team is included on all reviews +* @adulai @ccreutzi @debymf @MiriamScharnke @vpapanasta \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70a7d6c..6be9f48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,7 @@ jobs: - name: Pull models run: | ollama pull mistral + ollama pull moondream OLLAMA_HOST=127.0.0.1:11435 ollama pull qwen2:0.5b - name: Set up MATLAB uses: matlab-actions/setup-matlab@v2 diff --git a/.gitignore b/.gitignore index 98963a6..510c487 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,12 @@ *.env *.asv *.mat +!tests/recordings/*.mat startup.m papers_to_read.csv data/* examples/data/* +examples/mlx-scripts/data/* ._* .nfs* .DS_Store diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..0fbac1d --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,11 @@ +# Notes for Developers + +Nothing in this file should be required knowledge to use the repository. These are notes for people actually making changes that are going to be submitted and incorporated into the main branch. + +## Git Hooks + +After checkout, link or (on Windows) copy the files from `.githooks` into the local `.git/hooks` folder: + +``` +(cd .git/hooks/; ln -s ../../.githooks/pre-commit .) +``` diff --git a/README.md b/README.md index 4bcf377..dd2ecb4 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# Large Language Models (LLMs) with MATLAB® +# Large Language Models (LLMs) with MATLAB [![Open in MATLAB Online](https://www.mathworks.com/images/responsive/global/open-in-matlab-online.svg)](https://matlab.mathworks.com/open/github/v1?repo=matlab-deep-learning/llms-with-matlab) [![View Large Language Models (LLMs) with MATLAB on File Exchange](https://www.mathworks.com/matlabcentral/images/matlab-file-exchange.svg)](https://www.mathworks.com/matlabcentral/fileexchange/163796-large-language-models-llms-with-matlab) -This repository contains code to connect MATLAB to the [OpenAI™ Chat Completions API](https://platform.openai.com/docs/guides/text-generation/chat-completions-api) (which powers ChatGPT™), OpenAI Images API (which powers DALL·E™), [Azure® OpenAI Service](https://learn.microsoft.com/en-us/azure/ai-services/openai/), and both local and nonlocal [Ollama™](https://ollama.com/) models. This allows you to leverage the natural language processing capabilities of large language models directly within your MATLAB environment. +This repository contains code to connect MATLAB® to the [OpenAI® Chat Completions API](https://platform.openai.com/docs/guides/text-generation/chat-completions-api) (which powers ChatGPT™), OpenAI Images API (which powers DALL·E™), [Azure® OpenAI Service](https://learn.microsoft.com/en-us/azure/ai-services/openai/), and both local and nonlocal [Ollama™](https://ollama.com/) models. This allows you to leverage the natural language processing capabilities of large language models directly within your MATLAB environment. ## Requirements @@ -52,16 +52,16 @@ To use this repository with a local installation of MATLAB, first clone the repo ## Examples To learn how to use this in your workflows, see [Examples](/examples/). -- [ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.mlx](/examples/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.mlx): Learn to implement a simple chat that stream the response. -- [SummarizeLargeDocumentsUsingChatGPTandMATLAB.mlx](/examples/SummarizeLargeDocumentsUsingChatGPTandMATLAB.mlx): Learn to create concise summaries of long texts with ChatGPT. (Requires Text Analytics Toolbox™) -- [CreateSimpleChatBot.mlx](/examples/CreateSimpleChatBot.mlx): Build a conversational chatbot capable of handling various dialogue scenarios using ChatGPT. (Requires Text Analytics Toolbox) -- [AnalyzeScientificPapersUsingFunctionCalls.mlx](/examples/AnalyzeScientificPapersUsingFunctionCalls.mlx): Learn how to create agents capable of executing MATLAB functions. -- [AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mlx](/examples/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mlx): Learn how to take advantage of parallel function calling. -- [RetrievalAugmentedGenerationUsingChatGPTandMATLAB.mlx](/examples/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.mlx): Learn about retrieval augmented generation with a simple use case. (Requires Text Analytics Toolbox™) -- [DescribeImagesUsingChatGPT.mlx](/examples/DescribeImagesUsingChatGPT.mlx): Learn how to use GPT-4 Turbo with Vision to understand the content of an image. -- [AnalyzeSentimentinTextUsingChatGPTinJSONMode.mlx](/examples/AnalyzeSentimentinTextUsingChatGPTinJSONMode.mlx): Learn how to use JSON mode in chat completions -- [UsingDALLEToEditImages.mlx](/examples/UsingDALLEToEditImages.mlx): Learn how to generate images -- [UsingDALLEToGenerateImages.mlx](/examples/UsingDALLEToGenerateImages.mlx): Create variations of images and editimages. +- [ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.md](/examples/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.md): Learn to implement a simple chat that stream the response. +- [SummarizeLargeDocumentsUsingChatGPTandMATLAB.md](/examples/SummarizeLargeDocumentsUsingChatGPTandMATLAB.md): Learn to create concise summaries of long texts with ChatGPT. (Requires Text Analytics Toolbox™) +- [CreateSimpleChatBot.md](/examples/CreateSimpleChatBot.md): Build a conversational chatbot capable of handling various dialogue scenarios using ChatGPT. (Requires Text Analytics Toolbox) +- [AnalyzeScientificPapersUsingFunctionCalls.md](/examples/AnalyzeScientificPapersUsingFunctionCalls.md): Learn how to create agents capable of executing MATLAB functions. +- [AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.md](/examples/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.md): Learn how to take advantage of parallel function calling. +- [RetrievalAugmentedGenerationUsingChatGPTandMATLAB.md](/examples/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.md): Learn about retrieval augmented generation with a simple use case. (Requires Text Analytics Toolbox™) +- [DescribeImagesUsingChatGPT.md](/examples/DescribeImagesUsingChatGPT.md): Learn how to use GPT-4 Turbo with Vision to understand the content of an image. +- [AnalyzeSentimentinTextUsingChatGPTinJSONMode.md](/examples/AnalyzeSentimentinTextUsingChatGPTinJSONMode.md): Learn how to use JSON mode in chat completions +- [UsingDALLEToEditImages.md](/examples/UsingDALLEToEditImages.md): Learn how to generate images +- [UsingDALLEToGenerateImages.md](/examples/UsingDALLEToGenerateImages.md): Create variations of images and editimages. ## License diff --git a/azureChat.m b/azureChat.m index 27bdded..b4b8df5 100644 --- a/azureChat.m +++ b/azureChat.m @@ -191,7 +191,7 @@ if isstring(messages) && isscalar(messages) messagesStruct = {struct("role", "user", "content", messages)}; else - messagesStruct = messages.Messages; + messagesStruct = this.encodeImages(messages.Messages); end if ~isempty(this.SystemPrompt) @@ -251,6 +251,40 @@ function mustBeValidFunctionCall(this, functionCall) end end + + function messageStruct = encodeImages(~, messageStruct) + for k=1:numel(messageStruct) + if isfield(messageStruct{k},"images") + images = messageStruct{k}.images; + detail = messageStruct{k}.image_detail; + messageStruct{k} = rmfield(messageStruct{k},["images","image_detail"]); + messageStruct{k}.content = ... + {struct("type","text","text",messageStruct{k}.content)}; + for img = images(:).' + if startsWith(img,("https://"|"http://")) + s = struct( ... + "type","image_url", ... + "image_url",struct("url",img)); + else + [~,~,ext] = fileparts(img); + MIMEType = "data:image/" + erase(ext,".") + ";base64,"; + % Base64 encode the image using the given MIME type + fid = fopen(img); + im = fread(fid,'*uint8'); + fclose(fid); + b64 = matlab.net.base64encode(im); + s = struct( ... + "type","image_url", ... + "image_url",struct("url",MIMEType + b64)); + end + + s.image_url.detail = detail; + + messageStruct{k}.content{end+1} = s; + end + end + end + end end end diff --git a/doc/Azure.md b/doc/Azure.md index d5af221..6d9f79e 100644 --- a/doc/Azure.md +++ b/doc/Azure.md @@ -1,6 +1,6 @@ -# Connecting to Azure® OpenAI Service +# Connecting to Azure OpenAI Service -This repository contains code to connect MATLAB to the [Azure® OpenAI Service](https://learn.microsoft.com/en-us/azure/ai-services/openai/). +This repository contains code to connect MATLAB to the [Azure® OpenAI® Service](https://learn.microsoft.com/en-us/azure/ai-services/openai/). To use Azure OpenAI Services, you need to create a model deployment on your Azure account and obtain one of the keys for it. You are responsible for any fees Azure may charge for the use of their APIs. You should be familiar with the limitations and risks associated with using this technology, and you agree that you shall be solely responsible for full compliance with any terms that may apply to your use of the Azure APIs. @@ -31,7 +31,7 @@ loadenv(".env") ## Establishing a connection to Chat Completions API using Azure -To connect MATLAB to Chat Completions API via Azure, you will have to create an `azureChat` object. See [the Azure documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/chatgpt-quickstart) for details on the setup required and where to find your key, endpoint, and deployment name. As explained above, the endpoint, deployment, and key should be in the environment variables `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_DEPLOYMENYT`, and `AZURE_OPENAI_API_KEY`, or provided as `Endpoint=…`, `Deployment=…`, and `APIKey=…` in the `azureChat` call below. +To connect MATLAB® to Chat Completions API via Azure, you will have to create an `azureChat` object. See [the Azure documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/chatgpt-quickstart) for details on the setup required and where to find your key, endpoint, and deployment name. As explained above, the endpoint, deployment, and key should be in the environment variables `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_DEPLOYMENYT`, and `AZURE_OPENAI_API_KEY`, or provided as `Endpoint=…`, `Deployment=…`, and `APIKey=…` in the `azureChat` call below. In order to create the chat assistant, use the `azureChat` function, optionally providing a system prompt: ```matlab @@ -115,6 +115,19 @@ txt = generate(chat,"What is Model-Based Design and how is it related to Digital % Should stream the response token by token ``` +## Understanding the content of an image + +You can use gpt-4o, gpt-4o-mini, or gpt-4-turbo to experiment with image understanding. +```matlab +chat = azureChat("You are an AI assistant.",Deployment="gpt-4o"); +image_path = "peppers.png"; +messages = messageHistory; +messages = addUserMessageWithImages(messages,"What is in the image?",image_path); +[txt,response] = generate(chat,messages,MaxNumTokens=4096); +txt +% outputs a description of the image +``` + ## Calling MATLAB functions with the API Optionally, `Tools=functions` can be used to provide function specifications to the API. The purpose of this is to enable models to generate function arguments which adhere to the provided specifications. diff --git a/doc/Ollama.md b/doc/Ollama.md index 110a869..a6bf2b2 100644 --- a/doc/Ollama.md +++ b/doc/Ollama.md @@ -1,6 +1,6 @@ # Ollama -This repository contains code to connect MATLAB to an [Ollama™](https://ollama.com) server, running large language models (LLMs). +This repository contains code to connect MATLAB® to an [Ollama™](https://ollama.com) server, running large language models (LLMs). To use local models with Ollama, you will need to install and start an Ollama server, and “pull” models into it. Please follow the Ollama documentation for details. You should be familiar with the limitations and risks associated with using this technology, and you agree that you shall be solely responsible for full compliance with any terms that may apply to your use of any specific model. @@ -96,6 +96,24 @@ txt = generate(chat,"What is Model-Based Design and how is it related to Digital % Should stream the response token by token ``` +## Understanding the content of an image + +You can use multimodal models like `llava` to experiment with image understanding. + +> [!TIP] +> Many models available for Ollama allow you to include images in the prompt, even if the model does not support image inputs. In that case, the images are silently removed from the input. This can result in unexpected outputs. + + +```matlab +chat = ollamaChat("llava"); +image_path = "peppers.png"; +messages = messageHistory; +messages = addUserMessageWithImages(messages,"What is in the image?",image_path); +[txt,response] = generate(chat,messages,MaxNumTokens=4096); +txt +% outputs a description of the image +``` + ## Establishing a connection to remote LLMs using Ollama To connect to a remote Ollama server, use the `Endpoint` name-value pair. Include the server name and port number. Ollama starts on 11434 by default. diff --git a/doc/OpenAI.md b/doc/OpenAI.md index 51783ae..08bb501 100644 --- a/doc/OpenAI.md +++ b/doc/OpenAI.md @@ -1,6 +1,6 @@ -# OpenAI™ +# OpenAI -Several functions in this repository connect MATLAB to the [OpenAI™ Chat Completions API](https://platform.openai.com/docs/guides/text-generation/chat-completions-api) (which powers ChatGPT™) and the [OpenAI Images API](https://platform.openai.com/docs/guides/images/image-generation-beta) (which powers DALL·E™). +Several functions in this repository connect MATLAB® to the [OpenAI® Chat Completions API](https://platform.openai.com/docs/guides/text-generation/chat-completions-api) (which powers ChatGPT™) and the [OpenAI Images API](https://platform.openai.com/docs/guides/images/image-generation-beta) (which powers DALL·E™). To start using the OpenAI APIs, you first need to obtain OpenAI API keys. You are responsible for any fees OpenAI may charge for the use of their APIs. You should be familiar with the limitations and risks associated with using this technology, and you agree that you shall be solely responsible for full compliance with any terms that may apply to your use of the OpenAI APIs. @@ -250,14 +250,15 @@ You can extract the arguments and write the data to a table, for example. ## Understanding the content of an image -You can use gpt-4-turbo to experiment with image understanding. +You can use gpt-4o, gpt-4o-mini, or gpt-4-turbo to experiment with image understanding. ```matlab -chat = openAIChat("You are an AI assistant.", ModelName="gpt-4-turbo"); +chat = openAIChat("You are an AI assistant."); image_path = "peppers.png"; messages = messageHistory; messages = addUserMessageWithImages(messages,"What is in the image?",image_path); [txt,response] = generate(chat,messages,MaxNumTokens=4096); -% Should output the description of the image +txt +% outputs a description of the image ``` ## Obtaining embeddings diff --git a/doc/functions/openAIChat.md b/doc/functions/openAIChat.md new file mode 100644 index 0000000..6a55c03 --- /dev/null +++ b/doc/functions/openAIChat.md @@ -0,0 +1,250 @@ + +# openAIChat + +Connect to OpenAI® Chat Completion API from MATLAB® + +# Creation +## Syntax + +`chat = openAIChat` + + +`chat = openAIChat(systemPrompt)` + + +`chat = openAIChat(___,APIKey=key)` + + +`chat = openAIChat(___,Name=Value)` + + +## Description +Connect to the OpenAI Chat Completion API to generate text using large language models developed by OpenAI. + + +To connect to the OpenAI API, you need a valid API key. For information on how to obtain an API key, see [https://platform.openai.com/docs/quickstart](https://platform.openai.com/docs/quickstart). + + +`chat = openAIChat` creates an `openAIChat` object. Connecting to the OpenAI API requires a valid API key. Either set the environment variable `OPENAI_API_KEY` or specify the `APIKey` name\-value argument. + + +`chat = openAIChat(systemPrompt)` creates an `openAIChat` object with the specified system prompt. + + +`chat = openAIChat(___,APIKey=key)` uses the specified API key. + + +`chat = openAIChat(___,Name=Value)` specifies additional options using one or more name\-value arguments. + + +# Input Arguments + +### `systemPrompt` – System prompt + +character vector | string scalar + + +Specify the system prompt and set the `SystemPrompt` property. The system prompt is a natural\-language description that provides the framework in which a large language model generates its responses. The system prompt can include instructions about tone, communications style, language, etc. + + +**Example**: "You are a helpful assistant who provides answers to user queries in iambic pentameter." + +## Name\-Value Arguments + +### `APIKey` – OpenAI API key + +character vector | string scalar + + +OpenAI API key to access OpenAI APIs such as ChatGPT. + + +Instead of using the `APIKey` name\-value argument, you can also set the environment variable OPEN\_API\_KEY. For more information, see [OpenAI API](../OpenAI.md). + +--- + +### `Tools` – OpenAI functions to use during output generation + +`openAIFunction` object | array of `openAIFunction` objects + + +Custom functions used by the model to collect or generate additional data. + +For an example, see [Analyze Scientific Papers Using ChatGPT Function Calls](../../examples/AnalyzeScientificPapersUsingFunctionCalls.md). + +# Properties Settable at Construction +Optionally specify these properties at construction using name-value arguments. Specify `PropertyName1=PropertyValue1,...,PropertyNameN=PropertyValueN`, where `PropertyName` is the property name and `PropertyValue` is the corresponding value. + +### `ModelName` – Model name + +`"gpt-4o-mini"` (default) | `"gpt-4"` | `"gpt-3.5-turbo"` | `"dall-e-2"` | ... + + +Name of the OpenAI model to use for text or image generation. + + +For a list of currently supported models, see [OpenAI API](../OpenAI.md). + +--- + +### `Temperature` – Temperature + +`1` (default) | numeric scalar between `0` and `2` + + +Temperature value for controlling the randomness of the output. Higher temperature increases the randomness of the output. Setting the temperature to `0` results in fully deterministic output. + +--- + +### `TopP` – Top probability mass + +`1` (default) | numeric scalar between `0` and `1` + + +Top probability mass for controlling the diversity of the generated output using top-p sampling. Higher top probability mass corresponds to higher diversity. + +--- + +### `StopSequences` – Stop sequences + +`[]` (default) | string array with between `0` and `4` elements + + +Sequences that stop generation of tokens. + + +**Example:** `["The end.","And that is all she wrote."]` + +--- + +### `PresencePenalty` – Presence penalty + +`0` (default) | numeric scalar between `-2` and `2` + + +Penalty value for using a token that has already been used at least once in the generated output. Higher values reduce the repetition of tokens. Negative values increase the repetition of tokens. + + +The presence penalty is independent of the number of incidents of a token, so long as it has been used at least once. To increase the penalty for every additional time a token is generated, use the `FrequencyPenalty` name\-value argument. + +--- + +### `FrequencyPenalty` – Frequency penalty + +`0` (default) | numeric scalar between `-2` and `2` + + +Penalty value for repeatedly using the same token in the generated output. Higher values reduce the repetition of tokens. Negative values increase the repetition of tokens. + + +The frequence penalty increases with every instance of a token in the generated output. To use a constant penalty for a repeated token, independent of the number of instances that token is generated, use the `PresencePenalty` name\-value argument. + +--- + +### `TimeOut` – Connection timeout in seconds + +`10` (default) | nonnegative numeric scalar + +After construction, this property is read-only. + +If the OpenAI server does not respond within the timeout, then the function throws an error. + +--- + +### `StreamFun` – Custom streaming function + +function handle + + +Specify a custom streaming function to process the generated output token by token as it is being generated, rather than having to wait for the end of the generation. For example, you can use this function to print the output as it is generated. + +For an example, see [Process Generated Text in Real Time by Using ChatGPT™ in Streaming Mode](../../examples/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.md). + + +**Example:** `@(token) fprint("%s",token)` + +--- + +### `ResponseFormat` – Response format + +`"text"` (default) | `"json"` + + +After construction, this property is read\-only. + + +Format of generated output. + + +If the response format is `"text"`, then the generated output is a string. + + +If the response format is `"json"`, then the generated output is a string containing JSON encoded data. + + +To configure the format of the generated JSON file, describe the format using natural language and provide it to the model either in the system prompt or as a user message. The prompt or message describing the format must contain the word `"json"` or `"JSON"`. + + +For an example, see [Analyze Sentiment in Text Using ChatGPT in JSON Mode](../../examples/AnalyzeSentimentinTextUsingChatGPTinJSONMode.md). + + +The JSON response format is not supported for these models: + +- `ModelName="gpt-4"` +- `ModelName="gpt-4-0613"` + + +# Other Properties + +### `SystemPrompt` – System prompt + +character vector | string scalar + + +This property is read\-only. + + +The system prompt is a natural\-language description that provides the framework in which a large language model generates its responses. The system prompt can include instructions about tone, communications style, language, etc. + +Specify the `SystemPrompt` property at construction using the `systemPrompt` input argument. + + +**Example**: "You are a helpful assistant who provides answers to user queries in iambic pentameter." + +--- + +### `FunctionNames` – Names of OpenAI functions to use during output generation + +string array + + +This property is read\-only. + + +Names of the custom functions specified in the `Tools` name\-value argument. + + +# Object Functions + +`generate` – Generate output from large language models + +# Examples +## Create OpenAI Chat +```matlab +modelName = "gpt-4o-mini"; +chat = openAIChat("You are a helpful assistant awaiting further instructions.",ModelName=modelName); +``` + +## Generate and Stream Text +```matlab +sf = @(x) fprintf("%s",x); +chat = openAIChat(StreamFun=sf); +generate(chat,"Why is a raven like a writing desk?"); +``` +# See Also +- [Create Simple Chat Bot](../../examples/CreateSimpleChatBot.md) +- [Process Generated Text in Real Time Using ChatGPT in Streaming Mode](../../examples/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.md) +- [Analyze Scientific Papers Using Function Calls](../../examples/AnalyzeScientificPapersUsingFunctionCalls.md) +- [Analyze Sentiment in Text Using ChatGPT in JSON Mode](../../examples/AnalyzeSentimentinTextUsingChatGPTinJSONMode.md) + +*Copyright 2024 The MathWorks, Inc.* \ No newline at end of file diff --git a/examples/AnalyzeScientificPapersUsingFunctionCalls.md b/examples/AnalyzeScientificPapersUsingFunctionCalls.md new file mode 100644 index 0000000..72c96f0 --- /dev/null +++ b/examples/AnalyzeScientificPapersUsingFunctionCalls.md @@ -0,0 +1,158 @@ + +# Analyze Scientific Papers Using ChatGPT™ Function Calls + +To run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/AnalyzeScientificPapersUsingFunctionCalls.mlx](mlx-scripts/AnalyzeScientificPapersUsingFunctionCalls.mlx) + +This example shows how to extract recent scientific papers from ArXiv, summarize them using ChatGPT, and write the results to a CSV file using the `openAIFunction` function. + +- The example contains three steps: +- Define a custom function for ChatGPT to use to process its input and output. +- Extract papers from ArXiv. +- Use ChatGPT to assess whether a paper is relevant to your query, and to add an entry to the results table if so. + +To run this example, you need a valid API key from a paid OpenAI™ API account. + +```matlab +loadenv(".env") +addpath('../..') +``` +# Initialize OpenAI API Function and Chat + +Use `openAIFunction` to define functions that the model will be able to requests calls. + + +Set up the function to store paper details and initiate a chat with the OpenAI API with a defined role as a scientific paper expert. + + +Define the function that you want the model to have access to. In this example the used function is `writePaperDetails`. + +```matlab +f = openAIFunction("writePaperDetails", "Function to write paper details to a table."); +f = addParameter(f, "name", type="string", description="Name of the paper."); +f = addParameter(f, "url", type="string", description="URL containing the paper."); +f = addParameter(f, "explanation", type="string", description="Explanation on why the paper is related to the given topic."); + +paperVerifier = openAIChat("You are an expert in filtering scientific papers. " + ... + "Given a certain topic, you are able to decide if the paper" + ... + " fits the given topic or not."); + +paperExtractor = openAIChat("You are an expert in extracting information from a paper.", Tools=f); + +function writePaperDetails(name, url, desc) +filename = "papers_to_read.csv"; +T = table(name, url, desc, VariableNames=["Name", "URL", "Description"]); +writetable(T, filename, WriteMode="append"); +end +``` +# Extract Papers From ArXiv + +Specify the category of interest, the date range for the query, and the maximum number of results to retrieve from the ArXiv API. + +```matlab +category = "cs.CL"; +endDate = datetime("today", "Format","uuuuMMdd"); +startDate = datetime("today", "Format","uuuuMMdd") - 5; +maxResults = 40; +urlQuery = "https://export.arxiv.org/api/query?search_query=" + ... + "cat:" + category + ... + "&submittedDate=["+string(startDate)+"+TO+"+string(endDate)+"]"+... + "&max_results=" + maxResults + ... + "&sortBy=submittedDate&sortOrder=descending"; + +options = weboptions('Timeout',160); +code = webread(urlQuery,options); +``` + +Extract individual paper entries from the API response and use ChatGPT to determine whether each paper is related to the specified topic. + + +ChatGPT will parse the XML file, so we only need to extract the relevant entries. + +```matlab +entries = extractBetween(code, '', ''); +``` +# Write Relevant Information to Table + +Create empty file and determine the topic of interest. + +```matlab +filename = "papers_to_read.csv"; +T = table([], [], [], VariableNames=["Name", "URL", "Description"]); +writetable(T, filename); + +topic = "Large Language Models"; +``` + +Loop over the entries and see if they are relevant to the topic of interest. + +```matlab +for i = 1:length(entries) + prompt = "Given the following paper:" + newline +... + string(entries{i})+ newline +... + "Is it related to the topic: "+ topic +"?" + ... + " Answer 'yes' or 'no'."; + [text, response] = generate(paperVerifier, prompt); + +``` + +If the model classifies this entry as relevant, then it tries to request a function call. + +```matlab + if contains("yes", text, IgnoreCase=true) + prompt = "Given the following paper:" + newline + string(entries{i})+ newline +... + "Given the topic: "+ topic + newline + "Write the details to a table."; + [text, response] = generate(paperExtractor, prompt); +``` + +If `function_call` if part of the response, it means the model is requesting a function call. The function call request should contain the needed arguments to call the function specified at the end of this example and defined with `openAIFunctions`. + +```matlab + if isfield(response, "tool_calls") + funCall = response.tool_calls; + functionCallAttempt(funCall); + end + end +end +``` + +Read the generated file. + +```matlab +data = readtable("papers_to_read.csv", Delimiter=",") +``` +# Helper Function + +This function handles function call attempts from the model, checking the function name and arguments before calling the appropriate function to store the paper details. + +```matlab +function functionCallAttempt(funCall) +``` + +The model can sometimes hallucinate function names, so you need to ensure that it's suggesting the correct name. + +```matlab +if funCall.function.name == "writePaperDetails" + try +``` + +The model can sometimes return improperly formed JSON, which needs to be handled. + +```matlab + funArgs = jsondecode(funCall.function.arguments); + catch ME + error("Model returned improperly formed JSON."); + end +``` + +The model can hallucinate arguments. The code needs to ensure the arguments have been defined before calling the function. + +```matlab + if isfield(funArgs, "name") && isfield(funArgs, "url") && isfield(funArgs,"explanation") + writePaperDetails(string(funArgs.name), string(funArgs.url), string(funArgs.explanation)); + end +end +end +``` + +*Copyright 2023\-2024 The MathWorks, Inc.* + diff --git a/examples/AnalyzeScientificPapersUsingFunctionCalls.mlx b/examples/AnalyzeScientificPapersUsingFunctionCalls.mlx deleted file mode 100644 index f9aa0c9..0000000 Binary files a/examples/AnalyzeScientificPapersUsingFunctionCalls.mlx and /dev/null differ diff --git a/examples/AnalyzeSentimentinTextUsingChatGPTinJSONMode.md b/examples/AnalyzeSentimentinTextUsingChatGPTinJSONMode.md new file mode 100644 index 0000000..303ffe9 --- /dev/null +++ b/examples/AnalyzeSentimentinTextUsingChatGPTinJSONMode.md @@ -0,0 +1,88 @@ + +# Analyze Sentiment in Text Using ChatGPT™ in JSON Mode + +To run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/AnalyzeSentimentinTextUsingChatGPTinJSONMode.mlx](mlx-scripts/AnalyzeSentimentinTextUsingChatGPTinJSONMode.mlx) + +This example shows how to use ChatGPT for sentiment analysis and output the results in JSON format. + + +To run this example, you need a valid API key from a paid OpenAI™ API account. + +```matlab +loadenv(".env") +addpath('../..') +``` + +Define some text to analyze the sentiment. + +```matlab +inputText = ["I can't stand homework."; + "This sucks. I'm bored."; + "I can't wait for Halloween!!!"; + "I am neigher for or against the idea."; + "My cat is adorable ❤️❤️"; + "I hate chocolate"]; +``` + +Define the expected output JSON Schema. + +```matlab +jsonSchema = '{"sentiment": "string (positive, negative, neutral)","confidence_score": "number (0-1)"}'; +``` + +Define the the system prompt, combining your instructions and the JSON Schema. In order for the model to output JSON, you need to specify that in the prompt, for example, adding *"designed to output to JSON"* to the prompt. + +```matlab +systemPrompt = "You are an AI designed to output to JSON. You analyze the sentiment of the provided text and " + ... + "Determine whether the sentiment is positive, negative, or neutral and provide a confidence score using " + ... + "the schema: " + jsonSchema; +prompt = "Analyze the sentiment of the provided text. " + ... + "Determine whether the sentiment is positive, negative," + ... + " or neutral and provide a confidence score"; +``` + +Create a chat object with `ModelName gpt-3.5-turbo` and specify `ResponseFormat` as `"json".` + +```matlab +model = "gpt-3.5-turbo"; +chat = openAIChat(systemPrompt, ModelName=model, ResponseFormat="json"); +``` + +```matlabTextOutput +Warning: When using JSON mode, you must also prompt the model to produce JSON yourself via a system or user message. +``` + +Concatenate the prompt and input text and generate an answer with the model. + +```matlab +scores = cell(1,numel(inputText)); +for i = 1:numel(inputText) +``` + +Generate a response from the message. + +```matlab + [json, message, response] = generate(chat,prompt + newline + newline + inputText(i)); + scores{i} = jsondecode(json); +end +``` + +Extract the data from the JSON ouput. + +```matlab +T = struct2table([scores{:}]); +T.text = inputText; +T = movevars(T,"text","Before","sentiment") +``` +| |text|sentiment|confidence_score| +|:--:|:--:|:--:|:--:| +|1|"I can't stand homework."|'negative'|0.9500| +|2|"This sucks. I'm bored."|'negative'|0.9000| +|3|"I can't wait for Halloween!!!"|'positive'|0.9500| +|4|"I am neigher for or against the idea."|'neutral'|1| +|5|"My cat is adorable ❤️❤️"|'positive'|0.9500| +|6|"I hate chocolate"|'negative'|0.9000| + + +*Copyright 2024 The MathWorks, Inc.* + diff --git a/examples/AnalyzeSentimentinTextUsingChatGPTinJSONMode.mlx b/examples/AnalyzeSentimentinTextUsingChatGPTinJSONMode.mlx deleted file mode 100644 index 8fc5a63..0000000 Binary files a/examples/AnalyzeSentimentinTextUsingChatGPTinJSONMode.mlx and /dev/null differ diff --git a/examples/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.md b/examples/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.md new file mode 100644 index 0000000..2fcfa74 --- /dev/null +++ b/examples/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.md @@ -0,0 +1,218 @@ + +# Analyze Text Data Using Parallel Function Calls with ChatGPT™ + +To run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mlx](mlx-scripts/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mlx) + +This example shows how to detect multiple function calls in a single user prompt and use this to extract information from text data. + + +Function calls allow you to describe a function to ChatGPT in a structured way. When you pass a function to the model together with a prompt, the model detects how often the function needs to be called in the context of the prompt. If the function is called at least once, then the model creates a JSON object containing the function and argument to request. + + +This example contains four steps: + +- Create an unstructured text document containing fictional customer data. +- Create a prompt, asking ChatGPT to extract information about different types customers. +- Create a function that extracts information about customers. +- Combine the prompt and the function. Use ChatGPT to detect how many function calls are included in the prompt and to generate a JSON object with the function outputs. + +To run this example, you need a valid API key from a paid OpenAI™ API account. + +```matlab +loadenv(".env") +addpath('../..') +``` +# Extracting data from text + +The customer record contains fictional information. + +```matlab +record = ["Customer John Doe, 35 years old. Email: johndoe@email.com"; + "Jane Smith, age 28. Email address: janesmith@email.com"; + "Customer named Alex Lee, 29, with email alexlee@email.com"; + "Evelyn Carter, 32, email: evelyncarter32@email.com"; + "Jackson Briggs is 45 years old. Contact email: jacksonb45@email.com"; + "Aria Patel, 27 years old. Email contact: apatel27@email.com"; + "Liam Tanaka, aged 28. Email: liam.tanaka@email.com"; + "Sofia Russo, 24 years old, email: sofia.russo124@email.com"]; +``` + +Define the function that extract data from the customer record. + +```matlab +f = openAIFunction("extractCustomerData", "Extracts data from customer records"); +f = addParameter(f, "name", type="string", description="customer name", RequiredParameter=true); +f = addParameter(f, "age", type="number", description="customer age"); +f = addParameter(f, "email", type="string", description="customer email", RequiredParameter=true); +``` + +Create a message with the customer record and an instruction. + +```matlab +record = join(record); +messages = messageHistory; +messages = addUserMessage(messages,"Extract data from the record: " + record); +``` + +Create a chat object with a latest model. Only the new models support the parallel function calling. + +```matlab +model = "gpt-3.5-turbo"; +chat = openAIChat("You are an AI assistant designed to extract customer data.","ModelName",model,Tools=f); +``` + +Generate a response and extract the data. + +```matlab +[~, singleMessage, response] = generate(chat,messages); +if response.StatusCode == "OK" + funcData = [singleMessage.tool_calls.function]; + extractedData = arrayfun(@(x) jsondecode(x.arguments), funcData); + extractedData = struct2table(extractedData) +else + response.Body.Data.error +end +``` +| |name|age|email| +|:--:|:--:|:--:|:--:| +|1|'John Doe'|35|'johndoe@email.com'| +|2|'Jane Smith'|28|'janesmith@email.com'| +|3|'Alex Lee'|29|'alexlee@email.com'| +|4|'Evelyn Carter'|32|'evelyncarter32@email.com'| +|5|'Jackson Briggs'|45|'jacksonb45@email.com'| +|6|'Aria Patel'|27|'apatel27@email.com'| +|7|'Liam Tanaka'|28|'liam.tanaka@email.com'| +|8|'Sofia Russo'|24|'sofia.russo124@email.com'| + +# Calling an external function + +In this example, use a local function `searchCustomerData` defined at the end of this script. + + +Create a message with the information you would like to get from the local function. + +```matlab +prompt = "Who are our customers under 30 and older than 27?"; +messages = messageHistory; +messages = addUserMessage(messages,prompt); +``` + +Define the function that retrieves weather information for a given city based on the local function. + +```matlab +f = openAIFunction("searchCustomerData", "Get the customers who match the specified and age"); +f = addParameter(f,"minAge",type="integer",description="The minimum customer age",RequiredParameter=true); +f = addParameter(f,"maxAge",type="integer",description="The maximum customer age",RequiredParameter=true); +``` + +Create a chat object with a latest model. + +```matlab +model = "gpt-3.5-turbo"; +chat = openAIChat("You are an AI assistant designed to search customer data.",ModelName=model,Tools=f); +``` + +Generate a response + +```matlab +[~, singleMessage, response] = generate(chat,messages); +``` + +Check if the response contains a request for tool calling. + +```matlab +if isfield(singleMessage,'tool_calls') + tcalls = singleMessage.tool_calls +else + response.Body.Data.error +end +``` + +```matlabTextOutput +tcalls = struct with fields: + id: 'call_JKENvUzMbNclCTI8GTBmY7YT' + type: 'function' + function: [1x1 struct] + +``` + +And add the tool call to the messages. + +```matlab +messages = addResponseMessage(messages,singleMessage); +``` + +Call `searchCustomerData` function and add the results to the messages + +```matlab +messages = processToolCalls(extractedData,messages, tcalls); +``` + +Step 3: extend the conversation with the function result. + +```matlab +[txt, singleMessage, response] = generate(chat,messages); +if response.StatusCode == "OK" + txt +else + response.Body.Data.error +end +``` + +```matlabTextOutput +txt = + "The customers who are under 30 and older than 27 are: +1. Jane Smith (age 28) + 2. Alex Lee (age 29) + 3. Liam Tanaka (age 28)" + +``` +# Helper functions + +Function that searches specific customer based on age range + +```matlab +function json = searchCustomerData(data, minAge, maxAge) + result = data(data.age >= minAge & data.age <= maxAge,:); + result = table2struct(result); + json = jsonencode(result); +end + +``` + +Function that uses the tool calls to execute the function and add the results to the messages + +```matlab +function msg = processToolCalls(data, msg, toolCalls) + for ii = 1:numel(toolCalls) + funcId = string(toolCalls(ii).id); + funcName = string(toolCalls(ii).function.name); + if funcName == "searchCustomerData" + funcArgs = jsondecode(toolCalls(ii).function.arguments); + keys = fieldnames(funcArgs); + vals = cell(size(keys)); + if ismember(keys,["minAge","maxAge"]) + for jj = 1:numel(keys) + vals{jj} = funcArgs.(keys{jj}); + end + try + funcResult = searchCustomerData(data,vals{:}); + catch ME + error(ME.message) + return + end + msg = addToolMessage(msg, funcId, funcName, funcResult); + else + error("Unknown arguments provided: " + join(keys)) + return + end + else + error("Unknown function called: " + funcName) + return + end + end +end +``` + +*Copyright 2024 The MathWorks, Inc.* + diff --git a/examples/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mlx b/examples/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mlx deleted file mode 100644 index 8cfd809..0000000 Binary files a/examples/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mlx and /dev/null differ diff --git a/examples/CreateSimpleChatBot.md b/examples/CreateSimpleChatBot.md new file mode 100644 index 0000000..97bd59a --- /dev/null +++ b/examples/CreateSimpleChatBot.md @@ -0,0 +1,141 @@ + +# Create Simple ChatBot + +To run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/CreateSimpleChatBot.mlx](mlx-scripts/CreateSimpleChatBot.mlx) + +This example shows how to create a simple chatbot using the `openAIChat` and `messageHistory` functions. + + +When you run this example, an interactive AI chat starts in the MATLAB® Command Window. To leave the chat, type "end" or press **Ctrl+C**. + +- This example includes three steps: +- Define model parameters, such as the maximum word count, and a stop word. +- Create an openAIChat object and set up a meta prompt. +- Set up the chat loop. + +To run this example, you need a valid API key from a paid OpenAI™ API account. + +```matlab +loadenv(".env") +addpath('../..') +``` +# Setup Model + +Set the maximum allowable number of words per chat session and define the keyword that, when entered by the user, ends the chat session. This example uses the model `gpt-3.5-turbo`. + +```matlab +wordLimit = 2000; +stopWord = "end"; +modelName = "gpt-3.5-turbo"; +``` + +Create an instance of `openAIChat` to perform the chat and `messageHistory` to store the conversation history`.` + +```matlab +chat = openAIChat("You are a helpful assistant. You reply in a very concise way, keeping answers limited to short sentences.", ModelName=modelName); +messages = messageHistory; +``` +# Chat loop + +Start the chat and keep it going until it sees the word in `stopWord`. + +```matlab +totalWords = 0; +messagesSizes = []; +``` + +The main loop continues indefinitely until you input the stop word or press **Ctrl+C.** + +```matlab +while true + query = input("User: ", "s"); + query = string(query); + disp("User: " + query) +``` + +If the you input the stop word, display a farewell message and exit the loop. + +```matlab + if query == stopWord + disp("AI: Closing the chat. Have a great day!") + break; + end + + numWordsQuery = countNumWords(query); +``` + +If the query exceeds the word limit, display an error message and halt execution. + +```matlab + if numWordsQuery>wordLimit + error("Your query should have less than 2000 words. You query had " + numWordsQuery + " words") + end +``` + +Keep track of the size of each message and the total number of words used so far. + +```matlab + messagesSizes = [messagesSizes; numWordsQuery]; %#ok + totalWords = totalWords + numWordsQuery; +``` + +If the total word count exceeds the limit, remove messages from the start of the session until it no longer does. + +```matlab + while totalWords > wordLimit + totalWords = totalWords - messagesSizes(1); + messages = removeMessage(messages, 1); + messagesSizes(1) = []; + end +``` + +Add the new message to the session and generate a new response. + +```matlab + messages = addUserMessage(messages, query); + [text, response] = generate(chat, messages); + + disp("AI: " + text) +``` + +Count the number of words in the response and update the total word count. + +```matlab + numWordsResponse = countNumWords(text); + messagesSizes = [messagesSizes; numWordsResponse]; %#ok + totalWords = totalWords + numWordsResponse; +``` + +Add the response to the session. + +```matlab + messages = addResponseMessage(messages, response); +end +``` + +```matlabTextOutput +User: Hello, how much do you know about physics? +AI: I am knowledgeable about various topics in physics. How can I assist you today? +User: What is torque? +AI: Torque is a measure of the rotational force applied to an object. +User: What is force? +AI: Force is a push or pull that causes an object to accelerate or change its motion. +User: What is motion? +AI: Motion is the change in position of an object over time in relation to a reference point. +User: What is time? +AI: Time is a measurable period during which an action, process, or condition exists or continues. +User: end +AI: Closing the chat. Have a great day! +``` +# `countNumWords` function + +Function to count the number of words in a text string + +```matlab +function numWords = countNumWords(text) + numWords = doclength(tokenizedDocument(text)); +end +``` + +*Copyright 2023\-2024 The MathWorks, Inc.* + diff --git a/examples/CreateSimpleChatBot.mlx b/examples/CreateSimpleChatBot.mlx deleted file mode 100644 index b3eb66b..0000000 Binary files a/examples/CreateSimpleChatBot.mlx and /dev/null differ diff --git a/examples/CreateSimpleOllamaChatBot.md b/examples/CreateSimpleOllamaChatBot.md new file mode 100644 index 0000000..d2603a8 --- /dev/null +++ b/examples/CreateSimpleOllamaChatBot.md @@ -0,0 +1,238 @@ + +# Create Simple ChatBot + +To run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/CreateSimpleOllamaChatBot.mlx](mlx-scripts/CreateSimpleOllamaChatBot.mlx) + +This example shows how to create a simple chatbot using the `ollamaChat` and `messageHistory` functions. + + +When you run this example, an interactive AI chat starts in the MATLAB® Command Window. To leave the chat, type "end" or press **Ctrl+C**. + +- This example includes three steps: +- Define model parameters, such as the maximum word count, and a stop word. +- Create an ollamaChat object and set up a meta prompt. +- Set up the chat loop. + +To run this example, you need a running Ollama™ installation. To run it unchanged, you need to have Mistral® pulled into that Ollama server. + +```matlab +loadenv(".env") +addpath('../..') +``` +# Setup Model + +Set the maximum allowable number of words per chat session and define the keyword that, when entered by the user, ends the chat session. + +```matlab +wordLimit = 2000; +stopWord = "end"; +``` + +Create an instance of `ollamaChat` to perform the chat and `messageHistory` to store the conversation history`.` + +```matlab +chat = ollamaChat("mistral"); +messages = messageHistory; +``` +# Chat loop + +Start the chat and keep it going until it sees the word in `stopWord`. + +```matlab +totalWords = 0; +messagesSizes = []; +``` + +The main loop continues indefinitely until you input the stop word or press **Ctrl+C.** + +```matlab +while true + query = input("User: ", "s"); + query = string(query); + dispWrapped("User", query) +``` + +If the you input the stop word, display a farewell message and exit the loop. + +```matlab + if query == stopWord + disp("AI: Closing the chat. Have a great day!") + break; + end + + numWordsQuery = countNumWords(query); +``` + +If the query exceeds the word limit, display an error message and halt execution. + +```matlab + if numWordsQuery>wordLimit + error("Your query should have fewer than " + wordLimit + " words. You query had " + numWordsQuery + " words.") + end +``` + +Keep track of the size of each message and the total number of words used so far. + +```matlab + messagesSizes = [messagesSizes; numWordsQuery]; %#ok + totalWords = totalWords + numWordsQuery; +``` + +If the total word count exceeds the limit, remove messages from the start of the session until it no longer does. + +```matlab + while totalWords > wordLimit + totalWords = totalWords - messagesSizes(1); + messages = removeMessage(messages, 1); + messagesSizes(1) = []; + end +``` + +Add the new message to the session and generate a new response. + +```matlab + messages = addUserMessage(messages, query); + [text, response] = generate(chat, messages); + + dispWrapped("AI", text) +``` + +Count the number of words in the response and update the total word count. + +```matlab + numWordsResponse = countNumWords(text); + messagesSizes = [messagesSizes; numWordsResponse]; %#ok + totalWords = totalWords + numWordsResponse; +``` + +Add the response to the session. + +```matlab + messages = addResponseMessage(messages, response); +end +``` + +```matlabTextOutput +User: Please help me with creating a Butterworth bandpass filter in MATLAB. +AI: Sure, I can help you create a Butterworth bandpass filter in MATLAB. + Here's an example of how you can do it: + +1. First, we need to define the low-pass and high-pass cutoff + frequencies for our bandpass filter. Let's say we want a bandpass + filter with a lower cutoff frequency of 0.5 Hz and an upper cutoff + frequency of 2 Hz. + + ```matlab + lowcut = 0.5; % Lower cutoff frequency (in Hertz) + highcut = 2; % Upper cutoff frequency (in Hertz) + ``` + + 2. Next, we define the order of the filter. The order determines how + steep the transition between pass and stop bands will be. A higher + order means a steeper roll-off but may also introduce more phase + distortion. For this example, let's use an order of 5. + + ```matlab + order = 5; % Order of the filter + ``` + + 3. Now we can create the bandpass Butterworth filter using the + `butter()` function in MATLAB. This function takes three arguments: + the normalized frequencies (in rad/sec), an 's' vector for the + low-pass and high-pass filters, and the order of the filter. + + ```matlab + [num, den] = butter(order, [lowcut/samplingFrequency, 1 - + lowcut/highcut, highcut/samplingFrequency]); + ``` + + In this line, `samplingFrequency` should be replaced with the + sampling frequency of your data. For example, if you are working with + data sampled at 100 Hz, you would set `samplingFrequency = 100;`. + + 4. Finally, you can apply the filter to a signal using the `filter()` + function in MATLAB. + + ```matlab + filteredSignal = filter(num, den, inputSignal); + ``` + + Replace `inputSignal` with your actual input data. + + This should give you a bandpass-filtered version of your signal. You + can adjust the cutoff frequencies and the order of the filter to best + suit your specific needs. +User: I would like the frequency range to be 144-146 MHz, assuming a sample + rate of 2MS/s. +AI: In that case, here is how you could define the cutoff frequencies: + + ```matlab + lowcut = (144e6)/(2e6); % Lower cutoff frequency (in rad/sec) + highcut = (146e6)/(2e6); % Upper cutoff frequency (in rad/sec) + ``` + + Now you can proceed with the filter creation as described in my + previous response. Make sure to replace `samplingFrequency` with the + actual sample rate of your data, which in this case is 2e6 Hz. + + ```matlab + [num, den] = butter(order, [lowcut, 1 - lowcut/(highcut), highcut]); + ``` +User: How do I export the parameters to use in GNU Radio? +AI: To export the Butterworth filter coefficients from MATLAB and use + them in GNU Radio, you can save the numerator and denominator + coefficients as separate text files. Here's how: + +1. Save the numerator coefficients: + + ```matlab + filename_num = 'numerator_coefficients.txt'; + fid_num = fopen(filename_num, 'w'); + for i = 1 : length(num) + fprintf(fid_num, '%g\n', num(i)); + end + fclose(fid_num); + ``` + + 2. Save the denominator coefficients: + + ```matlab + filename_den = 'denominator_coefficients.txt'; + fid_den = fopen(filename_den, 'w'); + for i = 1 : length(den) + fprintf(fid_den, '%g\n', den(i)); + end + fclose(fid_den); + ``` + + Now you have two text files with the numerator and denominator + coefficients. You can import these coefficient lists in GNU Radio + using Polyphase FIR Filter blocks or any other filter blocks that + allow specifying taps directly. In the GNU Radio GUI, you'll need to + set the number of taps based on the length of the coefficient lists + loaded from the text files. +User: end +AI: Closing the chat. Have a great day! +``` +# `Helper Functions` + +Function to count the number of words in a text string + +```matlab +function numWords = countNumWords(text) + numWords = doclength(tokenizedDocument(text)); +end +``` + +Function to display wrapped text, with hanging indentation from a prefix + +```matlab +function dispWrapped(prefix, text) + indent = [newline, repmat(' ',1,strlength(prefix)+2)]; + text = strtrim(text); + disp(prefix + ": " + join(textwrap(text, 70),indent)) +end +``` + +*Copyright 2023\-2024 The MathWorks, Inc.* + diff --git a/examples/CreateSimpleOllamaChatBot.mlx b/examples/CreateSimpleOllamaChatBot.mlx deleted file mode 100644 index b87d21b..0000000 Binary files a/examples/CreateSimpleOllamaChatBot.mlx and /dev/null differ diff --git a/examples/DescribeImagesUsingChatGPT.md b/examples/DescribeImagesUsingChatGPT.md new file mode 100644 index 0000000..edc996f --- /dev/null +++ b/examples/DescribeImagesUsingChatGPT.md @@ -0,0 +1,66 @@ + +# Describe Images Using ChatGPT™ + +To run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/DescribeImagesUsingChatGPT.mlx](mlx-scripts/DescribeImagesUsingChatGPT.mlx) + +This example shows how to generate image descriptions using the addUserMessageWithImages function. To run this example, you need a valid API key from a paid OpenAI™ API account, and a history of successful payment. + +```matlab +loadenv(".env") +addpath('../..') +``` +# Load and Display Image Data + +Load the sample image from Wikipedia. Use the `imread` function to read images from URLs or filenames. + +```matlab +image_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg'; +im = imread(image_url); +imshow(im) +``` + +![figure_0.png](DescribeImagesUsingChatGPT_media/figure_0.png) +# Generate Image Descriptions + +Ask questions about the image with the URL. + +```matlab +chat = openAIChat("You are an AI assistant."); +``` + +Create a message and pass the image URL along with the prompt. + +```matlab +messages = messageHistory; +messages = addUserMessageWithImages(messages,"What is in the image?", string(image_url)); +``` + +Generate a response. By default, the model returns a very short response. To override it, set `MaxNumTokens` to 4096. + +```matlab +[txt,~,response] = generate(chat,messages,MaxNumTokens=4096); +if response.StatusCode == "OK" + wrappedText = wrapText(txt) +else + response.Body.Data.error +end +``` + +```matlabTextOutput +wrappedText = + "The image depicts a serene landscape featuring a wooden pathway that runs + through a lush, green marsh or meadow. The path is bordered by tall grass and + some shrubs, with a clear blue sky overhead dotted with clouds. The scene + evokes a sense of tranquility and natural beauty." + +``` +# Helper function +```matlab +function wrappedText = wrapText(text) + s = textwrap(text,80); + wrappedText = string(join(s,newline)); +end +``` + +*Copyright 2024 The MathWorks, Inc.* + diff --git a/examples/DescribeImagesUsingChatGPT_media/figure_0.png b/examples/DescribeImagesUsingChatGPT_media/figure_0.png new file mode 100644 index 0000000..7178d08 Binary files /dev/null and b/examples/DescribeImagesUsingChatGPT_media/figure_0.png differ diff --git a/examples/InformationRetrievalUsingOpenAIDocumentEmbedding.md b/examples/InformationRetrievalUsingOpenAIDocumentEmbedding.md new file mode 100644 index 0000000..331fe1a --- /dev/null +++ b/examples/InformationRetrievalUsingOpenAIDocumentEmbedding.md @@ -0,0 +1,135 @@ + +# Information Retrieval Using OpenAI™ Document Embedding + +To run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/InformationRetrievalUsingOpenAIDocumentEmbedding.mlx](mlx-scripts/InformationRetrievalUsingOpenAIDocumentEmbedding.mlx) + +This example shows how to find documents to answer queries using the 'text\-embedding\-3\-small' document embedding model. Embeddings are used to represent documents and queries in a high\-dimensional space, allowing for the efficient retrieval of relevant information based on semantic similarity. + + +The example consists of four steps: + +- Download and preprocess text from several MATLAB® documentation pages. +- Embed query document and document corpus using the "text\-embedding\-3\-small" document embedding. +- Find the most documentation page most relevant to the query using cosine similarity scores. +- Generate an answer to the query based on the most relevant documentation page. + +This process is sometimes referred to as Retrieval\-Augmented Generation (RAG), similar to the application found in the example [ExampleRetrievalAugmentedGeneration.mlx](./ExampleRetrievalAugmentedGeneration.mlx). + + +This example requires Text Analytics Toolbox™. + + +To run this example, you need a valid API key from a paid OpenAI API account. + +```matlab +loadenv(".env") +addpath('../..') +``` +# Embed Query Document + +Convert the query into a numerical vector using the extractOpenAIEmbeddings function. Specify the model as "text\-embedding\-3\-small". + +```matlab +query = "What is the best way to store data made up of rows and columns?"; +[qEmb, ~] = extractOpenAIEmbeddings(query, ModelName="text-embedding-3-small"); +qEmb(1:5) +``` + +```matlabTextOutput +ans = 1x5 + -0.0051 -0.0005 0.0362 -0.0069 0.0534 + +``` +# Download and Embed Source Text + +In this example, we will scrape content from several MATLAB documentation pages. + + +This requires the following steps: + +1. Start with a list of websites. This examples uses pages from MATLAB documentation. +2. Extract the context of the pags using `extractHTMLText`. +3. Embed the websites using `extractOpenAIEmbeddings`. +```matlab +metadata = ["https://www.mathworks.com/help/matlab/numeric-types.html"; + "https://www.mathworks.com/help/matlab/characters-and-strings.html"; + "https://www.mathworks.com/help/matlab/date-and-time-operations.html"; + "https://www.mathworks.com/help/matlab/categorical-arrays.html"; + "https://www.mathworks.com/help/matlab/tables.html"]; +id = (1:numel(metadata))'; +document = strings(numel(metadata),1); +embedding = []; +for ii = id' + page = webread(metadata(ii)); + tree = htmlTree(page); + subtree = findElement(tree,"body"); + document(ii) = extractHTMLText(subtree, ExtractionMethod="article"); + try + [emb, ~] = extractOpenAIEmbeddings(document(ii),ModelName="text-embedding-3-small"); + embedding = [embedding; emb]; + catch + end +end +vectorTable = table(id,document,metadata,embedding); +``` +# Generate Answer to Query + +Define the system prompt in `openAIChat` to answer questions based on context. + +```matlab +chat = openAIChat("You are a helpful MATLAB assistant. You will get a context for each question"); +``` + +Calculate the cosine similarity scores between the query and each of the documentation page using the `cosineSimilarity` function. + +```matlab +s = cosineSimilarity(vectorTable.embedding,qEmb); +``` + +Use the most similar documentation content to feed extra context into the prompt for generation. + +```matlab +[~,idx] = max(s); +context = vectorTable.document(idx); +prompt = "Context: " ... + + context + newline + "Answer the following question: " + query; +wrapText(prompt) +``` + +```matlabTextOutput +ans = + "Context: table is a data type suitable for column-oriented or tabular data that is often stored as columns in a text file or in a spreadsheet. + Tables consist of rows and column-oriented variables. + Each variable in a table can have a different data type and a different size with the one restriction that each variable must have the same number of rows. + For more information, see Create Tables and Assign Data to Them or watch Tables and Categorical Arrays. + Answer the following question: What is the best way to store data made up of rows and columns?" + +``` + +Pass the question and the context for generation to get a contextualized answer. + +```matlab +response = generate(chat, prompt); +wrapText(response) +``` + +```matlabTextOutput +ans = + "The best way to store data made up of rows and columns in MATLAB is by using the table data type. + Tables are designed for storing tabular data, where each column can have a different data type and size, but all columns must have the same number of rows. + This makes tables an ideal data structure for organizing and manipulating data in a tabular format." + +``` +# Helper Function + +Helper function to wrap text for easier reading in the live script. + +```matlab +function wrappedText = wrapText(text) + wrappedText = splitSentences(text); + wrappedText = join(wrappedText,newline); +end +``` + +*Copyright 2024 The MathWorks, Inc.* + diff --git a/examples/InformationRetrievalUsingOpenAIDocumentEmbedding.mlx b/examples/InformationRetrievalUsingOpenAIDocumentEmbedding.mlx deleted file mode 100644 index 7768de1..0000000 Binary files a/examples/InformationRetrievalUsingOpenAIDocumentEmbedding.mlx and /dev/null differ diff --git a/examples/ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode.md b/examples/ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode.md new file mode 100644 index 0000000..f00a247 --- /dev/null +++ b/examples/ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode.md @@ -0,0 +1,168 @@ + +# Process Generated Text in Real Time by Using Ollama™ in Streaming Mode + +To run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode.mlx](mlx-scripts/ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode.mlx) + +This example shows how to process generated text in real time by using Ollama in streaming mode. + + +By default, when you pass a prompt to Ollama using `ollamaChat`, it generates a response internally and then outputs it in full at the end. To print out and format generated text as the model is generating it, use the `StreamFun` name\-value argument of the `ollamaChat` class. The streaming function is a custom function handle that tells the model what to do with the output. + + +The example includes two parts: + +- First, define and use a custom streaming function to print out generated text directly as the model generates it. +- Then, create an HTML UI Component and define and use a custom streaming function to update the UI Component in real time as the model generates text. + +To run this example, you need a running Ollama server. As written, the example uses the Mistral model. + +```matlab +loadenv(".env") +addpath('../..') +``` +# Print Stream Directly to Screen + +In this example, the streamed output is printed directly to the screen. + + +Define the function to print the returned tokens. + +```matlab +function printToken(token) + fprintf("%s",token); +end +``` + +Create the chat object with the defined function as a handle. + +```matlab +chat = ollamaChat("mistral",StreamFun=@printToken); +``` + +Generate response to a prompt in streaming mode. + +```matlab +prompt = "What is Model-Based Design?"; +generate(chat, prompt, MaxNumTokens=500); +``` + +```matlabTextOutput + Model-Based Design (MBD) is an approach to system design, engineering, and modeling that uses graphical models to represent the functionality and behavior of systems. It emphasizes the creation of abstract, high-level models before implementing specific hardware or software components. + +The key features of Model-Based Design are: + +1. Graphical Representation: Instead of writing code, engineers create models using visual diagrams such as flowcharts, block diagrams, or state machines. This makes the system design more intuitive and easier for both experts and non-experts to understand. + +2. Simulation: By simulating a model, engineers can evaluate its performance and functionality without building any physical hardware or writing actual code. This allows for rapid prototyping and iterative improvements during the design phase. + +3. Code Generation: Once the model is validated through simulation and testing, it can be automatically converted into code ready for deployment on a variety of hardware platforms. This reduces development time and increases productivity by minimizing manual coding tasks. + +4. Model Reuse and Reusability: Models can be reused across different systems and applications, saving time and effort in the design process. Additionally, modular models can be combined to create larger, more complex systems. + +5. System Integration: MBD helps streamline system integration by providing a clear interface between subsystems and components. This makes it easier to integrate third-party libraries or software tools into a project. + +Model-Based Design is widely used in control systems design, embedded systems, and hardware-software co-design across various industries such as automotive, aerospace, and telecommunications. It facilitates faster development cycles, improved system performance, reduced debugging time, and increased reliability. +``` +# Print Stream to HTML UI Component + +In this example, the streamed output is printed to the HTML component. + + +Create the HTML UI component. + +```matlab +fig = uifigure; +h = uihtml(fig,Position=[10,10,fig.Position(3)-20,fig.Position(4)-20]); +``` + +Initialize the content of the HTML UI component. + +```matlab +resetTable(h); +``` + +Create the chat object with the function handle, which requires the `uihtml` object created earlier. + +```matlab +chat = ollamaChat("mistral",StreamFun=@(x) addChat(h,"assistant",x)); +``` + +Add the user prompt to the table in the HTML UI component. Use messageHistory to keep a running history of the exchange. + +```matlab +history = messageHistory; +userPrompt = "Tell me 5 jokes."; +addChat(h,"user",userPrompt); +history = addUserMessage(history,userPrompt); +``` + +Generate response to a prompt in streaming mode. + +```matlab +[txt,message] = generate(chat,history); +history = addResponseMessage(history,message); +``` + +Simulating a dialog, add another user input and AI response. + +```matlab +userPrompt = "Could you please explain the third one?"; +addChat(h,"user",userPrompt); +history = addUserMessage(history,userPrompt); +generate(chat,history); +``` +# Helper functions + +`resetTable`: + +1. Adds the basic HTML structure and the JavaScript that process the data change in MATLAB. +2. The JavaScript gets a reference to the table and changed data and if the 3rd element in the data is "new", adds a new row. +3. It populates the new row with two cells and update the cells from the first two elements of the data. +4. The new row is then appended to the table. +5. Otherwise, the JavaScript gets reference to the last cell of the last row of the table, and update it with the 2nd element of the data. +```matlab +function resetTable(obj) + %RESETTABLE initialize the HTML UI component in the input argument. + mustBeA(obj,'matlab.ui.control.HTML') + obj.HTMLSource = ['',... + '', ... + '', ... + '
RoleContent
']; + obj.Data = []; + drawnow +end +``` + +`addRow` adds text to the table in the HTML UI component + +```matlab +function addChat(obj,role,content) + %ADDCHAT adds a new row or updates the last row of the table + mustBeA(obj,'matlab.ui.control.HTML') + content = replace(content,newline,"
"); + obj.Data = {role,content}; + drawnow +end +``` + +*Copyright 2024 The MathWorks, Inc.* + diff --git a/examples/ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode.mlx b/examples/ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode.mlx deleted file mode 100644 index 176a604..0000000 Binary files a/examples/ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode.mlx and /dev/null differ diff --git a/examples/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.md b/examples/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.md new file mode 100644 index 0000000..85ffdbd --- /dev/null +++ b/examples/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.md @@ -0,0 +1,169 @@ + +# Process Generated Text in Real Time by Using ChatGPT™ in Streaming Mode + +To run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.mlx](mlx-scripts/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.mlx) + +This example shows how to process generated text in real time by using ChatGPT in streaming mode. + + +By default, when you pass a prompt to ChatGPT, it generates a response internally and then outputs it in full at the end. To print out and format generated text as the model is generating it, use the `StreamFun` name\-value argument of the `openAIChat` class. The streaming function is a custom function handle that tells the model what to do with the output. + + +The example includes two parts: + +- First, define and use a custom streaming function to print out generated text directly as the model generates it. +- Then, create an HTML UI Component and define and use a custom streaming function to update the UI Component in real time as the model generates text. + +To run this example, you need a valid API key from a paid OpenAI™ API account. + +```matlab +loadenv(".env") +addpath('../..') +``` +# Print Stream Directly to Screen + +In this example, the streamed output is printed directly to the screen. + + +Define the function to print the returned tokens. + +```matlab +function printToken(token) + fprintf("%s",token); +end +``` + +Create the chat object with the defined function as a handle. + +```matlab +chat = openAIChat(StreamFun=@printToken); +``` + +Generate response to a prompt in streaming mode. + +```matlab +prompt = "What is Model-Based Design?"; +generate(chat, prompt, MaxNumTokens=500); +``` + +```matlabTextOutput +Model-Based Design is an approach to system development that uses graphical models to design and simulate systems before implementing them in hardware or software. It involves creating models that represent the behavior and interactions of system components, and using these models to analyze, validate, and optimize the system before building it. Model-Based Design can help to improve the efficiency, reliability, and quality of system development by enabling engineers to explore design alternatives, detect errors early in the development process, and facilitate collaboration between different teams working on the same project. +``` +# Print Stream to HTML UI Component + +In this example, the streamed output is printed to the HTML component. + + +Create the HTML UI component. + +```matlab +fig = uifigure; +h = uihtml(fig,Position=[50,10,450,400]); +``` + +Initialize the content of the HTML UI component. + +```matlab +resetTable(h); +``` + +Create the chat object with the function handle, which requires the `uihtml` object created earlier. + +```matlab +chat = openAIChat(StreamFun=@(x)printStream(h,x)); +``` + +Add the user prompt to the table in the HTML UI component. + +```matlab +userPrompt = "Tell me 5 jokes."; +addChat(h,"user",userPrompt,"new") +``` + +Generate response to a prompt in streaming mode. + +```matlab +[txt, message, response] = generate(chat,userPrompt); +``` + +Update the last row with the final output. This is necessary if further update is needed to support additional HTML formatting. + +```matlab +addChat(h,"assistant",txt,"current") +``` +# Helper functions + +`resetTable`: + +1. Adds the basic HTML structure and the JavaScript that process the data change in MATLAB. +2. The JavaScript gets a reference to the table and changed data and if the 3rd element in the data is "new", adds a new row. +3. It populates the new row with two cells and update the cells from the first two elements of the data. +4. The new row is then appended to the table. +5. Otherwise, the JavaScript gets reference to the last cell of the last row of the table, and update it with the 2nd element of the data. +```matlab +function resetTable(obj) + %RESETTABLE initialize the HTML UI component in the input argument. + mustBeA(obj,'matlab.ui.control.HTML') + obj.HTMLSource = ['' ... + '
RoleContent
']; + obj.Data = []; + drawnow +end +``` + +`addRow` adds a new row to the table in the HTML UI component + +```matlab +function addChat(obj,role,content,row) + %ADDCHAT adds a new row or updates the last row of the table + mustBeA(obj,'matlab.ui.control.HTML') + content = replace(content,newline,"
"); + obj.Data = {role,content,row}; + drawnow +end +``` + +`printStream` is the streaming function and prints the stream in the table in the HTML UI component + +```matlab +function printStream(h,x) + %PRINTSTREAM prints the stream in a new row in the table + if strlength(x) == 0 + % if the first token is 0 length, add a new row + tokens = string(x); + h.Data = {"assistant",tokens,"new"}; + else + % otherwise append the new token to the previous tokens + % if the new token contains a line break, replace + % it with
+ if contains(x,newline) + x = replace(x,newline,"
"); + end + tokens = h.Data{2} + string(x); + % update the existing row. + h.Data = {"assistant",tokens,"current"}; + end + drawnow +end +``` + +*Copyright 2024 The MathWorks, Inc.* + diff --git a/examples/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.mlx b/examples/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.mlx deleted file mode 100644 index 2139f11..0000000 Binary files a/examples/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.mlx and /dev/null differ diff --git a/examples/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.md b/examples/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.md new file mode 100644 index 0000000..9479d45 --- /dev/null +++ b/examples/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.md @@ -0,0 +1,159 @@ + +# Retrieval\-Augmented Generation Using ChatGPT™ and MATLAB + +To run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.mlx](mlx-scripts/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.mlx) + +This example shows how to use retrieval\-augmented generation to generate answers to queries based on information contained in a document corpus. + + +The example contains three steps: + +- Download and preprocess documents. +- Find documents relevant to a query using keyword search. +- Generate a response using ChatGPT based on the both the query and the most relevant source document. \-> title "Generate Response" + +This example requires Text Analytics Toolbox™. + + +To run this example, you need a valid API key from a paid OpenAI™ API account. + +```matlab +loadenv(".env") +addpath('../..') +``` +# Download and Preprocess Documents + +Specify the URLs of the reports. + +```matlab +url = ["https://openknowledge.worldbank.org/bitstreams/0c18c872-91f0-51a4-ba91-c36b98893b4a/download" + "https://openknowledge.worldbank.org/bitstreams/476f037b-a17e-484f-9cc2-282a2e5a929f/download" + "https://openknowledge.worldbank.org/bitstreams/0c18c872-91f0-51a4-ba91-c36b98893b4a/download"]; +``` + +Define the local path where the reports will be saved and download the reports using the provided URLs and save them to the specified local path. + +```matlab +localpath = "./data/"; +if ~exist(localpath, "dir") + mkdir(localpath); +end +numFiles = numel(url); +for i = 1:numFiles + filename = "WBD_" + i + ".pdf"; + local_file_name = fullfile(localpath, filename); + if ~exist(local_file_name,"file") + websave(local_file_name, url{i}, weboptions(Timeout=30)); + end +end +``` + +Define the function to read the text from the downloaded files. + +```matlab +readFcn = @extractFileText; +file_pattern = [".txt",".pdf",".docx",".html",".htm"]; +fds = fileDatastore(localpath,'FileExtensions',file_pattern,'ReadFcn',readFcn); + +str = readall(fds); +str = [str{:}]; +``` + +Split the text data into paragraphs with the helper function `preprocessDocuments`. + +```matlab +documents = preprocessDocuments(str); +``` + +Initialize the chatbot with a system prompt and API key. Include your API key in the environment variable `OPENAI_API_KEY` or pass your key using the `APIKey` name\-value pair. + +```matlab +chat = openAIChat("You are a helpful assistant. You will get a " + ... + "context for each question, but only use the information " + ... + "in the context if that makes sense to answer the question. " + ... + "Let's think step-by-step, explaining how you reached the answer."); +``` +# Retrieve Relevant Documents + +Define the query, then retrieve and filter the relevant documents based on the query. + +```matlab +query = "What technical criteria can be used to streamline new approvals for grid-friendly DPV?"; +``` + +Tokenize the query and find similarity scores between the query and documents. + +```matlab +embQuery = bm25Similarity(documents, tokenizedDocument(query)); +``` + +Sort the documents in descending order of similarity scores. + +```matlab +[~, idx] = sort(embQuery, "descend"); +limitWords = 1000; +selectedDocs = []; +totalWords = 0; +``` + +Iterate over sorted document indices until word limit is reached + +```matlab +i = 1; +while totalWords <= limitWords && i <= length(idx) + totalWords = totalWords + doclength(documents(idx(i))); + selectedDocs = [selectedDocs; joinWords(documents(idx(i)))]; + i = i + 1; +end +``` +# Generate Response + +Define the prompt for the chatbot and generate a response. + +```matlab +prompt = "Context:" + join(selectedDocs, " ") + newline + ... + "Answer the following question: " + query; +response = generate(chat, prompt); +``` + +Wrap the text for easier visualization. + +```matlab +wrapText(response) +``` + +```matlabTextOutput +ans = + "The context provides information on how technical criteria can be used to + streamline new approvals for grid-friendly DPV. It mentions that technical + approvals for DPV installations to connect to the grid can be streamlined with + prudent screening criteria for systems that meet certain specifications. + Additionally, it emphasizes the importance of having a grid code that reflects + expected future growth of distributed energy resources. + + Therefore, the technical criteria that can be used to streamline new approvals + for grid-friendly DPV include having prudent screening criteria based on + specific specifications and ensuring that the grid code is in line with the + expected growth of distributed resources. This helps in facilitating the + connection of DPV installations to the grid efficiently and effectively." + +``` +# Helper Functions +```matlab +function allDocs = preprocessDocuments(str) +tokenized = tokenizedDocument(join(str,[newline newline])); +allDocs = splitParagraphs(tokenized); +end + +function wrappedText = wrapText(text) +s = textwrap(text,80); +wrappedText = string(join(s,newline)); +end +``` +# References + +*Energy Sector Management Assistance Program (ESMAP). 2023. From Sun to Roof to Grid: Power Systems and Distributed PV. Technical Report. Washington, DC: World Bank. License: Creative Commons Attribution CC BY 3.0 IGO* + + +*Copyright 2024 The MathWorks, Inc.* + diff --git a/examples/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.mlx b/examples/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.mlx deleted file mode 100644 index 9c6d61e..0000000 Binary files a/examples/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.mlx and /dev/null differ diff --git a/examples/RetrievalAugmentedGenerationUsingOllamaAndMATLAB.md b/examples/RetrievalAugmentedGenerationUsingOllamaAndMATLAB.md new file mode 100644 index 0000000..e711e7e --- /dev/null +++ b/examples/RetrievalAugmentedGenerationUsingOllamaAndMATLAB.md @@ -0,0 +1,167 @@ + +# Retrieval\-Augmented Generation Using Ollama™ and MATLAB + +To run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/RetrievalAugmentedGenerationUsingOllamaAndMATLAB.mlx](mlx-scripts/RetrievalAugmentedGenerationUsingOllamaAndMATLAB.mlx) + +This example shows how to use retrieval\-augmented generation to generate answers to queries based on information contained in a document corpus. + + +The example contains three steps: + +- Download and preprocess documents. +- Find documents relevant to a query using keyword search. +- Generate a response using Ollama based on the both the query and the most relevant source document. + +This example requires Text Analytics Toolbox™ and a running Ollama service. As written, it requires the Mistral model to be installed in that Ollama instance. + +```matlab +loadenv(".env") +addpath('../..') +``` +# Download and Preprocess Documents + +Specify the URLs of the reports. + +```matlab +url = ["https://openknowledge.worldbank.org/bitstreams/0c18c872-91f0-51a4-ba91-c36b98893b4a/download" + "https://openknowledge.worldbank.org/bitstreams/476f037b-a17e-484f-9cc2-282a2e5a929f/download" + "https://openknowledge.worldbank.org/bitstreams/0c18c872-91f0-51a4-ba91-c36b98893b4a/download"]; +``` + +Define the local path where the reports will be saved and download the reports using the provided URLs and save them to the specified local path. + +```matlab +localpath = "./data/"; +if ~exist(localpath, "dir") + mkdir(localpath); +end +numFiles = numel(url); +for i = 1:numFiles + filename = "WBD_" + i + ".pdf"; + localFileName = fullfile(localpath, filename); + if ~exist(localFileName,"file") + websave(localFileName, url{i}, weboptions(Timeout=30)); + end +end +``` + +Define the function to read the text from the downloaded files. + +```matlab +readFcn = @extractFileText; +filePattern = [".txt",".pdf",".docx",".html",".htm"]; +fds = fileDatastore(localpath,'FileExtensions',filePattern,'ReadFcn',readFcn); + +str = readall(fds); +str = [str{:}]; +``` + +Split the text data into paragraphs with the helper function `preprocessDocuments`. + +```matlab +documents = preprocessDocuments(str); +``` + +Initialize the chatbot with the model name (Mistral) and the a generic system prompt. Due to the long input created below, responses may take a long time on older machines; increase the accepted timeout. + +```matlab +chat = ollamaChat("mistral", ... + "You are a helpful assistant. You will get a " + ... + "context for each question, but only use the information " + ... + "in the context if that makes sense to answer the question. " + ... + "Let's think step-by-step, explaining how you reached the answer.", ... + TimeOut=600); +``` +# Retrieve Relevant Documents + +Define the query, then retrieve and filter the relevant documents based on the query. + +```matlab +query = "What technical criteria can be used to streamline new approvals for grid-friendly DPV?"; +``` + +Tokenize the query and find similarity scores between the query and documents. + +```matlab +embQuery = bm25Similarity(documents, tokenizedDocument(query)); +``` + +Sort the documents in descending order of similarity scores. + +```matlab +[~, idx] = sort(embQuery, "descend"); +limitWords = 1000; +selectedDocs = []; +totalWords = 0; +``` + +Iterate over sorted document indices until word limit is reached + +```matlab +i = 1; +while totalWords <= limitWords && i <= length(idx) + totalWords = totalWords + doclength(documents(idx(i))); + selectedDocs = [selectedDocs; joinWords(documents(idx(i)))]; + i = i + 1; +end +``` +# Generate Response + +Define the prompt for the chatbot and generate a response. + +```matlab +prompt = "Context:" + join(selectedDocs, " ") + newline + ... + "Answer the following question: " + query; +response = generate(chat, prompt); +``` + +Wrap the text for easier visualization. + +```matlab +wrapText(response) +``` + +```matlabTextOutput +ans = + " Technical criteria that can be used to streamline new approvals for + grid-friendly DPV include: + +1. Adopting a grid code that reflects expected future growth of distributed + energy resources, and updating it as necessary to balance deployment with the + technical requirements of distribution. + 2. Specifying advanced inverter functions today to reduce the need to change + systems in the future, making them easy to implement. + 3. Implementing prudent screening criteria for DPV installations that meet + certain specifications, which can streamline technical approvals. Such criteria + often rely on hosting capacity calculations to estimate the point where DPV + would induce technical impacts on system operations. + 4. Considering the value of ambitious investment options to accommodate + long-term developments and prioritizing the most cost-effective solutions in + assessing interventions for grid-friendly DPV. Costs can vary from one power + system to another, so case-by-case appraisal is important. + 5. Using metrics relevant to the impacts of DPV on low-voltage distribution + grids, such as DPV capacity penetration relative to minimum feeder daytime + load, to improve the use of technical screening criteria. Examples include + regulations that limit DPV installation size to a given percentage of the + customer's peak load, or countries with no limits on feeder penetration." + +``` +# Helper Functions +```matlab +function allDocs = preprocessDocuments(str) +tokenized = tokenizedDocument(join(str,[newline newline])); +allDocs = splitParagraphs(tokenized); +end + +function wrappedText = wrapText(text) +s = textwrap(text,80); +wrappedText = string(join(s,newline)); +end +``` +# References + +*Energy Sector Management Assistance Program (ESMAP). 2023. From Sun to Roof to Grid: Power Systems and Distributed PV. Technical Report. Washington, DC: World Bank. License: Creative Commons Attribution CC BY 3.0 IGO* + + +*Copyright 2024 The MathWorks, Inc.* + diff --git a/examples/RetrievalAugmentedGenerationUsingOllamaAndMATLAB.mlx b/examples/RetrievalAugmentedGenerationUsingOllamaAndMATLAB.mlx deleted file mode 100644 index c7f0776..0000000 Binary files a/examples/RetrievalAugmentedGenerationUsingOllamaAndMATLAB.mlx and /dev/null differ diff --git a/examples/SummarizeLargeDocumentsUsingChatGPTandMATLAB.md b/examples/SummarizeLargeDocumentsUsingChatGPTandMATLAB.md new file mode 100644 index 0000000..2a77c1e --- /dev/null +++ b/examples/SummarizeLargeDocumentsUsingChatGPTandMATLAB.md @@ -0,0 +1,152 @@ + +# Summarize Large Documents Using ChatGPT™ and MATLAB® + +To run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/SummarizeLargeDocumentsUsingChatGPTandMATLAB.mlx](mlx-scripts/SummarizeLargeDocumentsUsingChatGPTandMATLAB.mlx) + +This example shows how to use ChatGPT to summarize documents that are too large to be summarized at once. + + +To summarize short documents using ChatGPT, you can pass the documents directly as a prompt together with an instruction to summarize them. However, ChatGPT can only process prompts of limited size. + + +To summarize documents that are larger than this limit, split the documents up into smaller documents. Summarize the smaller document chunks, then pass all of the summaries to ChatGPT to generate one overall summary. + +- This example includes four steps: +- Download the complete text of "Alice in Wonderland" by Lewis Carroll from Project Gutenberg. +- Split the documents up into chunks of less than 3000 words. +- Use ChatGPT to create summaries of each chunk. +- Then use ChatGPT to create a summary of all of the summaries. + +To run this example, you need Text Analytics Toolbox™. + + +To run this example, you need a valid API key from a paid OpenAI™ API account. + +```matlab +loadenv(".env") +addpath('../..') +``` +# Download Text Data + +Download and read the content from Alice's Adventures in Wonderland by Lewis Carroll from Project Gutenberg. + + +First read the contents of the webpage. + +```matlab +options = weboptions(Timeout=30); +code = webread("https://www.gutenberg.org/files/11/11-h/11-h.htm", options); +longText = extractHTMLText(string(code)); +``` +# Split Document Into Chunks + +Large language models have a limit in terms of how much text they can accept as input, so if you try to summarize the complete book, you will likely get an error. A workaround is splitting the book into chunks and summarize each chunk individually. The chunk size is defined in `limitChunkWords`, which restricts the numbers of words in a chunk. + +```matlab +incrementalSummary = longText; +limitChunkWords = 3000; +chunks = createChunks(incrementalSummary, limitChunkWords); +``` +# Summarize Chunks + +Initialize a ChatGPT session with the role of summarizing text + +```matlab +summarizer = openAIChat("You are a professional summarizer."); +``` + +Looping process to gradually summarize the text chunk by chunk, reducing the chunk size with each iteration. + +```matlab +numCalls = 0; +while numel(chunks)>1 + summarizedChunks = strings(size(chunks)); + numCalls = numCalls + numel(chunks); +``` + +Add a limit to the number of calls, to ensure you are not making more calls than what is expected. You can change this value to match what is needed for your application. + +```matlab + if numCalls > 20 + error("Document is too long to be summarized.") + end + + for i = 1:length(chunks) + summarizedChunks(i) = generate(summarizer, "Summarize this content:" + newline + chunks(i)); + end +``` + +Merge the summarized chunks to serve as the base for the next iteration. + +```matlab + incrementalSummary = join(summarizedChunks); +``` + +Form new chunks with a reduced size for the subsequent iteration. + +```matlab + chunks = createChunks(incrementalSummary, limitChunkWords); +end +``` +# Summarize Document + +Compile the final summary by combining the summaries from all the chunks. + +```matlab +fullSummary = generate(summarizer, "The following text is a combination of summaries. " + ... + "Provide a cohese and coherent summary combining these smaller summaries, preserving as much information as possible:" + newline + incrementalSummary); +wrapText(fullSummary) +``` + +```matlabTextOutput +ans = + ""Alice's Adventures in Wonderland" by Lewis Carroll follows the whimsical journey of a young girl, Alice, who falls into a fantastical world through a rabbit hole. + Throughout her adventures, Alice encounters a series of peculiar characters and bizarre events while trying to find her way back home. + She navigates through surreal situations such as a Caucus-race with talking animals, converses with a cryptic Caterpillar about identity and size changes, and experiences a mad tea party with the March Hare and the Hatter. + Alice also interacts with the Queen of Hearts during a chaotic croquet game, intervenes in a trial involving the theft of tarts, and meets the Mock Turtle and Gryphon who share odd stories and engage in whimsical discussions about lobsters and fish tails. + The narrative is filled with illogical and imaginative elements, capturing readers' imaginations with its colorful and eccentric storytelling." + +``` +# `createChunks` function + +This function segments a long text into smaller parts of a predefined size to facilitate easier summarization. It preserves the structure of sentences. The `chunkSize` should be large enough to fit at least one sentence. + +```matlab +function chunks = createChunks(text, chunkSize) + % Tokenizing the input text for processing + text = tokenizedDocument(text); + + % Splitting the tokenized text into individual sentences + text = splitSentences(text); + chunks = []; + currentChunk = ""; + currentChunkSize = 0; + + % Iterating through the sentences to aggregate them into chunks until the chunk + % attains the predefined size, after which a new chunk is started + for i=1:length(text) + newChunkSize = currentChunkSize + doclength(text(i)); + if newChunkSize < chunkSize + currentChunkSize = currentChunkSize + doclength(text(i)); + currentChunk = currentChunk + " " + joinWords(text(i)); + else + chunks = [chunks; currentChunk]; %#ok + currentChunkSize = doclength(text(i)); + currentChunk = joinWords(text(i)); + end + end +end +``` +# `wrapText` function + +This function splits text into sentences and then concatenates them again using `newline` to make it easier to visualize text in this example + +```matlab +function wrappedText = wrapText(text) +wrappedText = splitSentences(text); +wrappedText = join(wrappedText,newline); +end +``` + +*Copyright 2023 The MathWorks, Inc.* + diff --git a/examples/SummarizeLargeDocumentsUsingChatGPTandMATLAB.mlx b/examples/SummarizeLargeDocumentsUsingChatGPTandMATLAB.mlx deleted file mode 100644 index daa8cbf..0000000 Binary files a/examples/SummarizeLargeDocumentsUsingChatGPTandMATLAB.mlx and /dev/null differ diff --git a/examples/UsingDALLEToEditImages.md b/examples/UsingDALLEToEditImages.md new file mode 100644 index 0000000..132fb0d --- /dev/null +++ b/examples/UsingDALLEToEditImages.md @@ -0,0 +1,82 @@ + +# Using DALL·E™ to Edit Images + +To run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/UsingDALLEToEditImages.mlx](mlx-scripts/UsingDALLEToEditImages.mlx) + +This example shows how to generate and edit images using the `openAIImages` object. + + +To run this example, you need a valid OpenAI™ API key. Creating images using DALL•E may incur a fee. + +```matlab +loadenv(".env") +addpath("../..") +``` + +We want to load images files relative to the project directory below: + +```matlab +projectDir = fileparts(which("openAIImages")); +``` +# Generate image variations + +Use the image variation feature in DALL•E 2. + +```matlab +mdl = openAIImages(ModelName="dall-e-2"); +``` + +Show the image to get variations for. + +```matlab +imagePath = fullfile(projectDir,"examples","images","bear.png"); +figure +imshow(imagePath) +``` + +![figure_0.png](UsingDALLEToEditImages_media/figure_0.png) + +Generate variations for that image. + +```matlab +[images,resp] = createVariation(mdl, imagePath, NumImages=4); +if ~isempty(images) + tiledlayout('flow') + for ii = 1:numel(images) + nexttile + imshow(images{ii}) + end +else + disp(resp.Body.Data.error) +end +``` + +![figure_1.png](UsingDALLEToEditImages_media/figure_1.png) +# Edit an Image with DALL·E + +Use an image containing a mask to apply modifications to the masked area. + +```matlab +maskImagePath = fullfile(projectDir,"examples","images","mask_bear.png"); +figure +imshow(maskImagePath) +``` + +![figure_2.png](UsingDALLEToEditImages_media/figure_2.png) + +Add a swan to the masked area using the function edit. + +```matlab +[images,resp] = edit(mdl, imagePath, "Swan", MaskImagePath=maskImagePath); +if isfield(resp.Body.Data,'data') + figure + imshow(images{1}); +else + disp(resp.Body.Data.error) +end +``` + +![figure_3.png](UsingDALLEToEditImages_media/figure_3.png) + +*Copyright 2024 The MathWorks, Inc.* + diff --git a/examples/UsingDALLEToEditImages_media/figure_0.png b/examples/UsingDALLEToEditImages_media/figure_0.png new file mode 100644 index 0000000..14164e6 Binary files /dev/null and b/examples/UsingDALLEToEditImages_media/figure_0.png differ diff --git a/examples/UsingDALLEToEditImages_media/figure_1.png b/examples/UsingDALLEToEditImages_media/figure_1.png new file mode 100644 index 0000000..9ba0281 Binary files /dev/null and b/examples/UsingDALLEToEditImages_media/figure_1.png differ diff --git a/examples/UsingDALLEToEditImages_media/figure_2.png b/examples/UsingDALLEToEditImages_media/figure_2.png new file mode 100644 index 0000000..98cfe3b Binary files /dev/null and b/examples/UsingDALLEToEditImages_media/figure_2.png differ diff --git a/examples/UsingDALLEToEditImages_media/figure_3.png b/examples/UsingDALLEToEditImages_media/figure_3.png new file mode 100644 index 0000000..c68e4ae Binary files /dev/null and b/examples/UsingDALLEToEditImages_media/figure_3.png differ diff --git a/examples/UsingDALLEToGenerateImages.md b/examples/UsingDALLEToGenerateImages.md new file mode 100644 index 0000000..d73243d --- /dev/null +++ b/examples/UsingDALLEToGenerateImages.md @@ -0,0 +1,44 @@ + +# Using DALL·E™ to generate images + +To run the code shown on this page, open the MLX file in MATLAB®: [mlx-scripts/UsingDALLEToGenerateImages.mlx](mlx-scripts/UsingDALLEToGenerateImages.mlx) + +This example shows how to generate images using the `openAIImages` object. + + +To run this example, you need a valid OpenAI™ API key. Creating images using DALL\-E may incur a fee. + +```matlab +loadenv(".env") +addpath('../..') +``` +# Image Generation with DALL·E 3 + +Create an `openAIImages` object with `ModelName` `dall-e-3`. + +```matlab +mdl = openAIImages(ModelName="dall-e-3"); +``` + +Generate and visualize an image. This model only supports the generation of one image per request. + +```matlab +images = generate(mdl,"A crispy fresh API key"); +figure +imshow(images{1}) +``` + +![figure_0.png](UsingDALLEToGenerateImages_media/figure_0.png) + +You can also define the style and quality of the image + +```matlab +images = generate(mdl,"A cat playing with yarn", Quality="hd", Style="natural"); +figure +imshow(images{1}) +``` + +![figure_1.png](UsingDALLEToGenerateImages_media/figure_1.png) + +*Copyright 2024 The MathWorks, Inc.* + diff --git a/examples/UsingDALLEToGenerateImages_media/figure_0.png b/examples/UsingDALLEToGenerateImages_media/figure_0.png new file mode 100644 index 0000000..85de485 Binary files /dev/null and b/examples/UsingDALLEToGenerateImages_media/figure_0.png differ diff --git a/examples/UsingDALLEToGenerateImages_media/figure_1.png b/examples/UsingDALLEToGenerateImages_media/figure_1.png new file mode 100644 index 0000000..7c4af1f Binary files /dev/null and b/examples/UsingDALLEToGenerateImages_media/figure_1.png differ diff --git a/examples/mlx-scripts/AnalyzeScientificPapersUsingFunctionCalls.mlx b/examples/mlx-scripts/AnalyzeScientificPapersUsingFunctionCalls.mlx new file mode 100644 index 0000000..def2bc3 Binary files /dev/null and b/examples/mlx-scripts/AnalyzeScientificPapersUsingFunctionCalls.mlx differ diff --git a/examples/mlx-scripts/AnalyzeSentimentinTextUsingChatGPTinJSONMode.mlx b/examples/mlx-scripts/AnalyzeSentimentinTextUsingChatGPTinJSONMode.mlx new file mode 100644 index 0000000..14e1ba2 Binary files /dev/null and b/examples/mlx-scripts/AnalyzeSentimentinTextUsingChatGPTinJSONMode.mlx differ diff --git a/examples/mlx-scripts/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mlx b/examples/mlx-scripts/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mlx new file mode 100644 index 0000000..7560f94 Binary files /dev/null and b/examples/mlx-scripts/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mlx differ diff --git a/examples/mlx-scripts/CreateSimpleChatBot.mlx b/examples/mlx-scripts/CreateSimpleChatBot.mlx new file mode 100644 index 0000000..eedb17e Binary files /dev/null and b/examples/mlx-scripts/CreateSimpleChatBot.mlx differ diff --git a/examples/mlx-scripts/CreateSimpleOllamaChatBot.mlx b/examples/mlx-scripts/CreateSimpleOllamaChatBot.mlx new file mode 100644 index 0000000..dc0c44a Binary files /dev/null and b/examples/mlx-scripts/CreateSimpleOllamaChatBot.mlx differ diff --git a/examples/DescribeImagesUsingChatGPT.mlx b/examples/mlx-scripts/DescribeImagesUsingChatGPT.mlx similarity index 97% rename from examples/DescribeImagesUsingChatGPT.mlx rename to examples/mlx-scripts/DescribeImagesUsingChatGPT.mlx index db73b30..0636c76 100644 Binary files a/examples/DescribeImagesUsingChatGPT.mlx and b/examples/mlx-scripts/DescribeImagesUsingChatGPT.mlx differ diff --git a/examples/mlx-scripts/InformationRetrievalUsingOpenAIDocumentEmbedding.mlx b/examples/mlx-scripts/InformationRetrievalUsingOpenAIDocumentEmbedding.mlx new file mode 100644 index 0000000..eb67cb5 Binary files /dev/null and b/examples/mlx-scripts/InformationRetrievalUsingOpenAIDocumentEmbedding.mlx differ diff --git a/examples/mlx-scripts/ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode.mlx b/examples/mlx-scripts/ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode.mlx new file mode 100644 index 0000000..cfb337d Binary files /dev/null and b/examples/mlx-scripts/ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode.mlx differ diff --git a/examples/mlx-scripts/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.mlx b/examples/mlx-scripts/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.mlx new file mode 100644 index 0000000..72bfad4 Binary files /dev/null and b/examples/mlx-scripts/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.mlx differ diff --git a/examples/mlx-scripts/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.mlx b/examples/mlx-scripts/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.mlx new file mode 100644 index 0000000..0b16ac4 Binary files /dev/null and b/examples/mlx-scripts/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.mlx differ diff --git a/examples/mlx-scripts/RetrievalAugmentedGenerationUsingOllamaAndMATLAB.mlx b/examples/mlx-scripts/RetrievalAugmentedGenerationUsingOllamaAndMATLAB.mlx new file mode 100644 index 0000000..a15f36c Binary files /dev/null and b/examples/mlx-scripts/RetrievalAugmentedGenerationUsingOllamaAndMATLAB.mlx differ diff --git a/examples/mlx-scripts/SummarizeLargeDocumentsUsingChatGPTandMATLAB.mlx b/examples/mlx-scripts/SummarizeLargeDocumentsUsingChatGPTandMATLAB.mlx new file mode 100644 index 0000000..c612023 Binary files /dev/null and b/examples/mlx-scripts/SummarizeLargeDocumentsUsingChatGPTandMATLAB.mlx differ diff --git a/examples/UsingDALLEToEditImages.mlx b/examples/mlx-scripts/UsingDALLEToEditImages.mlx similarity index 55% rename from examples/UsingDALLEToEditImages.mlx rename to examples/mlx-scripts/UsingDALLEToEditImages.mlx index f12dcf1..3d5e14d 100644 Binary files a/examples/UsingDALLEToEditImages.mlx and b/examples/mlx-scripts/UsingDALLEToEditImages.mlx differ diff --git a/examples/UsingDALLEToGenerateImages.mlx b/examples/mlx-scripts/UsingDALLEToGenerateImages.mlx similarity index 99% rename from examples/UsingDALLEToGenerateImages.mlx rename to examples/mlx-scripts/UsingDALLEToGenerateImages.mlx index 5fe2397..f1af68f 100644 Binary files a/examples/UsingDALLEToGenerateImages.mlx and b/examples/mlx-scripts/UsingDALLEToGenerateImages.mlx differ diff --git a/extractOpenAIEmbeddings.m b/extractOpenAIEmbeddings.m index 6813e0a..3f6b1a9 100644 --- a/extractOpenAIEmbeddings.m +++ b/extractOpenAIEmbeddings.m @@ -47,7 +47,7 @@ end -response = llms.internal.sendRequest(parameters,key, END_POINT, nvp.TimeOut); +response = llms.internal.sendRequestWrapper(parameters,key, END_POINT, nvp.TimeOut); if isfield(response.Body.Data, "data") emb = [response.Body.Data.data.embedding]; diff --git a/messageHistory.m b/messageHistory.m index 74c5e63..e2e0e60 100644 --- a/messageHistory.m +++ b/messageHistory.m @@ -111,32 +111,9 @@ nvp.Detail string {mustBeMember(nvp.Detail,["low","high","auto"])} = "auto" end - newMessage = struct("role", "user", "content", []); - newMessage.content = {struct("type","text","text",string(content))}; - for img = images(:).' - if startsWith(img,("https://"|"http://")) - s = struct( ... - "type","image_url", ... - "image_url",struct("url",img)); - else - [~,~,ext] = fileparts(img); - MIMEType = "data:image/" + erase(ext,".") + ";base64,"; - % Base64 encode the image using the given MIME type - fid = fopen(img); - im = fread(fid,'*uint8'); - fclose(fid); - b64 = matlab.net.base64encode(im); - s = struct( ... - "type","image_url", ... - "image_url",struct("url",MIMEType + b64)); - end - - s.image_url.detail = nvp.Detail; - - newMessage.content{end+1} = s; - this.Messages{end+1} = newMessage; - end - + newMessage = struct("role", "user", "content", string(content), ... + "images", images, "image_detail", nvp.Detail); + this.Messages{end+1} = newMessage; end function this = addToolMessage(this, id, name, content) diff --git a/ollamaChat.m b/ollamaChat.m index cdb2e35..6d9e5a0 100644 --- a/ollamaChat.m +++ b/ollamaChat.m @@ -136,7 +136,7 @@ if isstring(messages) && isscalar(messages) messagesStruct = {struct("role", "user", "content", messages)}; else - messagesStruct = messages.Messages; + messagesStruct = this.encodeImages(messages.Messages); end if ~isempty(this.SystemPrompt) @@ -160,6 +160,28 @@ end end + methods (Access=private) + function messageStruct = encodeImages(~, messageStruct) + for k=1:numel(messageStruct) + if isfield(messageStruct{k},"images") + images = messageStruct{k}.images; + % detail = messageStruct{k}.image_detail; + messageStruct{k} = rmfield(messageStruct{k},["images","image_detail"]); + imgs = cell(size(images)); + for n = 1:numel(images) + img = images(n); + % Base64 encode the image + fid = fopen(img); + im = fread(fid,'*uint8'); + fclose(fid); + imgs{n} = matlab.net.base64encode(im); + end + messageStruct{k}.images = imgs; + end + end + end + end + methods(Static) function mdls = models %ollamaChat.models - return models available on Ollama server diff --git a/openAIChat.m b/openAIChat.m index 7d73c75..cbd2440 100644 --- a/openAIChat.m +++ b/openAIChat.m @@ -181,7 +181,7 @@ if isstring(messages) && isscalar(messages) messagesStruct = {struct("role", "user", "content", messages)}; else - messagesStruct = messages.Messages; + messagesStruct = this.encodeImages(messages.Messages); end llms.openai.validateMessageSupported(messagesStruct{end}, this.ModelName); @@ -230,6 +230,40 @@ function mustBeValidFunctionCall(this, functionCall) end end + + function messageStruct = encodeImages(~, messageStruct) + for k=1:numel(messageStruct) + if isfield(messageStruct{k},"images") + images = messageStruct{k}.images; + detail = messageStruct{k}.image_detail; + messageStruct{k} = rmfield(messageStruct{k},["images","image_detail"]); + messageStruct{k}.content = ... + {struct("type","text","text",messageStruct{k}.content)}; + for img = images(:).' + if startsWith(img,("https://"|"http://")) + s = struct( ... + "type","image_url", ... + "image_url",struct("url",img)); + else + [~,~,ext] = fileparts(img); + MIMEType = "data:image/" + erase(ext,".") + ";base64,"; + % Base64 encode the image using the given MIME type + fid = fopen(img); + im = fread(fid,'*uint8'); + fclose(fid); + b64 = matlab.net.base64encode(im); + s = struct( ... + "type","image_url", ... + "image_url",struct("url",MIMEType + b64)); + end + + s.image_url.detail = detail; + + messageStruct{k}.content{end+1} = s; + end + end + end + end end end diff --git a/tests/private/recording-doubles/+llms/+internal/sendRequestWrapper.m b/tests/private/recording-doubles/+llms/+internal/sendRequestWrapper.m new file mode 100644 index 0000000..ab4858c --- /dev/null +++ b/tests/private/recording-doubles/+llms/+internal/sendRequestWrapper.m @@ -0,0 +1,40 @@ +function [response, streamedText] = sendRequestWrapper(parameters, token, varargin) +% This function is undocumented and will change in a future release + +% A wrapper around sendRequest to have a test seam +persistent seenCalls +if isempty(seenCalls) + seenCalls = cell(0,2); +end + +persistent filename + +if nargin == 1 && isequal(parameters,"close") + save(filename+".mat","seenCalls"); + seenCalls = cell(0,2); + return +end + +if nargin==2 && isequal(parameters,"open") + filename = token; + return +end + +streamFunCalls = {}; +hasCallback = nargin >= 5 && isa(varargin{3},'function_handle'); +if hasCallback + streamFun = varargin{3}; +end +function wrappedStreamFun(varargin) + streamFunCalls(end+1) = varargin; + streamFun(varargin{:}); +end +if hasCallback + varargin{3} = @wrappedStreamFun; +end + + +[response, streamedText] = llms.internal.sendRequest(parameters, token, varargin{:}); + +seenCalls(end+1,:) = {{parameters},{response,streamFunCalls,streamedText}}; +end diff --git a/tests/private/recording-doubles/addpath.m b/tests/private/recording-doubles/addpath.m new file mode 100644 index 0000000..d1f1059 --- /dev/null +++ b/tests/private/recording-doubles/addpath.m @@ -0,0 +1,2 @@ +function addpath(~) +% ignore addpath calls in examples diff --git a/tests/private/replaying-doubles/+llms/+internal/sendRequestWrapper.m b/tests/private/replaying-doubles/+llms/+internal/sendRequestWrapper.m new file mode 100644 index 0000000..0b689d7 --- /dev/null +++ b/tests/private/replaying-doubles/+llms/+internal/sendRequestWrapper.m @@ -0,0 +1,30 @@ +function [response, streamedText] = sendRequestWrapper(parameters, token, varargin) +% This function is undocumented and will change in a future release + +% A wrapper around sendRequest to have a test seam +persistent seenCalls +if isempty(seenCalls) + seenCalls = cell(0,2); +end + +if nargin == 1 && isequal(parameters,"close") + seenCalls = cell(0,2); + return +end + +if nargin==2 && isequal(parameters,"open") + load(token+".mat","seenCalls"); + return +end + +result = seenCalls{1,2}; +response = result{1}; +streamFunCalls = result{2}; +streamedText = result{3}; + +if nargin >= 5 && isa(varargin{3},'function_handle') + streamFun = varargin{3}; + cellfun(streamFun, streamFunCalls); +end + +seenCalls(1,:) = []; diff --git a/tests/private/replaying-doubles/addpath.m b/tests/private/replaying-doubles/addpath.m new file mode 100644 index 0000000..d1f1059 --- /dev/null +++ b/tests/private/replaying-doubles/addpath.m @@ -0,0 +1,2 @@ +function addpath(~) +% ignore addpath calls in examples diff --git a/tests/recordings/AnalyzeScientificPapersUsingFunctionCalls.mat b/tests/recordings/AnalyzeScientificPapersUsingFunctionCalls.mat new file mode 100644 index 0000000..db2a334 Binary files /dev/null and b/tests/recordings/AnalyzeScientificPapersUsingFunctionCalls.mat differ diff --git a/tests/recordings/AnalyzeSentimentinTextUsingChatGPTinJSONMode.mat b/tests/recordings/AnalyzeSentimentinTextUsingChatGPTinJSONMode.mat new file mode 100644 index 0000000..310d0e8 Binary files /dev/null and b/tests/recordings/AnalyzeSentimentinTextUsingChatGPTinJSONMode.mat differ diff --git a/tests/recordings/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mat b/tests/recordings/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mat new file mode 100644 index 0000000..c51cde7 Binary files /dev/null and b/tests/recordings/AnalyzeTextDataUsingParallelFunctionCallwithChatGPT.mat differ diff --git a/tests/recordings/CreateSimpleChatBot.mat b/tests/recordings/CreateSimpleChatBot.mat new file mode 100644 index 0000000..857ff8b Binary files /dev/null and b/tests/recordings/CreateSimpleChatBot.mat differ diff --git a/tests/recordings/CreateSimpleOllamaChatBot.mat b/tests/recordings/CreateSimpleOllamaChatBot.mat new file mode 100644 index 0000000..eadaa28 Binary files /dev/null and b/tests/recordings/CreateSimpleOllamaChatBot.mat differ diff --git a/tests/recordings/DescribeImagesUsingChatGPT.mat b/tests/recordings/DescribeImagesUsingChatGPT.mat new file mode 100644 index 0000000..227da1b Binary files /dev/null and b/tests/recordings/DescribeImagesUsingChatGPT.mat differ diff --git a/tests/recordings/InformationRetrievalUsingOpenAIDocumentEmbedding.mat b/tests/recordings/InformationRetrievalUsingOpenAIDocumentEmbedding.mat new file mode 100644 index 0000000..d82f627 Binary files /dev/null and b/tests/recordings/InformationRetrievalUsingOpenAIDocumentEmbedding.mat differ diff --git a/tests/recordings/ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode.mat b/tests/recordings/ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode.mat new file mode 100644 index 0000000..5bdf6d7 Binary files /dev/null and b/tests/recordings/ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode.mat differ diff --git a/tests/recordings/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.mat b/tests/recordings/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.mat new file mode 100644 index 0000000..cf73f21 Binary files /dev/null and b/tests/recordings/ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode.mat differ diff --git a/tests/recordings/README.md b/tests/recordings/README.md new file mode 100644 index 0000000..010208d --- /dev/null +++ b/tests/recordings/README.md @@ -0,0 +1,12 @@ +# Test Double Recordings + +Testing the examples typically takes a long time and tends to have false negatives relatively often, mostly due to timeout errors. + +The point of testing the examples is not to test that we can connect to the servers. We have other test points for that. Hence, we insert a “test double” while testing the examples that keeps recordings of previous interactions with the servers and just replays the responses. + +This directory contains those recordings. + +## Generating Recordings + +To generate or re-generate recordings (e.g., after changing an example, or making relevant software changes), open [`texampleTests.m`](../texampleTests.m) and in `setUpAndTearDowns`, change `capture = false;` to `capture = true;`. Then, run the test points relevant to the example(s) in question, and change `capture` back to `false`. + diff --git a/tests/recordings/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.mat b/tests/recordings/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.mat new file mode 100644 index 0000000..6bd5878 Binary files /dev/null and b/tests/recordings/RetrievalAugmentedGenerationUsingChatGPTandMATLAB.mat differ diff --git a/tests/recordings/RetrievalAugmentedGenerationUsingOllamaAndMATLAB.mat b/tests/recordings/RetrievalAugmentedGenerationUsingOllamaAndMATLAB.mat new file mode 100644 index 0000000..8329b23 Binary files /dev/null and b/tests/recordings/RetrievalAugmentedGenerationUsingOllamaAndMATLAB.mat differ diff --git a/tests/recordings/SummarizeLargeDocumentsUsingChatGPTandMATLAB.mat b/tests/recordings/SummarizeLargeDocumentsUsingChatGPTandMATLAB.mat new file mode 100644 index 0000000..f44077b Binary files /dev/null and b/tests/recordings/SummarizeLargeDocumentsUsingChatGPTandMATLAB.mat differ diff --git a/tests/recordings/UsingDALLEToEditImages.mat b/tests/recordings/UsingDALLEToEditImages.mat new file mode 100644 index 0000000..f20b183 Binary files /dev/null and b/tests/recordings/UsingDALLEToEditImages.mat differ diff --git a/tests/recordings/UsingDALLEToGenerateImages.mat b/tests/recordings/UsingDALLEToGenerateImages.mat new file mode 100644 index 0000000..58b842b Binary files /dev/null and b/tests/recordings/UsingDALLEToGenerateImages.mat differ diff --git a/tests/tazureChat.m b/tests/tazureChat.m index 531936f..a16a5da 100644 --- a/tests/tazureChat.m +++ b/tests/tazureChat.m @@ -55,6 +55,26 @@ function generateMultipleResponses(testCase) testCase.verifySize(response.Body.Data.choices,[3,1]); end + function generateWithImage(testCase) + chat = azureChat(Deployment="gpt-4o"); + image_path = "peppers.png"; + emptyMessages = messageHistory; + messages = addUserMessageWithImages(emptyMessages,"What is in the image?",image_path); + + text = generate(chat,messages); + testCase.verifyThat(text,matlab.unittest.constraints.ContainsSubstring("pepper")); + end + + function generateWithMultipleImages(testCase) + import matlab.unittest.constraints.ContainsSubstring + chat = azureChat(Deployment="gpt-4o"); + image_path = "peppers.png"; + emptyMessages = messageHistory; + messages = addUserMessageWithImages(emptyMessages,"Compare these images.",[image_path,image_path]); + + text = generate(chat,messages); + testCase.verifyThat(text,ContainsSubstring("same") | ContainsSubstring("identical")); + end function doReturnErrors(testCase) testCase.assumeTrue(isenv("AZURE_OPENAI_API_KEY"),"end-to-end test requires environment variables AZURE_OPENAI_API_KEY, AZURE_OPENAI_ENDPOINT, and AZURE_OPENAI_DEPLOYMENT."); @@ -65,6 +85,15 @@ function doReturnErrors(testCase) testCase.verifyError(@() generate(chat,wayTooLong), "llms:apiReturnedError"); end + function generateWithImageErrorsForGpt35(testCase) + chat = azureChat; + image_path = "peppers.png"; + emptyMessages = messageHistory; + messages = addUserMessageWithImages(emptyMessages,"What is in the image?",image_path); + + testCase.verifyError(@() generate(chat,messages), "llms:apiReturnedError"); + end + function seedFixesResult(testCase) testCase.assumeTrue(isenv("AZURE_OPENAI_API_KEY"),"end-to-end test requires environment variables AZURE_OPENAI_API_KEY, AZURE_OPENAI_ENDPOINT, and AZURE_OPENAI_DEPLOYMENT."); chat = azureChat; diff --git a/tests/texampleTests.m b/tests/texampleTests.m index a9864d1..e08a5c9 100644 --- a/tests/texampleTests.m +++ b/tests/texampleTests.m @@ -8,11 +8,27 @@ ChatBotExample = {"CreateSimpleChatBot", "CreateSimpleOllamaChatBot"}; end + properties + TestDir; + end methods (TestClassSetup) function setUpAndTearDowns(testCase) + % Capture and replay server interactions + testCase.TestDir = fileparts(mfilename("fullpath")); + import matlab.unittest.fixtures.PathFixture + capture = false; % run in capture or replay mode, cf. recordings/README.md + + if capture + testCase.applyFixture(PathFixture( ... + fullfile(testCase.TestDir,"private","recording-doubles"))); + else + testCase.applyFixture(PathFixture( ... + fullfile(testCase.TestDir,"private","replaying-doubles"))); + end + import matlab.unittest.fixtures.CurrentFolderFixture - testCase.applyFixture(CurrentFolderFixture("../examples")); + testCase.applyFixture(CurrentFolderFixture("../examples/mlx-scripts")); openAIEnvVar = "OPENAI_KEY"; secretKey = getenv(openAIEnvVar); @@ -29,22 +45,39 @@ function setUpAndTearDowns(testCase) testCase.addTeardown(@() iCloseAll()); end end - + + methods + function startCapture(testCase,testName) + llms.internal.sendRequestWrapper("open", ... + fullfile(testCase.TestDir,"recordings",testName)); + end + end + + methods(TestMethodTeardown) + function closeCapture(~) + llms.internal.sendRequestWrapper("close"); + end + end + methods(Test) - function testAnalyzeScientificPapersUsingFunctionCalls(~) + function testAnalyzeScientificPapersUsingFunctionCalls(testCase) + testCase.startCapture("AnalyzeScientificPapersUsingFunctionCalls"); AnalyzeScientificPapersUsingFunctionCalls; end function testAnalyzeSentimentinTextUsingChatGPTinJSONMode(testCase) + testCase.startCapture("AnalyzeSentimentinTextUsingChatGPTinJSONMode"); testCase.verifyWarning(@AnalyzeSentimentinTextUsingChatGPTinJSONMode,... "llms:warningJsonInstruction"); end - function testAnalyzeTextDataUsingParallelFunctionCallwithChatGPT(~) + function testAnalyzeTextDataUsingParallelFunctionCallwithChatGPT(testCase) + testCase.startCapture("AnalyzeTextDataUsingParallelFunctionCallwithChatGPT"); AnalyzeTextDataUsingParallelFunctionCallwithChatGPT; end function testCreateSimpleChatBot(testCase,ChatBotExample) + testCase.startCapture(ChatBotExample); % set up a fake input command, returning canned user prompts count = 0; prompts = [ @@ -85,43 +118,51 @@ function testCreateSimpleChatBot(testCase,ChatBotExample) testCase.verifySize(messages.Messages,[1 2*(count-1)]); end - function testDescribeImagesUsingChatGPT(~) + function testDescribeImagesUsingChatGPT(testCase) + testCase.startCapture("DescribeImagesUsingChatGPT"); DescribeImagesUsingChatGPT; end - function testInformationRetrievalUsingOpenAIDocumentEmbedding(~) + function testInformationRetrievalUsingOpenAIDocumentEmbedding(testCase) + testCase.startCapture("InformationRetrievalUsingOpenAIDocumentEmbedding"); InformationRetrievalUsingOpenAIDocumentEmbedding; end - function testProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode(~) + function testProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode(testCase) + testCase.startCapture("ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode"); ProcessGeneratedTextinRealTimebyUsingChatGPTinStreamingMode; end - function testProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode(~) + function testProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode(testCase) + testCase.startCapture("ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode"); ProcessGeneratedTextInRealTimeByUsingOllamaInStreamingMode; end - function testRetrievalAugmentedGenerationUsingChatGPTandMATLAB(~) + function testRetrievalAugmentedGenerationUsingChatGPTandMATLAB(testCase) + testCase.startCapture("RetrievalAugmentedGenerationUsingChatGPTandMATLAB"); RetrievalAugmentedGenerationUsingChatGPTandMATLAB; end - function testRetrievalAugmentedGenerationUsingOllamaAndMATLAB(~) + function testRetrievalAugmentedGenerationUsingOllamaAndMATLAB(testCase) + testCase.startCapture("RetrievalAugmentedGenerationUsingOllamaAndMATLAB"); RetrievalAugmentedGenerationUsingOllamaAndMATLAB; end - function testSummarizeLargeDocumentsUsingChatGPTandMATLAB(~) + function testSummarizeLargeDocumentsUsingChatGPTandMATLAB(testCase) + testCase.startCapture("SummarizeLargeDocumentsUsingChatGPTandMATLAB"); SummarizeLargeDocumentsUsingChatGPTandMATLAB; end - function testUsingDALLEToEditImages(~) + function testUsingDALLEToEditImages(testCase) + testCase.startCapture("UsingDALLEToEditImages"); UsingDALLEToEditImages; end - function testUsingDALLEToGenerateImages(~) + function testUsingDALLEToGenerateImages(testCase) + testCase.startCapture("UsingDALLEToGenerateImages"); UsingDALLEToGenerateImages; end - end - + end end function iCloseAll() diff --git a/tests/tollamaChat.m b/tests/tollamaChat.m index 95b9c3f..52d1376 100644 --- a/tests/tollamaChat.m +++ b/tests/tollamaChat.m @@ -98,6 +98,16 @@ function seedFixesResult(testCase) testCase.verifyEqual(response1,response2); end + function generateWithImages(testCase) + chat = ollamaChat("moondream"); + image_path = "peppers.png"; + emptyMessages = messageHistory; + messages = addUserMessageWithImages(emptyMessages,"What is in the image?",image_path); + + text = generate(chat,messages); + testCase.verifyThat(text,matlab.unittest.constraints.ContainsSubstring("pepper")); + end + function streamFunc(testCase) function seen = sf(str) persistent data; diff --git a/tests/topenAIChat.m b/tests/topenAIChat.m index 143335c..ad3f69e 100644 --- a/tests/topenAIChat.m +++ b/tests/topenAIChat.m @@ -97,6 +97,14 @@ function invalidInputsConstructor(testCase, InvalidConstructorInput) testCase.verifyError(@()openAIChat(InvalidConstructorInput.Input{:}), InvalidConstructorInput.Error); end + function generateWithStreamFunAndMaxNumTokens(testCase) + sf = @(x) fprintf("%s",x); + chat = openAIChat(StreamFun=sf); + result = generate(chat,"Why is a raven like a writing desk?",MaxNumTokens=5); + testCase.verifyClass(result,"string"); + testCase.verifyLessThan(strlength(result), 100); + end + function generateWithToolsAndStreamFunc(testCase) import matlab.unittest.constraints.HasField @@ -173,7 +181,7 @@ function generateWithToolsAndStreamFunc(testCase) testCase.verifyThat(data,HasField("explanation")); end - function generateWithImages(testCase) + function generateWithImage(testCase) chat = openAIChat; image_path = "peppers.png"; emptyMessages = messageHistory; @@ -183,6 +191,17 @@ function generateWithImages(testCase) testCase.verifyThat(text,matlab.unittest.constraints.ContainsSubstring("pepper")); end + function generateWithMultipleImages(testCase) + import matlab.unittest.constraints.ContainsSubstring + chat = openAIChat; + image_path = "peppers.png"; + emptyMessages = messageHistory; + messages = addUserMessageWithImages(emptyMessages,"Compare these images.",[image_path,image_path]); + + text = generate(chat,messages); + testCase.verifyThat(text,ContainsSubstring("same") | ContainsSubstring("identical")); + end + function invalidInputsGenerate(testCase, InvalidGenerateInput) f = openAIFunction("validfunction"); chat = openAIChat(Tools=f, APIKey="this-is-not-a-real-key"); @@ -377,7 +396,6 @@ function keyNotFound(testCase) function validConstructorInput = iGetValidConstructorInput() % while it is valid to provide the key via an environment variable, % this test set does not use that, for easier setup -validFunction = openAIFunction("funName"); validConstructorInput = struct( ... "JustKey", struct( ... "Input",{{"APIKey","this-is-not-a-real-key"}}, ... @@ -385,7 +403,7 @@ function keyNotFound(testCase) "VerifyProperties", struct( ... "Temperature", {1}, ... "TopP", {1}, ... - "StopSequences", {{}}, ... + "StopSequences", {string([])}, ... "PresencePenalty", {0}, ... "FrequencyPenalty", {0}, ... "TimeOut", {10}, ... @@ -401,7 +419,7 @@ function keyNotFound(testCase) "VerifyProperties", struct( ... "Temperature", {1}, ... "TopP", {1}, ... - "StopSequences", {{}}, ... + "StopSequences", {string([])}, ... "PresencePenalty", {0}, ... "FrequencyPenalty", {0}, ... "TimeOut", {10}, ... @@ -417,7 +435,7 @@ function keyNotFound(testCase) "VerifyProperties", struct( ... "Temperature", {2}, ... "TopP", {1}, ... - "StopSequences", {{}}, ... + "StopSequences", {string([])}, ... "PresencePenalty", {0}, ... "FrequencyPenalty", {0}, ... "TimeOut", {10}, ... @@ -433,7 +451,7 @@ function keyNotFound(testCase) "VerifyProperties", struct( ... "Temperature", {1}, ... "TopP", {0.2}, ... - "StopSequences", {{}}, ... + "StopSequences", {string([])}, ... "PresencePenalty", {0}, ... "FrequencyPenalty", {0}, ... "TimeOut", {10}, ... @@ -459,13 +477,29 @@ function keyNotFound(testCase) "ResponseFormat", {"text"} ... ) ... ), ... + "StopSequencesCharVector", struct( ... + "Input",{{"APIKey","this-is-not-a-real-key","StopSequences",'supercalifragilistic'}}, ... + "ExpectedWarning", '', ... + "VerifyProperties", struct( ... + "Temperature", {1}, ... + "TopP", {1}, ... + "StopSequences", {"supercalifragilistic"}, ... + "PresencePenalty", {0}, ... + "FrequencyPenalty", {0}, ... + "TimeOut", {10}, ... + "FunctionNames", {[]}, ... + "ModelName", {"gpt-4o-mini"}, ... + "SystemPrompt", {[]}, ... + "ResponseFormat", {"text"} ... + ) ... + ), ... "PresencePenalty", struct( ... "Input",{{"APIKey","this-is-not-a-real-key","PresencePenalty",0.1}}, ... "ExpectedWarning", '', ... "VerifyProperties", struct( ... "Temperature", {1}, ... "TopP", {1}, ... - "StopSequences", {{}}, ... + "StopSequences", {string([])}, ... "PresencePenalty", {0.1}, ... "FrequencyPenalty", {0}, ... "TimeOut", {10}, ... @@ -481,7 +515,7 @@ function keyNotFound(testCase) "VerifyProperties", struct( ... "Temperature", {1}, ... "TopP", {1}, ... - "StopSequences", {{}}, ... + "StopSequences", {string([])}, ... "PresencePenalty", {0}, ... "FrequencyPenalty", {0.1}, ... "TimeOut", {10}, ... @@ -497,7 +531,7 @@ function keyNotFound(testCase) "VerifyProperties", struct( ... "Temperature", {1}, ... "TopP", {1}, ... - "StopSequences", {{}}, ... + "StopSequences", {string([])}, ... "PresencePenalty", {0}, ... "FrequencyPenalty", {0}, ... "TimeOut", {0.1}, ... @@ -513,7 +547,7 @@ function keyNotFound(testCase) "VerifyProperties", struct( ... "Temperature", {1}, ... "TopP", {1}, ... - "StopSequences", {{}}, ... + "StopSequences", {string([])}, ... "PresencePenalty", {0}, ... "FrequencyPenalty", {0}, ... "TimeOut", {10}, ...