From b15bda12469e5e8720c9ed147b94f54176f72ccf Mon Sep 17 00:00:00 2001 From: Agustin Z Date: Sat, 13 Apr 2024 16:47:19 -0300 Subject: [PATCH 1/6] This resolves #27, creates a title for suggested pictos, also adds metadata to a board to be used with Midjourney --- run.ts | 23 ++++++++++++--- src/engine.ts | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 5 deletions(-) diff --git a/run.ts b/run.ts index d8f3494..d7ec7ad 100644 --- a/run.ts +++ b/run.ts @@ -33,8 +33,8 @@ const engineInstance = initEngine({ contentSafetyConfiguration, }); -const prompt = "A family of 5 people"; -const maxSuggestions = 5; +const prompt = "elements to tell a story about a wizard and a lizard"; +const maxSuggestions = 10; const symbolSet = "arasaac"; const language = "eng"; @@ -44,7 +44,7 @@ engineInstance.isContentSafe(prompt).then((result) => { console.log('Is content safe?', result); }); - +/* // Get suggestions with GlobalSymbols engineInstance .getSuggestions({ @@ -58,8 +58,23 @@ engineInstance "\nSuggestions -----------------------------------------------\n", suggestions, "length: " + suggestions.length - ) + ) ); +*/ + +engineInstance + .getFullBoard({ + prompt, + maxSuggestions, + symbolSet, + language, + }) + .then((board) => + console.log( + "\nFull board \n", + board + ) +); /* // Get suggestions with GlobalSymbol and Pictonizer images diff --git a/src/engine.ts b/src/engine.ts index 68d35b5..a5997ff 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -52,6 +52,12 @@ export type ContentSafetyConfiguration = { key: string; }; +export type Board = { + boardTitle: string; + boardContent: string; + pictos: Suggestion[]; +}; + export function init({ openAIConfiguration, globalSymbolsApiURL, @@ -83,9 +89,11 @@ export function init({ pictonizer, getSuggestionsAndProcessPictograms, isContentSafe, + getFullBoard, }; } +//Using a prompt get a list of suggested words. async function getWordSuggestions({ prompt, maxWords, @@ -129,6 +137,7 @@ async function getWordSuggestions({ throw new Error("ERROR: Suggestion list is empty"); } +//Based on a list of words fetch the Pictogram URL from GlobalSymbols API async function fetchPictogramsURLs({ words, symbolSet, @@ -254,6 +263,7 @@ async function processPictograms( return suggestionsWithAIImage; } +//Get suggested Pictograms based on a prompt. async function getSuggestions({ prompt, maxSuggestions = DEFAULT_MAX_SUGGESTIONS, @@ -276,7 +286,6 @@ async function getSuggestions({ symbolSet, language, }); - return suggestionsWithGlobalSymbolsImages; } @@ -303,6 +312,7 @@ const getSuggestionsAndProcessPictograms = async ({ return suggestionsWithAIImages; }; +//Use Azure ContentSafety API to check if a string is safe for all users. async function isContentSafe( textPrompt: string, ): Promise { @@ -328,3 +338,69 @@ async function isContentSafe( } } + + +//Using a list of words get a descriptive title. +async function getBoardTitle( + words: string[], + language: string +): Promise { + const max_tokens = Math.round(2 * words.length + 110); + const completionRequestParams = { + model: "text-davinci-003", + prompt: `act as a speech pathologist in language ${language} + usign this list of words {${words}} create a descriptive title for a communication board. + Here are mandatory instructions for the list: + -The title is 4 words maximum. + -It is very important to not repeat words. + -Do not add any other text or characters to the title.`, + temperature: 0, + max_tokens: max_tokens, + }; + + const response = await globalConfiguration.openAIInstance.createCompletion( + completionRequestParams + ); + const titleSuggestionsData = response.data?.choices[0]?.text; + if (titleSuggestionsData) { + return titleSuggestionsData; + } else { + return "No AI title."; + } +} + + +//Get a board with a title, a content desciption and Pictograms based on a prompt. +async function getFullBoard({ + prompt, + maxSuggestions = DEFAULT_MAX_SUGGESTIONS, + symbolSet, + language = DEFAULT_LANGUAGE, +}: { + prompt: string; + maxSuggestions: number; + symbolSet?: string; + language: string; +}): Promise { + const words: string[] = await getWordSuggestions({ + prompt, + maxWords: maxSuggestions, + language, + }); + //TODO we can check here if the word suggestions are safe @rodrisanchez + const title: string = await getBoardTitle(words,language); + console.log("Title: " + title); + const suggestionsWithGlobalSymbolsImages: Suggestion[] = + await fetchPictogramsURLs({ + words, + symbolSet, + language, + }); + return { + boardTitle: title, + boardContent: "MISC", //TODO change here to use a different midjourney model for different each category. + pictos:suggestionsWithGlobalSymbolsImages, + }; +} + + From bc14912bae1a374ab0f488f03837a0c41ae3242b Mon Sep 17 00:00:00 2001 From: Agustin Z Date: Sat, 13 Apr 2024 18:42:08 -0300 Subject: [PATCH 2/6] First commit to get a category for the board or for each picto to customize the Midjourney prompt with contextualization. --- run.ts | 2 +- src/engine.ts | 29 ++++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/run.ts b/run.ts index d7ec7ad..31ae32b 100644 --- a/run.ts +++ b/run.ts @@ -33,7 +33,7 @@ const engineInstance = initEngine({ contentSafetyConfiguration, }); -const prompt = "elements to tell a story about a wizard and a lizard"; +const prompt = "A japanese family"; const maxSuggestions = 10; const symbolSet = "arasaac"; const language = "eng"; diff --git a/src/engine.ts b/src/engine.ts index a5997ff..0a56384 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -390,6 +390,7 @@ async function getFullBoard({ //TODO we can check here if the word suggestions are safe @rodrisanchez const title: string = await getBoardTitle(words,language); console.log("Title: " + title); + const content: string = await getBoardContent(words,language); const suggestionsWithGlobalSymbolsImages: Suggestion[] = await fetchPictogramsURLs({ words, @@ -398,9 +399,35 @@ async function getFullBoard({ }); return { boardTitle: title, - boardContent: "MISC", //TODO change here to use a different midjourney model for different each category. + boardContent: content, pictos:suggestionsWithGlobalSymbolsImages, }; } +//Using a list of words get a descriptive title. +async function getBoardContent( + words: string[], + language: string +): Promise { + const max_tokens = Math.round(2 * words.length + 110); + const completionRequestParams = { + model: "text-davinci-003", + prompt: `given this list of words {${words}} you have to categorize all of them into one of the following categories only. + If you can't find a category use MISC. Categories:{PEOPLE, EMOTIONS, FOOD, PLACES, NATURE, MISC} + Use the following template for the response {Category1:word1; Category2:word2,word3, Category3:word4,...}`, + //TODO its a good idea to use the content category for each picto instead of the whole board. + temperature: 0, + max_tokens: max_tokens, + }; + + const response = await globalConfiguration.openAIInstance.createCompletion( + completionRequestParams + ); + const contentSuggestionData = response.data?.choices[0]?.text; + if (contentSuggestionData) { + return contentSuggestionData; + } else { + return "No AI content."; + } +} \ No newline at end of file From 4d54409bc5835309cda810e6587a1444ca6a3809 Mon Sep 17 00:00:00 2001 From: Agustin Z Date: Tue, 23 Apr 2024 16:23:14 -0300 Subject: [PATCH 3/6] Now there is a FullBoard that contains Title and each Pictogram has its category --- run.ts | 2 +- src/engine.ts | 55 ++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/run.ts b/run.ts index 31ae32b..4a511f6 100644 --- a/run.ts +++ b/run.ts @@ -33,7 +33,7 @@ const engineInstance = initEngine({ contentSafetyConfiguration, }); -const prompt = "A japanese family"; +const prompt = "school"; const maxSuggestions = 10; const symbolSet = "arasaac"; const language = "eng"; diff --git a/src/engine.ts b/src/engine.ts index 0a56384..a48d9a5 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -22,6 +22,7 @@ export type Suggestion = { id: string; label: string; locale: string; + category?: string; pictogram: { isAIGenerated: boolean; images: @@ -54,7 +55,6 @@ export type ContentSafetyConfiguration = { export type Board = { boardTitle: string; - boardContent: string; pictos: Suggestion[]; }; @@ -363,9 +363,9 @@ async function getBoardTitle( ); const titleSuggestionsData = response.data?.choices[0]?.text; if (titleSuggestionsData) { - return titleSuggestionsData; + return titleSuggestionsData.replace(/^\n\n/, ''); } else { - return "No AI title."; + throw new Error("Error fetching AI title"); } } @@ -390,16 +390,20 @@ async function getFullBoard({ //TODO we can check here if the word suggestions are safe @rodrisanchez const title: string = await getBoardTitle(words,language); console.log("Title: " + title); - const content: string = await getBoardContent(words,language); + const categories: Map = await getBoardContent(words,language); + //log + const categoryObject = Object.fromEntries(categories); + console.log("Categories:", categoryObject); + // const suggestionsWithGlobalSymbolsImages: Suggestion[] = await fetchPictogramsURLs({ words, symbolSet, language, }); + categorizePictos(suggestionsWithGlobalSymbolsImages,categories) return { - boardTitle: title, - boardContent: content, + boardTitle: title, pictos:suggestionsWithGlobalSymbolsImages, }; } @@ -409,25 +413,52 @@ async function getFullBoard({ async function getBoardContent( words: string[], language: string -): Promise { +): Promise> { const max_tokens = Math.round(2 * words.length + 110); const completionRequestParams = { model: "text-davinci-003", - prompt: `given this list of words {${words}} you have to categorize all of them into one of the following categories only. - If you can't find a category use MISC. Categories:{PEOPLE, EMOTIONS, FOOD, PLACES, NATURE, MISC} + prompt: `given this list of words {${words}} you have to sort all of them into one of the following categories only. + For default use DEF. Categories:{PEOPLE, EMOTIONS, FOOD, PLACES, NATURE, OTHER} Use the following template for the response {Category1:word1; Category2:word2,word3, Category3:word4,...}`, //TODO its a good idea to use the content category for each picto instead of the whole board. temperature: 0, max_tokens: max_tokens, }; - const response = await globalConfiguration.openAIInstance.createCompletion( completionRequestParams ); const contentSuggestionData = response.data?.choices[0]?.text; + console.log("contentSuggestionData" + contentSuggestionData); if (contentSuggestionData) { - return contentSuggestionData; + return parseContent(contentSuggestionData); } else { - return "No AI content."; + throw new Error("Error fetching AI categories"); + } +} + + +// Function to parse categories and words from the board content +function parseContent(content: string): Map { + const categoryMap = new Map(); + const matches = content.match(/([^}]*)/); + if (matches && matches[1]) { + matches[1].split(';').forEach(pair => { + const [category, wordsString] = pair.split(':').map(item => item.trim()); + const words = wordsString.split(',').map(word => word.trim()); + categoryMap.set(category, words); + }); } + + return categoryMap; +} + +// Function to add categories to pictos +function categorizePictos(pictos: Suggestion[], categoryMap: Map): void { + pictos.forEach(picto => { + categoryMap.forEach((words, category) => { + if (words.some(word => word.toLowerCase() === picto.label.toLowerCase())) { + picto.category = category; + } + }); + }); } \ No newline at end of file From 031357f4e91505144867eb9ed434e69fb5025723 Mon Sep 17 00:00:00 2001 From: Rodri Sanchez Date: Fri, 3 May 2024 14:03:04 -0300 Subject: [PATCH 4/6] Fix typo in getBoardContent prompt template --- src/engine.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine.ts b/src/engine.ts index a48d9a5..44d5a93 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -419,7 +419,7 @@ async function getBoardContent( model: "text-davinci-003", prompt: `given this list of words {${words}} you have to sort all of them into one of the following categories only. For default use DEF. Categories:{PEOPLE, EMOTIONS, FOOD, PLACES, NATURE, OTHER} - Use the following template for the response {Category1:word1; Category2:word2,word3, Category3:word4,...}`, + Use the following template for the response {Category1:word1; Category2:word2,word3; Category3:word4,...}`, //TODO its a good idea to use the content category for each picto instead of the whole board. temperature: 0, max_tokens: max_tokens, From e2174d69fd877882f1ce676e068ae066139336d1 Mon Sep 17 00:00:00 2001 From: Rodri Sanchez Date: Fri, 3 May 2024 14:03:23 -0300 Subject: [PATCH 5/6] Fix regex pattern in parseContent function --- src/engine.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine.ts b/src/engine.ts index 44d5a93..4abd710 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -440,7 +440,7 @@ async function getBoardContent( // Function to parse categories and words from the board content function parseContent(content: string): Map { const categoryMap = new Map(); - const matches = content.match(/([^}]*)/); + const matches = content.match(/([^.]*)/); if (matches && matches[1]) { matches[1].split(';').forEach(pair => { const [category, wordsString] = pair.split(':').map(item => item.trim()); From 14f45dde60ed51f2186080d8d4139ee9fd4bd8fb Mon Sep 17 00:00:00 2001 From: Rodri Sanchez Date: Tue, 11 Jun 2024 16:13:12 -0300 Subject: [PATCH 6/6] Update completion model --- src/engine.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engine.ts b/src/engine.ts index df31f93..f44c222 100644 --- a/src/engine.ts +++ b/src/engine.ts @@ -347,7 +347,7 @@ async function getBoardTitle( ): Promise { const max_tokens = Math.round(2 * words.length + 110); const completionRequestParams = { - model: "text-davinci-003", + model: "gpt-3.5-turbo-instruct", prompt: `act as a speech pathologist in language ${language} usign this list of words {${words}} create a descriptive title for a communication board. Here are mandatory instructions for the list: @@ -416,7 +416,7 @@ async function getBoardContent( ): Promise> { const max_tokens = Math.round(2 * words.length + 110); const completionRequestParams = { - model: "text-davinci-003", + model: "gpt-3.5-turbo-instruct", prompt: `given this list of words {${words}} you have to sort all of them into one of the following categories only. For default use DEF. Categories:{PEOPLE, EMOTIONS, FOOD, PLACES, NATURE, OTHER} Use the following template for the response {Category1:word1; Category2:word2,word3; Category3:word4,...}`,