Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add context pictonizer #38

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
19 changes: 17 additions & 2 deletions run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ engineInstance.isContentSafe(prompt).then((result) => {
console.log('Is content safe?', result);
});


/*
// Get suggestions with GlobalSymbols
engineInstance
.getSuggestions({
Expand All @@ -58,8 +58,23 @@ engineInstance
"\nSuggestions -----------------------------------------------\n",
suggestions,
"length: " + suggestions.length
)
)
);
*/

engineInstance
.getFullBoard({
prompt,
maxSuggestions,
symbolSet,
language,
})
.then((board) =>
console.log(
"\nFull board \n",
board
)
);

/*
// Get suggestions with GlobalSymbol and Pictonizer images
Expand Down
136 changes: 135 additions & 1 deletion src/engine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ export type Suggestion = {
id: string;
label: string;
locale: string;
category?: string;
pictogram: {
isAIGenerated: boolean;
images:
Expand Down Expand Up @@ -52,6 +53,11 @@ export type ContentSafetyConfiguration = {
key: string;
};

export type Board = {
boardTitle: string;
pictos: Suggestion[];
};

export function init({
openAIConfiguration,
globalSymbolsApiURL,
Expand Down Expand Up @@ -83,9 +89,11 @@ export function init({
pictonizer,
getSuggestionsAndProcessPictograms,
isContentSafe,
getFullBoard,
};
}

//Using a prompt get a list of suggested words.
async function getWordSuggestions({
prompt,
maxWords,
Expand Down Expand Up @@ -129,6 +137,7 @@ async function getWordSuggestions({
throw new Error("ERROR: Suggestion list is empty");
}

//Based on a list of words fetch the Pictogram URL from GlobalSymbols API
async function fetchPictogramsURLs({
words,
symbolSet,
Expand Down Expand Up @@ -254,6 +263,7 @@ async function processPictograms(
return suggestionsWithAIImage;
}

//Get suggested Pictograms based on a prompt.
async function getSuggestions({
prompt,
maxSuggestions = DEFAULT_MAX_SUGGESTIONS,
Expand All @@ -276,7 +286,6 @@ async function getSuggestions({
symbolSet,
language,
});

return suggestionsWithGlobalSymbolsImages;
}

Expand All @@ -303,6 +312,7 @@ const getSuggestionsAndProcessPictograms = async ({
return suggestionsWithAIImages;
};

//Use Azure ContentSafety API to check if a string is safe for all users.
async function isContentSafe(
textPrompt: string,
): Promise<boolean> {
Expand All @@ -328,3 +338,127 @@ async function isContentSafe(
}

}


//Using a list of words get a descriptive title.
async function getBoardTitle(
words: string[],
language: string
): Promise<string> {
const max_tokens = Math.round(2 * words.length + 110);
const completionRequestParams = {
model: "gpt-3.5-turbo-instruct",
prompt: `act as a speech pathologist in language ${language}
usign this list of words {${words}} create a descriptive title for a communication board.
Here are mandatory instructions for the list:
-The title is 4 words maximum.
-It is very important to not repeat words.
-Do not add any other text or characters to the title.`,
temperature: 0,
max_tokens: max_tokens,
};

const response = await globalConfiguration.openAIInstance.createCompletion(
completionRequestParams
);
const titleSuggestionsData = response.data?.choices[0]?.text;
if (titleSuggestionsData) {
return titleSuggestionsData.replace(/^\n\n/, '');
} else {
throw new Error("Error fetching AI title");
}
}


//Get a board with a title, a content desciption and Pictograms based on a prompt.
async function getFullBoard({
prompt,
maxSuggestions = DEFAULT_MAX_SUGGESTIONS,
symbolSet,
language = DEFAULT_LANGUAGE,
}: {
prompt: string;
maxSuggestions: number;
symbolSet?: string;
language: string;
}): Promise<Board> {
const words: string[] = await getWordSuggestions({
prompt,
maxWords: maxSuggestions,
language,
});
//TODO we can check here if the word suggestions are safe @rodrisanchez
const title: string = await getBoardTitle(words,language);
console.log("Title: " + title);
const categories: Map<string, string[]> = await getBoardContent(words,language);
//log
const categoryObject = Object.fromEntries(categories);
console.log("Categories:", categoryObject);
//
const suggestionsWithGlobalSymbolsImages: Suggestion[] =
await fetchPictogramsURLs({
words,
symbolSet,
language,
});
categorizePictos(suggestionsWithGlobalSymbolsImages,categories)
return {
boardTitle: title,
pictos:suggestionsWithGlobalSymbolsImages,
};
}


//Using a list of words get a descriptive title.
async function getBoardContent(
words: string[],
language: string
): Promise<Map<string, string[]>> {
const max_tokens = Math.round(2 * words.length + 110);
const completionRequestParams = {
model: "gpt-3.5-turbo-instruct",
prompt: `given this list of words {${words}} you have to sort all of them into one of the following categories only.
For default use DEF. Categories:{PEOPLE, EMOTIONS, FOOD, PLACES, NATURE, OTHER}
Use the following template for the response {Category1:word1; Category2:word2,word3; Category3:word4,...}`,
//TODO its a good idea to use the content category for each picto instead of the whole board.
temperature: 0,
max_tokens: max_tokens,
};
const response = await globalConfiguration.openAIInstance.createCompletion(
completionRequestParams
);
const contentSuggestionData = response.data?.choices[0]?.text;
console.log("contentSuggestionData" + contentSuggestionData);
if (contentSuggestionData) {
return parseContent(contentSuggestionData);
} else {
throw new Error("Error fetching AI categories");
}
}


// Function to parse categories and words from the board content
function parseContent(content: string): Map<string, string[]> {
const categoryMap = new Map<string, string[]>();
const matches = content.match(/([^.]*)/);
if (matches && matches[1]) {
matches[1].split(';').forEach(pair => {
const [category, wordsString] = pair.split(':').map(item => item.trim());
const words = wordsString.split(',').map(word => word.trim());
categoryMap.set(category, words);
});
}

return categoryMap;
}

// Function to add categories to pictos
function categorizePictos(pictos: Suggestion[], categoryMap: Map<string, string[]>): void {
pictos.forEach(picto => {
categoryMap.forEach((words, category) => {
if (words.some(word => word.toLowerCase() === picto.label.toLowerCase())) {
picto.category = category;
}
});
});
}