Update output schema to not use {"string": string_value} as output da…

…ta (#637)

Update output schema to not use {"string": string_value} as output data

See comment in
#610 (comment)

Now we're just going to directly check for the type being a pure string
and not embedded within the `OutputData` object. I also renamed
`OutputData` --> `OutputDataWithValue` to make this more explicit and
easier to understand

Rebuilt the schema using `yarn gen-schema` from `aiconfig/typescript`
dir

---
Stack created with [Sapling](https://sapling-scm.com). Best reviewed
with
[ReviewStack](https://reviewstack.dev/lastmile-ai/aiconfig/pull/637).
* #610
* __->__ #637

extensions/HuggingFace/python/src/aiconfig_extension_hugging_face/local_inference/text_2_image.py

@@ -16,7 +16,7 @@
 from aiconfig.schema import (
     ExecuteResult,
     Output,
-    OutputData,
+    OutputDataWithValue,
     Prompt,
     PromptMetadata,
 )
@@ -141,7 +141,7 @@ def pillow_image_to_base64_string(img : Image.Image):
         img.save(buffered, format="PNG")
         return base64.b64encode(buffered.getvalue()).decode("utf-8")
 
-    data = OutputData(
+    data = OutputDataWithValue(
         kind="base64",
         value=pillow_image_to_base64_string(image),
     )
@@ -346,7 +346,7 @@ def get_output_text(
         # TODO (rossdanlm): Handle multiple outputs in list
         # https://github.com/lastmile-ai/aiconfig/issues/467
         if output.output_type == "execute_result":
-            if isinstance(output.data, OutputData):
+            if isinstance(output.data, OutputDataWithValue):
                 return output.data.value
             elif isinstance(output.data, str):
                 return output.data

python/src/aiconfig/__init__.py

@@ -20,7 +20,7 @@
     JSONObject,
     ModelMetadata,
     Output,
-    OutputData,
+    OutputDataWithValue,
     Prompt,
     PromptInput,
     PromptMetadata,

python/src/aiconfig/default_parsers/dalle.py

@@ -6,7 +6,7 @@
 from aiconfig.schema import (
     ExecuteResult,
     Output,
-    OutputData,
+    OutputDataWithValue,
     Prompt,
     PromptMetadata,
 )
@@ -48,9 +48,9 @@ def refine_image_completion_params(model_settings):
 def construct_output(image_data: Image, execution_count: int) -> Output:
     data = None
     if image_data.b64_json is not None:
-        data = OutputData(kind="base64", value=image_data.b64_json)
+        data = OutputDataWithValue(kind="base64", value=image_data.b64_json)
     elif image_data.url is not None:
-        data = OutputData(kind="file_uri", value=image_data.url)
+        data = OutputDataWithValue(kind="file_uri", value=image_data.url)
     else:
         raise ValueError(
             f"Did not receive a valid image type from image_data: {image_data}"
@@ -215,7 +215,7 @@ def get_output_text(
         # TODO (rossdanlm): Handle multiple outputs in list
         # https://github.com/lastmile-ai/aiconfig/issues/467
         if output.output_type == "execute_result":
-            if isinstance(output.data, OutputData):
+            if isinstance(output.data, OutputDataWithValue):
                 return output.data.value
             elif isinstance(output.data, str):
                 return output.data

python/src/aiconfig/schema.py

@@ -12,10 +12,12 @@
 
 class OutputDataWithStringValue(BaseModel):
     """
-    OutputData represents the output content in a standard format.
+    This represents the output content that is storied as a string, but we use
+    both the `kind` field here and the `mime_type` in ExecuteResult to convert
+    the string into the output format we want.
     """
 
-    kind: Literal["string", "file_uri", "base64"]
+    kind: Literal["file_uri", "base64"]
     value: str
 
 
@@ -65,7 +67,10 @@ class OutputDataWithToolCallsValue(BaseModel):
     value: List[ToolCallData]
 
 
-OutputData = Union[OutputDataWithStringValue, OutputDataWithToolCallsValue]
+OutputDataWithValue = Union[
+    OutputDataWithStringValue,
+    OutputDataWithToolCallsValue,
+]
 
 
 class ExecuteResult(BaseModel):
@@ -78,7 +83,7 @@ class ExecuteResult(BaseModel):
     # nth choice.
     execution_count: Union[int, None] = None
     # The result of the executing prompt.
-    data: Union[OutputData, Any]
+    data: Union[OutputDataWithValue, str, Any]
     # The MIME type of the result. If not specified, the MIME type will be assumed to be plain text.
     mime_type: Optional[str] = None
     # Output metadata

schema/aiconfig.schema.json

@@ -245,13 +245,13 @@
                   }
                 },
                 {
+                  "description": "This represents the output content that is storied as a string, but we use\nboth the `kind` field here and the `mime_type` in ExecuteResult to convert\nthe string into the output format we want.",
                   "type": "object",
                   "properties": {
                     "kind": {
                       "enum": [
                         "base64",
-                        "file_uri",
-                        "string"
+                        "file_uri"
                       ],
                       "type": "string"
                     },

typescript/lib/parsers/openai.ts

@@ -252,7 +252,7 @@ export class OpenAIModelParser extends ParameterizedModelParser<CompletionCreate
     }
 
     if (output.output_type === "execute_result") {
-      // TODO: Add in OutputData another way to support function calls
+      // TODO: Add in OutputDataWithValue another way to support function calls
       if (typeof output.data === "string") {
         return output.data;
       }
@@ -645,7 +645,7 @@ export class OpenAIChatModelParser extends ParameterizedModelParser<Chat.ChatCom
     }
 
     if (output.output_type === "execute_result") {
-      // TODO: Add in OutputData another way to support function calls
+      // TODO: Add in OutputDataWithValue another way to support function calls
       if (typeof output.data === "string") {
         return output.data;
       }

typescript/types.ts

@@ -174,8 +174,13 @@ export type Prompt = {
  */
 export type Output = ExecuteResult | Error;
 
+/**
+ * This represents the output content that is storied as a string, but we use
+ * both the `kind` field here and the `mime_type` in ExecuteResult to convert
+ * the string into the output format we want.
+ */
 type OutputDataWithStringValue = {
-  kind: "string" | "file_uri" | "base64";
+  kind: "file_uri" | "base64";
   value: string;
 };
 
@@ -224,7 +229,7 @@ export type OutputDataWithToolCallsValue = {
  * The output type of the result from executing a prompt.
  * We use this the kind field to determine how to parse it.
  */
-export type OutputData =
+export type OutputDataWithValue =
   | OutputDataWithStringValue
   | OutputDataWithToolCallsValue;
 
@@ -245,7 +250,7 @@ export type ExecuteResult = {
   /**
    * The result of executing the prompt.
    */
-  data: OutputData | JSONValue;
+  data: OutputDataWithValue | string | JSONValue;
 
   /**
    * The MIME type of the result. If not specified, the MIME type will be assumed to be plain text.