feat: add support to ChatMessage for multimodal content

src/types.ts

@@ -21,6 +21,26 @@ export type ToolCall = {
   function: FunctionCall;
 };
 
+/** Used for multimodal chat messages. */
+export type ChatMessageContentPart =
+  | {
+      type: 'text';
+      text: string;
+    }
+  // User messages only.
+  | {
+      type: 'image_url';
+      image_url: {
+        url: string;
+        detail?: 'low' | 'high' | ('auto' & (string & {}));
+      };
+    }
+  // Assistant messages only.
+  | {
+      type: 'refusal';
+      refusal: string;
+    };
+
 export type ChatMessage = {
   /**
    * The role of the messages author. One of `system`, `user`, `assistant`,
@@ -32,8 +52,11 @@ export type ChatMessage = {
    * The contents of the message. `content` may be null for assistant messages
    * with function calls or for assistant messages if a `refusal` was given by
    * the model.
+   *
+   * `content` will most often be a `string` for simple text messages, but can
+   * also be an array for multimodal messages.
    */
-  content?: string | null;
+  content?: string | ChatMessageContentPart[] | null;
 
   /**
    * The refusal message if one was generated by the model.