Merge 77a6d7d into f6b59f6

firebase · Feb 11, 2025 · b05fee0 · b05fee0
2 parents f6b59f6 + 77a6d7d
commit b05fee0
Show file tree

Hide file tree

Showing 27 changed files with 965 additions and 111 deletions.
diff --git a/firebase-vertexai/CHANGELOG.md b/firebase-vertexai/CHANGELOG.md
@@ -1,6 +1,7 @@
 # Unreleased
 * [fixed] Added support for new values sent by the server for `FinishReason` and `BlockReason`.
 * [changed] Added support for modality-based token count. (#6658)
+* [feature] Added support for generating images with Imagen models.
 
 # 16.1.0
 * [changed] Internal improvements to correctly handle empty model responses.
@@ -65,3 +66,4 @@
 * [feature] Added support for `responseMimeType` in `GenerationConfig`.
 * [changed] Renamed `GoogleGenerativeAIException` to `FirebaseVertexAIException`.
 * [changed] Updated the KDocs for various classes and functions.
+
diff --git a/firebase-vertexai/api.txt b/firebase-vertexai/api.txt
@@ -25,6 +25,10 @@ package com.google.firebase.vertexai {
     method public static com.google.firebase.vertexai.FirebaseVertexAI getInstance(com.google.firebase.FirebaseApp app);
     method public static com.google.firebase.vertexai.FirebaseVertexAI getInstance(com.google.firebase.FirebaseApp app = Firebase.app, String location);
     method public static com.google.firebase.vertexai.FirebaseVertexAI getInstance(String location);
+    method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName);
+    method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null);
+    method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.vertexai.type.ImagenSafetySettings? safetySettings = null);
+    method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.vertexai.type.ImagenSafetySettings? safetySettings = null, com.google.firebase.vertexai.type.RequestOptions requestOptions = com.google.firebase.vertexai.type.RequestOptions());
     property public static final com.google.firebase.vertexai.FirebaseVertexAI instance;
     field public static final com.google.firebase.vertexai.FirebaseVertexAI.Companion Companion;
   }
@@ -55,6 +59,10 @@ package com.google.firebase.vertexai {
     method public com.google.firebase.vertexai.Chat startChat(java.util.List<com.google.firebase.vertexai.type.Content> history = emptyList());
   }
 
+  @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenModel {
+    method public suspend Object? generateImages(String prompt, kotlin.coroutines.Continuation<? super com.google.firebase.vertexai.type.ImagenGenerationResponse<com.google.firebase.vertexai.type.ImagenInlineImage>>);
+  }
+
 }
 
 package com.google.firebase.vertexai.java {
@@ -86,6 +94,17 @@ package com.google.firebase.vertexai.java {
     method public com.google.firebase.vertexai.java.GenerativeModelFutures from(com.google.firebase.vertexai.GenerativeModel model);
   }
 
+  @com.google.firebase.vertexai.type.PublicPreviewAPI public abstract class ImagenModelFutures {
+    method public static final com.google.firebase.vertexai.java.ImagenModelFutures from(com.google.firebase.vertexai.ImagenModel model);
+    method public abstract com.google.common.util.concurrent.ListenableFuture<com.google.firebase.vertexai.type.ImagenGenerationResponse<com.google.firebase.vertexai.type.ImagenInlineImage>> generateImages(String prompt);
+    method public abstract com.google.firebase.vertexai.ImagenModel getImageModel();
+    field public static final com.google.firebase.vertexai.java.ImagenModelFutures.Companion Companion;
+  }
+
+  public static final class ImagenModelFutures.Companion {
+    method public com.google.firebase.vertexai.java.ImagenModelFutures from(com.google.firebase.vertexai.ImagenModel model);
+  }
+
 }
 
 package com.google.firebase.vertexai.type {
@@ -163,6 +182,9 @@ package com.google.firebase.vertexai.type {
     property public final String? role;
   }
 
+  public final class ContentBlockedException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
+  }
+
   public final class ContentKt {
     method public static com.google.firebase.vertexai.type.Content content(String? role = "user", kotlin.jvm.functions.Function1<? super com.google.firebase.vertexai.type.Content.Builder,kotlin.Unit> init);
   }
@@ -376,6 +398,104 @@ package com.google.firebase.vertexai.type {
     property public final android.graphics.Bitmap image;
   }
 
+  @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenAspectRatio {
+    field public static final com.google.firebase.vertexai.type.ImagenAspectRatio.Companion Companion;
+    field public static final com.google.firebase.vertexai.type.ImagenAspectRatio LANDSCAPE_16x9;
+    field public static final com.google.firebase.vertexai.type.ImagenAspectRatio LANDSCAPE_4x3;
+    field public static final com.google.firebase.vertexai.type.ImagenAspectRatio PORTRAIT_3x4;
+    field public static final com.google.firebase.vertexai.type.ImagenAspectRatio PORTRAIT_9x16;
+    field public static final com.google.firebase.vertexai.type.ImagenAspectRatio SQUARE_1x1;
+  }
+
+  public static final class ImagenAspectRatio.Companion {
+  }
+
+  @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenGenerationConfig {
+    ctor public ImagenGenerationConfig(String? negativePrompt = null, Integer? numberOfImages = 1, com.google.firebase.vertexai.type.ImagenAspectRatio? aspectRatio = null, com.google.firebase.vertexai.type.ImagenImageFormat? imageFormat = null, Boolean? addWatermark = null);
+    method public Boolean? getAddWatermark();
+    method public com.google.firebase.vertexai.type.ImagenAspectRatio? getAspectRatio();
+    method public com.google.firebase.vertexai.type.ImagenImageFormat? getImageFormat();
+    method public String? getNegativePrompt();
+    method public Integer? getNumberOfImages();
+    property public final Boolean? addWatermark;
+    property public final com.google.firebase.vertexai.type.ImagenAspectRatio? aspectRatio;
+    property public final com.google.firebase.vertexai.type.ImagenImageFormat? imageFormat;
+    property public final String? negativePrompt;
+    property public final Integer? numberOfImages;
+    field public static final com.google.firebase.vertexai.type.ImagenGenerationConfig.Companion Companion;
+  }
+
+  public static final class ImagenGenerationConfig.Builder {
+    ctor public ImagenGenerationConfig.Builder();
+    method public com.google.firebase.vertexai.type.ImagenGenerationConfig build();
+    field public Boolean? addWatermark;
+    field public com.google.firebase.vertexai.type.ImagenAspectRatio? aspectRatio;
+    field public com.google.firebase.vertexai.type.ImagenImageFormat? imageFormat;
+    field public String? negativePrompt;
+    field public Integer? numberOfImages;
+  }
+
+  public static final class ImagenGenerationConfig.Companion {
+    method public com.google.firebase.vertexai.type.ImagenGenerationConfig.Builder builder();
+  }
+
+  public final class ImagenGenerationConfigKt {
+    method @com.google.firebase.vertexai.type.PublicPreviewAPI public static com.google.firebase.vertexai.type.ImagenGenerationConfig imagenGenerationConfig(kotlin.jvm.functions.Function1<? super com.google.firebase.vertexai.type.ImagenGenerationConfig.Builder,kotlin.Unit> init);
+  }
+
+  @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenGenerationResponse<T> {
+    method public String? getFilteredReason();
+    method public java.util.List<T> getImages();
+    property public final String? filteredReason;
+    property public final java.util.List<T> images;
+  }
+
+  @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenImageFormat {
+    method public Integer? getCompressionQuality();
+    method public String getMimeType();
+    property public final Integer? compressionQuality;
+    property public final String mimeType;
+    field public static final com.google.firebase.vertexai.type.ImagenImageFormat.Companion Companion;
+  }
+
+  public static final class ImagenImageFormat.Companion {
+    method public com.google.firebase.vertexai.type.ImagenImageFormat jpeg(Integer? compressionQuality = null);
+    method public com.google.firebase.vertexai.type.ImagenImageFormat png();
+  }
+
+  @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenInlineImage {
+    method public android.graphics.Bitmap asBitmap();
+    method public byte[] getData();
+    method public String getMimeType();
+    property public final byte[] data;
+    property public final String mimeType;
+  }
+
+  @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenPersonFilterLevel {
+    field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel ALLOW_ADULT;
+    field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel ALLOW_ALL;
+    field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel BLOCK_ALL;
+    field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel.Companion Companion;
+  }
+
+  public static final class ImagenPersonFilterLevel.Companion {
+  }
+
+  @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenSafetyFilterLevel {
+    field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_LOW_AND_ABOVE;
+    field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_MEDIUM_AND_ABOVE;
+    field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_NONE;
+    field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_ONLY_HIGH;
+    field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel.Companion Companion;
+  }
+
+  public static final class ImagenSafetyFilterLevel.Companion {
+  }
+
+  @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenSafetySettings {
+    ctor public ImagenSafetySettings(com.google.firebase.vertexai.type.ImagenSafetyFilterLevel safetyFilterLevel, com.google.firebase.vertexai.type.ImagenPersonFilterLevel personFilterLevel);
+  }
+
   public final class InlineDataPart implements com.google.firebase.vertexai.type.Part {
     ctor public InlineDataPart(byte[] inlineData, String mimeType);
     method public byte[] getInlineData();
@@ -413,8 +533,8 @@ package com.google.firebase.vertexai.type {
   }
 
   public final class PromptBlockedException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
-    method public com.google.firebase.vertexai.type.GenerateContentResponse getResponse();
-    property public final com.google.firebase.vertexai.type.GenerateContentResponse response;
+    method public com.google.firebase.vertexai.type.GenerateContentResponse? getResponse();
+    property public final com.google.firebase.vertexai.type.GenerateContentResponse? response;
   }
 
   public final class PromptFeedback {
@@ -427,6 +547,9 @@ package com.google.firebase.vertexai.type {
     property public final java.util.List<com.google.firebase.vertexai.type.SafetyRating> safetyRatings;
   }
 
+  @kotlin.RequiresOptIn(level=kotlin.RequiresOptIn.Level.ERROR, message="This API is part of an experimental public preview and may change in " + "backwards-incompatible ways without notice.") @kotlin.annotation.Retention(kotlin.annotation.AnnotationRetention.BINARY) public @interface PublicPreviewAPI {
+  }
+
   public final class RequestOptions {
     ctor public RequestOptions();
     ctor public RequestOptions(long timeoutInMillis = 180.seconds.inWholeMilliseconds);

diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/FirebaseVertexAI.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/FirebaseVertexAI.kt
@@ -24,7 +24,10 @@ import com.google.firebase.auth.internal.InternalAuthProvider
 import com.google.firebase.inject.Provider
 import com.google.firebase.vertexai.type.Content
 import com.google.firebase.vertexai.type.GenerationConfig
+import com.google.firebase.vertexai.type.ImagenGenerationConfig
+import com.google.firebase.vertexai.type.ImagenSafetySettings
 import com.google.firebase.vertexai.type.InvalidLocationException
+import com.google.firebase.vertexai.type.PublicPreviewAPI
 import com.google.firebase.vertexai.type.RequestOptions
 import com.google.firebase.vertexai.type.SafetySetting
 import com.google.firebase.vertexai.type.Tool
@@ -79,6 +82,37 @@ internal constructor(
     )
   }
 
+  /**
+   * Instantiates a new [ImagenModel] given the provided parameters.
+   *
+   * @param modelName The name of the model to use, for example `"imagen-3.0-generate-001"`.
+   * @param generationConfig The configuration parameters to use for image generation.
+   * @param safetySettings The safety bounds the model will abide by during image generation.
+   * @param requestOptions Configuration options for sending requests to the backend.
+   * @return The initialized [ImagenModel] instance.
+   */
+  @JvmOverloads
+  @PublicPreviewAPI
+  public fun imagenModel(
+    modelName: String,
+    generationConfig: ImagenGenerationConfig? = null,
+    safetySettings: ImagenSafetySettings? = null,
+    requestOptions: RequestOptions = RequestOptions(),
+  ): ImagenModel {
+    if (location.trim().isEmpty() || location.contains("/")) {
+      throw InvalidLocationException(location)
+    }
+    return ImagenModel(
+      "projects/${firebaseApp.options.projectId}/locations/${location}/publishers/google/models/${modelName}",
+      firebaseApp.options.apiKey,
+      generationConfig,
+      safetySettings,
+      requestOptions,
+      appCheckProvider.get(),
+      internalAuthProvider.get(),
+    )
+  }
+
   public companion object {
     /** The [FirebaseVertexAI] instance for the default [FirebaseApp] */
     @JvmStatic

diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/GenerativeModel.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/GenerativeModel.kt
@@ -17,13 +17,12 @@
 package com.google.firebase.vertexai
 
 import android.graphics.Bitmap
-import android.util.Log
 import com.google.firebase.appcheck.interop.InteropAppCheckTokenProvider
 import com.google.firebase.auth.internal.InternalAuthProvider
 import com.google.firebase.vertexai.common.APIController
+import com.google.firebase.vertexai.common.AppCheckHeaderProvider
 import com.google.firebase.vertexai.common.CountTokensRequest
 import com.google.firebase.vertexai.common.GenerateContentRequest
-import com.google.firebase.vertexai.common.HeaderProvider
 import com.google.firebase.vertexai.type.Content
 import com.google.firebase.vertexai.type.CountTokensResponse
 import com.google.firebase.vertexai.type.FinishReason
@@ -38,12 +37,9 @@ import com.google.firebase.vertexai.type.SerializationException
 import com.google.firebase.vertexai.type.Tool
 import com.google.firebase.vertexai.type.ToolConfig
 import com.google.firebase.vertexai.type.content
-import kotlin.time.Duration
-import kotlin.time.Duration.Companion.seconds
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.catch
 import kotlinx.coroutines.flow.map
-import kotlinx.coroutines.tasks.await
 
 /**
  * Represents a multimodal model (like Gemini), capable of generating content based on various input
@@ -57,10 +53,8 @@ internal constructor(
   private val tools: List<Tool>? = null,
   private val toolConfig: ToolConfig? = null,
   private val systemInstruction: Content? = null,
-  private val controller: APIController
+  private val controller: APIController,
 ) {
-
-  @JvmOverloads
   internal constructor(
     modelName: String,
     apiKey: String,
@@ -84,42 +78,8 @@ internal constructor(
       modelName,
       requestOptions,
       "gl-kotlin/${KotlinVersion.CURRENT} fire/${BuildConfig.VERSION_NAME}",
-      object : HeaderProvider {
-        override val timeout: Duration
-          get() = 10.seconds
-
-        override suspend fun generateHeaders(): Map<String, String> {
-          val headers = mutableMapOf<String, String>()
-          if (appCheckTokenProvider == null) {
-            Log.w(TAG, "AppCheck not registered, skipping")
-          } else {
-            val token = appCheckTokenProvider.getToken(false).await()
-
-            if (token.error != null) {
-              Log.w(TAG, "Error obtaining AppCheck token", token.error)
-            }
-            // The Firebase App Check backend can differentiate between apps without App Check, and
-            // wrongly configured apps by verifying the value of the token, so it always needs to be
-            // included.
-            headers["X-Firebase-AppCheck"] = token.token
-          }
-
-          if (internalAuthProvider == null) {
-            Log.w(TAG, "Auth not registered, skipping")
-          } else {
-            try {
-              val token = internalAuthProvider.getAccessToken(false).await()
-
-              headers["Authorization"] = "Firebase ${token.token!!}"
-            } catch (e: Exception) {
-              Log.w(TAG, "Error getting Auth token ", e)
-            }
-          }
-
-          return headers
-        }
-      }
-    )
+      AppCheckHeaderProvider(TAG, appCheckTokenProvider, internalAuthProvider),
+    ),
   )
 
   /**
@@ -247,7 +207,7 @@ internal constructor(
       generationConfig?.toInternal(),
       tools?.map { it.toInternal() },
       toolConfig?.toInternal(),
-      systemInstruction?.copy(role = "system")?.toInternal()
+      systemInstruction?.copy(role = "system")?.toInternal(),
     )
 
   private fun constructCountTokensRequest(vararg prompt: Content) =