diff --git a/firebase-vertexai/CHANGELOG.md b/firebase-vertexai/CHANGELOG.md index 557a2f10589..c0bfdfb214e 100644 --- a/firebase-vertexai/CHANGELOG.md +++ b/firebase-vertexai/CHANGELOG.md @@ -1,6 +1,7 @@ # Unreleased * [fixed] Added support for new values sent by the server for `FinishReason` and `BlockReason`. * [changed] Added support for modality-based token count. (#6658) +* [feature] Added support for generating images with Imagen models. # 16.1.0 * [changed] Internal improvements to correctly handle empty model responses. @@ -65,3 +66,4 @@ * [feature] Added support for `responseMimeType` in `GenerationConfig`. * [changed] Renamed `GoogleGenerativeAIException` to `FirebaseVertexAIException`. * [changed] Updated the KDocs for various classes and functions. + diff --git a/firebase-vertexai/api.txt b/firebase-vertexai/api.txt index 7bb2f629c51..b49da4c9769 100644 --- a/firebase-vertexai/api.txt +++ b/firebase-vertexai/api.txt @@ -25,6 +25,10 @@ package com.google.firebase.vertexai { method public static com.google.firebase.vertexai.FirebaseVertexAI getInstance(com.google.firebase.FirebaseApp app); method public static com.google.firebase.vertexai.FirebaseVertexAI getInstance(com.google.firebase.FirebaseApp app = Firebase.app, String location); method public static com.google.firebase.vertexai.FirebaseVertexAI getInstance(String location); + method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName); + method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null); + method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.vertexai.type.ImagenSafetySettings? safetySettings = null); + method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.vertexai.type.ImagenSafetySettings? safetySettings = null, com.google.firebase.vertexai.type.RequestOptions requestOptions = com.google.firebase.vertexai.type.RequestOptions()); property public static final com.google.firebase.vertexai.FirebaseVertexAI instance; field public static final com.google.firebase.vertexai.FirebaseVertexAI.Companion Companion; } @@ -55,6 +59,10 @@ package com.google.firebase.vertexai { method public com.google.firebase.vertexai.Chat startChat(java.util.List history = emptyList()); } + @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenModel { + method public suspend Object? generateImages(String prompt, kotlin.coroutines.Continuation>); + } + } package com.google.firebase.vertexai.java { @@ -86,6 +94,17 @@ package com.google.firebase.vertexai.java { method public com.google.firebase.vertexai.java.GenerativeModelFutures from(com.google.firebase.vertexai.GenerativeModel model); } + @com.google.firebase.vertexai.type.PublicPreviewAPI public abstract class ImagenModelFutures { + method public static final com.google.firebase.vertexai.java.ImagenModelFutures from(com.google.firebase.vertexai.ImagenModel model); + method public abstract com.google.common.util.concurrent.ListenableFuture> generateImages(String prompt); + method public abstract com.google.firebase.vertexai.ImagenModel getImageModel(); + field public static final com.google.firebase.vertexai.java.ImagenModelFutures.Companion Companion; + } + + public static final class ImagenModelFutures.Companion { + method public com.google.firebase.vertexai.java.ImagenModelFutures from(com.google.firebase.vertexai.ImagenModel model); + } + } package com.google.firebase.vertexai.type { @@ -163,6 +182,9 @@ package com.google.firebase.vertexai.type { property public final String? role; } + public final class ContentBlockedException extends com.google.firebase.vertexai.type.FirebaseVertexAIException { + } + public final class ContentKt { method public static com.google.firebase.vertexai.type.Content content(String? role = "user", kotlin.jvm.functions.Function1 init); } @@ -376,6 +398,104 @@ package com.google.firebase.vertexai.type { property public final android.graphics.Bitmap image; } + @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenAspectRatio { + field public static final com.google.firebase.vertexai.type.ImagenAspectRatio.Companion Companion; + field public static final com.google.firebase.vertexai.type.ImagenAspectRatio LANDSCAPE_16x9; + field public static final com.google.firebase.vertexai.type.ImagenAspectRatio LANDSCAPE_4x3; + field public static final com.google.firebase.vertexai.type.ImagenAspectRatio PORTRAIT_3x4; + field public static final com.google.firebase.vertexai.type.ImagenAspectRatio PORTRAIT_9x16; + field public static final com.google.firebase.vertexai.type.ImagenAspectRatio SQUARE_1x1; + } + + public static final class ImagenAspectRatio.Companion { + } + + @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenGenerationConfig { + ctor public ImagenGenerationConfig(String? negativePrompt = null, Integer? numberOfImages = 1, com.google.firebase.vertexai.type.ImagenAspectRatio? aspectRatio = null, com.google.firebase.vertexai.type.ImagenImageFormat? imageFormat = null, Boolean? addWatermark = null); + method public Boolean? getAddWatermark(); + method public com.google.firebase.vertexai.type.ImagenAspectRatio? getAspectRatio(); + method public com.google.firebase.vertexai.type.ImagenImageFormat? getImageFormat(); + method public String? getNegativePrompt(); + method public Integer? getNumberOfImages(); + property public final Boolean? addWatermark; + property public final com.google.firebase.vertexai.type.ImagenAspectRatio? aspectRatio; + property public final com.google.firebase.vertexai.type.ImagenImageFormat? imageFormat; + property public final String? negativePrompt; + property public final Integer? numberOfImages; + field public static final com.google.firebase.vertexai.type.ImagenGenerationConfig.Companion Companion; + } + + public static final class ImagenGenerationConfig.Builder { + ctor public ImagenGenerationConfig.Builder(); + method public com.google.firebase.vertexai.type.ImagenGenerationConfig build(); + field public Boolean? addWatermark; + field public com.google.firebase.vertexai.type.ImagenAspectRatio? aspectRatio; + field public com.google.firebase.vertexai.type.ImagenImageFormat? imageFormat; + field public String? negativePrompt; + field public Integer? numberOfImages; + } + + public static final class ImagenGenerationConfig.Companion { + method public com.google.firebase.vertexai.type.ImagenGenerationConfig.Builder builder(); + } + + public final class ImagenGenerationConfigKt { + method @com.google.firebase.vertexai.type.PublicPreviewAPI public static com.google.firebase.vertexai.type.ImagenGenerationConfig imagenGenerationConfig(kotlin.jvm.functions.Function1 init); + } + + @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenGenerationResponse { + method public String? getFilteredReason(); + method public java.util.List getImages(); + property public final String? filteredReason; + property public final java.util.List images; + } + + @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenImageFormat { + method public Integer? getCompressionQuality(); + method public String getMimeType(); + property public final Integer? compressionQuality; + property public final String mimeType; + field public static final com.google.firebase.vertexai.type.ImagenImageFormat.Companion Companion; + } + + public static final class ImagenImageFormat.Companion { + method public com.google.firebase.vertexai.type.ImagenImageFormat jpeg(Integer? compressionQuality = null); + method public com.google.firebase.vertexai.type.ImagenImageFormat png(); + } + + @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenInlineImage { + method public android.graphics.Bitmap asBitmap(); + method public byte[] getData(); + method public String getMimeType(); + property public final byte[] data; + property public final String mimeType; + } + + @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenPersonFilterLevel { + field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel ALLOW_ADULT; + field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel ALLOW_ALL; + field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel BLOCK_ALL; + field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel.Companion Companion; + } + + public static final class ImagenPersonFilterLevel.Companion { + } + + @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenSafetyFilterLevel { + field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_LOW_AND_ABOVE; + field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_MEDIUM_AND_ABOVE; + field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_NONE; + field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_ONLY_HIGH; + field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel.Companion Companion; + } + + public static final class ImagenSafetyFilterLevel.Companion { + } + + @com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenSafetySettings { + ctor public ImagenSafetySettings(com.google.firebase.vertexai.type.ImagenSafetyFilterLevel safetyFilterLevel, com.google.firebase.vertexai.type.ImagenPersonFilterLevel personFilterLevel); + } + public final class InlineDataPart implements com.google.firebase.vertexai.type.Part { ctor public InlineDataPart(byte[] inlineData, String mimeType); method public byte[] getInlineData(); @@ -413,8 +533,8 @@ package com.google.firebase.vertexai.type { } public final class PromptBlockedException extends com.google.firebase.vertexai.type.FirebaseVertexAIException { - method public com.google.firebase.vertexai.type.GenerateContentResponse getResponse(); - property public final com.google.firebase.vertexai.type.GenerateContentResponse response; + method public com.google.firebase.vertexai.type.GenerateContentResponse? getResponse(); + property public final com.google.firebase.vertexai.type.GenerateContentResponse? response; } public final class PromptFeedback { @@ -427,6 +547,9 @@ package com.google.firebase.vertexai.type { property public final java.util.List safetyRatings; } + @kotlin.RequiresOptIn(level=kotlin.RequiresOptIn.Level.ERROR, message="This API is part of an experimental public preview and may change in " + "backwards-incompatible ways without notice.") @kotlin.annotation.Retention(kotlin.annotation.AnnotationRetention.BINARY) public @interface PublicPreviewAPI { + } + public final class RequestOptions { ctor public RequestOptions(); ctor public RequestOptions(long timeoutInMillis = 180.seconds.inWholeMilliseconds); diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/FirebaseVertexAI.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/FirebaseVertexAI.kt index ff256482112..b89e5671992 100644 --- a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/FirebaseVertexAI.kt +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/FirebaseVertexAI.kt @@ -24,7 +24,10 @@ import com.google.firebase.auth.internal.InternalAuthProvider import com.google.firebase.inject.Provider import com.google.firebase.vertexai.type.Content import com.google.firebase.vertexai.type.GenerationConfig +import com.google.firebase.vertexai.type.ImagenGenerationConfig +import com.google.firebase.vertexai.type.ImagenSafetySettings import com.google.firebase.vertexai.type.InvalidLocationException +import com.google.firebase.vertexai.type.PublicPreviewAPI import com.google.firebase.vertexai.type.RequestOptions import com.google.firebase.vertexai.type.SafetySetting import com.google.firebase.vertexai.type.Tool @@ -79,6 +82,37 @@ internal constructor( ) } + /** + * Instantiates a new [ImagenModel] given the provided parameters. + * + * @param modelName The name of the model to use, for example `"imagen-3.0-generate-001"`. + * @param generationConfig The configuration parameters to use for image generation. + * @param safetySettings The safety bounds the model will abide by during image generation. + * @param requestOptions Configuration options for sending requests to the backend. + * @return The initialized [ImagenModel] instance. + */ + @JvmOverloads + @PublicPreviewAPI + public fun imagenModel( + modelName: String, + generationConfig: ImagenGenerationConfig? = null, + safetySettings: ImagenSafetySettings? = null, + requestOptions: RequestOptions = RequestOptions(), + ): ImagenModel { + if (location.trim().isEmpty() || location.contains("/")) { + throw InvalidLocationException(location) + } + return ImagenModel( + "projects/${firebaseApp.options.projectId}/locations/${location}/publishers/google/models/${modelName}", + firebaseApp.options.apiKey, + generationConfig, + safetySettings, + requestOptions, + appCheckProvider.get(), + internalAuthProvider.get(), + ) + } + public companion object { /** The [FirebaseVertexAI] instance for the default [FirebaseApp] */ @JvmStatic diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/GenerativeModel.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/GenerativeModel.kt index c2b9fcfd2f9..a49d4c279a8 100644 --- a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/GenerativeModel.kt +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/GenerativeModel.kt @@ -17,13 +17,12 @@ package com.google.firebase.vertexai import android.graphics.Bitmap -import android.util.Log import com.google.firebase.appcheck.interop.InteropAppCheckTokenProvider import com.google.firebase.auth.internal.InternalAuthProvider import com.google.firebase.vertexai.common.APIController +import com.google.firebase.vertexai.common.AppCheckHeaderProvider import com.google.firebase.vertexai.common.CountTokensRequest import com.google.firebase.vertexai.common.GenerateContentRequest -import com.google.firebase.vertexai.common.HeaderProvider import com.google.firebase.vertexai.type.Content import com.google.firebase.vertexai.type.CountTokensResponse import com.google.firebase.vertexai.type.FinishReason @@ -38,12 +37,9 @@ import com.google.firebase.vertexai.type.SerializationException import com.google.firebase.vertexai.type.Tool import com.google.firebase.vertexai.type.ToolConfig import com.google.firebase.vertexai.type.content -import kotlin.time.Duration -import kotlin.time.Duration.Companion.seconds import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.catch import kotlinx.coroutines.flow.map -import kotlinx.coroutines.tasks.await /** * Represents a multimodal model (like Gemini), capable of generating content based on various input @@ -57,10 +53,8 @@ internal constructor( private val tools: List? = null, private val toolConfig: ToolConfig? = null, private val systemInstruction: Content? = null, - private val controller: APIController + private val controller: APIController, ) { - - @JvmOverloads internal constructor( modelName: String, apiKey: String, @@ -84,42 +78,8 @@ internal constructor( modelName, requestOptions, "gl-kotlin/${KotlinVersion.CURRENT} fire/${BuildConfig.VERSION_NAME}", - object : HeaderProvider { - override val timeout: Duration - get() = 10.seconds - - override suspend fun generateHeaders(): Map { - val headers = mutableMapOf() - if (appCheckTokenProvider == null) { - Log.w(TAG, "AppCheck not registered, skipping") - } else { - val token = appCheckTokenProvider.getToken(false).await() - - if (token.error != null) { - Log.w(TAG, "Error obtaining AppCheck token", token.error) - } - // The Firebase App Check backend can differentiate between apps without App Check, and - // wrongly configured apps by verifying the value of the token, so it always needs to be - // included. - headers["X-Firebase-AppCheck"] = token.token - } - - if (internalAuthProvider == null) { - Log.w(TAG, "Auth not registered, skipping") - } else { - try { - val token = internalAuthProvider.getAccessToken(false).await() - - headers["Authorization"] = "Firebase ${token.token!!}" - } catch (e: Exception) { - Log.w(TAG, "Error getting Auth token ", e) - } - } - - return headers - } - } - ) + AppCheckHeaderProvider(TAG, appCheckTokenProvider, internalAuthProvider), + ), ) /** @@ -247,7 +207,7 @@ internal constructor( generationConfig?.toInternal(), tools?.map { it.toInternal() }, toolConfig?.toInternal(), - systemInstruction?.copy(role = "system")?.toInternal() + systemInstruction?.copy(role = "system")?.toInternal(), ) private fun constructCountTokensRequest(vararg prompt: Content) = diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/ImagenModel.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/ImagenModel.kt new file mode 100644 index 00000000000..583ef24bcc4 --- /dev/null +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/ImagenModel.kt @@ -0,0 +1,119 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.vertexai + +import com.google.firebase.appcheck.interop.InteropAppCheckTokenProvider +import com.google.firebase.auth.internal.InternalAuthProvider +import com.google.firebase.vertexai.common.APIController +import com.google.firebase.vertexai.common.AppCheckHeaderProvider +import com.google.firebase.vertexai.common.ContentBlockedException +import com.google.firebase.vertexai.common.GenerateImageRequest +import com.google.firebase.vertexai.type.FirebaseVertexAIException +import com.google.firebase.vertexai.type.ImagenGenerationConfig +import com.google.firebase.vertexai.type.ImagenGenerationResponse +import com.google.firebase.vertexai.type.ImagenInlineImage +import com.google.firebase.vertexai.type.ImagenSafetySettings +import com.google.firebase.vertexai.type.PublicPreviewAPI +import com.google.firebase.vertexai.type.RequestOptions + +/** + * Represents a generative model (like Imagen), capable of generating images based on various input + * types. + */ +@PublicPreviewAPI +public class ImagenModel +internal constructor( + private val modelName: String, + private val generationConfig: ImagenGenerationConfig? = null, + private val safetySettings: ImagenSafetySettings? = null, + private val controller: APIController, +) { + @JvmOverloads + internal constructor( + modelName: String, + apiKey: String, + generationConfig: ImagenGenerationConfig? = null, + safetySettings: ImagenSafetySettings? = null, + requestOptions: RequestOptions = RequestOptions(), + appCheckTokenProvider: InteropAppCheckTokenProvider? = null, + internalAuthProvider: InternalAuthProvider? = null, + ) : this( + modelName, + generationConfig, + safetySettings, + APIController( + apiKey, + modelName, + requestOptions, + "gl-kotlin/${KotlinVersion.CURRENT} fire/${BuildConfig.VERSION_NAME}", + AppCheckHeaderProvider(TAG, appCheckTokenProvider, internalAuthProvider), + ), + ) + + /** + * Generates an image, returning the result directly to the caller. + * + * @param prompt The input(s) given to the model as a prompt. + */ + public suspend fun generateImages(prompt: String): ImagenGenerationResponse = + try { + controller + .generateImage(constructRequest(prompt, null, generationConfig)) + .validate() + .toPublicInline() + } catch (e: Throwable) { + throw FirebaseVertexAIException.from(e) + } + + private fun constructRequest( + prompt: String, + gcsUri: String?, + config: ImagenGenerationConfig?, + ): GenerateImageRequest { + return GenerateImageRequest( + listOf(GenerateImageRequest.ImagenPrompt(prompt)), + GenerateImageRequest.ImagenParameters( + sampleCount = config?.numberOfImages ?: 1, + includeRaiReason = true, + addWatermark = generationConfig?.addWatermark, + personGeneration = safetySettings?.personFilterLevel?.internalVal, + negativePrompt = config?.negativePrompt, + safetySetting = safetySettings?.safetyFilterLevel?.internalVal, + storageUri = gcsUri, + aspectRatio = config?.aspectRatio?.internalVal, + imageOutputOptions = generationConfig?.imageFormat?.toInternal(), + ), + ) + } + + internal companion object { + private val TAG = ImagenModel::class.java.simpleName + internal const val DEFAULT_FILTERED_ERROR = + "Unable to show generated images. All images were filtered out because they violated Vertex AI's usage guidelines. You will not be charged for blocked images. Try rephrasing the prompt. If you think this was an error, send feedback." + } +} + +@OptIn(PublicPreviewAPI::class) +private fun ImagenGenerationResponse.Internal.validate(): ImagenGenerationResponse.Internal { + if (predictions.none { it.mimeType != null }) { + throw ContentBlockedException( + message = predictions.first { it.raiFilteredReason != null }.raiFilteredReason + ?: ImagenModel.DEFAULT_FILTERED_ERROR + ) + } + return this +} diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/common/APIController.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/common/APIController.kt index 286b8829241..f8bfe0bc24f 100644 --- a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/common/APIController.kt +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/common/APIController.kt @@ -25,6 +25,8 @@ import com.google.firebase.vertexai.type.CountTokensResponse import com.google.firebase.vertexai.type.FinishReason import com.google.firebase.vertexai.type.GRpcErrorResponse import com.google.firebase.vertexai.type.GenerateContentResponse +import com.google.firebase.vertexai.type.ImagenGenerationResponse +import com.google.firebase.vertexai.type.PublicPreviewAPI import com.google.firebase.vertexai.type.RequestOptions import com.google.firebase.vertexai.type.Response import io.ktor.client.HttpClient @@ -58,12 +60,15 @@ import kotlinx.coroutines.flow.channelFlow import kotlinx.coroutines.flow.map import kotlinx.coroutines.launch import kotlinx.coroutines.withTimeout +import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.json.Json +@OptIn(ExperimentalSerializationApi::class) internal val JSON = Json { ignoreUnknownKeys = true prettyPrint = false isLenient = true + explicitNulls = false } /** @@ -78,6 +83,7 @@ internal val JSON = Json { * @property apiClient The value to pass in the `x-goog-api-client` header. * @property headerProvider A provider that generates extra headers to include in all HTTP requests. */ +@OptIn(PublicPreviewAPI::class) internal class APIController internal constructor( private val key: String, @@ -122,6 +128,19 @@ internal constructor( throw FirebaseCommonAIException.from(e) } + suspend fun generateImage(request: GenerateImageRequest): ImagenGenerationResponse.Internal = + try { + client + .post("${requestOptions.endpoint}/${requestOptions.apiVersion}/$model:predict") { + applyCommonConfiguration(request) + applyHeaderProvider() + } + .also { validateResponse(it) } + .body() + } catch (e: Throwable) { + throw FirebaseCommonAIException.from(e) + } + fun generateContentStream( request: GenerateContentRequest ): Flow = @@ -151,6 +170,7 @@ internal constructor( when (request) { is GenerateContentRequest -> setBody(request) is CountTokensRequest -> setBody(request) + is GenerateImageRequest -> setBody(request) } contentType(ContentType.Application.Json) header("x-goog-api-key", key) @@ -258,6 +278,9 @@ private suspend fun validateResponse(response: HttpResponse) { if (message.contains("quota")) { throw QuotaExceededException(message) } + if (message.contains("The prompt could not be submitted")) { + throw PromptBlockedException(message) + } getServiceDisabledErrorDetailsOrNull(error)?.let { val errorMessage = if (it.metadata?.get("service") == "firebasevertexai.googleapis.com") { diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/common/AppCheckHeaderProvider.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/common/AppCheckHeaderProvider.kt new file mode 100644 index 00000000000..fb3b52ad46f --- /dev/null +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/common/AppCheckHeaderProvider.kt @@ -0,0 +1,64 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.vertexai.common + +import android.util.Log +import com.google.firebase.appcheck.interop.InteropAppCheckTokenProvider +import com.google.firebase.auth.internal.InternalAuthProvider +import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds +import kotlinx.coroutines.tasks.await + +internal class AppCheckHeaderProvider( + private val logTag: String, + private val appCheckTokenProvider: InteropAppCheckTokenProvider? = null, + private val internalAuthProvider: InternalAuthProvider? = null, +) : HeaderProvider { + override val timeout: Duration + get() = 10.seconds + + override suspend fun generateHeaders(): Map { + val headers = mutableMapOf() + if (appCheckTokenProvider == null) { + Log.w(logTag, "AppCheck not registered, skipping") + } else { + val token = appCheckTokenProvider.getToken(false).await() + + if (token.error != null) { + Log.w(logTag, "Error obtaining AppCheck token", token.error) + } + // The Firebase App Check backend can differentiate between apps without App Check, and + // wrongly configured apps by verifying the value of the token, so it always needs to be + // included. + headers["X-Firebase-AppCheck"] = token.token + } + + if (internalAuthProvider == null) { + Log.w(logTag, "Auth not registered, skipping") + } else { + try { + val token = internalAuthProvider.getAccessToken(false).await() + + headers["Authorization"] = "Firebase ${token.token!!}" + } catch (e: Exception) { + Log.w(logTag, "Error getting Auth token ", e) + } + } + + return headers + } +} diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/common/Exceptions.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/common/Exceptions.kt index 7567c384618..ad982f2daff 100644 --- a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/common/Exceptions.kt +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/common/Exceptions.kt @@ -66,14 +66,18 @@ internal class InvalidAPIKeyException(message: String, cause: Throwable? = null) * * @property response the full server response for the request. */ -internal class PromptBlockedException( - val response: GenerateContentResponse.Internal, - cause: Throwable? = null +internal class PromptBlockedException +internal constructor( + val response: GenerateContentResponse.Internal?, + cause: Throwable? = null, + message: String? = null, ) : FirebaseCommonAIException( - "Prompt was blocked: ${response.promptFeedback?.blockReason?.name}", + "Prompt was blocked: ${response?.promptFeedback?.blockReason?.name?: message}", cause, - ) + ) { + internal constructor(message: String, cause: Throwable? = null) : this(null, cause, message) +} /** * The user's location (region) is not supported by the API. @@ -127,6 +131,9 @@ internal class ServiceDisabledException(message: String, cause: Throwable? = nul internal class UnknownException(message: String, cause: Throwable? = null) : FirebaseCommonAIException(message, cause) +internal class ContentBlockedException(message: String, cause: Throwable? = null) : + FirebaseCommonAIException(message, cause) + internal fun makeMissingCaseException( source: String, ordinal: Int diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/common/Request.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/common/Request.kt index 040a38e0a0b..8696a090fc2 100644 --- a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/common/Request.kt +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/common/Request.kt @@ -19,13 +19,15 @@ package com.google.firebase.vertexai.common import com.google.firebase.vertexai.common.util.fullModelName import com.google.firebase.vertexai.type.Content import com.google.firebase.vertexai.type.GenerationConfig +import com.google.firebase.vertexai.type.ImagenImageFormat +import com.google.firebase.vertexai.type.PublicPreviewAPI import com.google.firebase.vertexai.type.SafetySetting import com.google.firebase.vertexai.type.Tool import com.google.firebase.vertexai.type.ToolConfig import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable -internal sealed interface Request +internal interface Request @Serializable internal data class GenerateContentRequest( @@ -65,3 +67,25 @@ internal data class CountTokensRequest( ) } } + +@Serializable +internal data class GenerateImageRequest( + val instances: List, + val parameters: ImagenParameters, +) : Request { + @Serializable internal data class ImagenPrompt(val prompt: String) + + @OptIn(PublicPreviewAPI::class) + @Serializable + internal data class ImagenParameters( + val sampleCount: Int, + val includeRaiReason: Boolean, + val storageUri: String?, + val negativePrompt: String?, + val aspectRatio: String?, + val safetySetting: String?, + val personGeneration: String?, + val addWatermark: Boolean?, + val imageOutputOptions: ImagenImageFormat.Internal?, + ) +} diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/internal/util/kotlin.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/internal/util/kotlin.kt deleted file mode 100644 index b4d68d2f14c..00000000000 --- a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/internal/util/kotlin.kt +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.firebase.vertexai.internal.util - -import java.lang.reflect.Field - -/** - * Removes the last character from the [StringBuilder]. - * - * If the StringBuilder is empty, calling this function will throw an [IndexOutOfBoundsException]. - * - * @return The [StringBuilder] used to make the call, for optional chaining. - * @throws IndexOutOfBoundsException if the StringBuilder is empty. - */ -internal fun StringBuilder.removeLast(): StringBuilder = - if (isEmpty()) throw IndexOutOfBoundsException("StringBuilder is empty.") - else deleteCharAt(length - 1) - -/** - * A variant of [getAnnotation][Field.getAnnotation] that provides implicit Kotlin support. - * - * Syntax sugar for: - * ``` - * getAnnotation(T::class.java) - * ``` - */ -internal inline fun Field.getAnnotation() = getAnnotation(T::class.java) diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/java/ImagenModelFutures.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/java/ImagenModelFutures.kt new file mode 100644 index 00000000000..97b043312c4 --- /dev/null +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/java/ImagenModelFutures.kt @@ -0,0 +1,59 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.vertexai.java + +import androidx.concurrent.futures.SuspendToFutureAdapter +import com.google.common.util.concurrent.ListenableFuture +import com.google.firebase.vertexai.ImagenModel +import com.google.firebase.vertexai.type.ImagenGenerationResponse +import com.google.firebase.vertexai.type.ImagenInlineImage +import com.google.firebase.vertexai.type.PublicPreviewAPI + +/** + * Wrapper class providing Java compatible methods for [ImagenModel]. + * + * @see [ImagenModel] + */ +@PublicPreviewAPI +public abstract class ImagenModelFutures internal constructor() { + /** + * Generates an image, returning the result directly to the caller. + * + * @param prompt The main text prompt from which the image is generated. + */ + public abstract fun generateImages( + prompt: String, + ): ListenableFuture> + + /** Returns the [ImagenModel] object wrapped by this object. */ + public abstract fun getImageModel(): ImagenModel + + private class FuturesImpl(private val model: ImagenModel) : ImagenModelFutures() { + override fun generateImages( + prompt: String, + ): ListenableFuture> = + SuspendToFutureAdapter.launchFuture { model.generateImages(prompt) } + + override fun getImageModel(): ImagenModel = model + } + + public companion object { + + /** @return a [ImagenModelFutures] created around the provided [ImagenModel] */ + @JvmStatic public fun from(model: ImagenModel): ImagenModelFutures = FuturesImpl(model) + } +} diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/Exceptions.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/Exceptions.kt index 4f4ca954f36..4890cd7ada3 100644 --- a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/Exceptions.kt +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/Exceptions.kt @@ -44,7 +44,7 @@ internal constructor(message: String, cause: Throwable? = null) : RuntimeExcepti is com.google.firebase.vertexai.common.InvalidAPIKeyException -> InvalidAPIKeyException(cause.message ?: "") is com.google.firebase.vertexai.common.PromptBlockedException -> - PromptBlockedException(cause.response.toPublic(), cause.cause) + PromptBlockedException(cause.response?.toPublic(), cause.cause) is com.google.firebase.vertexai.common.UnsupportedUserLocationException -> UnsupportedUserLocationException(cause.cause) is com.google.firebase.vertexai.common.InvalidStateException -> @@ -57,6 +57,8 @@ internal constructor(message: String, cause: Throwable? = null) : RuntimeExcepti ServiceDisabledException(cause.message ?: "", cause.cause) is com.google.firebase.vertexai.common.UnknownException -> UnknownException(cause.message ?: "", cause.cause) + is com.google.firebase.vertexai.common.ContentBlockedException -> + ContentBlockedException(cause.message ?: "", cause.cause) else -> UnknownException(cause.message ?: "", cause) } is TimeoutCancellationException -> @@ -87,13 +89,22 @@ internal constructor(message: String, cause: Throwable? = null) : * * @property response The full server response. */ -// TODO(rlazo): Add secondary constructor to pass through the message? public class PromptBlockedException -internal constructor(public val response: GenerateContentResponse, cause: Throwable? = null) : +internal constructor( + public val response: GenerateContentResponse?, + cause: Throwable? = null, + message: String? = null, +) : FirebaseVertexAIException( - "Prompt was blocked: ${response.promptFeedback?.blockReason?.name}", + "Prompt was blocked: ${response?.promptFeedback?.blockReason?.name?: message}", cause, - ) + ) { + internal constructor(message: String, cause: Throwable? = null) : this(null, cause, message) +} + +public class ContentBlockedException +internal constructor(message: String, cause: Throwable? = null) : + FirebaseVertexAIException(message, cause) /** * The user's location (region) is not supported by the API. diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenAspectRatio.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenAspectRatio.kt new file mode 100644 index 00000000000..e605a6e987e --- /dev/null +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenAspectRatio.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.vertexai.type + +/** Represents the aspect ratio that the generated image should conform to. */ +@PublicPreviewAPI +public class ImagenAspectRatio private constructor(internal val internalVal: String) { + public companion object { + /** A square image, useful for icons, profile pictures, etc. */ + @JvmField public val SQUARE_1x1: ImagenAspectRatio = ImagenAspectRatio("1:1") + /** A portrait image in 3:4, the aspect ratio of older TVs. */ + @JvmField public val PORTRAIT_3x4: ImagenAspectRatio = ImagenAspectRatio("3:4") + /** A landscape image in 4:3, the aspect ratio of older TVs. */ + @JvmField public val LANDSCAPE_4x3: ImagenAspectRatio = ImagenAspectRatio("4:3") + /** A portrait image in 9:16, the aspect ratio of modern monitors and phone screens. */ + @JvmField public val PORTRAIT_9x16: ImagenAspectRatio = ImagenAspectRatio("9:16") + /** A landscape image in 16:9, the aspect ratio of modern monitors and phone screens. */ + @JvmField public val LANDSCAPE_16x9: ImagenAspectRatio = ImagenAspectRatio("16:9") + } +} diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenGCSImage.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenGCSImage.kt new file mode 100644 index 00000000000..380bfa3c30b --- /dev/null +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenGCSImage.kt @@ -0,0 +1,27 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.vertexai.type + +/** + * Represents an Imagen-generated image that is contained in Google Cloud Storage. + * + * @param gcsUri Contains the `gs://` URI for the image. + * @param mimeType Contains the MIME type of the image (for example, `"image/png"`). + */ +@PublicPreviewAPI +internal class ImagenGCSImage +internal constructor(public val gcsUri: String, public val mimeType: String) {} diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenGenerationConfig.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenGenerationConfig.kt new file mode 100644 index 00000000000..bbf795f4d40 --- /dev/null +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenGenerationConfig.kt @@ -0,0 +1,99 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.vertexai.type + +/** + * Contains extra settings to configure image generation. + * + * @param negativePrompt This string contains things that should be explicitly excluded from + * generated images. + * @param numberOfImages How many images should be generated. + * @param aspectRatio The aspect ratio of the generated images. + * @param imageFormat The file format/compression of the generated images. + * @param addWatermark Adds an invisible watermark to mark the image as AI generated. + */ +@PublicPreviewAPI +public class ImagenGenerationConfig( + public val negativePrompt: String? = null, + public val numberOfImages: Int? = 1, + public val aspectRatio: ImagenAspectRatio? = null, + public val imageFormat: ImagenImageFormat? = null, + public val addWatermark: Boolean? = null, +) { + /** + * Builder for creating a [ImagenGenerationConfig]. + * + * This is mainly intended for Java interop. For Kotlin, use [imagenGenerationConfig] for a more + * idiomatic experience. + * + * @property negativePrompt See [ImagenGenerationConfig.negativePrompt]. + * @property numberOfImages See [ImagenGenerationConfig.numberOfImages]. + * @property aspectRatio See [ImagenGenerationConfig.aspectRatio]. + * @property imageFormat See [ImagenGenerationConfig.imageFormat] + * @property addWatermark See [ImagenGenerationConfig.addWatermark] + * @see [imagenGenerationConfig] + */ + public class Builder { + @JvmField public var negativePrompt: String? = null + @JvmField public var numberOfImages: Int? = 1 + @JvmField public var aspectRatio: ImagenAspectRatio? = null + @JvmField public var imageFormat: ImagenImageFormat? = null + @JvmField public var addWatermark: Boolean? = null + + /** + * Alternative casing for [ImagenGenerationConfig.Builder]: + * ``` + * val config = GenerationConfig.builder() + * ``` + */ + public fun build(): ImagenGenerationConfig = + ImagenGenerationConfig( + negativePrompt = negativePrompt, + numberOfImages = numberOfImages, + aspectRatio = aspectRatio, + imageFormat = imageFormat, + addWatermark = addWatermark, + ) + } + + public companion object { + public fun builder(): Builder = Builder() + } +} + +/** + * Helper method to construct a [ImagenGenerationConfig] in a DSL-like manner. + * + * Example Usage: + * ``` + * imagenGenerationConfig { + * negativePrompt = "People, black and white, painting" + * numberOfImages = 1 + * aspectRatio = ImagenAspecRatio.SQUARE_1x1 + * imageFormat = ImagenImageFormat.png() + * addWatermark = false + * } + * ``` + */ +@PublicPreviewAPI +public fun imagenGenerationConfig( + init: ImagenGenerationConfig.Builder.() -> Unit +): ImagenGenerationConfig { + val builder = ImagenGenerationConfig.builder() + builder.init() + return builder.build() +} diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenGenerationResponse.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenGenerationResponse.kt new file mode 100644 index 00000000000..a1a80360848 --- /dev/null +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenGenerationResponse.kt @@ -0,0 +1,59 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.vertexai.type + +import kotlinx.serialization.Serializable + +/** + * Represents a response from a call to [ImagenModel#generateImages] + * + * @param images contains the generated images + * @param filteredReason if fewer images were generated than were requested, this field will contain + * the reason they were filtered out. + */ +@PublicPreviewAPI +public class ImagenGenerationResponse +internal constructor(public val images: List, public val filteredReason: String?) { + + @Serializable + internal data class Internal(val predictions: List) { + internal fun toPublicGCS() = + ImagenGenerationResponse( + images = predictions.filter { it.mimeType != null }.map { it.toPublicGCS() }, + null, + ) + + internal fun toPublicInline() = + ImagenGenerationResponse( + images = predictions.filter { it.mimeType != null }.map { it.toPublicInline() }, + null, + ) + } + + @Serializable + internal data class ImagenImageResponse( + val bytesBase64Encoded: String? = null, + val gcsUri: String? = null, + val mimeType: String? = null, + val raiFilteredReason: String? = null, + ) { + internal fun toPublicInline() = + ImagenInlineImage(bytesBase64Encoded!!.toByteArray(), mimeType!!) + + internal fun toPublicGCS() = ImagenGCSImage(gcsUri!!, mimeType!!) + } +} diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenImageFormat.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenImageFormat.kt new file mode 100644 index 00000000000..41c85e98a7a --- /dev/null +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenImageFormat.kt @@ -0,0 +1,53 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.vertexai.type + +import kotlinx.serialization.Serializable + +/** + * Represents the format an image should be returned in. + * + * @param mimeType A string (like `"image/jpeg"`) specifying the encoding MIME type of the image. + * @param compressionQuality an int (1-100) representing the quality of the image; a lower number + * means the image is permitted to be lower quality to reduce size. This parameter is not relevant + * for every MIME type. + */ +@PublicPreviewAPI +public class ImagenImageFormat +private constructor(public val mimeType: String, public val compressionQuality: Int?) { + + internal fun toInternal() = Internal(mimeType, compressionQuality) + + @Serializable internal data class Internal(val mimeType: String, val compressionQuality: Int?) + + public companion object { + /** + * An [ImagenImageFormat] representing a JPEG image. + * + * @param compressionQuality an int (1-100) representing the quality of the image; a lower + * number means the image is permitted to be lower quality to reduce size. + */ + public fun jpeg(compressionQuality: Int? = null): ImagenImageFormat { + return ImagenImageFormat("image/jpeg", compressionQuality) + } + + /** An [ImagenImageFormat] representing a PNG image */ + public fun png(): ImagenImageFormat { + return ImagenImageFormat("image/png", null) + } + } +} diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenInlineImage.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenInlineImage.kt new file mode 100644 index 00000000000..03e93abf8e7 --- /dev/null +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenInlineImage.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.vertexai.type + +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.util.Base64 + +/** + * Represents an Imagen-generated image that is contained inline + * + * @param data Contains the raw bytes of the image + * @param mimeType Contains the MIME type of the image (for example, `"image/png"`) + */ +@PublicPreviewAPI +public class ImagenInlineImage +internal constructor(public val data: ByteArray, public val mimeType: String) { + + /** + * Returns the image as an Android OS native [Bitmap] so that it can be saved or sent to the UI. + */ + public fun asBitmap(): Bitmap { + val data = Base64.decode(data, Base64.NO_WRAP) + return BitmapFactory.decodeByteArray(data, 0, data.size) + } +} diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenPersonFilterLevel.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenPersonFilterLevel.kt new file mode 100644 index 00000000000..14031c86766 --- /dev/null +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenPersonFilterLevel.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.vertexai.type + +/** A filter used to prevent images from containing depictions of children or people. */ +@PublicPreviewAPI +public class ImagenPersonFilterLevel private constructor(internal val internalVal: String) { + public companion object { + /** No filters applied. */ + @JvmField public val ALLOW_ALL: ImagenPersonFilterLevel = ImagenPersonFilterLevel("allow_all") + /** Filters out any images containing depictions of children. */ + @JvmField + public val ALLOW_ADULT: ImagenPersonFilterLevel = ImagenPersonFilterLevel("allow_adult") + /** Filters out any images containing depictions of people. */ + @JvmField public val BLOCK_ALL: ImagenPersonFilterLevel = ImagenPersonFilterLevel("dont_allow") + } +} diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenSafetyFilterLevel.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenSafetyFilterLevel.kt new file mode 100644 index 00000000000..205538ebc0a --- /dev/null +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenSafetyFilterLevel.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.vertexai.type + +/** Used for safety filtering. */ +@PublicPreviewAPI +public class ImagenSafetyFilterLevel private constructor(internal val internalVal: String) { + public companion object { + /** Strongest filtering level, most strict blocking. */ + @JvmField + public val BLOCK_LOW_AND_ABOVE: ImagenSafetyFilterLevel = + ImagenSafetyFilterLevel("block_low_and_above") + /** Block some problematic prompts and responses. */ + @JvmField + public val BLOCK_MEDIUM_AND_ABOVE: ImagenSafetyFilterLevel = + ImagenSafetyFilterLevel("block_medium_and_above") + /** + * Reduces the number of requests blocked due to safety filters. May increase objectionable + * content generated by the Imagen model. + */ + @JvmField + public val BLOCK_ONLY_HIGH: ImagenSafetyFilterLevel = ImagenSafetyFilterLevel("block_only_high") + /** Turns off all optional safety filters. */ + @JvmField public val BLOCK_NONE: ImagenSafetyFilterLevel = ImagenSafetyFilterLevel("block_none") + } +} diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenSafetySettings.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenSafetySettings.kt new file mode 100644 index 00000000000..d5a00b557bd --- /dev/null +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/ImagenSafetySettings.kt @@ -0,0 +1,29 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.vertexai.type + +/** + * A configuration for filtering unsafe content or images containing people. + * + * @param safetyFilterLevel Used to filter unsafe content. + * @param personFilterLevel Used to filter images containing people. + */ +@PublicPreviewAPI +public class ImagenSafetySettings( + internal val safetyFilterLevel: ImagenSafetyFilterLevel, + internal val personFilterLevel: ImagenPersonFilterLevel, +) {} diff --git a/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/PublicPreviewAPI.kt b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/PublicPreviewAPI.kt new file mode 100644 index 00000000000..50f9880f3be --- /dev/null +++ b/firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/type/PublicPreviewAPI.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.firebase.vertexai.type + +@Retention(AnnotationRetention.BINARY) +@RequiresOptIn( + level = RequiresOptIn.Level.ERROR, + message = + "This API is part of an experimental public preview and may change in " + + "backwards-incompatible ways without notice.", +) +public annotation class PublicPreviewAPI() diff --git a/firebase-vertexai/src/test/java/com/google/firebase/vertexai/StreamingSnapshotTests.kt b/firebase-vertexai/src/test/java/com/google/firebase/vertexai/StreamingSnapshotTests.kt index 9a6beb057a1..ce53bcf9e33 100644 --- a/firebase-vertexai/src/test/java/com/google/firebase/vertexai/StreamingSnapshotTests.kt +++ b/firebase-vertexai/src/test/java/com/google/firebase/vertexai/StreamingSnapshotTests.kt @@ -119,7 +119,7 @@ internal class StreamingSnapshotTests { withTimeout(testTimeout) { val exception = shouldThrow { responses.collect() } - exception.response.promptFeedback?.blockReason shouldBe BlockReason.SAFETY + exception.response?.promptFeedback?.blockReason shouldBe BlockReason.SAFETY } } @@ -130,8 +130,8 @@ internal class StreamingSnapshotTests { withTimeout(testTimeout) { val exception = shouldThrow { responses.collect() } - exception.response.promptFeedback?.blockReason shouldBe BlockReason.SAFETY - exception.response.promptFeedback?.blockReasonMessage shouldBe "Reasons" + exception.response?.promptFeedback?.blockReason shouldBe BlockReason.SAFETY + exception.response?.promptFeedback?.blockReasonMessage shouldBe "Reasons" } } diff --git a/firebase-vertexai/src/test/java/com/google/firebase/vertexai/UnarySnapshotTests.kt b/firebase-vertexai/src/test/java/com/google/firebase/vertexai/UnarySnapshotTests.kt index e176fd8f7eb..1f439fea7e2 100644 --- a/firebase-vertexai/src/test/java/com/google/firebase/vertexai/UnarySnapshotTests.kt +++ b/firebase-vertexai/src/test/java/com/google/firebase/vertexai/UnarySnapshotTests.kt @@ -17,6 +17,7 @@ package com.google.firebase.vertexai import com.google.firebase.vertexai.type.BlockReason +import com.google.firebase.vertexai.type.ContentBlockedException import com.google.firebase.vertexai.type.ContentModality import com.google.firebase.vertexai.type.FinishReason import com.google.firebase.vertexai.type.FunctionCallPart @@ -25,6 +26,7 @@ import com.google.firebase.vertexai.type.HarmProbability import com.google.firebase.vertexai.type.HarmSeverity import com.google.firebase.vertexai.type.InvalidAPIKeyException import com.google.firebase.vertexai.type.PromptBlockedException +import com.google.firebase.vertexai.type.PublicPreviewAPI import com.google.firebase.vertexai.type.ResponseStoppedException import com.google.firebase.vertexai.type.SerializationException import com.google.firebase.vertexai.type.ServerException @@ -53,6 +55,7 @@ import kotlinx.serialization.json.jsonPrimitive import org.json.JSONArray import org.junit.Test +@OptIn(PublicPreviewAPI::class) internal class UnarySnapshotTests { private val testTimeout = 5.seconds @@ -125,7 +128,7 @@ internal class UnarySnapshotTests { withTimeout(testTimeout) { shouldThrow { model.generateContent("prompt") } should { - it.response.promptFeedback?.blockReason shouldBe BlockReason.UNKNOWN + it.response?.promptFeedback?.blockReason shouldBe BlockReason.UNKNOWN } } } @@ -180,7 +183,7 @@ internal class UnarySnapshotTests { withTimeout(testTimeout) { shouldThrow { model.generateContent("prompt") } should { - it.response.promptFeedback?.blockReason shouldBe BlockReason.SAFETY + it.response?.promptFeedback?.blockReason shouldBe BlockReason.SAFETY } } } @@ -191,8 +194,8 @@ internal class UnarySnapshotTests { withTimeout(testTimeout) { shouldThrow { model.generateContent("prompt") } should { - it.response.promptFeedback?.blockReason shouldBe BlockReason.SAFETY - it.response.promptFeedback?.blockReasonMessage shouldContain "Reasons" + it.response?.promptFeedback?.blockReason shouldBe BlockReason.SAFETY + it.response?.promptFeedback?.blockReasonMessage shouldContain "Reasons" } } } @@ -215,7 +218,7 @@ internal class UnarySnapshotTests { fun `user location error`() = goldenUnaryFile( "unary-failure-unsupported-user-location.json", - HttpStatusCode.PreconditionFailed + HttpStatusCode.PreconditionFailed, ) { withTimeout(testTimeout) { shouldThrow { model.generateContent("prompt") } @@ -513,4 +516,23 @@ internal class UnarySnapshotTests { goldenUnaryFile("unary-failure-model-not-found.json", HttpStatusCode.NotFound) { withTimeout(testTimeout) { shouldThrow { model.countTokens("prompt") } } } + + @Test + fun `generateImages should throw when all images filtered`() = + goldenUnaryFile("unary-failure-generate-images-all-filtered.json") { + withTimeout(testTimeout) { + shouldThrow { imagenModel.generateImages("prompt") } + } + } + + @Test + fun `generateImages should throw when prompt blocked`() = + goldenUnaryFile( + "unary-failure-generate-images-prompt-blocked.json", + HttpStatusCode.BadRequest, + ) { + withTimeout(testTimeout) { + shouldThrow { imagenModel.generateImages("prompt") } + } + } } diff --git a/firebase-vertexai/src/test/java/com/google/firebase/vertexai/common/StreamingSnapshotTests.kt b/firebase-vertexai/src/test/java/com/google/firebase/vertexai/common/StreamingSnapshotTests.kt index 2d29ad38ba7..6d14025b28c 100644 --- a/firebase-vertexai/src/test/java/com/google/firebase/vertexai/common/StreamingSnapshotTests.kt +++ b/firebase-vertexai/src/test/java/com/google/firebase/vertexai/common/StreamingSnapshotTests.kt @@ -105,7 +105,7 @@ internal class StreamingSnapshotTests { withTimeout(testTimeout) { val exception = shouldThrow { responses.collect() } - exception.response.promptFeedback?.blockReason shouldBe BlockReason.Internal.SAFETY + exception.response?.promptFeedback?.blockReason shouldBe BlockReason.Internal.SAFETY } } diff --git a/firebase-vertexai/src/test/java/com/google/firebase/vertexai/common/UnarySnapshotTests.kt b/firebase-vertexai/src/test/java/com/google/firebase/vertexai/common/UnarySnapshotTests.kt index 33ebdda5322..c316a9ece81 100644 --- a/firebase-vertexai/src/test/java/com/google/firebase/vertexai/common/UnarySnapshotTests.kt +++ b/firebase-vertexai/src/test/java/com/google/firebase/vertexai/common/UnarySnapshotTests.kt @@ -111,7 +111,7 @@ internal class UnarySnapshotTests { withTimeout(testTimeout) { shouldThrow { apiController.generateContent(textGenerateContentRequest("prompt")) - } should { it.response.promptFeedback?.blockReason shouldBe BlockReason.Internal.SAFETY } + } should { it.response?.promptFeedback?.blockReason shouldBe BlockReason.Internal.SAFETY } } } diff --git a/firebase-vertexai/src/test/java/com/google/firebase/vertexai/util/tests.kt b/firebase-vertexai/src/test/java/com/google/firebase/vertexai/util/tests.kt index 29b7923e35b..9428aea67ef 100644 --- a/firebase-vertexai/src/test/java/com/google/firebase/vertexai/util/tests.kt +++ b/firebase-vertexai/src/test/java/com/google/firebase/vertexai/util/tests.kt @@ -14,10 +14,14 @@ * limitations under the License. */ +@file:OptIn(PublicPreviewAPI::class) + package com.google.firebase.vertexai.util import com.google.firebase.vertexai.GenerativeModel +import com.google.firebase.vertexai.ImagenModel import com.google.firebase.vertexai.common.APIController +import com.google.firebase.vertexai.type.PublicPreviewAPI import com.google.firebase.vertexai.type.RequestOptions import io.kotest.matchers.collections.shouldNotBeEmpty import io.kotest.matchers.nulls.shouldNotBeNull @@ -57,7 +61,11 @@ internal suspend fun ByteChannel.send(bytes: ByteArray) { * @see commonTest * @see send */ -internal data class CommonTestScope(val channel: ByteChannel, val model: GenerativeModel) +internal data class CommonTestScope( + val channel: ByteChannel, + val model: GenerativeModel, + val imagenModel: ImagenModel, +) /** A test that runs under a [CommonTestScope]. */ internal typealias CommonTest = suspend CommonTestScope.() -> Unit @@ -104,7 +112,8 @@ internal fun commonTest( null, ) val model = GenerativeModel("cool-model-name", controller = apiController) - CommonTestScope(channel, model).block() + val imagenModel = ImagenModel("cooler-model-name", controller = apiController) + CommonTestScope(channel, model, imagenModel).block() } /**