Skip to content

Commit

Permalink
Fixes for Integration Tests
Browse files Browse the repository at this point in the history 
 - Multi modality image tests to assert with containsAny instead of contains specific values
 - Expand Mistral functionCall test's usage metrics range
  • Loading branch information
@ilayaperumalg
ilayaperumalg committed Feb 3, 2025
1 parent 4e1358a commit c5becf7
Show file tree
Hide file tree
Showing 15 changed files with 44 additions and 56 deletions.
3 changes: 2 additions & 1 deletion ...ing-ai-anthropic/src/test/java/org/springframework/ai/anthropic/AnthropicChatModelIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,8 @@ void multiModalityTest() throws IOException {
var response = this.chatModel.call(new Prompt(List.of(userMessage)));

logger.info(response.getResult().getOutput().getText());
assertThat(response.getResult().getOutput().getText()).contains("banan", "apple", "basket");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bananas", "apple", "bowl", "basket",
"fruit stand");
}

@Test
Expand Down
9 changes: 3 additions & 6 deletions ...nthropic/src/test/java/org/springframework/ai/anthropic/client/AnthropicChatClientIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,8 +298,7 @@ void multiModalityEmbeddedImage(String modelName) throws IOException {
// @formatter:on

logger.info(response);
assertThat(response).contains("bananas", "apple");
assertThat(response).containsAnyOf("bowl", "basket");
assertThat(response).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

@Disabled("Currently Anthropic API does not support external image URLs")
Expand All @@ -321,8 +320,7 @@ void multiModalityImageUrl(String modelName) throws IOException {
// @formatter:on

logger.info(response);
assertThat(response).contains("bananas", "apple");
assertThat(response).containsAnyOf("bowl", "basket");
assertThat(response).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

@Test
Expand All @@ -341,8 +339,7 @@ void streamingMultiModality() throws IOException {
String content = response.collectList().block().stream().collect(Collectors.joining());

logger.info("Response: {}", content);
assertThat(content).contains("bananas", "apple");
assertThat(content).containsAnyOf("bowl", "basket");
assertThat(content).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

record ActorsFilms(String actor, List<String> movies) {
Expand Down
5 changes: 2 additions & 3 deletions ...zure-openai/src/test/java/org/springframework/ai/azure/openai/AzureOpenAiChatModelIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,7 @@ void multiModalityImageUrl() throws IOException {
// @formatter:on

logger.info(response);
assertThat(response).contains("bananas", "apple");
assertThat(response).containsAnyOf("bowl", "basket");
assertThat(response).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

@Test
Expand All @@ -245,7 +244,7 @@ void multiModalityImageResource() {
.content();
// @formatter:on

assertThat(response).containsAnyOf("bananas", "apple", "apples");
assertThat(response).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

record ActorsFilms(String actor, List<String> movies) {
Expand Down
9 changes: 3 additions & 6 deletions ...se/src/test/java/org/springframework/ai/bedrock/converse/BedrockConverseChatClientIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -374,8 +374,7 @@ void multiModalityEmbeddedImage(String modelName) throws IOException {
// @formatter:on

logger.info(response);
assertThat(response).contains("bananas", "apple");
assertThat(response).containsAnyOf("bowl", "basket");
assertThat(response).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

@ParameterizedTest(name = "{0} : {displayName} ")
Expand All @@ -395,8 +394,7 @@ void multiModalityImageUrl(String modelName) throws IOException {
// @formatter:on

logger.info(response);
assertThat(response).contains("bananas", "apple");
assertThat(response).containsAnyOf("bowl", "basket");
assertThat(response).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

@Test
Expand All @@ -416,8 +414,7 @@ void streamingMultiModalityImageUrl() throws IOException {
String content = response.collectList().block().stream().collect(Collectors.joining());

logger.info("Response: {}", content);
assertThat(content).contains("bananas", "apple");
assertThat(content).containsAnyOf("bowl", "basket");
assertThat(content).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

record ActorsFilms(String actor, List<String> movies) {
Expand Down
3 changes: 2 additions & 1 deletion ...nverse/src/test/java/org/springframework/ai/bedrock/converse/BedrockProxyChatModelIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,8 @@ void multiModalityTest() throws IOException {
var response = this.chatModel.call(new Prompt(List.of(userMessage)));

logger.info(response.getResult().getOutput().getText());
assertThat(response.getResult().getOutput().getText()).contains("banan", "apple", "basket");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bananas", "apple", "bowl", "basket",
"fruit stand");
}

@Test
Expand Down
2 changes: 1 addition & 1 deletion ...src/test/java/org/springframework/ai/bedrock/converse/client/BedrockNovaChatClientIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ void imageMultiModalityTest() throws IOException {
.content();

logger.info(response);
assertThat(response).containsAnyOf("banan", "apple", "basket");
assertThat(response).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

@Test
Expand Down
9 changes: 4 additions & 5 deletions ...ng-ai-mistral-ai/src/test/java/org/springframework/ai/mistralai/MistralAiChatModelIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,8 +263,8 @@ void multiModalityEmbeddedImage(String modelName) {
.call(new Prompt(List.of(userMessage), ChatOptions.builder().model(modelName).build()));

logger.info(response.getResult().getOutput().getText());
assertThat(response.getResult().getOutput().getText()).contains("bananas", "apple");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bowl", "basket", "fruit stand");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bananas", "apple", "bowl", "basket",
"fruit stand");
}

@ParameterizedTest(name = "{0} : {displayName} ")
Expand Down Expand Up @@ -304,8 +304,7 @@ void streamingMultiModalityImageUrl() throws IOException {
.map(AssistantMessage::getText)
.collect(Collectors.joining());
logger.info("Response: {}", content);
assertThat(content).contains("bananas", "apple");
assertThat(content).containsAnyOf("bowl", "basket", "fruit stand");
assertThat(content).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

@Test
Expand All @@ -329,7 +328,7 @@ void streamFunctionCallUsageTest() {
logger.info("Response: {}", chatResponse);
assertThat(chatResponse.getMetadata()).isNotNull();
assertThat(chatResponse.getMetadata().getUsage()).isNotNull();
assertThat(chatResponse.getMetadata().getUsage().getTotalTokens()).isLessThan(1050).isGreaterThan(750);
assertThat(chatResponse.getMetadata().getUsage().getTotalTokens()).isLessThan(1050).isGreaterThan(650);
}

record ActorsFilmsRecord(String actor, List<String> movies) {
Expand Down
3 changes: 2 additions & 1 deletion ...ng-ai-ollama/src/test/java/org/springframework/ai/ollama/OllamaChatModelMultimodalIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ void multiModalityTest() {
var response = this.chatModel.call(new Prompt(List.of(userMessage)));

logger.info(response.getResult().getOutput().getText());
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bananas", "apple");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bananas", "apple", "bowl", "basket",
"fruit stand");
}

@SpringBootConfiguration
Expand Down
4 changes: 2 additions & 2 deletions .../spring-ai-openai/src/test/java/org/springframework/ai/openai/chat/OpenAiChatModelIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -478,8 +478,8 @@ void multiModalityImageUrl(String modelName) throws IOException {
.call(new Prompt(List.of(userMessage), OpenAiChatOptions.builder().model(modelName).build()));

logger.info(response.getResult().getOutput().getText());
assertThat(response.getResult().getOutput().getText()).contains("bananas", "apple");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bowl", "basket", "fruit stand");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bananas", "apple", "bowl", "basket",
"fruit stand");
}

@Test
Expand Down
9 changes: 3 additions & 6 deletions ...ai-openai/src/test/java/org/springframework/ai/openai/chat/client/OpenAiChatClientIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,8 +315,7 @@ void multiModalityEmbeddedImage(String modelName) throws IOException {
// @formatter:on

logger.info(response);
assertThat(response).contains("bananas", "apple");
assertThat(response).containsAnyOf("bowl", "basket");
assertThat(response).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

@ParameterizedTest(name = "{0} : {displayName} ")
Expand All @@ -336,8 +335,7 @@ void multiModalityImageUrl(String modelName) throws IOException {
// @formatter:on

logger.info(response);
assertThat(response).contains("bananas", "apple");
assertThat(response).containsAnyOf("bowl", "basket");
assertThat(response).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

@Test
Expand All @@ -359,8 +357,7 @@ void streamingMultiModalityImageUrl() throws IOException {
String content = response.collectList().block().stream().collect(Collectors.joining());

logger.info("Response: {}", content);
assertThat(content).contains("bananas", "apple");
assertThat(content).containsAnyOf("bowl", "basket");
assertThat(content).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

@Test
Expand Down
11 changes: 5 additions & 6 deletions ...nai/src/test/java/org/springframework/ai/openai/chat/proxy/GroqWithOpenAiChatModelIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,8 +308,8 @@ void multiModalityEmbeddedImage(String modelName) throws IOException {
.call(new Prompt(List.of(userMessage), OpenAiChatOptions.builder().model(modelName).build()));

logger.info(response.getResult().getOutput().getText());
assertThat(response.getResult().getOutput().getText()).contains("bananas", "apple");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bowl", "basket");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bananas", "apple", "bowl", "basket",
"fruit stand");
}

@Disabled("Groq does not support multi modality API")
Expand All @@ -327,8 +327,8 @@ void multiModalityImageUrl(String modelName) throws IOException {
.call(new Prompt(List.of(userMessage), OpenAiChatOptions.builder().model(modelName).build()));

logger.info(response.getResult().getOutput().getText());
assertThat(response.getResult().getOutput().getText()).contains("bananas", "apple");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bowl", "basket");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bananas", "apple", "bowl", "basket",
"fruit stand");
}

@Disabled("Groq does not support multi modality API")
Expand All @@ -352,8 +352,7 @@ void streamingMultiModalityImageUrl() throws IOException {
.map(AssistantMessage::getText)
.collect(Collectors.joining());
logger.info("Response: {}", content);
assertThat(content).contains("bananas", "apple");
assertThat(content).containsAnyOf("bowl", "basket");
assertThat(content).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

@ParameterizedTest(name = "{0} : {displayName} ")
Expand Down
7 changes: 3 additions & 4 deletions .../src/test/java/org/springframework/ai/openai/chat/proxy/MistralWithOpenAiChatModelIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,8 +312,8 @@ void multiModalityEmbeddedImage(String modelName) throws IOException {
.call(new Prompt(List.of(userMessage), OpenAiChatOptions.builder().model(modelName).build()));

logger.info(response.getResult().getOutput().getText());
assertThat(response.getResult().getOutput().getText()).contains("bananas", "apple");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bowl", "basket");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bananas", "apple", "bowl", "basket",
"fruit stand");
}

@Disabled("Mistral AI does not support multi modality API")
Expand Down Expand Up @@ -356,8 +356,7 @@ void streamingMultiModalityImageUrl() throws IOException {
.map(AssistantMessage::getText)
.collect(Collectors.joining());
logger.info("Response: {}", content);
assertThat(content).contains("bananas", "apple");
assertThat(content).containsAnyOf("bowl", "basket");
assertThat(content).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

@ParameterizedTest(name = "{0} : {displayName} ")
Expand Down
11 changes: 5 additions & 6 deletions ...i/src/test/java/org/springframework/ai/openai/chat/proxy/OllamaWithOpenAiChatModelIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,8 +335,8 @@ void multiModalityEmbeddedImage(String modelName) throws IOException {
.call(new Prompt(List.of(userMessage), OpenAiChatOptions.builder().model(modelName).build()));

logger.info(response.getResult().getOutput().getText());
assertThat(response.getResult().getOutput().getText()).contains("bananas", "apple");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bowl", "basket");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bananas", "apple", "bowl", "basket",
"fruit stand");
}

@Disabled("Not supported by the current Ollama API")
Expand All @@ -354,8 +354,8 @@ void multiModalityImageUrl(String modelName) throws IOException {
.call(new Prompt(List.of(userMessage), OpenAiChatOptions.builder().model(modelName).build()));

logger.info(response.getResult().getOutput().getText());
assertThat(response.getResult().getOutput().getText()).contains("bananas", "apple");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bowl", "basket");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bananas", "apple", "bowl", "basket",
"fruit stand");
}

@Disabled("Not supported by the current Ollama API")
Expand All @@ -381,8 +381,7 @@ void streamingMultiModalityImageUrl(String modelName) throws IOException {
.map(AssistantMessage::getText)
.collect(Collectors.joining());
logger.info("Response: {}", content);
assertThat(content).contains("bananas", "apple");
assertThat(content).containsAnyOf("bowl", "basket");
assertThat(content).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

@ParameterizedTest(name = "{0} : {displayName} ")
Expand Down
4 changes: 2 additions & 2 deletions ...emini/src/test/java/org/springframework/ai/vertexai/gemini/VertexAiGeminiChatModelIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,8 +254,8 @@ void multiModalityTest() throws IOException {
// List.of(new Media(MimeTypeDetector.getMimeType(imageUrl), imageUrl)));
// response = client.call(new Prompt(List.of(userMessage)));

// assertThat(response.getResult().getOutput().getContent()).contains("bananas",
// "apple", "basket");
// assertThat(response.getResult().getOutput().getContent())..containsAnyOf("bananas",
// "apple", "bowl", "basket", "fruit stand");

// https://github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/use-cases/intro_multimodal_use_cases.ipynb
}
Expand Down
11 changes: 5 additions & 6 deletions ...ring-ai-zhipuai/src/test/java/org/springframework/ai/zhipuai/chat/ZhiPuAiChatModelIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,8 +292,8 @@ void multiModalityEmbeddedImage(String modelName) throws IOException {
.call(new Prompt(List.of(userMessage), ZhiPuAiChatOptions.builder().model(modelName).build()));

logger.info(response.getResult().getOutput().getText());
assertThat(response.getResult().getOutput().getText()).contains("bananas", "apple");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bowl", "basket");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bananas", "apple", "bowl", "basket",
"fruit stand");
}

@ParameterizedTest(name = "{0} : {displayName} ")
Expand All @@ -310,8 +310,8 @@ void multiModalityImageUrl(String modelName) throws IOException {
.call(new Prompt(List.of(userMessage), ZhiPuAiChatOptions.builder().model(modelName).build()));

logger.info(response.getResult().getOutput().getText());
assertThat(response.getResult().getOutput().getText()).contains("bananas", "apple");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bowl", "basket");
assertThat(response.getResult().getOutput().getText()).containsAnyOf("bananas", "apple", "bowl", "basket",
"fruit stand");
}

@Test
Expand All @@ -334,8 +334,7 @@ void streamingMultiModalityImageUrl() throws IOException {
.map(AssistantMessage::getText)
.collect(Collectors.joining());
logger.info("Response: {}", content);
assertThat(content).contains("bananas", "apple");
assertThat(content).containsAnyOf("bowl", "basket");
assertThat(content).containsAnyOf("bananas", "apple", "bowl", "basket", "fruit stand");
}

record ActorsFilmsRecord(String actor, List<String> movies) {
Expand Down

0 comments on commit c5becf7

Please sign in to comment.