From a924a74e02b2fc6d2d7991f3365ff97e95a20d6b Mon Sep 17 00:00:00 2001 From: amitai stern <123amitai@gmail.com> Date: Fri, 17 Feb 2023 21:11:30 +0200 Subject: [PATCH 1/4] Upgrading compatibility to opensearch 2.5 (#367) * Upgrading compatibility to opensearch 2.5 Signed-off-by: amitai stern <123amitai@gmail.com> * latest 2.4 version Signed-off-by: amitai stern <123amitai@gmail.com> * keep existing compatibility matrix rows, this pr adds a new row for versions of opensearch after 2.3.0 Signed-off-by: amitai stern <123amitai@gmail.com> --------- Signed-off-by: amitai stern <123amitai@gmail.com> --- .github/workflows/test-integration.yml | 2 ++ COMPATIBILITY.md | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml index 921815ecb8..0c170ff080 100644 --- a/.github/workflows/test-integration.yml +++ b/.github/workflows/test-integration.yml @@ -17,6 +17,8 @@ jobs: - { opensearch_version: 2.1.0, java: 11 } - { opensearch_version: 2.2.1, java: 11 } - { opensearch_version: 2.3.0, java: 11 } + - { opensearch_version: 2.4.1, java: 11 } + - { opensearch_version: 2.5.0, java: 11 } steps: - name: Set up JDK ${{ matrix.java }} uses: actions/setup-java@v1 diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index c0b9044ee4..78b2b1372b 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -6,10 +6,11 @@ The below matrix shows the compatibility of the [`opensearch-java-client`](https://search.maven.org/artifact/org.opensearch.client/opensearch-java) with versions of [`OpenSearch`](https://opensearch.org/downloads.html#opensearch). | Client Version | OpenSearch Version | -| --- | --- | -| 1.0.0 | 1.0.0-1.3.3 | -| 2.0.0 | 1.3.3-2.0.1 | -| 2.1.0 | 1.3.3-2.3.0 | +|----------------|--------------------| +| 1.0.0 | 1.0.0-1.3.3 | +| 2.0.0 | 1.3.3-2.0.1 | +| 2.1.0 | 1.3.3-2.3.0 | +| 2.2.0 | 1.3.3-2.5.0 | ## Upgrading From 99d0e6a8c26eb6f3af688601f00da0555f8c5064 Mon Sep 17 00:00:00 2001 From: szczepanczykd <48957041+szczepanczykd@users.noreply.github.com> Date: Tue, 28 Feb 2023 19:04:45 +0000 Subject: [PATCH 2/4] Feature/range aggregation fix 369 (#370) * test: create integration tests for date_range and range aggregation (#369) Signed-off-by: Dominik Szczepanczyk * fix: add a missing key propery to the RangeBucket (#369) Signed-off-by: Dominik Szczepanczyk * docs: add CHANGELOG entry (#369) Signed-off-by: Dominik Szczepanczyk --------- Signed-off-by: Dominik Szczepanczyk --- CHANGELOG.md | 1 + .../_types/aggregations/RangeBucket.java | 29 +++ .../restclient/AggregationRequestIT.java | 173 ++++++++++++++++++ 3 files changed, 203 insertions(+) create mode 100644 java-client/src/test/java/org/opensearch/client/opensearch/integTest/restclient/AggregationRequestIT.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 650e921604..c79e975a22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Do not double-wrap OpenSearchException on error ([#323](https://github.com/opensearch-project/opensearch-java/pull/323)) - Fix AwsSdk2TransportOptions.responseCompression ([#322](https://github.com/opensearch-project/opensearch-java/pull/322)) - Bulk UpdateOperation misses upsert options ([#353](https://github.com/opensearch-project/opensearch-java/pull/353)) +- Fix missing key property in the RangeBucket ([#370](https://github.com/opensearch-project/opensearch-java/pull/370)) ### Security diff --git a/java-client/src/main/java/org/opensearch/client/opensearch/_types/aggregations/RangeBucket.java b/java-client/src/main/java/org/opensearch/client/opensearch/_types/aggregations/RangeBucket.java index 58b5a74b76..b8dd31a18b 100644 --- a/java-client/src/main/java/org/opensearch/client/opensearch/_types/aggregations/RangeBucket.java +++ b/java-client/src/main/java/org/opensearch/client/opensearch/_types/aggregations/RangeBucket.java @@ -51,6 +51,9 @@ @JsonpDeserializable public class RangeBucket extends MultiBucketBase { + + @Nullable + private final String key; @Nullable private final Double from; @@ -68,6 +71,7 @@ public class RangeBucket extends MultiBucketBase { private RangeBucket(Builder builder) { super(builder); + this.key = builder.key; this.from = builder.from; this.to = builder.to; this.fromAsString = builder.fromAsString; @@ -79,6 +83,14 @@ public static RangeBucket of(Function> fn) { return fn.apply(new Builder()).build(); } + /** + * API name: {@code key} + */ + @Nullable + public final String key() { + return this.key; + } + /** * API name: {@code from} */ @@ -114,6 +126,11 @@ public final String toAsString() { protected void serializeInternal(JsonGenerator generator, JsonpMapper mapper) { super.serializeInternal(generator, mapper); + if (this.key != null) { + generator.writeKey("key"); + generator.write(this.key); + + } if (this.from != null) { generator.writeKey("from"); generator.write(this.from); @@ -144,6 +161,9 @@ protected void serializeInternal(JsonGenerator generator, JsonpMapper mapper) { */ public static class Builder extends MultiBucketBase.AbstractBuilder implements ObjectBuilder { + + @Nullable + private String key; @Nullable private Double from; @@ -156,6 +176,14 @@ public static class Builder extends MultiBucketBase.AbstractBuilder imp @Nullable private String toAsString; + /** + * API name: {@code key} + */ + public final Builder key(String value) { + this.key = value; + return this; + } + /** * API name: {@code from} */ @@ -216,6 +244,7 @@ public RangeBucket build() { protected static void setupRangeBucketDeserializer(ObjectDeserializer op) { setupMultiBucketBaseDeserializer(op); + op.add(Builder::key, JsonpDeserializer.stringDeserializer(), "key"); op.add(Builder::from, JsonpDeserializer.doubleDeserializer(), "from"); op.add(Builder::to, JsonpDeserializer.doubleDeserializer(), "to"); op.add(Builder::fromAsString, JsonpDeserializer.stringDeserializer(), "from_as_string"); diff --git a/java-client/src/test/java/org/opensearch/client/opensearch/integTest/restclient/AggregationRequestIT.java b/java-client/src/test/java/org/opensearch/client/opensearch/integTest/restclient/AggregationRequestIT.java new file mode 100644 index 0000000000..151c32867d --- /dev/null +++ b/java-client/src/test/java/org/opensearch/client/opensearch/integTest/restclient/AggregationRequestIT.java @@ -0,0 +1,173 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.client.opensearch.integTest.restclient; + +import com.fasterxml.jackson.annotation.JsonFormat; +import org.apache.hc.core5.http.HttpHost; +import org.junit.Test; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch._types.Refresh; +import org.opensearch.client.opensearch._types.aggregations.Aggregation; +import org.opensearch.client.opensearch._types.aggregations.AggregationRange; +import org.opensearch.client.opensearch._types.aggregations.DateRangeAggregation; +import org.opensearch.client.opensearch._types.aggregations.DateRangeExpression; +import org.opensearch.client.opensearch._types.aggregations.FieldDateMath; +import org.opensearch.client.opensearch._types.aggregations.RangeAggregation; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.integTest.AbstractRequestIT; +import org.opensearch.client.transport.OpenSearchTransport; +import org.opensearch.client.transport.rest_client.RestClientTransport; +import org.opensearch.common.settings.Settings; + +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.Date; +import java.util.List; + +public class AggregationRequestIT extends AbstractRequestIT { + + @Override + public OpenSearchTransport buildTransport(Settings settings, HttpHost[] hosts) throws IOException { + return new RestClientTransport(buildClient(settings, hosts), new JacksonJsonpMapper()); + } + + @Test + public void testValueRangeAggregation() throws Exception { + var index = "test-value-range-aggregation"; + createDateRangeDocuments(index); + var searchResponse = sendAggregateRequest(index, "cost_ranges", getCostValueRangeAggregation()); + var costRangesAggregations = searchResponse.aggregations().get("cost_ranges"); + var buckets = costRangesAggregations._get() + ._toAggregate() + .range() + .buckets() + .array(); + + assertEquals(3, buckets.size()); + assertEquals(2, buckets.get(0).docCount()); + assertEquals(2, buckets.get(1).docCount()); + assertEquals(2, buckets.get(2).docCount()); + } + + @Test + public void testDateRangeAggregation() throws Exception { + var index = "test-date-range-aggregation"; + createDateRangeDocuments(index); + var searchResponse = sendAggregateRequest(index, "expiry_ranges", getExpiryDateRangeAggregation()); + var expiryRangesAggregations = searchResponse.aggregations().get("expiry_ranges"); + var buckets = expiryRangesAggregations._get() + ._toAggregate() + .dateRange() + .buckets() + .array(); + + assertEquals(3, buckets.size()); + assertEquals(2, buckets.get(0).docCount()); + assertEquals(2, buckets.get(1).docCount()); + assertEquals(2, buckets.get(2).docCount()); + } + + private Aggregation getExpiryDateRangeAggregation() { + DateRangeAggregation expiryDateRangeAggregation = new DateRangeAggregation.Builder() + .field("expDate") + .ranges(getDateAggregationRanges()) + .build(); + return new Aggregation.Builder().dateRange(expiryDateRangeAggregation).build(); + } + + private Aggregation getCostValueRangeAggregation() { + RangeAggregation costValueRangeAggregation = new RangeAggregation.Builder() + .field("cost") + .ranges(getValueAggregationRanges()) + .build(); + return new Aggregation.Builder().range(costValueRangeAggregation).build(); + } + + private SearchResponse sendAggregateRequest(String index, String key, Aggregation value) throws IOException { + return javaClient().search( + request -> request.index(index) + .size(0) + .aggregations(key, value), + Void.class); + } + + private List getDateAggregationRanges() { + return List.of( + new DateRangeExpression.Builder() + .from(builder -> builder.value((double) getDatePlusDays(1).getTime())) + .to(FieldDateMath.of(builder -> builder.value((double) getDatePlusDays(3).getTime() - 1000))) + .key("from-1-to-2-days") + .build(), + new DateRangeExpression.Builder() + .from(builder -> builder.value((double) getDatePlusDays(3).getTime())) + .to(FieldDateMath.of(builder -> builder.value((double) getDatePlusDays(5).getTime() - 1000))) + .key("from-3-to-4-days") + .build(), + new DateRangeExpression.Builder() + .from(builder -> builder.value((double) getDatePlusDays(5).getTime())) + .to(FieldDateMath.of(builder -> builder.value((double) getDatePlusDays(7).getTime() - 1000))) + .key("from-5-to-6-days") + .build() + ); + } + + private List getValueAggregationRanges() { + return List.of( + new AggregationRange.Builder().to("10").build(), + new AggregationRange.Builder().from("10").to("30").build(), + new AggregationRange.Builder().from("30").build() + ); + } + + private void createDateRangeDocuments(String index) throws IOException { + javaClient().create(_1 -> _1.index(index).id("1").document(createProduct("egg", 2, 1)).refresh(Refresh.True)); + javaClient().create(_1 -> _1.index(index).id("2").document(createProduct("meat", 15, 2)).refresh(Refresh.True)); + javaClient().create(_1 -> _1.index(index).id("3").document(createProduct("ham", 30, 3)).refresh(Refresh.True)); + javaClient().create(_1 -> _1.index(index).id("4").document(createProduct("cheese", 25, 4)).refresh(Refresh.True)); + javaClient().create(_1 -> _1.index(index).id("5").document(createProduct("pasta", 8, 5)).refresh(Refresh.True)); + javaClient().create(_1 -> _1.index(index).id("6").document(createProduct("oil", 50, 6)).refresh(Refresh.True)); + } + + private ProductDetails createProduct(String name, int cost, int plusDays) { + return new ProductDetails(name, cost, getDatePlusDays(plusDays)); + } + + private Date getDatePlusDays(int plusDays) { + return java.sql.Date.from(LocalDateTime.of(2023, 2, 20, 0, 0, 0).plusDays(plusDays).toInstant(ZoneOffset.UTC)); + } + + public static class ProductDetails { + private String name; + private int cost; + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSS") + private Date expDate; + + public ProductDetails() { + } + + public ProductDetails(String name, int cost, Date expDate) { + this.name = name; + this.cost = cost; + this.expDate = expDate; + } + + public String getName() { + return name; + } + + public int getCost() { + return cost; + } + + public Date getExpDate() { + return expDate; + } + } +} From 701ba47cf005b3a4b70c08d64b13b948c329ba11 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Tue, 28 Feb 2023 11:05:24 -0800 Subject: [PATCH 3/4] Fixing integration tests (#375) * Fixing integration tests Signed-off-by: Vacha Shah * Updating the test with a non-exact check Signed-off-by: Vacha Shah --------- Signed-off-by: Vacha Shah --- CHANGELOG.md | 1 + .../opensearch/integTest/AbstractClusterClientIT.java | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c79e975a22..e6beb30845 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Prevent SPI calls at runtime ([#293](https://github.com/opensearch-project/opensearch-java/pull/293)) - Add support for OpenSearch Serverless ([#339](https://github.com/opensearch-project/opensearch-java/pull/339)) - Fix issue where completion suggestions were failing, due to being parsed as term suggestions ([#107](https://github.com/opensearch-project/opensearch-java/issues/107)) +- Fix integration tests ([#375](https://github.com/opensearch-project/opensearch-java/pull/375)) ### Deprecated diff --git a/java-client/src/test/java/org/opensearch/client/opensearch/integTest/AbstractClusterClientIT.java b/java-client/src/test/java/org/opensearch/client/opensearch/integTest/AbstractClusterClientIT.java index f709de64ee..e695f7cf87 100644 --- a/java-client/src/test/java/org/opensearch/client/opensearch/integTest/AbstractClusterClientIT.java +++ b/java-client/src/test/java/org/opensearch/client/opensearch/integTest/AbstractClusterClientIT.java @@ -33,6 +33,7 @@ import static java.util.Collections.emptyMap; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertTrue; public abstract class AbstractClusterClientIT extends OpenSearchJavaClientTestCase { public void testClusterPutSettings() throws IOException { @@ -95,11 +96,7 @@ public void testClusterUpdateSettingNonExistent() throws IOException { } catch (OpenSearchException e) { assertNotNull(e); assertEquals(e.response().status(), 400); - assertEquals( - e.getMessage(), - "Request failed: [illegal_argument_exception] " + - "transient setting [no_idea_what_you_are_talking_about], not recognized" - ); + assertTrue(e.getMessage().contains("transient setting [no_idea_what_you_are_talking_about], not recognized")); } } From 75c5e595f01014942482464d00a84311ea28f962 Mon Sep 17 00:00:00 2001 From: Vacha Shah Date: Tue, 28 Feb 2023 11:07:40 -0800 Subject: [PATCH 4/4] Adding bulk request example in user guide (#373) Signed-off-by: Vacha Shah --- USER_GUIDE.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 6cd1b1deda..4ab999df6d 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -10,6 +10,7 @@ - [Index data](#index-data) - [Search for the documents](#search-for-the-documents) - [Search documents using a match query](#search-documents-using-a-match-query) + - [Bulk requests](#bulk-requests) - [Aggregations](#aggregations) - [Delete the document](#delete-the-document) - [Delete the index](#delete-the-index) @@ -142,6 +143,30 @@ for (int i = 0; i < searchResponse.hits().hits().size(); i++) { } ``` +## Bulk requests + +```java +ArrayList ops = new ArrayList<>(); +SimplePojo doc1 = new SimplePojo("Document 1", "The text of document 1"); +ops.add(new BulkOperation.Builder().index( + IndexOperation.of(io -> io.index(TEST_INDEX).id("id1").document(doc1)) +).build()); +SimplePojo doc2 = new SimplePojo("Document 2", "The text of document 2"); +ops.add(new BulkOperation.Builder().index( + IndexOperation.of(io -> io.index(TEST_INDEX).id("id2").document(doc2)) +).build()); +SimplePojo doc3 = getLongDoc("Long Document 3", 100000); +ops.add(new BulkOperation.Builder().index( + IndexOperation.of(io -> io.index(TEST_INDEX).id("id3").document(doc3)) +).build()); + +BulkRequest.Builder bulkReq = new BulkRequest.Builder() + .index(index) + .operations(ops) + .refresh(Refresh.WaitFor); +BulkResponse bulkResponse = client.bulk(bulkReq.build()); +``` + ## Aggregations ```java