From 5fa4d279c5ae50ad9aaec5ddb749a358463a3a26 Mon Sep 17 00:00:00 2001 From: oalam Date: Thu, 14 Oct 2021 16:21:39 +0200 Subject: [PATCH 01/16] update junit version --- .../com/hurence/historian/spark/compactor/Compactor.scala | 1 - .../com/hurence/historian/spark/compactor/CompactorTest.java | 2 +- pom.xml | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/historian-spark/src/main/scala/com/hurence/historian/spark/compactor/Compactor.scala b/historian-spark/src/main/scala/com/hurence/historian/spark/compactor/Compactor.scala index 139871e9..61c4c4cd 100644 --- a/historian-spark/src/main/scala/com/hurence/historian/spark/compactor/Compactor.scala +++ b/historian-spark/src/main/scala/com/hurence/historian/spark/compactor/Compactor.scala @@ -118,7 +118,6 @@ class Compactor(val options: CompactorConf) extends Serializable with Runnable { val measuresDS = convertChunksToMeasures(uncompactedChunks) val compactedChunksDS = convertMeasuresToChunks(measuresDS) - compactedChunksDS.show(10,false) writeCompactedChunksToSolr(compactedChunksDS) deleteOldChunks(day) diff --git a/historian-spark/src/test/java/com/hurence/historian/spark/compactor/CompactorTest.java b/historian-spark/src/test/java/com/hurence/historian/spark/compactor/CompactorTest.java index a73aa7e9..faa232c5 100644 --- a/historian-spark/src/test/java/com/hurence/historian/spark/compactor/CompactorTest.java +++ b/historian-spark/src/test/java/com/hurence/historian/spark/compactor/CompactorTest.java @@ -1147,7 +1147,7 @@ public void testCompactorHourly(SparkSession sparkSession) throws InterruptedExc SolrITHelper.COLLECTION_HISTORIAN, "", "compactor", - "yyyy-MM-dd.HH"); + "yyyy-MM-dd"); Compactor compactor = new Compactor(conf); compactor.setSolrClient(cloudClient); diff --git a/pom.xml b/pom.xml index 8bed060e..96824807 100644 --- a/pom.xml +++ b/pom.xml @@ -83,8 +83,8 @@ 2.3.2 3.6.6 - 4.13.1 - 1.2.0 + 4.13 + 1.5.2 5.5.2 3.8.1 From 7e493be41de3c389270ef82d4d5da38debb0ce91 Mon Sep 17 00:00:00 2001 From: oalam Date: Thu, 14 Oct 2021 16:37:53 +0200 Subject: [PATCH 02/16] add a new repo for org.restlet.jee --- pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pom.xml b/pom.xml index 96824807..3a05b742 100644 --- a/pom.xml +++ b/pom.xml @@ -110,6 +110,13 @@ historian-scrapper + + + spring plugins repo + https://repo.spring.io/plugins-release/ + + + From e6691dab7168c241cba5d43b758725e294cb6265 Mon Sep 17 00:00:00 2001 From: oalam Date: Thu, 14 Oct 2021 16:39:01 +0200 Subject: [PATCH 03/16] add a new repo for org.restlet.jee --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 3a05b742..1647fb33 100644 --- a/pom.xml +++ b/pom.xml @@ -112,6 +112,7 @@ + spring-plug-repo spring plugins repo https://repo.spring.io/plugins-release/ From ed922e8b9fb586010f053023fd11c72fa59bc16e Mon Sep 17 00:00:00 2001 From: oalam Date: Thu, 14 Oct 2021 16:51:06 +0200 Subject: [PATCH 04/16] add a new repo for org.restlet.jee --- integration-tests/pom.xml | 5 ++++- pom.xml | 7 ------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 6ee6984e..ee9caad1 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -54,6 +54,10 @@ commons-codec commons-codec + + org.restlet.jee + * + @@ -93,7 +97,6 @@ com.hurence.historian historian-tools - 1.3.8 compile diff --git a/pom.xml b/pom.xml index 1647fb33..4ab374a5 100644 --- a/pom.xml +++ b/pom.xml @@ -110,13 +110,6 @@ historian-scrapper - - - spring-plug-repo - spring plugins repo - https://repo.spring.io/plugins-release/ - - From f56715905f435f7a8c3df04a585b436ebbbaf651 Mon Sep 17 00:00:00 2001 From: oalam Date: Thu, 14 Oct 2021 16:53:20 +0200 Subject: [PATCH 05/16] add a new repo for org.restlet.jee --- integration-tests/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index ee9caad1..cd9bab8b 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -97,6 +97,7 @@ com.hurence.historian historian-tools + ${project.version} compile From 09df908eab32d994be94d97aa58aacdc79ea1d7a Mon Sep 17 00:00:00 2001 From: oalam Date: Thu, 14 Oct 2021 17:01:54 +0200 Subject: [PATCH 06/16] remove dep to org.restlet.jee --- historian-server/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/historian-server/pom.xml b/historian-server/pom.xml index 59377910..0ffd3df6 100644 --- a/historian-server/pom.xml +++ b/historian-server/pom.xml @@ -123,6 +123,10 @@ org.apache.hadoop * + + org.restlet.jee + * + From d047b5ddfc1c85e2084df265dd414632c5057899 Mon Sep 17 00:00:00 2001 From: oalam Date: Thu, 14 Oct 2021 17:07:44 +0200 Subject: [PATCH 07/16] remove dep to org.restlet.jee --- historian-spark/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/historian-spark/pom.xml b/historian-spark/pom.xml index 92f02341..21cf1c13 100644 --- a/historian-spark/pom.xml +++ b/historian-spark/pom.xml @@ -174,10 +174,10 @@ org.apache.hadoop * - + com.google.guava * From ac073e5534b806b8f479f7dc0911d0fe592073b3 Mon Sep 17 00:00:00 2001 From: oalam Date: Thu, 14 Oct 2021 17:12:08 +0200 Subject: [PATCH 08/16] remove dep to org.restlet.jee --- historian-spark/pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/historian-spark/pom.xml b/historian-spark/pom.xml index 21cf1c13..b5078f52 100644 --- a/historian-spark/pom.xml +++ b/historian-spark/pom.xml @@ -174,10 +174,6 @@ org.apache.hadoop * - - org.restlet.jee - * - com.google.guava * @@ -224,6 +220,10 @@ com.google.guava * + + org.restlet.jee + * + From 8e006e278707646536fc26d61c88c3a10c7f170f Mon Sep 17 00:00:00 2001 From: oalam Date: Thu, 14 Oct 2021 17:15:39 +0200 Subject: [PATCH 09/16] remove dep to org.restlet.jee --- historian-spark/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/historian-spark/pom.xml b/historian-spark/pom.xml index b5078f52..30e0d22f 100644 --- a/historian-spark/pom.xml +++ b/historian-spark/pom.xml @@ -178,6 +178,10 @@ com.google.guava * + + org.restlet.jee + * + From 41e9742d2abc60399448939c0d1e75cda1bf1cb2 Mon Sep 17 00:00:00 2001 From: oalam Date: Thu, 14 Oct 2021 17:22:34 +0200 Subject: [PATCH 10/16] remove dep to org.restlet.jee --- historian-spark/pom.xml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/historian-spark/pom.xml b/historian-spark/pom.xml index 30e0d22f..c40596f2 100644 --- a/historian-spark/pom.xml +++ b/historian-spark/pom.xml @@ -303,14 +303,10 @@ commons-cli commons-cli - - - - - - - - + + org.restlet.jee + * + com.google.guava * @@ -326,7 +322,7 @@ com.hurence.historian historian-tools - 1.3.8 + ${project.version} From 0d03b55cdd80d3644d474a5b41bfbcf1d843717e Mon Sep 17 00:00:00 2001 From: oalam Date: Mon, 18 Oct 2021 10:37:20 +0200 Subject: [PATCH 11/16] (re) add dep to org.restlet.jee --- historian-spark/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/historian-spark/pom.xml b/historian-spark/pom.xml index c40596f2..a32ac610 100644 --- a/historian-spark/pom.xml +++ b/historian-spark/pom.xml @@ -303,10 +303,6 @@ commons-cli commons-cli - - org.restlet.jee - * - com.google.guava * From 73963d98b6f32c9a17e22c5d0ef5078e9ffcc8b3 Mon Sep 17 00:00:00 2001 From: oalam Date: Mon, 18 Oct 2021 10:44:17 +0200 Subject: [PATCH 12/16] add repo maven restlet --- pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pom.xml b/pom.xml index 4ab374a5..67901149 100644 --- a/pom.xml +++ b/pom.xml @@ -111,6 +111,14 @@ + + + maven-restlet + Public online Restlet repository + https://maven.restlet.talend.com + + + From 8d85d72eb5765affa7051248f1967de67a785ee6 Mon Sep 17 00:00:00 2001 From: oalam Date: Mon, 18 Oct 2021 11:00:00 +0200 Subject: [PATCH 13/16] setup it tests --- .github/workflows/integ-tests.yml | 2 +- .github/workflows/unit-tests.yml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integ-tests.yml b/.github/workflows/integ-tests.yml index 82825ed7..904e8237 100644 --- a/.github/workflows/integ-tests.yml +++ b/.github/workflows/integ-tests.yml @@ -5,7 +5,7 @@ name: integration tests on: push: - branches: [ master, release-1.3.5 ] + branches: [ master, feature/it-tests ] pull_request: branches: [ master, release-1.3.5 ] workflow_dispatch: diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 77fc85ca..3a1670ed 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,6 +1,9 @@ name: build -on: [push] +on: + push: + branches-ignore: + - 'feature/it-test' jobs: build: From 21c343d8a0641b514d0079ae0eb83e2c23c44108 Mon Sep 17 00:00:00 2001 From: oalam Date: Mon, 18 Oct 2021 16:15:23 +0200 Subject: [PATCH 14/16] setup it tests --- .github/workflows/unit-tests.yml | 1 - historian-server/pom.xml | 3 +-- ...ntFocusSamplingReturnEnoughPointsCurrentVersionIT.java | 2 +- .../http/api/grafana/simplejson/SearchEndPointIT.java | 2 +- integration-tests/pom.xml | 5 ----- .../com/hurence/historian/solr/util/SolrITHelper.java | 2 +- .../java/com/hurence/unit5/extensions/SolrExtension.java | 8 ++++---- pom.xml | 2 +- 8 files changed, 9 insertions(+), 16 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 3a1670ed..1c16e169 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -3,7 +3,6 @@ name: build on: push: branches-ignore: - - 'feature/it-test' jobs: build: diff --git a/historian-server/pom.xml b/historian-server/pom.xml index 0ffd3df6..f2f662b0 100644 --- a/historian-server/pom.xml +++ b/historian-server/pom.xml @@ -16,7 +16,6 @@ 3.8.1 1.5.0 8.2.0 - 1.12.2 @@ -185,7 +184,7 @@ org.testcontainers testcontainers - 1.12.2 + ${test.containers} test diff --git a/historian-server/src/integration-test/java/com/hurence/webapiservice/http/api/grafana/hurence/QueryEndPointFocusSamplingReturnEnoughPointsCurrentVersionIT.java b/historian-server/src/integration-test/java/com/hurence/webapiservice/http/api/grafana/hurence/QueryEndPointFocusSamplingReturnEnoughPointsCurrentVersionIT.java index f574695b..37801065 100644 --- a/historian-server/src/integration-test/java/com/hurence/webapiservice/http/api/grafana/hurence/QueryEndPointFocusSamplingReturnEnoughPointsCurrentVersionIT.java +++ b/historian-server/src/integration-test/java/com/hurence/webapiservice/http/api/grafana/hurence/QueryEndPointFocusSamplingReturnEnoughPointsCurrentVersionIT.java @@ -24,7 +24,6 @@ import io.vertx.reactivex.ext.web.codec.BodyCodec; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; -import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -33,6 +32,7 @@ import org.slf4j.LoggerFactory; import org.testcontainers.containers.DockerComposeContainer; +import javax.validation.constraints.NotNull; import java.io.IOException; import java.util.Arrays; import java.util.List; diff --git a/historian-server/src/integration-test/java/com/hurence/webapiservice/http/api/grafana/simplejson/SearchEndPointIT.java b/historian-server/src/integration-test/java/com/hurence/webapiservice/http/api/grafana/simplejson/SearchEndPointIT.java index 5b3703fc..0e04deef 100644 --- a/historian-server/src/integration-test/java/com/hurence/webapiservice/http/api/grafana/simplejson/SearchEndPointIT.java +++ b/historian-server/src/integration-test/java/com/hurence/webapiservice/http/api/grafana/simplejson/SearchEndPointIT.java @@ -24,13 +24,13 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.common.SolrInputDocument; -import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.*; import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.DockerComposeContainer; +import javax.validation.constraints.NotNull; import java.io.IOException; import java.util.HashSet; import java.util.List; diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index cd9bab8b..2794b382 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -11,11 +11,6 @@ integration-tests jar - - 2.11 - 8.2.0 - 1.12.2 - diff --git a/integration-tests/src/main/java/com/hurence/historian/solr/util/SolrITHelper.java b/integration-tests/src/main/java/com/hurence/historian/solr/util/SolrITHelper.java index 05207cc9..53e1ab99 100644 --- a/integration-tests/src/main/java/com/hurence/historian/solr/util/SolrITHelper.java +++ b/integration-tests/src/main/java/com/hurence/historian/solr/util/SolrITHelper.java @@ -16,12 +16,12 @@ import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.params.SolrParams; -import org.jetbrains.annotations.NotNull; import org.noggit.JSONUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.DockerComposeContainer; +import javax.validation.constraints.NotNull; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; diff --git a/integration-tests/src/main/java/com/hurence/unit5/extensions/SolrExtension.java b/integration-tests/src/main/java/com/hurence/unit5/extensions/SolrExtension.java index 4ff04ca7..9b95e460 100644 --- a/integration-tests/src/main/java/com/hurence/unit5/extensions/SolrExtension.java +++ b/integration-tests/src/main/java/com/hurence/unit5/extensions/SolrExtension.java @@ -41,7 +41,7 @@ public class SolrExtension implements BeforeAllCallback, AfterAllCallback, Param public final static String SOLR1_SERVICE_NAME = "solr1_1"; public final static int SOLR_1_PORT = 8983; public final static int SOLR_2_PORT = 8983; - public final static String ZOOKEEPER_SERVICE_NAME = "zookeeper_1"; + public final static String ZOOKEEPER_SERVICE_NAME = "zoo1_1"; public final static int ZOOKEEPER_PORT = 2181; private final static String IMAGE = "solr:8"; public final static String SOLR_CONF_TEMPLATE_HISTORIAN_CURRENT = "historian-current"; @@ -77,11 +77,11 @@ public void afterAll(ExtensionContext extensionContext) throws Exception { @Override public void beforeAll(ExtensionContext extensionContext) throws Exception { this.dockerComposeContainer = new DockerComposeContainer( - new File(getClass().getResource("/shared-resources/docker-compose-test.yml").getFile()) + new File(getClass().getResource("/docker-compose-for-grafana-tests.yml").getFile()) ) - .withExposedService(ZOOKEEPER_SERVICE_NAME, ZOOKEEPER_PORT, Wait.forListeningPort()) + ;/* .withExposedService(ZOOKEEPER_SERVICE_NAME, ZOOKEEPER_PORT, Wait.forListeningPort()) .withExposedService(SOLR1_SERVICE_NAME, SOLR_1_PORT, Wait.forListeningPort()) - .waitingFor(SOLR2_SERVICE_NAME, Wait.forListeningPort()); + .waitingFor(SOLR2_SERVICE_NAME, Wait.forListeningPort());*/ logger.info("Starting docker compose"); this.dockerComposeContainer.start(); diff --git a/pom.xml b/pom.xml index 67901149..7a0164df 100644 --- a/pom.xml +++ b/pom.xml @@ -90,7 +90,7 @@ 3.8.1 1.5.0 - 1.12.2 + 1.16.0 1.18.12 1.18.12.0 From 70ba030304305c6ac993f80631065b9341a956d1 Mon Sep 17 00:00:00 2001 From: oalam Date: Mon, 18 Oct 2021 16:27:08 +0200 Subject: [PATCH 15/16] setup it tests --- .github/workflows/integ-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integ-tests.yml b/.github/workflows/integ-tests.yml index 904e8237..2d1b42dc 100644 --- a/.github/workflows/integ-tests.yml +++ b/.github/workflows/integ-tests.yml @@ -5,12 +5,12 @@ name: integration tests on: push: - branches: [ master, feature/it-tests ] + branches: [ master ] pull_request: - branches: [ master, release-1.3.5 ] + branches: [ master ] workflow_dispatch: inputs: - branches: [ master, release-1.3.5 ] + branches: [ master] jobs: From 1686d9b6f02a617ca1f79a80c273bb7c92630c4c Mon Sep 17 00:00:00 2001 From: oalam Date: Tue, 19 Oct 2021 12:11:37 +0200 Subject: [PATCH 16/16] fix chunk writer to store tags --- historian-spark/docker/README.md | 9 ----- .../sql/writer/solr/SolrChunksWriter.scala | 33 ++++++++----------- 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/historian-spark/docker/README.md b/historian-spark/docker/README.md index 349521cf..05028f3d 100644 --- a/historian-spark/docker/README.md +++ b/historian-spark/docker/README.md @@ -29,15 +29,6 @@ docker tag hurence/historian-spark:latest hurence/historian-spark:1.3.8 Deploy the image to Docker hub ------------------------------ -tag the image as latest - -verify image build : - -```shell script -docker images -docker tag latest -``` - then login and push the latest image ```shell script diff --git a/historian-spark/src/main/scala/com/hurence/historian/spark/sql/writer/solr/SolrChunksWriter.scala b/historian-spark/src/main/scala/com/hurence/historian/spark/sql/writer/solr/SolrChunksWriter.scala index 93003794..e17ae278 100644 --- a/historian-spark/src/main/scala/com/hurence/historian/spark/sql/writer/solr/SolrChunksWriter.scala +++ b/historian-spark/src/main/scala/com/hurence/historian/spark/sql/writer/solr/SolrChunksWriter.scala @@ -12,12 +12,12 @@ import com.hurence.timeseries.model.Definitions._ import scala.collection.JavaConverters._ /** - * val options.config = Map( - * "zkhost" -> options.zkHosts, - * "collection" -> options.collectionName - * ) - * - */ + * val options.config = Map( + * "zkhost" -> options.zkHosts, + * "collection" -> options.collectionName + * ) + * + */ class SolrChunksWriter extends Writer[Chunk] { @@ -32,22 +32,15 @@ class SolrChunksWriter extends Writer[Chunk] { else options.config - var someTags : Boolean = true - val tagCols : List[Column] = if (options.config.contains(TAG_NAMES)) { - options.config(TAG_NAMES).split(",").toList - .map(tag => col(FIELD_TAGS)(tag).as(tag)) - } else { - // No tags specified - someTags = false - List[Column]() - } + // build column names with tags + val mainCols = FIELDS.asScala.toList.map(name => col(name).as(getColumnFromField(name))) + val keysDF = ds.select(explode(map_keys(col(FIELD_TAGS)))).distinct() + val keys = keysDF.collect().map(f=>f.get(0)) + val tagCols = keys.map(f=> col(FIELD_TAGS).getItem(f).as(f.toString)).toList - val mainCols = FIELDS.asScala.toList - .map(name => col(name).as(getColumnFromField(name))) - - // todo manage dateFormatbucket and date interval + // write the dataset to SolR ds - .select(mainCols ::: tagCols: _*) + .select(mainCols ::: tagCols:_*) .withColumn(SOLR_COLUMN_VALUE, base64(col(SOLR_COLUMN_VALUE))) .write .format("solr")