diff --git a/.github/workflows/sync-jira-versions.yml b/.github/workflows/sync-jira-versions.yml new file mode 100644 index 0000000..f31c5d4 --- /dev/null +++ b/.github/workflows/sync-jira-versions.yml @@ -0,0 +1,13 @@ +name: Add GitHub release version to Jira issues + +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + call-jira-sync: + name: Call Jira versions update + uses: reportportal/.github/.github/workflows/update-jira-versions.yaml@main + with: + jira-server: ${{ vars.JIRA_SERVER }} + secrets: inherit \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 9a1ceee..7208af9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM gradle:8.4.0-jdk21 AS build +FROM --platform=$BUILDPLATFORM gradle:8.10.0-jdk21 AS build ARG RELEASE_MODE ARG APP_VERSION WORKDIR /usr/app @@ -10,7 +10,7 @@ RUN if [ "${RELEASE_MODE}" = true ]; then \ else gradle build --exclude-task test -Dorg.gradle.project.version=${APP_VERSION}; fi # For ARM build use flag: `--platform linux/arm64` -FROM --platform=$BUILDPLATFORM amazoncorretto:21.0.1 +FROM amazoncorretto:21.0.4 LABEL version=${APP_VERSION} description="EPAM Report portal. Jobs Service" maintainer="Andrei Varabyeu , Hleb Kanonik " ARG APP_VERSION=${APP_VERSION} ENV APP_DIR=/usr/app diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 63f0b02..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1,43 +0,0 @@ -#!groovy - -node { - - load "$JENKINS_HOME/jobvars.env" - - env.JAVA_HOME = "${tool 'openjdk-11'}" - env.PATH = "${env.JAVA_HOME}/bin:${env.PATH}" - - stage('Checkout') { - checkout scm - } - stage('Assemble') { - sh "./gradlew clean assemble -P buildNumber=${env.BUILD_NUMBER}" - } - stage('Test') { - sh './gradlew test --full-stacktrace' - } - stage('Build') { - sh './gradlew build' - } - stage('Docker image') { - sh "./gradlew buildDocker" - } - stage('Push to registries') { - withEnv(["AWS_URI=${AWS_URI}", "AWS_REGION=${AWS_REGION}"]) { - sh 'docker tag reportportal-dev/service-jobs ${AWS_URI}/service-jobs:SNAPSHOT-${BUILD_NUMBER}' - def image = env.AWS_URI + '/service-jobs' + ':SNAPSHOT-' + env.BUILD_NUMBER - def url = 'https://' + env.AWS_URI - def credentials = 'ecr:' + env.AWS_REGION + ':aws_credentials' - echo image - docker.withRegistry(url, credentials) { - docker.image(image).push() - } - } - } - stage('Cleanup') { - withEnv(["AWS_URI=${AWS_URI}"]) { - sh 'docker rmi ${AWS_URI}/service-jobs:SNAPSHOT-${BUILD_NUMBER}' - sh 'docker rmi reportportal-dev/service-jobs:latest' - } - } -} diff --git a/Jenkinsfile-candidate b/Jenkinsfile-candidate deleted file mode 100644 index dae8716..0000000 --- a/Jenkinsfile-candidate +++ /dev/null @@ -1,57 +0,0 @@ -#!groovy -properties([ - parameters([ - string( - name: "VERSION", - defaultValue: "", - description: "Release candidate version tag" - ), - string( - name: "BRANCH", - defaultValue: "", - description: "Specify the GitHub branch from which the image will be built" - ) - ]) -]) - -node { - - load "$JENKINS_HOME/jobvars.env" - - env.JAVA_HOME = "${tool 'openjdk-11'}" - env.PATH = "${env.JAVA_HOME}/bin:${env.PATH}" - - stage('Checkout') { - checkout scm - } - - stage('Assemble') { - sh "./gradlew clean assemble -P buildNumber=${env.BUILD_NUMBER}" - } - - stage('Test') { - sh './gradlew test --full-stacktrace' - } - - stage('Build') { - sh './gradlew build' - } - - stage('Push to ECR') { - withEnv(["AWS_URI=${AWS_URI}", "AWS_REGION=${AWS_REGION}", "TAG=${VERSION}"]) { - def image = env.AWS_URI + '/service-jobs:' + env.TAG + '-RC-' + env.BUILD_NUMBER - def url = 'https://' + env.AWS_URI - def credentials = 'ecr:' + env.AWS_REGION + ':aws_credentials' - sh './gradlew buildDocker -P dockerTag=$AWS_URI/service-jobs:$VERSION-RC-$BUILD_NUMBER' - docker.withRegistry(url, credentials) { - docker.image(image).push() - } - } - } - - stage('Cleanup') { - withEnv(["AWS_URI=${AWS_URI}"]) { - sh 'docker rmi ${AWS_URI}/service-jobs:${VERSION}-RC-${BUILD_NUMBER}' - } - } -} diff --git a/build.gradle b/build.gradle index 0d2ccff..365b30b 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,6 @@ plugins { - id 'org.springframework.boot' version '2.7.17' - id 'io.spring.dependency-management' version '1.0.11.RELEASE' + id 'org.springframework.boot' version '2.7.18' + id 'io.spring.dependency-management' version '1.1.4' id 'java' } @@ -11,9 +11,8 @@ project.ext { ext['junit-jupiter.version'] = '5.10.0' -def scriptsUrl = 'https://raw.githubusercontent.com/reportportal/gradle-scripts/' + (releaseMode ? '5.11.0' : 'EPMRPP-85756') +def scriptsUrl = 'https://raw.githubusercontent.com/reportportal/gradle-scripts/' + (releaseMode ? '5.12.0' : 'develop') -apply from: "$scriptsUrl/build-docker.gradle" apply from: "$scriptsUrl/build-commons.gradle" apply from: "$scriptsUrl/build-info.gradle" //apply from: "$scriptsUrl/build-quality.gradle" @@ -26,7 +25,7 @@ tasks.withType(JavaCompile).configureEach { } wrapper { - gradleVersion = '8.4' + gradleVersion = '8.10' } bootJar { @@ -54,7 +53,8 @@ ext['log4j2.version'] = '2.21.1' ext['log4j-to-slf4j.version'] = '2.21.1' //https://nvd.nist.gov/vuln/detail/CVE-2022-26520 ext['postgresql.version'] = '42.6.1' -ext['snakeyaml.version'] = '1.33' +ext['snakeyaml.version'] = '2.2' +ext['spring-boot.version'] = '2.7.18' // dependencies { @@ -91,6 +91,7 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-jdbc' implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.springframework.boot:spring-boot-starter-amqp' + implementation 'org.springframework:spring-jdbc:6.1.5' implementation 'org.apache.jclouds.api:s3:2.5.0' implementation 'org.apache.jclouds.provider:aws-s3:2.5.0' implementation 'org.apache.jclouds.api:filesystem:2.5.0' diff --git a/gradle.properties b/gradle.properties index dd3706e..268b736 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=5.11.2 +version=5.12.0 description=EPAM Report portal. Service jobs dockerServerUrl=unix:///var/run/docker.sock dockerPrepareEnvironment= diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 7f93135..a4b76b9 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 3fa8f86..9355b41 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 1aa94a4..f5feea6 100755 --- a/gradlew +++ b/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -55,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -84,7 +86,8 @@ done # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} # Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) -APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum diff --git a/gradlew.bat b/gradlew.bat index 6689b85..9b42019 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -13,6 +13,8 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem @if "%DEBUG%"=="" @echo off @rem ########################################################################## @@ -43,11 +45,11 @@ set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -57,11 +59,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail diff --git a/src/main/java/com/epam/reportportal/config/JacksonConfiguration.java b/src/main/java/com/epam/reportportal/config/JacksonConfiguration.java new file mode 100644 index 0000000..8cd7255 --- /dev/null +++ b/src/main/java/com/epam/reportportal/config/JacksonConfiguration.java @@ -0,0 +1,47 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.config; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author Siarhei Hrabko + */ +@Configuration +public class JacksonConfiguration { + + /** + * @return Configured object mapper + */ + @Bean(name = "objectMapper") + public ObjectMapper objectMapper() { + ObjectMapper om = new ObjectMapper(); + om.setAnnotationIntrospector(new JacksonAnnotationIntrospector()); + om.configure(MapperFeature.DEFAULT_VIEW_INCLUSION, true); + om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + om.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + om.registerModule(new JavaTimeModule()); + return om; + } +} diff --git a/src/main/java/com/epam/reportportal/config/rabbit/BackgroundProcessingConfiguration.java b/src/main/java/com/epam/reportportal/config/rabbit/BackgroundProcessingConfiguration.java index 2e05a11..079052d 100644 --- a/src/main/java/com/epam/reportportal/config/rabbit/BackgroundProcessingConfiguration.java +++ b/src/main/java/com/epam/reportportal/config/rabbit/BackgroundProcessingConfiguration.java @@ -15,7 +15,7 @@ * @author Maksim Antonov */ @Configuration -@ConditionalOnProperty(prefix = "rp.elasticsearch", name = "host") +@ConditionalOnProperty(prefix = "rp.searchengine", name = "host") public class BackgroundProcessingConfiguration { public static final String LOG_MESSAGE_SAVING_QUEUE_NAME = "log_message_saving"; diff --git a/src/main/java/com/epam/reportportal/elastic/EmptyElasticSearchClient.java b/src/main/java/com/epam/reportportal/elastic/EmptySearchEngineClient.java similarity index 78% rename from src/main/java/com/epam/reportportal/elastic/EmptyElasticSearchClient.java rename to src/main/java/com/epam/reportportal/elastic/EmptySearchEngineClient.java index 11bc6ab..779fa38 100644 --- a/src/main/java/com/epam/reportportal/elastic/EmptyElasticSearchClient.java +++ b/src/main/java/com/epam/reportportal/elastic/EmptySearchEngineClient.java @@ -5,12 +5,12 @@ import org.springframework.stereotype.Service; /** - * Empty client to work with Elasticsearch. + * Empty client to work with Search engine. * * @author Maksim Antonov */ @Service -public class EmptyElasticSearchClient implements ElasticSearchClient { +public class EmptySearchEngineClient implements SearchEngineClient { @Override public void save(List logMessageList) { diff --git a/src/main/java/com/epam/reportportal/elastic/ElasticSearchClient.java b/src/main/java/com/epam/reportportal/elastic/SearchEngineClient.java similarity index 78% rename from src/main/java/com/epam/reportportal/elastic/ElasticSearchClient.java rename to src/main/java/com/epam/reportportal/elastic/SearchEngineClient.java index b81ed57..8efd483 100644 --- a/src/main/java/com/epam/reportportal/elastic/ElasticSearchClient.java +++ b/src/main/java/com/epam/reportportal/elastic/SearchEngineClient.java @@ -4,11 +4,11 @@ import java.util.List; /** - * Client interface to work with Elasticsearch. + * Client interface to work with Search engine. * * @author Maksim Antonov */ -public interface ElasticSearchClient { +public interface SearchEngineClient { void save(List logMessageList); diff --git a/src/main/java/com/epam/reportportal/elastic/SimpleElasticSearchClient.java b/src/main/java/com/epam/reportportal/elastic/SimpleSearchEngineClient.java similarity index 89% rename from src/main/java/com/epam/reportportal/elastic/SimpleElasticSearchClient.java rename to src/main/java/com/epam/reportportal/elastic/SimpleSearchEngineClient.java index 233f437..3eab580 100644 --- a/src/main/java/com/epam/reportportal/elastic/SimpleElasticSearchClient.java +++ b/src/main/java/com/epam/reportportal/elastic/SimpleSearchEngineClient.java @@ -19,23 +19,23 @@ import org.springframework.web.client.RestTemplate; /** - * Simple client to work with Elasticsearch. + * Simple client to work with Search engine. * * @author Maksim Antonov */ @Primary @Service -@ConditionalOnProperty(prefix = "rp.elasticsearch", name = "host") -public class SimpleElasticSearchClient implements ElasticSearchClient { +@ConditionalOnProperty(prefix = "rp.searchengine", name = "host") +public class SimpleSearchEngineClient implements SearchEngineClient { - protected final Logger LOGGER = LoggerFactory.getLogger(SimpleElasticSearchClient.class); + protected final Logger LOGGER = LoggerFactory.getLogger(SimpleSearchEngineClient.class); private final String host; private final RestTemplate restTemplate; - public SimpleElasticSearchClient(@Value("${rp.elasticsearch.host}") String host, - @Value("${rp.elasticsearch.username:}") String username, - @Value("${rp.elasticsearch.password:}") String password) { + public SimpleSearchEngineClient(@Value("${rp.searchengine.host}") String host, + @Value("${rp.searchengine.username:}") String username, + @Value("${rp.searchengine.password:}") String password) { restTemplate = new RestTemplate(); if (!username.isEmpty() && !password.isEmpty()) { diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java index afc14a5..80a0b0e 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanAttachmentJob.java @@ -17,30 +17,41 @@ @Service public class CleanAttachmentJob extends BaseCleanJob { - private static final String MOVING_QUERY = - """ - WITH moved_rows AS (DELETE FROM attachment WHERE project_id = ? AND creation_date <= ?::TIMESTAMP RETURNING *) \s - INSERT INTO attachment_deletion (id, file_id, thumbnail_id, creation_attachment_date, deletion_date)\s - SELECT id, file_id, thumbnail_id, creation_date, NOW() FROM moved_rows;"""; + private static final String MOVING_QUERY = """ + WITH moved_rows AS ( + DELETE FROM attachment\s + WHERE project_id = ?\s + AND creation_date <= ?::TIMESTAMP\s + AND launch_id NOT IN ( + SELECT id FROM launch WHERE retention_policy='IMPORTANT' + )\s + RETURNING * + ) + INSERT INTO attachment_deletion (id, file_id, thumbnail_id, creation_attachment_date, + deletion_date) + SELECT id, file_id, thumbnail_id, creation_date, NOW() FROM moved_rows;"""; public CleanAttachmentJob(JdbcTemplate jdbcTemplate) { super(jdbcTemplate); } @Override - @Scheduled(cron = "${rp.environment.variable.clean.attachment.cron}") - @SchedulerLock(name = "cleanAttachment", lockAtMostFor = "24h") - public void execute() { - moveAttachments(); - } + @Scheduled(cron = "${rp.environment.variable.clean.attachment.cron}") + @SchedulerLock(name = "cleanAttachment", lockAtMostFor = "24h") + public void execute() { + moveAttachments(); + } - void moveAttachments() { - AtomicInteger counter = new AtomicInteger(0); - getProjectsWithAttribute(KEEP_SCREENSHOTS).forEach((projectId, duration) -> { - LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration); - int movedCount = jdbcTemplate.update(MOVING_QUERY, projectId, lessThanDate); - counter.addAndGet(movedCount); - LOGGER.info("Moved {} attachments to the deletion table for project {}, lessThanDate {} ", movedCount, projectId, lessThanDate); - }); - } + void moveAttachments() { + AtomicInteger counter = new AtomicInteger(0); + getProjectsWithAttribute(KEEP_SCREENSHOTS).forEach((projectId, duration) -> { + LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration); + int movedCount = jdbcTemplate.update(MOVING_QUERY, projectId, lessThanDate); + counter.addAndGet(movedCount); + LOGGER.info( + "Moved {} attachments to the deletion table for project {}, lessThanDate {} ", movedCount, + projectId, lessThanDate + ); + }); + } } diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java index 0ea68ac..d519999 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanLaunchJob.java @@ -1,7 +1,7 @@ package com.epam.reportportal.jobs.clean; import com.epam.reportportal.analyzer.index.IndexerServiceClient; -import com.epam.reportportal.elastic.ElasticSearchClient; +import com.epam.reportportal.elastic.SearchEngineClient; import com.google.common.collect.Lists; import java.time.LocalDateTime; import java.time.ZoneOffset; @@ -27,49 +27,51 @@ public class CleanLaunchJob extends BaseCleanJob { private static final String IDS_PARAM = "ids"; private static final String PROJECT_ID_PARAM = "projectId"; private static final String START_TIME_PARAM = "startTime"; - private static final String SELECT_LAUNCH_ID_QUERY = "SELECT id FROM launch WHERE project_id = :projectId AND start_time <= :startTime::TIMESTAMP;"; - private static final String DELETE_CLUSTER_QUERY = "DELETE FROM clusters WHERE clusters.launch_id IN (:ids);"; + private static final String SELECT_LAUNCH_ID_QUERY = + "SELECT id FROM launch WHERE project_id = :projectId AND start_time <= " + + ":startTime::TIMESTAMP AND retention_policy = 'REGULAR'"; + private static final String DELETE_CLUSTER_QUERY = + "DELETE FROM clusters WHERE clusters.launch_id IN (:ids);"; private static final String DELETE_LAUNCH_QUERY = "DELETE FROM launch WHERE id IN (:ids);"; private final Integer batchSize; private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; private final CleanLogJob cleanLogJob; private final IndexerServiceClient indexerServiceClient; private final ApplicationEventPublisher eventPublisher; - private final ElasticSearchClient elasticSearchClient; + private final SearchEngineClient searchEngineClient; public CleanLaunchJob( @Value("${rp.environment.variable.elements-counter.batch-size}") Integer batchSize, - JdbcTemplate jdbcTemplate, - NamedParameterJdbcTemplate namedParameterJdbcTemplate, CleanLogJob cleanLogJob, - IndexerServiceClient indexerServiceClient, - ApplicationEventPublisher eventPublisher, ElasticSearchClient elasticSearchClient) { + JdbcTemplate jdbcTemplate, NamedParameterJdbcTemplate namedParameterJdbcTemplate, + CleanLogJob cleanLogJob, IndexerServiceClient indexerServiceClient, + ApplicationEventPublisher eventPublisher, SearchEngineClient searchEngineClient) { super(jdbcTemplate); this.batchSize = batchSize; this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; this.cleanLogJob = cleanLogJob; this.indexerServiceClient = indexerServiceClient; this.eventPublisher = eventPublisher; - this.elasticSearchClient = elasticSearchClient; + this.searchEngineClient = searchEngineClient; } @Override - @Scheduled(cron = "${rp.environment.variable.clean.launch.cron}") - @SchedulerLock(name = "cleanLaunch", lockAtMostFor = "24h") - public void execute() { - removeLaunches(); - cleanLogJob.removeLogs(); - } + @Scheduled(cron = "${rp.environment.variable.clean.launch.cron}") + @SchedulerLock(name = "cleanLaunch", lockAtMostFor = "24h") + public void execute() { + removeLaunches(); + cleanLogJob.removeLogs(); + } - private void removeLaunches() { - AtomicInteger counter = new AtomicInteger(0); - getProjectsWithAttribute(KEEP_LAUNCHES).forEach((projectId, duration) -> { - final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration); - final List launchIds = getLaunchIds(projectId, lessThanDate); - if (!launchIds.isEmpty()) { - deleteClusters(launchIds); -// final Long numberOfLaunchElements = countNumberOfLaunchElements(launchIds); - int deleted = namedParameterJdbcTemplate.update(DELETE_LAUNCH_QUERY, - Map.of(IDS_PARAM, launchIds)); + private void removeLaunches() { + AtomicInteger counter = new AtomicInteger(0); + getProjectsWithAttribute(KEEP_LAUNCHES).forEach((projectId, duration) -> { + final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration); + final List launchIds = getLaunchIds(projectId, lessThanDate); + if (!launchIds.isEmpty()) { + deleteClusters(launchIds); + // final Long numberOfLaunchElements = countNumberOfLaunchElements(launchIds); + int deleted = + namedParameterJdbcTemplate.update(DELETE_LAUNCH_QUERY, Map.of(IDS_PARAM, launchIds)); counter.addAndGet(deleted); LOGGER.info("Delete {} launches for project {}", deleted, projectId); // to avoid error message in analyzer log, doesn't find index @@ -77,27 +79,28 @@ private void removeLaunches() { indexerServiceClient.removeFromIndexLessThanLaunchDate(projectId, lessThanDate); LOGGER.info("Send message for deletion to analyzer for project {}", projectId); - deleteLogsFromElasticsearchByLaunchIdsAndProjectId(launchIds, projectId); + deleteLogsFromSearchEngineByLaunchIdsAndProjectId(launchIds, projectId); -// eventPublisher.publishEvent(new ElementsDeletedEvent(launchIds, projectId, numberOfLaunchElements)); -// LOGGER.info("Send event with elements deleted number {} for project {}", deleted, projectId); - } - } - }); - } + // eventPublisher.publishEvent(new ElementsDeletedEvent(launchIds, + // projectId, numberOfLaunchElements)); + // LOGGER.info("Send event with elements deleted number {} for + // project {}", deleted, projectId); + } + } + }); + } - private void deleteLogsFromElasticsearchByLaunchIdsAndProjectId(List launchIds, + private void deleteLogsFromSearchEngineByLaunchIdsAndProjectId(List launchIds, Long projectId) { for (Long launchId : launchIds) { - elasticSearchClient.deleteLogsByLaunchIdAndProjectId(launchId, projectId); + searchEngineClient.deleteLogsByLaunchIdAndProjectId(launchId, projectId); LOGGER.info("Delete logs from ES by launch {} and project {}", launchId, projectId); } } private List getLaunchIds(Long projectId, LocalDateTime lessThanDate) { return namedParameterJdbcTemplate.queryForList(SELECT_LAUNCH_ID_QUERY, - Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate), - Long.class + Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate), Long.class ); } @@ -111,20 +114,16 @@ private Long countNumberOfLaunchElements(List launchIds) { "SELECT item_id FROM test_item WHERE launch_id IN (:ids) UNION " + "SELECT item_id FROM test_item WHERE retry_of IS NOT NULL AND retry_of IN " + "(SELECT item_id FROM test_item WHERE launch_id IN (:ids))", - Map.of(IDS_PARAM, launchIds), - Long.class + Map.of(IDS_PARAM, launchIds), Long.class ); resultedNumber.addAndGet(itemIds.size()); - Lists.partition(itemIds, batchSize) - .forEach(batch -> resultedNumber.addAndGet( - Optional.ofNullable(namedParameterJdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM log WHERE item_id IN (:ids);", - Map.of(IDS_PARAM, batch), - Long.class - )).orElse(0L))); + Lists.partition(itemIds, batchSize).forEach(batch -> resultedNumber.addAndGet( + Optional.ofNullable(namedParameterJdbcTemplate.queryForObject( + "SELECT COUNT(*) FROM log WHERE item_id IN (:ids);", Map.of(IDS_PARAM, batch), + Long.class + )).orElse(0L))); resultedNumber.addAndGet(Optional.ofNullable(namedParameterJdbcTemplate.queryForObject( - "SELECT COUNT(*) FROM log WHERE log.launch_id IN (:ids);", - Map.of(IDS_PARAM, launchIds), + "SELECT COUNT(*) FROM log WHERE log.launch_id IN (:ids);", Map.of(IDS_PARAM, launchIds), Long.class )).orElse(0L)); return resultedNumber.longValue(); diff --git a/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java b/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java index 0446356..8d5da38 100644 --- a/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java +++ b/src/main/java/com/epam/reportportal/jobs/clean/CleanLogJob.java @@ -1,7 +1,7 @@ package com.epam.reportportal.jobs.clean; import com.epam.reportportal.analyzer.index.IndexerServiceClient; -import com.epam.reportportal.elastic.ElasticSearchClient; +import com.epam.reportportal.elastic.SearchEngineClient; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.List; @@ -23,69 +23,79 @@ public class CleanLogJob extends BaseCleanJob { private static final String PROJECT_ID_PARAM = "projectId"; private static final String START_TIME_PARAM = "startTime"; - private static final String DELETE_LOGS_QUERY = "DELETE FROM log WHERE project_id = ? AND log_time <= ?::TIMESTAMP;"; - private static final String SELECT_LAUNCH_ID_QUERY = "SELECT id FROM launch WHERE project_id = :projectId AND start_time <= :startTime::TIMESTAMP;"; + private static final String DELETE_LOGS_QUERY = """ + DELETE FROM log + WHERE log.project_id = ? AND log.log_time <= ?::TIMESTAMP + AND COALESCE(log.launch_id, + (SELECT test_item.launch_id FROM test_item WHERE test_item.item_id = log.item_id), + (SELECT test_item.launch_id FROM test_item WHERE test_item.item_id = + (SELECT ti.retry_of FROM test_item ti WHERE ti.item_id = log.item_id) + ) + ) IN (SELECT launch.id FROM launch WHERE launch.retention_policy = 'REGULAR'); + """; + private static final String SELECT_LAUNCH_ID_QUERY = + "SELECT id FROM launch WHERE project_id = :projectId AND start_time <= " + + ":startTime::TIMESTAMP;"; private final CleanAttachmentJob cleanAttachmentJob; private final IndexerServiceClient indexerServiceClient; private final ApplicationEventPublisher eventPublisher; - private final ElasticSearchClient elasticSearchClient; + private final SearchEngineClient searchEngineClient; private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; public CleanLogJob(JdbcTemplate jdbcTemplate, CleanAttachmentJob cleanAttachmentJob, IndexerServiceClient indexerServiceClient, ApplicationEventPublisher eventPublisher, - ElasticSearchClient elasticSearchClient, + SearchEngineClient searchEngineClient, NamedParameterJdbcTemplate namedParameterJdbcTemplate) { super(jdbcTemplate); this.cleanAttachmentJob = cleanAttachmentJob; this.indexerServiceClient = indexerServiceClient; this.eventPublisher = eventPublisher; - this.elasticSearchClient = elasticSearchClient; + this.searchEngineClient = searchEngineClient; this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; } @Override - @Scheduled(cron = "${rp.environment.variable.clean.log.cron}") - @SchedulerLock(name = "cleanLog", lockAtMostFor = "24h") - public void execute() { - removeLogs(); - cleanAttachmentJob.moveAttachments(); - } + @Scheduled(cron = "${rp.environment.variable.clean.log.cron}") + @SchedulerLock(name = "cleanLog", lockAtMostFor = "24h") + public void execute() { + removeLogs(); + } - void removeLogs() { - AtomicInteger counter = new AtomicInteger(0); - // TODO: Need to refactor Logs to keep real it's launchId and combine code with - // CleanLaunch to avoid duplication - getProjectsWithAttribute(KEEP_LOGS).forEach((projectId, duration) -> { - final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration); - int deleted = jdbcTemplate.update(DELETE_LOGS_QUERY, projectId, lessThanDate); - counter.addAndGet(deleted); - LOGGER.info("Delete {} logs for project {}", deleted, projectId); - // to avoid error message in analyzer log, doesn't find index - if (deleted > 0) { - indexerServiceClient.removeFromIndexLessThanLogDate(projectId, lessThanDate); - LOGGER.info("Send message for deletion to analyzer for project {}", projectId); + void removeLogs() { + AtomicInteger counter = new AtomicInteger(0); + // TODO: Need to refactor Logs to keep real it's launchId and combine code with + // CleanLaunch to avoid duplication + getProjectsWithAttribute(KEEP_LOGS).forEach((projectId, duration) -> { + final LocalDateTime lessThanDate = LocalDateTime.now(ZoneOffset.UTC).minus(duration); + int deleted = jdbcTemplate.update(DELETE_LOGS_QUERY, projectId, lessThanDate); + counter.addAndGet(deleted); + LOGGER.info("Delete {} logs for project {}", deleted, projectId); + // to avoid error message in analyzer log, doesn't find index + if (deleted > 0) { + indexerServiceClient.removeFromIndexLessThanLogDate(projectId, lessThanDate); + LOGGER.info("Send message for deletion to analyzer for project {}", projectId); final List launchIds = getLaunchIds(projectId, lessThanDate); if (!launchIds.isEmpty()) { - deleteLogsFromElasticsearchByLaunchIdsAndProjectId(launchIds, projectId); + deleteLogsFromSearchEngineByLaunchIdsAndProjectId(launchIds, projectId); } - } - }); - } + } + }); + cleanAttachmentJob.moveAttachments(); + } - private void deleteLogsFromElasticsearchByLaunchIdsAndProjectId(List launchIds, + private void deleteLogsFromSearchEngineByLaunchIdsAndProjectId(List launchIds, Long projectId) { for (Long launchId : launchIds) { - elasticSearchClient.deleteLogsByLaunchIdAndProjectId(launchId, projectId); + searchEngineClient.deleteLogsByLaunchIdAndProjectId(launchId, projectId); LOGGER.info("Delete logs from ES by launch {} and project {}", launchId, projectId); } } private List getLaunchIds(Long projectId, LocalDateTime lessThanDate) { return namedParameterJdbcTemplate.queryForList(SELECT_LAUNCH_ID_QUERY, - Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate), - Long.class + Map.of(PROJECT_ID_PARAM, projectId, START_TIME_PARAM, lessThanDate), Long.class ); } } diff --git a/src/main/java/com/epam/reportportal/jobs/processing/SaveLogMessageJob.java b/src/main/java/com/epam/reportportal/jobs/processing/SaveLogMessageJob.java index 4a93e6b..fb121fe 100644 --- a/src/main/java/com/epam/reportportal/jobs/processing/SaveLogMessageJob.java +++ b/src/main/java/com/epam/reportportal/jobs/processing/SaveLogMessageJob.java @@ -14,7 +14,7 @@ * @author Maksim Antonov */ @Service -@ConditionalOnProperty(prefix = "rp.elasticsearch", name = "host") +@ConditionalOnProperty(prefix = "rp.searchengine", name = "host") public class SaveLogMessageJob { public static final String LOG_MESSAGE_SAVING_QUEUE_NAME = "log_message_saving"; diff --git a/src/main/java/com/epam/reportportal/jobs/statistics/DefectUpdateStatisticsJob.java b/src/main/java/com/epam/reportportal/jobs/statistics/DefectUpdateStatisticsJob.java new file mode 100644 index 0000000..82e4ae8 --- /dev/null +++ b/src/main/java/com/epam/reportportal/jobs/statistics/DefectUpdateStatisticsJob.java @@ -0,0 +1,196 @@ +/* + * Copyright 2024 EPAM Systems + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.epam.reportportal.jobs.statistics; + +import static org.springframework.http.HttpMethod.POST; + +import com.epam.reportportal.jobs.BaseJob; +import java.security.SecureRandom; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; +import java.util.HashSet; +import java.util.Set; +import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; +import org.apache.commons.lang3.StringUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.client.RestTemplate; + +/** + * Sends statistics about amounts of manual analyzed items to the GA4 service. + * + * @author Maksim Antonov + */ +@Service +public class DefectUpdateStatisticsJob extends BaseJob { + + private static final String GA_URL = "https://www.google-analytics.com/mp/collect?measurement_id=%s&api_secret=%s"; + private static final String DATE_BEFORE = "date_before"; + + private static final String SELECT_INSTANCE_ID_QUERY = "SELECT value FROM server_settings WHERE key = 'server.details.instance';"; + private static final String SELECT_STATISTICS_QUERY = "SELECT * FROM analytics_data WHERE type = 'DEFECT_UPDATE_STATISTICS' AND created_at >= :date_before::TIMESTAMP;"; + private static final String DELETE_STATISTICS_QUERY = "DELETE FROM analytics_data WHERE type = 'DEFECT_UPDATE_STATISTICS';"; + + private final NamedParameterJdbcTemplate namedParameterJdbcTemplate; + + private final RestTemplate restTemplate; + + private final String mId; + private final String gaId; + + + /** + * Initializes {@link DefectUpdateStatisticsJob}. + * + * @param jdbcTemplate {@link JdbcTemplate} + */ + @Autowired + public DefectUpdateStatisticsJob(JdbcTemplate jdbcTemplate, + @Value("${rp.environment.variable.ga.mId}") String mId, + @Value("${rp.environment.variable.ga.id}") String gaId, + NamedParameterJdbcTemplate namedParameterJdbcTemplate) { + super(jdbcTemplate); + this.mId = mId; + this.gaId = gaId; + this.namedParameterJdbcTemplate = namedParameterJdbcTemplate; + this.restTemplate = new RestTemplate(); + } + + + /** + * Sends analyzed items statistics. + */ + @Override + @Scheduled(cron = "${rp.environment.variable.ga.cron}") + @SchedulerLock(name = "defectUpdateStatisticsJob", lockAtMostFor = "24h") + @Transactional + public void execute() { + LOGGER.info("Start sending items defect update statistics"); + if (StringUtils.isEmpty(mId) || StringUtils.isEmpty(gaId)) { + LOGGER.info( + "Both 'mId' and 'id' environment variables should be provided in order to run the job 'defectUpdateStatisticsJob'"); + return; + } + + var now = Instant.now(); + var dateBefore = now.minus(1, ChronoUnit.DAYS) + .atOffset(ZoneOffset.UTC) + .toLocalDateTime(); + MapSqlParameterSource queryParams = new MapSqlParameterSource(); + queryParams.addValue(DATE_BEFORE, dateBefore); + + namedParameterJdbcTemplate.query(SELECT_STATISTICS_QUERY, queryParams, rs -> { + int autoAnalyzed = 0; + int userAnalyzed = 0; + int sentToAnalyze = 0; + String version; + boolean analyzerEnabled; + Set status = new HashSet<>(); + Set autoAnalysisState = new HashSet<>(); + + do { + var metadata = new JSONObject(rs.getString("metadata")) + .getJSONObject("metadata"); + + analyzerEnabled = metadata.optBoolean("analyzerEnabled"); + if (analyzerEnabled) { + autoAnalysisState.add(metadata.getBoolean("autoAnalysisOn") ? "on" : "off"); + } + + if (metadata.optInt("userAnalyzed") > 0) { + status.add("manually"); + sentToAnalyze += metadata.optInt("userAnalyzed"); + } else { + status.add("automatically"); + sentToAnalyze += metadata.optInt("sentToAnalyze"); + } + + userAnalyzed += metadata.optInt("userAnalyzed"); + autoAnalyzed += metadata.optInt("analyzed"); + version = metadata.getString("version"); + + } while (rs.next()); + + var instanceId = jdbcTemplate.queryForObject(SELECT_INSTANCE_ID_QUERY, String.class); + var params = new JSONObject(); + params.put("category", "analyzer"); + params.put("instanceID", instanceId); + params.put("timestamp", now.toEpochMilli()); + params.put("version", version); + params.put("type", analyzerEnabled ? "is_analyzer" : "not_analyzer"); + if (analyzerEnabled) { + params.put("number", autoAnalyzed + "#" + userAnalyzed + "#" + sentToAnalyze); + params.put("auto_analysis", String.join("#", autoAnalysisState)); + params.put("status", String.join("#", status)); + } + + var event = new JSONObject(); + event.put("name", "analyze_analyzer"); + event.put("params", params); + + JSONArray events = new JSONArray(); + events.put(event); + + JSONObject requestBody = new JSONObject(); + requestBody.put("client_id", + now.toEpochMilli() + "." + new SecureRandom().nextInt(100_000, 999_999)); + requestBody.put("events", events); + + sendRequest(requestBody); + + }); + + LOGGER.info("Completed items defect update statistics job"); + + } + + private void sendRequest(JSONObject requestBody) { + try { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Sending statistics data: {}", requestBody); + } + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + HttpEntity request = new HttpEntity<>(requestBody.toString(), headers); + + String url = String.format(GA_URL, mId, gaId); + + var response = restTemplate.exchange(url, POST, request, String.class); + if (response.getStatusCodeValue() != 204) { + LOGGER.error("Failed to send statistics: {}", response); + } + } catch (Exception e) { + LOGGER.error("Failed to send statistics", e); + } finally { + jdbcTemplate.execute(DELETE_STATISTICS_QUERY); + } + } + +} diff --git a/src/main/java/com/epam/reportportal/log/LogProcessing.java b/src/main/java/com/epam/reportportal/log/LogProcessing.java index 2781596..e07b55c 100644 --- a/src/main/java/com/epam/reportportal/log/LogProcessing.java +++ b/src/main/java/com/epam/reportportal/log/LogProcessing.java @@ -1,7 +1,7 @@ package com.epam.reportportal.log; import com.epam.reportportal.calculation.BatchProcessing; -import com.epam.reportportal.elastic.ElasticSearchClient; +import com.epam.reportportal.elastic.SearchEngineClient; import java.util.List; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -15,22 +15,22 @@ * @author Maksim Antonov */ @Component -@ConditionalOnProperty(prefix = "rp.elasticsearch", name = "host") +@ConditionalOnProperty(prefix = "rp.searchengine", name = "host") public class LogProcessing extends BatchProcessing { - private final ElasticSearchClient elasticSearchClient; + private final SearchEngineClient searchEngineClient; - public LogProcessing(ElasticSearchClient elasticSearchClient, + public LogProcessing(SearchEngineClient searchEngineClient, @Value("${rp.processing.log.maxBatchSize}") int batchSize, @Value("${rp.processing.log.maxBatchTimeout}") int timeout) { super(batchSize, timeout, new DefaultManagedTaskScheduler()); - this.elasticSearchClient = elasticSearchClient; + this.searchEngineClient = searchEngineClient; } @Override protected void process(List logMessageList) { if (!CollectionUtils.isEmpty(logMessageList)) { - elasticSearchClient.save(logMessageList); + searchEngineClient.save(logMessageList); } } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 529387e..d75c71c 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -9,5 +9,6 @@ management.endpoints.web.base-path=/ management.endpoint.info.enabled=true management.info.env.enabled=true + # Health info management.endpoint.health.show-details=always diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index d60d20d..6c4d0a1 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -39,13 +39,17 @@ rp: project: ## 1 minute cron: '0 */1 * * * *' + ga: + id: ovxfTlz7QOeaZDPbroXZQA + mId: G-Z22WZS0E4E + cron: '0 0 */24 * * *' executor: pool: storage: project: core: 5 max: 10 -# elasticsearch: +# searchengine: # host: http://elasticsearch:9200 # username: # password: