From 6f82ee8a7dcd2663bb9828de676f45ea2514c2ea Mon Sep 17 00:00:00 2001 From: Robin Mattis Date: Mon, 19 Sep 2022 17:19:17 +0200 Subject: [PATCH 01/52] Add S3/MinIO support for application settings --- docs/application-settings.md | 44 +++ pom.xml | 11 + .../settings/S3ApplicationSettings.java | 205 ++++++++++ .../service/S3PeriodicRefreshService.java | 192 ++++++++++ .../spring/config/SettingsConfiguration.java | 121 +++++- .../settings/S3ApplicationSettingsTest.java | 350 ++++++++++++++++++ .../service/S3PeriodicRefreshServiceTest.java | 208 +++++++++++ 7 files changed, 1128 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/prebid/server/settings/S3ApplicationSettings.java create mode 100644 src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java create mode 100644 src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java create mode 100644 src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java diff --git a/docs/application-settings.md b/docs/application-settings.md index 885c836a257..f174f2eb065 100644 --- a/docs/application-settings.md +++ b/docs/application-settings.md @@ -208,6 +208,50 @@ Here's an example YAML file containing account-specific settings: default-coop-sync: true ``` +## Setting Account Configuration in S3 + +This is identical to the account configuration in a file system, with the main difference that your file system is +[AWS S3](https://aws.amazon.com/de/s3/) or any S3 compatible storage, such as [MinIO](https://min.io/). + + +The general idea is that you'll place all the account-specific settings in a separate YAML file and point to that file. + +```yaml +settings: + s3: + accessKeyId: + secretAccessKey: + endpoint: # http://s3.storage.com + bucket: # prebid-application-settings + accounts-dir: accounts + stored-imps-dir: stored-impressions + stored-requests-dir: stored-requests + stored-responses-dir: stored-responses + + # recommended to configure an in memory cache, but this is optional + in-memory-cache: + # example settings, tailor to your needs + cache-size: 100000 + ttl-seconds: 1200 # 20 minutes + # recommended to configure + s3-update: + refresh-rate: 900000 # Refresh every 15 minutes + timeout: 5000 +``` + +### File format + +We recommend using the `json` format for your account configuration. A minimal configuration may look like this. + +```json +{ + "id" : "979c7116-1f5a-43d4-9a87-5da3ccc4f52c", + "status" : "active" +} +``` + +This pairs nicely if you have a default configuration defined in your prebid server config under `settings.default-account-config`. + ## Setting Account Configuration in the Database In database approach account properties are stored in database table(s). diff --git a/pom.xml b/pom.xml index fd67628d5f9..5875197727b 100644 --- a/pom.xml +++ b/pom.xml @@ -107,6 +107,13 @@ pom import + + software.amazon.awssdk + bom + 2.17.274 + pom + import + @@ -275,6 +282,10 @@ postgresql ${postgresql.version} + + software.amazon.awssdk + s3 + com.github.ben-manes.caffeine caffeine diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java new file mode 100644 index 00000000000..985dfa25b2b --- /dev/null +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -0,0 +1,205 @@ +package org.prebid.server.settings; + +import io.vertx.core.CompositeFuture; +import io.vertx.core.Future; +import io.vertx.core.Vertx; +import org.prebid.server.auction.model.Tuple2; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.execution.Timeout; +import org.prebid.server.json.DecodeException; +import org.prebid.server.json.JacksonMapper; +import org.prebid.server.settings.model.Account; +import org.prebid.server.settings.model.StoredDataResult; +import org.prebid.server.settings.model.StoredResponseDataResult; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Implementation of {@link ApplicationSettings}. + *

+ * Reads an application settings from JSON file in an s3 bucket, stores and serves them in and from the memory. + *

+ * Immediately loads stored request data from local files. These are stored in memory for low-latency reads. + * This expects each file in the directory to be named "{config_id}.json". + */ +public class S3ApplicationSettings implements ApplicationSettings { + + private static final String JSON_SUFFIX = ".json"; + + final S3AsyncClient asyncClient; + final String bucket; + final String accountsDirectory; + final String storedImpressionsDirectory; + final String storedRequestsDirectory; + final String storedResponsesDirectory; + final JacksonMapper jacksonMapper; + final Vertx vertx; + + public S3ApplicationSettings( + S3AsyncClient asyncClient, + String bucket, + String accountsDirectory, + String storedImpressionsDirectory, + String storedRequestsDirectory, + String storedResponsesDirectory, + JacksonMapper jacksonMapper, + Vertx vertx) { + this.asyncClient = asyncClient; + this.bucket = bucket; + this.accountsDirectory = accountsDirectory; + this.storedImpressionsDirectory = storedImpressionsDirectory; + this.storedRequestsDirectory = storedRequestsDirectory; + this.storedResponsesDirectory = storedResponsesDirectory; + this.jacksonMapper = jacksonMapper; + this.vertx = vertx; + } + + @Override + public Future getAccountById(String accountId, Timeout timeout) { + return downloadFile(accountsDirectory + "/" + accountId + JSON_SUFFIX) + .map(fileContentOpt -> + fileContentOpt.map(fileContent -> jacksonMapper.decodeValue(fileContent, Account.class))) + .compose(accountOpt -> { + if (accountOpt.isPresent()) { + return Future.succeededFuture(accountOpt.get()); + } else { + return Future + .failedFuture(new PreBidException("Account with id %s not found".formatted(accountId))); + } + }) + .recover(ex -> { + if (ex instanceof DecodeException) { + return Future + .failedFuture( + new PreBidException( + "Invalid json for account with id %s".formatted(accountId))); + } + return Future + .failedFuture(new PreBidException("Account with id %s not found".formatted(accountId))); + }); + } + + @Override + public Future getStoredData( + String accountId, + Set requestIds, + Set impIds, + Timeout timeout) { + + return getFileContents(storedRequestsDirectory, requestIds).compose(storedIdToRequest -> + getFileContents(storedImpressionsDirectory, impIds) + .map(storedIdToImp -> { + final List missingStoredRequestIds = + getMissingStoredDataIds(storedIdToRequest).stream() + .map("No stored request found for id: %s"::formatted).toList(); + final List missingStoredImpressionIds = + getMissingStoredDataIds(storedIdToImp).stream() + .map("No stored impression found for id: %s"::formatted).toList(); + + return StoredDataResult.of( + filterOptionalFileContent(storedIdToRequest), + filterOptionalFileContent(storedIdToImp), + Stream.concat( + missingStoredImpressionIds.stream(), + missingStoredRequestIds.stream()).toList()); + } + )); + } + + private Map filterOptionalFileContent(Map> fileContents) { + return fileContents + .entrySet() + .stream() + .filter(e -> e.getValue().isPresent()) + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().get())); + } + + private List getMissingStoredDataIds(Map> fileContents) { + return fileContents.entrySet().stream().filter(e -> e.getValue().isEmpty()).map(Map.Entry::getKey).toList(); + } + + @Override + public Future getAmpStoredData( + String accountId, + Set requestIds, + Set impIds, + Timeout timeout) { + return getStoredData(accountId, requestIds, Collections.emptySet(), timeout); + } + + @Override + public Future getVideoStoredData( + String accountId, + Set requestIds, + Set impIds, + Timeout timeout) { + return getStoredData(accountId, requestIds, impIds, timeout); + } + + @Override + public Future getStoredResponses(Set responseIds, Timeout timeout) { + return getFileContents(storedResponsesDirectory, responseIds).map(storedIdToResponse -> { + final List missingStoredResponseIds = + getMissingStoredDataIds(storedIdToResponse).stream() + .map("No stored response found for id: %s"::formatted).toList(); + + return StoredResponseDataResult.of( + filterOptionalFileContent(storedIdToResponse), + missingStoredResponseIds + ); + }); + } + + @Override + public Future> getCategories(String primaryAdServer, String publisher, Timeout timeout) { + return Future.succeededFuture(Collections.emptyMap()); + } + + private Future>> getFileContents(String directory, Set ids) { + final List>>> futureListContents = ids.stream() + .map(impressionId -> + downloadFile(directory + withInitialSlash(impressionId) + JSON_SUFFIX) + .map(fileContent -> Tuple2.of(impressionId, fileContent))) + .collect(Collectors.toCollection(ArrayList::new)); + + final Future>>> composedFutures = CompositeFuture + .all(new ArrayList<>(futureListContents)) + .map(CompositeFuture::list); + + return composedFutures.map(one -> one.stream().collect(Collectors.toMap(Tuple2::getLeft, Tuple2::getRight))); + } + + /** + * When the impression id is the ad unit path it may already start with a slash and there's no need to add + * another one. + * + * @param impressionId from the bid request + * @return impression id with only a single slash at the beginning + */ + private String withInitialSlash(String impressionId) { + if (impressionId.startsWith("/")) { + return impressionId; + } + return "/" + impressionId; + } + + private Future> downloadFile(String key) { + final GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(key).build(); + + return Future.fromCompletionStage( + asyncClient.getObject(request, AsyncResponseTransformer.toBytes()), + vertx.getOrCreateContext()) + .map(test -> Optional.of(test.asUtf8String())).recover(ex -> Future.succeededFuture(Optional.empty())); + } + +} diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java new file mode 100644 index 00000000000..81f1146d81c --- /dev/null +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -0,0 +1,192 @@ +package org.prebid.server.settings.service; + +import io.vertx.core.CompositeFuture; +import io.vertx.core.Future; +import io.vertx.core.Vertx; +import io.vertx.core.logging.Logger; +import io.vertx.core.logging.LoggerFactory; +import org.prebid.server.auction.model.Tuple2; +import org.prebid.server.metric.MetricName; +import org.prebid.server.metric.Metrics; +import org.prebid.server.settings.CacheNotificationListener; +import org.prebid.server.settings.model.StoredDataResult; +import org.prebid.server.vertx.Initializable; +import software.amazon.awssdk.core.BytesWrapper; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.S3Object; + +import java.time.Clock; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + *

+ * Service that periodically calls s3 for stored request updates. + * If refreshRate is negative, then the data will never be refreshed. + *

+ * Fetches all files from the specified folders/prefixes in s3 and downloads all files. + */ +public class S3PeriodicRefreshService implements Initializable { + + private static final String JSON_SUFFIX = ".json"; + + private static final Logger logger = LoggerFactory.getLogger(S3PeriodicRefreshService.class); + + private final S3AsyncClient asyncClient; + private final String bucket; + private final String storedImpressionsDirectory; + private final String storedRequestsDirectory; + private final long refreshPeriod; + private final long timeout; + private final MetricName cacheType; + private final CacheNotificationListener cacheNotificationListener; + private final Vertx vertx; + private final Metrics metrics; + private final Clock clock; + private StoredDataResult lastResult; + + public S3PeriodicRefreshService(S3AsyncClient asyncClient, + String bucket, + String storedRequestsDirectory, + String storedImpressionsDirectory, + long refreshPeriod, + long timeout, + MetricName cacheType, + CacheNotificationListener cacheNotificationListener, + Vertx vertx, + Metrics metrics, + Clock clock) { + + this.asyncClient = asyncClient; + this.bucket = bucket; + this.storedRequestsDirectory = storedRequestsDirectory; + this.storedImpressionsDirectory = storedImpressionsDirectory; + this.refreshPeriod = refreshPeriod; + this.timeout = timeout; + this.cacheType = Objects.requireNonNull(cacheType); + this.cacheNotificationListener = Objects.requireNonNull(cacheNotificationListener); + this.vertx = Objects.requireNonNull(vertx); + this.metrics = Objects.requireNonNull(metrics); + this.clock = Objects.requireNonNull(clock); + } + + private static List getInvalidatedKeys(Map newMap, Map oldMap) { + return oldMap.keySet().stream().filter(s -> !newMap.containsKey(s)).toList(); + } + + @Override + public void initialize() { + getAll(); + if (refreshPeriod > 0) { + vertx.setPeriodic(refreshPeriod, aLong -> refresh()); + } + } + + private void getAll() { + final long startTime = clock.millis(); + + getFileContentsForDirectory(storedRequestsDirectory) + .compose(storedIdToRequest -> getFileContentsForDirectory(storedImpressionsDirectory) + .map(storedIdToImp -> + StoredDataResult.of(storedIdToRequest, storedIdToImp, Collections.emptyList()))) + .map(storedDataResult -> handleResult(storedDataResult, startTime, MetricName.initialize)) + .recover(exception -> handleFailure(exception, startTime, MetricName.initialize)); + } + + private void refresh() { + final long startTime = clock.millis(); + + getFileContentsForDirectory(storedRequestsDirectory) + .compose(storedIdToRequest -> getFileContentsForDirectory(storedImpressionsDirectory) + .map(storedIdToImp -> + StoredDataResult.of(storedIdToRequest, storedIdToImp, Collections.emptyList()))) + .map(storedDataResult -> handleResult(invalidate(storedDataResult), startTime, MetricName.update)) + .recover(exception -> handleFailure(exception, startTime, MetricName.update)); + } + + private Void handleResult(StoredDataResult storedDataResult, + long startTime, + MetricName refreshType) { + + lastResult = storedDataResult; + + cacheNotificationListener.save(storedDataResult.getStoredIdToRequest(), storedDataResult.getStoredIdToImp()); + + metrics.updateSettingsCacheRefreshTime(cacheType, refreshType, clock.millis() - startTime); + + return null; + } + + private Future handleFailure(Throwable exception, long startTime, MetricName refreshType) { + logger.warn("Error occurred while request to s3 refresh service", exception); + + metrics.updateSettingsCacheRefreshTime(cacheType, refreshType, clock.millis() - startTime); + metrics.updateSettingsCacheRefreshErrorMetric(cacheType, refreshType); + + return Future.failedFuture(exception); + } + + private StoredDataResult invalidate(StoredDataResult storedDataResult) { + final List invalidatedRequests = getInvalidatedKeys( + storedDataResult.getStoredIdToRequest(), + lastResult != null ? lastResult.getStoredIdToRequest() : Collections.emptyMap()); + final List invalidatedImps = getInvalidatedKeys( + storedDataResult.getStoredIdToImp(), + lastResult != null ? lastResult.getStoredIdToImp() : Collections.emptyMap()); + + if (!invalidatedRequests.isEmpty() || !invalidatedImps.isEmpty()) { + cacheNotificationListener.invalidate(invalidatedRequests, invalidatedImps); + } + + return storedDataResult; + } + + private Future> listFiles(String prefix) { + final ListObjectsRequest listObjectsRequest = + ListObjectsRequest.builder().bucket(bucket).prefix(prefix).build(); + + return Future.fromCompletionStage(asyncClient.listObjects(listObjectsRequest)) + .map(response -> response.contents().stream().map(S3Object::key).toList()); + } + + private Future> getFileContentsForDirectory(String directory) { + return listFiles(directory) + .compose(files -> + getFileContents(new HashSet<>(files)) + .map(map -> map.entrySet().stream().collect( + Collectors.toMap( + e -> stripFileName(directory, e.getKey()), + Map.Entry::getValue)))); + } + + private String stripFileName(String directory, String name) { + return name.replace(directory + "/", "").replace(JSON_SUFFIX, ""); + } + + private Future> getFileContents(Set fileNames) { + final List>> futureListContents = fileNames.stream() + .map(fileName -> downloadFile(fileName).map(fileContent -> Tuple2.of(fileName, fileContent))) + .collect(Collectors.toCollection(ArrayList::new)); + + final Future>> composedFutures = + CompositeFuture.all(new ArrayList<>(futureListContents)).map(CompositeFuture::list); + + return composedFutures.map(one -> one.stream().collect(Collectors.toMap(Tuple2::getLeft, Tuple2::getRight))); + } + + private Future downloadFile(String key) { + final GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(key).build(); + + return Future.fromCompletionStage(asyncClient.getObject(request, AsyncResponseTransformer.toBytes())) + .map(BytesWrapper::asUtf8String); + } +} diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java index 37b079d9ef4..9b410fa308f 100644 --- a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java @@ -19,10 +19,12 @@ import org.prebid.server.settings.FileApplicationSettings; import org.prebid.server.settings.HttpApplicationSettings; import org.prebid.server.settings.JdbcApplicationSettings; +import org.prebid.server.settings.S3ApplicationSettings; import org.prebid.server.settings.SettingsCache; import org.prebid.server.settings.service.HttpPeriodicRefreshService; import org.prebid.server.settings.service.JdbcPeriodicRefreshService; -import org.prebid.server.spring.config.database.DatabaseConfiguration; +import org.prebid.server.settings.service.S3PeriodicRefreshService; +import org.prebid.server.spring.config.database.DatabaseConfiguration import org.prebid.server.vertx.http.HttpClient; import org.prebid.server.vertx.jdbc.JdbcClient; import org.springframework.beans.factory.annotation.Autowired; @@ -35,9 +37,16 @@ import org.springframework.context.annotation.Configuration; import org.springframework.stereotype.Component; import org.springframework.validation.annotation.Validated; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; import javax.validation.constraints.Min; +import javax.validation.constraints.NotBlank; import javax.validation.constraints.NotNull; +import java.net.URI; +import java.net.URISyntaxException; import java.time.Clock; import java.util.List; import java.util.Objects; @@ -213,6 +222,110 @@ public JdbcPeriodicRefreshService ampJdbcPeriodicRefreshService( } } + @Configuration + @ConditionalOnProperty(prefix = "settings.s3", name = {"accounts-dir", "stored-imps-dir", "stored-requests-dir"}) + static class S3SettingsConfiguration { + + @Component + @ConfigurationProperties(prefix = "settings.s3") + @ConditionalOnProperty(prefix = "settings.s3", name = {"accessKeyId", "secretAccessKey"}) + @Validated + @Data + @NoArgsConstructor + private static class S3ConfigurationProperties { + @NotBlank + private String accessKeyId; + @NotBlank + private String secretAccessKey; + @NotBlank + private String endpoint; + @NotBlank + private String bucket; + @NotBlank + private String accountsDir; + @NotBlank + private String storedImpsDir; + @NotBlank + private String storedRequestsDir; + @NotBlank + private String storedResponsesDir; + } + + @Bean + S3AsyncClient s3AsyncClient(S3ConfigurationProperties s3ConfigurationProperties) throws URISyntaxException { + final AwsBasicCredentials credentials = AwsBasicCredentials.create( + s3ConfigurationProperties.getAccessKeyId(), + s3ConfigurationProperties.getSecretAccessKey()); + + return S3AsyncClient + .builder() + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .endpointOverride(new URI(s3ConfigurationProperties.getEndpoint())) + .region(Region.EU_CENTRAL_1) + .build(); + } + + @Bean + S3ApplicationSettings s3ApplicationSettings( + JacksonMapper mapper, + S3ConfigurationProperties s3ConfigurationProperties, + S3AsyncClient s3AsyncClient, + Vertx vertx) { + + return new S3ApplicationSettings( + s3AsyncClient, + s3ConfigurationProperties.getBucket(), + s3ConfigurationProperties.getAccountsDir(), + s3ConfigurationProperties.getStoredImpsDir(), + s3ConfigurationProperties.getStoredRequestsDir(), + s3ConfigurationProperties.getStoredResponsesDir(), + mapper, + vertx); + } + } + + @Configuration + @ConditionalOnProperty(prefix = "settings.in-memory-cache.s3-update", + name = {"refresh-rate", "timeout"}) + static class S3PeriodicRefreshServiceConfiguration { + + @Value("${settings.in-memory-cache.s3-update.refresh-rate}") + long refreshPeriod; + + @Value("${settings.in-memory-cache.s3-update.timeout}") + long timeout; + + @Autowired + Vertx vertx; + + @Autowired + HttpClient httpClient; + @Autowired + Metrics metrics; + @Autowired + Clock clock; + + @Bean + public S3PeriodicRefreshService s3PeriodicRefreshService( + S3AsyncClient s3AsyncClient, + S3SettingsConfiguration.S3ConfigurationProperties s3ConfigurationProperties, + SettingsCache settingsCache, + JacksonMapper mapper) { + return new S3PeriodicRefreshService( + s3AsyncClient, + s3ConfigurationProperties.getBucket(), + s3ConfigurationProperties.getStoredRequestsDir(), + s3ConfigurationProperties.getStoredImpsDir(), + refreshPeriod, + timeout, + MetricName.stored_request, + settingsCache, + vertx, + metrics, + clock); + } + } + /** * This configuration defines a collection of application settings fetchers and its ordering. */ @@ -223,10 +336,12 @@ static class CompositeSettingsConfiguration { CompositeApplicationSettings compositeApplicationSettings( @Autowired(required = false) FileApplicationSettings fileApplicationSettings, @Autowired(required = false) JdbcApplicationSettings jdbcApplicationSettings, - @Autowired(required = false) HttpApplicationSettings httpApplicationSettings) { + @Autowired(required = false) HttpApplicationSettings httpApplicationSettings, + @Autowired(required = false) S3ApplicationSettings s3ApplicationSettings) { final List applicationSettingsList = - Stream.of(fileApplicationSettings, + Stream.of(s3ApplicationSettings, + fileApplicationSettings, jdbcApplicationSettings, httpApplicationSettings) .filter(Objects::nonNull) diff --git a/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java b/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java new file mode 100644 index 00000000000..dec091bdb9a --- /dev/null +++ b/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java @@ -0,0 +1,350 @@ +package org.prebid.server.settings; + +import com.fasterxml.jackson.core.JsonProcessingException; +import io.vertx.core.Future; +import io.vertx.core.Vertx; +import io.vertx.ext.unit.Async; +import io.vertx.ext.unit.TestContext; +import io.vertx.ext.unit.junit.VertxUnitRunner; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.prebid.server.VertxTest; +import org.prebid.server.exception.PreBidException; +import org.prebid.server.execution.Timeout; +import org.prebid.server.execution.TimeoutFactory; +import org.prebid.server.settings.model.Account; +import org.prebid.server.settings.model.AccountAuctionConfig; +import org.prebid.server.settings.model.AccountPrivacyConfig; +import org.prebid.server.settings.model.StoredDataResult; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneId; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +import static java.util.Collections.singletonList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.verify; + +@RunWith(VertxUnitRunner.class) +public class S3ApplicationSettingsTest extends VertxTest { + + private static final String BUCKET = "bucket"; + private static final String ACCOUNTS_DIR = "accounts"; + private static final String STORED_IMPS_DIR = "stored-imps"; + private static final String STORED_REQUESTS_DIR = "stored-requests"; + private static final String STORED_RESPONSES_DIR = "stored-responses"; + @Rule + public final MockitoRule mockitoRule = MockitoJUnit.rule(); + private Timeout timeout; + + @Mock + private S3AsyncClient s3AsyncClient; + private Vertx vertx; + + private S3ApplicationSettings s3ApplicationSettings; + + @Before + public void setUp() { + vertx = Vertx.vertx(); + s3ApplicationSettings = new S3ApplicationSettings(s3AsyncClient, BUCKET, ACCOUNTS_DIR, + STORED_IMPS_DIR, STORED_REQUESTS_DIR, STORED_RESPONSES_DIR, jacksonMapper, vertx); + + final Clock clock = Clock.fixed(Instant.now(), ZoneId.systemDefault()); + final TimeoutFactory timeoutFactory = new TimeoutFactory(clock); + timeout = timeoutFactory.create(500L); + } + + @After + public void tearDown(TestContext context) { + vertx.close(context.asyncAssertSuccess()); + } + + @Test + public void getAccountByIdShouldReturnFetchedAccount(TestContext context) throws JsonProcessingException { + // given + final Account account = Account.builder() + .id("someId") + .auction(AccountAuctionConfig.builder() + .priceGranularity("testPriceGranularity") + .build()) + .privacy(AccountPrivacyConfig.of(null, null, null, null)) + .build(); + + given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) + .willReturn(CompletableFuture.completedFuture( + ResponseBytes.fromByteArray( + GetObjectResponse.builder().build(), + mapper.writeValueAsString(account).getBytes()))); + + // when + final Future future = s3ApplicationSettings.getAccountById("someId", timeout); + + // then + final Async async = context.async(); + + future.onComplete(context.asyncAssertSuccess(returnedAccount -> { + assertThat(returnedAccount.getId()).isEqualTo("someId"); + assertThat(returnedAccount.getAuction().getPriceGranularity()).isEqualTo("testPriceGranularity"); + + verify(s3AsyncClient).getObject( + eq(GetObjectRequest.builder().bucket(BUCKET).key(ACCOUNTS_DIR + "/someId.json").build()), + any(AsyncResponseTransformer.class)); + async.complete(); + })); + } + + @Test + public void getAccountByIdNoSuchKey(TestContext context) { + // given + given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) + .willReturn(CompletableFuture.failedFuture( + NoSuchKeyException.create( + "The specified key does not exist.", + new IllegalStateException("")))); + + // when + final Future future = s3ApplicationSettings.getAccountById("notFoundId", timeout); + + // then + final Async async = context.async(); + + future.onComplete(context.asyncAssertFailure(cause -> { + assertThat(cause) + .isInstanceOf(PreBidException.class) + .hasMessage("Account with id notFoundId not found"); + + async.complete(); + })); + } + + @Test + public void getAccountByIdInvalidJson(TestContext context) { + // given + given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) + .willReturn(CompletableFuture.completedFuture( + ResponseBytes.fromByteArray( + GetObjectResponse.builder().build(), + "invalidJson".getBytes()))); + + // when + final Future future = s3ApplicationSettings.getAccountById("invalidJsonId", timeout); + + // then + final Async async = context.async(); + + future.onComplete(context.asyncAssertFailure(cause -> { + assertThat(cause) + .isInstanceOf(PreBidException.class) + .hasMessage("Invalid json for account with id invalidJsonId"); + async.complete(); + })); + } + + @Test + public void getStoredDataShouldReturnFetchedStoredRequest(TestContext context) { + // given + given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) + .willReturn(CompletableFuture.completedFuture( + ResponseBytes.fromByteArray( + GetObjectResponse.builder().build(), + "req1Result".getBytes()))); + + // when + final Future future = s3ApplicationSettings + .getStoredData("someId", Set.of("req1"), Collections.emptySet(), timeout); + + // then + final Async async = context.async(); + + future.onComplete(context.asyncAssertSuccess(account -> { + assertThat(account.getStoredIdToRequest().size()).isEqualTo(1); + assertThat(account.getStoredIdToImp().size()).isEqualTo(0); + assertThat(account.getStoredIdToRequest()).isEqualTo(Map.of("req1", "req1Result")); + assertThat(account.getErrors()).isEmpty(); + + verify(s3AsyncClient).getObject( + eq(GetObjectRequest.builder().bucket(BUCKET).key(STORED_REQUESTS_DIR + "/req1.json").build()), + any(AsyncResponseTransformer.class)); + + async.complete(); + })); + } + + @Test + public void getStoredDataShouldReturnFetchedStoredImpression(TestContext context) { + // given + given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) + .willReturn(CompletableFuture.completedFuture( + ResponseBytes.fromByteArray( + GetObjectResponse.builder().build(), + "imp1Result".getBytes()))); + + // when + final Future future = s3ApplicationSettings + .getStoredData("someId", Collections.emptySet(), Set.of("imp1"), timeout); + + // then + final Async async = context.async(); + + future.onComplete(context.asyncAssertSuccess(account -> { + assertThat(account.getStoredIdToRequest().size()).isEqualTo(0); + assertThat(account.getStoredIdToImp().size()).isEqualTo(1); + assertThat(account.getStoredIdToImp()).isEqualTo(Map.of("imp1", "imp1Result")); + assertThat(account.getErrors()).isEmpty(); + + verify(s3AsyncClient).getObject( + eq(GetObjectRequest.builder().bucket(BUCKET).key(STORED_IMPS_DIR + "/imp1.json").build()), + any(AsyncResponseTransformer.class)); + + async.complete(); + })); + } + + @Test + public void getStoredDataShouldReturnFetchedStoredImpressionWithAdUnitPathStoredId(TestContext context) { + // given + given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) + .willReturn(CompletableFuture.completedFuture( + ResponseBytes.fromByteArray( + GetObjectResponse.builder().build(), + "imp1Result".getBytes()))); + + // when + final Future future = s3ApplicationSettings + .getStoredData("/123/root/position-1", Collections.emptySet(), Set.of("imp1"), timeout); + + // then + final Async async = context.async(); + + future.onComplete(context.asyncAssertSuccess(account -> { + assertThat(account.getStoredIdToRequest().size()).isEqualTo(0); + assertThat(account.getStoredIdToImp().size()).isEqualTo(1); + assertThat(account.getStoredIdToImp()).isEqualTo(Map.of("imp1", "imp1Result")); + assertThat(account.getErrors()).isEmpty(); + + verify(s3AsyncClient).getObject( + eq(GetObjectRequest.builder().bucket(BUCKET).key(STORED_IMPS_DIR + "/imp1.json").build()), + any(AsyncResponseTransformer.class)); + + async.complete(); + })); + } + + @Test + public void getStoredDataShouldReturnFetchedStoredImpressionAndStoredRequest(TestContext context) { + // given + given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) + .willReturn( + CompletableFuture.completedFuture( + ResponseBytes.fromByteArray( + GetObjectResponse.builder().build(), + "req1Result".getBytes())), + CompletableFuture.completedFuture( + ResponseBytes.fromByteArray( + GetObjectResponse.builder().build(), + "imp1Result".getBytes()))); + + // when + final Future future = s3ApplicationSettings + .getStoredData("someId", Set.of("req1"), Set.of("imp1"), timeout); + + // then + final Async async = context.async(); + + future.onComplete(context.asyncAssertSuccess(account -> { + assertThat(account.getStoredIdToRequest().size()).isEqualTo(1); + assertThat(account.getStoredIdToRequest()).isEqualTo(Map.of("req1", "req1Result")); + assertThat(account.getStoredIdToImp().size()).isEqualTo(1); + assertThat(account.getStoredIdToImp()).isEqualTo(Map.of("imp1", "imp1Result")); + assertThat(account.getErrors()).isEmpty(); + + verify(s3AsyncClient).getObject( + eq(GetObjectRequest.builder().bucket(BUCKET).key(STORED_IMPS_DIR + "/imp1.json").build()), + any(AsyncResponseTransformer.class)); + verify(s3AsyncClient).getObject( + eq(GetObjectRequest.builder().bucket(BUCKET).key(STORED_REQUESTS_DIR + "/req1.json").build()), + any(AsyncResponseTransformer.class)); + + async.complete(); + })); + } + + @Test + public void getStoredDataReturnsErrorsForNotFoundRequests(TestContext context) { + // given + given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) + .willReturn(CompletableFuture.failedFuture( + NoSuchKeyException.create( + "The specified key does not exist.", + new IllegalStateException("")))); + + // when + final Future future = s3ApplicationSettings + .getStoredData("someId", Set.of("req1"), Collections.emptySet(), timeout); + + // then + final Async async = context.async(); + + future.onComplete(context.asyncAssertSuccess(account -> { + assertThat(account.getStoredIdToImp()).isEmpty(); + assertThat(account.getStoredIdToRequest()).isEmpty(); + assertThat(account.getErrors().size()).isEqualTo(1); + assertThat(account.getErrors()) + .isNotNull() + .hasSize(1) + .isEqualTo(singletonList("No stored request found for id: req1")); + + async.complete(); + })); + } + + @Test + public void getStoredDataReturnsErrorsForNotFoundImpressions(TestContext context) { + // given + given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) + .willReturn( + CompletableFuture.failedFuture( + NoSuchKeyException.create( + "The specified key does not exist.", + new IllegalStateException("")))); + + // when + final Future future = s3ApplicationSettings + .getStoredData("someId", Collections.emptySet(), Set.of("imp1"), timeout); + + // then + final Async async = context.async(); + + future.onComplete(context.asyncAssertSuccess(account -> { + assertThat(account.getStoredIdToImp()).isEmpty(); + assertThat(account.getStoredIdToRequest()).isEmpty(); + assertThat(account.getErrors().size()).isEqualTo(1); + assertThat(account.getErrors()) + .isNotNull() + .hasSize(1) + .isEqualTo(singletonList("No stored impression found for id: imp1")); + + async.complete(); + })); + } + +} diff --git a/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java new file mode 100644 index 00000000000..2a1d6e1b248 --- /dev/null +++ b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java @@ -0,0 +1,208 @@ +package org.prebid.server.settings.service; + +import io.vertx.core.Handler; +import io.vertx.core.Vertx; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.mockito.stubbing.Answer; +import org.prebid.server.VertxTest; +import org.prebid.server.metric.MetricName; +import org.prebid.server.metric.Metrics; +import org.prebid.server.settings.CacheNotificationListener; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsResponse; +import software.amazon.awssdk.services.s3.model.S3Object; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class S3PeriodicRefreshServiceTest extends VertxTest { + + private static final String BUCKET = "bucket"; + private static final String STORED_REQ_DIR = "stored-req"; + private static final String STORED_IMP_DIR = "stored-imp"; + + @Rule + public final MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + private CacheNotificationListener cacheNotificationListener; + @Mock + private Vertx vertx; + @Mock + private S3AsyncClient s3AsyncClient; + private final Clock clock = Clock.fixed(Instant.now(), ZoneId.systemDefault()); + @Mock + private Metrics metrics; + + private final Map expectedRequests = singletonMap("id1", "value1"); + private final Map expectedImps = singletonMap("id2", "value2"); + + @Before + public void setUp() { + given(s3AsyncClient.listObjects(any(ListObjectsRequest.class))) + .willReturn(listObjectResponse(STORED_REQ_DIR + "/id1.json"), + listObjectResponse(STORED_IMP_DIR + "/id2.json")); + + given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) + .willReturn(CompletableFuture.completedFuture( + ResponseBytes.fromByteArray( + GetObjectResponse.builder().build(), + "value1".getBytes())), + CompletableFuture.completedFuture( + ResponseBytes.fromByteArray( + GetObjectResponse.builder().build(), + "value2".getBytes()))); + } + + @Test + public void shouldCallSaveWithExpectedParameters() { + // when + createAndInitService(1000); + + // then + verify(cacheNotificationListener).save(expectedRequests, expectedImps); + } + + @Test + public void shouldCallInvalidateAndSaveWithExpectedParameters() { + // given + given(vertx.setPeriodic(anyLong(), any())) + .willAnswer(withSelfAndPassObjectToHandler(1L)); + given(s3AsyncClient.listObjects(any(ListObjectsRequest.class))) + .willReturn(listObjectResponse(STORED_REQ_DIR + "/id1.json"), + listObjectResponse(STORED_IMP_DIR + "/id2.json"), + listObjectResponse(), + listObjectResponse(STORED_IMP_DIR + "/id2.json")); + given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) + .willReturn(getObjectResponse("value1"), + getObjectResponse("value2"), + getObjectResponse("changed_value")); + + // when + createAndInitService(1000); + + // then + verify(cacheNotificationListener).save(expectedRequests, expectedImps); + verify(cacheNotificationListener).invalidate(singletonList("id1"), emptyList()); + verify(cacheNotificationListener).save(emptyMap(), singletonMap("id2", "changed_value")); + } + + @Test + public void initializeShouldMakeOneInitialRequestAndTwoScheduledRequestsWithParam() { + // given + given(vertx.setPeriodic(anyLong(), any())) + .willAnswer(withSelfAndPassObjectToHandler(1L, 2L)); + + // when + createAndInitService(1000); + + // then + verify(s3AsyncClient, times(6)).listObjects(any(ListObjectsRequest.class)); + verify(s3AsyncClient, times(6)).getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class)); + } + + @Test + public void initializeShouldMakeOnlyOneInitialRequestIfRefreshPeriodIsNegative() { + // when + createAndInitService(-1); + + // then + verify(vertx, never()).setPeriodic(anyLong(), any()); + verify(s3AsyncClient, times(2)).listObjects(any(ListObjectsRequest.class)); + } + + @Test + public void shouldUpdateTimerMetric() { + // when + createAndInitService(1000); + + // then + verify(metrics).updateSettingsCacheRefreshTime( + eq(MetricName.stored_request), eq(MetricName.initialize), anyLong()); + } + + @Test + public void shouldUpdateTimerAndErrorMetric() { + // given + given(s3AsyncClient.listObjects(any(ListObjectsRequest.class))) + .willReturn(CompletableFuture.failedFuture(new IllegalStateException("Failed"))); + + // when + createAndInitService(1000); + + // then + verify(metrics).updateSettingsCacheRefreshTime( + eq(MetricName.stored_request), eq(MetricName.initialize), anyLong()); + verify(metrics).updateSettingsCacheRefreshErrorMetric( + eq(MetricName.stored_request), eq(MetricName.initialize)); + } + + private CompletableFuture listObjectResponse(String... keys) { + return CompletableFuture.completedFuture( + ListObjectsResponse + .builder() + .contents(Arrays.stream(keys).map(key -> S3Object.builder().key(key).build()).toList()) + .build()); + } + + private CompletableFuture> getObjectResponse(String value) { + return CompletableFuture.completedFuture( + ResponseBytes.fromByteArray( + GetObjectResponse.builder().build(), + value.getBytes())); + } + + private void createAndInitService(long refreshPeriod) { + final S3PeriodicRefreshService s3PeriodicRefreshService = new S3PeriodicRefreshService( + s3AsyncClient, + BUCKET, + STORED_REQ_DIR, + STORED_IMP_DIR, + refreshPeriod, + 2000, + MetricName.stored_request, + cacheNotificationListener, + vertx, + metrics, + clock); + s3PeriodicRefreshService.initialize(); + } + + @SuppressWarnings("unchecked") + private static Answer withSelfAndPassObjectToHandler(T... objects) { + return inv -> { + // invoking handler right away passing mock to it + for (T obj : objects) { + ((Handler) inv.getArgument(1)).handle(obj); + } + return 0L; + }; + } + +} From b433a07b86dd4a84444c48197f76f19ab6f72cd3 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Fri, 27 Oct 2023 20:59:12 +0200 Subject: [PATCH 02/52] Fix checkstyle warning --- .../org/prebid/server/spring/config/SettingsConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java index 9b410fa308f..7f87ae798b5 100644 --- a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java @@ -24,7 +24,7 @@ import org.prebid.server.settings.service.HttpPeriodicRefreshService; import org.prebid.server.settings.service.JdbcPeriodicRefreshService; import org.prebid.server.settings.service.S3PeriodicRefreshService; -import org.prebid.server.spring.config.database.DatabaseConfiguration +import org.prebid.server.spring.config.database.DatabaseConfiguration; import org.prebid.server.vertx.http.HttpClient; import org.prebid.server.vertx.jdbc.JdbcClient; import org.springframework.beans.factory.annotation.Autowired; From 017c5439954687b4147c6b3edddfd563d76f4c6c Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 21 Dec 2023 08:49:47 +0100 Subject: [PATCH 03/52] Use property for version in pom.xml --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5875197727b..b38f7e260f3 100644 --- a/pom.xml +++ b/pom.xml @@ -60,6 +60,7 @@ 3.21.7 3.17.0 1.0.7 + 2.17.274 4.13.2 @@ -110,7 +111,7 @@ software.amazon.awssdk bom - 2.17.274 + ${aws.awssdk.version} pom import From b93a2817ecfe6a5b2a1f4709eb117f31009af9a0 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 21 Dec 2023 08:50:36 +0100 Subject: [PATCH 04/52] Specify version for awssdk s3 --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index b38f7e260f3..388c2f75141 100644 --- a/pom.xml +++ b/pom.xml @@ -286,6 +286,7 @@ software.amazon.awssdk s3 + ${aws.awssdk.version} com.github.ben-manes.caffeine From c163511aed4b7ed2ea644e112d0600618f030166 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 21 Dec 2023 08:52:43 +0100 Subject: [PATCH 05/52] Adding requrireNonNull checks in S3ApplicationSettings --- .../server/settings/S3ApplicationSettings.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index 985dfa25b2b..00f13ee6253 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -19,6 +19,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -54,14 +55,15 @@ public S3ApplicationSettings( String storedResponsesDirectory, JacksonMapper jacksonMapper, Vertx vertx) { - this.asyncClient = asyncClient; - this.bucket = bucket; - this.accountsDirectory = accountsDirectory; - this.storedImpressionsDirectory = storedImpressionsDirectory; - this.storedRequestsDirectory = storedRequestsDirectory; - this.storedResponsesDirectory = storedResponsesDirectory; - this.jacksonMapper = jacksonMapper; - this.vertx = vertx; + + this.asyncClient = Objects.requireNonNull(asyncClient); + this.bucket = Objects.requireNonNull(bucket); + this.accountsDirectory = Objects.requireNonNull(accountsDirectory); + this.storedImpressionsDirectory = Objects.requireNonNull(storedImpressionsDirectory); + this.storedRequestsDirectory = Objects.requireNonNull(storedRequestsDirectory); + this.storedResponsesDirectory = Objects.requireNonNull(storedResponsesDirectory); + this.jacksonMapper = Objects.requireNonNull(jacksonMapper); + this.vertx = Objects.requireNonNull(vertx); } @Override From a17952c585963e8cdfa3bc582954e2ebc263795e Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 21 Dec 2023 09:04:23 +0100 Subject: [PATCH 06/52] Recfactor lambda into method for less nesting --- .../settings/S3ApplicationSettings.java | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index 00f13ee6253..f1556cac85c 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -100,22 +100,23 @@ public Future getStoredData( return getFileContents(storedRequestsDirectory, requestIds).compose(storedIdToRequest -> getFileContents(storedImpressionsDirectory, impIds) - .map(storedIdToImp -> { - final List missingStoredRequestIds = - getMissingStoredDataIds(storedIdToRequest).stream() - .map("No stored request found for id: %s"::formatted).toList(); - final List missingStoredImpressionIds = - getMissingStoredDataIds(storedIdToImp).stream() - .map("No stored impression found for id: %s"::formatted).toList(); - - return StoredDataResult.of( - filterOptionalFileContent(storedIdToRequest), - filterOptionalFileContent(storedIdToImp), - Stream.concat( - missingStoredImpressionIds.stream(), - missingStoredRequestIds.stream()).toList()); - } - )); + .map(storedIdToImp -> buildStoredDataResult(storedIdToRequest, storedIdToImp))); + } + + private StoredDataResult buildStoredDataResult(Map> storedIdToRequest, Map> storedIdToImp) { + final List missingStoredRequestIds = + getMissingStoredDataIds(storedIdToRequest).stream() + .map("No stored request found for id: %s"::formatted).toList(); + final List missingStoredImpressionIds = + getMissingStoredDataIds(storedIdToImp).stream() + .map("No stored impression found for id: %s"::formatted).toList(); + + return StoredDataResult.of( + filterOptionalFileContent(storedIdToRequest), + filterOptionalFileContent(storedIdToImp), + Stream.concat( + missingStoredImpressionIds.stream(), + missingStoredRequestIds.stream()).toList()); } private Map filterOptionalFileContent(Map> fileContents) { From d1c677058d13443856dd04c47de639d1700a761e Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 21 Dec 2023 14:17:50 +0100 Subject: [PATCH 07/52] Remove Option as map value --- .../settings/S3ApplicationSettings.java | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index f1556cac85c..75e9d548dcd 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -100,35 +100,34 @@ public Future getStoredData( return getFileContents(storedRequestsDirectory, requestIds).compose(storedIdToRequest -> getFileContents(storedImpressionsDirectory, impIds) - .map(storedIdToImp -> buildStoredDataResult(storedIdToRequest, storedIdToImp))); + .map(storedIdToImp -> buildStoredDataResult(storedIdToRequest, storedIdToImp, requestIds, impIds))); } - private StoredDataResult buildStoredDataResult(Map> storedIdToRequest, Map> storedIdToImp) { + private StoredDataResult buildStoredDataResult( + Map storedIdToRequest, + Map storedIdToImp, + Set requestIds, + Set impIds + ) { final List missingStoredRequestIds = - getMissingStoredDataIds(storedIdToRequest).stream() + getMissingStoredDataIds(storedIdToRequest, requestIds).stream() .map("No stored request found for id: %s"::formatted).toList(); final List missingStoredImpressionIds = - getMissingStoredDataIds(storedIdToImp).stream() + getMissingStoredDataIds(storedIdToImp, impIds).stream() .map("No stored impression found for id: %s"::formatted).toList(); return StoredDataResult.of( - filterOptionalFileContent(storedIdToRequest), - filterOptionalFileContent(storedIdToImp), + storedIdToRequest, + storedIdToImp, Stream.concat( missingStoredImpressionIds.stream(), missingStoredRequestIds.stream()).toList()); } - private Map filterOptionalFileContent(Map> fileContents) { - return fileContents - .entrySet() - .stream() - .filter(e -> e.getValue().isPresent()) - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().get())); - } - - private List getMissingStoredDataIds(Map> fileContents) { - return fileContents.entrySet().stream().filter(e -> e.getValue().isEmpty()).map(Map.Entry::getKey).toList(); + private List getMissingStoredDataIds(Map fileContents, Set responseIds) { + final List missingStoredDataIds = new ArrayList<>(responseIds); + missingStoredDataIds.removeAll(fileContents.keySet()); + return missingStoredDataIds; } @Override @@ -153,13 +152,10 @@ public Future getVideoStoredData( public Future getStoredResponses(Set responseIds, Timeout timeout) { return getFileContents(storedResponsesDirectory, responseIds).map(storedIdToResponse -> { final List missingStoredResponseIds = - getMissingStoredDataIds(storedIdToResponse).stream() + getMissingStoredDataIds(storedIdToResponse, responseIds).stream() .map("No stored response found for id: %s"::formatted).toList(); - return StoredResponseDataResult.of( - filterOptionalFileContent(storedIdToResponse), - missingStoredResponseIds - ); + return StoredResponseDataResult.of(storedIdToResponse, missingStoredResponseIds); }); } @@ -168,7 +164,7 @@ public Future> getCategories(String primaryAdServer, String return Future.succeededFuture(Collections.emptyMap()); } - private Future>> getFileContents(String directory, Set ids) { + private Future> getFileContents(String directory, Set ids) { final List>>> futureListContents = ids.stream() .map(impressionId -> downloadFile(directory + withInitialSlash(impressionId) + JSON_SUFFIX) @@ -179,7 +175,10 @@ private Future>> getFileContents(String directory, .all(new ArrayList<>(futureListContents)) .map(CompositeFuture::list); - return composedFutures.map(one -> one.stream().collect(Collectors.toMap(Tuple2::getLeft, Tuple2::getRight))); + // filter out IDs that had no stored request present and return a map from ids to stored request content + return composedFutures.map(one -> one.stream().flatMap(idContentTuple -> + idContentTuple.getRight().stream().map(content -> Tuple2.of(idContentTuple.getLeft(), content)) + )).map(one -> one.collect(Collectors.toMap(Tuple2::getLeft, Tuple2::getRight))); } /** From c2be492e69a04894533a6d3dcc3b8a5c5664658f Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Fri, 22 Dec 2023 15:51:30 +0100 Subject: [PATCH 08/52] Add empty lines and make withInitialSlash static --- .../org/prebid/server/settings/S3ApplicationSettings.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index 75e9d548dcd..cc11c2737d3 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -127,6 +127,7 @@ private StoredDataResult buildStoredDataResult( private List getMissingStoredDataIds(Map fileContents, Set responseIds) { final List missingStoredDataIds = new ArrayList<>(responseIds); missingStoredDataIds.removeAll(fileContents.keySet()); + return missingStoredDataIds; } @@ -136,6 +137,7 @@ public Future getAmpStoredData( Set requestIds, Set impIds, Timeout timeout) { + return getStoredData(accountId, requestIds, Collections.emptySet(), timeout); } @@ -145,6 +147,7 @@ public Future getVideoStoredData( Set requestIds, Set impIds, Timeout timeout) { + return getStoredData(accountId, requestIds, impIds, timeout); } @@ -188,7 +191,7 @@ private Future> getFileContents(String directory, Set Date: Fri, 22 Dec 2023 15:52:54 +0100 Subject: [PATCH 09/52] Fix checkstyle issues --- .../org/prebid/server/settings/S3ApplicationSettings.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index cc11c2737d3..833ceeb3c38 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -98,9 +98,10 @@ public Future getStoredData( Set impIds, Timeout timeout) { - return getFileContents(storedRequestsDirectory, requestIds).compose(storedIdToRequest -> - getFileContents(storedImpressionsDirectory, impIds) - .map(storedIdToImp -> buildStoredDataResult(storedIdToRequest, storedIdToImp, requestIds, impIds))); + return getFileContents(storedRequestsDirectory, requestIds) + .compose(storedIdToRequest -> getFileContents(storedImpressionsDirectory, impIds) + .map(storedIdToImp -> buildStoredDataResult(storedIdToRequest, storedIdToImp, requestIds, impIds)) + ); } private StoredDataResult buildStoredDataResult( From 5520b548372c2828ed3946c6c012d1da32218983 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Wed, 3 Jan 2024 10:09:45 +0100 Subject: [PATCH 10/52] Add requireNonNull checks --- .../server/settings/service/S3PeriodicRefreshService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 81f1146d81c..15c5aa564f9 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -66,10 +66,10 @@ public S3PeriodicRefreshService(S3AsyncClient asyncClient, Metrics metrics, Clock clock) { - this.asyncClient = asyncClient; - this.bucket = bucket; - this.storedRequestsDirectory = storedRequestsDirectory; - this.storedImpressionsDirectory = storedImpressionsDirectory; + this.asyncClient = Objects.requireNonNull(asyncClient); + this.bucket = Objects.requireNonNull(bucket); + this.storedRequestsDirectory = Objects.requireNonNull(storedRequestsDirectory); + this.storedImpressionsDirectory = Objects.requireNonNull(storedImpressionsDirectory); this.refreshPeriod = refreshPeriod; this.timeout = timeout; this.cacheType = Objects.requireNonNull(cacheType); From 6ab79c909d2856f2db5a7405060b40dcb7617407 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Wed, 3 Jan 2024 10:24:57 +0100 Subject: [PATCH 11/52] Check that accountId and file path match --- .../settings/S3ApplicationSettings.java | 14 +++++++- .../settings/S3ApplicationSettingsTest.java | 34 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index 833ceeb3c38..7a4eeb724cf 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -79,6 +79,14 @@ public Future getAccountById(String accountId, Timeout timeout) { .failedFuture(new PreBidException("Account with id %s not found".formatted(accountId))); } }) + .flatMap(account -> { + if (!Objects.equals(account.getId(), accountId)) { + return Future.failedFuture(new PreBidException( + "Account with id %s does not match id %s in file".formatted(accountId, account.getId())) + ); + } + return Future.succeededFuture(account); + }) .recover(ex -> { if (ex instanceof DecodeException) { return Future @@ -86,6 +94,10 @@ public Future getAccountById(String accountId, Timeout timeout) { new PreBidException( "Invalid json for account with id %s".formatted(accountId))); } + // if a previous validation already yielded a PreBidException, just return it + if(ex instanceof PreBidException) { + return Future.failedFuture(ex); + } return Future .failedFuture(new PreBidException("Account with id %s not found".formatted(accountId))); }); @@ -100,7 +112,7 @@ public Future getStoredData( return getFileContents(storedRequestsDirectory, requestIds) .compose(storedIdToRequest -> getFileContents(storedImpressionsDirectory, impIds) - .map(storedIdToImp -> buildStoredDataResult(storedIdToRequest, storedIdToImp, requestIds, impIds)) + .map(storedIdToImp -> buildStoredDataResult(storedIdToRequest, storedIdToImp, requestIds, impIds)) ); } diff --git a/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java b/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java index dec091bdb9a..989fbbc2f4c 100644 --- a/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java +++ b/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java @@ -159,6 +159,40 @@ public void getAccountByIdInvalidJson(TestContext context) { })); } + @Test + public void getAccountByIdWithAccountIdMismatch(TestContext context) throws JsonProcessingException { + // given + final Account account = Account.builder() + .id("wrong-id") + .auction(AccountAuctionConfig.builder().build()) + .privacy(AccountPrivacyConfig.of(null, null, null, null)) + .build(); + + given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) + .willReturn(CompletableFuture.completedFuture( + ResponseBytes.fromByteArray( + GetObjectResponse.builder().build(), + mapper.writeValueAsString(account).getBytes()))); + + // when + final Future future = s3ApplicationSettings.getAccountById("another-id", timeout); + + // then + final Async async = context.async(); + + future.onComplete(context.asyncAssertFailure(cause -> { + assertThat(cause) + .isInstanceOf(PreBidException.class) + .hasMessage("Account with id another-id does not match id wrong-id in file"); + + + verify(s3AsyncClient).getObject( + eq(GetObjectRequest.builder().bucket(BUCKET).key(ACCOUNTS_DIR + "/another-id.json").build()), + any(AsyncResponseTransformer.class)); + async.complete(); + })); + } + @Test public void getStoredDataShouldReturnFetchedStoredRequest(TestContext context) { // given From 183107a39ee5656c1cdbd21035df91e3501b4a14 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Mon, 22 Jan 2024 20:32:24 +0100 Subject: [PATCH 12/52] Fix linting issues --- .../java/org/prebid/server/settings/S3ApplicationSettings.java | 3 ++- .../org/prebid/server/settings/S3ApplicationSettingsTest.java | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index 7a4eeb724cf..e1419b704a5 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -112,7 +112,8 @@ public Future getStoredData( return getFileContents(storedRequestsDirectory, requestIds) .compose(storedIdToRequest -> getFileContents(storedImpressionsDirectory, impIds) - .map(storedIdToImp -> buildStoredDataResult(storedIdToRequest, storedIdToImp, requestIds, impIds)) + .map(storedIdToImp -> + buildStoredDataResult(storedIdToRequest, storedIdToImp, requestIds, impIds)) ); } diff --git a/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java b/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java index 989fbbc2f4c..ec0ead1d95b 100644 --- a/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java +++ b/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java @@ -185,7 +185,6 @@ public void getAccountByIdWithAccountIdMismatch(TestContext context) throws Json .isInstanceOf(PreBidException.class) .hasMessage("Account with id another-id does not match id wrong-id in file"); - verify(s3AsyncClient).getObject( eq(GetObjectRequest.builder().bucket(BUCKET).key(ACCOUNTS_DIR + "/another-id.json").build()), any(AsyncResponseTransformer.class)); From 9ad80b50e88e544f8d69d9844c2ca416dd1176ad Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Mon, 22 Jan 2024 20:34:37 +0100 Subject: [PATCH 13/52] Fix linting issue #2 --- .../java/org/prebid/server/settings/S3ApplicationSettings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index e1419b704a5..d33a0cd7825 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -95,7 +95,7 @@ public Future getAccountById(String accountId, Timeout timeout) { "Invalid json for account with id %s".formatted(accountId))); } // if a previous validation already yielded a PreBidException, just return it - if(ex instanceof PreBidException) { + if (ex instanceof PreBidException) { return Future.failedFuture(ex); } return Future From f02c42445bd49289b509dc838f62d8586fb75dfd Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Mon, 4 Mar 2024 13:02:50 +0100 Subject: [PATCH 14/52] Fix pom.xml --- pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index 2c88116f5bc..23f6f9ac87a 100644 --- a/pom.xml +++ b/pom.xml @@ -107,6 +107,8 @@ bom ${aws.awssdk.version} pom + + com.google.code.gson gson From 79c2f946182463558c7b2d3000d7a50b739a62f5 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Mon, 4 Mar 2024 13:03:16 +0100 Subject: [PATCH 15/52] Use Set instead of List --- .../server/settings/service/S3PeriodicRefreshService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 15c5aa564f9..100c1a88541 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -150,18 +150,18 @@ private StoredDataResult invalidate(StoredDataResult storedDataResult) { return storedDataResult; } - private Future> listFiles(String prefix) { + private Future> listFiles(String prefix) { final ListObjectsRequest listObjectsRequest = ListObjectsRequest.builder().bucket(bucket).prefix(prefix).build(); return Future.fromCompletionStage(asyncClient.listObjects(listObjectsRequest)) - .map(response -> response.contents().stream().map(S3Object::key).toList()); + .map(response -> response.contents().stream().map(S3Object::key).collect(Collectors.toSet())); } private Future> getFileContentsForDirectory(String directory) { return listFiles(directory) .compose(files -> - getFileContents(new HashSet<>(files)) + getFileContents(files) .map(map -> map.entrySet().stream().collect( Collectors.toMap( e -> stripFileName(directory, e.getKey()), From dc0f98563f11ead2cc45a73b69fa216f96fd332c Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Mon, 4 Mar 2024 13:07:11 +0100 Subject: [PATCH 16/52] Add empty lines after multi-line parameter function --- .../java/org/prebid/server/settings/S3ApplicationSettings.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index d33a0cd7825..7b00b5a6d27 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -123,6 +123,7 @@ private StoredDataResult buildStoredDataResult( Set requestIds, Set impIds ) { + final List missingStoredRequestIds = getMissingStoredDataIds(storedIdToRequest, requestIds).stream() .map("No stored request found for id: %s"::formatted).toList(); From a18a9b79390abf3e941536431ce270b6741b68a8 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Mon, 4 Mar 2024 13:07:22 +0100 Subject: [PATCH 17/52] Fix compile error in S3ApplicationSettingsTest --- .../org/prebid/server/settings/S3ApplicationSettingsTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java b/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java index ec0ead1d95b..86fbdd350c9 100644 --- a/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java +++ b/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java @@ -86,7 +86,7 @@ public void getAccountByIdShouldReturnFetchedAccount(TestContext context) throws .auction(AccountAuctionConfig.builder() .priceGranularity("testPriceGranularity") .build()) - .privacy(AccountPrivacyConfig.of(null, null, null, null)) + .privacy(AccountPrivacyConfig.of(null, null, null, null, null)) .build(); given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) @@ -165,7 +165,7 @@ public void getAccountByIdWithAccountIdMismatch(TestContext context) throws Json final Account account = Account.builder() .id("wrong-id") .auction(AccountAuctionConfig.builder().build()) - .privacy(AccountPrivacyConfig.of(null, null, null, null)) + .privacy(AccountPrivacyConfig.of(null, null, null, null, null)) .build(); given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) From 67f047ca4fc0578d59b787ee695e0f081a1ae2db Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Mon, 4 Mar 2024 13:18:28 +0100 Subject: [PATCH 18/52] Remove optional --- .../settings/S3ApplicationSettings.java | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index 7b00b5a6d27..54d002206fe 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -11,6 +11,7 @@ import org.prebid.server.settings.model.Account; import org.prebid.server.settings.model.StoredDataResult; import org.prebid.server.settings.model.StoredResponseDataResult; +import software.amazon.awssdk.core.BytesWrapper; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectRequest; @@ -28,7 +29,7 @@ /** * Implementation of {@link ApplicationSettings}. *

- * Reads an application settings from JSON file in an s3 bucket, stores and serves them in and from the memory. + * Reads an application settings from JSON file in a s3 bucket, stores and serves them in and from the memory. *

* Immediately loads stored request data from local files. These are stored in memory for low-latency reads. * This expects each file in the directory to be named "{config_id}.json". @@ -69,16 +70,7 @@ public S3ApplicationSettings( @Override public Future getAccountById(String accountId, Timeout timeout) { return downloadFile(accountsDirectory + "/" + accountId + JSON_SUFFIX) - .map(fileContentOpt -> - fileContentOpt.map(fileContent -> jacksonMapper.decodeValue(fileContent, Account.class))) - .compose(accountOpt -> { - if (accountOpt.isPresent()) { - return Future.succeededFuture(accountOpt.get()); - } else { - return Future - .failedFuture(new PreBidException("Account with id %s not found".formatted(accountId))); - } - }) + .map(fileContent -> jacksonMapper.decodeValue(fileContent, Account.class)) .flatMap(account -> { if (!Objects.equals(account.getId(), accountId)) { return Future.failedFuture(new PreBidException( @@ -183,7 +175,7 @@ public Future> getCategories(String primaryAdServer, String } private Future> getFileContents(String directory, Set ids) { - final List>>> futureListContents = ids.stream() + final List>> futureListContents = ids.stream() .map(impressionId -> downloadFile(directory + withInitialSlash(impressionId) + JSON_SUFFIX) .map(fileContent -> Tuple2.of(impressionId, fileContent))) @@ -213,13 +205,13 @@ private static String withInitialSlash(String impressionId) { return "/" + impressionId; } - private Future> downloadFile(String key) { + private Future downloadFile(String key) { final GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(key).build(); return Future.fromCompletionStage( asyncClient.getObject(request, AsyncResponseTransformer.toBytes()), vertx.getOrCreateContext()) - .map(test -> Optional.of(test.asUtf8String())).recover(ex -> Future.succeededFuture(Optional.empty())); + .map(BytesWrapper::asUtf8String); } } From 5ec3976292c3643d12fb06e21b42e498a2a05c66 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Sat, 9 Mar 2024 19:29:16 +0100 Subject: [PATCH 19/52] Remove unused import --- .../prebid/server/settings/service/S3PeriodicRefreshService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 100c1a88541..d99d3314a34 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -21,7 +21,6 @@ import java.time.Clock; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; From a3e350f565d032777e0120dc4fbadbd8775e8c5c Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Sat, 9 Mar 2024 20:05:52 +0100 Subject: [PATCH 20/52] Use AccountPrivacyConfig.builder --- .../org/prebid/server/settings/S3ApplicationSettingsTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java b/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java index 86fbdd350c9..4d84f88dbae 100644 --- a/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java +++ b/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java @@ -86,7 +86,7 @@ public void getAccountByIdShouldReturnFetchedAccount(TestContext context) throws .auction(AccountAuctionConfig.builder() .priceGranularity("testPriceGranularity") .build()) - .privacy(AccountPrivacyConfig.of(null, null, null, null, null)) + .privacy(AccountPrivacyConfig.builder().build()) .build(); given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) @@ -165,7 +165,7 @@ public void getAccountByIdWithAccountIdMismatch(TestContext context) throws Json final Account account = Account.builder() .id("wrong-id") .auction(AccountAuctionConfig.builder().build()) - .privacy(AccountPrivacyConfig.of(null, null, null, null, null)) + .privacy(AccountPrivacyConfig.builder().build()) .build(); given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) From a0aa64a3aab5720699a9acdec57f7f4128a05d73 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Mon, 25 Mar 2024 18:44:15 +0100 Subject: [PATCH 21/52] GD-7732 handle non existing stored impressions gracefully --- .../settings/S3ApplicationSettings.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index 54d002206fe..a0f9ded64ca 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -11,7 +11,6 @@ import org.prebid.server.settings.model.Account; import org.prebid.server.settings.model.StoredDataResult; import org.prebid.server.settings.model.StoredResponseDataResult; -import software.amazon.awssdk.core.BytesWrapper; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectRequest; @@ -70,6 +69,11 @@ public S3ApplicationSettings( @Override public Future getAccountById(String accountId, Timeout timeout) { return downloadFile(accountsDirectory + "/" + accountId + JSON_SUFFIX) + .flatMap(fileContentOpt -> fileContentOpt.map(Future::succeededFuture) + .orElseGet(() -> Future.failedFuture( + new PreBidException("Account with id %s not found".formatted(accountId))) + ) + ) .map(fileContent -> jacksonMapper.decodeValue(fileContent, Account.class)) .flatMap(account -> { if (!Objects.equals(account.getId(), accountId)) { @@ -175,20 +179,20 @@ public Future> getCategories(String primaryAdServer, String } private Future> getFileContents(String directory, Set ids) { - final List>> futureListContents = ids.stream() + final List>>> futureListContents = ids.stream() .map(impressionId -> downloadFile(directory + withInitialSlash(impressionId) + JSON_SUFFIX) - .map(fileContent -> Tuple2.of(impressionId, fileContent))) + .map(fileContentOpt -> fileContentOpt + .map(fileContent -> Tuple2.of(impressionId, fileContent)))) .collect(Collectors.toCollection(ArrayList::new)); - final Future>>> composedFutures = CompositeFuture + final Future>>> composedFutures = CompositeFuture .all(new ArrayList<>(futureListContents)) .map(CompositeFuture::list); // filter out IDs that had no stored request present and return a map from ids to stored request content - return composedFutures.map(one -> one.stream().flatMap(idContentTuple -> - idContentTuple.getRight().stream().map(content -> Tuple2.of(idContentTuple.getLeft(), content)) - )).map(one -> one.collect(Collectors.toMap(Tuple2::getLeft, Tuple2::getRight))); + return composedFutures.map(one -> one.stream().flatMap(Optional::stream)) + .map(one -> one.collect(Collectors.toMap(Tuple2::getLeft, Tuple2::getRight))); } /** @@ -205,13 +209,13 @@ private static String withInitialSlash(String impressionId) { return "/" + impressionId; } - private Future downloadFile(String key) { + private Future> downloadFile(String key) { final GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(key).build(); return Future.fromCompletionStage( asyncClient.getObject(request, AsyncResponseTransformer.toBytes()), vertx.getOrCreateContext()) - .map(BytesWrapper::asUtf8String); + .map(test -> Optional.of(test.asUtf8String())).recover(ex -> Future.succeededFuture(Optional.empty())); } } From 53ce17bf20c9966a0c7628722e62e30e76df566e Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Mon, 25 Mar 2024 22:00:20 +0100 Subject: [PATCH 22/52] GD-7732 Use SetUtils for calculating missing stored impressions --- .../org/prebid/server/settings/S3ApplicationSettings.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index a0f9ded64ca..eec4b4f80c8 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -3,6 +3,7 @@ import io.vertx.core.CompositeFuture; import io.vertx.core.Future; import io.vertx.core.Vertx; +import org.apache.commons.collections4.SetUtils; import org.prebid.server.auction.model.Tuple2; import org.prebid.server.exception.PreBidException; import org.prebid.server.execution.Timeout; @@ -136,10 +137,7 @@ private StoredDataResult buildStoredDataResult( } private List getMissingStoredDataIds(Map fileContents, Set responseIds) { - final List missingStoredDataIds = new ArrayList<>(responseIds); - missingStoredDataIds.removeAll(fileContents.keySet()); - - return missingStoredDataIds; + return SetUtils.difference(responseIds, fileContents.keySet()).stream().toList(); } @Override From 593967097959c3ee5136d166166b2886c333754d Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Tue, 26 Mar 2024 12:05:37 +0100 Subject: [PATCH 23/52] GD-7732 Use atomic reference and remove timeout --- .../settings/service/S3PeriodicRefreshService.java | 12 +++++------- .../server/spring/config/SettingsConfiguration.java | 1 - .../service/S3PeriodicRefreshServiceTest.java | 1 - 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index d99d3314a34..a1595670511 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -18,6 +18,7 @@ import software.amazon.awssdk.services.s3.model.ListObjectsRequest; import software.amazon.awssdk.services.s3.model.S3Object; +import java.util.concurrent.atomic.AtomicReference; import java.time.Clock; import java.util.ArrayList; import java.util.Collections; @@ -45,20 +46,18 @@ public class S3PeriodicRefreshService implements Initializable { private final String storedImpressionsDirectory; private final String storedRequestsDirectory; private final long refreshPeriod; - private final long timeout; private final MetricName cacheType; private final CacheNotificationListener cacheNotificationListener; private final Vertx vertx; private final Metrics metrics; private final Clock clock; - private StoredDataResult lastResult; + private AtomicReference lastResult; public S3PeriodicRefreshService(S3AsyncClient asyncClient, String bucket, String storedRequestsDirectory, String storedImpressionsDirectory, long refreshPeriod, - long timeout, MetricName cacheType, CacheNotificationListener cacheNotificationListener, Vertx vertx, @@ -70,7 +69,6 @@ public S3PeriodicRefreshService(S3AsyncClient asyncClient, this.storedRequestsDirectory = Objects.requireNonNull(storedRequestsDirectory); this.storedImpressionsDirectory = Objects.requireNonNull(storedImpressionsDirectory); this.refreshPeriod = refreshPeriod; - this.timeout = timeout; this.cacheType = Objects.requireNonNull(cacheType); this.cacheNotificationListener = Objects.requireNonNull(cacheNotificationListener); this.vertx = Objects.requireNonNull(vertx); @@ -116,7 +114,7 @@ private Void handleResult(StoredDataResult storedDataResult, long startTime, MetricName refreshType) { - lastResult = storedDataResult; + lastResult.set(storedDataResult); cacheNotificationListener.save(storedDataResult.getStoredIdToRequest(), storedDataResult.getStoredIdToImp()); @@ -137,10 +135,10 @@ private Future handleFailure(Throwable exception, long startTime, MetricNa private StoredDataResult invalidate(StoredDataResult storedDataResult) { final List invalidatedRequests = getInvalidatedKeys( storedDataResult.getStoredIdToRequest(), - lastResult != null ? lastResult.getStoredIdToRequest() : Collections.emptyMap()); + lastResult != null ? lastResult.get().getStoredIdToRequest() : Collections.emptyMap()); final List invalidatedImps = getInvalidatedKeys( storedDataResult.getStoredIdToImp(), - lastResult != null ? lastResult.getStoredIdToImp() : Collections.emptyMap()); + lastResult != null ? lastResult.get().getStoredIdToImp() : Collections.emptyMap()); if (!invalidatedRequests.isEmpty() || !invalidatedImps.isEmpty()) { cacheNotificationListener.invalidate(invalidatedRequests, invalidatedImps); diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java index df51b4e8fa9..29099a1523b 100644 --- a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java @@ -318,7 +318,6 @@ public S3PeriodicRefreshService s3PeriodicRefreshService( s3ConfigurationProperties.getStoredRequestsDir(), s3ConfigurationProperties.getStoredImpsDir(), refreshPeriod, - timeout, MetricName.stored_request, settingsCache, vertx, diff --git a/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java index 2a1d6e1b248..e3448069670 100644 --- a/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java +++ b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java @@ -185,7 +185,6 @@ private void createAndInitService(long refreshPeriod) { STORED_REQ_DIR, STORED_IMP_DIR, refreshPeriod, - 2000, MetricName.stored_request, cacheNotificationListener, vertx, From a085ae62e8911cf6a2bd3b6f27363ced19fe92d9 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Tue, 26 Mar 2024 12:06:46 +0100 Subject: [PATCH 24/52] GD-7732 Use SetUtils.difference --- .../server/settings/service/S3PeriodicRefreshService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index a1595670511..06131fdc4a9 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -5,6 +5,7 @@ import io.vertx.core.Vertx; import io.vertx.core.logging.Logger; import io.vertx.core.logging.LoggerFactory; +import org.apache.commons.collections4.SetUtils; import org.prebid.server.auction.model.Tuple2; import org.prebid.server.metric.MetricName; import org.prebid.server.metric.Metrics; @@ -77,7 +78,7 @@ public S3PeriodicRefreshService(S3AsyncClient asyncClient, } private static List getInvalidatedKeys(Map newMap, Map oldMap) { - return oldMap.keySet().stream().filter(s -> !newMap.containsKey(s)).toList(); + return SetUtils.difference(newMap.keySet(), oldMap.keySet()).stream().toList(); } @Override From 62f5096d83e32717af3f6f07e92cd2f4f03736a3 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Tue, 26 Mar 2024 12:08:07 +0100 Subject: [PATCH 25/52] GD-7732 Use onSuccess/onFailure instead of map/recover --- .../server/settings/service/S3PeriodicRefreshService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 06131fdc4a9..8b1702d1a9c 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -96,8 +96,8 @@ private void getAll() { .compose(storedIdToRequest -> getFileContentsForDirectory(storedImpressionsDirectory) .map(storedIdToImp -> StoredDataResult.of(storedIdToRequest, storedIdToImp, Collections.emptyList()))) - .map(storedDataResult -> handleResult(storedDataResult, startTime, MetricName.initialize)) - .recover(exception -> handleFailure(exception, startTime, MetricName.initialize)); + .onSuccess(storedDataResult -> handleResult(storedDataResult, startTime, MetricName.initialize)) + .onFailure(exception -> handleFailure(exception, startTime, MetricName.initialize)); } private void refresh() { @@ -107,8 +107,8 @@ private void refresh() { .compose(storedIdToRequest -> getFileContentsForDirectory(storedImpressionsDirectory) .map(storedIdToImp -> StoredDataResult.of(storedIdToRequest, storedIdToImp, Collections.emptyList()))) - .map(storedDataResult -> handleResult(invalidate(storedDataResult), startTime, MetricName.update)) - .recover(exception -> handleFailure(exception, startTime, MetricName.update)); + .onSuccess(storedDataResult -> handleResult(invalidate(storedDataResult), startTime, MetricName.update)) + .onFailure(exception -> handleFailure(exception, startTime, MetricName.update)); } private Void handleResult(StoredDataResult storedDataResult, From 8132c2600c37137c459072dce3256a75c7df3c37 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Tue, 26 Mar 2024 21:53:13 +0100 Subject: [PATCH 26/52] GD-7732 Remove redundant Set/Stream/List conversions --- .../settings/S3ApplicationSettings.java | 21 ++++++++----------- .../service/S3PeriodicRefreshService.java | 13 +++++++----- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index eec4b4f80c8..d7ae630850b 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -121,23 +121,23 @@ private StoredDataResult buildStoredDataResult( Set impIds ) { - final List missingStoredRequestIds = + final Stream missingStoredRequestIds = getMissingStoredDataIds(storedIdToRequest, requestIds).stream() - .map("No stored request found for id: %s"::formatted).toList(); - final List missingStoredImpressionIds = + .map("No stored request found for id: %s"::formatted); + final Stream missingStoredImpressionIds = getMissingStoredDataIds(storedIdToImp, impIds).stream() - .map("No stored impression found for id: %s"::formatted).toList(); + .map("No stored impression found for id: %s"::formatted); return StoredDataResult.of( storedIdToRequest, storedIdToImp, Stream.concat( - missingStoredImpressionIds.stream(), - missingStoredRequestIds.stream()).toList()); + missingStoredImpressionIds, + missingStoredRequestIds).toList()); } - private List getMissingStoredDataIds(Map fileContents, Set responseIds) { - return SetUtils.difference(responseIds, fileContents.keySet()).stream().toList(); + private Set getMissingStoredDataIds(Map fileContents, Set responseIds) { + return SetUtils.difference(responseIds, fileContents.keySet()); } @Override @@ -201,10 +201,7 @@ private Future> getFileContents(String directory, Set> downloadFile(String key) { diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 8b1702d1a9c..9d95871c809 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -77,8 +77,8 @@ public S3PeriodicRefreshService(S3AsyncClient asyncClient, this.clock = Objects.requireNonNull(clock); } - private static List getInvalidatedKeys(Map newMap, Map oldMap) { - return SetUtils.difference(newMap.keySet(), oldMap.keySet()).stream().toList(); + private static Set getInvalidatedKeys(Map newMap, Map oldMap) { + return SetUtils.difference(newMap.keySet(), oldMap.keySet()); } @Override @@ -134,15 +134,18 @@ private Future handleFailure(Throwable exception, long startTime, MetricNa } private StoredDataResult invalidate(StoredDataResult storedDataResult) { - final List invalidatedRequests = getInvalidatedKeys( + final Set invalidatedRequests = getInvalidatedKeys( storedDataResult.getStoredIdToRequest(), lastResult != null ? lastResult.get().getStoredIdToRequest() : Collections.emptyMap()); - final List invalidatedImps = getInvalidatedKeys( + final Set invalidatedImps = getInvalidatedKeys( storedDataResult.getStoredIdToImp(), lastResult != null ? lastResult.get().getStoredIdToImp() : Collections.emptyMap()); if (!invalidatedRequests.isEmpty() || !invalidatedImps.isEmpty()) { - cacheNotificationListener.invalidate(invalidatedRequests, invalidatedImps); + cacheNotificationListener.invalidate( + invalidatedRequests.stream().toList(), + invalidatedImps.stream().toList() + ); } return storedDataResult; From 754e31884e337632129292534b0c9b48e938ff88 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Tue, 26 Mar 2024 21:53:53 +0100 Subject: [PATCH 27/52] GD-7732 Rename aLong var to ignored --- .../server/settings/service/S3PeriodicRefreshService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 9d95871c809..96c0b505276 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -85,7 +85,7 @@ private static Set getInvalidatedKeys(Map newMap, Map 0) { - vertx.setPeriodic(refreshPeriod, aLong -> refresh()); + vertx.setPeriodic(refreshPeriod, ignored -> refresh()); } } From 1e40c2528c5af08c5e2ebf596d23af09e9a87f8f Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Tue, 26 Mar 2024 21:58:44 +0100 Subject: [PATCH 28/52] GD-7732 getFileContents runs in parallel --- .../settings/S3ApplicationSettings.java | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index d7ae630850b..184a1cc11f7 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -177,20 +177,15 @@ public Future> getCategories(String primaryAdServer, String } private Future> getFileContents(String directory, Set ids) { - final List>>> futureListContents = ids.stream() - .map(impressionId -> - downloadFile(directory + withInitialSlash(impressionId) + JSON_SUFFIX) + return CompositeFuture.all(ids.stream() + .map(impId -> downloadFile(directory + withInitialSlash(impId) + JSON_SUFFIX) .map(fileContentOpt -> fileContentOpt - .map(fileContent -> Tuple2.of(impressionId, fileContent)))) - .collect(Collectors.toCollection(ArrayList::new)); - - final Future>>> composedFutures = CompositeFuture - .all(new ArrayList<>(futureListContents)) - .map(CompositeFuture::list); - - // filter out IDs that had no stored request present and return a map from ids to stored request content - return composedFutures.map(one -> one.stream().flatMap(Optional::stream)) - .map(one -> one.collect(Collectors.toMap(Tuple2::getLeft, Tuple2::getRight))); + .map(fileContent -> Tuple2.of(impId, fileContent)))) + .toList()) + .map(CompositeFuture::>>list) + .map(impIdToFileContent -> impIdToFileContent.stream() + .flatMap(Optional::stream) + .collect(Collectors.toMap(Tuple2::getLeft, Tuple2::getRight))); } /** From 20cdf45f519f7f289df65e878793e07bee6fa107 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Tue, 26 Mar 2024 22:05:04 +0100 Subject: [PATCH 29/52] GD-7732 Use CompositeFutura.join instead of all --- .../java/org/prebid/server/settings/S3ApplicationSettings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index 184a1cc11f7..e70b005cd67 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -177,7 +177,7 @@ public Future> getCategories(String primaryAdServer, String } private Future> getFileContents(String directory, Set ids) { - return CompositeFuture.all(ids.stream() + return CompositeFuture.join(ids.stream() .map(impId -> downloadFile(directory + withInitialSlash(impId) + JSON_SUFFIX) .map(fileContentOpt -> fileContentOpt .map(fileContent -> Tuple2.of(impId, fileContent)))) From d089b27e4c8e8daa28c9d2d4fe9c3dbe6e592b52 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Sat, 25 May 2024 22:37:46 +0200 Subject: [PATCH 30/52] Fix compile error in SettingsConfiguration --- .../org/prebid/server/spring/config/SettingsConfiguration.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java index b84a72f6e4a..df8b0bdc943 100644 --- a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java @@ -343,8 +343,6 @@ CompositeApplicationSettings compositeApplicationSettings( @Autowired(required = false) DatabaseApplicationSettings databaseApplicationSettings, @Autowired(required = false) HttpApplicationSettings httpApplicationSettings, @Autowired(required = false) S3ApplicationSettings s3ApplicationSettings) { - @Autowired(required = false) DatabaseApplicationSettings databaseApplicationSettings, - @Autowired(required = false) HttpApplicationSettings httpApplicationSettings) { final List applicationSettingsList = Stream.of(s3ApplicationSettings, From f55befff26dbb8ad0e70718aff96b766e3c0b400 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Sun, 26 May 2024 12:35:38 +0200 Subject: [PATCH 31/52] Remove unused imports --- .../java/org/prebid/server/settings/S3ApplicationSettings.java | 1 - .../org/prebid/server/spring/config/SettingsConfiguration.java | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java index e70b005cd67..6ae0ec7f887 100644 --- a/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java +++ b/src/main/java/org/prebid/server/settings/S3ApplicationSettings.java @@ -16,7 +16,6 @@ import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java index df8b0bdc943..81a8441b673 100644 --- a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java @@ -20,13 +20,11 @@ import org.prebid.server.settings.EnrichingApplicationSettings; import org.prebid.server.settings.FileApplicationSettings; import org.prebid.server.settings.HttpApplicationSettings; -import org.prebid.server.settings.JdbcApplicationSettings; import org.prebid.server.settings.S3ApplicationSettings; import org.prebid.server.settings.SettingsCache; import org.prebid.server.settings.helper.ParametrizedQueryHelper; import org.prebid.server.settings.service.DatabasePeriodicRefreshService; import org.prebid.server.settings.service.HttpPeriodicRefreshService; -import org.prebid.server.settings.service.JdbcPeriodicRefreshService; import org.prebid.server.settings.service.S3PeriodicRefreshService; import org.prebid.server.spring.config.database.DatabaseConfiguration; import org.prebid.server.vertx.database.DatabaseClient; From bdc37f24ca5a33eadca8fe09ea6e9af30de46ef2 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Sun, 26 May 2024 14:30:01 +0200 Subject: [PATCH 32/52] Proper initialize implementation --- .../server/settings/service/S3PeriodicRefreshService.java | 5 ++++- .../settings/service/S3PeriodicRefreshServiceTest.java | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 96c0b505276..232e6ec518d 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -2,6 +2,7 @@ import io.vertx.core.CompositeFuture; import io.vertx.core.Future; +import io.vertx.core.Promise; import io.vertx.core.Vertx; import io.vertx.core.logging.Logger; import io.vertx.core.logging.LoggerFactory; @@ -82,11 +83,13 @@ private static Set getInvalidatedKeys(Map newMap, Map initializePromise) { getAll(); if (refreshPeriod > 0) { vertx.setPeriodic(refreshPeriod, ignored -> refresh()); } + + initializePromise.tryComplete(); } private void getAll() { diff --git a/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java index e3448069670..d2c2b5fa3a4 100644 --- a/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java +++ b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java @@ -1,6 +1,7 @@ package org.prebid.server.settings.service; import io.vertx.core.Handler; +import io.vertx.core.Promise; import io.vertx.core.Vertx; import org.junit.Before; import org.junit.Rule; @@ -190,7 +191,7 @@ private void createAndInitService(long refreshPeriod) { vertx, metrics, clock); - s3PeriodicRefreshService.initialize(); + s3PeriodicRefreshService.initialize(Promise.promise()); } @SuppressWarnings("unchecked") From a4469e271dbe53d5390e306108fa0b59b621dbfb Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Tue, 16 Jul 2024 20:15:29 +0200 Subject: [PATCH 33/52] Adding region property --- docs/application-settings.md | 1 + .../server/spring/config/SettingsConfiguration.java | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/application-settings.md b/docs/application-settings.md index 51954ccb2d4..39fd52c7e5a 100644 --- a/docs/application-settings.md +++ b/docs/application-settings.md @@ -274,6 +274,7 @@ settings: secretAccessKey: endpoint: # http://s3.storage.com bucket: # prebid-application-settings + region: # if not provided AWS_GLOBAL will be used. Example value: 'eu-central-1' accounts-dir: accounts stored-imps-dir: stored-impressions stored-requests-dir: stored-requests diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java index 81a8441b673..da42a0ab467 100644 --- a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java @@ -241,6 +241,10 @@ private static class S3ConfigurationProperties { private String accessKeyId; @NotBlank private String secretAccessKey; + /** + * If not provided AWS_GLOBAL will be used as a region + */ + private String region; @NotBlank private String endpoint; @NotBlank @@ -260,12 +264,14 @@ S3AsyncClient s3AsyncClient(S3ConfigurationProperties s3ConfigurationProperties) final AwsBasicCredentials credentials = AwsBasicCredentials.create( s3ConfigurationProperties.getAccessKeyId(), s3ConfigurationProperties.getSecretAccessKey()); - + final String awsRegionName = s3ConfigurationProperties.getRegion(); + final Region awsRegion = Objects.isNull(awsRegionName) ? Region.AWS_GLOBAL + : Region.of(s3ConfigurationProperties.getRegion()); return S3AsyncClient .builder() .credentialsProvider(StaticCredentialsProvider.create(credentials)) .endpointOverride(new URI(s3ConfigurationProperties.getEndpoint())) - .region(Region.EU_CENTRAL_1) + .region(awsRegion) .build(); } From 87466a731a7611af3c619cf4c996804322fc1807 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 18 Jul 2024 23:22:06 +0200 Subject: [PATCH 34/52] Migrate to Junit5 - one test case still broken --- .../service/S3PeriodicRefreshService.java | 1 + .../settings/S3ApplicationSettingsTest.java | 131 ++++++++---------- .../service/S3PeriodicRefreshServiceTest.java | 18 ++- 3 files changed, 63 insertions(+), 87 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 232e6ec518d..9dc1bf59cd8 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -76,6 +76,7 @@ public S3PeriodicRefreshService(S3AsyncClient asyncClient, this.vertx = Objects.requireNonNull(vertx); this.metrics = Objects.requireNonNull(metrics); this.clock = Objects.requireNonNull(clock); + this.lastResult = new AtomicReference<>(); } private static Set getInvalidatedKeys(Map newMap, Map oldMap) { diff --git a/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java b/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java index 4d84f88dbae..1747b6f3a4f 100644 --- a/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java +++ b/src/test/java/org/prebid/server/settings/S3ApplicationSettingsTest.java @@ -3,17 +3,14 @@ import com.fasterxml.jackson.core.JsonProcessingException; import io.vertx.core.Future; import io.vertx.core.Vertx; -import io.vertx.ext.unit.Async; -import io.vertx.ext.unit.TestContext; -import io.vertx.ext.unit.junit.VertxUnitRunner; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.runner.RunWith; +import io.vertx.junit5.VertxExtension; +import io.vertx.junit5.VertxTestContext; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; import org.prebid.server.VertxTest; import org.prebid.server.exception.PreBidException; import org.prebid.server.execution.Timeout; @@ -44,7 +41,7 @@ import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.verify; -@RunWith(VertxUnitRunner.class) +@ExtendWith({MockitoExtension.class, VertxExtension.class}) public class S3ApplicationSettingsTest extends VertxTest { private static final String BUCKET = "bucket"; @@ -52,20 +49,18 @@ public class S3ApplicationSettingsTest extends VertxTest { private static final String STORED_IMPS_DIR = "stored-imps"; private static final String STORED_REQUESTS_DIR = "stored-requests"; private static final String STORED_RESPONSES_DIR = "stored-responses"; - @Rule - public final MockitoRule mockitoRule = MockitoJUnit.rule(); private Timeout timeout; @Mock private S3AsyncClient s3AsyncClient; private Vertx vertx; - private S3ApplicationSettings s3ApplicationSettings; + private S3ApplicationSettings target; - @Before + @BeforeEach public void setUp() { vertx = Vertx.vertx(); - s3ApplicationSettings = new S3ApplicationSettings(s3AsyncClient, BUCKET, ACCOUNTS_DIR, + target = new S3ApplicationSettings(s3AsyncClient, BUCKET, ACCOUNTS_DIR, STORED_IMPS_DIR, STORED_REQUESTS_DIR, STORED_RESPONSES_DIR, jacksonMapper, vertx); final Clock clock = Clock.fixed(Instant.now(), ZoneId.systemDefault()); @@ -73,13 +68,13 @@ public void setUp() { timeout = timeoutFactory.create(500L); } - @After - public void tearDown(TestContext context) { - vertx.close(context.asyncAssertSuccess()); + @AfterEach + public void tearDown(VertxTestContext context) { + vertx.close(context.succeedingThenComplete()); } @Test - public void getAccountByIdShouldReturnFetchedAccount(TestContext context) throws JsonProcessingException { + public void getAccountByIdShouldReturnFetchedAccount(VertxTestContext context) throws JsonProcessingException { // given final Account account = Account.builder() .id("someId") @@ -96,24 +91,23 @@ public void getAccountByIdShouldReturnFetchedAccount(TestContext context) throws mapper.writeValueAsString(account).getBytes()))); // when - final Future future = s3ApplicationSettings.getAccountById("someId", timeout); + final Future future = target.getAccountById("someId", timeout); // then - final Async async = context.async(); - future.onComplete(context.asyncAssertSuccess(returnedAccount -> { + future.onComplete(context.succeeding(returnedAccount -> { assertThat(returnedAccount.getId()).isEqualTo("someId"); assertThat(returnedAccount.getAuction().getPriceGranularity()).isEqualTo("testPriceGranularity"); verify(s3AsyncClient).getObject( eq(GetObjectRequest.builder().bucket(BUCKET).key(ACCOUNTS_DIR + "/someId.json").build()), any(AsyncResponseTransformer.class)); - async.complete(); + context.completeNow(); })); } @Test - public void getAccountByIdNoSuchKey(TestContext context) { + public void getAccountByIdNoSuchKey(VertxTestContext context) { // given given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) .willReturn(CompletableFuture.failedFuture( @@ -122,22 +116,21 @@ public void getAccountByIdNoSuchKey(TestContext context) { new IllegalStateException("")))); // when - final Future future = s3ApplicationSettings.getAccountById("notFoundId", timeout); + final Future future = target.getAccountById("notFoundId", timeout); // then - final Async async = context.async(); - future.onComplete(context.asyncAssertFailure(cause -> { + future.onComplete(context.failing(cause -> { assertThat(cause) .isInstanceOf(PreBidException.class) .hasMessage("Account with id notFoundId not found"); - async.complete(); + context.completeNow(); })); } @Test - public void getAccountByIdInvalidJson(TestContext context) { + public void getAccountByIdInvalidJson(VertxTestContext context) { // given given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) .willReturn(CompletableFuture.completedFuture( @@ -146,21 +139,20 @@ public void getAccountByIdInvalidJson(TestContext context) { "invalidJson".getBytes()))); // when - final Future future = s3ApplicationSettings.getAccountById("invalidJsonId", timeout); + final Future future = target.getAccountById("invalidJsonId", timeout); // then - final Async async = context.async(); - future.onComplete(context.asyncAssertFailure(cause -> { + future.onComplete(context.failing(cause -> { assertThat(cause) .isInstanceOf(PreBidException.class) .hasMessage("Invalid json for account with id invalidJsonId"); - async.complete(); + context.completeNow(); })); } @Test - public void getAccountByIdWithAccountIdMismatch(TestContext context) throws JsonProcessingException { + public void getAccountByIdWithAccountIdMismatch(VertxTestContext context) throws JsonProcessingException { // given final Account account = Account.builder() .id("wrong-id") @@ -175,12 +167,10 @@ public void getAccountByIdWithAccountIdMismatch(TestContext context) throws Json mapper.writeValueAsString(account).getBytes()))); // when - final Future future = s3ApplicationSettings.getAccountById("another-id", timeout); + final Future future = target.getAccountById("another-id", timeout); // then - final Async async = context.async(); - - future.onComplete(context.asyncAssertFailure(cause -> { + future.onComplete(context.failing(cause -> { assertThat(cause) .isInstanceOf(PreBidException.class) .hasMessage("Account with id another-id does not match id wrong-id in file"); @@ -188,12 +178,12 @@ public void getAccountByIdWithAccountIdMismatch(TestContext context) throws Json verify(s3AsyncClient).getObject( eq(GetObjectRequest.builder().bucket(BUCKET).key(ACCOUNTS_DIR + "/another-id.json").build()), any(AsyncResponseTransformer.class)); - async.complete(); + context.completeNow(); })); } @Test - public void getStoredDataShouldReturnFetchedStoredRequest(TestContext context) { + public void getStoredDataShouldReturnFetchedStoredRequest(VertxTestContext context) { // given given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) .willReturn(CompletableFuture.completedFuture( @@ -202,13 +192,11 @@ public void getStoredDataShouldReturnFetchedStoredRequest(TestContext context) { "req1Result".getBytes()))); // when - final Future future = s3ApplicationSettings + final Future future = target .getStoredData("someId", Set.of("req1"), Collections.emptySet(), timeout); // then - final Async async = context.async(); - - future.onComplete(context.asyncAssertSuccess(account -> { + future.onComplete(context.succeeding(account -> { assertThat(account.getStoredIdToRequest().size()).isEqualTo(1); assertThat(account.getStoredIdToImp().size()).isEqualTo(0); assertThat(account.getStoredIdToRequest()).isEqualTo(Map.of("req1", "req1Result")); @@ -217,13 +205,12 @@ public void getStoredDataShouldReturnFetchedStoredRequest(TestContext context) { verify(s3AsyncClient).getObject( eq(GetObjectRequest.builder().bucket(BUCKET).key(STORED_REQUESTS_DIR + "/req1.json").build()), any(AsyncResponseTransformer.class)); - - async.complete(); + context.completeNow(); })); } @Test - public void getStoredDataShouldReturnFetchedStoredImpression(TestContext context) { + public void getStoredDataShouldReturnFetchedStoredImpression(VertxTestContext context) { // given given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) .willReturn(CompletableFuture.completedFuture( @@ -232,13 +219,11 @@ public void getStoredDataShouldReturnFetchedStoredImpression(TestContext context "imp1Result".getBytes()))); // when - final Future future = s3ApplicationSettings + final Future future = target .getStoredData("someId", Collections.emptySet(), Set.of("imp1"), timeout); // then - final Async async = context.async(); - - future.onComplete(context.asyncAssertSuccess(account -> { + future.onComplete(context.succeeding(account -> { assertThat(account.getStoredIdToRequest().size()).isEqualTo(0); assertThat(account.getStoredIdToImp().size()).isEqualTo(1); assertThat(account.getStoredIdToImp()).isEqualTo(Map.of("imp1", "imp1Result")); @@ -248,12 +233,12 @@ public void getStoredDataShouldReturnFetchedStoredImpression(TestContext context eq(GetObjectRequest.builder().bucket(BUCKET).key(STORED_IMPS_DIR + "/imp1.json").build()), any(AsyncResponseTransformer.class)); - async.complete(); + context.completeNow(); })); } @Test - public void getStoredDataShouldReturnFetchedStoredImpressionWithAdUnitPathStoredId(TestContext context) { + public void getStoredDataShouldReturnFetchedStoredImpressionWithAdUnitPathStoredId(VertxTestContext context) { // given given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) .willReturn(CompletableFuture.completedFuture( @@ -262,13 +247,11 @@ public void getStoredDataShouldReturnFetchedStoredImpressionWithAdUnitPathStored "imp1Result".getBytes()))); // when - final Future future = s3ApplicationSettings + final Future future = target .getStoredData("/123/root/position-1", Collections.emptySet(), Set.of("imp1"), timeout); // then - final Async async = context.async(); - - future.onComplete(context.asyncAssertSuccess(account -> { + future.onComplete(context.succeeding(account -> { assertThat(account.getStoredIdToRequest().size()).isEqualTo(0); assertThat(account.getStoredIdToImp().size()).isEqualTo(1); assertThat(account.getStoredIdToImp()).isEqualTo(Map.of("imp1", "imp1Result")); @@ -278,12 +261,12 @@ public void getStoredDataShouldReturnFetchedStoredImpressionWithAdUnitPathStored eq(GetObjectRequest.builder().bucket(BUCKET).key(STORED_IMPS_DIR + "/imp1.json").build()), any(AsyncResponseTransformer.class)); - async.complete(); + context.completeNow(); })); } @Test - public void getStoredDataShouldReturnFetchedStoredImpressionAndStoredRequest(TestContext context) { + public void getStoredDataShouldReturnFetchedStoredImpressionAndStoredRequest(VertxTestContext context) { // given given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) .willReturn( @@ -297,13 +280,11 @@ public void getStoredDataShouldReturnFetchedStoredImpressionAndStoredRequest(Tes "imp1Result".getBytes()))); // when - final Future future = s3ApplicationSettings + final Future future = target .getStoredData("someId", Set.of("req1"), Set.of("imp1"), timeout); // then - final Async async = context.async(); - - future.onComplete(context.asyncAssertSuccess(account -> { + future.onComplete(context.succeeding(account -> { assertThat(account.getStoredIdToRequest().size()).isEqualTo(1); assertThat(account.getStoredIdToRequest()).isEqualTo(Map.of("req1", "req1Result")); assertThat(account.getStoredIdToImp().size()).isEqualTo(1); @@ -317,12 +298,12 @@ public void getStoredDataShouldReturnFetchedStoredImpressionAndStoredRequest(Tes eq(GetObjectRequest.builder().bucket(BUCKET).key(STORED_REQUESTS_DIR + "/req1.json").build()), any(AsyncResponseTransformer.class)); - async.complete(); + context.completeNow(); })); } @Test - public void getStoredDataReturnsErrorsForNotFoundRequests(TestContext context) { + public void getStoredDataReturnsErrorsForNotFoundRequests(VertxTestContext context) { // given given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) .willReturn(CompletableFuture.failedFuture( @@ -331,13 +312,11 @@ public void getStoredDataReturnsErrorsForNotFoundRequests(TestContext context) { new IllegalStateException("")))); // when - final Future future = s3ApplicationSettings + final Future future = target .getStoredData("someId", Set.of("req1"), Collections.emptySet(), timeout); // then - final Async async = context.async(); - - future.onComplete(context.asyncAssertSuccess(account -> { + future.onComplete(context.succeeding(account -> { assertThat(account.getStoredIdToImp()).isEmpty(); assertThat(account.getStoredIdToRequest()).isEmpty(); assertThat(account.getErrors().size()).isEqualTo(1); @@ -346,12 +325,12 @@ public void getStoredDataReturnsErrorsForNotFoundRequests(TestContext context) { .hasSize(1) .isEqualTo(singletonList("No stored request found for id: req1")); - async.complete(); + context.completeNow(); })); } @Test - public void getStoredDataReturnsErrorsForNotFoundImpressions(TestContext context) { + public void getStoredDataReturnsErrorsForNotFoundImpressions(VertxTestContext context) { // given given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) .willReturn( @@ -361,13 +340,11 @@ public void getStoredDataReturnsErrorsForNotFoundImpressions(TestContext context new IllegalStateException("")))); // when - final Future future = s3ApplicationSettings + final Future future = target .getStoredData("someId", Collections.emptySet(), Set.of("imp1"), timeout); // then - final Async async = context.async(); - - future.onComplete(context.asyncAssertSuccess(account -> { + future.onComplete(context.succeeding(account -> { assertThat(account.getStoredIdToImp()).isEmpty(); assertThat(account.getStoredIdToRequest()).isEmpty(); assertThat(account.getErrors().size()).isEqualTo(1); @@ -376,7 +353,7 @@ public void getStoredDataReturnsErrorsForNotFoundImpressions(TestContext context .hasSize(1) .isEqualTo(singletonList("No stored impression found for id: imp1")); - async.complete(); + context.completeNow(); })); } diff --git a/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java index d2c2b5fa3a4..e46662045c2 100644 --- a/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java +++ b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java @@ -3,12 +3,11 @@ import io.vertx.core.Handler; import io.vertx.core.Promise; import io.vertx.core.Vertx; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; +import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.stubbing.Answer; import org.prebid.server.VertxTest; import org.prebid.server.metric.MetricName; @@ -38,24 +37,23 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; +import static org.mockito.Mock.Strictness.LENIENT; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +@ExtendWith(MockitoExtension.class) public class S3PeriodicRefreshServiceTest extends VertxTest { private static final String BUCKET = "bucket"; private static final String STORED_REQ_DIR = "stored-req"; private static final String STORED_IMP_DIR = "stored-imp"; - @Rule - public final MockitoRule mockitoRule = MockitoJUnit.rule(); - @Mock private CacheNotificationListener cacheNotificationListener; @Mock private Vertx vertx; - @Mock + @Mock(strictness = LENIENT) private S3AsyncClient s3AsyncClient; private final Clock clock = Clock.fixed(Instant.now(), ZoneId.systemDefault()); @Mock @@ -64,7 +62,7 @@ public class S3PeriodicRefreshServiceTest extends VertxTest { private final Map expectedRequests = singletonMap("id1", "value1"); private final Map expectedImps = singletonMap("id2", "value2"); - @Before + @BeforeEach public void setUp() { given(s3AsyncClient.listObjects(any(ListObjectsRequest.class))) .willReturn(listObjectResponse(STORED_REQ_DIR + "/id1.json"), From 9c2b22c35802e6a648813279b4dd558b6c2b2ae1 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 25 Jul 2024 11:42:58 +0200 Subject: [PATCH 35/52] Use prebid logger implementation, mark vars as final and return void instead of Void --- .../settings/service/S3PeriodicRefreshService.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 9dc1bf59cd8..46927377508 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -4,10 +4,10 @@ import io.vertx.core.Future; import io.vertx.core.Promise; import io.vertx.core.Vertx; -import io.vertx.core.logging.Logger; -import io.vertx.core.logging.LoggerFactory; import org.apache.commons.collections4.SetUtils; import org.prebid.server.auction.model.Tuple2; +import org.prebid.server.log.Logger; +import org.prebid.server.log.LoggerFactory; import org.prebid.server.metric.MetricName; import org.prebid.server.metric.Metrics; import org.prebid.server.settings.CacheNotificationListener; @@ -53,7 +53,7 @@ public class S3PeriodicRefreshService implements Initializable { private final Vertx vertx; private final Metrics metrics; private final Clock clock; - private AtomicReference lastResult; + private final AtomicReference lastResult; public S3PeriodicRefreshService(S3AsyncClient asyncClient, String bucket, @@ -115,7 +115,7 @@ private void refresh() { .onFailure(exception -> handleFailure(exception, startTime, MetricName.update)); } - private Void handleResult(StoredDataResult storedDataResult, + private void handleResult(StoredDataResult storedDataResult, long startTime, MetricName refreshType) { @@ -124,8 +124,6 @@ private Void handleResult(StoredDataResult storedDataResult, cacheNotificationListener.save(storedDataResult.getStoredIdToRequest(), storedDataResult.getStoredIdToImp()); metrics.updateSettingsCacheRefreshTime(cacheType, refreshType, clock.millis() - startTime); - - return null; } private Future handleFailure(Throwable exception, long startTime, MetricName refreshType) { From 6a5d1fa28e5e281383049b31f7381bed059eb6fc Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 25 Jul 2024 11:46:56 +0200 Subject: [PATCH 36/52] Use proposed refactoring --- .../service/S3PeriodicRefreshService.java | 40 ++++++++----------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 46927377508..783920907fa 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -85,40 +85,36 @@ private static Set getInvalidatedKeys(Map newMap, Map initializePromise) { - getAll(); + final long startTime = clock.millis(); + + fetchStoredDataResult() + .onSuccess(storedDataResult -> handleResult(storedDataResult, startTime, MetricName.initialize)) + .onFailure(exception -> handleFailure(exception, startTime, MetricName.initialize)) + .mapEmpty() + .onComplete(initializePromise); + if (refreshPeriod > 0) { vertx.setPeriodic(refreshPeriod, ignored -> refresh()); } - - initializePromise.tryComplete(); } - private void getAll() { - final long startTime = clock.millis(); - - getFileContentsForDirectory(storedRequestsDirectory) - .compose(storedIdToRequest -> getFileContentsForDirectory(storedImpressionsDirectory) - .map(storedIdToImp -> - StoredDataResult.of(storedIdToRequest, storedIdToImp, Collections.emptyList()))) - .onSuccess(storedDataResult -> handleResult(storedDataResult, startTime, MetricName.initialize)) - .onFailure(exception -> handleFailure(exception, startTime, MetricName.initialize)); + private Future fetchStoredDataResult() { + return Future.all( + getFileContentsForDirectory(storedRequestsDirectory), + getFileContentsForDirectory(storedImpressionsDirectory)) + .map(CompositeFuture::>list) + .map(results -> StoredDataResult.of(results.getFirst(), results.get(1), Collections.emptyList())); } private void refresh() { final long startTime = clock.millis(); - getFileContentsForDirectory(storedRequestsDirectory) - .compose(storedIdToRequest -> getFileContentsForDirectory(storedImpressionsDirectory) - .map(storedIdToImp -> - StoredDataResult.of(storedIdToRequest, storedIdToImp, Collections.emptyList()))) + fetchStoredDataResult() .onSuccess(storedDataResult -> handleResult(invalidate(storedDataResult), startTime, MetricName.update)) .onFailure(exception -> handleFailure(exception, startTime, MetricName.update)); } - private void handleResult(StoredDataResult storedDataResult, - long startTime, - MetricName refreshType) { - + private void handleResult(StoredDataResult storedDataResult, long startTime, MetricName refreshType) { lastResult.set(storedDataResult); cacheNotificationListener.save(storedDataResult.getStoredIdToRequest(), storedDataResult.getStoredIdToImp()); @@ -126,13 +122,11 @@ private void handleResult(StoredDataResult storedDataResult, metrics.updateSettingsCacheRefreshTime(cacheType, refreshType, clock.millis() - startTime); } - private Future handleFailure(Throwable exception, long startTime, MetricName refreshType) { + private void handleFailure(Throwable exception, long startTime, MetricName refreshType) { logger.warn("Error occurred while request to s3 refresh service", exception); metrics.updateSettingsCacheRefreshTime(cacheType, refreshType, clock.millis() - startTime); metrics.updateSettingsCacheRefreshErrorMetric(cacheType, refreshType); - - return Future.failedFuture(exception); } private StoredDataResult invalidate(StoredDataResult storedDataResult) { From 6a61e58ab9b45dc1e7f1048c18256bff12fe78a1 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 25 Jul 2024 11:50:14 +0200 Subject: [PATCH 37/52] Use vertx.getOrCreateContext() --- .../server/settings/service/S3PeriodicRefreshService.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 783920907fa..6edf6952f52 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -151,7 +151,7 @@ private Future> listFiles(String prefix) { final ListObjectsRequest listObjectsRequest = ListObjectsRequest.builder().bucket(bucket).prefix(prefix).build(); - return Future.fromCompletionStage(asyncClient.listObjects(listObjectsRequest)) + return Future.fromCompletionStage(asyncClient.listObjects(listObjectsRequest), vertx.getOrCreateContext()) .map(response -> response.contents().stream().map(S3Object::key).collect(Collectors.toSet())); } @@ -183,7 +183,8 @@ private Future> getFileContents(Set fileNames) { private Future downloadFile(String key) { final GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(key).build(); - return Future.fromCompletionStage(asyncClient.getObject(request, AsyncResponseTransformer.toBytes())) - .map(BytesWrapper::asUtf8String); + return Future.fromCompletionStage( + asyncClient.getObject(request, AsyncResponseTransformer.toBytes()), vertx.getOrCreateContext() + ).map(BytesWrapper::asUtf8String); } } From 70be3ca18ecb7d465c5a72db7e323c17e5ea556c Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 25 Jul 2024 12:03:48 +0200 Subject: [PATCH 38/52] Add proposed refactoring --- .../service/S3PeriodicRefreshService.java | 55 +++++++++---------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 6edf6952f52..9bd96d84035 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -147,44 +147,41 @@ private StoredDataResult invalidate(StoredDataResult storedDataResult) { return storedDataResult; } - private Future> listFiles(String prefix) { - final ListObjectsRequest listObjectsRequest = - ListObjectsRequest.builder().bucket(bucket).prefix(prefix).build(); - - return Future.fromCompletionStage(asyncClient.listObjects(listObjectsRequest), vertx.getOrCreateContext()) - .map(response -> response.contents().stream().map(S3Object::key).collect(Collectors.toSet())); - } - private Future> getFileContentsForDirectory(String directory) { return listFiles(directory) - .compose(files -> - getFileContents(files) - .map(map -> map.entrySet().stream().collect( - Collectors.toMap( - e -> stripFileName(directory, e.getKey()), - Map.Entry::getValue)))); - } - - private String stripFileName(String directory, String name) { - return name.replace(directory + "/", "").replace(JSON_SUFFIX, ""); + .map(files -> files.stream().map(this::downloadFile).toList()) + .compose(Future::all) + .map(CompositeFuture::>list) + .map(fileNameToContent -> fileNameToContent.stream() + .collect(Collectors.toMap( + entry -> stripFileName(directory, entry.getLeft()), + Tuple2::getRight))); } - private Future> getFileContents(Set fileNames) { - final List>> futureListContents = fileNames.stream() - .map(fileName -> downloadFile(fileName).map(fileContent -> Tuple2.of(fileName, fileContent))) - .collect(Collectors.toCollection(ArrayList::new)); - - final Future>> composedFutures = - CompositeFuture.all(new ArrayList<>(futureListContents)).map(CompositeFuture::list); + private Future> listFiles(String prefix) { + final ListObjectsRequest listObjectsRequest = ListObjectsRequest.builder() + .bucket(bucket) + .prefix(prefix) + .build(); - return composedFutures.map(one -> one.stream().collect(Collectors.toMap(Tuple2::getLeft, Tuple2::getRight))); + return Future.fromCompletionStage(asyncClient.listObjects(listObjectsRequest), vertx.getOrCreateContext()) + .map(response -> response.contents().stream() + .map(S3Object::key) + .collect(Collectors.toList())); } - private Future downloadFile(String key) { + private Future> downloadFile(String key) { final GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(key).build(); return Future.fromCompletionStage( - asyncClient.getObject(request, AsyncResponseTransformer.toBytes()), vertx.getOrCreateContext() - ).map(BytesWrapper::asUtf8String); + asyncClient.getObject(request, AsyncResponseTransformer.toBytes()), + vertx.getOrCreateContext()) + .map(content -> Tuple2.of(key, content.asUtf8String())); + } + + private String stripFileName(String directory, String name) { + return name + .replace(directory + "/", "") + .replace(JSON_SUFFIX, ""); } } From c97e0bfb86ab33e0f4769452736ad7ac3efe50c3 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 25 Jul 2024 22:48:50 +0200 Subject: [PATCH 39/52] Add sample config with s3 --- sample/configs/prebid-config-s3.yaml | 59 ++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 sample/configs/prebid-config-s3.yaml diff --git a/sample/configs/prebid-config-s3.yaml b/sample/configs/prebid-config-s3.yaml new file mode 100644 index 00000000000..b65d7a34b3b --- /dev/null +++ b/sample/configs/prebid-config-s3.yaml @@ -0,0 +1,59 @@ +status-response: "ok" + +server: + enable-quickack: true + enable-reuseport: true + +adapters: + appnexus: + enabled: true + ix: + enabled: true + openx: + enabled: true + pubmatic: + enabled: true + rubicon: + enabled: true +metrics: + prefix: prebid +cache: + scheme: http + host: localhost + path: /cache + query: uuid= +settings: + enforce-valid-account: false + generate-storedrequest-bidrequest-id: true + s3: + accessKeyId: prebid-server-test + secretAccessKey: nq9h6whXQURNL2NnWg3rcMlLMtGGDJeWrdl8hC9g + endpoint: http://localhost:9000 + bucket: prebid-server-configs-dev.h5v.eu # prebid-application-settings + # region: # if not provided AWS_GLOBAL will be used. Example value: 'eu-central-1' + accounts-dir: accounts + stored-imps-dir: stored-impressions + stored-requests-dir: stored-requests + stored-responses-dir: stored-responses + + in-memory-cache: + cache-size: 10000 + ttl-seconds: 1200 # 20 minutes + s3-update: + refresh-rate: 900000 # Refresh every 15 minutes + timeout: 5000 + +gdpr: + default-value: 1 + vendorlist: + v2: + cache-dir: /var/tmp/vendor2 + v3: + cache-dir: /var/tmp/vendor3 + +admin-endpoints: + logging-changelevel: + enabled: true + path: /logging/changelevel + on-application-port: true + protected: false From 7ea3745dd18f969fa853083839dcf41c5e8630f1 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 25 Jul 2024 22:49:01 +0200 Subject: [PATCH 40/52] Update aws depdendency --- extra/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extra/pom.xml b/extra/pom.xml index 08d177b988c..52f145ad28a 100644 --- a/extra/pom.xml +++ b/extra/pom.xml @@ -51,7 +51,7 @@ 3.21.7 3.17.3 1.0.7 - 2.17.274 + 2.26.24 3.5.4 From 8655f558bdd5f04b7d6e60f143605812b310a5cb Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Fri, 26 Jul 2024 14:43:05 +0200 Subject: [PATCH 41/52] Remove unnecessary beans --- .../spring/config/SettingsConfiguration.java | 25 +++++-------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java index da42a0ab467..e5bf7c944b9 100644 --- a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java @@ -295,32 +295,19 @@ S3ApplicationSettings s3ApplicationSettings( } @Configuration - @ConditionalOnProperty(prefix = "settings.in-memory-cache.s3-update", - name = {"refresh-rate", "timeout"}) + @ConditionalOnProperty(prefix = "settings.in-memory-cache.s3-update", name = {"refresh-rate", "timeout"}) static class S3PeriodicRefreshServiceConfiguration { - @Value("${settings.in-memory-cache.s3-update.refresh-rate}") - long refreshPeriod; - - @Value("${settings.in-memory-cache.s3-update.timeout}") - long timeout; - - @Autowired - Vertx vertx; - - @Autowired - HttpClient httpClient; - @Autowired - Metrics metrics; - @Autowired - Clock clock; - @Bean public S3PeriodicRefreshService s3PeriodicRefreshService( S3AsyncClient s3AsyncClient, S3SettingsConfiguration.S3ConfigurationProperties s3ConfigurationProperties, + @Value("${settings.in-memory-cache.s3-update.refresh-rate}") long refreshPeriod, SettingsCache settingsCache, - JacksonMapper mapper) { + Vertx vertx, + Metrics metrics, + Clock clock) { + return new S3PeriodicRefreshService( s3AsyncClient, s3ConfigurationProperties.getBucket(), From 5b21d386af40b85582b6572d92dbe9e455f84ed4 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Fri, 26 Jul 2024 14:46:12 +0200 Subject: [PATCH 42/52] Remove invalidate cache logic --- .../service/S3PeriodicRefreshService.java | 28 +------------------ .../service/S3PeriodicRefreshServiceTest.java | 24 ---------------- 2 files changed, 1 insertion(+), 51 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 9bd96d84035..51def464b62 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -4,7 +4,6 @@ import io.vertx.core.Future; import io.vertx.core.Promise; import io.vertx.core.Vertx; -import org.apache.commons.collections4.SetUtils; import org.prebid.server.auction.model.Tuple2; import org.prebid.server.log.Logger; import org.prebid.server.log.LoggerFactory; @@ -13,7 +12,6 @@ import org.prebid.server.settings.CacheNotificationListener; import org.prebid.server.settings.model.StoredDataResult; import org.prebid.server.vertx.Initializable; -import software.amazon.awssdk.core.BytesWrapper; import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectRequest; @@ -22,12 +20,10 @@ import java.util.concurrent.atomic.AtomicReference; import java.time.Clock; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.stream.Collectors; /** @@ -79,10 +75,6 @@ public S3PeriodicRefreshService(S3AsyncClient asyncClient, this.lastResult = new AtomicReference<>(); } - private static Set getInvalidatedKeys(Map newMap, Map oldMap) { - return SetUtils.difference(newMap.keySet(), oldMap.keySet()); - } - @Override public void initialize(Promise initializePromise) { final long startTime = clock.millis(); @@ -110,7 +102,7 @@ private void refresh() { final long startTime = clock.millis(); fetchStoredDataResult() - .onSuccess(storedDataResult -> handleResult(invalidate(storedDataResult), startTime, MetricName.update)) + .onSuccess(storedDataResult -> handleResult(storedDataResult, startTime, MetricName.update)) .onFailure(exception -> handleFailure(exception, startTime, MetricName.update)); } @@ -129,24 +121,6 @@ private void handleFailure(Throwable exception, long startTime, MetricName refre metrics.updateSettingsCacheRefreshErrorMetric(cacheType, refreshType); } - private StoredDataResult invalidate(StoredDataResult storedDataResult) { - final Set invalidatedRequests = getInvalidatedKeys( - storedDataResult.getStoredIdToRequest(), - lastResult != null ? lastResult.get().getStoredIdToRequest() : Collections.emptyMap()); - final Set invalidatedImps = getInvalidatedKeys( - storedDataResult.getStoredIdToImp(), - lastResult != null ? lastResult.get().getStoredIdToImp() : Collections.emptyMap()); - - if (!invalidatedRequests.isEmpty() || !invalidatedImps.isEmpty()) { - cacheNotificationListener.invalidate( - invalidatedRequests.stream().toList(), - invalidatedImps.stream().toList() - ); - } - - return storedDataResult; - } - private Future> getFileContentsForDirectory(String directory) { return listFiles(directory) .map(files -> files.stream().map(this::downloadFile).toList()) diff --git a/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java index e46662045c2..dca501e8d1f 100644 --- a/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java +++ b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java @@ -88,30 +88,6 @@ public void shouldCallSaveWithExpectedParameters() { verify(cacheNotificationListener).save(expectedRequests, expectedImps); } - @Test - public void shouldCallInvalidateAndSaveWithExpectedParameters() { - // given - given(vertx.setPeriodic(anyLong(), any())) - .willAnswer(withSelfAndPassObjectToHandler(1L)); - given(s3AsyncClient.listObjects(any(ListObjectsRequest.class))) - .willReturn(listObjectResponse(STORED_REQ_DIR + "/id1.json"), - listObjectResponse(STORED_IMP_DIR + "/id2.json"), - listObjectResponse(), - listObjectResponse(STORED_IMP_DIR + "/id2.json")); - given(s3AsyncClient.getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class))) - .willReturn(getObjectResponse("value1"), - getObjectResponse("value2"), - getObjectResponse("changed_value")); - - // when - createAndInitService(1000); - - // then - verify(cacheNotificationListener).save(expectedRequests, expectedImps); - verify(cacheNotificationListener).invalidate(singletonList("id1"), emptyList()); - verify(cacheNotificationListener).save(emptyMap(), singletonMap("id2", "changed_value")); - } - @Test public void initializeShouldMakeOneInitialRequestAndTwoScheduledRequestsWithParam() { // given From f8986485975fa5fc2dae0cab27a45eab76af7894 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Fri, 26 Jul 2024 14:47:51 +0200 Subject: [PATCH 43/52] Change private to protected to satisfy IDEA --- .../prebid/server/spring/config/SettingsConfiguration.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java index e5bf7c944b9..508d754dae2 100644 --- a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java @@ -236,7 +236,7 @@ static class S3SettingsConfiguration { @Validated @Data @NoArgsConstructor - private static class S3ConfigurationProperties { + protected static class S3ConfigurationProperties { @NotBlank private String accessKeyId; @NotBlank @@ -445,7 +445,7 @@ SettingsCache videoSettingCache(ApplicationSettingsCacheProperties cacheProperti @Validated @Data @NoArgsConstructor - private static class ApplicationSettingsCacheProperties { + protected static class ApplicationSettingsCacheProperties { @NotNull @Min(1) From e89c0f01b028f956b4af4cd9f2ef21a0c5b91dd2 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Fri, 26 Jul 2024 14:49:10 +0200 Subject: [PATCH 44/52] Formatting --- .../prebid/server/spring/config/SettingsConfiguration.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java index 508d754dae2..7eea7895764 100644 --- a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java @@ -265,8 +265,9 @@ S3AsyncClient s3AsyncClient(S3ConfigurationProperties s3ConfigurationProperties) s3ConfigurationProperties.getAccessKeyId(), s3ConfigurationProperties.getSecretAccessKey()); final String awsRegionName = s3ConfigurationProperties.getRegion(); - final Region awsRegion = Objects.isNull(awsRegionName) ? Region.AWS_GLOBAL - : Region.of(s3ConfigurationProperties.getRegion()); + final Region awsRegion = awsRegionName != null + ? Region.of(s3ConfigurationProperties.getRegion()) + : Region.AWS_GLOBAL; return S3AsyncClient .builder() .credentialsProvider(StaticCredentialsProvider.create(credentials)) From 122937268154094b9289597d89e41d1932b743c0 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Mon, 29 Jul 2024 10:47:04 +0200 Subject: [PATCH 45/52] remove unused imports --- .../server/settings/service/S3PeriodicRefreshServiceTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java index dca501e8d1f..f93ef16b865 100644 --- a/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java +++ b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java @@ -29,9 +29,6 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; From 31febb49b7634c30e1fac58d11c879aa3216a26b Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Mon, 19 Aug 2024 15:28:53 +0200 Subject: [PATCH 46/52] Remove vertx.getOrCreateContext() call --- .../server/settings/service/S3PeriodicRefreshService.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 51def464b62..7cb92b2496b 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -138,7 +138,7 @@ private Future> listFiles(String prefix) { .prefix(prefix) .build(); - return Future.fromCompletionStage(asyncClient.listObjects(listObjectsRequest), vertx.getOrCreateContext()) + return Future.fromCompletionStage(asyncClient.listObjects(listObjectsRequest)) .map(response -> response.contents().stream() .map(S3Object::key) .collect(Collectors.toList())); @@ -147,9 +147,7 @@ private Future> listFiles(String prefix) { private Future> downloadFile(String key) { final GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(key).build(); - return Future.fromCompletionStage( - asyncClient.getObject(request, AsyncResponseTransformer.toBytes()), - vertx.getOrCreateContext()) + return Future.fromCompletionStage(asyncClient.getObject(request, AsyncResponseTransformer.toBytes())) .map(content -> Tuple2.of(key, content.asUtf8String())); } From cc2972114c8df54bd86093a5277859961d86eab4 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Tue, 20 Aug 2024 21:54:49 +0200 Subject: [PATCH 47/52] Revert "Remove vertx.getOrCreateContext() call" This reverts commit 31febb49b7634c30e1fac58d11c879aa3216a26b. --- .../server/settings/service/S3PeriodicRefreshService.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 7cb92b2496b..51def464b62 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -138,7 +138,7 @@ private Future> listFiles(String prefix) { .prefix(prefix) .build(); - return Future.fromCompletionStage(asyncClient.listObjects(listObjectsRequest)) + return Future.fromCompletionStage(asyncClient.listObjects(listObjectsRequest), vertx.getOrCreateContext()) .map(response -> response.contents().stream() .map(S3Object::key) .collect(Collectors.toList())); @@ -147,7 +147,9 @@ private Future> listFiles(String prefix) { private Future> downloadFile(String key) { final GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(key).build(); - return Future.fromCompletionStage(asyncClient.getObject(request, AsyncResponseTransformer.toBytes())) + return Future.fromCompletionStage( + asyncClient.getObject(request, AsyncResponseTransformer.toBytes()), + vertx.getOrCreateContext()) .map(content -> Tuple2.of(key, content.asUtf8String())); } From ea36f03c040f2a45933a872770f5907b3b24a9f0 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Tue, 20 Aug 2024 22:18:14 +0200 Subject: [PATCH 48/52] Reintroduce vertx.getOrCreateContext --- .../service/S3PeriodicRefreshServiceTest.java | 89 +++++++++++++------ 1 file changed, 60 insertions(+), 29 deletions(-) diff --git a/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java index f93ef16b865..b8a7b2abd6d 100644 --- a/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java +++ b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java @@ -1,8 +1,12 @@ package org.prebid.server.settings.service; +import io.vertx.core.Future; import io.vertx.core.Handler; import io.vertx.core.Promise; import io.vertx.core.Vertx; +import io.vertx.junit5.VertxExtension; +import io.vertx.junit5.VertxTestContext; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -40,6 +44,7 @@ import static org.mockito.Mockito.verify; @ExtendWith(MockitoExtension.class) +@ExtendWith(VertxExtension.class) public class S3PeriodicRefreshServiceTest extends VertxTest { private static final String BUCKET = "bucket"; @@ -48,8 +53,11 @@ public class S3PeriodicRefreshServiceTest extends VertxTest { @Mock private CacheNotificationListener cacheNotificationListener; + @Mock private Vertx vertx; + private Vertx vertxImpl; + @Mock(strictness = LENIENT) private S3AsyncClient s3AsyncClient; private final Clock clock = Clock.fixed(Instant.now(), ZoneId.systemDefault()); @@ -61,6 +69,7 @@ public class S3PeriodicRefreshServiceTest extends VertxTest { @BeforeEach public void setUp() { + vertxImpl = Vertx.vertx(); given(s3AsyncClient.listObjects(any(ListObjectsRequest.class))) .willReturn(listObjectResponse(STORED_REQ_DIR + "/id1.json"), listObjectResponse(STORED_IMP_DIR + "/id2.json")); @@ -74,65 +83,85 @@ public void setUp() { ResponseBytes.fromByteArray( GetObjectResponse.builder().build(), "value2".getBytes()))); + + given(vertx.getOrCreateContext()).willReturn(vertxImpl.getOrCreateContext()); + } + + @AfterEach + public void tearDown(VertxTestContext context) { + vertxImpl.close(context.succeedingThenComplete()); } @Test - public void shouldCallSaveWithExpectedParameters() { + public void shouldCallSaveWithExpectedParameters(VertxTestContext context) { // when - createAndInitService(1000); + createAndInitService(1000) + .onSuccess((unused) -> { + verify(cacheNotificationListener).save(expectedRequests, expectedImps); + }) + .onComplete(context.succeedingThenComplete()); - // then - verify(cacheNotificationListener).save(expectedRequests, expectedImps); } @Test - public void initializeShouldMakeOneInitialRequestAndTwoScheduledRequestsWithParam() { + public void initializeShouldMakeOneInitialRequestAndTwoScheduledRequestsWithParam(VertxTestContext context) { // given given(vertx.setPeriodic(anyLong(), any())) .willAnswer(withSelfAndPassObjectToHandler(1L, 2L)); // when - createAndInitService(1000); + createAndInitService(1000) + .onSuccess((unused) -> { + // then + verify(s3AsyncClient, times(6)).listObjects(any(ListObjectsRequest.class)); + verify(s3AsyncClient, times(6)).getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class)); + }) + .onComplete(context.succeedingThenComplete()); - // then - verify(s3AsyncClient, times(6)).listObjects(any(ListObjectsRequest.class)); - verify(s3AsyncClient, times(6)).getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class)); } @Test - public void initializeShouldMakeOnlyOneInitialRequestIfRefreshPeriodIsNegative() { + public void initializeShouldMakeOnlyOneInitialRequestIfRefreshPeriodIsNegative(VertxTestContext context) { // when - createAndInitService(-1); + createAndInitService(-1) + .onSuccess((unused) -> { + // then + verify(vertx, never()).setPeriodic(anyLong(), any()); + verify(s3AsyncClient, times(2)).listObjects(any(ListObjectsRequest.class)); + }) + .onComplete(context.succeedingThenComplete());; - // then - verify(vertx, never()).setPeriodic(anyLong(), any()); - verify(s3AsyncClient, times(2)).listObjects(any(ListObjectsRequest.class)); } @Test - public void shouldUpdateTimerMetric() { + public void shouldUpdateTimerMetric(VertxTestContext context) { // when - createAndInitService(1000); - - // then - verify(metrics).updateSettingsCacheRefreshTime( - eq(MetricName.stored_request), eq(MetricName.initialize), anyLong()); + createAndInitService(1000) + .onSuccess((unused) -> { + // then + verify(metrics).updateSettingsCacheRefreshTime( + eq(MetricName.stored_request), eq(MetricName.initialize), anyLong()); + }) + .onComplete(context.succeedingThenComplete()); } @Test - public void shouldUpdateTimerAndErrorMetric() { + public void shouldUpdateTimerAndErrorMetric(VertxTestContext context) { // given given(s3AsyncClient.listObjects(any(ListObjectsRequest.class))) .willReturn(CompletableFuture.failedFuture(new IllegalStateException("Failed"))); // when - createAndInitService(1000); + createAndInitService(1000) + .onFailure((unused) -> { + // then + verify(metrics).updateSettingsCacheRefreshTime( + eq(MetricName.stored_request), eq(MetricName.initialize), anyLong()); + verify(metrics).updateSettingsCacheRefreshErrorMetric( + eq(MetricName.stored_request), eq(MetricName.initialize)); + }) + .onComplete(context.failingThenComplete()); - // then - verify(metrics).updateSettingsCacheRefreshTime( - eq(MetricName.stored_request), eq(MetricName.initialize), anyLong()); - verify(metrics).updateSettingsCacheRefreshErrorMetric( - eq(MetricName.stored_request), eq(MetricName.initialize)); } private CompletableFuture listObjectResponse(String... keys) { @@ -150,7 +179,7 @@ private CompletableFuture> getObjectResponse(St value.getBytes())); } - private void createAndInitService(long refreshPeriod) { + private Future createAndInitService(long refreshPeriod) { final S3PeriodicRefreshService s3PeriodicRefreshService = new S3PeriodicRefreshService( s3AsyncClient, BUCKET, @@ -162,7 +191,9 @@ private void createAndInitService(long refreshPeriod) { vertx, metrics, clock); - s3PeriodicRefreshService.initialize(Promise.promise()); + Promise init = Promise.promise(); + s3PeriodicRefreshService.initialize(init); + return init.future(); } @SuppressWarnings("unchecked") From 8f6cddd25755b216f191ba96a04ed0efd061fa66 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Tue, 20 Aug 2024 22:21:22 +0200 Subject: [PATCH 49/52] hello checkstyle my old friend ... --- .../service/S3PeriodicRefreshServiceTest.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java index b8a7b2abd6d..92619c0f320 100644 --- a/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java +++ b/src/test/java/org/prebid/server/settings/service/S3PeriodicRefreshServiceTest.java @@ -96,7 +96,7 @@ public void tearDown(VertxTestContext context) { public void shouldCallSaveWithExpectedParameters(VertxTestContext context) { // when createAndInitService(1000) - .onSuccess((unused) -> { + .onSuccess(unused -> { verify(cacheNotificationListener).save(expectedRequests, expectedImps); }) .onComplete(context.succeedingThenComplete()); @@ -111,10 +111,12 @@ public void initializeShouldMakeOneInitialRequestAndTwoScheduledRequestsWithPara // when createAndInitService(1000) - .onSuccess((unused) -> { + .onSuccess(unused -> { // then verify(s3AsyncClient, times(6)).listObjects(any(ListObjectsRequest.class)); - verify(s3AsyncClient, times(6)).getObject(any(GetObjectRequest.class), any(AsyncResponseTransformer.class)); + verify(s3AsyncClient, times(6)).getObject( + any(GetObjectRequest.class), any(AsyncResponseTransformer.class) + ); }) .onComplete(context.succeedingThenComplete()); @@ -124,12 +126,12 @@ public void initializeShouldMakeOneInitialRequestAndTwoScheduledRequestsWithPara public void initializeShouldMakeOnlyOneInitialRequestIfRefreshPeriodIsNegative(VertxTestContext context) { // when createAndInitService(-1) - .onSuccess((unused) -> { + .onSuccess(unused -> { // then verify(vertx, never()).setPeriodic(anyLong(), any()); verify(s3AsyncClient, times(2)).listObjects(any(ListObjectsRequest.class)); }) - .onComplete(context.succeedingThenComplete());; + .onComplete(context.succeedingThenComplete()); } @@ -137,7 +139,7 @@ public void initializeShouldMakeOnlyOneInitialRequestIfRefreshPeriodIsNegative(V public void shouldUpdateTimerMetric(VertxTestContext context) { // when createAndInitService(1000) - .onSuccess((unused) -> { + .onSuccess(unused -> { // then verify(metrics).updateSettingsCacheRefreshTime( eq(MetricName.stored_request), eq(MetricName.initialize), anyLong()); @@ -153,7 +155,7 @@ public void shouldUpdateTimerAndErrorMetric(VertxTestContext context) { // when createAndInitService(1000) - .onFailure((unused) -> { + .onFailure(unused -> { // then verify(metrics).updateSettingsCacheRefreshTime( eq(MetricName.stored_request), eq(MetricName.initialize), anyLong()); @@ -191,7 +193,7 @@ private Future createAndInitService(long refreshPeriod) { vertx, metrics, clock); - Promise init = Promise.promise(); + final Promise init = Promise.promise(); s3PeriodicRefreshService.initialize(init); return init.future(); } From cda0e5abbabc187dd38dac5292669eb7847ee2fa Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 22 Aug 2024 16:58:36 +0200 Subject: [PATCH 50/52] Add force-path-style value --- sample/configs/prebid-config-s3.yaml | 3 ++- .../prebid/server/spring/config/SettingsConfiguration.java | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sample/configs/prebid-config-s3.yaml b/sample/configs/prebid-config-s3.yaml index b65d7a34b3b..277ad94613c 100644 --- a/sample/configs/prebid-config-s3.yaml +++ b/sample/configs/prebid-config-s3.yaml @@ -29,7 +29,8 @@ settings: accessKeyId: prebid-server-test secretAccessKey: nq9h6whXQURNL2NnWg3rcMlLMtGGDJeWrdl8hC9g endpoint: http://localhost:9000 - bucket: prebid-server-configs-dev.h5v.eu # prebid-application-settings + bucket: prebid-server-configs.example.com # prebid-application-settings + force-path-style: true # virtual bucketing # region: # if not provided AWS_GLOBAL will be used. Example value: 'eu-central-1' accounts-dir: accounts stored-imps-dir: stored-impressions diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java index 7eea7895764..cbe3d503e4b 100644 --- a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java @@ -250,6 +250,8 @@ protected static class S3ConfigurationProperties { @NotBlank private String bucket; @NotBlank + private Boolean forcePathStyle; + @NotBlank private String accountsDir; @NotBlank private String storedImpsDir; @@ -268,10 +270,12 @@ S3AsyncClient s3AsyncClient(S3ConfigurationProperties s3ConfigurationProperties) final Region awsRegion = awsRegionName != null ? Region.of(s3ConfigurationProperties.getRegion()) : Region.AWS_GLOBAL; + System.out.println("Path Style: " + s3ConfigurationProperties.getForcePathStyle()); return S3AsyncClient .builder() .credentialsProvider(StaticCredentialsProvider.create(credentials)) .endpointOverride(new URI(s3ConfigurationProperties.getEndpoint())) + .forcePathStyle(s3ConfigurationProperties.getForcePathStyle()) .region(awsRegion) .build(); } From 332d5d4f6945ff2ae87244e6f3bcb3711c0c8121 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 22 Aug 2024 17:00:58 +0200 Subject: [PATCH 51/52] Log refresh period --- .../prebid/server/settings/service/S3PeriodicRefreshService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java index 51def464b62..8b31811cdd3 100644 --- a/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java +++ b/src/main/java/org/prebid/server/settings/service/S3PeriodicRefreshService.java @@ -86,6 +86,7 @@ public void initialize(Promise initializePromise) { .onComplete(initializePromise); if (refreshPeriod > 0) { + logger.info("Starting s3 periodic refresh for " + cacheType + " every " + refreshPeriod + " s"); vertx.setPeriodic(refreshPeriod, ignored -> refresh()); } } From 68f29750c126051e93a90ca12f481dc3865d5e81 Mon Sep 17 00:00:00 2001 From: Nepomuk Seiler Date: Thu, 22 Aug 2024 17:04:51 +0200 Subject: [PATCH 52/52] Remove println debugging --- .../org/prebid/server/spring/config/SettingsConfiguration.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java index cbe3d503e4b..90f56672b00 100644 --- a/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/SettingsConfiguration.java @@ -270,7 +270,6 @@ S3AsyncClient s3AsyncClient(S3ConfigurationProperties s3ConfigurationProperties) final Region awsRegion = awsRegionName != null ? Region.of(s3ConfigurationProperties.getRegion()) : Region.AWS_GLOBAL; - System.out.println("Path Style: " + s3ConfigurationProperties.getForcePathStyle()); return S3AsyncClient .builder() .credentialsProvider(StaticCredentialsProvider.create(credentials))