From c6bfb41370fb0055685640d3c446509734eb23b8 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 13 Dec 2024 15:10:45 +0100 Subject: [PATCH] Quarkus-based Polaris service runtime --- Dockerfile | 36 +- LICENSE-BINARY-DIST | 857 +++++++++++++++++- aggregated-license-report/build.gradle.kts | 2 +- api/iceberg-service/build.gradle.kts | 3 + api/management-model/build.gradle.kts | 3 + api/management-service/build.gradle.kts | 3 + build.gradle.kts | 3 + docker-compose.yml | 9 +- dropwizard/service/README-quarkus.md | 97 ++ dropwizard/service/build.gradle.kts | 190 ++-- .../service/src/main/docker/Dockerfile.jvm | 49 + .../dropwizard/BootstrapRealmsCommand.java | 77 -- .../dropwizard/PolarisApplication.java | 584 ------------ .../dropwizard/PurgeRealmsCommand.java | 50 - .../TimedApplicationEventListener.java | 121 --- .../QuarkusAuthenticatorConfiguration.java | 61 ++ .../dropwizard/auth/QuarkusOAuthFilter.java | 159 ++++ .../io/QuarkusCatalogIOConfiguration.java} | 24 +- .../dropwizard/config/CorsConfiguration.java | 95 -- .../config/PolarisApplicationConfig.java | 494 ---------- .../config/QuarkusFeaturesConfiguration.java | 37 + .../config/QuarkusJacksonConfig.java | 67 ++ .../dropwizard/config/QuarkusProducers.java | 206 +++++ .../context/QuarkusContextConfiguration.java | 40 + .../dropwizard/context/RealmScopeContext.java | 99 -- .../JerseyViolationExceptionMapper.java | 41 - .../logging/PolarisJsonLayoutFactory.java | 242 ----- .../logging/QuarkusLoggingConfiguration.java} | 26 +- .../logging/QuarkusLoggingMDCFilter.java | 72 ++ .../metrics/QuarkusMeterFilterProducer.java | 40 + .../metrics/QuarkusMetricsConfiguration.java | 22 +- .../QuarkusValueExpressionResolver.java | 38 + .../metrics/RealmIdTagContributor.java | 55 ++ .../monitor/PolarisMetricRegistry.java | 219 ----- .../QuarkusPersistenceConfiguration.java | 33 + .../persistence/cache/EntityCacheFactory.java | 47 - .../QuarkusRateLimiterConfiguration.java | 47 + .../storage/QuarkusStorageConfiguration.java | 47 + .../task/QuarkusTaskExecutorImpl.java | 79 ++ .../task/QuarkusTaskHandlerConfiguration.java | 37 + .../StreamReadConstraintsExceptionMapper.java | 42 - .../tracing/HeadersMapAccessor.java | 57 -- .../QuarkusTelemetryConfiguration.java} | 20 +- .../tracing/QuarkusTracingFilter.java | 54 ++ .../dropwizard/tracing/TracingFilter.java | 102 --- .../resources/META-INF/hk2-locator/default | 93 -- ...o.smallrye.config.ConfigSourceInterceptor} | 2 +- .../src/main/resources/application.properties | 160 ++++ .../src/main/resources/log4j.properties | 24 - .../PolarisApplicationConfigurationTest.java | 79 -- .../PolarisApplicationIntegrationTest.java | 348 ++++--- .../service/dropwizard/TestServices.java | 41 +- .../TimedApplicationEventListenerTest.java | 250 ++--- .../admin/PolarisAdminServiceAuthzTest.java | 4 + .../admin/PolarisAuthzTestBase.java | 111 ++- .../admin/PolarisRealmEntityCacheTest.java | 278 ------ .../PolarisServiceImplIntegrationTest.java | 593 +++++------- .../dropwizard/auth/JWTRSAKeyPairTest.java | 85 +- .../auth/JWTSymmetricKeyGeneratorTest.java | 9 +- .../service/dropwizard/auth/TokenUtils.java | 12 +- .../catalog/BasePolarisCatalogTest.java | 59 +- .../catalog/BasePolarisCatalogViewTest.java | 84 +- ...PolarisCatalogHandlerWrapperAuthzTest.java | 29 +- .../PolarisRestCatalogIntegrationTest.java | 317 ++++--- ...arisRestCatalogViewAwsIntegrationTest.java | 2 + ...isRestCatalogViewAzureIntegrationTest.java | 2 + ...risRestCatalogViewFileIntegrationTest.java | 2 + ...arisRestCatalogViewGcpIntegrationTest.java | 2 + ...PolarisRestCatalogViewIntegrationTest.java | 91 +- .../catalog/PolarisSparkIntegrationTest.java | 133 ++- .../service/dropwizard/catalog/TestUtil.java | 89 +- .../catalog/io/TestFileIOFactory.java | 8 +- .../ratelimiter/MockTokenBucketFactory.java | 22 +- .../ratelimiter/RateLimiterFilterTest.java | 207 +++-- .../RealmTokenBucketRateLimiterTest.java | 66 -- .../dropwizard/ratelimiter/TestUtil.java | 22 +- ...eLimiterTest.java => TokenBucketTest.java} | 5 +- .../ManifestFileCleanupTaskHandlerTest.java | 15 +- .../task/TableCleanupTaskHandlerTest.java | 15 +- .../test/DefaultTestEnvironmentResolver.java | 35 + .../DropwizardTestEnvironmentResolver.java | 85 -- .../test/PolarisConnectionExtension.java | 215 ----- .../test/PolarisIntegrationTestFixture.java | 240 +++++ .../test/PolarisIntegrationTestHelper.java | 44 + .../test/SnowmanCredentialsExtension.java | 227 ----- .../dropwizard/test/TestEnvironment.java | 51 +- .../test/TestEnvironmentExtension.java | 5 +- .../dropwizard/test/TestMetricsUtil.java | 63 +- .../polaris-server-integrationtest.yml | 164 ---- .../persistence/eclipselink/build.gradle.kts | 29 +- .../eclipselink/EclipseLinkConfiguration.java | 29 + ...pseLinkPolarisMetaStoreManagerFactory.java | 17 +- gradle/libs.versions.toml | 18 +- gradle/projects.main.properties | 2 +- polaris-core/build.gradle.kts | 3 + .../polaris/core/PolarisCallContext.java | 8 + .../polaris/core/context/CallContext.java | 5 +- .../LocalPolarisMetaStoreManagerFactory.java | 12 + .../persistence/MetaStoreManagerFactory.java | 8 +- .../core/persistence/cache/EntityCache.java | 4 - server-templates/api.mustache | 3 +- service/common/build.gradle.kts | 23 +- .../service/admin/PolarisServiceImpl.java | 18 +- .../auth/AuthenticatorConfiguration.java | 64 ++ .../auth/BasePolarisAuthenticator.java | 22 +- .../service/auth/DefaultOAuth2ApiService.java | 21 +- .../auth/DefaultPolarisAuthenticator.java | 22 +- .../polaris/service/auth/JWTBroker.java | 10 +- .../polaris/service/auth/JWTRSAKeyPair.java | 15 +- .../service/auth/JWTRSAKeyPairFactory.java | 48 +- .../service/auth/JWTSymmetricKeyFactory.java | 56 +- .../service/auth/LocalRSAKeyProvider.java | 22 +- .../service/auth/NoneTokenBrokerFactory.java | 70 ++ .../apache/polaris/service/auth/PemUtils.java | 76 +- ...InlineBearerTokenPolarisAuthenticator.java | 23 +- .../service/auth/TestOAuth2ApiService.java | 8 +- .../polaris/service/auth/TokenBroker.java | 13 +- .../catalog/IcebergCatalogAdapter.java | 12 +- .../catalog/io/DefaultFileIOFactory.java | 2 + .../service/catalog/io/FileIOFactory.java | 5 - .../io/WasbTranslatingFileIOFactory.java | 7 +- .../config/DefaultConfigurationStore.java | 40 +- .../service/config/FeaturesConfiguration.java | 89 ++ .../config/RealmEntityManagerFactory.java | 19 +- .../config/TaskHandlerConfiguration.java | 52 -- .../service/context/ContextConfiguration.java | 31 + .../context/DefaultCallContextResolver.java | 17 +- .../context/DefaultRealmContextResolver.java | 31 +- .../PolarisCallContextCatalogFactory.java | 2 + .../service/context/RealmContextResolver.java | 4 - ...nMemoryPolarisMetaStoreManagerFactory.java | 24 +- .../DefaultTokenBucketFactory.java | 27 +- .../service/ratelimiter/NoOpRateLimiter.java | 2 + .../ratelimiter/RateLimiterConfiguration.java | 18 +- .../ratelimiter/RateLimiterFilter.java | 2 +- .../RealmTokenBucketRateLimiter.java | 26 +- ...PolarisStorageIntegrationProviderImpl.java | 10 +- .../service/storage/StorageConfiguration.java | 97 ++ .../service/task/TaskExecutorImpl.java | 128 +-- .../service/task/TaskFileIOSupplier.java | 4 + .../task/TaskHandlerConfiguration.java | 11 +- 141 files changed, 4902 insertions(+), 5481 deletions(-) create mode 100644 dropwizard/service/README-quarkus.md create mode 100644 dropwizard/service/src/main/docker/Dockerfile.jvm delete mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/BootstrapRealmsCommand.java delete mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisApplication.java delete mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PurgeRealmsCommand.java delete mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/TimedApplicationEventListener.java create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/auth/QuarkusAuthenticatorConfiguration.java create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/auth/QuarkusOAuthFilter.java rename dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/{throttling/RequestThrottlingErrorResponse.java => catalog/io/QuarkusCatalogIOConfiguration.java} (65%) delete mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/CorsConfiguration.java delete mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/PolarisApplicationConfig.java create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/QuarkusFeaturesConfiguration.java create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/QuarkusJacksonConfig.java create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/QuarkusProducers.java create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/context/QuarkusContextConfiguration.java delete mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/context/RealmScopeContext.java delete mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/exception/JerseyViolationExceptionMapper.java delete mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/logging/PolarisJsonLayoutFactory.java rename dropwizard/service/src/{test/java/org/apache/polaris/service/dropwizard/test/PolarisRealm.java => main/java/org/apache/polaris/service/dropwizard/logging/QuarkusLoggingConfiguration.java} (63%) create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/logging/QuarkusLoggingMDCFilter.java create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/QuarkusMeterFilterProducer.java rename polaris-core/src/main/java/org/apache/polaris/core/context/RealmScoped.java => dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/QuarkusMetricsConfiguration.java (67%) create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/QuarkusValueExpressionResolver.java create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/RealmIdTagContributor.java delete mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/monitor/PolarisMetricRegistry.java create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/persistence/QuarkusPersistenceConfiguration.java delete mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/persistence/cache/EntityCacheFactory.java create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/ratelimiter/QuarkusRateLimiterConfiguration.java create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/storage/QuarkusStorageConfiguration.java create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/task/QuarkusTaskExecutorImpl.java create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/task/QuarkusTaskHandlerConfiguration.java delete mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/throttling/StreamReadConstraintsExceptionMapper.java delete mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/HeadersMapAccessor.java rename dropwizard/service/src/{test/resources/META-INF/hk2-locator/default => main/java/org/apache/polaris/service/dropwizard/tracing/QuarkusTelemetryConfiguration.java} (66%) create mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/QuarkusTracingFilter.java delete mode 100644 dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/TracingFilter.java delete mode 100644 dropwizard/service/src/main/resources/META-INF/hk2-locator/default rename dropwizard/service/src/main/resources/META-INF/services/{io.dropwizard.logging.common.layout.DiscoverableLayoutFactory => io.smallrye.config.ConfigSourceInterceptor} (91%) create mode 100644 dropwizard/service/src/main/resources/application.properties delete mode 100644 dropwizard/service/src/main/resources/log4j.properties delete mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationConfigurationTest.java delete mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisRealmEntityCacheTest.java delete mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/RealmTokenBucketRateLimiterTest.java rename dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/{TokenBucketRateLimiterTest.java => TokenBucketTest.java} (96%) create mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/DefaultTestEnvironmentResolver.java delete mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/DropwizardTestEnvironmentResolver.java delete mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisConnectionExtension.java create mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisIntegrationTestFixture.java create mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisIntegrationTestHelper.java delete mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/SnowmanCredentialsExtension.java delete mode 100644 dropwizard/service/src/test/resources/polaris-server-integrationtest.yml create mode 100644 extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkConfiguration.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/auth/AuthenticatorConfiguration.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/auth/NoneTokenBrokerFactory.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/config/FeaturesConfiguration.java delete mode 100644 service/common/src/main/java/org/apache/polaris/service/config/TaskHandlerConfiguration.java create mode 100644 service/common/src/main/java/org/apache/polaris/service/context/ContextConfiguration.java rename dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisHealthCheck.java => service/common/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterConfiguration.java (73%) create mode 100644 service/common/src/main/java/org/apache/polaris/service/storage/StorageConfiguration.java rename extension/persistence/eclipselink/src/main/resources/META-INF/hk2-locator/default => service/common/src/main/java/org/apache/polaris/service/task/TaskHandlerConfiguration.java (76%) diff --git a/Dockerfile b/Dockerfile index 9477cf07f..8fd17ef6a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,16 +32,36 @@ WORKDIR /app RUN rm -rf build # Build the rest catalog -RUN ./gradlew --no-daemon --info ${ECLIPSELINK_DEPS+"-PeclipseLinkDeps=$ECLIPSELINK_DEPS"} -PeclipseLink=$ECLIPSELINK clean prepareDockerDist +RUN ./gradlew --no-daemon --info ${ECLIPSELINK_DEPS+"-PeclipseLinkDeps=$ECLIPSELINK_DEPS"} -PeclipseLink=$ECLIPSELINK clean :polaris-quarkus-service:build -x test FROM registry.access.redhat.com/ubi9/openjdk-21-runtime:1.20-2.1729089285 -WORKDIR /app -COPY --from=build /app/dropwizard/service/build/docker-dist/bin /app/bin -COPY --from=build /app/dropwizard/service/build/docker-dist/lib /app/lib -COPY --from=build /app/polaris-server.yml /app + +LABEL org.opencontainers.image.source=https://github.com/apache/polaris +LABEL org.opencontainers.image.description="Apache Polaris (incubating)" +LABEL org.opencontainers.image.licenses=Apache-2.0 + +ENV LANGUAGE='en_US:en' + +USER root +RUN groupadd --gid 10001 polaris \ + && useradd --uid 10000 --gid polaris polaris \ + && chown -R polaris:polaris /opt/jboss/container \ + && chown -R polaris:polaris /deployments + +USER polaris +WORKDIR /home/polaris +ENV USER=polaris +ENV UID=10000 +ENV HOME=/home/polaris + +# We make four distinct layers so if there are application changes the library layers can be re-used +COPY --from=build --chown=polaris:polaris /app/dropwizard/service/build/quarkus-app/lib/ /deployments/lib/ +COPY --from=build --chown=polaris:polaris /app/dropwizard/service/build/quarkus-app/*.jar /deployments/ +COPY --from=build --chown=polaris:polaris /app/dropwizard/service/build/quarkus-app/app/ /deployments/app/ +COPY --from=build --chown=polaris:polaris /app/dropwizard/service/build/quarkus-app/quarkus/ /deployments/quarkus/ EXPOSE 8181 +EXPOSE 8182 -# Run the resulting java binary -ENTRYPOINT ["/app/bin/polaris-service"] -CMD ["server", "polaris-server.yml"] +ENV AB_JOLOKIA_OFF="" +ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" diff --git a/LICENSE-BINARY-DIST b/LICENSE-BINARY-DIST index 6ab97ac6d..9fb503538 100644 --- a/LICENSE-BINARY-DIST +++ b/LICENSE-BINARY-DIST @@ -208,6 +208,8 @@ Apache Polaris distributions contain some or all of the following dependencies, licensed under Apache License: +com.aayushatharva.brotli4j:brotli4j +com.aayushatharva.brotli4j:service com.fasterxml.jackson.core:jackson-annotations com.fasterxml.jackson.core:jackson-core com.fasterxml.jackson.core:jackson-databind @@ -229,6 +231,7 @@ com.google.android:annotations com.google.api-client:google-api-client com.google.api.grpc:gapic-google-cloud-storage-v2 com.google.api.grpc:grpc-google-cloud-storage-v2 +com.google.api.grpc:proto-google-cloud-monitoring-v3 com.google.api.grpc:proto-google-cloud-storage-v2 com.google.api.grpc:proto-google-common-protos com.google.api.grpc:proto-google-iam-v1 @@ -236,20 +239,19 @@ com.google.api:api-common com.google.api:gax com.google.api:gax-grpc com.google.api:gax-httpjson -com.google.api.grpc:proto-google-cloud-monitoring-v3 com.google.apis:google-api-services-storage com.google.auth:google-auth-library-credentials com.google.auth:google-auth-library-oauth2-http com.google.auto.value:auto-value-annotations -com.google.cloud:google-cloud-core -com.google.cloud:google-cloud-core-grpc -com.google.cloud:google-cloud-core-http -com.google.cloud:google-cloud-storage -com.google.cloud:google-cloud-monitoring com.google.cloud.opentelemetry:detector-resources-support com.google.cloud.opentelemetry:exporter-metrics com.google.cloud.opentelemetry:shared-resourcemapping +com.google.cloud:google-cloud-core +com.google.cloud:google-cloud-core-grpc +com.google.cloud:google-cloud-core-http com.google.cloud:google-cloud-monitorin +com.google.cloud:google-cloud-monitoring +com.google.cloud:google-cloud-storage com.google.code.findbugs:jsr305 com.google.code.gson:gson com.google.errorprone:error_prone_annotations @@ -279,33 +281,6 @@ commons-logging:commons-logging commons-net:commons-net dev.failsafe:failsafe io.airlift:aircompressor -io.dropwizard.logback:logback-throttling-appender -io.dropwizard.metrics:metrics-annotation -io.dropwizard.metrics:metrics-caffeine -io.dropwizard.metrics:metrics-core -io.dropwizard.metrics:metrics-healthchecks -io.dropwizard.metrics:metrics-jakarta-servlets -io.dropwizard.metrics:metrics-jersey3 -io.dropwizard.metrics:metrics-jetty11 -io.dropwizard.metrics:metrics-jmx -io.dropwizard.metrics:metrics-json -io.dropwizard.metrics:metrics-jvm -io.dropwizard.metrics:metrics-logback -io.dropwizard:dropwizard-auth -io.dropwizard:dropwizard-configuration -io.dropwizard:dropwizard-core -io.dropwizard:dropwizard-health -io.dropwizard:dropwizard-jackson -io.dropwizard:dropwizard-jersey -io.dropwizard:dropwizard-jetty -io.dropwizard:dropwizard-json-logging -io.dropwizard:dropwizard-lifecycle -io.dropwizard:dropwizard-logging -io.dropwizard:dropwizard-metrics -io.dropwizard:dropwizard-request-logging -io.dropwizard:dropwizard-servlets -io.dropwizard:dropwizard-util -io.dropwizard:dropwizard-validation io.grpc:grpc-alts io.grpc:grpc-api io.grpc:grpc-auth @@ -314,6 +289,7 @@ io.grpc:grpc-core io.grpc:grpc-googleapis io.grpc:grpc-grpclb io.grpc:grpc-inprocess +io.grpc:grpc-netty io.grpc:grpc-netty-shaded io.grpc:grpc-opentelemetry io.grpc:grpc-protobuf @@ -327,9 +303,11 @@ io.micrometer:micrometer-commons io.micrometer:micrometer-core io.micrometer:micrometer-observation io.micrometer:micrometer-registry-prometheus +io.micrometer:micrometer-registry-prometheus-simpleclient io.netty:netty-buffer io.netty:netty-codec io.netty:netty-codec-dns +io.netty:netty-codec-haproxy io.netty:netty-codec-http io.netty:netty-codec-http2 io.netty:netty-codec-socks @@ -351,18 +329,30 @@ io.netty:netty-transport-native-unix-common io.opencensus:opencensus-api io.opencensus:opencensus-contrib-http-util io.opencensus:opencensus-proto +io.opentelemetry.contrib:opentelemetry-gcp-resources +io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations +io.opentelemetry.instrumentation:opentelemetry-instrumentation-annotations-support +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-incubator +io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java17 +io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java8 io.opentelemetry.semconv:opentelemetry-semconv +io.opentelemetry.semconv:opentelemetry-semconv-incubating io.opentelemetry:opentelemetry-api io.opentelemetry:opentelemetry-api-incubator io.opentelemetry:opentelemetry-context +io.opentelemetry:opentelemetry-exporter-common io.opentelemetry:opentelemetry-exporter-logging +io.opentelemetry:opentelemetry-exporter-otlp +io.opentelemetry:opentelemetry-exporter-otlp-common +io.opentelemetry:opentelemetry-exporter-sender-okhttp io.opentelemetry:opentelemetry-sdk io.opentelemetry:opentelemetry-sdk-common +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi io.opentelemetry:opentelemetry-sdk-logs io.opentelemetry:opentelemetry-sdk-metrics io.opentelemetry:opentelemetry-sdk-trace -io.opentelemetry.contrib:opentelemetry-gcp-resources io.perfmark:perfmark-api io.projectreactor.netty:reactor-netty-core io.projectreactor.netty:reactor-netty-http @@ -375,18 +365,103 @@ io.prometheus:prometheus-metrics-exposition-formats io.prometheus:prometheus-metrics-model io.prometheus:prometheus-metrics-shaded-protobuf io.prometheus:prometheus-metrics-tracer-common +io.prometheus:simpleclient +io.prometheus:simpleclient_common +io.prometheus:simpleclient_tracer_common +io.prometheus:simpleclient_tracer_otel +io.prometheus:simpleclient_tracer_otel_agent +io.quarkus.arc:arc +io.quarkus.resteasy.reactive:resteasy-reactive +io.quarkus.resteasy.reactive:resteasy-reactive-common +io.quarkus.resteasy.reactive:resteasy-reactive-common-types +io.quarkus.resteasy.reactive:resteasy-reactive-jackson +io.quarkus.resteasy.reactive:resteasy-reactive-vertx +io.quarkus.security:quarkus-security +io.quarkus.vertx.utils:quarkus-vertx-utils +io.quarkus:quarkus-arc +io.quarkus:quarkus-bootstrap-runner +io.quarkus:quarkus-classloader-commons +io.quarkus:quarkus-container-image +io.quarkus:quarkus-container-image-docker +io.quarkus:quarkus-container-image-docker-common +io.quarkus:quarkus-core +io.quarkus:quarkus-credentials +io.quarkus:quarkus-development-mode-spi +io.quarkus:quarkus-fs-util +io.quarkus:quarkus-grpc-common +io.quarkus:quarkus-hibernate-validator +io.quarkus:quarkus-ide-launcher +io.quarkus:quarkus-jackson +io.quarkus:quarkus-jsonp +io.quarkus:quarkus-logging-json +io.quarkus:quarkus-micrometer +io.quarkus:quarkus-micrometer-registry-prometheus +io.quarkus:quarkus-mutiny +io.quarkus:quarkus-netty +io.quarkus:quarkus-opentelemetry +io.quarkus:quarkus-reactive-routes +io.quarkus:quarkus-rest +io.quarkus:quarkus-rest-common +io.quarkus:quarkus-rest-jackson +io.quarkus:quarkus-rest-jackson-common +io.quarkus:quarkus-security-runtime-spi +io.quarkus:quarkus-smallrye-context-propagation +io.quarkus:quarkus-smallrye-health +io.quarkus:quarkus-tls-registry +io.quarkus:quarkus-vertx +io.quarkus:quarkus-vertx-http +io.quarkus:quarkus-vertx-latebound-mdc-provider +io.quarkus:quarkus-virtual-threads io.smallrye.common:smallrye-common-annotation io.smallrye.common:smallrye-common-classloader io.smallrye.common:smallrye-common-constraint +io.smallrye.common:smallrye-common-cpu io.smallrye.common:smallrye-common-expression io.smallrye.common:smallrye-common-function +io.smallrye.common:smallrye-common-io +io.smallrye.common:smallrye-common-net +io.smallrye.common:smallrye-common-os +io.smallrye.common:smallrye-common-ref +io.smallrye.common:smallrye-common-vertx-context io.smallrye.config:smallrye-config io.smallrye.config:smallrye-config-common io.smallrye.config:smallrye-config-core +io.smallrye.config:smallrye-config-validator +io.smallrye.reactive:mutiny +io.smallrye.reactive:mutiny-smallrye-context-propagation +io.smallrye.reactive:mutiny-zero-flow-adapters +io.smallrye.reactive:smallrye-mutiny-vertx-auth-common +io.smallrye.reactive:smallrye-mutiny-vertx-bridge-common +io.smallrye.reactive:smallrye-mutiny-vertx-core +io.smallrye.reactive:smallrye-mutiny-vertx-runtime +io.smallrye.reactive:smallrye-mutiny-vertx-uri-template +io.smallrye.reactive:smallrye-mutiny-vertx-web +io.smallrye.reactive:smallrye-mutiny-vertx-web-common +io.smallrye.reactive:vertx-mutiny-generator +io.smallrye:smallrye-context-propagation +io.smallrye:smallrye-context-propagation-api +io.smallrye:smallrye-context-propagation-storage +io.smallrye:smallrye-fault-tolerance-vertx +io.smallrye:smallrye-health +io.smallrye:smallrye-health-api +io.smallrye:smallrye-health-provided-checks io.swagger:swagger-annotations io.swagger:swagger-core io.swagger:swagger-jaxrs io.swagger:swagger-models +io.vertx:vertx-auth-common +io.vertx:vertx-bridge-common +io.vertx:vertx-codegen +io.vertx:vertx-core +io.vertx:vertx-grpc +io.vertx:vertx-grpc-client +io.vertx:vertx-grpc-common +io.vertx:vertx-grpc-server +io.vertx:vertx-uri-template +io.vertx:vertx-web +io.vertx:vertx-web-common +jakarta.enterprise:jakarta.enterprise.cdi-api +jakarta.enterprise:jakarta.enterprise.lang-model jakarta.inject:jakarta.inject-api jakarta.validation:jakarta.validation-api javax.inject:javax.inject @@ -405,13 +480,14 @@ org.apache.curator:curator-client org.apache.curator:curator-framework org.apache.curator:curator-recipes org.apache.hadoop.thirdparty:hadoop-shaded-guava +org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_21 org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_7 org.apache.hadoop:hadoop-annotations org.apache.hadoop:hadoop-auth org.apache.hadoop:hadoop-client-api +org.apache.hadoop:hadoop-client-runtime org.apache.hadoop:hadoop-common org.apache.hadoop:hadoop-hdfs-client -org.apache.hadoop.thirdparty:hadoop-shaded-protobuf_3_21 org.apache.httpcomponents.client5:httpclient5 org.apache.httpcomponents.core5:httpcore5 org.apache.httpcomponents.core5:httpcore5-h2 @@ -452,9 +528,12 @@ org.eclipse.jetty:jetty-server org.eclipse.jetty:jetty-servlet org.eclipse.jetty:jetty-servlets org.eclipse.jetty:jetty-util +org.eclipse.jetty:jetty-util-ajax org.eclipse.jetty:jetty-webapp org.eclipse.jetty:jetty-xml org.eclipse.microprofile.config:microprofile-config-api +org.eclipse.microprofile.context-propagation:microprofile-context-propagation-api +org.eclipse.microprofile.health:microprofile-health-api org.glassfish.jersey.containers:jersey-container-servlet org.glassfish.jersey.containers:jersey-container-servlet-core org.glassfish.jersey.core:jersey-client @@ -465,7 +544,13 @@ org.glassfish.jersey.ext:jersey-metainf-services org.glassfish.jersey.inject:jersey-hk2 org.hibernate.validator:hibernate-validator org.javassist:javassist +org.jboss.logging:commons-logging-jboss-logging org.jboss.logging:jboss-logging +org.jboss.logging:jboss-logging-annotations +org.jboss.logmanager:jboss-logmanager +org.jboss.slf4j:slf4j-jboss-logmanager +org.jboss.threads:jboss-threads +org.jctools:jctools-core org.jetbrains.kotlin:kotlin-stdlib org.jetbrains.kotlin:kotlin-stdlib-common org.ow2.asm:asm @@ -473,6 +558,7 @@ org.reflections:reflections org.roaringbitmap:RoaringBitmap org.slf4j:jcl-over-slf4j org.slf4j:log4j-over-slf4j +org.wildfly.common:wildfly-common org.xerial.snappy:snappy-java org.yaml:snakeyaml software.amazon.awssdk:annotations @@ -6919,4 +7005,703 @@ org.threeten:threetenbp PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +--- +io.github.crac:org-crac +jakarta.interceptor:jakarta.interceptor-api +jakarta.json:jakarta.json-api +jakarta.transaction:jakarta.transaction-api +org.eclipse.parsson:jakarta.json +org.eclipse.parsson:parsson +org.glassfish.expressly:expressly + + +# Eclipse Public License - v 2.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + + 1. DEFINITIONS + + "Contribution" means: + + a) in the case of the initial Contributor, the initial content + Distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + i) changes to the Program, and + ii) additions to the Program; + where such changes and/or additions to the Program originate from + and are Distributed by that particular Contributor. A Contribution + "originates" from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include changes or additions to the Program that + are not Modified Works. + + "Contributor" means any person or entity that Distributes the Program. + + "Licensed Patents" mean patent claims licensable by a Contributor which + are necessarily infringed by the use or sale of its Contribution alone + or when combined with the Program. + + "Program" means the Contributions Distributed in accordance with this + Agreement. + + "Recipient" means anyone who receives the Program under this Agreement + or any Secondary License (as applicable), including Contributors. + + "Derivative Works" shall mean any work, whether in Source Code or other + form, that is based on (or derived from) the Program and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. + + "Modified Works" shall mean any work in Source Code or other form that + results from an addition to, deletion from, or modification of the + contents of the Program, including, for purposes of clarity any new file + in Source Code form that contains any contents of the Program. Modified + Works shall not include works that contain only declarations, + interfaces, types, classes, structures, or files of the Program solely + in each case in order to link to, bind by name, or subclass the Program + or Modified Works thereof. + + "Distribute" means the acts of a) distributing or b) making available + in any manner that enables the transfer of a copy. + + "Source Code" means the form of a Program preferred for making + modifications, including but not limited to software source code, + documentation source, and configuration files. + + "Secondary License" means either the GNU General Public License, + Version 2.0, or any later versions of that license, including any + exceptions or additional permissions as identified by the initial + Contributor. + + 2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free copyright + license to reproduce, prepare Derivative Works of, publicly display, + publicly perform, Distribute and sublicense the Contribution of such + Contributor, if any, and such Derivative Works. + + b) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free patent + license under Licensed Patents to make, use, sell, offer to sell, + import and otherwise transfer the Contribution of such Contributor, + if any, in Source Code or other form. This patent license shall + apply to the combination of the Contribution and the Program if, at + the time the Contribution is added by the Contributor, such addition + of the Contribution causes such combination to be covered by the + Licensed Patents. The patent license shall not apply to any other + combinations which include the Contribution. No hardware per se is + licensed hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the + rights and licenses granted hereunder, each Recipient hereby + assumes sole responsibility to secure any other intellectual + property rights needed, if any. For example, if a third party + patent license is required to allow Recipient to Distribute the + Program, it is Recipient's responsibility to acquire that license + before distributing the Program. + + d) Each Contributor represents that to its knowledge it has + sufficient copyright rights in its Contribution, if any, to grant + the copyright license set forth in this Agreement. + + e) Notwithstanding the terms of any Secondary License, no + Contributor makes additional grants to any Recipient (other than + those set forth in this Agreement) as a result of such Recipient's + receipt of the Program under the terms of a Secondary License + (if permitted under the terms of Section 3). + + 3. REQUIREMENTS + + 3.1 If a Contributor Distributes the Program in any form, then: + + a) the Program must also be made available as Source Code, in + accordance with section 3.2, and the Contributor must accompany + the Program with a statement that the Source Code for the Program + is available under this Agreement, and informs Recipients how to + obtain it in a reasonable manner on or through a medium customarily + used for software exchange; and + + b) the Contributor may Distribute the Program under a license + different than this Agreement, provided that such license: + i) effectively disclaims on behalf of all other Contributors all + warranties and conditions, express and implied, including + warranties or conditions of title and non-infringement, and + implied warranties or conditions of merchantability and fitness + for a particular purpose; + + ii) effectively excludes on behalf of all other Contributors all + liability for damages, including direct, indirect, special, + incidental and consequential damages, such as lost profits; + + iii) does not attempt to limit or alter the recipients' rights + in the Source Code under section 3.2; and + + iv) requires any subsequent distribution of the Program by any + party to be under a license that satisfies the requirements + of this section 3. + + 3.2 When the Program is Distributed as Source Code: + + a) it must be made available under this Agreement, or if the + Program (i) is combined with other material in a separate file or + files made available under a Secondary License, and (ii) the initial + Contributor attached to the Source Code the notice described in + Exhibit A of this Agreement, then the Program may be made available + under the terms of such Secondary Licenses, and + + b) a copy of this Agreement must be included with each copy of + the Program. + + 3.3 Contributors may not remove or alter any copyright, patent, + trademark, attribution notices, disclaimers of warranty, or limitations + of liability ("notices") contained within the Program from any copy of + the Program which they Distribute, provided that Contributors may add + their own appropriate notices. + + 4. COMMERCIAL DISTRIBUTION + + Commercial distributors of software may accept certain responsibilities + with respect to end users, business partners and the like. While this + license is intended to facilitate the commercial use of the Program, + the Contributor who includes the Program in a commercial product + offering should do so in a manner which does not create potential + liability for other Contributors. Therefore, if a Contributor includes + the Program in a commercial product offering, such Contributor + ("Commercial Contributor") hereby agrees to defend and indemnify every + other Contributor ("Indemnified Contributor") against any losses, + damages and costs (collectively "Losses") arising from claims, lawsuits + and other legal actions brought by a third party against the Indemnified + Contributor to the extent caused by the acts or omissions of such + Commercial Contributor in connection with its distribution of the Program + in a commercial product offering. The obligations in this section do not + apply to any claims or Losses relating to any actual or alleged + intellectual property infringement. In order to qualify, an Indemnified + Contributor must: a) promptly notify the Commercial Contributor in + writing of such claim, and b) allow the Commercial Contributor to control, + and cooperate with the Commercial Contributor in, the defense and any + related settlement negotiations. The Indemnified Contributor may + participate in any such claim at its own expense. + + For example, a Contributor might include the Program in a commercial + product offering, Product X. That Contributor is then a Commercial + Contributor. If that Commercial Contributor then makes performance + claims, or offers warranties related to Product X, those performance + claims and warranties are such Commercial Contributor's responsibility + alone. Under this section, the Commercial Contributor would have to + defend claims against the other Contributors related to those performance + claims and warranties, and if a court requires any other Contributor to + pay any damages as a result, the Commercial Contributor must pay + those damages. + + 5. NO WARRANTY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT + PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS" + BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR + IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF + TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR + PURPOSE. Each Recipient is solely responsible for determining the + appropriateness of using and distributing the Program and assumes all + risks associated with its exercise of rights under this Agreement, + including but not limited to the risks and costs of program errors, + compliance with applicable laws, damage to or loss of data, programs + or equipment, and unavailability or interruption of operations. + + 6. DISCLAIMER OF LIABILITY + + EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT + PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS + SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST + PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE + EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGES. + + 7. GENERAL + + If any provision of this Agreement is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this Agreement, and without further + action by the parties hereto, such provision shall be reformed to the + minimum extent necessary to make such provision valid and enforceable. + + If Recipient institutes patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that the + Program itself (excluding combinations of the Program with other software + or hardware) infringes such Recipient's patent(s), then such Recipient's + rights granted under Section 2(b) shall terminate as of the date such + litigation is filed. + + All Recipient's rights under this Agreement shall terminate if it + fails to comply with any of the material terms or conditions of this + Agreement and does not cure such failure in a reasonable period of + time after becoming aware of such noncompliance. If all Recipient's + rights under this Agreement terminate, Recipient agrees to cease use + and distribution of the Program as soon as reasonably practicable. + However, Recipient's obligations under this Agreement and any licenses + granted by Recipient relating to the Program shall continue and survive. + + Everyone is permitted to copy and distribute copies of this Agreement, + but in order to avoid inconsistency the Agreement is copyrighted and + may only be modified in the following manner. The Agreement Steward + reserves the right to publish new versions (including revisions) of + this Agreement from time to time. No one other than the Agreement + Steward has the right to modify this Agreement. The Eclipse Foundation + is the initial Agreement Steward. The Eclipse Foundation may assign the + responsibility to serve as the Agreement Steward to a suitable separate + entity. Each new version of the Agreement will be given a distinguishing + version number. The Program (including Contributions) may always be + Distributed subject to the version of the Agreement under which it was + received. In addition, after a new version of the Agreement is published, + Contributor may elect to Distribute the Program (including its + Contributions) under the new version. + + Except as expressly stated in Sections 2(a) and 2(b) above, Recipient + receives no rights or licenses to the intellectual property of any + Contributor under this Agreement, whether expressly, by implication, + estoppel or otherwise. All rights in the Program not expressly granted + under this Agreement are reserved. Nothing in this Agreement is intended + to be enforceable by any entity that is not a Contributor or Recipient. + No third-party beneficiary rights are created under this Agreement. + + Exhibit A - Form of Secondary Licenses Notice + + "This Source Code may also be made available under the following + Secondary Licenses when the conditions for such availability set forth + in the Eclipse Public License, v. 2.0 are satisfied: {name license(s), + version(s), and exceptions or additional permissions here}." + + Simply including a copy of this Agreement, including this Exhibit A + is not sufficient to license the Source Code under Secondary Licenses. + + If it is not possible or desirable to put the notice in a particular + file, then You may include the notice in a location (such as a LICENSE + file in a relevant directory) where a recipient would be likely to + look for such a notice. + + You may add additional accurate notices of copyright ownership. + +--- + +## The GNU General Public License (GPL) Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor + Boston, MA 02110-1335 + USA + + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your freedom to + share and change it. By contrast, the GNU General Public License is + intended to guarantee your freedom to share and change free software--to + make sure the software is free for all its users. This General Public + License applies to most of the Free Software Foundation's software and + to any other program whose authors commit to using it. (Some other Free + Software Foundation software is covered by the GNU Library General + Public License instead.) You can apply it to your programs, too. + + When we speak of free software, we are referring to freedom, not price. + Our General Public Licenses are designed to make sure that you have the + freedom to distribute copies of free software (and charge for this + service if you wish), that you receive source code or can get it if you + want it, that you can change the software or use pieces of it in new + free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid anyone + to deny you these rights or to ask you to surrender the rights. These + restrictions translate to certain responsibilities for you if you + distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether gratis + or for a fee, you must give the recipients all the rights that you have. + You must make sure that they, too, receive or can get the source code. + And you must show them these terms so they know their rights. + + We protect your rights with two steps: (1) copyright the software, and + (2) offer you this license which gives you legal permission to copy, + distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain + that everyone understands that there is no warranty for this free + software. If the software is modified by someone else and passed on, we + want its recipients to know that what they have is not the original, so + that any problems introduced by others will not reflect on the original + authors' reputations. + + Finally, any free program is threatened constantly by software patents. + We wish to avoid the danger that redistributors of a free program will + individually obtain patent licenses, in effect making the program + proprietary. To prevent this, we have made it clear that any patent must + be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and + modification follow. + + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains a + notice placed by the copyright holder saying it may be distributed under + the terms of this General Public License. The "Program", below, refers + to any such program or work, and a "work based on the Program" means + either the Program or any derivative work under copyright law: that is + to say, a work containing the Program or a portion of it, either + verbatim or with modifications and/or translated into another language. + (Hereinafter, translation is included without limitation in the term + "modification".) Each licensee is addressed as "you". + + Activities other than copying, distribution and modification are not + covered by this License; they are outside its scope. The act of running + the Program is not restricted, and the output from the Program is + covered only if its contents constitute a work based on the Program + (independent of having been made by running the Program). Whether that + is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's source + code as you receive it, in any medium, provided that you conspicuously + and appropriately publish on each copy an appropriate copyright notice + and disclaimer of warranty; keep intact all the notices that refer to + this License and to the absence of any warranty; and give any other + recipients of the Program a copy of this License along with the Program. + + You may charge a fee for the physical act of transferring a copy, and + you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion of + it, thus forming a work based on the Program, and copy and distribute + such modifications or work under the terms of Section 1 above, provided + that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any part + thereof, to be licensed as a whole at no charge to all third parties + under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a notice + that there is no warranty (or else, saying that you provide a + warranty) and that users may redistribute the program under these + conditions, and telling the user how to view a copy of this License. + (Exception: if the Program itself is interactive but does not + normally print such an announcement, your work based on the Program + is not required to print an announcement.) + + These requirements apply to the modified work as a whole. If + identifiable sections of that work are not derived from the Program, and + can be reasonably considered independent and separate works in + themselves, then this License, and its terms, do not apply to those + sections when you distribute them as separate works. But when you + distribute the same sections as part of a whole which is a work based on + the Program, the distribution of the whole must be on the terms of this + License, whose permissions for other licensees extend to the entire + whole, and thus to each and every part regardless of who wrote it. + + Thus, it is not the intent of this section to claim rights or contest + your rights to work written entirely by you; rather, the intent is to + exercise the right to control the distribution of derivative or + collective works based on the Program. + + In addition, mere aggregation of another work not based on the Program + with the Program (or with a work based on the Program) on a volume of a + storage or distribution medium does not bring the other work under the + scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, + under Section 2) in object code or executable form under the terms of + Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections 1 + and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your cost + of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer to + distribute corresponding source code. (This alternative is allowed + only for noncommercial distribution and only if you received the + program in object code or executable form with such an offer, in + accord with Subsection b above.) + + The source code for a work means the preferred form of the work for + making modifications to it. For an executable work, complete source code + means all the source code for all modules it contains, plus any + associated interface definition files, plus the scripts used to control + compilation and installation of the executable. However, as a special + exception, the source code distributed need not include anything that is + normally distributed (in either source or binary form) with the major + components (compiler, kernel, and so on) of the operating system on + which the executable runs, unless that component itself accompanies the + executable. + + If distribution of executable or object code is made by offering access + to copy from a designated place, then offering equivalent access to copy + the source code from the same place counts as distribution of the source + code, even though third parties are not compelled to copy the source + along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program + except as expressly provided under this License. Any attempt otherwise + to copy, modify, sublicense or distribute the Program is void, and will + automatically terminate your rights under this License. However, parties + who have received copies, or rights, from you under this License will + not have their licenses terminated so long as such parties remain in + full compliance. + + 5. You are not required to accept this License, since you have not + signed it. However, nothing else grants you permission to modify or + distribute the Program or its derivative works. These actions are + prohibited by law if you do not accept this License. Therefore, by + modifying or distributing the Program (or any work based on the + Program), you indicate your acceptance of this License to do so, and all + its terms and conditions for copying, distributing or modifying the + Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the + Program), the recipient automatically receives a license from the + original licensor to copy, distribute or modify the Program subject to + these terms and conditions. You may not impose any further restrictions + on the recipients' exercise of the rights granted herein. You are not + responsible for enforcing compliance by third parties to this License. + + 7. If, as a consequence of a court judgment or allegation of patent + infringement or for any other reason (not limited to patent issues), + conditions are imposed on you (whether by court order, agreement or + otherwise) that contradict the conditions of this License, they do not + excuse you from the conditions of this License. If you cannot distribute + so as to satisfy simultaneously your obligations under this License and + any other pertinent obligations, then as a consequence you may not + distribute the Program at all. For example, if a patent license would + not permit royalty-free redistribution of the Program by all those who + receive copies directly or indirectly through you, then the only way you + could satisfy both it and this License would be to refrain entirely from + distribution of the Program. + + If any portion of this section is held invalid or unenforceable under + any particular circumstance, the balance of the section is intended to + apply and the section as a whole is intended to apply in other + circumstances. + + It is not the purpose of this section to induce you to infringe any + patents or other property right claims or to contest validity of any + such claims; this section has the sole purpose of protecting the + integrity of the free software distribution system, which is implemented + by public license practices. Many people have made generous + contributions to the wide range of software distributed through that + system in reliance on consistent application of that system; it is up to + the author/donor to decide if he or she is willing to distribute + software through any other system and a licensee cannot impose that choice. + + This section is intended to make thoroughly clear what is believed to be + a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in + certain countries either by patents or by copyrighted interfaces, the + original copyright holder who places the Program under this License may + add an explicit geographical distribution limitation excluding those + countries, so that distribution is permitted only in or among countries + not thus excluded. In such case, this License incorporates the + limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new + versions of the General Public License from time to time. Such new + versions will be similar in spirit to the present version, but may + differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the Program + specifies a version number of this License which applies to it and "any + later version", you have the option of following the terms and + conditions either of that version or of any later version published by + the Free Software Foundation. If the Program does not specify a version + number of this License, you may choose any version ever published by the + Free Software Foundation. + + 10. If you wish to incorporate parts of the Program into other free + programs whose distribution conditions are different, write to the + author to ask for permission. For software which is copyrighted by the + Free Software Foundation, write to the Free Software Foundation; we + sometimes make exceptions for this. Our decision will be guided by the + two goals of preserving the free status of all derivatives of our free + software and of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO + WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. + EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR + OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, + EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE + ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH + YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL + NECESSARY SERVICING, REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN + WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY + AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR + DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL + DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM + (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED + INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF + THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR + OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest + possible use to the public, the best way to achieve this is to make it + free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest to + attach them to the start of each source file to most effectively convey + the exclusion of warranty; and each file should have at least the + "copyright" line and a pointer to where the full notice is found. + + One line to give the program's name and a brief idea of what it does. + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + + Also add information on how to contact you by electronic and paper mail. + + If the program is interactive, make it output a short notice like this + when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type + `show w'. This is free software, and you are welcome to redistribute + it under certain conditions; type `show c' for details. + + The hypothetical commands `show w' and `show c' should show the + appropriate parts of the General Public License. Of course, the commands + you use may be called something other than `show w' and `show c'; they + could even be mouse-clicks or menu items--whatever suits your program. + + You should also get your employer (if you work as a programmer) or your + school, if any, to sign a "copyright disclaimer" for the program, if + necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + program `Gnomovision' (which makes passes at compilers) written by + James Hacker. + + signature of Ty Coon, 1 April 1989 + Ty Coon, President of Vice + + This General Public License does not permit incorporating your program + into proprietary programs. If your program is a subroutine library, you + may consider it more useful to permit linking proprietary applications + with the library. If this is what you want to do, use the GNU Library + General Public License instead of this License. + +--- + +## CLASSPATH EXCEPTION + + Linking this library statically or dynamically with other modules is + making a combined work based on this library. Thus, the terms and + conditions of the GNU General Public License version 2 cover the whole + combination. + + As a special exception, the copyright holders of this library give you + permission to link this library with independent modules to produce an + executable, regardless of the license terms of these independent + modules, and to copy and distribute the resulting executable under + terms of your choice, provided that you also meet, for each linked + independent module, the terms and conditions of the license of that + module. An independent module is a module which is not derived from or + based on this library. If you modify this library, you may extend this + exception to your version of the library, but you are not obligated to + do so. If you do not wish to do so, delete this exception statement + from your version. + + + +# Notices for Eclipse Project for Interceptors + +This content is produced and maintained by the Eclipse Project for Interceptors +project. + +* Project home: https://projects.eclipse.org/projects/ee4j.interceptors + +## Trademarks + +Eclipse Project for Interceptors is a trademark of the Eclipse Foundation. + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v. 2.0 which is available at +http://www.eclipse.org/legal/epl-2.0. This Source Code may also be made +available under the following Secondary Licenses when the conditions for such +availability set forth in the Eclipse Public License v. 2.0 are satisfied: GNU +General Public License, version 2 with the GNU Classpath Exception which is +available at https://www.gnu.org/software/classpath/license.html. + +SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/interceptor-api + +## Third-party Content + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. + diff --git a/aggregated-license-report/build.gradle.kts b/aggregated-license-report/build.gradle.kts index 5fe3de643..1e70825fb 100644 --- a/aggregated-license-report/build.gradle.kts +++ b/aggregated-license-report/build.gradle.kts @@ -22,7 +22,7 @@ import org.gradle.kotlin.dsl.support.unzipTo val licenseReports by configurations.creating { description = "Used to generate license reports" } dependencies { - licenseReports(project(":polaris-dropwizard-service", "licenseReports")) + licenseReports(project(":polaris-quarkus-service", "licenseReports")) } val collectLicenseReportJars by diff --git a/api/iceberg-service/build.gradle.kts b/api/iceberg-service/build.gradle.kts index 0b05626e4..e1d8e6abc 100644 --- a/api/iceberg-service/build.gradle.kts +++ b/api/iceberg-service/build.gradle.kts @@ -20,6 +20,7 @@ plugins { alias(libs.plugins.openapi.generator) id("polaris-client") + alias(libs.plugins.jandex) } dependencies { @@ -111,3 +112,5 @@ listOf("sourcesJar", "compileJava").forEach { task -> sourceSets { main { java { srcDir(project.layout.buildDirectory.dir("generated/src/main/java")) } } } + +tasks.named("javadoc") { dependsOn("jandex") } diff --git a/api/management-model/build.gradle.kts b/api/management-model/build.gradle.kts index 313d28d65..488681410 100644 --- a/api/management-model/build.gradle.kts +++ b/api/management-model/build.gradle.kts @@ -20,6 +20,7 @@ plugins { alias(libs.plugins.openapi.generator) id("polaris-client") + alias(libs.plugins.jandex) } dependencies { @@ -60,3 +61,5 @@ listOf("sourcesJar", "compileJava").forEach { task -> sourceSets { main { java { srcDir(project.layout.buildDirectory.dir("generated/src/main/java")) } } } + +tasks.named("javadoc") { dependsOn("jandex") } diff --git a/api/management-service/build.gradle.kts b/api/management-service/build.gradle.kts index 548c12596..a5cd11d63 100644 --- a/api/management-service/build.gradle.kts +++ b/api/management-service/build.gradle.kts @@ -20,6 +20,7 @@ plugins { alias(libs.plugins.openapi.generator) id("polaris-client") + alias(libs.plugins.jandex) } dependencies { @@ -74,3 +75,5 @@ listOf("sourcesJar", "compileJava").forEach { task -> sourceSets { main { java { srcDir(project.layout.buildDirectory.dir("generated/src/main/java")) } } } + +tasks.named("javadoc") { dependsOn("jandex") } diff --git a/build.gradle.kts b/build.gradle.kts index e5fadafd4..373a04a4e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -24,6 +24,7 @@ buildscript { repositories { maven { url = java.net.URI("https://plugins.gradle.org/m2/") } } dependencies { classpath("com.diffplug.spotless:spotless-plugin-gradle:${libs.plugins.spotless.get().version}") + classpath("org.kordamp.gradle:jandex-gradle-plugin:${libs.plugins.jandex.get().version}") } } @@ -32,6 +33,8 @@ plugins { id("eclipse") id("polaris-root") alias(libs.plugins.rat) + // workaround for https://github.com/kordamp/jandex-gradle-plugin/issues/25 + alias(libs.plugins.jandex) apply false } val projectName = rootProject.file("ide-name.txt").readText().trim() diff --git a/docker-compose.yml b/docker-compose.yml index 0757fa9cb..6144ad093 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,17 +26,20 @@ services: - "8182" environment: AWS_REGION: us-west-2 + AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY GOOGLE_APPLICATION_CREDENTIALS: $GOOGLE_APPLICATION_CREDENTIALS AZURE_TENANT_ID: $AZURE_TENANT_ID AZURE_CLIENT_ID: $AZURE_CLIENT_ID AZURE_CLIENT_SECRET: $AZURE_CLIENT_SECRET - # add aws keys as dropwizard config - JAVA_OPTS: -Ddw.awsAccessKey=$AWS_ACCESS_KEY_ID -Ddw.awsSecretKey=$AWS_SECRET_ACCESS_KEY + POLARIS_AUTHENTICATION_TYPE: test + POLARIS_AUTHENTICATION_OAUTH2_SERVICE_TYPE: test + QUARKUS_OTEL_SDK_DISABLED: "true" volumes: - ./regtests/credentials:/tmp/credentials/ healthcheck: - test: ["CMD", "curl", "http://localhost:8182/healthcheck"] + test: ["CMD", "curl", "http://localhost:8182/q/health"] interval: 10s timeout: 10s retries: 5 diff --git a/dropwizard/service/README-quarkus.md b/dropwizard/service/README-quarkus.md new file mode 100644 index 000000000..1add64333 --- /dev/null +++ b/dropwizard/service/README-quarkus.md @@ -0,0 +1,97 @@ + + +This module contains the Polaris Service powered by Quarkus (instead of Dropwizard). + +# Main differences + +* Bean injection (CDI) is made using `@ApplicationScoped` annotation on class and injected in other classes using `@Inject` annotation (https://quarkus.io/guides/cdi-reference). +* Codehale metrics registry and opentelemetry boilerplate (prometheus exporter included) are not needed anymore: Quarkus provides it "out of the box" (https://quarkus.io/guides/opentelemetry) +* `PolarisHealthCheck` is not needed anymore: Quarkus provides it "out of the box" (https://quarkus.io/guides/smallrye-health) +* `TimedApplicationEventListener` and the `@TimedApi` annotation are replaced by Quarkus (micrometer) `@Timed` annotation (https://quarkus.io/guides/telemetry-micrometer) +* `PolarisJsonLayoutFactory` is not needed anymore: Quarkus provides it by configuration (using `quarkus.log.*` configuration) +* `PolarisApplication` is not needed, Quarkus provide a "main" application out of the box (it's possible to provide `QuarkusApplication` for control the startup and also using `@Startup` annotation) +* CORS boilerplate is not needed anymore: Quarkus supports it via configuration (using `quarkus.http.cors.*` configuration) +* CLI is not part of `polaris-service` anymore, we have (will have) a dedicated module (`polaris-cli`) + +# Build and run + +To build `polaris-service` you simply do: + +``` +./gradlew :polaris-service:build +``` + +The build creates ready to run package: +* in the `build/quarkus-app` folder, you can run with `java -jar quarkus-run.jar` +* the `build/distributions` folder contains tar/zip distributions you can extract + +You can directly run Polaris service (in the build scope) using: + +``` +./gradlew :polaris-quarkus-service:quarkusRun +``` + +You can run in Dev mode as well: + +``` +./gradlew --console=plain :polaris-quarkus-service:quarkusDev +``` + +You can directly build a Docker image using: + +``` +./gradlew :polaris-quarkus-service:imageBuild +``` + +# Configuration + +The main configuration file is not the `application.properties`. The default configuration is +packaged as part of the `polaris-quarkus-service`. `polaris-quarkus-service` uses several +configuration sources (in this order): +* system properties +* environment variables +* `.env` file in the current working directory +* `$PWD/config/application.properties` file +* the `application.properties` packaged in the `polaris-quarkus-service` application + +It means you can override some configuration property using environment variables for example. + +By default, `polaris-quarkus-service` uses 8181 as the HTTP port (defined in the `quarkus.http.port` +configuration property) and 8182 as the management port (defined in the `quarkus.management.port` +configuration property). + +You can find more details here: https://quarkus.io/guides/config + +# TODO + +* Modify `CallContext` and remove all usages of `ThreadLocal`, replace with proper context propagation. +* Remove `PolarisCallContext` – it's just an aggregation of CDI beans +* Use `@QuarkustIntegrationTest` for integration tests +* Remove `OAuthCredentialAuthFilter` and replace with Quarkus OIDC security +* Create `polaris-cli` module, add Bootstrap and Purge commands +* Adapt Helm charts, Dockerfiles, K8s examples +* Fix intermittent Gradle build error : SRCFG00011: Could not expand value + platform.quarkus.native.builder-image in property quarkus.native.builder-image + (https://github.com/quarkusio/quarkus/issues/19139). + The Quarkus issue says that using the Quarkus-platform bom helps (it's really what we should depend on). But IIRC there were some dependency issues with Spark/Scala, which prevents us from using the Quarkus-platform bom. + +* Update documentation/README/... + +* Do we want to support existing json configuration file as configuration source ? diff --git a/dropwizard/service/build.gradle.kts b/dropwizard/service/build.gradle.kts index 294279946..c4103b120 100644 --- a/dropwizard/service/build.gradle.kts +++ b/dropwizard/service/build.gradle.kts @@ -17,13 +17,11 @@ * under the License. */ -import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar - plugins { + alias(libs.plugins.quarkus) alias(libs.plugins.openapi.generator) id("polaris-server") id("polaris-license-report") - id("polaris-shadow-jar") id("application") } @@ -33,11 +31,6 @@ dependencies { implementation(project(":polaris-api-iceberg-service")) implementation(project(":polaris-service-common")) - implementation(platform(libs.dropwizard.bom)) - implementation("io.dropwizard:dropwizard-core") - implementation("io.dropwizard:dropwizard-auth") - implementation("io.dropwizard:dropwizard-json-logging") - implementation(platform(libs.iceberg.bom)) implementation("org.apache.iceberg:iceberg-api") implementation("org.apache.iceberg:iceberg-core") @@ -46,64 +39,57 @@ dependencies { // override dnsjava version in dependencies due to https://github.com/dnsjava/dnsjava/issues/329 implementation(platform(libs.dnsjava)) - implementation(libs.hadoop.common) { - exclude("org.slf4j", "slf4j-reload4j") - exclude("org.slf4j", "slf4j-log4j12") - exclude("ch.qos.reload4j", "reload4j") - exclude("log4j", "log4j") - exclude("org.apache.zookeeper", "zookeeper") - exclude("org.apache.hadoop.thirdparty", "hadoop-shaded-protobuf_3_25") - exclude("com.github.pjfanning", "jersey-json") - exclude("com.sun.jersey", "jersey-core") - exclude("com.sun.jersey", "jersey-server") - exclude("com.sun.jersey", "jersey-servlet") - } + implementation(platform(libs.quarkus.bom)) + implementation("io.quarkus:quarkus-logging-json") + implementation("io.quarkus:quarkus-rest") + implementation("io.quarkus:quarkus-rest-jackson") + implementation("io.quarkus:quarkus-reactive-routes") + implementation("io.quarkus:quarkus-hibernate-validator") + implementation("io.quarkus:quarkus-smallrye-health") + implementation("io.quarkus:quarkus-micrometer") + implementation("io.quarkus:quarkus-micrometer-registry-prometheus") + implementation("io.quarkus:quarkus-opentelemetry") + implementation("io.quarkus:quarkus-container-image-docker") + implementation("io.quarkus:quarkus-smallrye-context-propagation") - compileOnly(libs.jakarta.annotation.api) - compileOnly(libs.jakarta.inject.api) - compileOnly(libs.jakarta.servlet.api) - compileOnly(libs.jakarta.validation.api) - compileOnly(libs.jakarta.ws.rs.api) + implementation(libs.jakarta.enterprise.cdi.api) + implementation(libs.jakarta.inject.api) + implementation(libs.jakarta.validation.api) + implementation(libs.jakarta.ws.rs.api) + + implementation(libs.caffeine) + implementation(libs.guava) + implementation(libs.slf4j.api) + + implementation("org.jboss.slf4j:slf4j-jboss-logmanager") + + implementation(libs.hadoop.client.api) + implementation(libs.hadoop.client.runtime) - compileOnly(libs.smallrye.common.annotation) + implementation(libs.auth0.jwt) + + implementation(libs.bouncycastle.bcprov) + + compileOnly(libs.jakarta.annotation.api) + compileOnly(libs.spotbugs.annotations) implementation(platform(libs.google.cloud.storage.bom)) implementation("com.google.cloud:google-cloud-storage") - implementation(platform(libs.awssdk.bom)) implementation("software.amazon.awssdk:sts") implementation("software.amazon.awssdk:iam-policy-builder") implementation("software.amazon.awssdk:s3") - implementation(platform(libs.azuresdk.bom)) implementation("com.azure:azure-core") - implementation(platform(libs.micrometer.bom)) - implementation("io.micrometer:micrometer-core") - implementation("io.micrometer:micrometer-registry-prometheus") - implementation(libs.prometheus.metrics.exporter.servlet.jakarta) - - implementation(platform(libs.opentelemetry.bom)) - implementation("io.opentelemetry:opentelemetry-api") - implementation("io.opentelemetry:opentelemetry-sdk-trace") - implementation("io.opentelemetry:opentelemetry-exporter-logging") - implementation(libs.opentelemetry.semconv) - - implementation(libs.smallrye.common.annotation) - compileOnly(libs.swagger.annotations) - compileOnly(libs.spotbugs.annotations) - compileOnly(libs.jakarta.annotation.api) - compileOnly(libs.spotbugs.annotations) + + implementation(libs.jakarta.servlet.api) testImplementation(project(":polaris-api-management-model")) testImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests") testImplementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}:tests") - testImplementation("io.dropwizard:dropwizard-testing") - testImplementation(platform(libs.testcontainers.bom)) - testImplementation("org.testcontainers:testcontainers") - testImplementation(libs.s3mock.testcontainers) testImplementation("org.apache.iceberg:iceberg-spark-3.5_2.12") testImplementation("org.apache.iceberg:iceberg-spark-extensions-3.5_2.12") @@ -112,28 +98,41 @@ dependencies { exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") exclude("org.apache.logging.log4j", "log4j-api") exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.slf4j", "jul-to-slf4j") } - testImplementation(platform(libs.awssdk.bom)) testImplementation("software.amazon.awssdk:glue") testImplementation("software.amazon.awssdk:kms") testImplementation("software.amazon.awssdk:dynamodb") - testImplementation(libs.auth0.jwt) + testImplementation(platform(libs.junit.bom)) + testImplementation(libs.bundles.junit.testing) - testCompileOnly(libs.smallrye.common.annotation) + testImplementation(platform(libs.quarkus.bom)) + testImplementation("io.quarkus:quarkus-junit5") + testImplementation("io.quarkus:quarkus-junit5-mockito") + testImplementation("io.quarkus:quarkus-rest-client") + testImplementation("io.quarkus:quarkus-rest-client-jackson") + testImplementation("io.rest-assured:rest-assured") - testImplementation(platform(libs.junit.bom)) - testImplementation("org.junit.jupiter:junit-jupiter") - testImplementation(libs.assertj.core) - testImplementation(libs.mockito.core) - testRuntimeOnly("org.junit.platform:junit-platform-launcher") + testImplementation(platform(libs.testcontainers.bom)) + testImplementation("org.testcontainers:testcontainers") + testImplementation(libs.s3mock.testcontainers) - testImplementation(project(":polaris-eclipselink")) + // required for PolarisSparkIntegrationTest + testImplementation(enforcedPlatform("org.scala-lang:scala-library:2.12.18")) + testImplementation(enforcedPlatform("org.scala-lang:scala-reflect:2.12.18")) + testImplementation(libs.javax.servlet.api) + testImplementation( + enforcedPlatform("org.antlr:antlr4-runtime:4.9.3") + ) // cannot be higher than 4.9.3 + + testImplementation("org.hawkular.agent:prometheus-scraper:0.23.0.Final") } -if (project.properties.get("eclipseLink") == "true") { - dependencies { implementation(project(":polaris-eclipselink")) } +tasks.withType(Test::class.java).configureEach { + systemProperty("java.util.logging.manager", "org.jboss.logmanager.LogManager") + addSparkJvmOptions() } tasks.named("test").configure { @@ -145,45 +144,38 @@ tasks.named("test").configure { maxParallelForks = 4 } -tasks.register("runApp").configure { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") - } - classpath = sourceSets["main"].runtimeClasspath - mainClass = "org.apache.polaris.service.dropwizard.PolarisApplication" - args("server", "$rootDir/polaris-server.yml") -} - -application { mainClass = "org.apache.polaris.service.dropwizard.PolarisApplication" } - -tasks.named("jar") { - manifest { attributes["Main-Class"] = "org.apache.polaris.service.dropwizard.PolarisApplication" } -} - -tasks.register("testJar") { - archiveClassifier.set("tests") - from(sourceSets.test.get().output) +/** + * Adds the JPMS options required for Spark to run on Java 17, taken from the + * `DEFAULT_MODULE_OPTIONS` constant in `org.apache.spark.launcher.JavaModuleOptions`. + */ +fun JavaForkOptions.addSparkJvmOptions() { + jvmArgs = + (jvmArgs ?: emptyList()) + + listOf( + // Spark 3.3+ + "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens=java.base/sun.security.action=ALL-UNNAMED", + "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED", + "--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED", + // Spark 3.4+ + "-Djdk.reflect.useDirectMethodHandle=false" + ) } -val shadowJar = - tasks.named("shadowJar") { - append("META-INF/hk2-locator/default") - finalizedBy("startShadowScripts") - } - -val startScripts = - tasks.named("startScripts") { applicationName = "polaris-service" } - -tasks.named("startShadowScripts") { applicationName = "polaris-service" } - -tasks.register("prepareDockerDist") { - into(project.layout.buildDirectory.dir("docker-dist")) - from(startScripts) { into("bin") } - from(configurations.runtimeClasspath) { into("lib") } - from(tasks.named("jar")) { into("lib") } -} +tasks.named("compileJava") { dependsOn("compileQuarkusGeneratedSourcesJava") } -tasks.named("build").configure { dependsOn("prepareDockerDist") } +tasks.named("sourcesJar") { dependsOn("compileQuarkusGeneratedSourcesJava") } distributions { main { @@ -192,12 +184,4 @@ distributions { from("../../LICENSE-BINARY-DIST").rename("LICENSE-BINARY-DIST", "LICENSE") } } - named("shadow") { - contents { - from("../../NOTICE") - from("../../LICENSE-BINARY-DIST").rename("LICENSE-BINARY-DIST", "LICENSE") - } - } } - -tasks.named("assemble").configure { dependsOn("testJar") } diff --git a/dropwizard/service/src/main/docker/Dockerfile.jvm b/dropwizard/service/src/main/docker/Dockerfile.jvm new file mode 100644 index 000000000..c54d1294b --- /dev/null +++ b/dropwizard/service/src/main/docker/Dockerfile.jvm @@ -0,0 +1,49 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +FROM registry.access.redhat.com/ubi9/openjdk-21:1.20-2.1726695192 + +LABEL org.opencontainers.image.source=https://github.com/apache/polaris +LABEL org.opencontainers.image.description="Apache Polaris" +LABEL org.opencontainers.image.licenses=Apache-2.0 + +ENV LANGUAGE='en_US:en' + +USER root +RUN groupadd --gid 10001 polaris \ + && useradd --uid 10000 --gid polaris polaris \ + && chown -R polaris:polaris /opt/jboss/container \ + && chown -R polaris:polaris /deployments + +USER polaris +WORKDIR /home/polaris +ENV USER=polaris +ENV UID=10000 +ENV HOME=/home/polaris + +# We make four distinct layers so if there are application changes the library layers can be re-used +COPY --chown=polaris:polaris build/quarkus-app/lib/ /deployments/lib/ +COPY --chown=polaris:polaris build/quarkus-app/*.jar /deployments/ +COPY --chown=polaris:polaris build/quarkus-app/app/ /deployments/app/ +COPY --chown=polaris:polaris build/quarkus-app/quarkus/ /deployments/quarkus/ + +EXPOSE 8181 +EXPOSE 8182 + +ENV AB_JOLOKIA_OFF="" +ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/BootstrapRealmsCommand.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/BootstrapRealmsCommand.java deleted file mode 100644 index b68d2d761..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/BootstrapRealmsCommand.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard; - -import io.dropwizard.core.cli.ConfiguredCommand; -import io.dropwizard.core.setup.Bootstrap; -import java.util.Map; -import net.sourceforge.argparse4j.inf.Namespace; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Command for bootstrapping root level service principals for each realm. This command will invoke - * a default implementation which generates random user id and secret. These credentials will be - * printed out to the log and standard output (stdout). - */ -public class BootstrapRealmsCommand extends ConfiguredCommand { - private static final Logger LOGGER = LoggerFactory.getLogger(BootstrapRealmsCommand.class); - - public BootstrapRealmsCommand() { - super("bootstrap", "bootstraps principal credentials for all realms and prints them to log"); - } - - @Override - protected void run( - Bootstrap bootstrap, - Namespace namespace, - PolarisApplicationConfig configuration) { - MetaStoreManagerFactory metaStoreManagerFactory = - configuration.findService(MetaStoreManagerFactory.class); - - PolarisConfigurationStore configurationStore = - configuration.findService(PolarisConfigurationStore.class); - - // Execute the bootstrap - Map results = - metaStoreManagerFactory.bootstrapRealms(configuration.getDefaultRealms()); - - // Log any errors: - boolean success = true; - for (Map.Entry result : results.entrySet()) { - if (!result.getValue().isSuccess()) { - LOGGER.error( - "Bootstrapping `{}` failed: {}", - result.getKey(), - result.getValue().getReturnStatus().toString()); - success = false; - } - } - - if (success) { - LOGGER.info("Bootstrap completed successfully."); - } else { - LOGGER.error("Bootstrap encountered errors during operation."); - } - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisApplication.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisApplication.java deleted file mode 100644 index 29afa31a7..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisApplication.java +++ /dev/null @@ -1,584 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNull; -import static org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig.REQUEST_BODY_BYTES_NO_LIMIT; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.core.StreamReadConstraints; -import com.fasterxml.jackson.databind.DeserializationConfig; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.JavaType; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.deser.std.StdValueInstantiator; -import com.fasterxml.jackson.databind.jsontype.NamedType; -import com.fasterxml.jackson.databind.module.SimpleModule; -import com.fasterxml.jackson.databind.module.SimpleValueInstantiators; -import com.fasterxml.jackson.databind.type.TypeFactory; -import io.dropwizard.auth.AuthDynamicFeature; -import io.dropwizard.auth.AuthFilter; -import io.dropwizard.auth.oauth.OAuthCredentialAuthFilter; -import io.dropwizard.configuration.EnvironmentVariableSubstitutor; -import io.dropwizard.configuration.SubstitutingSourceProvider; -import io.dropwizard.core.Application; -import io.dropwizard.core.setup.Bootstrap; -import io.dropwizard.core.setup.Environment; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.prometheusmetrics.PrometheusConfig; -import io.micrometer.prometheusmetrics.PrometheusMeterRegistry; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; -import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; -import io.opentelemetry.context.propagation.ContextPropagators; -import io.opentelemetry.context.propagation.TextMapPropagator; -import io.opentelemetry.exporter.logging.LoggingSpanExporter; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; -import io.opentelemetry.semconv.ServiceAttributes; -import io.prometheus.metrics.exporter.servlet.jakarta.PrometheusMetricsServlet; -import jakarta.inject.Inject; -import jakarta.inject.Provider; -import jakarta.inject.Singleton; -import jakarta.servlet.DispatcherType; -import jakarta.servlet.Filter; -import jakarta.servlet.FilterChain; -import jakarta.servlet.FilterRegistration; -import jakarta.servlet.ServletException; -import jakarta.servlet.ServletRequest; -import jakarta.servlet.ServletResponse; -import jakarta.servlet.http.HttpServletRequest; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.util.Collections; -import java.util.EnumSet; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.Executors; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.iceberg.rest.RESTSerializers; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.auth.PolarisAuthorizer; -import org.apache.polaris.core.auth.PolarisAuthorizerImpl; -import org.apache.polaris.core.auth.PolarisGrantManager; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.context.RealmScoped; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.cache.EntityCache; -import org.apache.polaris.core.persistence.cache.PolarisRemoteCache; -import org.apache.polaris.service.admin.PolarisServiceImpl; -import org.apache.polaris.service.admin.api.PolarisCatalogsApi; -import org.apache.polaris.service.admin.api.PolarisCatalogsApiService; -import org.apache.polaris.service.admin.api.PolarisPrincipalRolesApi; -import org.apache.polaris.service.admin.api.PolarisPrincipalRolesApiService; -import org.apache.polaris.service.admin.api.PolarisPrincipalsApi; -import org.apache.polaris.service.admin.api.PolarisPrincipalsApiService; -import org.apache.polaris.service.auth.Authenticator; -import org.apache.polaris.service.catalog.IcebergCatalogAdapter; -import org.apache.polaris.service.catalog.api.IcebergRestCatalogApi; -import org.apache.polaris.service.catalog.api.IcebergRestCatalogApiService; -import org.apache.polaris.service.catalog.api.IcebergRestConfigurationApi; -import org.apache.polaris.service.catalog.api.IcebergRestConfigurationApiService; -import org.apache.polaris.service.catalog.api.IcebergRestOAuth2Api; -import org.apache.polaris.service.catalog.io.FileIOFactory; -import org.apache.polaris.service.config.RealmEntityManagerFactory; -import org.apache.polaris.service.config.Serializers; -import org.apache.polaris.service.config.TaskHandlerConfiguration; -import org.apache.polaris.service.context.CallContextCatalogFactory; -import org.apache.polaris.service.context.CallContextResolver; -import org.apache.polaris.service.context.PolarisCallContextCatalogFactory; -import org.apache.polaris.service.context.RealmContextResolver; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.context.RealmScopeContext; -import org.apache.polaris.service.dropwizard.exception.JerseyViolationExceptionMapper; -import org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry; -import org.apache.polaris.service.dropwizard.persistence.cache.EntityCacheFactory; -import org.apache.polaris.service.dropwizard.throttling.StreamReadConstraintsExceptionMapper; -import org.apache.polaris.service.dropwizard.tracing.TracingFilter; -import org.apache.polaris.service.exception.IcebergExceptionMapper; -import org.apache.polaris.service.exception.IcebergJsonProcessingExceptionMapper; -import org.apache.polaris.service.exception.PolarisExceptionMapper; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; -import org.apache.polaris.service.ratelimiter.RateLimiterFilter; -import org.apache.polaris.service.task.ManifestFileCleanupTaskHandler; -import org.apache.polaris.service.task.TableCleanupTaskHandler; -import org.apache.polaris.service.task.TaskExecutor; -import org.apache.polaris.service.task.TaskExecutorImpl; -import org.apache.polaris.service.task.TaskFileIOSupplier; -import org.eclipse.jetty.servlets.CrossOriginFilter; -import org.glassfish.hk2.api.Context; -import org.glassfish.hk2.api.Factory; -import org.glassfish.hk2.api.ServiceLocator; -import org.glassfish.hk2.api.TypeLiteral; -import org.glassfish.hk2.utilities.ServiceLocatorUtilities; -import org.glassfish.hk2.utilities.binding.AbstractBinder; -import org.glassfish.jersey.process.internal.RequestScoped; -import org.glassfish.jersey.servlet.ServletProperties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; - -public class PolarisApplication extends Application { - private static final Logger LOGGER = LoggerFactory.getLogger(PolarisApplication.class); - private ServiceLocator serviceLocator; - - public static void main(final String[] args) throws Exception { - new PolarisApplication().run(args); - printAsciiArt(); - } - - private static void printAsciiArt() throws IOException { - URL url = PolarisApplication.class.getResource("/org/apache/polaris/service/banner.txt"); - try (InputStream in = - requireNonNull(url, "banner.txt not found on classpath") - .openConnection() - .getInputStream()) { - System.out.println(new String(in.readAllBytes(), UTF_8)); - } - } - - @Override - public void initialize(Bootstrap bootstrap) { - // Enable variable substitution with environment variables - EnvironmentVariableSubstitutor substitutor = new EnvironmentVariableSubstitutor(false); - SubstitutingSourceProvider provider = - new SubstitutingSourceProvider(bootstrap.getConfigurationSourceProvider(), substitutor); - bootstrap.setConfigurationSourceProvider(provider); - - bootstrap.addCommand(new BootstrapRealmsCommand()); - bootstrap.addCommand(new PurgeRealmsCommand()); - serviceLocator = ServiceLocatorUtilities.createAndPopulateServiceLocator(); - ObjectMapper objectMapper = bootstrap.getObjectMapper(); - - // Register the PolarisApplicationConfig class with the ServiceLocator so that we can inject - // instances of the configuration or certain of the configuration fields into other classes. - // The configuration's postConstruct method registers all the configured fields as factories - // so that they are used for DI. - ServiceLocatorUtilities.addClasses(serviceLocator, PolarisApplicationConfig.class); - TypeFactory typeFactory = TypeFactory.defaultInstance(); - SimpleValueInstantiators instantiators = new SimpleValueInstantiators(); - instantiators.addValueInstantiator( - PolarisApplicationConfig.class, - new ServiceLocatorValueInstantiator( - objectMapper.getDeserializationConfig(), - typeFactory.constructType(PolarisApplicationConfig.class), - serviceLocator)); - - // Use the default ServiceLocator to discover the implementations of various contract providers - // and register them as subtypes with the ObjectMapper. This allows Jackson to discover the - // various implementations and use the type annotations to determine which instance to use when - // parsing the YAML configuration. - SimpleModule module = new SimpleModule(); - serviceLocator - .getDescriptors((c) -> true) - .forEach( - descriptor -> { - try { - Class klazz = - PolarisApplication.class - .getClassLoader() - .loadClass(descriptor.getImplementation()); - String name = descriptor.getName(); - if (name == null) { - module.registerSubtypes(klazz); - } else { - module.registerSubtypes(new NamedType(klazz, name)); - } - } catch (ClassNotFoundException e) { - LOGGER.error("Error loading class {}", descriptor.getImplementation(), e); - throw new RuntimeException("Unable to start Polaris application"); - } - }); - - ServiceLocatorUtilities.addClasses(serviceLocator, PolarisMetricRegistry.class); - ServiceLocatorUtilities.bind( - serviceLocator, - new AbstractBinder() { - @Override - protected void configure() { - bind(setupTracing()).to(OpenTelemetry.class); - bind(new PrometheusMeterRegistry(PrometheusConfig.DEFAULT)).to(MeterRegistry.class); - } - }); - module.setValueInstantiators(instantiators); - objectMapper.registerModule(module); - } - - /** - * Value instantiator that uses the ServiceLocator to create instances of the various service - * types - */ - private static class ServiceLocatorValueInstantiator extends StdValueInstantiator { - private final ServiceLocator serviceLocator; - - public ServiceLocatorValueInstantiator( - DeserializationConfig config, JavaType valueType, ServiceLocator serviceLocator) { - super(config, valueType); - this.serviceLocator = serviceLocator; - } - - @Override - public boolean canCreateUsingDefault() { - return true; - } - - @Override - public boolean canInstantiate() { - return true; - } - - @Override - public Object createUsingDefault(DeserializationContext ctxt) throws IOException { - return ServiceLocatorUtilities.findOrCreateService(serviceLocator, getValueClass()); - } - } - - @Override - public void run(PolarisApplicationConfig configuration, Environment environment) { - PolarisMetricRegistry polarisMetricRegistry = - configuration.findService(PolarisMetricRegistry.class); - - MetaStoreManagerFactory metaStoreManagerFactory = - configuration.findService(MetaStoreManagerFactory.class); - - // Use the PolarisApplicationConfig to register dependencies in the Jersey resource - // configuration. This uses a different ServiceLocator from the one in the bootstrap step - environment - .getApplicationContext() - .setAttribute(ServletProperties.SERVICE_LOCATOR, configuration.getServiceLocator()); - - environment - .jersey() - .register( - new AbstractBinder() { - @Override - protected void configure() { - bindFactory(CallContextFactory.class).to(CallContext.class).in(RequestScoped.class); - bindFactory(RealmContextFactory.class) - .to(RealmContext.class) - .in(RequestScoped.class); - bind(RealmScopeContext.class) - .in(Singleton.class) - .to(new TypeLiteral>() {}); - bindFactory(PolarisMetaStoreManagerFactory.class) - .to(PolarisMetaStoreManager.class) - .in(RealmScoped.class); - bindFactory(EntityCacheFactory.class).in(RealmScoped.class).to(EntityCache.class); - - bindFactory(PolarisRemoteCacheFactory.class) - .in(RealmScoped.class) - .to(PolarisRemoteCache.class); - - // factory to use a cache delegating grant cache - // currently depends explicitly on the metaStoreManager as the delegate grant - // manager - bindFactory(PolarisMetaStoreManagerFactory.class) - .in(RealmScoped.class) - .to(PolarisGrantManager.class); - polarisMetricRegistry.init( - IcebergRestCatalogApi.class, - IcebergRestConfigurationApi.class, - IcebergRestOAuth2Api.class, - PolarisCatalogsApi.class, - PolarisPrincipalsApi.class, - PolarisPrincipalRolesApi.class); - bindAsContract(RealmEntityManagerFactory.class).in(Singleton.class); - bind(PolarisCallContextCatalogFactory.class) - .to(CallContextCatalogFactory.class) - .in(Singleton.class); - bind(PolarisAuthorizerImpl.class).in(Singleton.class).to(PolarisAuthorizer.class); - bind(IcebergCatalogAdapter.class) - .in(Singleton.class) - .to(IcebergRestCatalogApiService.class) - .to(IcebergRestConfigurationApiService.class); - bind(PolarisServiceImpl.class) - .in(Singleton.class) - .to(PolarisCatalogsApiService.class) - .to(PolarisPrincipalsApiService.class) - .to(PolarisPrincipalRolesApiService.class); - FileIOFactory fileIOFactory = configuration.findService(FileIOFactory.class); - - TaskHandlerConfiguration taskConfig = configuration.getTaskHandler(); - TaskExecutorImpl taskExecutor = - new TaskExecutorImpl(taskConfig.executorService(), metaStoreManagerFactory); - TaskFileIOSupplier fileIOSupplier = - new TaskFileIOSupplier( - metaStoreManagerFactory, - fileIOFactory, - configuration.findService(PolarisConfigurationStore.class)); - taskExecutor.addTaskHandler( - new TableCleanupTaskHandler( - taskExecutor, metaStoreManagerFactory, fileIOSupplier)); - taskExecutor.addTaskHandler( - new ManifestFileCleanupTaskHandler( - fileIOSupplier, Executors.newVirtualThreadPerTaskExecutor())); - - bind(taskExecutor).to(TaskExecutor.class); - } - }); - - // servlet filters don't use the underlying DI - environment - .servlets() - .addFilter( - "realmContext", - new ContextResolverFilter( - configuration.findService(RealmContextResolver.class), - configuration.findService(CallContextResolver.class))) - .addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*"); - - LOGGER.info( - "Initializing PolarisCallContextCatalogFactory for metaStoreManagerType {}", - metaStoreManagerFactory); - - environment.jersey().register(IcebergRestCatalogApi.class); - environment.jersey().register(IcebergRestConfigurationApi.class); - - FilterRegistration.Dynamic corsRegistration = - environment.servlets().addFilter("CORS", CrossOriginFilter.class); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOWED_ORIGINS_PARAM, - String.join(",", configuration.getCorsConfiguration().getAllowedOrigins())); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOWED_TIMING_ORIGINS_PARAM, - String.join(",", configuration.getCorsConfiguration().getAllowedTimingOrigins())); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOWED_METHODS_PARAM, - String.join(",", configuration.getCorsConfiguration().getAllowedMethods())); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOWED_HEADERS_PARAM, - String.join(",", configuration.getCorsConfiguration().getAllowedHeaders())); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOW_CREDENTIALS_PARAM, - String.join(",", configuration.getCorsConfiguration().getAllowCredentials())); - corsRegistration.setInitParameter( - CrossOriginFilter.PREFLIGHT_MAX_AGE_PARAM, - Objects.toString(configuration.getCorsConfiguration().getPreflightMaxAge())); - corsRegistration.setInitParameter( - CrossOriginFilter.ALLOW_CREDENTIALS_PARAM, - configuration.getCorsConfiguration().getAllowCredentials()); - corsRegistration.addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*"); - - OpenTelemetry openTelemetry = configuration.findService(OpenTelemetry.class); - environment - .servlets() - .addFilter("tracing", new TracingFilter(openTelemetry)) - .addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*"); - - if (configuration.hasRateLimiter()) { - environment.jersey().register(RateLimiterFilter.class); - } - Authenticator authenticator = - configuration.findService(new TypeLiteral<>() {}); - AuthFilter oauthCredentialAuthFilter = - new OAuthCredentialAuthFilter.Builder() - .setAuthenticator(authenticator::authenticate) - .setPrefix("Bearer") - .buildAuthFilter(); - environment.jersey().register(new AuthDynamicFeature(oauthCredentialAuthFilter)); - environment.healthChecks().register("polaris", new PolarisHealthCheck()); - - environment.jersey().register(IcebergRestOAuth2Api.class); - environment.jersey().register(IcebergExceptionMapper.class); - environment.jersey().register(PolarisExceptionMapper.class); - - environment.jersey().register(PolarisCatalogsApi.class); - environment.jersey().register(PolarisPrincipalsApi.class); - environment.jersey().register(PolarisPrincipalRolesApi.class); - - ObjectMapper objectMapper = environment.getObjectMapper(); - objectMapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - objectMapper.setPropertyNamingStrategy(new PropertyNamingStrategies.KebabCaseStrategy()); - - long maxRequestBodyBytes = configuration.getMaxRequestBodyBytes(); - if (maxRequestBodyBytes != REQUEST_BODY_BYTES_NO_LIMIT) { - objectMapper - .getFactory() - .setStreamReadConstraints( - StreamReadConstraints.builder().maxDocumentLength(maxRequestBodyBytes).build()); - LOGGER.info("Limiting request body size to {} bytes", maxRequestBodyBytes); - } - - environment.jersey().register(StreamReadConstraintsExceptionMapper.class); - RESTSerializers.registerAll(objectMapper); - Serializers.registerSerializers(objectMapper); - environment.jersey().register(IcebergJsonProcessingExceptionMapper.class); - environment.jersey().register(JerseyViolationExceptionMapper.class); - - // for tests, we have to instantiate the TimedApplicationEventListener directly - environment.jersey().register(new TimedApplicationEventListener(polarisMetricRegistry)); - - environment - .admin() - .addServlet( - "metrics", - new PrometheusMetricsServlet( - ((PrometheusMeterRegistry) polarisMetricRegistry.getMeterRegistry()) - .getPrometheusRegistry())) - .addMapping("/metrics"); - - // For in-memory metastore we need to bootstrap Service and Service principal at startup (for - // default realm) - // We can not utilize dropwizard Bootstrap command as command and server will be running two - // different processes - // and in-memory state will be lost b/w invocation of bootstrap command and running a server - if (metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory) { - metaStoreManagerFactory.getOrCreateMetaStoreManager(configuration::getDefaultRealm); - } - - LOGGER.info("Server started successfully."); - } - - private static OpenTelemetry setupTracing() { - Resource resource = - Resource.getDefault().toBuilder() - .put(ServiceAttributes.SERVICE_NAME, "polaris") - .put(ServiceAttributes.SERVICE_VERSION, "0.1.0") - .build(); - SdkTracerProvider sdkTracerProvider = - SdkTracerProvider.builder() - .addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter.create())) - .setResource(resource) - .build(); - return OpenTelemetrySdk.builder() - .setTracerProvider(sdkTracerProvider) - .setPropagators( - ContextPropagators.create( - TextMapPropagator.composite( - W3CTraceContextPropagator.getInstance(), W3CBaggagePropagator.getInstance()))) - .build(); - } - - /** Resolves and sets ThreadLocal CallContext/RealmContext based on the request contents. */ - private static class ContextResolverFilter implements Filter { - private final RealmContextResolver realmContextResolver; - private final CallContextResolver callContextResolver; - - @Inject - public ContextResolverFilter( - RealmContextResolver realmContextResolver, CallContextResolver callContextResolver) { - this.realmContextResolver = realmContextResolver; - this.callContextResolver = callContextResolver; - } - - @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) - throws IOException, ServletException { - HttpServletRequest httpRequest = (HttpServletRequest) request; - Stream headerNames = Collections.list(httpRequest.getHeaderNames()).stream(); - Map headers = - headerNames.collect(Collectors.toMap(Function.identity(), httpRequest::getHeader)); - RealmContext currentRealmContext = - realmContextResolver.resolveRealmContext( - httpRequest.getRequestURL().toString(), - httpRequest.getMethod(), - httpRequest.getRequestURI().substring(1), - request.getParameterMap().entrySet().stream() - .collect( - Collectors.toMap(Map.Entry::getKey, (e) -> ((String[]) e.getValue())[0])), - headers); - CallContext currentCallContext = - callContextResolver.resolveCallContext( - currentRealmContext, - httpRequest.getMethod(), - httpRequest.getRequestURI().substring(1), - request.getParameterMap().entrySet().stream() - .collect( - Collectors.toMap(Map.Entry::getKey, (e) -> ((String[]) e.getValue())[0])), - headers); - CallContext.setCurrentContext(currentCallContext); - try (MDC.MDCCloseable ignored1 = - MDC.putCloseable("realm", currentRealmContext.getRealmIdentifier()); - MDC.MDCCloseable ignored2 = - MDC.putCloseable("request_id", httpRequest.getHeader("request_id"))) { - chain.doFilter(request, response); - } finally { - currentCallContext.close(); - } - } - } - - /** Factory to create a CallContext based on the request contents. */ - @RequestScoped - private static class CallContextFactory implements Factory { - - @RequestScoped - @Override - public CallContext provide() { - return CallContext.getCurrentContext(); - } - - @Override - public void dispose(CallContext instance) {} - } - - @RequestScoped - private static class RealmContextFactory implements Factory { - @Inject Provider callContext; - - @RequestScoped - @Override - public RealmContext provide() { - return callContext.get().getRealmContext(); - } - - @Override - public void dispose(RealmContext instance) {} - } - - private static class PolarisMetaStoreManagerFactory implements Factory { - @Inject MetaStoreManagerFactory metaStoreManagerFactory; - - @RealmScoped - @Override - public PolarisMetaStoreManager provide() { - RealmContext realmContext = CallContext.getCurrentContext().getRealmContext(); - return metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); - } - - @Override - public void dispose(PolarisMetaStoreManager instance) {} - } - - private static class PolarisRemoteCacheFactory implements Factory { - @Inject PolarisMetaStoreManager metaStoreManager; - - @RealmScoped - @Override - public PolarisRemoteCache provide() { - return metaStoreManager; - } - - @Override - public void dispose(PolarisRemoteCache instance) {} - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PurgeRealmsCommand.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PurgeRealmsCommand.java deleted file mode 100644 index 259f1f07d..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PurgeRealmsCommand.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard; - -import io.dropwizard.core.cli.ConfiguredCommand; -import io.dropwizard.core.setup.Bootstrap; -import net.sourceforge.argparse4j.inf.Namespace; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Command for purging all metadata associated with a realm */ -public class PurgeRealmsCommand extends ConfiguredCommand { - private static final Logger LOGGER = LoggerFactory.getLogger(PurgeRealmsCommand.class); - - public PurgeRealmsCommand() { - super("purge", "purge principal credentials for all realms and prints them to log"); - } - - @Override - protected void run( - Bootstrap bootstrap, - Namespace namespace, - PolarisApplicationConfig configuration) - throws Exception { - MetaStoreManagerFactory metaStoreManagerFactory = - configuration.findService(MetaStoreManagerFactory.class); - - metaStoreManagerFactory.purgeRealms(configuration.getDefaultRealms()); - - LOGGER.info("Purge completed successfully."); - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/TimedApplicationEventListener.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/TimedApplicationEventListener.java deleted file mode 100644 index 88a41c957..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/TimedApplicationEventListener.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard; - -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.TAG_RESP_CODE; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Stopwatch; -import io.micrometer.core.annotation.Timed; -import io.micrometer.core.instrument.Tag; -import jakarta.inject.Inject; -import jakarta.ws.rs.ext.Provider; -import java.lang.reflect.Method; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry; -import org.glassfish.jersey.server.monitoring.ApplicationEvent; -import org.glassfish.jersey.server.monitoring.ApplicationEventListener; -import org.glassfish.jersey.server.monitoring.RequestEvent; -import org.glassfish.jersey.server.monitoring.RequestEventListener; - -/** - * An ApplicationEventListener that supports timing and error counting of Jersey resource methods - * annotated by {@link Timed}. It uses the {@link PolarisMetricRegistry} for metric collection and - * properly times the resource on success and increments the error counter on failure. - */ -@Provider -public class TimedApplicationEventListener implements ApplicationEventListener { - - /** - * Each API will increment a common counter (SINGLETON_METRIC_NAME) but have its API name tagged - * (TAG_API_NAME). - */ - public static final String SINGLETON_METRIC_NAME = "polaris.api"; - - public static final String TAG_API_NAME = "api_name"; - - // The PolarisMetricRegistry instance used for recording metrics and error counters. - private final PolarisMetricRegistry polarisMetricRegistry; - - @Inject - public TimedApplicationEventListener(PolarisMetricRegistry polarisMetricRegistry) { - this.polarisMetricRegistry = polarisMetricRegistry; - } - - @VisibleForTesting - public PolarisMetricRegistry getMetricRegistry() { - return polarisMetricRegistry; - } - - @Override - public void onEvent(ApplicationEvent event) {} - - @Override - public RequestEventListener onRequest(RequestEvent event) { - return new TimedRequestEventListener(); - } - - /** - * A RequestEventListener implementation that handles timing of resource method execution and - * increments error counters on failures. The lifetime of the listener is tied to a single HTTP - * request. - */ - private class TimedRequestEventListener implements RequestEventListener { - private String metric; - private Stopwatch sw; - - /** Handles various types of RequestEvents to start timing, stop timing, and record metrics. */ - @Override - public void onEvent(RequestEvent event) { - String realmId = CallContext.getCurrentContext().getRealmContext().getRealmIdentifier(); - if (event.getType() == RequestEvent.Type.REQUEST_MATCHED) { - Method method = - event.getUriInfo().getMatchedResourceMethod().getInvocable().getHandlingMethod(); - if (method.isAnnotationPresent(Timed.class)) { - Timed timedApi = method.getAnnotation(Timed.class); - metric = timedApi.value(); - - // Increment both the counter with the API name in the metric name and a common metric - polarisMetricRegistry.incrementCounter(metric, realmId); - polarisMetricRegistry.incrementCounter( - SINGLETON_METRIC_NAME, realmId, Tag.of(TAG_API_NAME, metric)); - } - } else if (event.getType() == RequestEvent.Type.RESOURCE_METHOD_START && metric != null) { - sw = Stopwatch.createStarted(); - } else if (event.getType() == RequestEvent.Type.FINISHED && metric != null) { - if (event.isSuccess()) { - sw.stop(); - polarisMetricRegistry.recordTimer(metric, sw.elapsed(TimeUnit.MILLISECONDS), realmId); - } else { - int statusCode = event.getContainerResponse().getStatus(); - - // Increment both the counter with the API name in the metric name and a common metric - polarisMetricRegistry.incrementErrorCounter(metric, statusCode, realmId); - polarisMetricRegistry.incrementErrorCounter( - SINGLETON_METRIC_NAME, - realmId, - List.of( - Tag.of(TAG_API_NAME, metric), Tag.of(TAG_RESP_CODE, String.valueOf(statusCode)))); - } - } - } - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/auth/QuarkusAuthenticatorConfiguration.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/auth/QuarkusAuthenticatorConfiguration.java new file mode 100644 index 000000000..0441d60f0 --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/auth/QuarkusAuthenticatorConfiguration.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.auth; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithName; +import org.apache.polaris.service.auth.AuthenticatorConfiguration; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.authentication") +public interface QuarkusAuthenticatorConfiguration extends AuthenticatorConfiguration { + + /** + * The type of the authenticator to use. Must be a registered {@link + * org.apache.polaris.service.auth.Authenticator} identifier. + */ + String type(); + + /** The configuration for the OAuth2 service that delivers and validates tokens. */ + @WithName("oauth2-service") + QuarkusServiceConfiguration service(); + + /** The configuration for the token broker factory. */ + @Override + QuarkusTokenBrokerFactoryConfiguration tokenBrokerFactory(); + + interface QuarkusServiceConfiguration { + + /** + * The type of the OAuth2 service. Must be a registered {@link + * org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService} identifier. + */ + String type(); + } + + interface QuarkusTokenBrokerFactoryConfiguration extends TokenBrokerFactoryConfiguration { + + /** + * The type of the token broker factory. Must be a registered {@link + * org.apache.polaris.service.auth.TokenBrokerFactory} identifier. + */ + String type(); + } +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/auth/QuarkusOAuthFilter.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/auth/QuarkusOAuthFilter.java new file mode 100644 index 000000000..50c56640b --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/auth/QuarkusOAuthFilter.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.auth; + +import jakarta.inject.Inject; +import jakarta.ws.rs.NotAuthorizedException; +import jakarta.ws.rs.Priorities; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.HttpHeaders; +import jakarta.ws.rs.core.SecurityContext; +import java.security.Principal; +import java.util.Optional; +import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.service.auth.Authenticator; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.jboss.resteasy.reactive.server.ServerRequestFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// TODO replace with all authN with Quarkus native authN +public class QuarkusOAuthFilter { + + private static final Logger LOGGER = LoggerFactory.getLogger(QuarkusOAuthFilter.class); + + private static final String CHALLENGE_FORMAT = "Bearer realm=\"%s\""; + + /** + * Query parameter used to pass Bearer token + * + * @see The OAuth 2.0 Authorization + * Framework: Bearer Token Usage + */ + public static final String OAUTH_ACCESS_TOKEN_PARAM = "access_token"; + + @Inject Authenticator authenticator; + @Inject CallContext callContext; + + @ServerRequestFilter(priority = Priorities.AUTHENTICATION) + public void authenticate(ContainerRequestContext requestContext) { + + if (requestContext.getUriInfo().getPath().equals("/api/catalog/v1/oauth/tokens")) { + return; + } + + String credentials = + getCredentials(requestContext.getHeaders().getFirst(HttpHeaders.AUTHORIZATION)); + + // If Authorization header is not used, check query parameter where token can be passed as well + if (credentials == null) { + credentials = + requestContext.getUriInfo().getQueryParameters().getFirst(OAUTH_ACCESS_TOKEN_PARAM); + } + + if (!authenticate(requestContext, credentials, SecurityContext.BASIC_AUTH)) { + throw new NotAuthorizedException( + "Credentials are required to access this resource.", + String.format(CHALLENGE_FORMAT, callContext.getRealmContext().getRealmIdentifier())); + } + } + + /** + * Parses a value of the `Authorization` header in the form of `Bearer a892bf3e284da9bb40648ab10`. + * + * @param header the value of the `Authorization` header + * @return a token + */ + @Nullable + private String getCredentials(String header) { + if (header == null) { + return null; + } + + final int space = header.indexOf(' '); + if (space <= 0) { + return null; + } + + final String method = header.substring(0, space); + if (!"Bearer".equalsIgnoreCase(method)) { + return null; + } + + return header.substring(space + 1); + } + + /** + * Authenticates a request with user credentials and setup the security context. + * + * @param requestContext the context of the request + * @param credentials the user credentials + * @param scheme the authentication scheme; one of {@code BASIC_AUTH, FORM_AUTH, CLIENT_CERT_AUTH, + * DIGEST_AUTH}. See {@link SecurityContext} + * @return {@code true}, if the request is authenticated, otherwise {@code false} + */ + protected boolean authenticate( + ContainerRequestContext requestContext, @Nullable String credentials, String scheme) { + if (credentials == null) { + return false; + } + + CallContext.setCurrentContext(callContext); + + Optional principal = authenticator.authenticate(credentials); + if (principal.isEmpty()) { + return false; + } + + LOGGER.debug("Authenticated user: {}", principal.get().getName()); + + AuthenticatedPolarisPrincipal prince = principal.get(); + SecurityContext securityContext = augmentSecurityContext(requestContext, scheme, prince); + requestContext.setSecurityContext(securityContext); + return true; + } + + private static SecurityContext augmentSecurityContext( + ContainerRequestContext requestContext, String scheme, AuthenticatedPolarisPrincipal prince) { + SecurityContext securityContext = requestContext.getSecurityContext(); + boolean secure = securityContext != null && securityContext.isSecure(); + return new SecurityContext() { + @Override + public Principal getUserPrincipal() { + return prince; + } + + @Override + public boolean isUserInRole(String role) { + return true; // TODO: implement role-based access control + } + + @Override + public boolean isSecure() { + return secure; + } + + @Override + public String getAuthenticationScheme() { + return scheme; + } + }; + } +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/throttling/RequestThrottlingErrorResponse.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/catalog/io/QuarkusCatalogIOConfiguration.java similarity index 65% rename from dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/throttling/RequestThrottlingErrorResponse.java rename to dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/catalog/io/QuarkusCatalogIOConfiguration.java index fa07647c1..122442312 100644 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/throttling/RequestThrottlingErrorResponse.java +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/catalog/io/QuarkusCatalogIOConfiguration.java @@ -16,18 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.throttling; +package org.apache.polaris.service.dropwizard.catalog.io; -import com.fasterxml.jackson.annotation.JsonProperty; +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; -/** - * Response object for errors caused by DoS-prevention throttling mechanisms, such as request size - * limits - */ -public record RequestThrottlingErrorResponse( - @JsonProperty("error_type") RequestThrottlingErrorType errorType) { - public enum RequestThrottlingErrorType { - REQUEST_TOO_LARGE, - ; - } +@StaticInitSafe +@ConfigMapping(prefix = "polaris.catalog.io") +public interface QuarkusCatalogIOConfiguration { + + /** + * The type of the catalog IO to use. Must be a registered {@link + * org.apache.polaris.service.catalog.io.FileIOFactory} identifier. + */ + String type(); } diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/CorsConfiguration.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/CorsConfiguration.java deleted file mode 100644 index 9055d73a7..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/CorsConfiguration.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.config; - -import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.List; - -public class CorsConfiguration { - private List allowedOrigins = List.of("*"); - private List allowedTimingOrigins = List.of("*"); - private List allowedMethods = List.of("*"); - private List allowedHeaders = List.of("*"); - private List exposedHeaders = List.of("*"); - private Integer preflightMaxAge = 600; - private String allowCredentials = "true"; - - public List getAllowedOrigins() { - return allowedOrigins; - } - - @JsonProperty("allowed-origins") - public void setAllowedOrigins(List allowedOrigins) { - this.allowedOrigins = allowedOrigins; - } - - public void setAllowedTimingOrigins(List allowedTimingOrigins) { - this.allowedTimingOrigins = allowedTimingOrigins; - } - - @JsonProperty("allowed-timing-origins") - public List getAllowedTimingOrigins() { - return allowedTimingOrigins; - } - - public List getAllowedMethods() { - return allowedMethods; - } - - @JsonProperty("allowed-methods") - public void setAllowedMethods(List allowedMethods) { - this.allowedMethods = allowedMethods; - } - - public List getAllowedHeaders() { - return allowedHeaders; - } - - @JsonProperty("allowed-headers") - public void setAllowedHeaders(List allowedHeaders) { - this.allowedHeaders = allowedHeaders; - } - - public List getExposedHeaders() { - return exposedHeaders; - } - - @JsonProperty("exposed-headers") - public void setExposedHeaders(List exposedHeaders) { - this.exposedHeaders = exposedHeaders; - } - - public Integer getPreflightMaxAge() { - return preflightMaxAge; - } - - @JsonProperty("preflight-max-age") - public void setPreflightMaxAge(Integer preflightMaxAge) { - this.preflightMaxAge = preflightMaxAge; - } - - public String getAllowCredentials() { - return allowCredentials; - } - - @JsonProperty("allowed-credentials") - public void setAllowCredentials(String allowCredentials) { - this.allowCredentials = allowCredentials; - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/PolarisApplicationConfig.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/PolarisApplicationConfig.java deleted file mode 100644 index e5f94e1c4..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/PolarisApplicationConfig.java +++ /dev/null @@ -1,494 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.config; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.google.auth.oauth2.AccessToken; -import com.google.auth.oauth2.GoogleCredentials; -import com.google.common.base.Preconditions; -import io.dropwizard.core.Configuration; -import jakarta.annotation.Nonnull; -import jakarta.annotation.Nullable; -import jakarta.annotation.PostConstruct; -import jakarta.inject.Inject; -import java.io.IOException; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.function.Supplier; -import org.apache.commons.lang3.StringUtils; -import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; -import org.apache.polaris.service.auth.Authenticator; -import org.apache.polaris.service.auth.DecodedToken; -import org.apache.polaris.service.auth.TokenBroker; -import org.apache.polaris.service.auth.TokenBrokerFactory; -import org.apache.polaris.service.auth.TokenResponse; -import org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService; -import org.apache.polaris.service.catalog.io.FileIOFactory; -import org.apache.polaris.service.config.DefaultConfigurationStore; -import org.apache.polaris.service.config.TaskHandlerConfiguration; -import org.apache.polaris.service.context.CallContextResolver; -import org.apache.polaris.service.context.RealmContextResolver; -import org.apache.polaris.service.ratelimiter.RateLimiter; -import org.apache.polaris.service.ratelimiter.TokenBucketFactory; -import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; -import org.apache.polaris.service.types.TokenType; -import org.glassfish.hk2.api.Factory; -import org.glassfish.hk2.api.ServiceLocator; -import org.glassfish.hk2.api.TypeLiteral; -import org.glassfish.hk2.utilities.ServiceLocatorUtilities; -import org.glassfish.hk2.utilities.binding.AbstractBinder; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.services.sts.StsClient; -import software.amazon.awssdk.services.sts.StsClientBuilder; - -/** - * Configuration specific to a Polaris REST Service. Place these entries in a YML file for them to - * be picked up, i.e. `iceberg-rest-server.yml` - */ -public class PolarisApplicationConfig extends Configuration { - /** - * Override the default binding of registered services so that the configured instances are used. - */ - private static final int OVERRIDE_BINDING_RANK = 10; - - private MetaStoreManagerFactory metaStoreManagerFactory; - private String defaultRealm = "default-realm"; - private RealmContextResolver realmContextResolver; - private CallContextResolver callContextResolver; - private Authenticator polarisAuthenticator; - private CorsConfiguration corsConfiguration = new CorsConfiguration(); - private TaskHandlerConfiguration taskHandler = new TaskHandlerConfiguration(); - private Map globalFeatureConfiguration = Map.of(); - private Map> realmConfiguration = Map.of(); - private List defaultRealms; - private String awsAccessKey; - private String awsSecretKey; - private FileIOFactory fileIOFactory; - private RateLimiter rateLimiter; - private TokenBucketFactory tokenBucketFactory; - private TokenBrokerFactory tokenBrokerFactory; - - private AccessToken gcpAccessToken; - - public static final long REQUEST_BODY_BYTES_NO_LIMIT = -1; - private long maxRequestBodyBytes = REQUEST_BODY_BYTES_NO_LIMIT; - - @Inject ServiceLocator serviceLocator; - - @PostConstruct - public void bindToServiceLocator() { - ServiceLocatorUtilities.bind(serviceLocator, binder()); - } - - public ServiceLocator getServiceLocator() { - return serviceLocator; - } - - @Nonnull - public AbstractBinder binder() { - PolarisApplicationConfig config = this; - return new AbstractBinder() { - @Override - protected void configure() { - bindFactory(SupplierFactory.create(serviceLocator, config::getStorageIntegrationProvider)) - .to(PolarisStorageIntegrationProvider.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getMetaStoreManagerFactory)) - .to(MetaStoreManagerFactory.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::createConfigurationStore)) - .to(PolarisConfigurationStore.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getFileIOFactory)) - .to(FileIOFactory.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getPolarisAuthenticator)) - .to(Authenticator.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getTokenBrokerFactory)) - .to(TokenBrokerFactory.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getOauth2Service)) - .to(IcebergRestOAuth2ApiService.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getCallContextResolver)) - .to(CallContextResolver.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getRealmContextResolver)) - .to(RealmContextResolver.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getRateLimiter)) - .to(RateLimiter.class) - .ranked(OVERRIDE_BINDING_RANK); - bindFactory(SupplierFactory.create(serviceLocator, config::getTokenBucketFactory)) - .to(TokenBucketFactory.class) - .ranked(OVERRIDE_BINDING_RANK); - } - }; - } - - /** - * Factory implementation that uses the provided supplier method to retrieve the instance and then - * uses the {@link #serviceLocator} to inject dependencies into the instance. This is necessary - * since the DI framework doesn't automatically inject dependencies into the instances created. - * - * @param - */ - private static final class SupplierFactory implements Factory { - private final ServiceLocator serviceLocator; - private final Supplier supplier; - - private static SupplierFactory create( - ServiceLocator serviceLocator, Supplier supplier) { - return new SupplierFactory<>(serviceLocator, supplier); - } - - private SupplierFactory(ServiceLocator serviceLocator, Supplier supplier) { - this.serviceLocator = serviceLocator; - this.supplier = supplier; - } - - @Override - public T provide() { - T obj = supplier.get(); - serviceLocator.inject(obj); - return obj; - } - - @Override - public void dispose(T instance) {} - } - - public T findService(Class serviceClass) { - return serviceLocator.getService(serviceClass); - } - - public T findService(TypeLiteral serviceClass) { - return serviceLocator.getService(serviceClass.getRawType()); - } - - @JsonProperty("metaStoreManager") - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - } - - private MetaStoreManagerFactory getMetaStoreManagerFactory() { - return metaStoreManagerFactory; - } - - @JsonProperty("io") - @JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "factoryType") - public void setFileIOFactory(FileIOFactory fileIOFactory) { - this.fileIOFactory = fileIOFactory; - } - - private FileIOFactory getFileIOFactory() { - return fileIOFactory; - } - - @JsonProperty("authenticator") - @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "class") - public void setPolarisAuthenticator( - Authenticator polarisAuthenticator) { - this.polarisAuthenticator = polarisAuthenticator; - } - - private Authenticator getPolarisAuthenticator() { - return polarisAuthenticator; - } - - @JsonProperty("tokenBroker") - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") - public void setTokenBrokerFactory(TokenBrokerFactory tokenBrokerFactory) { - this.tokenBrokerFactory = tokenBrokerFactory; - } - - private TokenBrokerFactory getTokenBrokerFactory() { - // return a no-op implementation if none is specified - return Objects.requireNonNullElseGet( - tokenBrokerFactory, - () -> - (rc) -> - new TokenBroker() { - @Override - public boolean supportsGrantType(String grantType) { - return false; - } - - @Override - public boolean supportsRequestedTokenType(TokenType tokenType) { - return false; - } - - @Override - public TokenResponse generateFromClientSecrets( - String clientId, String clientSecret, String grantType, String scope) { - return null; - } - - @Override - public TokenResponse generateFromToken( - TokenType tokenType, String subjectToken, String grantType, String scope) { - return null; - } - - @Override - public DecodedToken verify(String token) { - return null; - } - }); - } - - private RealmContextResolver getRealmContextResolver() { - realmContextResolver.setDefaultRealm(this.defaultRealm); - return realmContextResolver; - } - - @JsonProperty("realmContextResolver") - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") - public void setRealmContextResolver(RealmContextResolver realmContextResolver) { - this.realmContextResolver = realmContextResolver; - } - - private CallContextResolver getCallContextResolver() { - return callContextResolver; - } - - @JsonProperty("callContextResolver") - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") - public void setCallContextResolver(CallContextResolver callContextResolver) { - this.callContextResolver = callContextResolver; - } - - private IcebergRestOAuth2ApiService oauth2Service; - - @JsonProperty("oauth2") - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") - public void setOauth2Service(IcebergRestOAuth2ApiService oauth2Service) { - this.oauth2Service = oauth2Service; - } - - private IcebergRestOAuth2ApiService getOauth2Service() { - return oauth2Service; - } - - public String getDefaultRealm() { - return defaultRealm; - } - - @JsonProperty("defaultRealm") - public void setDefaultRealm(String defaultRealm) { - this.defaultRealm = defaultRealm; - realmContextResolver.setDefaultRealm(defaultRealm); - } - - @JsonProperty("cors") - public CorsConfiguration getCorsConfiguration() { - return corsConfiguration; - } - - @JsonProperty("cors") - public void setCorsConfiguration(CorsConfiguration corsConfiguration) { - this.corsConfiguration = corsConfiguration; - } - - @JsonProperty("rateLimiter") - private RateLimiter getRateLimiter() { - return rateLimiter; - } - - public boolean hasRateLimiter() { - return rateLimiter != null; - } - - @JsonProperty("rateLimiter") - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") - public void setRateLimiter(@Nullable RateLimiter rateLimiter) { - this.rateLimiter = rateLimiter; - } - - @JsonProperty("tokenBucketFactory") - private TokenBucketFactory getTokenBucketFactory() { - return tokenBucketFactory; - } - - @JsonProperty("tokenBucketFactory") - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") - public void setTokenBucketFactory(@Nullable TokenBucketFactory tokenBucketFactory) { - this.tokenBucketFactory = tokenBucketFactory; - } - - public void setTaskHandler(TaskHandlerConfiguration taskHandler) { - this.taskHandler = taskHandler; - } - - public TaskHandlerConfiguration getTaskHandler() { - return taskHandler; - } - - @JsonProperty("featureConfiguration") - public void setFeatureConfiguration(Map featureConfiguration) { - this.globalFeatureConfiguration = featureConfiguration; - } - - @JsonProperty("realmFeatureConfiguration") - public void setRealmFeatureConfiguration(Map> realmConfiguration) { - this.realmConfiguration = realmConfiguration; - } - - @JsonProperty("maxRequestBodyBytes") - public void setMaxRequestBodyBytes(long maxRequestBodyBytes) { - // The underlying library that we use to implement the limit treats all values <= 0 as the - // same, so we block all but -1 to prevent ambiguity. - Preconditions.checkArgument( - maxRequestBodyBytes == -1 || maxRequestBodyBytes > 0, - "maxRequestBodyBytes must be a positive integer or %s to specify no limit.", - REQUEST_BODY_BYTES_NO_LIMIT); - - this.maxRequestBodyBytes = maxRequestBodyBytes; - } - - public long getMaxRequestBodyBytes() { - return maxRequestBodyBytes; - } - - private PolarisConfigurationStore createConfigurationStore() { - return new DefaultConfigurationStore(globalFeatureConfiguration, realmConfiguration); - } - - public List getDefaultRealms() { - return defaultRealms; - } - - private AwsCredentialsProvider credentialsProvider() { - if (StringUtils.isNotBlank(awsAccessKey) && StringUtils.isNotBlank(awsSecretKey)) { - LoggerFactory.getLogger(PolarisApplicationConfig.class) - .warn("Using hard-coded AWS credentials - this is not recommended for production"); - return StaticCredentialsProvider.create( - AwsBasicCredentials.create(awsAccessKey, awsSecretKey)); - } - return null; - } - - public void setAwsAccessKey(String awsAccessKey) { - this.awsAccessKey = awsAccessKey; - } - - public void setAwsSecretKey(String awsSecretKey) { - this.awsSecretKey = awsSecretKey; - } - - public void setDefaultRealms(List defaultRealms) { - this.defaultRealms = defaultRealms; - } - - private PolarisStorageIntegrationProvider storageIntegrationProvider; - - public void setStorageIntegrationProvider( - PolarisStorageIntegrationProvider storageIntegrationProvider) { - this.storageIntegrationProvider = storageIntegrationProvider; - } - - private PolarisStorageIntegrationProvider getStorageIntegrationProvider() { - if (storageIntegrationProvider == null) { - storageIntegrationProvider = - new PolarisStorageIntegrationProviderImpl( - () -> { - StsClientBuilder stsClientBuilder = StsClient.builder(); - AwsCredentialsProvider awsCredentialsProvider = credentialsProvider(); - if (awsCredentialsProvider != null) { - stsClientBuilder.credentialsProvider(awsCredentialsProvider); - } - return stsClientBuilder.build(); - }, - getGcpCredentialsProvider()); - } - return storageIntegrationProvider; - } - - private Supplier getGcpCredentialsProvider() { - return () -> - Optional.ofNullable(gcpAccessToken) - .map(GoogleCredentials::create) - .orElseGet( - () -> { - try { - return GoogleCredentials.getApplicationDefault(); - } catch (IOException e) { - throw new RuntimeException("Failed to get GCP credentials", e); - } - }); - } - - @JsonProperty("gcp_credentials") - void setGcpCredentials(GcpAccessToken token) { - this.gcpAccessToken = - new AccessToken( - token.getAccessToken(), - new Date(System.currentTimeMillis() + token.getExpiresIn() * 1000)); - } - - /** - * A static AccessToken representation used to store a static token and expiration date. This - * should strictly be used for testing. - */ - static class GcpAccessToken { - private String accessToken; - private long expiresIn; - - public GcpAccessToken() {} - - public GcpAccessToken(String accessToken, long expiresIn) { - this.accessToken = accessToken; - this.expiresIn = expiresIn; - } - - public String getAccessToken() { - return accessToken; - } - - @JsonProperty("access_token") - public void setAccessToken(String accessToken) { - this.accessToken = accessToken; - } - - public long getExpiresIn() { - return expiresIn; - } - - @JsonProperty("expires_in") - public void setExpiresIn(long expiresIn) { - this.expiresIn = expiresIn; - } - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/QuarkusFeaturesConfiguration.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/QuarkusFeaturesConfiguration.java new file mode 100644 index 000000000..41616100b --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/QuarkusFeaturesConfiguration.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.config; + +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithParentName; +import java.util.Map; +import org.apache.polaris.service.config.FeaturesConfiguration; + +@ConfigMapping(prefix = "polaris.config") +public interface QuarkusFeaturesConfiguration extends FeaturesConfiguration { + + @Override + Map realmOverrides(); + + interface QuarkusRealmOverrides extends RealmOverrides { + @WithParentName + @Override + Map overrides(); + } +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/QuarkusJacksonConfig.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/QuarkusJacksonConfig.java new file mode 100644 index 000000000..19213d420 --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/QuarkusJacksonConfig.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.config; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import io.quarkus.jackson.ObjectMapperCustomizer; +import io.quarkus.runtime.configuration.MemorySize; +import io.quarkus.runtime.configuration.MemorySizeConverter; +import io.smallrye.config.WithConverter; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import org.apache.iceberg.rest.RESTSerializers; +import org.apache.polaris.service.config.Serializers; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Singleton +public class QuarkusJacksonConfig implements ObjectMapperCustomizer { + + private static final Logger LOGGER = LoggerFactory.getLogger(QuarkusJacksonConfig.class); + + private final long maxBodySize; + + @Inject + public QuarkusJacksonConfig( + @ConfigProperty(name = "quarkus.http.limits.max-body-size") + @WithConverter(MemorySizeConverter.class) + MemorySize maxBodySize) { + this.maxBodySize = maxBodySize.asLongValue(); + } + + @Override + public void customize(ObjectMapper objectMapper) { + objectMapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + objectMapper.setPropertyNamingStrategy(new PropertyNamingStrategies.KebabCaseStrategy()); + RESTSerializers.registerAll(objectMapper); + Serializers.registerSerializers(objectMapper); + objectMapper + .getFactory() + .setStreamReadConstraints( + StreamReadConstraints.builder().maxDocumentLength(maxBodySize).build()); + LOGGER.info("Limiting request body size to {} bytes", maxBodySize); + } +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/QuarkusProducers.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/QuarkusProducers.java new file mode 100644 index 000000000..48794e7bd --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/config/QuarkusProducers.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.config; + +import io.smallrye.common.annotation.Identifier; +import io.smallrye.context.SmallRyeManagedExecutor; +import io.vertx.core.http.HttpServerRequest; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.context.RequestScoped; +import jakarta.enterprise.inject.Any; +import jakarta.enterprise.inject.Disposes; +import jakarta.enterprise.inject.Instance; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Singleton; +import jakarta.ws.rs.core.Context; +import java.time.Clock; +import java.util.HashMap; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.PolarisConfigurationStore; +import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; +import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.auth.PolarisAuthorizerImpl; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.apache.polaris.core.storage.cache.StorageCredentialCache; +import org.apache.polaris.service.auth.Authenticator; +import org.apache.polaris.service.auth.TokenBrokerFactory; +import org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService; +import org.apache.polaris.service.catalog.io.FileIOFactory; +import org.apache.polaris.service.context.RealmContextResolver; +import org.apache.polaris.service.dropwizard.auth.QuarkusAuthenticatorConfiguration; +import org.apache.polaris.service.dropwizard.catalog.io.QuarkusCatalogIOConfiguration; +import org.apache.polaris.service.dropwizard.context.QuarkusContextConfiguration; +import org.apache.polaris.service.dropwizard.persistence.QuarkusPersistenceConfiguration; +import org.apache.polaris.service.dropwizard.ratelimiter.QuarkusRateLimiterConfiguration; +import org.apache.polaris.service.ratelimiter.RateLimiter; +import org.apache.polaris.service.ratelimiter.TokenBucketFactory; +import org.apache.polaris.service.task.TaskHandlerConfiguration; +import org.eclipse.microprofile.context.ManagedExecutor; +import org.eclipse.microprofile.context.ThreadContext; + +public class QuarkusProducers { + + @Produces + @ApplicationScoped // cannot be singleton because it is mocked in tests + public Clock clock() { + return Clock.systemDefaultZone(); + } + + // Polaris core beans - application scope + + @Produces + @ApplicationScoped + public StorageCredentialCache storageCredentialCache() { + return new StorageCredentialCache(); + } + + @Produces + @ApplicationScoped + public PolarisAuthorizer polarisAuthorizer(PolarisConfigurationStore configurationStore) { + return new PolarisAuthorizerImpl(configurationStore); + } + + @Produces + @ApplicationScoped + public PolarisDiagnostics polarisDiagnostics() { + return new PolarisDefaultDiagServiceImpl(); + } + + // Polaris core beans - request scope + + @Produces + @RequestScoped + public RealmContext realmContext( + @Context HttpServerRequest request, RealmContextResolver realmContextResolver) { + return realmContextResolver.resolveRealmContext( + request.absoluteURI(), + request.method().name(), + request.path(), + request.params().entries().stream() + .collect(HashMap::new, (m, e) -> m.put(e.getKey(), e.getValue()), HashMap::putAll), + request.headers().entries().stream() + .collect(HashMap::new, (m, e) -> m.put(e.getKey(), e.getValue()), HashMap::putAll)); + } + + @Produces + @RequestScoped + public PolarisCallContext polarisCallContext( + RealmContext realmContext, + PolarisDiagnostics diagServices, + PolarisConfigurationStore configurationStore, + MetaStoreManagerFactory metaStoreManagerFactory, + Clock clock) { + PolarisMetaStoreSession metaStoreSession = + metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); + return new PolarisCallContext(metaStoreSession, diagServices, configurationStore, clock); + } + + @Produces + @RequestScoped + public CallContext callContext(RealmContext realmContext, PolarisCallContext polarisCallContext) { + return CallContext.of(realmContext, polarisCallContext); + } + + public void closeCallContext(@Disposes CallContext callContext) { + callContext.close(); + } + + // Polaris service beans - selected from @Identifier-annotated beans + + @Produces + public RealmContextResolver realmContextResolver( + QuarkusContextConfiguration config, + @Any Instance realmContextResolvers) { + return realmContextResolvers + .select(Identifier.Literal.of(config.realmContextResolver().type())) + .get(); + } + + @Produces + public FileIOFactory fileIOFactory( + QuarkusCatalogIOConfiguration config, @Any Instance fileIOFactories) { + return fileIOFactories.select(Identifier.Literal.of(config.type())).get(); + } + + @Produces + public MetaStoreManagerFactory metaStoreManagerFactory( + QuarkusPersistenceConfiguration config, + @Any Instance metaStoreManagerFactories) { + return metaStoreManagerFactories.select(Identifier.Literal.of(config.type())).get(); + } + + @Produces + public RateLimiter rateLimiter( + QuarkusRateLimiterConfiguration config, @Any Instance rateLimiters) { + return rateLimiters.select(Identifier.Literal.of(config.type())).get(); + } + + @Produces + public TokenBucketFactory tokenBucketFactory( + QuarkusRateLimiterConfiguration config, + @Any Instance tokenBucketFactories) { + return tokenBucketFactories.select(Identifier.Literal.of(config.tokenBucket().type())).get(); + } + + @Produces + public Authenticator authenticator( + QuarkusAuthenticatorConfiguration config, + @Any Instance> authenticators) { + return authenticators.select(Identifier.Literal.of(config.type())).get(); + } + + @Produces + public IcebergRestOAuth2ApiService icebergRestOAuth2ApiService( + QuarkusAuthenticatorConfiguration config, + @Any Instance services) { + return services.select(Identifier.Literal.of(config.service().type())).get(); + } + + @Produces + public TokenBrokerFactory tokenBrokerFactory( + QuarkusAuthenticatorConfiguration config, + @Any Instance tokenBrokerFactories) { + return tokenBrokerFactories + .select(Identifier.Literal.of(config.tokenBrokerFactory().type())) + .get(); + } + + // other beans + + @Produces + @Singleton + @Identifier("task-executor") + public ManagedExecutor taskExecutor(TaskHandlerConfiguration config) { + return SmallRyeManagedExecutor.builder() + .injectionPointName("task-executor") + .propagated(ThreadContext.ALL_REMAINING) + .maxAsync(config.maxConcurrentTasks()) + .maxQueued(config.maxQueuedTasks()) + .build(); + } + + public void closeTaskExecutor(@Disposes @Identifier("task-executor") ManagedExecutor executor) { + executor.close(); + } +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/context/QuarkusContextConfiguration.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/context/QuarkusContextConfiguration.java new file mode 100644 index 000000000..bcb22b6fd --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/context/QuarkusContextConfiguration.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.context; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import org.apache.polaris.service.context.ContextConfiguration; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.context") +public interface QuarkusContextConfiguration extends ContextConfiguration { + + @Override + QuarkusRealmContextResolverConfiguration realmContextResolver(); + + interface QuarkusRealmContextResolverConfiguration extends RealmContextResolverConfiguration { + + /** + * The type of the realm context resolver. Must be a registered {@link + * org.apache.polaris.service.context.RealmContextResolver} identifier. + */ + String type(); + } +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/context/RealmScopeContext.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/context/RealmScopeContext.java deleted file mode 100644 index f89ea64e7..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/context/RealmScopeContext.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.context; - -import jakarta.inject.Inject; -import jakarta.inject.Singleton; -import java.lang.annotation.Annotation; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.context.RealmScoped; -import org.glassfish.hk2.api.ActiveDescriptor; -import org.glassfish.hk2.api.Context; -import org.glassfish.hk2.api.IterableProvider; -import org.glassfish.hk2.api.ServiceHandle; -import org.glassfish.hk2.api.ServiceLocator; - -@Singleton -public class RealmScopeContext implements Context { - private final Map, Object>> contexts = new ConcurrentHashMap<>(); - - @Inject private ServiceLocator locator; - @Inject private IterableProvider realmContextProvider; - - @Override - public Class getScope() { - return RealmScoped.class; - } - - @SuppressWarnings("unchecked") - @Override - public U findOrCreate(ActiveDescriptor activeDescriptor, ServiceHandle root) { - RealmContext realmContext = realmContextProvider.iterator().next(); - Map, Object> contextMap = - contexts.computeIfAbsent(realmContext.getRealmIdentifier(), k -> new ConcurrentHashMap<>()); - return (U) contextMap.computeIfAbsent(activeDescriptor, k -> activeDescriptor.create(root)); - } - - @Override - public boolean containsKey(ActiveDescriptor descriptor) { - RealmContext realmContext = realmContextProvider.iterator().next(); - Map, Object> contextMap = - contexts.computeIfAbsent(realmContext.getRealmIdentifier(), k -> new HashMap<>()); - return contextMap.containsKey(descriptor); - } - - @Override - public void destroyOne(ActiveDescriptor descriptor) { - RealmContext realmContext = realmContextProvider.iterator().next(); - Map, Object> contextMap = - contexts.computeIfAbsent(realmContext.getRealmIdentifier(), k -> new HashMap<>()); - contextMap.remove(descriptor); - } - - @Override - public boolean supportsNullCreation() { - return false; - } - - @Override - public boolean isActive() { - Optional first = - locator.getAllServices(Context.class).stream() - .filter( - context -> - context - .getScope() - .equals( - realmContextProvider - .getHandle() - .getActiveDescriptor() - .getScopeAnnotation())) - .findFirst(); - return first.map(Context::isActive).orElse(false); - } - - @Override - public void shutdown() { - contexts.clear(); - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/exception/JerseyViolationExceptionMapper.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/exception/JerseyViolationExceptionMapper.java deleted file mode 100644 index 4885b3a54..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/exception/JerseyViolationExceptionMapper.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.exception; - -import io.dropwizard.jersey.validation.JerseyViolationException; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.ext.ExceptionMapper; -import jakarta.ws.rs.ext.Provider; -import org.apache.polaris.service.exception.IcebergConstraintViolationExceptionMapper; - -/** - * Override of the default JerseyViolationExceptionMapper to provide an Iceberg ErrorResponse with - * the exception details. - */ -@Provider -public class JerseyViolationExceptionMapper implements ExceptionMapper { - - private IcebergConstraintViolationExceptionMapper icebergMapper = - new IcebergConstraintViolationExceptionMapper(); - - @Override - public Response toResponse(JerseyViolationException exception) { - return icebergMapper.toResponse(exception); - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/logging/PolarisJsonLayoutFactory.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/logging/PolarisJsonLayoutFactory.java deleted file mode 100644 index 26aec8bcc..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/logging/PolarisJsonLayoutFactory.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.logging; - -import ch.qos.logback.classic.LoggerContext; -import ch.qos.logback.classic.pattern.ExtendedThrowableProxyConverter; -import ch.qos.logback.classic.pattern.RootCauseFirstThrowableProxyConverter; -import ch.qos.logback.classic.pattern.ThrowableHandlingConverter; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.LayoutBase; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonTypeName; -import com.google.common.collect.ImmutableMap; -import io.dropwizard.logging.json.AbstractJsonLayoutBaseFactory; -import io.dropwizard.logging.json.EventAttribute; -import io.dropwizard.logging.json.layout.EventJsonLayout; -import io.dropwizard.logging.json.layout.ExceptionFormat; -import io.dropwizard.logging.json.layout.JsonFormatter; -import io.dropwizard.logging.json.layout.TimestampFormatter; -import java.util.ArrayList; -import java.util.Collections; -import java.util.EnumSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; -import java.util.stream.Collectors; -import org.checkerframework.checker.nullness.qual.Nullable; - -/** - * Basically a direct copy of {@link io.dropwizard.logging.json.EventJsonLayoutBaseFactory} that - * adds support for {@link ILoggingEvent#getKeyValuePairs()} in the output. By default, additional - * key/value pairs are included as the `params` field of the json output, but they can optionally be - * flattened into the log event output. - * - *

To use this appender, change the appender type to `polaris` - * loggers: - * org.apache.iceberg.rest: DEBUG - * org.apache.iceberg.polaris: DEBUG - * appenders: - * - type: console - * threshold: ALL - * layout: - * type: polaris - * flattenKeyValues: false - * includeKeyValues: true - * - */ -@JsonTypeName("polaris") -public class PolarisJsonLayoutFactory extends AbstractJsonLayoutBaseFactory { - private EnumSet includes = - EnumSet.of( - EventAttribute.LEVEL, - EventAttribute.THREAD_NAME, - EventAttribute.MDC, - EventAttribute.MARKER, - EventAttribute.LOGGER_NAME, - EventAttribute.MESSAGE, - EventAttribute.EXCEPTION, - EventAttribute.TIMESTAMP); - - private Set includesMdcKeys = Collections.emptySet(); - private boolean flattenMdc = false; - private boolean includeKeyValues = true; - private boolean flattenKeyValues = false; - - @Nullable private ExceptionFormat exceptionFormat; - - @JsonProperty - public EnumSet getIncludes() { - return includes; - } - - @JsonProperty - public void setIncludes(EnumSet includes) { - this.includes = includes; - } - - @JsonProperty - public Set getIncludesMdcKeys() { - return includesMdcKeys; - } - - @JsonProperty - public void setIncludesMdcKeys(Set includesMdcKeys) { - this.includesMdcKeys = includesMdcKeys; - } - - @JsonProperty - public boolean isFlattenMdc() { - return flattenMdc; - } - - @JsonProperty - public void setFlattenMdc(boolean flattenMdc) { - this.flattenMdc = flattenMdc; - } - - @JsonProperty - public boolean isIncludeKeyValues() { - return includeKeyValues; - } - - @JsonProperty - public void setIncludeKeyValues(boolean includeKeyValues) { - this.includeKeyValues = includeKeyValues; - } - - @JsonProperty - public boolean isFlattenKeyValues() { - return flattenKeyValues; - } - - @JsonProperty - public void setFlattenKeyValues(boolean flattenKeyValues) { - this.flattenKeyValues = flattenKeyValues; - } - - /** - * @since 2.0 - */ - @JsonProperty("exception") - public void setExceptionFormat(ExceptionFormat exceptionFormat) { - this.exceptionFormat = exceptionFormat; - } - - /** - * @since 2.0 - */ - @JsonProperty("exception") - @Nullable - public ExceptionFormat getExceptionFormat() { - return exceptionFormat; - } - - @Override - public LayoutBase build(LoggerContext context, TimeZone timeZone) { - final PolarisJsonLayout jsonLayout = - new PolarisJsonLayout( - createDropwizardJsonFormatter(), - createTimestampFormatter(timeZone), - createThrowableProxyConverter(context), - includes, - getCustomFieldNames(), - getAdditionalFields(), - includesMdcKeys, - flattenMdc, - includeKeyValues, - flattenKeyValues); - jsonLayout.setContext(context); - return jsonLayout; - } - - public static class PolarisJsonLayout extends EventJsonLayout { - private final boolean includeKeyValues; - private final boolean flattenKeyValues; - - public PolarisJsonLayout( - JsonFormatter jsonFormatter, - TimestampFormatter timestampFormatter, - ThrowableHandlingConverter throwableProxyConverter, - Set includes, - Map customFieldNames, - Map additionalFields, - Set includesMdcKeys, - boolean flattenMdc, - boolean includeKeyValues, - boolean flattenKeyValues) { - super( - jsonFormatter, - timestampFormatter, - throwableProxyConverter, - includes, - customFieldNames, - additionalFields, - includesMdcKeys, - flattenMdc); - this.includeKeyValues = includeKeyValues; - this.flattenKeyValues = flattenKeyValues; - } - - @Override - protected Map toJsonMap(ILoggingEvent event) { - Map jsonMap = super.toJsonMap(event); - if (!includeKeyValues) { - return jsonMap; - } - Map keyValueMap = - event.getKeyValuePairs() == null - ? Map.of() - : event.getKeyValuePairs().stream() - .collect(Collectors.toMap(kv -> kv.key, kv -> kv.value)); - ImmutableMap.Builder builder = - ImmutableMap.builder().putAll(jsonMap); - if (flattenKeyValues) { - builder.putAll(keyValueMap); - } else { - builder.put("params", keyValueMap); - } - return builder.build(); - } - } - - protected ThrowableHandlingConverter createThrowableProxyConverter(LoggerContext context) { - if (exceptionFormat == null) { - return new RootCauseFirstThrowableProxyConverter(); - } - - ThrowableHandlingConverter throwableHandlingConverter; - if (exceptionFormat.isRootFirst()) { - throwableHandlingConverter = new RootCauseFirstThrowableProxyConverter(); - } else { - throwableHandlingConverter = new ExtendedThrowableProxyConverter(); - } - - List options = new ArrayList<>(); - // depth must be added first - options.add(exceptionFormat.getDepth()); - options.addAll(exceptionFormat.getEvaluators()); - - throwableHandlingConverter.setOptionList(options); - throwableHandlingConverter.setContext(context); - - return throwableHandlingConverter; - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisRealm.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/logging/QuarkusLoggingConfiguration.java similarity index 63% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisRealm.java rename to dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/logging/QuarkusLoggingConfiguration.java index 88d5696b3..ace4d62e9 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisRealm.java +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/logging/QuarkusLoggingConfiguration.java @@ -16,17 +16,19 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.test; +package org.apache.polaris.service.dropwizard.logging; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import java.util.Map; -/** - * Annotation used to specify where to inject the Polaris test realm identifier. This is provided by - * PolarisConnectionExtension. - */ -@Target({ElementType.PARAMETER}) -@Retention(RetentionPolicy.RUNTIME) -public @interface PolarisRealm {} +@StaticInitSafe +@ConfigMapping(prefix = "polaris.log") +public interface QuarkusLoggingConfiguration { + + /** The name of the header that contains the request ID. */ + String requestIdHeaderName(); + + /** Additional MDC values to include in the log context. */ + Map mdc(); +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/logging/QuarkusLoggingMDCFilter.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/logging/QuarkusLoggingMDCFilter.java new file mode 100644 index 000000000..eebb6b995 --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/logging/QuarkusLoggingMDCFilter.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.logging; + +import io.quarkus.vertx.web.RouteFilter; +import io.vertx.ext.web.RoutingContext; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.apache.polaris.core.context.RealmContext; +import org.slf4j.MDC; + +@ApplicationScoped +public class QuarkusLoggingMDCFilter { + + public static final int PRIORITY = RouteFilter.DEFAULT_PRIORITY + 100; + + private static final String REQUEST_ID_KEY = "requestId"; + private static final String REALM_ID_KEY = "realmId"; + + @Inject RealmContext realmContext; + + @Inject QuarkusLoggingConfiguration loggingConfiguration; + + public static String requestId(RoutingContext rc) { + return rc.get(REQUEST_ID_KEY); + } + + public static String realmId(RoutingContext rc) { + return rc.get(REALM_ID_KEY); + } + + @RouteFilter(value = PRIORITY) + public void applyMDCContext(RoutingContext rc) { + // The request scope is active here, so any MDC values set here will be propagated to + // threads handling the request. + // Also put the MDC values in the request context for use by other filters and handlers + loggingConfiguration.mdc().forEach(MDC::put); + loggingConfiguration.mdc().forEach(rc::put); + var requestId = rc.request().getHeader(loggingConfiguration.requestIdHeaderName()); + if (requestId != null) { + MDC.put(REQUEST_ID_KEY, requestId); + rc.put(REQUEST_ID_KEY, requestId); + } + MDC.put(REALM_ID_KEY, realmContext.getRealmIdentifier()); + rc.put(REALM_ID_KEY, realmContext.getRealmIdentifier()); + // Do not explicitly remove the MDC values from the request context with an end handler, + // as this could remove MDC context still in use in TaskExecutor threads + // rc.addEndHandler( + // (v) -> { + // MDC.remove(REQUEST_ID_MDC_KEY); + // MDC.remove(REALM_ID_MDC_KEY); + // loggingConfiguration.mdc().keySet().forEach(MDC::remove); + // }); + rc.next(); + } +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/QuarkusMeterFilterProducer.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/QuarkusMeterFilterProducer.java new file mode 100644 index 000000000..d49dd40ec --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/QuarkusMeterFilterProducer.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.metrics; + +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.config.MeterFilter; +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import java.util.stream.Collectors; + +public class QuarkusMeterFilterProducer { + + @Inject QuarkusMetricsConfiguration configuration; + + @Produces + @Singleton + public MeterFilter produceGlobalMeterFilter() { + return MeterFilter.commonTags( + this.configuration.tags().entrySet().stream() + .map(e -> Tag.of(e.getKey(), e.getValue())) + .collect(Collectors.toSet())); + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/context/RealmScoped.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/QuarkusMetricsConfiguration.java similarity index 67% rename from polaris-core/src/main/java/org/apache/polaris/core/context/RealmScoped.java rename to dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/QuarkusMetricsConfiguration.java index 9bf456f58..a4c819de8 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/context/RealmScoped.java +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/QuarkusMetricsConfiguration.java @@ -16,16 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.core.context; +package org.apache.polaris.service.dropwizard.metrics; -import jakarta.inject.Scope; -import java.lang.annotation.Documented; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.Target; +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import java.util.Map; -@Scope -@Documented -@Retention(java.lang.annotation.RetentionPolicy.RUNTIME) -@Target({ElementType.TYPE, ElementType.METHOD}) -public @interface RealmScoped {} +@StaticInitSafe +@ConfigMapping(prefix = "polaris.metrics") +public interface QuarkusMetricsConfiguration { + + /** Additional tags to include in the metrics. */ + Map tags(); +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/QuarkusValueExpressionResolver.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/QuarkusValueExpressionResolver.java new file mode 100644 index 000000000..8054bfb8f --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/QuarkusValueExpressionResolver.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.metrics; + +import io.micrometer.common.annotation.ValueExpressionResolver; +import io.micrometer.common.lang.Nullable; +import jakarta.annotation.Nonnull; +import jakarta.enterprise.context.ApplicationScoped; +import org.apache.polaris.core.context.RealmContext; + +@ApplicationScoped +public class QuarkusValueExpressionResolver implements ValueExpressionResolver { + + @Override + public String resolve(@Nonnull String expression, @Nullable Object parameter) { + // TODO maybe replace with CEL of some expression engine and make this more generic + if (parameter instanceof RealmContext realmContext && expression.equals("realmIdentifier")) { + return realmContext.getRealmIdentifier(); + } + return null; + } +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/RealmIdTagContributor.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/RealmIdTagContributor.java new file mode 100644 index 000000000..ad1e96b42 --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/metrics/RealmIdTagContributor.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.metrics; + +import io.micrometer.core.instrument.Tags; +import io.quarkus.micrometer.runtime.HttpServerMetricsTagsContributor; +import io.vertx.core.http.HttpServerRequest; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.util.HashMap; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.service.context.RealmContextResolver; + +@ApplicationScoped +public class RealmIdTagContributor implements HttpServerMetricsTagsContributor { + + public static final String TAG_REALM = "realm_id"; + + @Inject RealmContextResolver realmContextResolver; + + @Override + public Tags contribute(Context context) { + // FIXME request scope does not work here, so we have to resolve the realm context manually + HttpServerRequest request = context.request(); + RealmContext realmContext = resolveRealmContext(request); + return Tags.of(TAG_REALM, realmContext.getRealmIdentifier()); + } + + private RealmContext resolveRealmContext(HttpServerRequest request) { + return realmContextResolver.resolveRealmContext( + request.absoluteURI(), + request.method().name(), + request.path(), + request.params().entries().stream() + .collect(HashMap::new, (m, e) -> m.put(e.getKey(), e.getValue()), HashMap::putAll), + request.headers().entries().stream() + .collect(HashMap::new, (m, e) -> m.put(e.getKey(), e.getValue()), HashMap::putAll)); + } +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/monitor/PolarisMetricRegistry.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/monitor/PolarisMetricRegistry.java deleted file mode 100644 index c34249a85..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/monitor/PolarisMetricRegistry.java +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.monitor; - -import com.google.common.annotations.VisibleForTesting; -import io.micrometer.core.annotation.Timed; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Tag; -import io.micrometer.core.instrument.Timer; -import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; -import io.micrometer.core.instrument.binder.system.ProcessorMetrics; -import jakarta.inject.Inject; -import java.lang.reflect.Method; -import java.util.Collections; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; - -/** - * Wrapper around the Micrometer {@link MeterRegistry} providing additional metric management - * functions for the Polaris application. Implements in-memory caching of timers and counters. - * Records two metrics for each instrument with one tagged by the realm ID (realm-specific metric) - * and one without. The realm-specific metric is suffixed with ".realm". - * - *

Uppercase tag names are now deprecated. Prefer snake_casing instead. Old metrics are emitted - * with both variations but the uppercase versions may eventually be removed. New methods for tag - * emission (those that allow you to pass in an Iterable) only emit the snake_case version. - */ -public class PolarisMetricRegistry { - private final MeterRegistry meterRegistry; - private final ConcurrentMap timers = new ConcurrentHashMap<>(); - private final ConcurrentMap counters = new ConcurrentHashMap<>(); - - /** - * @deprecated See class Javadoc. - */ - @Deprecated public static final String TAG_REALM_DEPRECATED = "REALM_ID"; - - public static final String TAG_REALM = "realm_id"; - - /** - * @deprecated See class Javadoc. - */ - @Deprecated public static final String TAG_RESP_CODE_DEPRECATED = "HTTP_RESPONSE_CODE"; - - public static final String TAG_RESP_CODE = "http_response_code"; - - public static final String SUFFIX_COUNTER = ".count"; - public static final String SUFFIX_ERROR = ".error"; - public static final String SUFFIX_REALM = ".realm"; - - @Inject - public PolarisMetricRegistry(MeterRegistry meterRegistry) { - this.meterRegistry = meterRegistry; - new ClassLoaderMetrics().bindTo(meterRegistry); - new JvmMemoryMetrics().bindTo(meterRegistry); - new JvmGcMetrics().bindTo(meterRegistry); - new ProcessorMetrics().bindTo(meterRegistry); - new JvmThreadMetrics().bindTo(meterRegistry); - } - - public MeterRegistry getMeterRegistry() { - return meterRegistry; - } - - @VisibleForTesting - public void clear() { - meterRegistry.clear(); - counters.clear(); - } - - public void init(Class... classes) { - for (Class clazz : classes) { - Method[] methods = clazz.getDeclaredMethods(); - for (Method method : methods) { - if (method.isAnnotationPresent(Timed.class)) { - Timed timedApi = method.getAnnotation(Timed.class); - String metric = timedApi.value(); - timers.put(metric, Timer.builder(metric).register(meterRegistry)); - counters.put( - metric + SUFFIX_COUNTER, - Counter.builder(metric + SUFFIX_COUNTER).register(meterRegistry)); - - // Error counters contain the HTTP response code in a tag, thus caching them would not be - // meaningful. - Counter.builder(metric + SUFFIX_ERROR) - .tags(TAG_RESP_CODE, "400", TAG_RESP_CODE_DEPRECATED, "400") - .register(meterRegistry); - Counter.builder(metric + SUFFIX_ERROR) - .tags(TAG_RESP_CODE, "500", TAG_RESP_CODE_DEPRECATED, "500") - .register(meterRegistry); - } - } - } - } - - public void recordTimer(String metric, long elapsedTimeMs, String realmId) { - Timer timer = - timers.computeIfAbsent(metric, m -> Timer.builder(metric).register(meterRegistry)); - timer.record(elapsedTimeMs, TimeUnit.MILLISECONDS); - - Timer timerRealm = - timers.computeIfAbsent( - metric + SUFFIX_REALM, - m -> - Timer.builder(metric + SUFFIX_REALM) - .tag(TAG_REALM, realmId) - .tag(TAG_REALM_DEPRECATED, realmId) - .register(meterRegistry)); - timerRealm.record(elapsedTimeMs, TimeUnit.MILLISECONDS); - } - - /** - * Increments metric.count and metric.count.realm. The realmId is tagged on the latter metric. - * Counters are cached. - */ - public void incrementCounter(String metric, String realmId) { - String counterMetric = metric + SUFFIX_COUNTER; - Counter counter = - counters.computeIfAbsent( - counterMetric, m -> Counter.builder(counterMetric).register(meterRegistry)); - counter.increment(); - - Counter counterRealm = - counters.computeIfAbsent( - counterMetric + SUFFIX_REALM, - m -> - Counter.builder(counterMetric + SUFFIX_REALM) - .tag(TAG_REALM, realmId) - .tag(TAG_REALM_DEPRECATED, realmId) - .register(meterRegistry)); - counterRealm.increment(); - } - - /** - * Increments metric.error and metric.error.realm. The realmId is tagged on the latter metric. - * Both metrics are tagged with the statusCode and counters are not cached. - */ - public void incrementErrorCounter(String metric, int statusCode, String realmId) { - String errorMetric = metric + SUFFIX_ERROR; - Counter.builder(errorMetric) - .tag(TAG_RESP_CODE, String.valueOf(statusCode)) - .tag(TAG_RESP_CODE_DEPRECATED, String.valueOf(statusCode)) - .register(meterRegistry) - .increment(); - - Counter.builder(errorMetric + SUFFIX_REALM) - .tag(TAG_RESP_CODE, String.valueOf(statusCode)) - .tag(TAG_RESP_CODE_DEPRECATED, String.valueOf(statusCode)) - .tag(TAG_REALM, realmId) - .tag(TAG_REALM_DEPRECATED, realmId) - .register(meterRegistry) - .increment(); - } - - /** - * Increments metric.count and metric.count.realm. The realmId is tagged on the latter metric. - * Arbitrary tags can be specified and counters are not cached. - */ - public void incrementCounter(String metric, String realmId, Iterable tags) { - Counter.builder(metric + SUFFIX_COUNTER).tags(tags).register(meterRegistry).increment(); - - Counter.builder(metric + SUFFIX_COUNTER + SUFFIX_REALM) - .tags(tags) - .tag(TAG_REALM, realmId) - .register(meterRegistry) - .increment(); - } - - /** - * Increments metric.count and metric.count.realm. The realmId is tagged on the latter metric. - * Arbitrary tags can be specified and counters are not cached. - */ - public void incrementCounter(String metric, String realmId, Tag tag) { - incrementCounter(metric, realmId, Collections.singleton(tag)); - } - - /** - * Increments metric.error and metric.error.realm. The realmId is tagged on the latter metric. - * Arbitrary tags can be specified and counters are not cached. - */ - public void incrementErrorCounter(String metric, String realmId, Iterable tags) { - Counter.builder(metric + SUFFIX_ERROR).tags(tags).register(meterRegistry).increment(); - - Counter.builder(metric + SUFFIX_ERROR + SUFFIX_REALM) - .tags(tags) - .tag(TAG_REALM, realmId) - .register(meterRegistry) - .increment(); - } - - /** - * Increments metric.error and metric.error.realm. The realmId is tagged on the latter metric. - * Arbitrary tags can be specified and counters are not cached. - */ - public void incrementErrorCounter(String metric, String realmId, Tag tag) { - incrementErrorCounter(metric, realmId, Collections.singleton(tag)); - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/persistence/QuarkusPersistenceConfiguration.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/persistence/QuarkusPersistenceConfiguration.java new file mode 100644 index 000000000..cb4fbeeaa --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/persistence/QuarkusPersistenceConfiguration.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.persistence; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.persistence") +public interface QuarkusPersistenceConfiguration { + + /** + * The type of the persistence to use. Must be a registered {@link + * org.apache.polaris.core.persistence.MetaStoreManagerFactory} identifier. + */ + String type(); +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/persistence/cache/EntityCacheFactory.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/persistence/cache/EntityCacheFactory.java deleted file mode 100644 index 0313fd85c..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/persistence/cache/EntityCacheFactory.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.persistence.cache; - -import jakarta.inject.Inject; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmScoped; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.cache.EntityCache; -import org.glassfish.hk2.api.Factory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class EntityCacheFactory implements Factory { - private static Logger LOGGER = LoggerFactory.getLogger(EntityCacheFactory.class); - @Inject PolarisMetaStoreManager metaStoreManager; - - @RealmScoped - @Override - public EntityCache provide() { - LOGGER.debug( - "Creating new EntityCache instance for realm {}", - CallContext.getCurrentContext().getRealmContext().getRealmIdentifier()); - return new EntityCache(metaStoreManager); - } - - @Override - public void dispose(EntityCache instance) { - // no-op - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/ratelimiter/QuarkusRateLimiterConfiguration.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/ratelimiter/QuarkusRateLimiterConfiguration.java new file mode 100644 index 000000000..5dfaad2fc --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/ratelimiter/QuarkusRateLimiterConfiguration.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.ratelimiter; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import org.apache.polaris.service.ratelimiter.RateLimiterConfiguration; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.rate-limiter") +public interface QuarkusRateLimiterConfiguration extends RateLimiterConfiguration { + + /** + * The type of the rate limiter. Must be a registered {@link + * org.apache.polaris.service.ratelimiter.RateLimiter} identifier. + */ + String type(); + + /** The configuration for the token bucket rate limiter. */ + @Override + QuarkusTokenBucketConfiguration tokenBucket(); + + interface QuarkusTokenBucketConfiguration extends TokenBucketConfiguration { + + /** + * The type of the token bucket factory. Must be a registered {@link + * org.apache.polaris.service.ratelimiter.TokenBucketFactory} identifier. + */ + String type(); + } +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/storage/QuarkusStorageConfiguration.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/storage/QuarkusStorageConfiguration.java new file mode 100644 index 000000000..f4078518b --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/storage/QuarkusStorageConfiguration.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.storage; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithName; +import java.time.Duration; +import java.util.*; +import org.apache.polaris.service.storage.StorageConfiguration; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.storage") +public interface QuarkusStorageConfiguration extends StorageConfiguration { + + @WithName("aws.access-key") + @Override + Optional awsAccessKey(); + + @WithName("aws.secret-key") + @Override + Optional awsSecretKey(); + + @WithName("gcp.token") + @Override + Optional gcpAccessToken(); + + @WithName("gcp.lifespan") + @Override + Optional gcpAccessTokenLifespan(); +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/task/QuarkusTaskExecutorImpl.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/task/QuarkusTaskExecutorImpl.java new file mode 100644 index 000000000..0ec96ee07 --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/task/QuarkusTaskExecutorImpl.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.task; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.quarkus.runtime.Startup; +import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.util.concurrent.ExecutorService; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.service.dropwizard.tracing.QuarkusTracingFilter; +import org.apache.polaris.service.task.TaskExecutorImpl; +import org.apache.polaris.service.task.TaskFileIOSupplier; + +@ApplicationScoped +public class QuarkusTaskExecutorImpl extends TaskExecutorImpl { + + private final Tracer tracer; + + public QuarkusTaskExecutorImpl() { + this(null, null, null, null); + } + + @Inject + public QuarkusTaskExecutorImpl( + @Identifier("task-executor") ExecutorService executorService, + MetaStoreManagerFactory metaStoreManagerFactory, + TaskFileIOSupplier fileIOSupplier, + Tracer tracer) { + super(executorService, metaStoreManagerFactory, fileIOSupplier); + this.tracer = tracer; + } + + @Startup + @Override + public void init() { + super.init(); + } + + @Override + protected void handleTask(long taskEntityId, CallContext callContext, int attempt) { + Span span = + tracer + .spanBuilder("polaris.task") + .setParent(Context.current()) + .setAttribute( + QuarkusTracingFilter.REALM_ID_ATTRIBUTE, + callContext.getRealmContext().getRealmIdentifier()) + .setAttribute("polaris.task.entity.id", taskEntityId) + .setAttribute("polaris.task.attempt", attempt) + .startSpan(); + try (Scope ignored = span.makeCurrent()) { + super.handleTask(taskEntityId, callContext, attempt); + } finally { + span.end(); + } + } +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/task/QuarkusTaskHandlerConfiguration.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/task/QuarkusTaskHandlerConfiguration.java new file mode 100644 index 000000000..75c43f6e6 --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/task/QuarkusTaskHandlerConfiguration.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.task; + +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithDefault; +import org.apache.polaris.service.task.TaskHandlerConfiguration; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.tasks") +public interface QuarkusTaskHandlerConfiguration extends TaskHandlerConfiguration { + + @WithDefault("-1") + @Override + int maxConcurrentTasks(); + + @WithDefault("-1") + @Override + int maxQueuedTasks(); +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/throttling/StreamReadConstraintsExceptionMapper.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/throttling/StreamReadConstraintsExceptionMapper.java deleted file mode 100644 index d29f2de5d..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/throttling/StreamReadConstraintsExceptionMapper.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.throttling; - -import static org.apache.polaris.service.dropwizard.throttling.RequestThrottlingErrorResponse.RequestThrottlingErrorType.REQUEST_TOO_LARGE; - -import com.fasterxml.jackson.core.exc.StreamConstraintsException; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.ext.ExceptionMapper; - -/** - * Handles exceptions during the request that are a result of stream constraints such as the request - * being too large - */ -public class StreamReadConstraintsExceptionMapper - implements ExceptionMapper { - - @Override - public Response toResponse(StreamConstraintsException exception) { - return Response.status(Response.Status.BAD_REQUEST) - .type(MediaType.APPLICATION_JSON_TYPE) - .entity(new RequestThrottlingErrorResponse(REQUEST_TOO_LARGE)) - .build(); - } -} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/HeadersMapAccessor.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/HeadersMapAccessor.java deleted file mode 100644 index f6e52ac55..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/HeadersMapAccessor.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.tracing; - -import io.opentelemetry.context.propagation.TextMapGetter; -import io.opentelemetry.context.propagation.TextMapSetter; -import jakarta.annotation.Nullable; -import jakarta.servlet.http.HttpServletRequest; -import java.net.http.HttpRequest; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.stream.StreamSupport; - -/** - * Implementation of {@link TextMapSetter} and {@link TextMapGetter} that can handle an {@link - * HttpServletRequest} for extracting headers and sets headers on a {@link HttpRequest.Builder}. - */ -public class HeadersMapAccessor - implements TextMapSetter, TextMapGetter { - @Override - public Iterable keys(HttpServletRequest carrier) { - return StreamSupport.stream( - Spliterators.spliteratorUnknownSize( - carrier.getHeaderNames().asIterator(), Spliterator.IMMUTABLE), - false) - .toList(); - } - - @Nullable - @Override - public String get(@Nullable HttpServletRequest carrier, String key) { - return carrier == null ? null : carrier.getHeader(key); - } - - @Override - public void set(@Nullable HttpRequest.Builder carrier, String key, String value) { - if (carrier != null) { - carrier.setHeader(key, value); - } - } -} diff --git a/dropwizard/service/src/test/resources/META-INF/hk2-locator/default b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/QuarkusTelemetryConfiguration.java similarity index 66% rename from dropwizard/service/src/test/resources/META-INF/hk2-locator/default rename to dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/QuarkusTelemetryConfiguration.java index 92b32e5af..c3915ebcd 100644 --- a/dropwizard/service/src/test/resources/META-INF/hk2-locator/default +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/QuarkusTelemetryConfiguration.java @@ -16,12 +16,16 @@ * specific language governing permissions and limitations * under the License. */ -[org.apache.polaris.service.dropwizard.catalog.io.TestFileIOFactory]S -contract={org.apache.polaris.service.catalog.io.FileIOFactory} -name=test -qualifier={io.smallrye.common.annotation.Identifier} +package org.apache.polaris.service.dropwizard.tracing; -[org.apache.polaris.service.dropwizard.ratelimiter.MockTokenBucketFactory]S -contract={org.apache.polaris.service.ratelimiter.TokenBucketFactory} -name=mock -qualifier={io.smallrye.common.annotation.Identifier} +import io.quarkus.runtime.annotations.StaticInitSafe; +import io.smallrye.config.ConfigMapping; +import java.util.Map; + +@StaticInitSafe +@ConfigMapping(prefix = "polaris.otel") +public interface QuarkusTelemetryConfiguration { + + /** Additional attributes to include in generated spans. */ + Map spanAttributes(); +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/QuarkusTracingFilter.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/QuarkusTracingFilter.java new file mode 100644 index 000000000..960128871 --- /dev/null +++ b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/QuarkusTracingFilter.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.tracing; + +import io.opentelemetry.api.trace.Span; +import io.quarkus.vertx.web.RouteFilter; +import io.vertx.ext.web.RoutingContext; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.apache.polaris.service.dropwizard.logging.QuarkusLoggingMDCFilter; +import org.eclipse.microprofile.config.inject.ConfigProperty; + +@ApplicationScoped +public class QuarkusTracingFilter { + + public static final String REQUEST_ID_ATTRIBUTE = "polaris.request.id"; + public static final String REALM_ID_ATTRIBUTE = "polaris.realm"; + + @ConfigProperty(name = "quarkus.otel.sdk.disabled") + boolean sdkDisabled; + + @Inject QuarkusTelemetryConfiguration telemetryConfiguration; + + @RouteFilter(QuarkusLoggingMDCFilter.PRIORITY - 1) + public void applySpanAttributes(RoutingContext rc) { + if (!sdkDisabled) { + Span span = Span.current(); + telemetryConfiguration.spanAttributes().forEach(span::setAttribute); + String requestId = QuarkusLoggingMDCFilter.requestId(rc); + String realmId = QuarkusLoggingMDCFilter.realmId(rc); + if (requestId != null) { + span.setAttribute(REQUEST_ID_ATTRIBUTE, requestId); + } + span.setAttribute(REALM_ID_ATTRIBUTE, realmId); + } + rc.next(); + } +} diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/TracingFilter.java b/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/TracingFilter.java deleted file mode 100644 index 30f5919a5..000000000 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/tracing/TracingFilter.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.tracing; - -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.Scope; -import io.opentelemetry.semconv.HttpAttributes; -import io.opentelemetry.semconv.ServerAttributes; -import io.opentelemetry.semconv.UrlAttributes; -import jakarta.annotation.Priority; -import jakarta.inject.Inject; -import jakarta.servlet.Filter; -import jakarta.servlet.FilterChain; -import jakarta.servlet.ServletException; -import jakarta.servlet.ServletRequest; -import jakarta.servlet.ServletResponse; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.ws.rs.Priorities; -import java.io.IOException; -import org.apache.polaris.core.context.CallContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; - -/** - * Servlet {@link Filter} that starts an OpenTracing {@link Span}, propagating the calling context - * from HTTP headers, if present. "spanId" and "traceId" are added to the logging MDC so that all - * logs recorded in the request will contain the current span and trace id. Downstream HTTP calls - * should use the OpenTelemetry {@link io.opentelemetry.context.propagation.ContextPropagators} to - * include the current trace id in the request headers. - */ -@Priority(Priorities.AUTHENTICATION - 1) -public class TracingFilter implements Filter { - private static final Logger LOGGER = LoggerFactory.getLogger(TracingFilter.class); - private final OpenTelemetry openTelemetry; - - @Inject - public TracingFilter(OpenTelemetry openTelemetry) { - this.openTelemetry = openTelemetry; - } - - @Override - public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) - throws IOException, ServletException { - HttpServletRequest httpRequest = (HttpServletRequest) request; - Context extractedContext = - openTelemetry - .getPropagators() - .getTextMapPropagator() - .extract(Context.current(), httpRequest, new HeadersMapAccessor()); - try (Scope scope = extractedContext.makeCurrent()) { - Tracer tracer = openTelemetry.getTracer(httpRequest.getPathInfo()); - Span span = - tracer - .spanBuilder(httpRequest.getMethod() + " " + httpRequest.getPathInfo()) - .setSpanKind(SpanKind.SERVER) - .setAttribute( - "realm", CallContext.getCurrentContext().getRealmContext().getRealmIdentifier()) - .startSpan(); - - try (Scope ignored = span.makeCurrent(); - MDC.MDCCloseable spanId = MDC.putCloseable("spanId", span.getSpanContext().getSpanId()); - MDC.MDCCloseable traceId = - MDC.putCloseable("traceId", span.getSpanContext().getTraceId())) { - LOGGER - .atInfo() - .addKeyValue("spanId", span.getSpanContext().getSpanId()) - .addKeyValue("traceId", span.getSpanContext().getTraceId()) - .addKeyValue("parentContext", extractedContext) - .log("Started span with parent"); - span.setAttribute(HttpAttributes.HTTP_REQUEST_METHOD, httpRequest.getMethod()); - span.setAttribute(ServerAttributes.SERVER_ADDRESS, httpRequest.getServerName()); - span.setAttribute(UrlAttributes.URL_SCHEME, httpRequest.getScheme()); - span.setAttribute(UrlAttributes.URL_PATH, httpRequest.getPathInfo()); - - chain.doFilter(request, response); - } finally { - span.end(); - } - } - } -} diff --git a/dropwizard/service/src/main/resources/META-INF/hk2-locator/default b/dropwizard/service/src/main/resources/META-INF/hk2-locator/default deleted file mode 100644 index 1771a6ad8..000000000 --- a/dropwizard/service/src/main/resources/META-INF/hk2-locator/default +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -[org.apache.polaris.service.auth.DefaultPolarisAuthenticator]S -contract={org.apache.polaris.service.auth.Authenticator} -name=default -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.auth.TestInlineBearerTokenPolarisAuthenticator]S -contract={org.apache.polaris.service.auth.Authenticator} -name=test -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory]S -contract={org.apache.polaris.core.persistence.MetaStoreManagerFactory} -name=in-memory -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.auth.DefaultOAuth2ApiService]S -contract={org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService} -name=default -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.auth.TestOAuth2ApiService]S -contract={org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService} -name=test -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.auth.JWTSymmetricKeyFactory]S -contract={org.apache.polaris.service.auth.TokenBrokerFactory} -name=symmetric-key -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.auth.JWTRSAKeyPairFactory]S -contract={org.apache.polaris.service.auth.TokenBrokerFactory} -name=rsa-key-pair -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.context.DefaultRealmContextResolver]S -contract={org.apache.polaris.service.context.RealmContextResolver} -name=default -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.context.DefaultCallContextResolver]S -contract={org.apache.polaris.service.context.CallContextResolver} -name=default -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.core.storage.PolarisStorageIntegrationProvider]S -contract={org.apache.polaris.core.storage.PolarisStorageIntegrationProvider} -name=default -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.catalog.io.DefaultFileIOFactory]S -contract={org.apache.polaris.service.catalog.io.FileIOFactory} -name=default -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.catalog.io.WasbTranslatingFileIOFactory]S -contract={org.apache.polaris.service.catalog.io.FileIOFactory} -name=wasb -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.ratelimiter.NoOpRateLimiter]S -contract={org.apache.polaris.service.ratelimiter.RateLimiter} -name=no-op -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.ratelimiter.RealmTokenBucketRateLimiter]S -contract={org.apache.polaris.service.ratelimiter.RateLimiter} -name=realm-token-bucket -qualifier={io.smallrye.common.annotation.Identifier} - -[org.apache.polaris.service.ratelimiter.DefaultTokenBucketFactory]S -contract={org.apache.polaris.service.ratelimiter.TokenBucketFactory} -name=default -qualifier={io.smallrye.common.annotation.Identifier} - diff --git a/dropwizard/service/src/main/resources/META-INF/services/io.dropwizard.logging.common.layout.DiscoverableLayoutFactory b/dropwizard/service/src/main/resources/META-INF/services/io.smallrye.config.ConfigSourceInterceptor similarity index 91% rename from dropwizard/service/src/main/resources/META-INF/services/io.dropwizard.logging.common.layout.DiscoverableLayoutFactory rename to dropwizard/service/src/main/resources/META-INF/services/io.smallrye.config.ConfigSourceInterceptor index 2e1cd0210..b92210f72 100644 --- a/dropwizard/service/src/main/resources/META-INF/services/io.dropwizard.logging.common.layout.DiscoverableLayoutFactory +++ b/dropwizard/service/src/main/resources/META-INF/services/io.smallrye.config.ConfigSourceInterceptor @@ -17,4 +17,4 @@ # under the License. # -org.apache.polaris.service.dropwizard.logging.PolarisJsonLayoutFactory \ No newline at end of file +io.smallrye.config.LoggingConfigSourceInterceptor \ No newline at end of file diff --git a/dropwizard/service/src/main/resources/application.properties b/dropwizard/service/src/main/resources/application.properties new file mode 100644 index 000000000..cfb1c0ddc --- /dev/null +++ b/dropwizard/service/src/main/resources/application.properties @@ -0,0 +1,160 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +quarkus.application.name=Apache Polaris (incubating) +quarkus.banner.path=/org/apache/polaris/service/banner.txt + +quarkus.config.mapping.validate-unknown=true + +quarkus.container-image.build=false +quarkus.container-image.push=false +# quarkus.container-image.registry=ghcr.io +quarkus.container-image.group=apache +quarkus.container-image.name=polaris +# quarkus.container-image.tag=latest + +# multi-architecture builds require pushing to a registry +# quarkus.docker.buildx.platform=linux/amd64,linux/arm64 + +quarkus.http.auth.basic=false +quarkus.http.access-log.enabled=true +# quarkus.http.access-log.pattern=common +quarkus.http.enable-compression=true +quarkus.http.enable-decompression=true +quarkus.http.body.handle-file-uploads=false +quarkus.http.limits.max-body-size=10240K +quarkus.http.compress-media-types=application/json,text/html,text/plain + +quarkus.http.cors.origins=http://localhost:8080 +quarkus.http.cors.methods=PATCH, POST, DELETE, GET, PUT +quarkus.http.cors.headers=* +quarkus.http.cors.exposed-headers=* +quarkus.http.cors.access-control-max-age=PT10M +quarkus.http.cors.access-control-allow-credentials=true + +quarkus.http.port=8181 +quarkus.http.test-port=0 + +quarkus.log.level=INFO +quarkus.log.console.enable=true +quarkus.log.console.level=ALL +quarkus.log.console.json=false +quarkus.log.console.format=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] [%X{requestId},%X{realmId}] [%X{traceId},%X{parentId},%X{spanId},%X{sampled}] (%t) %s%e%n +quarkus.log.file.enable=true +quarkus.log.file.level=ALL +quarkus.log.file.json=false +quarkus.log.file.format=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] [%X{requestId},%X{realmId}] [%X{traceId},%X{parentId},%X{spanId},%X{sampled}] (%t) %s%e%n +quarkus.log.file.path=./logs/polaris.log +quarkus.log.file.rotation.file-suffix=.yyyy-MM-dd.gz +quarkus.log.file.rotation.max-file-size=10M +quarkus.log.file.rotation.max-backup-index=14 +quarkus.log.category."org.apache.polaris".level=INFO +quarkus.log.category."org.apache.iceberg.rest".level=INFO +quarkus.log.category."io.smallrye.config".level=INFO + +quarkus.management.enabled=true +quarkus.management.port=8182 +quarkus.management.test-port=0 + +quarkus.micrometer.enabled=true +quarkus.micrometer.export.prometheus.enabled=true + +quarkus.otel.enabled=true +quarkus.otel.sdk.disabled=false +# quarkus.otel.exporter.otlp.endpoint=http://otlp-collector:4317 +# quarkus.otel.resource.attributes=deployment.environment=dev +# quarkus.otel.service.name=polaris +# quarkus.otel.traces.sampler=parentbased_always_on +# quarkus.otel.traces.sampler.arg=1.0d + +polaris.context.realm-context-resolver.default-realm=default-realm +polaris.context.realm-context-resolver.type=default + +polaris.config.defaults.ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING=false +polaris.config.defaults.SUPPORTED_CATALOG_STORAGE_TYPES=["S3","GCS","AZURE","FILE"] +# realm overrides +# polaris.config.realm-overrides."my-realm".INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST=true +# polaris.config.realm-overrides."my-realm".SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION=true + +polaris.persistence.type=in-memory + +polaris.catalog.io.type=default + +polaris.log.request-id-header-name=request_id +# polaris.log.mdc.aid=polaris +# polaris.log.mdc.sid=polaris-service + +polaris.metrics.tags.application=Polaris +# polaris.metrics.tags.service=polaris +# polaris.metrics.tags.environment=prod +# polaris.metrics.tags.region=us-west-2 + +# polaris.otel.span-attributes.service=polaris +# polaris.otel.span-attributes.environment=prod +# polaris.otel.span-attributes.region=us-west-2 + +# polaris.tasks.max-concurrent-tasks=100 +# polaris.tasks.max-queued-tasks=1000 + +polaris.rate-limiter.type=default +polaris.rate-limiter.token-bucket.type=default +polaris.rate-limiter.token-bucket.requests-per-second=9999 +polaris.rate-limiter.token-bucket.window=PT10S + +polaris.authentication.type=default +polaris.authentication.oauth2-service.type=default +polaris.authentication.token-broker-factory.type=rsa-key-pair +polaris.authentication.token-broker-factory.max-token-generation=PT1H +# polaris.authentication.token-broker-factory.rsa-key-pair.public-key-file=/tmp/public.key +# polaris.authentication.token-broker-factory.rsa-key-pair.private-key-file=/tmp/private.key +# polaris.authentication.token-broker-factory.symmetric-key.secret=secret +# polaris.authentication.token-broker-factory.symmetric-key.file=/tmp/symmetric.key + +# If the following properties are unset, default credentials will be used +# polaris.storage.aws.access-key=accessKey +# polaris.storage.aws.secret-key=secretKey +# polaris.storage.gcp.token=token +# polaris.storage.gcp.lifespan=PT1H + +%test.quarkus.log.file.enable=false +%test.quarkus.log.category."org.apache.polaris".level=INFO +%test.quarkus.log.category."org.apache.iceberg.rest".level=INFO +%test.quarkus.log.category."org.apache.iceberg.rest.RESTSessionCatalog".level=ERROR +%test.quarkus.log.category."org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl".level=ERROR +%test.quarkus.log.category."org.apache.polaris.service.context.DefaultRealmContextResolver".level=ERROR +%test.quarkus.log.category."org.apache.polaris.service.catalog.PolarisCatalogHandlerWrapper".level=ERROR +%test.quarkus.log.category."org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl".level=ERROR +%test.quarkus.http.limits.max-body-size=1000000 +%test.quarkus.otel.sdk.disabled=true +%test.polaris.authentication.token-broker-factory.type=symmetric-key +%test.polaris.authentication.token-broker-factory.symmetric-key.secret=polaris +%test.polaris.config.defaults.ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING=false +%test.polaris.config.defaults.ALLOW_EXTERNAL_METADATA_FILE_LOCATION=false +%test.polaris.config.defaults.ALLOW_OVERLAPPING_CATALOG_URLS=true +%test.polaris.config.defaults.ALLOW_SPECIFYING_FILE_IO_IMPL=true +%test.polaris.config.defaults.ALLOW_WILDCARD_LOCATION=true +%test.polaris.config.defaults.ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING=true +%test.polaris.config.defaults.INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST=true +%test.polaris.config.defaults.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION=true +%test.polaris.config.defaults.SUPPORTED_CATALOG_STORAGE_TYPES=["FILE","S3","GCS","AZURE"] +%test.polaris.context.realm-context-resolver.default-realm=POLARIS +%test.polaris.storage.aws.access-key=accessKey +%test.polaris.storage.aws.secret-key=secretKey +%test.polaris.storage.gcp.token=token +%test.polaris.storage.gcp.lifespan=PT1H diff --git a/dropwizard/service/src/main/resources/log4j.properties b/dropwizard/service/src/main/resources/log4j.properties deleted file mode 100644 index 37c1696bd..000000000 --- a/dropwizard/service/src/main/resources/log4j.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -log4j.rootLogger=INFO, stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.Target=System.out -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd'T'HH:mm:ss.SSS} %-5p [%c] - %m%n diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationConfigurationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationConfigurationTest.java deleted file mode 100644 index 112b121c3..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationConfigurationTest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard; - -import static org.assertj.core.api.Assertions.assertThat; - -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.extension.persistence.impl.eclipselink.EclipseLinkPolarisMetaStoreManagerFactory; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; - -@ExtendWith(DropwizardExtensionsSupport.class) -public class PolarisApplicationConfigurationTest { - - public static final String CONFIG_PATH = - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"); - // Bind to random ports to support parallelism - public static final ConfigOverride RANDOM_APP_PORT = - ConfigOverride.config("server.applicationConnectors[0].port", "0"); - public static final ConfigOverride RANDOM_ADMIN_PORT = - ConfigOverride.config("server.adminConnectors[0].port", "0"); - - @Nested - class DefaultMetastore { - private final DropwizardAppExtension app = - new DropwizardAppExtension<>( - PolarisApplication.class, CONFIG_PATH, RANDOM_APP_PORT, RANDOM_ADMIN_PORT); - - @Test - void testMetastoreType() { - assertThat(app.getConfiguration().findService(MetaStoreManagerFactory.class)) - .isInstanceOf(InMemoryPolarisMetaStoreManagerFactory.class); - } - } - - @Nested - class EclipseLinkMetastore { - private final DropwizardAppExtension app = - new DropwizardAppExtension<>( - PolarisApplication.class, - CONFIG_PATH, - RANDOM_APP_PORT, - RANDOM_ADMIN_PORT, - ConfigOverride.config("metaStoreManager.type", "eclipse-link"), - ConfigOverride.config("metaStoreManager.persistence-unit", "test-unit"), - ConfigOverride.config("metaStoreManager.conf-file", "/test-conf-file")); - - @Test - void testMetastoreType() { - assertThat(app.getConfiguration().findService(MetaStoreManagerFactory.class)) - .isInstanceOf(EclipseLinkPolarisMetaStoreManagerFactory.class) - .extracting("persistenceUnitName", "confFile") - .containsExactly("test-unit", "/test-conf-file"); - } - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java index af2013fb0..2bbd87201 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java @@ -18,29 +18,26 @@ */ package org.apache.polaris.service.dropwizard; +import static org.apache.polaris.service.auth.BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL; import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.apache.polaris.service.dropwizard.throttling.RequestThrottlingErrorResponse.RequestThrottlingErrorType.REQUEST_TOO_LARGE; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import jakarta.ws.rs.ProcessingException; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientBuilder; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; -import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; -import java.util.function.Supplier; -import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; import org.apache.iceberg.PartitionData; @@ -65,7 +62,6 @@ import org.apache.iceberg.rest.RESTClient; import org.apache.iceberg.rest.RESTSessionCatalog; import org.apache.iceberg.rest.auth.AuthConfig; -import org.apache.iceberg.rest.auth.ImmutableAuthConfig; import org.apache.iceberg.rest.auth.OAuth2Properties; import org.apache.iceberg.rest.auth.OAuth2Util; import org.apache.iceberg.types.Types; @@ -81,128 +77,91 @@ import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestHelper; +import org.apache.polaris.service.dropwizard.test.TestEnvironment; import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.apache.polaris.service.dropwizard.throttling.RequestThrottlingErrorResponse; import org.assertj.core.api.Assertions; import org.assertj.core.api.InstanceOfAssertFactories; +import org.eclipse.microprofile.config.inject.ConfigProperty; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; - -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) + +@QuarkusTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@ExtendWith(TestEnvironmentExtension.class) public class PolarisApplicationIntegrationTest { - @TempDir private static Path tempDir; - private static final Supplier CURRENT_LOG = - () -> tempDir.resolve("application.log").toString(); private static final Logger LOGGER = LoggerFactory.getLogger(PolarisApplicationIntegrationTest.class); public static final String PRINCIPAL_ROLE_NAME = "admin"; - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0"), // Bind to random port to support parallelism - ConfigOverride.config("logging.appenders[1].type", "file"), - ConfigOverride.config("logging.appenders[1].currentLogFilename", CURRENT_LOG)); - - private static String userToken; - private static SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials; - private static Path testDir; - private static String realm; - @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken userToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, - @PolarisRealm String polarisRealm) - throws IOException { - realm = polarisRealm; + @Inject PolarisIntegrationTestHelper helper; - assertThat(new File(CURRENT_LOG.get())) - .exists() - .content() - .contains("PolarisApplication: Server started successfully"); + @ConfigProperty(name = "quarkus.http.limits.max-body-size") + long maxBodySize; - testDir = Path.of("build/test_data/iceberg/" + realm); - FileUtils.deleteQuietly(testDir.toFile()); - Files.createDirectories(testDir); - PolarisApplicationIntegrationTest.userToken = userToken.token(); - PolarisApplicationIntegrationTest.snowmanCredentials = snowmanCredentials; + private TestEnvironment testEnv; + private PolarisIntegrationTestFixture fixture; + @BeforeAll + public void createFixture(TestEnvironment testEnv, TestInfo testInfo) { + this.testEnv = testEnv; + fixture = helper.createFixture(testEnv, testInfo); PrincipalRole principalRole = new PrincipalRole(PRINCIPAL_ROLE_NAME); try (Response createPrResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) + fixture + .client + .target(String.format("%s/api/management/v1/principal-roles", testEnv.baseUri())) .request("application/json") - .header("Authorization", "Bearer " + userToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(principalRole))) { assertThat(createPrResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response assignPrResponse = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/principals/%s/principal-roles", - EXT.getLocalPort(), snowmanCredentials.identifier().principalName())) + "%s/api/management/v1/principals/%s/principal-roles", + testEnv.baseUri(), fixture.snowmanCredentials.identifier().principalName())) .request("application/json") - .header("Authorization", "Bearer " + PolarisApplicationIntegrationTest.userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .put(Entity.json(principalRole))) { assertThat(assignPrResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - - assertZeroErrorsInApplicationLog(); } @AfterAll - public static void deletePrincipalRole() { - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles/%s", - EXT.getLocalPort(), PRINCIPAL_ROLE_NAME)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .delete() - .close(); - } - - private static void assertZeroErrorsInApplicationLog() { - assertThat(new File(CURRENT_LOG.get())) - .exists() - .content() - .hasSizeGreaterThan(0) - .doesNotContain("ERROR", "FATAL"); + public void destroyFixture() { + if (fixture != null) { + fixture + .client + .target( + String.format( + "%s/api/management/v1/principal-roles/%s", + testEnv.baseUri(), PRINCIPAL_ROLE_NAME)) + .request("application/json") + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) + .delete() + .close(); + fixture.destroy(); + } } /** @@ -212,7 +171,7 @@ private static void assertZeroErrorsInApplicationLog() { * @param testInfo */ @BeforeEach - public void before(TestInfo testInfo) { + public void createTestCatalog(TestInfo testInfo) { testInfo .getTestMethod() .ifPresent( @@ -223,7 +182,7 @@ public void before(TestInfo testInfo) { }); } - private static void createCatalog( + private void createCatalog( String catalogName, Catalog.TypeEnum catalogType, String principalRoleName) { createCatalog( catalogName, @@ -238,7 +197,7 @@ private static void createCatalog( "s3://my-bucket/path/to/data"); } - private static void createCatalog( + private void createCatalog( String catalogName, Catalog.TypeEnum catalogType, String principalRoleName, @@ -265,39 +224,41 @@ private static void createCatalog( .setStorageConfigInfo(storageConfig) .build(); try (Response response = - EXT.client() - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) + fixture + .client + .target(String.format("%s/api/management/v1/catalogs", testEnv.baseUri())) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(catalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s", - EXT.getLocalPort(), + "%s/api/management/v1/catalogs/%s/catalog-roles/%s", + testEnv.baseUri(), catalogName, PolarisEntityConstants.getNameOfCatalogAdminRole())) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); CatalogRole catalogRole = response.readEntity(CatalogRole.class); try (Response assignResponse = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/principal-roles/%s/catalog-roles/%s", - EXT.getLocalPort(), principalRoleName, catalogName)) + "%s/api/management/v1/principal-roles/%s/catalog-roles/%s", + testEnv.baseUri(), principalRoleName, catalogName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .put(Entity.json(catalogRole))) { assertThat(assignResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -305,21 +266,21 @@ private static void createCatalog( } } - private static RESTSessionCatalog newSessionCatalog(String catalog) { + private RESTSessionCatalog newSessionCatalog(String catalog) { RESTSessionCatalog sessionCatalog = new RESTSessionCatalog(); sessionCatalog.initialize( "polaris_catalog_test", Map.of( "uri", - "http://localhost:" + EXT.getLocalPort() + "/api/catalog", + testEnv.baseUri() + "/api/catalog", OAuth2Properties.CREDENTIAL, - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(), + fixture.snowmanCredentials.clientId() + ":" + fixture.snowmanCredentials.clientSecret(), OAuth2Properties.SCOPE, - BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL, + PRINCIPAL_ROLE_ALL, "warehouse", catalog, "header." + REALM_PROPERTY_KEY, - realm)); + fixture.realm)); return sessionCatalog; } @@ -508,16 +469,17 @@ public void testIcebergCreateTablesWithWritePathBlocked(TestInfo testInfo) throw } @Test - public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws IOException { + public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo, @TempDir Path tempDir) + throws IOException { String catalogName = testInfo.getTestMethod().get().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME, FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath())) + .setAllowedLocations(List.of("file://" + tempDir.toFile().getAbsolutePath())) .build(), - "file://" + testDir.toFile().getAbsolutePath()); + "file://" + tempDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -526,7 +488,7 @@ public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table"); String location = "file://" - + testDir.toFile().getAbsolutePath() + + tempDir.toFile().getAbsolutePath() + "/" + testInfo.getTestMethod().get().getName(); String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; @@ -555,16 +517,17 @@ public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws } @Test - public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IOException { + public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo, @TempDir Path tempDir) + throws IOException { String catalogName = testInfo.getTestMethod().get().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME, FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath())) + .setAllowedLocations(List.of("file://" + tempDir.toFile().getAbsolutePath())) .build(), - "file://" + testDir.toFile().getAbsolutePath()); + "file://" + tempDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -573,7 +536,7 @@ public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IO TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table"); String location = "file://" - + testDir.toFile().getAbsolutePath() + + tempDir.toFile().getAbsolutePath() + "/" + testInfo.getTestMethod().get().getName(); String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; @@ -608,16 +571,17 @@ public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IO } @Test - public void testIcebergDropTableInExternalCatalog(TestInfo testInfo) throws IOException { + public void testIcebergDropTableInExternalCatalog(TestInfo testInfo, @TempDir Path tempDir) + throws IOException { String catalogName = testInfo.getTestMethod().get().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME, FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath())) + .setAllowedLocations(List.of("file://" + tempDir.toFile().getAbsolutePath())) .build(), - "file://" + testDir.toFile().getAbsolutePath()); + "file://" + tempDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -626,7 +590,7 @@ public void testIcebergDropTableInExternalCatalog(TestInfo testInfo) throws IOEx TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table"); String location = "file://" - + testDir.toFile().getAbsolutePath() + + tempDir.toFile().getAbsolutePath() + "/" + testInfo.getTestMethod().get().getName(); String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; @@ -663,15 +627,17 @@ public void testWarehouseNotSpecified() throws IOException { "polaris_catalog_test", Map.of( "uri", - "http://localhost:" + EXT.getLocalPort() + "/api/catalog", + testEnv.baseUri() + "/api/catalog", OAuth2Properties.CREDENTIAL, - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(), + fixture.snowmanCredentials.clientId() + + ":" + + fixture.snowmanCredentials.clientSecret(), OAuth2Properties.SCOPE, - BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL, + PRINCIPAL_ROLE_ALL, "warehouse", emptyEnvironmentVariable, "header." + REALM_PROPERTY_KEY, - realm))) + fixture.realm))) .isInstanceOf(BadRequestException.class) .hasMessage("Malformed request: Please specify a warehouse"); } @@ -679,91 +645,93 @@ public void testWarehouseNotSpecified() throws IOException { @Test public void testRequestHeaderTooLarge() { - Invocation.Builder request = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) - .request("application/json"); - - // The default limit is 8KiB and each of these headers is at least 8 bytes, so 1500 definitely - // exceeds the limit - for (int i = 0; i < 1500; i++) { - request = request.header("header" + i, "" + i); - } + try (Client client = ClientBuilder.newClient()) { + Invocation.Builder request = + client + .target(String.format("%s/api/management/v1/principal-roles", testEnv.baseUri())) + .request("application/json"); + + // The default limit is 8KiB and each of these headers is at least 8 bytes, so 1500 definitely + // exceeds the limit + for (int i = 0; i < 1500; i++) { + request = request.header("header" + i, "" + i); + } - try { - try (Response response = - request - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(new PrincipalRole("r")))) { - assertThat(response) - .returns( - Response.Status.REQUEST_HEADER_FIELDS_TOO_LARGE.getStatusCode(), - Response::getStatus); + try { + try (Response response = + request + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) + .post(Entity.json(new PrincipalRole("r")))) { + assertThat(response) + .returns( + Response.Status.REQUEST_HEADER_FIELDS_TOO_LARGE.getStatusCode(), + Response::getStatus); + } + } catch (ProcessingException e) { + // In some runtime environments the request above will return a 431 but in others it'll + // result + // in a ProcessingException from the socket being closed. The test asserts that one of those + // things happens. + assertThat(e).hasMessageContaining("Connection was closed"); } - } catch (ProcessingException e) { - // In some runtime environments the request above will return a 431 but in others it'll result - // in a ProcessingException from the socket being closed. The test asserts that one of those - // things happens. } } @Test public void testRequestBodyTooLarge() { - // The size is set to be higher than the limit in polaris-server-integrationtest.yml - Entity largeRequest = Entity.json(new PrincipalRole("r".repeat(1000001))); + Entity largeRequest = + Entity.json(new PrincipalRole("r".repeat((int) (maxBodySize + 1)))); - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(largeRequest)) { - assertThat(response) - .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) - .matches( - r -> - r.readEntity(RequestThrottlingErrorResponse.class) - .errorType() - .equals(REQUEST_TOO_LARGE)); + try (Client client = ClientBuilder.newClient()) { + try (Response response = + client + .target(String.format("%s/api/management/v1/principal-roles", testEnv.baseUri())) + .request("application/json") + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) + .post(largeRequest)) { + assertThat(response) + .returns(Response.Status.REQUEST_ENTITY_TOO_LARGE.getStatusCode(), Response::getStatus); + } catch (ProcessingException e) { + // In some runtime environments the request above will return a 431 but in others it'll + // result + // in a ProcessingException from the socket being closed. The test asserts that one of those + // things happens. + assertThat(e).hasMessageContaining("Connection was closed"); + } } } @Test public void testRefreshToken() throws IOException { - String path = - String.format("http://localhost:%d/api/catalog/v1/oauth/tokens", EXT.getLocalPort()); + String path = String.format("%s/api/catalog/v1/oauth/tokens", testEnv.baseUri()); try (RESTClient client = - HTTPClient.builder(ImmutableMap.of()) - .withHeader(REALM_PROPERTY_KEY, realm) + HTTPClient.builder(Map.of()) + .withHeader(REALM_PROPERTY_KEY, fixture.realm) .uri(path) .build()) { String credentialString = - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(); - var authConfig = - AuthConfig.builder().credential(credentialString).scope("PRINCIPAL_ROLE:ALL").build(); - ImmutableAuthConfig configSpy = spy(authConfig); - when(configSpy.expiresAtMillis()).thenReturn(0L); - assertThat(configSpy.expiresAtMillis()).isEqualTo(0L); - when(configSpy.oauth2ServerUri()).thenReturn(path); - - var parentSession = new OAuth2Util.AuthSession(Map.of(), configSpy); + fixture.snowmanCredentials.clientId() + ":" + fixture.snowmanCredentials.clientSecret(); + AuthConfig configMock = mock(AuthConfig.class); + when(configMock.credential()).thenReturn(credentialString); + when(configMock.scope()).thenReturn(PRINCIPAL_ROLE_ALL); + when(configMock.expiresAtMillis()).thenReturn(0L); + when(configMock.oauth2ServerUri()).thenReturn(path); + + var parentSession = new OAuth2Util.AuthSession(Map.of(), configMock); var session = - OAuth2Util.AuthSession.fromAccessToken(client, null, userToken, 0L, parentSession); + OAuth2Util.AuthSession.fromAccessToken( + client, null, fixture.adminToken, 0L, parentSession); OAuth2Util.AuthSession sessionSpy = spy(session); when(sessionSpy.expiresAtMillis()).thenReturn(0L); assertThat(sessionSpy.expiresAtMillis()).isEqualTo(0L); - assertThat(sessionSpy.token()).isEqualTo(userToken); + assertThat(sessionSpy.token()).isEqualTo(fixture.adminToken); sessionSpy.refresh(client); assertThat(sessionSpy.credential()).isNotNull(); - assertThat(sessionSpy.credential()).isNotEqualTo(userToken); + assertThat(sessionSpy.credential()).isNotEqualTo(fixture.adminToken); } } } diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/TestServices.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/TestServices.java index c9dacaf45..faeb42963 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/TestServices.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/TestServices.java @@ -37,7 +37,6 @@ import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; -import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.service.admin.PolarisServiceImpl; import org.apache.polaris.service.admin.api.PolarisCatalogsApi; import org.apache.polaris.service.catalog.IcebergCatalogAdapter; @@ -59,6 +58,7 @@ public record TestServices( PolarisCatalogsApi catalogsApi, RealmContext realmContext, SecurityContext securityContext) { + private static final RealmContext testRealm = () -> "test-realm"; public static TestServices inMemory(Map config) { @@ -71,17 +71,27 @@ public static TestServices inMemory(FileIOFactory ioFactory) { public static TestServices inMemory(FileIOFactory ioFactory, Map config) { InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - metaStoreManagerFactory.setStorageIntegrationProvider( - new PolarisStorageIntegrationProviderImpl( - Mockito::mock, () -> GoogleCredentials.create(new AccessToken("abc", new Date())))); + new InMemoryPolarisMetaStoreManagerFactory( + new PolarisStorageIntegrationProviderImpl( + Mockito::mock, () -> GoogleCredentials.create(new AccessToken("abc", new Date())))); PolarisMetaStoreManager metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(testRealm); - EntityCache cache = new EntityCache(metaStoreManager); + PolarisMetaStoreSession session = + metaStoreManagerFactory.getOrCreateSessionSupplier(testRealm).get(); + + PolarisCallContext context = + new PolarisCallContext( + session, + Mockito.mock(PolarisDiagnostics.class), + new DefaultConfigurationStore(config), + Clock.systemDefaultZone()); + + CallContext callContext = CallContext.of(testRealm, context); + RealmEntityManagerFactory realmEntityManagerFactory = - new RealmEntityManagerFactory(metaStoreManagerFactory, () -> cache) {}; + new RealmEntityManagerFactory(metaStoreManagerFactory) {}; CallContextCatalogFactory callContextFactory = new PolarisCallContextCatalogFactory( realmEntityManagerFactory, @@ -91,17 +101,13 @@ public static TestServices inMemory(FileIOFactory ioFactory, Map PolarisAuthorizer authorizer = Mockito.mock(PolarisAuthorizer.class); IcebergRestCatalogApiService service = new IcebergCatalogAdapter( - callContextFactory, realmEntityManagerFactory, metaStoreManagerFactory, authorizer); + callContext, + callContextFactory, + realmEntityManagerFactory, + metaStoreManagerFactory, + authorizer); IcebergRestCatalogApi restApi = new IcebergRestCatalogApi(service); - PolarisMetaStoreSession session = - metaStoreManagerFactory.getOrCreateSessionSupplier(testRealm).get(); - PolarisCallContext context = - new PolarisCallContext( - session, - Mockito.mock(PolarisDiagnostics.class), - new DefaultConfigurationStore(config), - Clock.systemDefaultZone()); PolarisMetaStoreManager.CreatePrincipalResult createdPrincipal = metaStoreManager.createPrincipal( context, @@ -140,7 +146,8 @@ public String getAuthenticationScheme() { PolarisCatalogsApi catalogsApi = new PolarisCatalogsApi( - new PolarisServiceImpl(realmEntityManagerFactory, metaStoreManagerFactory, authorizer)); + new PolarisServiceImpl( + realmEntityManagerFactory, metaStoreManagerFactory, authorizer, callContext)); CallContext.setCurrentContext(CallContext.of(testRealm, context)); return new TestServices(restApi, catalogsApi, testRealm, securityContext); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/TimedApplicationEventListenerTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/TimedApplicationEventListenerTest.java index 76c40cfe2..1783cbd63 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/TimedApplicationEventListenerTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/TimedApplicationEventListenerTest.java @@ -19,205 +19,125 @@ package org.apache.polaris.service.dropwizard; import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.apache.polaris.service.dropwizard.TimedApplicationEventListener.SINGLETON_METRIC_NAME; -import static org.apache.polaris.service.dropwizard.TimedApplicationEventListener.TAG_API_NAME; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.SUFFIX_COUNTER; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.SUFFIX_ERROR; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.SUFFIX_REALM; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.TAG_REALM; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.TAG_REALM_DEPRECATED; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.TAG_RESP_CODE; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.TAG_RESP_CODE_DEPRECATED; - -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; -import io.micrometer.core.annotation.Timed; -import io.micrometer.core.instrument.Tag; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.type; + +import io.micrometer.core.instrument.MeterRegistry; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import org.apache.polaris.service.admin.api.PolarisPrincipalsApi; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; +import java.util.Map; +import org.apache.polaris.service.dropwizard.TimedApplicationEventListenerTest.Profile; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestHelper; +import org.apache.polaris.service.dropwizard.test.TestEnvironment; import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; import org.apache.polaris.service.dropwizard.test.TestMetricsUtil; +import org.hawkular.agent.prometheus.types.MetricFamily; +import org.hawkular.agent.prometheus.types.Summary; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) +@QuarkusTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@ExtendWith(TestEnvironmentExtension.class) +@TestProfile(Profile.class) public class TimedApplicationEventListenerTest { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + return Map.of("polaris.metrics.tags.environment", "prod"); + } + } private static final int ERROR_CODE = Response.Status.NOT_FOUND.getStatusCode(); private static final String ENDPOINT = "api/management/v1/principals"; - private static final String API_ANNOTATION = - Arrays.stream(PolarisPrincipalsApi.class.getMethods()) - .filter(m -> m.getName().contains("getPrincipal")) - .findFirst() - .orElseThrow() - .getAnnotation(Timed.class) - .value(); + private static final String METRIC_NAME = "polaris_principals_getPrincipal_seconds"; + + @Inject PolarisIntegrationTestHelper helper; + @Inject MeterRegistry registry; - private static PolarisConnectionExtension.PolarisToken userToken; - private static SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials; - private static String realm; + private TestEnvironment testEnv; + private PolarisIntegrationTestFixture fixture; @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken userToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm) - throws IOException { - TimedApplicationEventListenerTest.userToken = userToken; - TimedApplicationEventListenerTest.snowmanCredentials = snowmanCredentials; - TimedApplicationEventListenerTest.realm = realm; + public void createFixture(TestEnvironment testEnv, TestInfo testInfo) { + this.testEnv = testEnv; + fixture = helper.createFixture(testEnv, testInfo); } @BeforeEach public void clearMetrics() { - getPolarisMetricRegistry().clear(); + registry.clear(); } @Test public void testMetricsEmittedOnSuccessfulRequest() { sendSuccessfulRequest(); - Assertions.assertTrue(getPerApiMetricCount() > 0); - Assertions.assertTrue(getPerApiRealmMetricCount() > 0); - Assertions.assertTrue(getCommonMetricCount() > 0); - Assertions.assertTrue(getCommonRealmMetricCount() > 0); - Assertions.assertEquals(0, getPerApiMetricErrorCount()); - Assertions.assertEquals(0, getPerApiRealmMetricErrorCount()); - Assertions.assertEquals(0, getCommonMetricErrorCount()); - Assertions.assertEquals(0, getCommonRealmMetricErrorCount()); + Map allMetrics = + TestMetricsUtil.fetchMetrics(fixture.client, testEnv.baseManagementUri()); + assertThat(allMetrics).containsKey(METRIC_NAME); + assertThat(allMetrics.get(METRIC_NAME).getMetrics()) + .satisfiesOnlyOnce( + metric -> { + assertThat(metric.getLabels()) + .contains( + Map.entry("application", "Polaris"), + Map.entry("environment", "prod"), + Map.entry("realm_id", fixture.realm), + Map.entry( + "class", "org.apache.polaris.service.admin.api.PolarisPrincipalsApi"), + Map.entry("exception", "none"), + Map.entry("method", "getPrincipal")); + assertThat(metric) + .asInstanceOf(type(Summary.class)) + .extracting(Summary::getSampleCount) + .isEqualTo(1L); + }); } @Test public void testMetricsEmittedOnFailedRequest() { sendFailingRequest(); - Assertions.assertTrue(getPerApiMetricCount() > 0); - Assertions.assertTrue(getPerApiRealmMetricCount() > 0); - Assertions.assertTrue(getCommonMetricCount() > 0); - Assertions.assertTrue(getCommonRealmMetricCount() > 0); - Assertions.assertTrue(getPerApiMetricErrorCount() > 0); - Assertions.assertTrue(getPerApiRealmMetricErrorCount() > 0); - Assertions.assertTrue(getCommonMetricErrorCount() > 0); - Assertions.assertTrue(getCommonRealmMetricErrorCount() > 0); - } - - private PolarisMetricRegistry getPolarisMetricRegistry() { - TimedApplicationEventListener listener = - (TimedApplicationEventListener) - EXT.getEnvironment().jersey().getResourceConfig().getSingletons().stream() - .filter( - s -> - TimedApplicationEventListener.class - .getName() - .equals(s.getClass().getName())) - .findAny() - .orElseThrow(); - return listener.getMetricRegistry(); - } - - private double getPerApiMetricCount() { - return TestMetricsUtil.getTotalCounter( - EXT, API_ANNOTATION + SUFFIX_COUNTER, Collections.emptyList()); - } - - @SuppressWarnings("deprecation") - private double getPerApiRealmMetricCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - API_ANNOTATION + SUFFIX_COUNTER + SUFFIX_REALM, - List.of(Tag.of(TAG_REALM, realm), Tag.of(TAG_REALM_DEPRECATED, realm))); - } - - @SuppressWarnings("deprecation") - private double getPerApiMetricErrorCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - API_ANNOTATION + SUFFIX_ERROR, - List.of( - Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE)), - Tag.of(TAG_RESP_CODE_DEPRECATED, String.valueOf(ERROR_CODE)))); - } - - @SuppressWarnings("deprecation") - private double getPerApiRealmMetricErrorCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - API_ANNOTATION + SUFFIX_ERROR + SUFFIX_REALM, - List.of( - Tag.of(TAG_REALM, realm), - Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE)), - Tag.of(TAG_REALM_DEPRECATED, realm), - Tag.of(TAG_RESP_CODE_DEPRECATED, String.valueOf(ERROR_CODE)))); - } - - private double getCommonMetricCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - SINGLETON_METRIC_NAME + SUFFIX_COUNTER, - Collections.singleton(Tag.of(TAG_API_NAME, API_ANNOTATION))); - } - - private double getCommonRealmMetricCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - SINGLETON_METRIC_NAME + SUFFIX_COUNTER + SUFFIX_REALM, - List.of(Tag.of(TAG_API_NAME, API_ANNOTATION), Tag.of(TAG_REALM, realm))); - } - - private double getCommonMetricErrorCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - SINGLETON_METRIC_NAME + SUFFIX_ERROR, - List.of( - Tag.of(TAG_API_NAME, API_ANNOTATION), - Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE)))); - } - - private double getCommonRealmMetricErrorCount() { - return TestMetricsUtil.getTotalCounter( - EXT, - SINGLETON_METRIC_NAME + SUFFIX_ERROR + SUFFIX_REALM, - List.of( - Tag.of(TAG_API_NAME, API_ANNOTATION), - Tag.of(TAG_REALM, realm), - Tag.of(TAG_RESP_CODE, String.valueOf(ERROR_CODE)))); + Map allMetrics = + TestMetricsUtil.fetchMetrics(fixture.client, testEnv.baseManagementUri()); + assertThat(allMetrics).containsKey(METRIC_NAME); + assertThat(allMetrics.get(METRIC_NAME).getMetrics()) + .satisfiesOnlyOnce( + metric -> { + assertThat(metric.getLabels()) + .contains( + Map.entry("application", "Polaris"), + Map.entry("environment", "prod"), + Map.entry("realm_id", fixture.realm), + Map.entry( + "class", "org.apache.polaris.service.admin.api.PolarisPrincipalsApi"), + Map.entry("exception", "NotFoundException"), + Map.entry("method", "getPrincipal")); + assertThat(metric) + .asInstanceOf(type(Summary.class)) + .extracting(Summary::getSampleCount) + .isEqualTo(1L); + }); } private int sendRequest(String principalName) { try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/%s/%s", EXT.getLocalPort(), ENDPOINT, principalName)) + fixture + .client + .target(String.format("%s/%s/%s", testEnv.baseUri(), ENDPOINT, principalName)) .request("application/json") - .header("Authorization", "Bearer " + userToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { return response.getStatus(); } @@ -226,7 +146,7 @@ private int sendRequest(String principalName) { private void sendSuccessfulRequest() { Assertions.assertEquals( Response.Status.OK.getStatusCode(), - sendRequest(snowmanCredentials.identifier().principalName())); + sendRequest(fixture.snowmanCredentials.identifier().principalName())); } private void sendFailingRequest() { diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisAdminServiceAuthzTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisAdminServiceAuthzTest.java index 292b9bc5a..03873c2d4 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisAdminServiceAuthzTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisAdminServiceAuthzTest.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.service.dropwizard.admin; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; import java.util.List; import java.util.Map; import java.util.Set; @@ -36,6 +38,8 @@ import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; +@QuarkusTest +@TestProfile(PolarisAuthzTestBase.Profile.class) public class PolarisAdminServiceAuthzTest extends PolarisAuthzTestBase { private PolarisAdminService newTestAdminService() { return newTestAdminService(Set.of()); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisAuthzTestBase.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisAuthzTestBase.java index 58a2e6ee1..ba43131f5 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisAuthzTestBase.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisAuthzTestBase.java @@ -23,8 +23,13 @@ import com.google.auth.oauth2.AccessToken; import com.google.auth.oauth2.GoogleCredentials; import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusMock; +import io.quarkus.test.junit.QuarkusTestProfile; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Alternative; +import jakarta.inject.Inject; import java.io.IOException; import java.time.Clock; import java.util.Date; @@ -42,7 +47,6 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; @@ -62,27 +66,39 @@ import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.PrincipalRoleEntity; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; -import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.BasePolarisCatalog; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; +import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.config.DefaultConfigurationStore; import org.apache.polaris.service.config.RealmEntityManagerFactory; +import org.apache.polaris.service.context.CallContextCatalogFactory; import org.apache.polaris.service.context.PolarisCallContextCatalogFactory; import org.apache.polaris.service.dropwizard.catalog.PolarisPassthroughResolutionView; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; +import org.apache.polaris.service.task.TaskExecutor; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; import org.mockito.Mockito; /** Base class for shared test setup logic used by various Polaris authz-related tests. */ public abstract class PolarisAuthzTestBase { + + public static class Profile implements QuarkusTestProfile { + + @Override + public Set> getEnabledAlternatives() { + return Set.of(TestPolarisCallContextCatalogFactory.class); + } + } + protected static final String CATALOG_NAME = "polaris-catalog"; protected static final String PRINCIPAL_NAME = "snowman"; @@ -139,6 +155,12 @@ public abstract class PolarisAuthzTestBase { PolarisConfiguration.ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING.key, true))); + @Inject protected MetaStoreManagerFactory managerFactory; + @Inject protected RealmEntityManagerFactory realmEntityManagerFactory; + @Inject protected CallContextCatalogFactory callContextCatalogFactory; + @Inject protected PolarisDiagnostics diagServices; + @Inject protected Clock clock; + protected BasePolarisCatalog baseCatalog; protected PolarisAdminService adminService; protected PolarisEntityManager entityManager; @@ -147,38 +169,39 @@ public abstract class PolarisAuthzTestBase { protected PrincipalEntity principalEntity; protected CallContext callContext; protected AuthenticatedPolarisPrincipal authenticatedRoot; - protected InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory; - @BeforeEach - @SuppressWarnings("unchecked") - public void before() { - PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); - metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); - metaStoreManagerFactory.setStorageIntegrationProvider( + private PolarisCallContext polarisContext; + + @BeforeAll + public static void setUpMocks() { + PolarisStorageIntegrationProviderImpl mock = new PolarisStorageIntegrationProviderImpl( - Mockito::mock, () -> GoogleCredentials.create(new AccessToken("abc", new Date())))); - RealmContext realmContext = () -> "realm"; - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); + Mockito::mock, () -> GoogleCredentials.create(new AccessToken("abc", new Date()))); + QuarkusMock.installMockForType(mock, PolarisStorageIntegrationProviderImpl.class); + } + + @BeforeEach + public void before(TestInfo testInfo) { + RealmContext realmContext = testInfo::getDisplayName; + metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); Map configMap = Map.of( "ALLOW_SPECIFYING_FILE_IO_IMPL", true, "ALLOW_EXTERNAL_METADATA_FILE_LOCATION", true); - PolarisCallContext polarisContext = + polarisContext = new PolarisCallContext( - metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(), + managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, new PolarisConfigurationStore() { @Override public @Nullable T getConfiguration(PolarisCallContext ctx, String configName) { - return (T) configMap.get(configName); + @SuppressWarnings("unchecked") + var r = (T) configMap.get(configName); + return r; } }, - Clock.systemDefaultZone()); - this.entityManager = - new PolarisEntityManager( - metaStoreManager, new StorageCredentialCache(), new EntityCache(metaStoreManager)); - this.metaStoreManager = metaStoreManager; + clock); + this.entityManager = realmEntityManagerFactory.getOrCreateEntityManager(realmContext); callContext = CallContext.of(realmContext, polarisContext); CallContext.setCurrentContext(callContext); @@ -295,13 +318,17 @@ public void before() { @AfterEach public void after() { - if (this.baseCatalog != null) { - try { - this.baseCatalog.close(); - this.baseCatalog = null; - } catch (IOException e) { - throw new RuntimeException(e); + try { + if (this.baseCatalog != null) { + try { + this.baseCatalog.close(); + this.baseCatalog = null; + } catch (IOException e) { + throw new RuntimeException(e); + } } + } finally { + metaStoreManager.purge(polarisContext); } } @@ -368,18 +395,22 @@ private void initBaseCatalog() { CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); } - public class TestPolarisCallContextCatalogFactory extends PolarisCallContextCatalogFactory { + @Alternative + @ApplicationScoped + public static class TestPolarisCallContextCatalogFactory + extends PolarisCallContextCatalogFactory { + public TestPolarisCallContextCatalogFactory() { - super( - new RealmEntityManagerFactory() { - @Override - public PolarisEntityManager getOrCreateEntityManager(RealmContext realmContext) { - return entityManager; - } - }, - metaStoreManagerFactory, - Mockito.mock(), - new DefaultFileIOFactory()); + super(null, null, null, null); + } + + @Inject + public TestPolarisCallContextCatalogFactory( + RealmEntityManagerFactory entityManagerFactory, + MetaStoreManagerFactory metaStoreManagerFactory, + TaskExecutor taskExecutor, + FileIOFactory fileIOFactory) { + super(entityManagerFactory, metaStoreManagerFactory, taskExecutor, fileIOFactory); } @Override diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisRealmEntityCacheTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisRealmEntityCacheTest.java deleted file mode 100644 index c1a082533..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisRealmEntityCacheTest.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.admin; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.assertj.core.api.Assertions.assertThat; - -import io.dropwizard.core.setup.Environment; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; -import jakarta.inject.Inject; -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.client.Invocation; -import jakarta.ws.rs.core.Feature; -import jakarta.ws.rs.core.FeatureContext; -import jakarta.ws.rs.core.Response; -import java.io.IOException; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.CatalogProperties; -import org.apache.polaris.core.admin.model.CatalogRole; -import org.apache.polaris.core.admin.model.CreateCatalogRequest; -import org.apache.polaris.core.admin.model.CreateCatalogRoleRequest; -import org.apache.polaris.core.admin.model.PolarisCatalog; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.PolarisBaseEntity; -import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.persistence.cache.EntityCache; -import org.apache.polaris.core.persistence.cache.EntityCacheByNameKey; -import org.apache.polaris.core.persistence.cache.EntityCacheEntry; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.persistence.cache.EntityCacheFactory; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.glassfish.hk2.api.ServiceLocator; -import org.glassfish.jersey.process.internal.RequestScope; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - * tests around the {@link EntityCacheFactory} and ensuring that the {@link - * org.apache.polaris.core.persistence.cache.EntityCache} is managed per realm. - */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) -public class PolarisRealmEntityCacheTest { - private static ServiceLocatorAccessor accessor = new ServiceLocatorAccessor(); - - /** - * Injectable {@link Feature} that allows us to access the {@link ServiceLocator} from the jersey - * resource configuration. - */ - private static final class ServiceLocatorAccessor implements Feature { - @Inject ServiceLocator serviceLocator; - - @Override - public boolean configure(FeatureContext context) { - return true; - } - - public ServiceLocator getServiceLocator() { - return serviceLocator; - } - } - - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config("server.adminConnectors[0].port", "0")) - .addListener( - new DropwizardAppExtension.ServiceListener() { - @Override - public void onRun( - PolarisApplicationConfig configuration, - Environment environment, - DropwizardAppExtension rule) - throws Exception { - environment.jersey().register(accessor); - } - }); - private static String userToken; - private static String realm; - - @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken adminToken, @PolarisRealm String polarisRealm) - throws IOException { - userToken = adminToken.token(); - realm = polarisRealm; - - // Set up test location - PolarisConnectionExtension.createTestDir(realm); - } - - @Test - public void testRealmEntityCacheEquality() { - ServiceLocator serviceLocator = accessor.getServiceLocator(); - RequestScope requestScope = serviceLocator.getService(RequestScope.class); - EntityCache cache1; - // check that multiple calls to the serviceLocator return the same instance of the EntityCache - // within the same call context - try (CallContext ctx = - CallContext.setCurrentContext( - CallContext.of(() -> realm, new PolarisCallContext(null, null)))) { - cache1 = requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); - EntityCache cache2 = - requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); - assertThat(cache1).isSameAs(cache2); - } - - // in a new call context with a different realm, the EntityCache should be different - try (CallContext ctx = - CallContext.setCurrentContext( - CallContext.of(() -> "anotherrealm", new PolarisCallContext(null, null)))) { - EntityCache cache2 = - requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); - assertThat(cache1).isNotSameAs(cache2); - } - - // but if we start a new call context with the original realm, we'll get the same EntityCache - // instance - try (CallContext ctx = - CallContext.setCurrentContext( - CallContext.of(() -> realm, new PolarisCallContext(null, null)))) { - EntityCache cache2 = - requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); - assertThat(cache1).isSameAs(cache2); - } - } - - @Test - public void testCacheForRealm() { - // create a catalog - String catalogName = "mycachecatalog"; - ServiceLocator serviceLocator = accessor.getServiceLocator(); - RequestScope requestScope = serviceLocator.getService(RequestScope.class); - listCatalogs(); - - // check for the catalog - it should not exist - // the service_admin role, however, should exist - try (CallContext ctx = - CallContext.setCurrentContext( - CallContext.of(() -> realm, new PolarisCallContext(null, null)))) { - EntityCache cache = - requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); - assertThat(cache).isNotNull(); - EntityCacheEntry cachedCatalog = - cache.getEntityByName(new EntityCacheByNameKey(PolarisEntityType.CATALOG, catalogName)); - assertThat(cachedCatalog).isNull(); - EntityCacheEntry serviceAdmin = - cache.getEntityByName( - new EntityCacheByNameKey( - PolarisEntityType.PRINCIPAL_ROLE, - PolarisEntityConstants.getNameOfPrincipalServiceAdminRole())); - assertThat(serviceAdmin) - .isNotNull() - .extracting(EntityCacheEntry::getEntity) - .returns(PolarisEntityType.PRINCIPAL_ROLE, PolarisBaseEntity::getType) - .returns( - PolarisEntityConstants.getNameOfPrincipalServiceAdminRole(), - PolarisBaseEntity::getName); - } - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName) - .setStorageConfigInfo( - new AwsStorageConfigInfo( - "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) - .setProperties(new CatalogProperties("s3://bucket1/")) - .build(); - createCatalog(catalog); - createCatalogRole(catalogName, "my_cr", userToken); - - // now check again for the catalog - it should exist - try (CallContext ctx = - CallContext.setCurrentContext( - CallContext.of(() -> realm, new PolarisCallContext(null, null)))) { - EntityCache cache = - requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); - assertThat(cache).isNotNull(); - EntityCacheEntry cachedCatalog = - cache.getEntityByName(new EntityCacheByNameKey(PolarisEntityType.CATALOG, catalogName)); - assertThat(cachedCatalog) - .isNotNull() - .extracting(EntityCacheEntry::getEntity) - .returns(catalogName, PolarisBaseEntity::getName); - } - - // but if we check a different realm, the catalog should not exist in the cache - // the service_admin role also does not exist, since it's never been used - try (CallContext ctx = - CallContext.setCurrentContext( - CallContext.of(() -> "another-realm", new PolarisCallContext(null, null)))) { - EntityCache cache = - requestScope.runInScope(() -> serviceLocator.getService(EntityCache.class)); - assertThat(cache).isNotNull(); - EntityCacheEntry cachedCatalog = - cache.getEntityByName(new EntityCacheByNameKey(PolarisEntityType.CATALOG, catalogName)); - assertThat(cachedCatalog).isNull(); - EntityCacheEntry serviceAdmin = - cache.getEntityByName( - new EntityCacheByNameKey( - PolarisEntityType.PRINCIPAL_ROLE, - PolarisEntityConstants.getNameOfPrincipalServiceAdminRole())); - assertThat(serviceAdmin).isNull(); - } - } - - private static Invocation.Builder newRequest(String url, String token) { - return EXT.client() - .target(String.format(url, EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + token) - .header(REALM_PROPERTY_KEY, realm); - } - - private static Invocation.Builder newRequest(String url) { - return newRequest(url, userToken); - } - - private static void listCatalogs() { - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } - - private static void createCatalog(Catalog catalog) { - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private static void createCatalogRole( - String catalogName, String catalogRoleName, String catalogAdminToken) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName + "/catalog-roles", - catalogAdminToken) - .post(Entity.json(new CreateCatalogRoleRequest(new CatalogRole(catalogRoleName))))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java index b55b951fb..34ede6936 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java @@ -18,7 +18,6 @@ */ package org.apache.polaris.service.dropwizard.admin; -import static io.dropwizard.jackson.Jackson.newObjectMapper; import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; @@ -29,10 +28,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; @@ -85,28 +84,39 @@ import org.apache.polaris.core.admin.model.UpdatePrincipalRequest; import org.apache.polaris.core.admin.model.UpdatePrincipalRoleRequest; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.dropwizard.PolarisApplication; import org.apache.polaris.service.dropwizard.auth.TokenUtils; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestHelper; +import org.apache.polaris.service.dropwizard.test.TestEnvironment; import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.LoggerFactory; import org.testcontainers.shaded.org.awaitility.Awaitility; -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) +@QuarkusTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestProfile(PolarisServiceImplIntegrationTest.Profile.class) +@ExtendWith(TestEnvironmentExtension.class) public class PolarisServiceImplIntegrationTest { + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + // disallow FILE urls for the sake of tests below + return Map.of( + "polaris.config.defaults.SUPPORTED_CATALOG_STORAGE_TYPES", "[\"S3\",\"GCS\",\"AZURE\"]"); + } + } + private static final int MAX_IDENTIFIER_LENGTH = 256; private static final String ISSUER_KEY = "polaris"; private static final String CLAIM_KEY_ACTIVE = "active"; @@ -117,40 +127,27 @@ public class PolarisServiceImplIntegrationTest { // TODO: Add a test-only hook that fully clobbers all persistence state so we can have a fresh // slate on every test case; otherwise, leftover state from one test from failures will interfere // with other test cases. - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config("server.adminConnectors[0].port", "0"), - - // disallow FILE urls for the sake of tests below - ConfigOverride.config( - "featureConfiguration.SUPPORTED_CATALOG_STORAGE_TYPES", "S3,GCS,AZURE"), - ConfigOverride.config("gcp_credentials.access_token", "abc"), - ConfigOverride.config("gcp_credentials.expires_in", "12345")); - private static String userToken; - private static String realm; - private static String clientId; + + @Inject ObjectMapper mapper; + @Inject PolarisIntegrationTestHelper helper; + + private TestEnvironment testEnv; + private PolarisIntegrationTestFixture fixture; @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken adminToken, - PolarisPrincipalSecrets adminSecrets, - @PolarisRealm String polarisRealm) - throws IOException { - userToken = adminToken.token(); - realm = polarisRealm; - clientId = adminSecrets.getPrincipalClientId(); - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + public void createFixture(TestEnvironment testEnv, TestInfo testInfo) { + this.testEnv = testEnv; + fixture = helper.createFixture(testEnv, testInfo); + } + + @AfterAll + public void destroyFixture() { + fixture.destroy(); } @AfterEach - public void tearDown() { - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + public void after() { + try (Response response = newRequest("%s/api/management/v1/catalogs").get()) { response .readEntity(Catalogs.class) .getCatalogs() @@ -160,11 +157,7 @@ public void tearDown() { // delete all the namespaces try (Response res = - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalog.getName() - + "/namespaces") - .get()) { + newRequest("%s/api/catalog/v1/" + catalog.getName() + "/namespaces").get()) { if (res.getStatus() != Response.Status.OK.getStatusCode()) { LoggerFactory.getLogger(getClass()) .warn( @@ -177,7 +170,7 @@ public void tearDown() { .forEach( namespace -> { newRequest( - "http://localhost:%d/api/catalog/v1/" + "%s/api/catalog/v1/" + catalog.getName() + "/namespaces/" + RESTUtil.encodeNamespace(namespace)) @@ -190,9 +183,7 @@ public void tearDown() { // delete all the catalog roles except catalog_admin try (Response res = newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalog.getName() - + "/catalog-roles") + "%s/api/management/v1/catalogs/" + catalog.getName() + "/catalog-roles") .get()) { if (res.getStatus() != Response.Status.OK.getStatusCode()) { LoggerFactory.getLogger(getClass()) @@ -207,7 +198,7 @@ public void tearDown() { .forEach( cr -> newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + "%s/api/management/v1/catalogs/" + catalog.getName() + "/catalog-roles/" + cr.getName()) @@ -216,9 +207,7 @@ public void tearDown() { } Response deleteResponse = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalog.getName()) - .delete(); + newRequest("%s/api/management/v1/catalogs/" + catalog.getName()).delete(); if (deleteResponse.getStatus() != Response.Status.NO_CONTENT.getStatusCode()) { LoggerFactory.getLogger(getClass()) .warn( @@ -229,21 +218,19 @@ public void tearDown() { deleteResponse.close(); }); } - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { + try (Response response = newRequest("%s/api/management/v1/principals").get()) { response.readEntity(Principals.class).getPrincipals().stream() .filter( principal -> !principal.getName().equals(PolarisEntityConstants.getRootPrincipalName())) .forEach( principal -> { - newRequest( - "http://localhost:%d/api/management/v1/principals/" + principal.getName()) + newRequest("%s/api/management/v1/principals/" + principal.getName()) .delete() .close(); }); } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { + try (Response response = newRequest("%s/api/management/v1/principal-roles").get()) { response.readEntity(PrincipalRoles.class).getRoles().stream() .filter( principalRole -> @@ -252,9 +239,7 @@ public void tearDown() { .equals(PolarisEntityConstants.getNameOfPrincipalServiceAdminRole())) .forEach( principalRole -> { - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRole.getName()) + newRequest("%s/api/management/v1/principal-roles/" + principalRole.getName()) .delete() .close(); }); @@ -289,7 +274,7 @@ public void testCatalogSerializing() throws IOException { @Test public void testListCatalogs() { - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Catalogs.class)) @@ -307,20 +292,18 @@ public void testListCatalogsUnauthorized() { Principal principal = new Principal("a_new_user"); String newToken = null; try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(principal))) { + newRequest("%s/api/management/v1/principals").post(Entity.json(principal))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); PrincipalWithCredentials creds = response.readEntity(PrincipalWithCredentials.class); newToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + fixture.client, + testEnv.baseUri(), creds.getCredentials().getClientId(), creds.getCredentials().getClientSecret(), - realm); + fixture.realm); } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", newToken).get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs", newToken).get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } } @@ -328,7 +311,7 @@ public void testListCatalogsUnauthorized() { @Test public void testCreateCatalog() { try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + newRequest("%s/api/management/v1/catalogs") .post( Entity.json( "{\"catalog\":{\"type\":\"INTERNAL\",\"name\":\"my-catalog\",\"properties\":{\"default-base-location\":\"s3://my-bucket/path/to/data\"},\"storageConfigInfo\":{\"storageType\":\"S3\",\"roleArn\":\"arn:aws:iam::123456789012:role/my-role\",\"externalId\":\"externalId\",\"userArn\":\"userArn\",\"allowedLocations\":[\"s3://my-old-bucket/path/to/data\"]}}}"))) { @@ -336,8 +319,7 @@ public void testCreateCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/my-catalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -355,8 +337,6 @@ public void testCreateCatalogWithInvalidName() { String goodName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH, true, true); - ObjectMapper mapper = newObjectMapper(); - Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) @@ -365,7 +345,7 @@ public void testCreateCatalogWithInvalidName() { .setStorageConfigInfo(awsConfigModel) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + newRequest("%s/api/management/v1/catalogs") .post(Entity.json(mapper.writeValueAsString(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } catch (JsonProcessingException e) { @@ -392,7 +372,7 @@ public void testCreateCatalogWithInvalidName() { .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + newRequest("%s/api/management/v1/catalogs") .post(Entity.json(mapper.writeValueAsString(catalog)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -424,12 +404,11 @@ public void testCreateCatalogWithAzureStorageConfig() { .setStorageConfigInfo(azureConfigInfo) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + newRequest("%s/api/management/v1/catalogs") .post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/my-catalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catResponse = response.readEntity(Catalog.class); assertThat(catResponse.getStorageConfigInfo()) @@ -457,12 +436,11 @@ public void testCreateCatalogWithGcpStorageConfig() { .setStorageConfigInfo(gcpConfigModel) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + newRequest("%s/api/management/v1/catalogs") .post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/my-catalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catResponse = response.readEntity(Catalog.class); assertThat(catResponse.getStorageConfigInfo()) @@ -492,8 +470,7 @@ public void testCreateCatalogWithNullBaseLocation() { ObjectNode requestNode = mapper.createObjectNode(); requestNode.set("catalog", catalogNode); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(requestNode))) { + newRequest("%s/api/management/v1/catalogs").post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); } @@ -519,7 +496,7 @@ public void testCreateCatalogWithoutProperties() { requestNode.set("catalog", catalogNode); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) + newRequest("%s/api/management/v1/catalogs", fixture.adminToken) .post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -537,7 +514,7 @@ public void testCreateCatalogWithoutStorageConfig() throws JsonProcessingExcepti String catalogString = "{\"catalog\": {\"type\":\"INTERNAL\",\"name\":\"my-catalog\",\"properties\":{\"default-base-location\":\"s3://my-bucket/path/to/data\"}}}"; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) + newRequest("%s/api/management/v1/catalogs", fixture.adminToken) .post(Entity.json(catalogString))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -554,7 +531,7 @@ public void testCreateCatalogWithoutStorageConfig() throws JsonProcessingExcepti public void testCreateCatalogWithUnparsableJson() throws JsonProcessingException { String catalogString = "{\"catalog\": {{\"bad data}"; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) + newRequest("%s/api/management/v1/catalogs", fixture.adminToken) .post(Entity.json(catalogString))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -562,8 +539,7 @@ public void testCreateCatalogWithUnparsableJson() throws JsonProcessingException assertThat(error) .isNotNull() .extracting(ErrorResponse::message) - .asString() - .startsWith("Invalid JSON: Unexpected character"); + .isEqualTo("HTTP 400 Bad Request"); } } @@ -582,7 +558,7 @@ public void testCreateCatalogWithDisallowedStorageConfig() throws JsonProcessing .setStorageConfigInfo(fileStorage) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) + newRequest("%s/api/management/v1/catalogs", fixture.adminToken) .post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -612,7 +588,7 @@ public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonPro .setStorageConfigInfo(awsConfigModel) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) + newRequest("%s/api/management/v1/catalogs", fixture.adminToken) .post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -620,8 +596,7 @@ public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonPro // 200 successful GET after creation Catalog fetchedCatalog = null; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) - .get()) { + newRequest("%s/api/management/v1/catalogs/" + catalogName, fixture.adminToken).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -640,7 +615,7 @@ public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonPro // Successfully update Catalog updatedCatalog = null; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) + newRequest("%s/api/management/v1/catalogs/" + catalogName, fixture.adminToken) .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); updatedCatalog = response.readEntity(Catalog.class); @@ -672,7 +647,7 @@ public void testUpdateCatalogWithDisallowedStorageConfig() throws JsonProcessing .setStorageConfigInfo(awsConfigModel) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) + newRequest("%s/api/management/v1/catalogs", fixture.adminToken) .post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -680,8 +655,7 @@ public void testUpdateCatalogWithDisallowedStorageConfig() throws JsonProcessing // 200 successful GET after creation Catalog fetchedCatalog = null; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) - .get()) { + newRequest("%s/api/management/v1/catalogs/" + catalogName, fixture.adminToken).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -703,7 +677,7 @@ public void testUpdateCatalogWithDisallowedStorageConfig() throws JsonProcessing // failure to update try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) + newRequest("%s/api/management/v1/catalogs/" + catalogName, fixture.adminToken) .put(Entity.json(updateRequest))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -735,8 +709,7 @@ public void testCreateExternalCatalog() { .build(); createCatalog(catalog); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName).get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/" + catalogName).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog fetchedCatalog = response.readEntity(Catalog.class); assertThat(fetchedCatalog) @@ -752,8 +725,7 @@ public void testCreateExternalCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName).delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/" + catalogName).delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -781,8 +753,7 @@ public void testCreateCatalogWithoutDefaultLocation() { requestNode.set("catalog", catalogNode); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(requestNode))) { + newRequest("%s/api/management/v1/catalogs").post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); } @@ -814,8 +785,7 @@ public void testCreateAndUpdateAzureCatalog() { // 200 successful GET after creation Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/myazurecatalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -834,7 +804,7 @@ public void testCreateAndUpdateAzureCatalog() { Map.of("default-base-location", "abfss://newcontainer@acct1.dfs.core.windows.net/"), modifiedStorageConfig); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog") + newRequest("%s/api/management/v1/catalogs/myazurecatalog") .put(Entity.json(badUpdateRequest))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -854,7 +824,7 @@ public void testCreateAndUpdateAzureCatalog() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog") + newRequest("%s/api/management/v1/catalogs/myazurecatalog") .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -865,8 +835,7 @@ public void testCreateAndUpdateAzureCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog").delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/myazurecatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -888,15 +857,14 @@ public void testCreateListUpdateAndDeleteCatalog() { // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + newRequest("%s/api/management/v1/catalogs") .post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -907,7 +875,7 @@ public void testCreateListUpdateAndDeleteCatalog() { } // Should list the catalog. - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Catalogs.class)) @@ -927,8 +895,7 @@ public void testCreateListUpdateAndDeleteCatalog() { Map.of("default-base-location", "s3://newbucket/"), invalidModifiedStorageConfig); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog") - .put(Entity.json(badUpdateRequest))) { + newRequest("%s/api/management/v1/catalogs/mycatalog").put(Entity.json(badUpdateRequest))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -953,8 +920,7 @@ public void testCreateListUpdateAndDeleteCatalog() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog") - .put(Entity.json(updateRequest))) { + newRequest("%s/api/management/v1/catalogs/mycatalog").put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -966,8 +932,7 @@ public void testCreateListUpdateAndDeleteCatalog() { } // 200 GET after update should show new properties - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -976,19 +941,17 @@ public void testCreateListUpdateAndDeleteCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // NOT_FOUND after deletion - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Catalogs.class)) @@ -1001,23 +964,23 @@ public void testCreateListUpdateAndDeleteCatalog() { } } - private static Invocation.Builder newRequest(String url, String token) { - return EXT.client() - .target(String.format(url, EXT.getLocalPort())) + private Invocation.Builder newRequest(String url, String token) { + return fixture + .client + .target(String.format(url, testEnv.baseUri())) .request("application/json") .header("Authorization", "Bearer " + token) - .header(REALM_PROPERTY_KEY, realm); + .header(REALM_PROPERTY_KEY, fixture.realm); } - private static Invocation.Builder newRequest(String url) { - return newRequest(url, userToken); + private Invocation.Builder newRequest(String url) { + return newRequest(url, fixture.adminToken); } @Test public void testGetCatalogNotFound() { // there's no catalog yet. Expect 404 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } } @@ -1036,8 +999,7 @@ public void testGetCatalogInvalidName() { for (String invalidCatalogName : invalidCatalogNames) { // there's no catalog yet. Expect 404 try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + invalidCatalogName) - .get()) { + newRequest("%s/api/management/v1/catalogs/" + invalidCatalogName).get()) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -1072,7 +1034,7 @@ public void testCatalogRoleInvalidName() { for (String invalidCatalogRoleName : invalidCatalogRoleNames) { try (Response response = newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/" + "%s/api/management/v1/catalogs/mycatalog1/catalog-roles/" + invalidCatalogRoleName) .get()) { @@ -1090,20 +1052,18 @@ public void testListPrincipalsUnauthorized() { Principal principal = new Principal("new_admin"); String newToken = null; try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(principal))) { + newRequest("%s/api/management/v1/principals").post(Entity.json(principal))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); PrincipalWithCredentials creds = response.readEntity(PrincipalWithCredentials.class); newToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + fixture.client, + testEnv.baseUri(), creds.getCredentials().getClientId(), creds.getCredentials().getClientSecret(), - realm); + fixture.realm); } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + try (Response response = newRequest("%s/api/management/v1/principals", newToken).get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } } @@ -1119,7 +1079,7 @@ public void testCreatePrincipalAndRotateCredentials() { PrincipalWithCredentialsCredentials creds = null; Principal returnedPrincipal = null; try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, true)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); PrincipalWithCredentials parsed = response.readEntity(PrincipalWithCredentials.class); @@ -1135,21 +1095,18 @@ public void testCreatePrincipalAndRotateCredentials() { // newly created principal's credentials, we should fail; rotateCredentials is only // a "self" privilege that even admins can't inherit. try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal/rotate") - .post(Entity.json(""))) { + newRequest("%s/api/management/v1/principals/myprincipal/rotate").post(Entity.json(""))) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } // Get a fresh token associate with the principal itself. String newPrincipalToken = TokenUtils.getTokenFromSecrets( - EXT.client(), EXT.getLocalPort(), oldClientId, oldSecret, realm); + fixture.client, testEnv.baseUri(), oldClientId, oldSecret, fixture.realm); // Any call should initially fail with error indicating that rotation is needed. try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal", newPrincipalToken) - .get()) { + newRequest("%s/api/management/v1/principals/myprincipal", newPrincipalToken).get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); assertThat(error) @@ -1161,9 +1118,7 @@ public void testCreatePrincipalAndRotateCredentials() { // Now try to rotate using the principal's token. try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal/rotate", - newPrincipalToken) + newRequest("%s/api/management/v1/principals/myprincipal/rotate", newPrincipalToken) .post(Entity.json(""))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); PrincipalWithCredentials parsed = response.readEntity(PrincipalWithCredentials.class); @@ -1188,22 +1143,21 @@ public void testCreateListUpdateAndDeletePrincipal() { .setProperties(Map.of("custom-tag", "foo")) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, null)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, false)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation Principal fetchedPrincipal = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { + try (Response response = newRequest("%s/api/management/v1/principals/myprincipal").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipal = response.readEntity(Principal.class); @@ -1213,7 +1167,7 @@ public void testCreateListUpdateAndDeletePrincipal() { } // Should list the principal. - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { + try (Response response = newRequest("%s/api/management/v1/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Principals.class)) @@ -1228,8 +1182,7 @@ public void testCreateListUpdateAndDeletePrincipal() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal") - .put(Entity.json(updateRequest))) { + newRequest("%s/api/management/v1/principals/myprincipal").put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipal = response.readEntity(Principal.class); @@ -1237,8 +1190,7 @@ public void testCreateListUpdateAndDeletePrincipal() { } // 200 GET after update should show new properties - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { + try (Response response = newRequest("%s/api/management/v1/principals/myprincipal").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipal = response.readEntity(Principal.class); @@ -1246,19 +1198,17 @@ public void testCreateListUpdateAndDeletePrincipal() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").delete()) { + try (Response response = newRequest("%s/api/management/v1/principals/myprincipal").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // NOT_FOUND after deletion - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { + try (Response response = newRequest("%s/api/management/v1/principals/myprincipal").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { + try (Response response = newRequest("%s/api/management/v1/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Principals.class)) @@ -1277,7 +1227,7 @@ public void testCreatePrincipalWithInvalidName() { .setProperties(Map.of("custom-tag", "good_principal")) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, null)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -1300,7 +1250,7 @@ public void testCreatePrincipalWithInvalidName() { .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, false)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -1324,8 +1274,7 @@ public void testGetPrincipalWithInvalidName() { for (String invalidPrincipalName : invalidPrincipalNames) { try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/" + invalidPrincipalName) - .get()) { + newRequest("%s/api/management/v1/principals/" + invalidPrincipalName).get()) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -1343,7 +1292,7 @@ public void testCreateListUpdateAndDeletePrincipalRole() { // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") + newRequest("%s/api/management/v1/principal-roles") .post(Entity.json(new CreatePrincipalRoleRequest(principalRole)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); @@ -1352,7 +1301,7 @@ public void testCreateListUpdateAndDeletePrincipalRole() { // 200 successful GET after creation PrincipalRole fetchedPrincipalRole = null; try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { + newRequest("%s/api/management/v1/principal-roles/myprincipalrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipalRole = response.readEntity(PrincipalRole.class); @@ -1363,8 +1312,7 @@ public void testCreateListUpdateAndDeletePrincipalRole() { } // Should list the principalRole. - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { + try (Response response = newRequest("%s/api/management/v1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1380,7 +1328,7 @@ public void testCreateListUpdateAndDeletePrincipalRole() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") + newRequest("%s/api/management/v1/principal-roles/myprincipalrole") .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipalRole = response.readEntity(PrincipalRole.class); @@ -1390,7 +1338,7 @@ public void testCreateListUpdateAndDeletePrincipalRole() { // 200 GET after update should show new properties try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { + newRequest("%s/api/management/v1/principal-roles/myprincipalrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipalRole = response.readEntity(PrincipalRole.class); @@ -1399,22 +1347,20 @@ public void testCreateListUpdateAndDeletePrincipalRole() { // 204 Successful delete try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") - .delete()) { + newRequest("%s/api/management/v1/principal-roles/myprincipalrole").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // NOT_FOUND after deletion try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { + newRequest("%s/api/management/v1/principal-roles/myprincipalrole").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { + try (Response response = newRequest("%s/api/management/v1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1448,7 +1394,7 @@ public void testCreatePrincipalRoleInvalidName() { invalidPrincipalRoleName, Map.of("custom-tag", "bad_principal_role"), 0L, 0L, 1); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") + newRequest("%s/api/management/v1/principal-roles") .post(Entity.json(new CreatePrincipalRoleRequest(principalRole)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -1472,10 +1418,7 @@ public void testGetPrincipalRoleInvalidName() { for (String invalidPrincipalRoleName : invalidPrincipalRoleNames) { try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + invalidPrincipalRoleName) - .get()) { + newRequest("%s/api/management/v1/principal-roles/" + invalidPrincipalRoleName).get()) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -1512,7 +1455,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { CatalogRole catalogRole = new CatalogRole("mycatalogrole", Map.of("custom-tag", "foo"), 0L, 0L, 1); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1520,7 +1463,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); @@ -1529,9 +1472,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 200 successful GET after creation CatalogRole fetchedCatalogRole = null; try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .get()) { + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalogRole = response.readEntity(CatalogRole.class); @@ -1543,8 +1484,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // Should list the catalogRole. try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") - .get()) { + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1556,8 +1496,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // Empty list if listing in catalog2 try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog2/catalog-roles") - .get()) { + newRequest("%s/api/management/v1/catalogs/mycatalog2/catalog-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1578,8 +1517,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 200 successful update try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalogRole = response.readEntity(CatalogRole.class); @@ -1589,9 +1527,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 200 GET after update should show new properties try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .get()) { + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalogRole = response.readEntity(CatalogRole.class); @@ -1600,8 +1536,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 204 Successful delete try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -1609,17 +1544,14 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // NOT_FOUND after deletion try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .get()) { + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") - .get()) { + newRequest("%s/api/management/v1/catalogs/mycatalog1/catalog-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1630,15 +1562,13 @@ public void testCreateListUpdateAndDeleteCatalogRole() { } // 204 Successful delete mycatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1").delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog1").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mycatalog2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog2").delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog2").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1649,7 +1579,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Create two Principals Principal principal1 = new Principal("myprincipal1"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal1, false)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1657,7 +1587,7 @@ public void testAssignListAndRevokePrincipalRoles() { Principal principal2 = new Principal("myprincipal2"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal2, false)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1669,7 +1599,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Assign the role to myprincipal1 try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") + newRequest("%s/api/management/v1/principals/myprincipal1/principal-roles") .put(Entity.json(principalRole))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1677,8 +1607,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Should list myprincipalrole try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") - .get()) { + newRequest("%s/api/management/v1/principals/myprincipal1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1692,9 +1621,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Should list myprincipal1 if listing assignees of myprincipalrole try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/myprincipalrole/principals") - .get()) { + newRequest("%s/api/management/v1/principal-roles/myprincipalrole/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1707,8 +1634,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Empty list if listing in principal2 try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal2/principal-roles") - .get()) { + newRequest("%s/api/management/v1/principals/myprincipal2/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1718,8 +1644,7 @@ public void testAssignListAndRevokePrincipalRoles() { // 204 Successful revoke try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles/myprincipalrole") + newRequest("%s/api/management/v1/principals/myprincipal1/principal-roles/myprincipalrole") .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -1727,8 +1652,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Empty list try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") - .get()) { + newRequest("%s/api/management/v1/principals/myprincipal1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1736,9 +1660,7 @@ public void testAssignListAndRevokePrincipalRoles() { .returns(List.of(), PrincipalRoles::getRoles); } try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/myprincipalrole/principals") - .get()) { + newRequest("%s/api/management/v1/principal-roles/myprincipalrole/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1747,23 +1669,20 @@ public void testAssignListAndRevokePrincipalRoles() { } // 204 Successful delete myprincipal1 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1").delete()) { + try (Response response = newRequest("%s/api/management/v1/principals/myprincipal1").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete myprincipal2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal2").delete()) { + try (Response response = newRequest("%s/api/management/v1/principals/myprincipal2").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete myprincipalrole try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") - .delete()) { + newRequest("%s/api/management/v1/principal-roles/myprincipalrole").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1792,7 +1711,7 @@ public void testAssignListAndRevokeCatalogRoles() { CatalogRole catalogRole = new CatalogRole("mycr"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles") + newRequest("%s/api/management/v1/catalogs/mycatalog/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1812,7 +1731,7 @@ public void testAssignListAndRevokeCatalogRoles() { CatalogRole otherCatalogRole = new CatalogRole("myothercr"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/othercatalog/catalog-roles") + newRequest("%s/api/management/v1/catalogs/othercatalog/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(otherCatalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1820,15 +1739,13 @@ public void testAssignListAndRevokeCatalogRoles() { // Assign both the roles to mypr1 try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") + newRequest("%s/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") .put(Entity.json(new GrantCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/othercatalog") + newRequest("%s/api/management/v1/principal-roles/mypr1/catalog-roles/othercatalog") .put(Entity.json(new GrantCatalogRoleRequest(otherCatalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1836,9 +1753,7 @@ public void testAssignListAndRevokeCatalogRoles() { // Should list only mycr try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") - .get()) { + newRequest("%s/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1851,8 +1766,7 @@ public void testAssignListAndRevokeCatalogRoles() { // Should list mypr1 if listing assignees of mycr try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") + newRequest("%s/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") .get()) { assertThat(response) @@ -1866,9 +1780,7 @@ public void testAssignListAndRevokeCatalogRoles() { // Empty list if listing in principalRole2 try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr2/catalog-roles/mycatalog") - .get()) { + newRequest("%s/api/management/v1/principal-roles/mypr2/catalog-roles/mycatalog").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1878,8 +1790,7 @@ public void testAssignListAndRevokeCatalogRoles() { // 204 Successful revoke try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog/mycr") + newRequest("%s/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog/mycr") .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -1887,9 +1798,7 @@ public void testAssignListAndRevokeCatalogRoles() { // Empty list try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") - .get()) { + newRequest("%s/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1897,8 +1806,7 @@ public void testAssignListAndRevokeCatalogRoles() { .returns(List.of(), CatalogRoles::getRoles); } try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") + newRequest("%s/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") .get()) { assertThat(response) @@ -1908,46 +1816,39 @@ public void testAssignListAndRevokeCatalogRoles() { } // 204 Successful delete mypr1 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/mypr1").delete()) { + try (Response response = newRequest("%s/api/management/v1/principal-roles/mypr1").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mypr2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/mypr2").delete()) { + try (Response response = newRequest("%s/api/management/v1/principal-roles/mypr2").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mycr try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr") - .delete()) { + newRequest("%s/api/management/v1/catalogs/mycatalog/catalog-roles/mycr").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mycatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/mycatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete myothercr try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/othercatalog/catalog-roles/myothercr") - .delete()) { + newRequest("%s/api/management/v1/catalogs/othercatalog/catalog-roles/myothercr").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete othercatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/othercatalog").delete()) { + try (Response response = newRequest("%s/api/management/v1/catalogs/othercatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1974,7 +1875,8 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { createCatalog(catalog); CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + grantCatalogRoleToPrincipalRole( + principalRoleName, catalogName, catalogAdminRole, fixture.adminToken); PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); @@ -1982,11 +1884,11 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { String catalogAdminToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + fixture.client, + testEnv.baseUri(), catalogAdminPrincipal.getCredentials().getClientId(), catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + fixture.realm); // Create a second principal role. Use the catalog admin principal to list principal roles and // grant a catalog role to the new principal role @@ -2014,7 +1916,7 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { // PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE try (Response response = newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + "%s/api/management/v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName @@ -2029,13 +1931,13 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { // PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE privilege try (Response response = newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + "%s/api/management/v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName + "/" + catalogRoleName, - userToken) + fixture.adminToken) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -2062,7 +1964,8 @@ public void testServiceAdminCanTransferCatalogAdmin() { createCatalog(catalog); CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + grantCatalogRoleToPrincipalRole( + principalRoleName, catalogName, catalogAdminRole, fixture.adminToken); PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); @@ -2070,20 +1973,20 @@ public void testServiceAdminCanTransferCatalogAdmin() { String catalogAdminToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + fixture.client, + testEnv.baseUri(), catalogAdminPrincipal.getCredentials().getClientId(), catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + fixture.realm); // service_admin revokes the catalog_admin privilege from its principal role try { try (Response response = newRequest( - "http://localhost:%d/api/management/v1/principal-roles/service_admin/catalog-roles/" + "%s/api/management/v1/principal-roles/service_admin/catalog-roles/" + catalogName + "/catalog_admin", - userToken) + fixture.adminToken) .delete()) { assertThat(response) .returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -2092,7 +1995,7 @@ public void testServiceAdminCanTransferCatalogAdmin() { // the service_admin can not revoke the catalog_admin privilege from the new principal role try (Response response = newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + "%s/api/management/v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName @@ -2145,11 +2048,12 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { // create a catalog role *in the second catalog* and grant it manage_content privilege String catalogRoleName = "mycr1"; - createCatalogRole(catalogName2, catalogRoleName, userToken); + createCatalogRole(catalogName2, catalogRoleName, fixture.adminToken); // Get the catalog admin role from the *first* catalog and grant that role to the principal role CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + grantCatalogRoleToPrincipalRole( + principalRoleName, catalogName, catalogAdminRole, fixture.adminToken); // Create a principal and grant the principal role to it PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); @@ -2157,11 +2061,11 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { String catalogAdminToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + fixture.client, + testEnv.baseUri(), catalogAdminPrincipal.getCredentials().getClientId(), catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + fixture.realm); // Create a second principal role. String principalRoleName2 = "mypr2"; @@ -2172,7 +2076,7 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { // catalog role is in the wrong catalog try (Response response = newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + "%s/api/management/v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName2, @@ -2203,7 +2107,7 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { createCatalog(catalog); // create a valid target CatalogRole in this catalog - createCatalogRole(catalogName, "target_catalog_role", userToken); + createCatalogRole(catalogName, "target_catalog_role", fixture.adminToken); // create a second catalog String catalogName2 = "anothertablemanagecatalog"; @@ -2219,7 +2123,7 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { createCatalog(catalog2); // create an *invalid* target CatalogRole in second catalog - createCatalogRole(catalogName2, "invalid_target_catalog_role", userToken); + createCatalogRole(catalogName2, "invalid_target_catalog_role", fixture.adminToken); // create the namespace "c" in *both* namespaces String namespaceName = "c"; @@ -2230,7 +2134,7 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { // namespace level // grant that role to the PrincipalRole String catalogRoleName = "ns_manage_access_role"; - createCatalogRole(catalogName, catalogRoleName, userToken); + createCatalogRole(catalogName, catalogRoleName, fixture.adminToken); grantPrivilegeToCatalogRole( catalogName, catalogRoleName, @@ -2238,11 +2142,11 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { List.of(namespaceName), NamespacePrivilege.CATALOG_MANAGE_ACCESS, GrantResource.TypeEnum.NAMESPACE), - userToken, + fixture.adminToken, Response.Status.CREATED); grantCatalogRoleToPrincipalRole( - principalRoleName, catalogName, new CatalogRole(catalogRoleName), userToken); + principalRoleName, catalogName, new CatalogRole(catalogRoleName), fixture.adminToken); // Create a principal and grant the principal role to it PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("ns_manage_access_user"); @@ -2250,11 +2154,11 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { String manageAccessUserToken = TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), + fixture.client, + testEnv.baseUri(), catalogAdminPrincipal.getCredentials().getClientId(), catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + fixture.realm); // Use the ns_manage_access_user to grant TABLE_CREATE access to the target catalog role // This works because the user has CATALOG_MANAGE_ACCESS within the namespace and the target @@ -2277,7 +2181,7 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { // as a securable try (Response response = newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + "%s/api/management/v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName, @@ -2331,7 +2235,7 @@ public void testTokenExpiry() { .untilAsserted( () -> { try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + newRequest("%s/api/management/v1/principals", newToken).get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2343,8 +2247,7 @@ public void testTokenInactive() { // InvalidClaimException - if a claim contained a different value than the expected one. String newToken = defaultJwt().withClaim(CLAIM_KEY_ACTIVE, false).sign(Algorithm.HMAC256("polaris")); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + try (Response response = newRequest("%s/api/management/v1/principals", newToken).get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2354,8 +2257,7 @@ public void testTokenInactive() { public void testTokenInvalidSignature() { // SignatureVerificationException - if the signature is invalid. String newToken = defaultJwt().sign(Algorithm.HMAC256("invalid_secret")); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + try (Response response = newRequest("%s/api/management/v1/principals", newToken).get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2365,8 +2267,21 @@ public void testTokenInvalidSignature() { public void testTokenInvalidPrincipalId() { String newToken = defaultJwt().withClaim(CLAIM_KEY_PRINCIPAL_ID, 0).sign(Algorithm.HMAC256("polaris")); + try (Response response = newRequest("%s/api/management/v1/principals", newToken).get()) { + assertThat(response) + .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); + } + } + + @Test + public void testNoAuth() { try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + fixture + .client + .target(String.format("%s/api/management/v1/principals", testEnv.baseUri())) + .request("application/json") + .header(REALM_PROPERTY_KEY, fixture.realm) + .get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2394,11 +2309,8 @@ public void testNamespaceExistsStatus() { // check if a namespace existed try (Response response = newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalogName - + "/namespaces/" - + namespaceName, - userToken) + "%s/api/catalog/v1/" + catalogName + "/namespaces/" + namespaceName, + fixture.adminToken) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -2426,17 +2338,14 @@ public void testDropNamespaceStatus() { // drop a namespace try (Response response = newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalogName - + "/namespaces/" - + namespaceName, - userToken) + "%s/api/catalog/v1/" + catalogName + "/namespaces/" + namespaceName, + fixture.adminToken) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } - public static JWTCreator.Builder defaultJwt() { + public JWTCreator.Builder defaultJwt() { Instant now = Instant.now(); return JWT.create() .withIssuer(ISSUER_KEY) @@ -2445,14 +2354,14 @@ public static JWTCreator.Builder defaultJwt() { .withExpiresAt(now.plus(10, ChronoUnit.SECONDS)) .withJWTId(UUID.randomUUID().toString()) .withClaim(CLAIM_KEY_ACTIVE, true) - .withClaim(CLAIM_KEY_CLIENT_ID, clientId) + .withClaim(CLAIM_KEY_CLIENT_ID, fixture.adminSecrets.getPrincipalClientId()) .withClaim(CLAIM_KEY_PRINCIPAL_ID, 1) .withClaim(CLAIM_KEY_SCOPE, BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL); } - private static void createNamespace(String catalogName, String namespaceName) { + private void createNamespace(String catalogName, String namespaceName) { try (Response response = - newRequest("http://localhost:%d/api/catalog/v1/" + catalogName + "/namespaces", userToken) + newRequest("%s/api/catalog/v1/" + catalogName + "/namespaces", fixture.adminToken) .post( Entity.json( CreateNamespaceRequest.builder() @@ -2462,16 +2371,16 @@ private static void createNamespace(String catalogName, String namespaceName) { } } - private static void createCatalog(Catalog catalog) { + private void createCatalog(Catalog catalog) { try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + newRequest("%s/api/management/v1/catalogs") .post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } } - private static void grantPrivilegeToCatalogRole( + private void grantPrivilegeToCatalogRole( String catalogName, String catalogRoleName, GrantResource grant, @@ -2479,7 +2388,7 @@ private static void grantPrivilegeToCatalogRole( Response.Status expectedStatus) { try (Response response = newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + "%s/api/management/v1/catalogs/" + catalogName + "/catalog-roles/" + catalogRoleName @@ -2490,33 +2399,29 @@ private static void grantPrivilegeToCatalogRole( } } - private static void createCatalogRole( + private void createCatalogRole( String catalogName, String catalogRoleName, String catalogAdminToken) { try (Response response = newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName + "/catalog-roles", + "%s/api/management/v1/catalogs/" + catalogName + "/catalog-roles", catalogAdminToken) .post(Entity.json(new CreateCatalogRoleRequest(new CatalogRole(catalogRoleName))))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } } - private static void grantPrincipalRoleToPrincipal( - String principalName, PrincipalRole principalRole) { + private void grantPrincipalRoleToPrincipal(String principalName, PrincipalRole principalRole) { try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/" - + principalName - + "/principal-roles") + newRequest("%s/api/management/v1/principals/" + principalName + "/principal-roles") .put(Entity.json(new GrantPrincipalRoleRequest(principalRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } } - private static PrincipalWithCredentials createPrincipal(String principalName) { + private PrincipalWithCredentials createPrincipal(String principalName) { PrincipalWithCredentials catalogAdminPrincipal; try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + newRequest("%s/api/management/v1/principals") .post(Entity.json(new CreatePrincipalRequest(new Principal(principalName), false)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); catalogAdminPrincipal = response.readEntity(PrincipalWithCredentials.class); @@ -2524,11 +2429,11 @@ private static PrincipalWithCredentials createPrincipal(String principalName) { return catalogAdminPrincipal; } - private static void grantCatalogRoleToPrincipalRole( + private void grantCatalogRoleToPrincipalRole( String principalRoleName, String catalogName, CatalogRole catalogRole, String token) { try (Response response = newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + "%s/api/management/v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName, @@ -2538,13 +2443,9 @@ private static void grantCatalogRoleToPrincipalRole( } } - private static CatalogRole readCatalogRole(String catalogName, String roleName) { + private CatalogRole readCatalogRole(String catalogName, String roleName) { try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalogName - + "/catalog-roles/" - + roleName) + newRequest("%s/api/management/v1/catalogs/" + catalogName + "/catalog-roles/" + roleName) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); @@ -2552,9 +2453,9 @@ private static CatalogRole readCatalogRole(String catalogName, String roleName) } } - private static void createPrincipalRole(PrincipalRole principalRole1) { + private void createPrincipalRole(PrincipalRole principalRole1) { try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") + newRequest("%s/api/management/v1/principal-roles") .post(Entity.json(new CreatePrincipalRoleRequest(principalRole1)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/JWTRSAKeyPairTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/JWTRSAKeyPairTest.java index a5519e288..65e914e04 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/JWTRSAKeyPairTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/JWTRSAKeyPairTest.java @@ -18,27 +18,19 @@ */ package org.apache.polaris.service.dropwizard.auth; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; import com.auth0.jwt.JWT; import com.auth0.jwt.JWTVerifier; import com.auth0.jwt.algorithms.Algorithm; import com.auth0.jwt.interfaces.DecodedJWT; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.security.KeyPair; -import java.security.KeyPairGenerator; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.interfaces.RSAPrivateKey; import java.security.interfaces.RSAPublicKey; -import java.util.Base64; import java.util.HashMap; -import java.util.Map; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; @@ -46,6 +38,7 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.service.auth.JWTRSAKeyPair; import org.apache.polaris.service.auth.LocalRSAKeyProvider; +import org.apache.polaris.service.auth.PemUtils; import org.apache.polaris.service.auth.TokenBroker; import org.apache.polaris.service.auth.TokenRequestValidator; import org.apache.polaris.service.auth.TokenResponse; @@ -55,70 +48,17 @@ public class JWTRSAKeyPairTest { - private void writePemToTmpFile(String privateFileLocation, String publicFileLocation) - throws Exception { - new File(privateFileLocation).delete(); - new File(publicFileLocation).delete(); - KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA"); - kpg.initialize(2048); - KeyPair kp = kpg.generateKeyPair(); - try (BufferedWriter writer = - new BufferedWriter(new FileWriter(privateFileLocation, UTF_8, true))) { - writer.write("-----BEGIN PRIVATE KEY-----"); // pragma: allowlist secret - writer.newLine(); - writer.write(Base64.getMimeEncoder().encodeToString(kp.getPrivate().getEncoded())); - writer.newLine(); - writer.write("-----END PRIVATE KEY-----"); - writer.newLine(); - } - try (BufferedWriter writer = - new BufferedWriter(new FileWriter(publicFileLocation, UTF_8, true))) { - writer.write("-----BEGIN PUBLIC KEY-----"); - writer.newLine(); - writer.write(Base64.getMimeEncoder().encodeToString(kp.getPublic().getEncoded())); - writer.newLine(); - writer.write("-----END PUBLIC KEY-----"); - writer.newLine(); - } - } - - public CallContext getTestCallContext(PolarisCallContext polarisCallContext) { - return CallContext.setCurrentContext( - new CallContext() { - @Override - public RealmContext getRealmContext() { - return () -> "realm"; - } - - @Override - public PolarisCallContext getPolarisCallContext() { - return polarisCallContext; - } - - @Override - public Map contextVariables() { - return Map.of("token", "me"); - } - }); - } - @Test public void testSuccessfulTokenGeneration() throws Exception { - String privateFileLocation = "/tmp/test-private.pem"; - String publicFileLocation = "/tmp/test-public.pem"; - writePemToTmpFile(privateFileLocation, publicFileLocation); + Path privateFileLocation = Files.createTempFile("test-private", ".pem"); + Path publicFileLocation = Files.createTempFile("test-public", ".pem"); + PemUtils.generateKeyPair(privateFileLocation, publicFileLocation); final String clientId = "test-client-id"; final String scope = "PRINCIPAL_ROLE:TEST"; - Map config = new HashMap<>(); - - config.put("LOCAL_PRIVATE_KEY_LOCATION_KEY", privateFileLocation); - config.put("LOCAL_PUBLIC_LOCATION_KEY", publicFileLocation); - - DefaultConfigurationStore store = new DefaultConfigurationStore(config); + DefaultConfigurationStore store = new DefaultConfigurationStore(new HashMap<>()); PolarisCallContext polarisCallContext = new PolarisCallContext(null, null, store, null); - CallContext.setCurrentContext(getTestCallContext(polarisCallContext)); PolarisMetaStoreManager metastoreManager = Mockito.mock(PolarisMetaStoreManager.class); String mainSecret = "client-secret"; PolarisPrincipalSecrets principalSecrets = @@ -135,14 +75,19 @@ public void testSuccessfulTokenGeneration() throws Exception { "principal"); Mockito.when(metastoreManager.loadEntity(polarisCallContext, 0L, 1L)) .thenReturn(new PolarisMetaStoreManager.EntityResult(principal)); - TokenBroker tokenBroker = new JWTRSAKeyPair(metastoreManager, 420); + TokenBroker tokenBroker = + new JWTRSAKeyPair(metastoreManager, 420, publicFileLocation, privateFileLocation); TokenResponse token = tokenBroker.generateFromClientSecrets( - clientId, mainSecret, TokenRequestValidator.CLIENT_CREDENTIALS, scope); + clientId, + mainSecret, + TokenRequestValidator.CLIENT_CREDENTIALS, + scope, + polarisCallContext); assertThat(token).isNotNull(); assertThat(token.getExpiresIn()).isEqualTo(420); - LocalRSAKeyProvider provider = new LocalRSAKeyProvider(); + LocalRSAKeyProvider provider = new LocalRSAKeyProvider(publicFileLocation, privateFileLocation); assertThat(provider.getPrivateKey()).isNotNull(); assertThat(provider.getPublicKey()).isNotNull(); JWTVerifier verifier = diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/JWTSymmetricKeyGeneratorTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/JWTSymmetricKeyGeneratorTest.java index 3384a89e7..a5851ae33 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/JWTSymmetricKeyGeneratorTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/JWTSymmetricKeyGeneratorTest.java @@ -26,7 +26,6 @@ import com.auth0.jwt.interfaces.DecodedJWT; import java.util.Map; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -70,7 +69,7 @@ public Map contextVariables() { PolarisPrincipalSecrets principalSecrets = new PolarisPrincipalSecrets(1L, clientId, mainSecret, "otherSecret"); Mockito.when(metastoreManager.loadPrincipalSecrets(polarisCallContext, clientId)) - .thenReturn(new PrincipalSecretsResult(principalSecrets)); + .thenReturn(new PolarisMetaStoreManager.PrincipalSecretsResult(principalSecrets)); PolarisBaseEntity principal = new PolarisBaseEntity( 0L, @@ -84,7 +83,11 @@ public Map contextVariables() { TokenBroker generator = new JWTSymmetricKeyBroker(metastoreManager, 666, () -> "polaris"); TokenResponse token = generator.generateFromClientSecrets( - clientId, mainSecret, TokenRequestValidator.CLIENT_CREDENTIALS, "PRINCIPAL_ROLE:TEST"); + clientId, + mainSecret, + TokenRequestValidator.CLIENT_CREDENTIALS, + "PRINCIPAL_ROLE:TEST", + polarisCallContext); assertThat(token).isNotNull(); JWTVerifier verifier = JWT.require(Algorithm.HMAC256("polaris")).withIssuer("polaris").build(); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/TokenUtils.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/TokenUtils.java index db3043b25..af08c5a8a 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/TokenUtils.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/auth/TokenUtils.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.dropwizard.auth; +import static org.apache.polaris.service.auth.BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL; import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; @@ -26,6 +27,7 @@ import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.MultivaluedHashMap; import jakarta.ws.rs.core.Response; +import java.net.URI; import java.util.Map; import org.apache.iceberg.rest.responses.OAuthTokenResponse; @@ -33,13 +35,7 @@ public class TokenUtils { /** Get token against specified realm */ public static String getTokenFromSecrets( - Client client, int port, String clientId, String clientSecret, String realm) { - return getTokenFromSecrets( - client, String.format("http://localhost:%d", port), clientId, clientSecret, realm); - } - - public static String getTokenFromSecrets( - Client client, String baseUrl, String clientId, String clientSecret, String realm) { + Client client, URI baseUrl, String clientId, String clientSecret, String realm) { String token; Invocation.Builder builder = @@ -58,7 +54,7 @@ public static String getTokenFromSecrets( "grant_type", "client_credentials", "scope", - "PRINCIPAL_ROLE:ALL", + PRINCIPAL_ROLE_ALL, "client_id", clientId, "client_secret", diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogTest.java index 1b7c59e22..e212c4848 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogTest.java @@ -25,12 +25,14 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; -import jakarta.annotation.Nullable; +import io.quarkus.test.junit.QuarkusMock; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import java.io.IOException; +import java.lang.reflect.Method; import java.time.Clock; import java.util.Arrays; import java.util.EnumMap; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -60,7 +62,6 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; import org.apache.polaris.core.admin.model.StorageConfigInfo; @@ -93,25 +94,27 @@ import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.dropwizard.catalog.io.TestFileIOFactory; import org.apache.polaris.service.exception.IcebergExceptionMapper; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; import org.apache.polaris.service.task.TableCleanupTaskHandler; import org.apache.polaris.service.task.TaskExecutor; import org.apache.polaris.service.task.TaskFileIOSupplier; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; import org.apache.polaris.service.types.TableUpdateNotification; -import org.assertj.core.api.AbstractBooleanAssert; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.mockito.Mockito; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; import software.amazon.awssdk.services.sts.model.Credentials; +@QuarkusTest public class BasePolarisCatalogTest extends CatalogTests { protected static final Namespace NS = Namespace.of("newdb"); protected static final TableIdentifier TABLE = TableIdentifier.of(NS, "table"); @@ -124,9 +127,15 @@ public class BasePolarisCatalogTest extends CatalogTests { public static final String SECRET_ACCESS_KEY = "secret_access_key"; public static final String SESSION_TOKEN = "session_token"; + @Inject MetaStoreManagerFactory managerFactory; + @Inject PolarisConfigurationStore configurationStore; + @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; + @Inject PolarisDiagnostics diagServices; + private BasePolarisCatalog catalog; private AwsStorageConfigInfo storageConfigModel; private StsClient stsClient; + private String realmName; private PolarisMetaStoreManager metaStoreManager; private PolarisCallContext polarisContext; private PolarisAdminService adminService; @@ -134,29 +143,27 @@ public class BasePolarisCatalogTest extends CatalogTests { private AuthenticatedPolarisPrincipal authenticatedRoot; private PolarisEntity catalogEntity; + @BeforeAll + public static void setUpMocks() { + PolarisStorageIntegrationProviderImpl mock = + Mockito.mock(PolarisStorageIntegrationProviderImpl.class); + QuarkusMock.installMockForType(mock, PolarisStorageIntegrationProviderImpl.class); + } + @BeforeEach @SuppressWarnings("unchecked") - public void before() { - PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); - RealmContext realmContext = () -> "realm"; - PolarisStorageIntegrationProvider storageIntegrationProvider = Mockito.mock(); - InMemoryPolarisMetaStoreManagerFactory managerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - managerFactory.setStorageIntegrationProvider(storageIntegrationProvider); + public void before(TestInfo testInfo) { + realmName = + "realm_%s_%s" + .formatted( + testInfo.getTestMethod().map(Method::getName).orElse("test"), System.nanoTime()); + RealmContext realmContext = () -> realmName; metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); - Map configMap = new HashMap<>(); - configMap.put("ALLOW_SPECIFYING_FILE_IO_IMPL", true); - configMap.put("INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST", true); polarisContext = new PolarisCallContext( managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, - new PolarisConfigurationStore() { - @Override - public @Nullable T getConfiguration(PolarisCallContext ctx, String configName) { - return (T) configMap.get(configName); - } - }, + configurationStore, Clock.systemDefaultZone()); entityManager = new PolarisEntityManager( @@ -246,6 +253,7 @@ public void before() { @AfterEach public void after() throws IOException { catalog().close(); + metaStoreManager.purge(polarisContext); } @Override @@ -290,6 +298,11 @@ public StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext rea return new StorageCredentialCache(); } + @Override + public EntityCache getOrCreateEntityCache(RealmContext realmContext) { + return new EntityCache(metaStoreManager); + } + @Override public Map bootstrapRealms(List realms) { throw new NotImplementedException("Bootstrapping realms is not supported"); @@ -1330,8 +1343,8 @@ public void testDropTableWithPurge() { TableMetadata tableMetadata = ((BaseTable) table).operations().current(); boolean dropped = catalog.dropTable(TABLE, true); - ((AbstractBooleanAssert) - Assertions.assertThat(dropped).as("Should drop a table that does exist", new Object[0])) + Assertions.assertThat(dropped) + .as("Should drop a table that does exist", new Object[0]) .isTrue(); Assertions.assertThatPredicate(catalog::tableExists) .as("Table should not exist after drop") diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogViewTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogViewTest.java index 17f1cf019..1f5a0bd32 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogViewTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/BasePolarisCatalogViewTest.java @@ -18,15 +18,16 @@ */ package org.apache.polaris.service.dropwizard.catalog; -import com.google.auth.oauth2.AccessToken; -import com.google.auth.oauth2.GoogleCredentials; import com.google.common.collect.ImmutableMap; -import jakarta.annotation.Nullable; +import io.quarkus.test.junit.QuarkusMock; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.file.Path; import java.time.Clock; -import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Set; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.catalog.Catalog; @@ -34,7 +35,6 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfiguration; import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.StorageConfigInfo; @@ -47,6 +47,7 @@ import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.cache.EntityCache; @@ -54,47 +55,64 @@ import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.BasePolarisCatalog; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.io.TempDir; import org.mockito.Mockito; +@QuarkusTest public class BasePolarisCatalogViewTest extends ViewCatalogTests { public static final String CATALOG_NAME = "polaris-catalog"; + + @Inject MetaStoreManagerFactory managerFactory; + @Inject PolarisConfigurationStore configurationStore; + @Inject PolarisDiagnostics diagServices; + private BasePolarisCatalog catalog; + private String realmName; + private PolarisMetaStoreManager metaStoreManager; + private PolarisCallContext polarisContext; + + @BeforeAll + public static void setUpMocks() { + PolarisStorageIntegrationProviderImpl mock = + Mockito.mock(PolarisStorageIntegrationProviderImpl.class); + QuarkusMock.installMockForType(mock, PolarisStorageIntegrationProviderImpl.class); + } + @BeforeEach - @SuppressWarnings("unchecked") - public void before() { - PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); - RealmContext realmContext = () -> "realm"; - InMemoryPolarisMetaStoreManagerFactory managerFactory = - new InMemoryPolarisMetaStoreManagerFactory(); - managerFactory.setStorageIntegrationProvider( - new PolarisStorageIntegrationProviderImpl( - Mockito::mock, () -> GoogleCredentials.create(new AccessToken("abc", new Date())))); - PolarisMetaStoreManager metaStoreManager = - managerFactory.getOrCreateMetaStoreManager(realmContext); - Map configMap = new HashMap<>(); - configMap.put("ALLOW_WILDCARD_LOCATION", true); - configMap.put("ALLOW_SPECIFYING_FILE_IO_IMPL", true); - PolarisCallContext polarisContext = + public void setUpTempDir(@TempDir Path tempDir) throws Exception { + // see https://github.com/quarkusio/quarkus/issues/13261 + Field field = ViewCatalogTests.class.getDeclaredField("tempDir"); + field.setAccessible(true); + field.set(this, tempDir); + } + + @BeforeEach + public void before(TestInfo testInfo) { + realmName = + "realm_%s_%s" + .formatted( + testInfo.getTestMethod().map(Method::getName).orElse("test"), System.nanoTime()); + RealmContext realmContext = () -> realmName; + + metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); + polarisContext = new PolarisCallContext( managerFactory.getOrCreateSessionSupplier(realmContext).get(), diagServices, - new PolarisConfigurationStore() { - @Override - public @Nullable T getConfiguration(PolarisCallContext ctx, String configName) { - return (T) configMap.get(configName); - } - }, + configurationStore, Clock.systemDefaultZone()); PolarisEntityManager entityManager = new PolarisEntityManager( metaStoreManager, new StorageCredentialCache(), new EntityCache(metaStoreManager)); - CallContext callContext = CallContext.of(null, polarisContext); + CallContext callContext = CallContext.of(realmContext, polarisContext); CallContext.setCurrentContext(callContext); PrincipalEntity rootEntity = @@ -149,6 +167,12 @@ public void before() { CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); } + @AfterEach + public void after() throws IOException { + catalog().close(); + metaStoreManager.purge(polarisContext); + } + @Override protected BasePolarisCatalog catalog() { return catalog; diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisCatalogHandlerWrapperAuthzTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisCatalogHandlerWrapperAuthzTest.java index db80ac1f6..8eade2344 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisCatalogHandlerWrapperAuthzTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisCatalogHandlerWrapperAuthzTest.java @@ -19,6 +19,8 @@ package org.apache.polaris.service.dropwizard.catalog; import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; import java.time.Instant; import java.util.List; import java.util.Map; @@ -63,6 +65,7 @@ import org.apache.polaris.service.catalog.PolarisCatalogHandlerWrapper; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; import org.apache.polaris.service.config.RealmEntityManagerFactory; +import org.apache.polaris.service.context.CallContextCatalogFactory; import org.apache.polaris.service.context.PolarisCallContextCatalogFactory; import org.apache.polaris.service.dropwizard.admin.PolarisAuthzTestBase; import org.apache.polaris.service.types.NotificationRequest; @@ -72,20 +75,28 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; +@QuarkusTest +@TestProfile(PolarisCatalogHandlerWrapperAuthzTest.Profile.class) public class PolarisCatalogHandlerWrapperAuthzTest extends PolarisAuthzTestBase { + + public static class Profile extends PolarisAuthzTestBase.Profile { + + @Override + public Map getConfigOverrides() { + return Map.of("polaris.config.defaults.ALLOW_EXTERNAL_METADATA_FILE_LOCATION", "true"); + } + } + private PolarisCatalogHandlerWrapper newWrapper() { return newWrapper(Set.of()); } private PolarisCatalogHandlerWrapper newWrapper(Set activatedPrincipalRoles) { - return newWrapper( - activatedPrincipalRoles, CATALOG_NAME, new TestPolarisCallContextCatalogFactory()); + return newWrapper(activatedPrincipalRoles, CATALOG_NAME, callContextCatalogFactory); } private PolarisCatalogHandlerWrapper newWrapper( - Set activatedPrincipalRoles, - String catalogName, - PolarisCallContextCatalogFactory factory) { + Set activatedPrincipalRoles, String catalogName, CallContextCatalogFactory factory) { final AuthenticatedPolarisPrincipal authenticatedPrincipal = new AuthenticatedPolarisPrincipal(principalEntity, activatedPrincipalRoles); return new PolarisCatalogHandlerWrapper( @@ -230,7 +241,7 @@ public void testInsufficientPermissionsPriorToSecretRotation() { entityManager, metaStoreManager, authenticatedPrincipal, - new TestPolarisCallContextCatalogFactory(), + callContextCatalogFactory, CATALOG_NAME, polarisAuthorizer); @@ -261,7 +272,7 @@ public void testInsufficientPermissionsPriorToSecretRotation() { entityManager, metaStoreManager, authenticatedPrincipal1, - new TestPolarisCallContextCatalogFactory(), + callContextCatalogFactory, CATALOG_NAME, polarisAuthorizer); @@ -1676,13 +1687,13 @@ public void testSendNotificationSufficientPrivileges() { PolarisCallContextCatalogFactory factory = new PolarisCallContextCatalogFactory( - new RealmEntityManagerFactory() { + new RealmEntityManagerFactory(null) { @Override public PolarisEntityManager getOrCreateEntityManager(RealmContext realmContext) { return entityManager; } }, - metaStoreManagerFactory, + managerFactory, Mockito.mock(), new DefaultFileIOFactory()) { @Override diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java index 948f6d581..ac04aac98 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java @@ -23,15 +23,13 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.common.collect.ImmutableMap; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; -import java.io.IOException; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -79,36 +77,32 @@ import org.apache.polaris.core.admin.model.ViewPrivilege; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.auth.TokenUtils; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension.PolarisToken; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension.SnowmanCredentials; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestHelper; +import org.apache.polaris.service.dropwizard.test.TestEnvironment; import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; import org.apache.polaris.service.types.TableUpdateNotification; import org.assertj.core.api.Assertions; import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; /** * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog * client. */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) +@QuarkusTest +@TestInstance(Lifecycle.PER_CLASS) +@ExtendWith(TestEnvironmentExtension.class) public class PolarisRestCatalogIntegrationTest extends CatalogTests { private static final String TEST_ROLE_ARN = Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN")) @@ -116,22 +110,16 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests private static final String S3_BUCKET_BASE = Optional.ofNullable(System.getenv("INTEGRATION_TEST_S3_PATH")) .orElse("file:///tmp/buckets/my-bucket"); - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism protected static final String VIEW_QUERY = "select * from ns1.layer1_table"; + @Inject PolarisIntegrationTestHelper helper; + + private TestEnvironment testEnv; + private PolarisIntegrationTestFixture fixture; private RESTCatalog restCatalog; private String currentCatalogName; - private String userToken; - private String realm; + private Path tempDir; private final String catalogBaseLocation = S3_BUCKET_BASE + "/" + System.getenv("USER") + "/path/to/data"; @@ -157,25 +145,18 @@ String[] properties() default { } @BeforeAll - public static void setup(@PolarisRealm String realm) throws IOException { - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + public void createFixture(TestEnvironment testEnv, TestInfo testInfo) { + this.testEnv = testEnv; + fixture = helper.createFixture(testEnv, testInfo); + } + + @BeforeEach + public void destroyFixture(@TempDir Path tempDir) { + this.tempDir = tempDir; } @BeforeEach - public void before( - TestInfo testInfo, - PolarisToken adminToken, - SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm) { - this.realm = realm; - userToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - snowmanCredentials.clientId(), - snowmanCredentials.clientSecret(), - realm); + void before(TestInfo testInfo) { testInfo .getTestMethod() .ifPresent( @@ -237,15 +218,15 @@ public void before( }); restCatalog = TestUtil.createSnowmanManagedCatalog( - EXT, - adminToken, - snowmanCredentials, - realm, - catalog, - extraPropertiesBuilder.build()); + testEnv, fixture, catalog, extraPropertiesBuilder.build()); }); } + @AfterAll + public void tearDown() { + fixture.destroy(); + } + @Override protected RESTCatalog catalog() { return restCatalog; @@ -274,14 +255,15 @@ protected boolean overridesRequestedLocation() { private void createCatalogRole(String catalogRoleName) { CatalogRole catalogRole = new CatalogRole(catalogRoleName); try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles", - EXT.getLocalPort(), currentCatalogName)) + "%s/api/management/v1/catalogs/%s/catalog-roles", + testEnv.baseUri(), currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.userToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(catalogRole))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -289,14 +271,15 @@ private void createCatalogRole(String catalogRoleName) { private void addGrant(String catalogRoleName, GrantResource grant) { try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, catalogRoleName)) + "%s/api/management/v1/catalogs/%s/catalog-roles/%s/grants", + testEnv.baseUri(), currentCatalogName, catalogRoleName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.userToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .put(Entity.json(grant))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -424,14 +407,15 @@ public void testListGrantsOnCatalogObjectsToCatalogRoles() { // List grants for catalogrole1 try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole1")) + "%s/api/management/v1/catalogs/%s/catalog-roles/%s/grants", + testEnv.baseUri(), currentCatalogName, "catalogrole1")) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.userToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -443,14 +427,15 @@ public void testListGrantsOnCatalogObjectsToCatalogRoles() { // List grants for catalogrole2 try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole2")) + "%s/api/management/v1/catalogs/%s/catalog-roles/%s/grants", + testEnv.baseUri(), currentCatalogName, "catalogrole2")) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.userToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -494,14 +479,15 @@ public void testListGrantsAfterRename() { GrantResource.TypeEnum.TABLE); try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole1")) + "%s/api/management/v1/catalogs/%s/catalog-roles/%s/grants", + testEnv.baseUri(), currentCatalogName, "catalogrole1")) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.userToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -513,16 +499,16 @@ public void testListGrantsAfterRename() { } @Test - public void testCreateTableWithOverriddenBaseLocation(PolarisToken adminToken) { + public void testCreateTableWithOverriddenBaseLocation() { try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) + "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catalog = response.readEntity(Catalog.class); @@ -530,14 +516,14 @@ public void testCreateTableWithOverriddenBaseLocation(PolarisToken adminToken) { catalogProps.put( PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); try (Response updateResponse = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) + "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), catalog.getName())) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .put( Entity.json( new UpdateCatalogRequest( @@ -569,17 +555,16 @@ public void testCreateTableWithOverriddenBaseLocation(PolarisToken adminToken) { } @Test - public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( - PolarisToken adminToken) { + public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling() { try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) + "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catalog = response.readEntity(Catalog.class); @@ -587,14 +572,14 @@ public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( catalogProps.put( PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); try (Response updateResponse = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) + "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), catalog.getName())) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .put( Entity.json( new UpdateCatalogRequest( @@ -635,17 +620,16 @@ public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( } @Test - public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory( - PolarisToken adminToken) { + public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory() { try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) + "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catalog = response.readEntity(Catalog.class); @@ -653,14 +637,14 @@ public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory( catalogProps.put( PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); try (Response updateResponse = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) + "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), catalog.getName())) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .put( Entity.json( new UpdateCatalogRequest( @@ -699,16 +683,17 @@ public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory( public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigDisabled() { Namespace ns1 = Namespace.of("ns1"); restCatalog.createNamespace(ns1); + String baseLocation = "file://" + tempDir.toString() + "/ns1/my_table"; TableMetadata tableMetadata = TableMetadata.newTableMetadata( new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), - "file:///tmp/ns1/my_table", + baseLocation, Map.of()); try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { resolvingFileIO.initialize(Map.of()); resolvingFileIO.setConf(new Configuration()); - String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json"; + String fileLocation = baseLocation + "/metadata/v1.metadata.json"; TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); try { @@ -734,16 +719,17 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigDisable public void testLoadTableWithoutAccessDelegationForExternalCatalogWithConfigDisabled() { Namespace ns1 = Namespace.of("ns1"); restCatalog.createNamespace(ns1); + String baseLocation = "file://" + tempDir.toString() + "/ns1/my_table"; TableMetadata tableMetadata = TableMetadata.newTableMetadata( new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), - "file:///tmp/ns1/my_table", + baseLocation, Map.of()); try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { resolvingFileIO.initialize(Map.of()); resolvingFileIO.setConf(new Configuration()); - String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json"; + String fileLocation = baseLocation + "/metadata/v1.metadata.json"; TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); try { @@ -768,16 +754,17 @@ public void testLoadTableWithoutAccessDelegationForExternalCatalogWithConfigDisa public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigEnabledForCatalog() { Namespace ns1 = Namespace.of("ns1"); restCatalog.createNamespace(ns1); + String baseLocation = "file://" + tempDir.toString() + "/ns1/my_table"; TableMetadata tableMetadata = TableMetadata.newTableMetadata( new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), - "file:///tmp/ns1/my_table", + baseLocation, Map.of()); try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { resolvingFileIO.initialize(Map.of()); resolvingFileIO.setConf(new Configuration()); - String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json"; + String fileLocation = baseLocation + "/metadata/v1.metadata.json"; TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); try { @@ -802,14 +789,15 @@ public void testSendNotificationInternalCatalog() { restCatalog.createNamespace(Namespace.of("ns1")); String notificationUrl = String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/ns1/tables/tbl1/notifications", - EXT.getLocalPort(), currentCatalogName); + "%s/api/catalog/v1/%s/namespaces/ns1/tables/tbl1/notifications", + testEnv.baseUri(), currentCatalogName); try (Response response = - EXT.client() + fixture + .client .target(notificationUrl) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.userToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(notification))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) @@ -820,11 +808,12 @@ public void testSendNotificationInternalCatalog() { // NotificationType.VALIDATE should also surface the same error. notification.setNotificationType(NotificationType.VALIDATE); try (Response response = - EXT.client() + fixture + .client .target(notificationUrl) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.userToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(notification))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) @@ -1033,14 +1022,15 @@ public void testTableExistsStatus() { catalog().buildTable(identifier, SCHEMA).create(); try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), currentCatalogName, namespace.toString(), tableName)) + "%s/api/catalog/v1/%s/namespaces/%s/tables/%s", + testEnv.baseUri(), currentCatalogName, namespace.toString(), tableName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1059,14 +1049,15 @@ public void testDropTableStatus() { catalog().buildTable(identifier, SCHEMA).create(); try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), currentCatalogName, namespace.toString(), tableName)) + "%s/api/catalog/v1/%s/namespaces/%s/tables/%s", + testEnv.baseUri(), currentCatalogName, namespace.toString(), tableName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1093,14 +1084,15 @@ public void testViewExistsStatus() { .create(); try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, viewName)) + "%s/api/catalog/v1/%s/namespaces/%s/views/%s", + testEnv.baseUri(), currentCatalogName, namespace, viewName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1127,14 +1119,15 @@ public void testDropViewStatus() { .create(); try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, viewName)) + "%s/api/catalog/v1/%s/namespaces/%s/views/%s", + testEnv.baseUri(), currentCatalogName, namespace, viewName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1168,42 +1161,44 @@ public void testRenameViewStatus() { // Perform view rename try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/views/rename", - EXT.getLocalPort(), currentCatalogName)) + "%s/api/catalog/v1/%s/views/rename", testEnv.baseUri(), currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(payload))) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // Original view should no longer exists try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, viewName)) + "%s/api/catalog/v1/%s/namespaces/%s/views/%s", + testEnv.baseUri(), currentCatalogName, namespace, viewName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .head()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // New view should exists try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, newViewName)) + "%s/api/catalog/v1/%s/namespaces/%s/views/%s", + testEnv.baseUri(), currentCatalogName, namespace, newViewName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java index b7fdbee57..fb378ed8b 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.dropwizard.catalog; +import io.quarkus.test.junit.QuarkusTest; import java.util.List; import java.util.Optional; import java.util.stream.Stream; @@ -26,6 +27,7 @@ import org.assertj.core.util.Strings; /** Runs PolarisRestCatalogViewIntegrationTest on AWS. */ +@QuarkusTest public class PolarisRestCatalogViewAwsIntegrationTest extends PolarisRestCatalogViewIntegrationTest { public static final String ROLE_ARN = diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java index be755526f..9742c198c 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.dropwizard.catalog; +import io.quarkus.test.junit.QuarkusTest; import java.util.List; import java.util.stream.Stream; import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; @@ -25,6 +26,7 @@ import org.assertj.core.util.Strings; /** Runs PolarisRestCatalogViewIntegrationTest on Azure. */ +@QuarkusTest public class PolarisRestCatalogViewAzureIntegrationTest extends PolarisRestCatalogViewIntegrationTest { public static final String TENANT_ID = System.getenv("INTEGRATION_TEST_AZURE_TENANT_ID"); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java index c7853df97..c8045baa3 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java @@ -18,11 +18,13 @@ */ package org.apache.polaris.service.dropwizard.catalog; +import io.quarkus.test.junit.QuarkusTest; import java.util.List; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.StorageConfigInfo; /** Runs PolarisRestCatalogViewIntegrationTest on the local filesystem. */ +@QuarkusTest public class PolarisRestCatalogViewFileIntegrationTest extends PolarisRestCatalogViewIntegrationTest { public static final String BASE_LOCATION = "file:///tmp/buckets/my-bucket"; diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java index b74677275..d12646eb0 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.dropwizard.catalog; +import io.quarkus.test.junit.QuarkusTest; import java.util.List; import java.util.stream.Stream; import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; @@ -25,6 +26,7 @@ import org.assertj.core.util.Strings; /** Runs PolarisRestCatalogViewIntegrationTest on GCP. */ +@QuarkusTest public class PolarisRestCatalogViewGcpIntegrationTest extends PolarisRestCatalogViewIntegrationTest { public static final String SERVICE_ACCOUNT = diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java index 215288dbc..a1ab497aa 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java @@ -20,12 +20,10 @@ import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; -import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Path; import java.util.Map; import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.view.ViewCatalogTests; @@ -34,75 +32,64 @@ import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension.PolarisToken; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension.SnowmanCredentials; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestHelper; import org.apache.polaris.service.dropwizard.test.TestEnvironment; import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; /** * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog * client. */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@ExtendWith(TestEnvironmentExtension.class) public abstract class PolarisRestCatalogViewIntegrationTest extends ViewCatalogTests { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism + @Inject PolarisIntegrationTestHelper helper; + + private TestEnvironment testEnv; + private PolarisIntegrationTestFixture fixture; private RESTCatalog restCatalog; @BeforeAll - public static void setup(@PolarisRealm String realm) throws IOException { - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + public void createFixture(TestEnvironment testEnv, TestInfo testInfo) { + Assumptions.assumeFalse(shouldSkip()); + this.testEnv = testEnv; + fixture = helper.createFixture(testEnv, testInfo); } @BeforeEach - public void before( - TestInfo testInfo, - PolarisToken adminToken, - SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm, - TestEnvironment testEnv) { - - Assumptions.assumeFalse(shouldSkip()); + public void setUpTempDir(@TempDir Path tempDir) throws Exception { + // see https://github.com/quarkusio/quarkus/issues/13261 + Field field = ViewCatalogTests.class.getDeclaredField("tempDir"); + field.setAccessible(true); + field.set(this, tempDir); + } - String userToken = adminToken.token(); + @BeforeEach + void before(TestInfo testInfo) { testInfo .getTestMethod() .ifPresent( method -> { - String catalogName = method.getName() + testEnv.testId(); + String catalogName = method.getName(); try (Response response = - testEnv - .apiClient() + fixture + .client .target( String.format( "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), catalogName)) .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "Bearer " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { if (response.getStatus() == Response.Status.OK.getStatusCode()) { // Already exists! Must be in a parameterized test. @@ -139,17 +126,17 @@ public void before( .setStorageConfigInfo(storageConfig) .build(); restCatalog = - TestUtil.createSnowmanManagedCatalog( - testEnv.apiClient(), - testEnv.baseUri().toString(), - adminToken, - snowmanCredentials, - realm, - catalog, - Map.of()); + TestUtil.createSnowmanManagedCatalog(testEnv, fixture, catalog, Map.of()); }); } + @AfterAll + public void destroyFixture() { + if (fixture != null) { + fixture.destroy(); + } + } + /** * @return The catalog's storage config. */ diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java index 173e2bda6..047f356f5 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java @@ -23,13 +23,10 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.adobe.testing.s3mock.testcontainers.S3MockContainer; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; -import java.io.IOException; import java.time.Instant; import java.util.List; import java.util.Map; @@ -41,10 +38,9 @@ import org.apache.polaris.core.admin.model.ExternalCatalog; import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestHelper; +import org.apache.polaris.service.dropwizard.test.TestEnvironment; import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; import org.apache.polaris.service.types.NotificationRequest; import org.apache.polaris.service.types.NotificationType; @@ -58,52 +54,43 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.LoggerFactory; -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) +@QuarkusTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@ExtendWith(TestEnvironmentExtension.class) public class PolarisSparkIntegrationTest { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism public static final String CATALOG_NAME = "mycatalog"; public static final String EXTERNAL_CATALOG_NAME = "external_catalog"; - private static final S3MockContainer s3Container = + + private final S3MockContainer s3Container = new S3MockContainer("3.11.0").withInitialBuckets("my-bucket,my-old-bucket"); - private static PolarisConnectionExtension.PolarisToken polarisToken; - private static SparkSession spark; - private String realm; + + @Inject PolarisIntegrationTestHelper helper; + + private TestEnvironment testEnv; + private PolarisIntegrationTestFixture fixture; + private SparkSession spark; @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken polarisToken, @PolarisRealm String realm) - throws IOException { + public void createFixture(TestEnvironment testEnv, TestInfo testInfo) { + this.testEnv = testEnv; s3Container.start(); - PolarisSparkIntegrationTest.polarisToken = polarisToken; - - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + fixture = helper.createFixture(testEnv, testInfo); } @AfterAll - public static void cleanup() { + public void destroyFixture() { + fixture.destroy(); s3Container.stop(); } @BeforeEach - public void before(@PolarisRealm String realm) { - this.realm = realm; + public void before() { AwsStorageConfigInfo awsConfigModel = AwsStorageConfigInfo.builder() .setRoleArn("arn:aws:iam::123456789012:role/my-role") @@ -140,12 +127,12 @@ public void before(@PolarisRealm String realm) { .build(); try (Response response = - EXT.client() - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) + fixture + .client + .target(String.format("%s/api/management/v1/catalogs", testEnv.baseUri())) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(catalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -178,12 +165,12 @@ public void before(@PolarisRealm String realm) { .setRemoteUrl("http://dummy_url") .build(); try (Response response = - EXT.client() - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) + fixture + .client + .target(String.format("%s/api/management/v1/catalogs", testEnv.baseUri())) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(externalCatalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -215,11 +202,11 @@ private SparkSession.Builder withCatalog(SparkSession.Builder builder, String ca .config(String.format("spark.sql.catalog.%s.type", catalogName), "rest") .config( String.format("spark.sql.catalog.%s.uri", catalogName), - "http://localhost:" + EXT.getLocalPort() + "/api/catalog") + testEnv.baseUri() + "/api/catalog") .config(String.format("spark.sql.catalog.%s.warehouse", catalogName), catalogName) .config(String.format("spark.sql.catalog.%s.scope", catalogName), "PRINCIPAL_ROLE:ALL") - .config(String.format("spark.sql.catalog.%s.header.realm", catalogName), realm) - .config(String.format("spark.sql.catalog.%s.token", catalogName), polarisToken.token()) + .config(String.format("spark.sql.catalog.%s.header.realm", catalogName), fixture.realm) + .config(String.format("spark.sql.catalog.%s.token", catalogName), fixture.adminToken) .config(String.format("spark.sql.catalog.%s.s3.access-key-id", catalogName), "fakekey") .config( String.format("spark.sql.catalog.%s.s3.secret-access-key", catalogName), "fakesecret") @@ -254,14 +241,13 @@ private void cleanupCatalog(String catalogName) { onSpark("DROP NAMESPACE " + namespace.getString(0)); } try (Response response = - EXT.client() + fixture + .client .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName, - EXT.getLocalPort())) + String.format("%s/api/management/v1/catalogs/" + catalogName, testEnv.baseUri())) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -303,16 +289,17 @@ public void testCreateAndUpdateExternalTable() { LoadTableResponse tableResponse = loadTable(CATALOG_NAME, "ns1", "tb1"); try (Response registerResponse = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/" + "%s/api/catalog/v1/" + EXTERNAL_CATALOG_NAME + "/namespaces/externalns1/register", - EXT.getLocalPort())) + testEnv.baseUri())) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post( Entity.json( ImmutableRegisterTableRequest.builder() @@ -344,14 +331,15 @@ public void testCreateAndUpdateExternalTable() { notificationRequest.setPayload(updateNotification); notificationRequest.setNotificationType(NotificationType.UPDATE); try (Response notifyResponse = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/externalns1/tables/mytb1/notifications", - EXT.getLocalPort(), EXTERNAL_CATALOG_NAME)) + "%s/api/catalog/v1/%s/namespaces/externalns1/tables/mytb1/notifications", + testEnv.baseUri(), EXTERNAL_CATALOG_NAME)) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .post(Entity.json(notificationRequest))) { assertThat(notifyResponse) .returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -378,21 +366,22 @@ public void testCreateView() { private LoadTableResponse loadTable(String catalog, String namespace, String table) { try (Response response = - EXT.client() + fixture + .client .target( String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), catalog, namespace, table)) + "%s/api/catalog/v1/%s/namespaces/%s/tables/%s", + testEnv.baseUri(), catalog, namespace, table)) .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + .header("Authorization", "BEARER " + fixture.adminToken) + .header(REALM_PROPERTY_KEY, fixture.realm) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); return response.readEntity(LoadTableResponse.class); } } - private static Dataset onSpark(@Language("SQL") String sql) { + private Dataset onSpark(@Language("SQL") String sql) { return spark.sql(sql); } } diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/TestUtil.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/TestUtil.java index c2342d8b8..2f5a9c898 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/TestUtil.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/TestUtil.java @@ -22,10 +22,9 @@ import static org.assertj.core.api.Assertions.assertThat; import com.google.common.collect.ImmutableMap; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import jakarta.ws.rs.client.Client; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; +import java.net.URI; import java.util.Map; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.catalog.SessionCatalog; @@ -38,50 +37,45 @@ import org.apache.polaris.core.admin.model.CatalogRole; import org.apache.polaris.core.admin.model.GrantResource; import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.dropwizard.test.TestEnvironment; /** Test utilities for catalog tests */ public class TestUtil { - /** Performs createSnowmanManagedCatalog() on a Dropwizard instance of Polaris */ + + /** + * Creates a catalog and grants the snowman principal permission to manage it. + * + * @return A client to interact with the catalog. + */ public static RESTCatalog createSnowmanManagedCatalog( - DropwizardAppExtension EXT, - PolarisConnectionExtension.PolarisToken adminToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, - String realm, + TestEnvironment testEnv, + PolarisIntegrationTestFixture fixture, Catalog catalog, Map extraProperties) { return createSnowmanManagedCatalog( - EXT.client(), - String.format("http://localhost:%d", EXT.getLocalPort()), - adminToken, - snowmanCredentials, - realm, - catalog, - extraProperties); + testEnv, fixture, testEnv.baseUri(), fixture.realm, catalog, extraProperties); } /** - * Creates a catalog and grants the snowman principal from SnowmanCredentialsExtension permission - * to manage it. + * Creates a catalog and grants the snowman principal permission to manage it. * * @return A client to interact with the catalog. */ public static RESTCatalog createSnowmanManagedCatalog( - Client client, - String baseUrl, - PolarisConnectionExtension.PolarisToken adminToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, + TestEnvironment testEnv, + PolarisIntegrationTestFixture fixture, + URI baseUri, String realm, Catalog catalog, Map extraProperties) { String currentCatalogName = catalog.getName(); try (Response response = - client - .target(String.format("%s/api/management/v1/catalogs", baseUrl)) + fixture + .client + .target(String.format("%s/api/management/v1/catalogs", baseUri)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + fixture.adminToken) .header(REALM_PROPERTY_KEY, realm) .post(Entity.json(catalog))) { assertStatusCode(response, Response.Status.CREATED.getStatusCode()); @@ -90,12 +84,13 @@ public static RESTCatalog createSnowmanManagedCatalog( // Create a new CatalogRole that has CATALOG_MANAGE_CONTENT and CATALOG_MANAGE_ACCESS CatalogRole newRole = new CatalogRole("custom-admin"); try (Response response = - client + fixture + .client .target( String.format( - "%s/api/management/v1/catalogs/%s/catalog-roles", baseUrl, currentCatalogName)) + "%s/api/management/v1/catalogs/%s/catalog-roles", baseUri, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + fixture.adminToken) .header(REALM_PROPERTY_KEY, realm) .post(Entity.json(newRole))) { assertStatusCode(response, Response.Status.CREATED.getStatusCode()); @@ -103,13 +98,14 @@ public static RESTCatalog createSnowmanManagedCatalog( CatalogGrant grantResource = new CatalogGrant(CatalogPrivilege.CATALOG_MANAGE_CONTENT, GrantResource.TypeEnum.CATALOG); try (Response response = - client + fixture + .client .target( String.format( "%s/api/management/v1/catalogs/%s/catalog-roles/custom-admin/grants", - baseUrl, currentCatalogName)) + baseUri, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + fixture.adminToken) .header(REALM_PROPERTY_KEY, realm) .put(Entity.json(grantResource))) { assertStatusCode(response, Response.Status.CREATED.getStatusCode()); @@ -117,13 +113,14 @@ public static RESTCatalog createSnowmanManagedCatalog( CatalogGrant grantAccessResource = new CatalogGrant(CatalogPrivilege.CATALOG_MANAGE_ACCESS, GrantResource.TypeEnum.CATALOG); try (Response response = - client + fixture + .client .target( String.format( "%s/api/management/v1/catalogs/%s/catalog-roles/custom-admin/grants", - baseUrl, currentCatalogName)) + baseUri, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + fixture.adminToken) .header(REALM_PROPERTY_KEY, realm) .put(Entity.json(grantAccessResource))) { assertStatusCode(response, Response.Status.CREATED.getStatusCode()); @@ -131,28 +128,30 @@ public static RESTCatalog createSnowmanManagedCatalog( // Assign this new CatalogRole to the service_admin PrincipalRole try (Response response = - client + fixture + .client .target( String.format( "%s/api/management/v1/catalogs/%s/catalog-roles/custom-admin", - baseUrl, currentCatalogName)) + baseUri, currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + fixture.adminToken) .header(REALM_PROPERTY_KEY, realm) .get()) { assertStatusCode(response, Response.Status.OK.getStatusCode()); CatalogRole catalogRole = response.readEntity(CatalogRole.class); try (Response assignResponse = - client + fixture + .client .target( String.format( "%s/api/management/v1/principal-roles/%s/catalog-roles/%s", - baseUrl, - snowmanCredentials.identifier().principalRoleName(), + baseUri, + fixture.snowmanCredentials.identifier().principalRoleName(), currentCatalogName)) .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) + .header("Authorization", "Bearer " + fixture.adminToken) .header(REALM_PROPERTY_KEY, realm) .put(Entity.json(catalogRole))) { assertStatusCode(assignResponse, Response.Status.CREATED.getStatusCode()); @@ -167,10 +166,12 @@ public static RESTCatalog createSnowmanManagedCatalog( ImmutableMap.Builder propertiesBuilder = ImmutableMap.builder() - .put(CatalogProperties.URI, baseUrl + "/api/catalog") + .put(CatalogProperties.URI, baseUri + "/api/catalog") .put( OAuth2Properties.CREDENTIAL, - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret()) + fixture.snowmanCredentials.clientId() + + ":" + + fixture.snowmanCredentials.clientSecret()) .put(OAuth2Properties.SCOPE, BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL) .put(CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO") .put("warehouse", currentCatalogName) diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/TestFileIOFactory.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/TestFileIOFactory.java index fa59f5678..9c0b04875 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/TestFileIOFactory.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/io/TestFileIOFactory.java @@ -18,7 +18,7 @@ */ package org.apache.polaris.service.dropwizard.catalog.io; -import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.inject.Vetoed; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -27,14 +27,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.io.FileIO; -import org.apache.polaris.service.catalog.io.FileIOFactory; +import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; /** * A FileIOFactory that measures the number of bytes read, files written, and files deleted. It can * inject exceptions at various parts of the IO construction. */ -@Identifier("test") -public class TestFileIOFactory implements FileIOFactory { +@Vetoed +public class TestFileIOFactory extends DefaultFileIOFactory { private final List ios = new ArrayList<>(); // When present, the following will be used to throw exceptions at various parts of the IO diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/MockTokenBucketFactory.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/MockTokenBucketFactory.java index 6516e7c79..509d2af42 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/MockTokenBucketFactory.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/MockTokenBucketFactory.java @@ -18,23 +18,27 @@ */ package org.apache.polaris.service.dropwizard.ratelimiter; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Alternative; +import jakarta.inject.Inject; import java.time.Instant; import java.time.ZoneOffset; import org.apache.polaris.service.ratelimiter.DefaultTokenBucketFactory; +import org.apache.polaris.service.ratelimiter.RateLimiterConfiguration; import org.threeten.extra.MutableClock; /** TokenBucketFactory with a mock clock */ -@Identifier("mock") +@Alternative +@ApplicationScoped public class MockTokenBucketFactory extends DefaultTokenBucketFactory { public static MutableClock CLOCK = MutableClock.of(Instant.now(), ZoneOffset.UTC); - @JsonCreator - public MockTokenBucketFactory( - @JsonProperty("requestsPerSecond") long requestsPerSecond, - @JsonProperty("windowSeconds") long windowSeconds) { - super(requestsPerSecond, windowSeconds, CLOCK); + public MockTokenBucketFactory() { + super(0, null, CLOCK); + } + + @Inject + public MockTokenBucketFactory(RateLimiterConfiguration configuration) { + super(configuration, CLOCK); } } diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/RateLimiterFilterTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/RateLimiterFilterTest.java index a2dc8370a..1717658f2 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/RateLimiterFilterTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/RateLimiterFilterTest.java @@ -18,112 +18,173 @@ */ package org.apache.polaris.service.dropwizard.ratelimiter; -import static org.apache.polaris.service.dropwizard.TimedApplicationEventListener.SINGLETON_METRIC_NAME; -import static org.apache.polaris.service.dropwizard.TimedApplicationEventListener.TAG_API_NAME; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.SUFFIX_ERROR; -import static org.apache.polaris.service.dropwizard.monitor.PolarisMetricRegistry.TAG_RESP_CODE; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; -import io.micrometer.core.instrument.Tag; -import jakarta.ws.rs.core.Response; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.type; + +import io.micrometer.core.instrument.MeterRegistry; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import jakarta.ws.rs.core.Response.Status; import java.time.Duration; -import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.function.Consumer; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; +import org.apache.polaris.service.dropwizard.ratelimiter.RateLimiterFilterTest.Profile; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestHelper; +import org.apache.polaris.service.dropwizard.test.TestEnvironment; import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; import org.apache.polaris.service.dropwizard.test.TestMetricsUtil; +import org.hawkular.agent.prometheus.types.MetricFamily; +import org.hawkular.agent.prometheus.types.Summary; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; import org.junit.jupiter.api.extension.ExtendWith; -import org.threeten.extra.MutableClock; /** Main integration tests for rate limiting */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) +@QuarkusTest +@TestInstance(Lifecycle.PER_CLASS) +@TestProfile(Profile.class) +@ExtendWith(TestEnvironmentExtension.class) public class RateLimiterFilterTest { + + public static class Profile implements QuarkusTestProfile { + + @Override + public Set> getEnabledAlternatives() { + return Set.of(MockTokenBucketFactory.class); + } + + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.rate-limiter.type", + "default", + "polaris.rate-limiter.token-bucket.type", + "default", + "polaris.rate-limiter.token-bucket.requests-per-second", + String.valueOf(REQUESTS_PER_SECOND), + "polaris.rate-limiter.token-bucket.window", + WINDOW.toString(), + "polaris.metrics.tags.environment", + "prod"); + } + } + private static final long REQUESTS_PER_SECOND = 5; - private static final long WINDOW_SECONDS = 10; - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config("server.adminConnectors[0].port", "0"), - ConfigOverride.config("tokenBucketFactory.type", "mock"), - ConfigOverride.config( - "tokenBucketFactory.requestsPerSecond", String.valueOf(REQUESTS_PER_SECOND)), - ConfigOverride.config( - "tokenBucketFactory.windowSeconds", String.valueOf(WINDOW_SECONDS))); - - private static String userToken; - private static String realm; - private static MutableClock clock = MockTokenBucketFactory.CLOCK; + private static final Duration WINDOW = Duration.ofSeconds(10); + + @Inject PolarisIntegrationTestHelper helper; + @Inject MeterRegistry meterRegistry; + + private TestEnvironment testEnv; + private PolarisIntegrationTestFixture fixture; @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken userToken, @PolarisRealm String polarisRealm) { - realm = polarisRealm; - RateLimiterFilterTest.userToken = userToken.token(); + public void createFixture(TestEnvironment testEnv, TestInfo testInfo) { + this.testEnv = testEnv; + fixture = helper.createFixture(testEnv, testInfo); + } + + @AfterAll + public void destroyFixture() { + if (fixture != null) { + fixture.destroy(); + } } @BeforeEach @AfterEach public void resetRateLimiter() { - clock.add( - Duration.ofSeconds(2 * WINDOW_SECONDS)); // Clear any counters from before/after this test + MockTokenBucketFactory.CLOCK.add( + WINDOW.multipliedBy(2)); // Clear any counters from before/after this test + } + + @BeforeEach + public void resetMeterRegistry() { + meterRegistry.clear(); } @Test public void testRateLimiter() { - Consumer requestAsserter = - TestUtil.constructRequestAsserter(EXT, userToken, realm); + Consumer requestAsserter = + TestUtil.constructRequestAsserter(testEnv, fixture, fixture.realm); - for (int i = 0; i < REQUESTS_PER_SECOND * WINDOW_SECONDS; i++) { - requestAsserter.accept(Response.Status.OK); + for (int i = 0; i < REQUESTS_PER_SECOND * WINDOW.toSeconds(); i++) { + requestAsserter.accept(Status.OK); } - requestAsserter.accept(Response.Status.TOO_MANY_REQUESTS); + requestAsserter.accept(Status.TOO_MANY_REQUESTS); // Ensure that a different realm identifier gets a separate limit - Consumer requestAsserter2 = - TestUtil.constructRequestAsserter(EXT, userToken, realm + "2"); - requestAsserter2.accept(Response.Status.OK); + Consumer requestAsserter2 = + TestUtil.constructRequestAsserter(testEnv, fixture, fixture.realm + "2"); + requestAsserter2.accept(Status.OK); } @Test public void testMetricsAreEmittedWhenRateLimiting() { - Consumer requestAsserter = - TestUtil.constructRequestAsserter(EXT, userToken, realm); + Consumer requestAsserter = + TestUtil.constructRequestAsserter(testEnv, fixture, fixture.realm); - for (int i = 0; i < REQUESTS_PER_SECOND * WINDOW_SECONDS; i++) { - requestAsserter.accept(Response.Status.OK); + for (int i = 0; i < REQUESTS_PER_SECOND * WINDOW.toSeconds(); i++) { + requestAsserter.accept(Status.OK); } - requestAsserter.accept(Response.Status.TOO_MANY_REQUESTS); - - assertTrue( - TestMetricsUtil.getTotalCounter( - EXT, - SINGLETON_METRIC_NAME + SUFFIX_ERROR, - List.of( - Tag.of(TAG_API_NAME, "polaris.principal-roles.listPrincipalRoles"), - Tag.of( - TAG_RESP_CODE, - String.valueOf(Response.Status.TOO_MANY_REQUESTS.getStatusCode())))) - > 0); + requestAsserter.accept(Status.TOO_MANY_REQUESTS); + + // Examples of expected metrics: + // http_server_requests_seconds_count{application="Polaris",environment="prod",method="GET",outcome="CLIENT_ERROR",realm_id="org_apache_polaris_service_ratelimiter_RateLimiterFilterTest",status="429",uri="/api/management/v1/principal-roles"} 1.0 + // polaris_principal_roles_listPrincipalRoles_seconds_count{application="Polaris",class="org.apache.polaris.service.admin.api.PolarisPrincipalRolesApi",environment="prod",exception="none",method="listPrincipalRoles"} 50.0 + + Map metrics = + TestMetricsUtil.fetchMetrics(fixture.client, testEnv.baseManagementUri()); + + assertThat(metrics) + .isNotEmpty() + .containsKey("http_server_requests_seconds") + .containsKey("polaris_principal_roles_listPrincipalRoles_seconds"); + + assertThat(metrics.get("http_server_requests_seconds").getMetrics()) + .satisfiesOnlyOnce( + metric -> { + assertThat(metric.getLabels()) + .contains( + Map.entry("application", "Polaris"), + Map.entry("environment", "prod"), + Map.entry("realm_id", fixture.realm), + Map.entry("method", "GET"), + Map.entry("outcome", "CLIENT_ERROR"), + Map.entry("status", String.valueOf(Status.TOO_MANY_REQUESTS.getStatusCode())), + Map.entry("uri", "/api/management/v1/principal-roles")); + assertThat(metric) + .asInstanceOf(type(Summary.class)) + .extracting(Summary::getSampleCount) + .isEqualTo(1L); + }); + + assertThat(metrics.get("polaris_principal_roles_listPrincipalRoles_seconds").getMetrics()) + .satisfiesOnlyOnce( + metric -> { + assertThat(metric.getLabels()) + .contains( + Map.entry("application", "Polaris"), + Map.entry("environment", "prod"), + Map.entry("realm_id", fixture.realm), + Map.entry( + "class", "org.apache.polaris.service.admin.api.PolarisPrincipalRolesApi"), + Map.entry("exception", "none"), + Map.entry("method", "listPrincipalRoles")); + assertThat(metric) + .asInstanceOf(type(Summary.class)) + .extracting(Summary::getSampleCount) + .isEqualTo(REQUESTS_PER_SECOND * WINDOW.toSeconds()); + }); } } diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/RealmTokenBucketRateLimiterTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/RealmTokenBucketRateLimiterTest.java deleted file mode 100644 index efeee130c..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/RealmTokenBucketRateLimiterTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.ratelimiter; - -import static org.apache.polaris.service.dropwizard.ratelimiter.MockTokenBucketFactory.CLOCK; - -import java.time.Duration; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.service.ratelimiter.DefaultTokenBucketFactory; -import org.apache.polaris.service.ratelimiter.RealmTokenBucketRateLimiter; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -/** Main unit test class for TokenBucketRateLimiter */ -public class RealmTokenBucketRateLimiterTest { - @Test - void testDifferentBucketsDontTouch() { - RealmTokenBucketRateLimiter rateLimiter = new RealmTokenBucketRateLimiter(); - rateLimiter.setTokenBucketFactory(new DefaultTokenBucketFactory(10, 10, CLOCK)); - - for (int i = 0; i < 202; i++) { - String realm = (i % 2 == 0) ? "realm1" : "realm2"; - CallContext.setCurrentContext(CallContext.of(() -> realm, null)); - - if (i < 200) { - Assertions.assertTrue(rateLimiter.canProceed()); - } else { - assertCannotProceed(rateLimiter); - } - } - - CLOCK.add(Duration.ofSeconds(1)); - for (int i = 0; i < 22; i++) { - String realm = (i % 2 == 0) ? "realm1" : "realm2"; - CallContext.setCurrentContext(CallContext.of(() -> realm, null)); - - if (i < 20) { - Assertions.assertTrue(rateLimiter.canProceed()); - } else { - assertCannotProceed(rateLimiter); - } - } - } - - private void assertCannotProceed(RealmTokenBucketRateLimiter rateLimiter) { - for (int i = 0; i < 5; i++) { - Assertions.assertFalse(rateLimiter.canProceed()); - } - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TestUtil.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TestUtil.java index 1cd5789df..f33cb6517 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TestUtil.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TestUtil.java @@ -21,10 +21,11 @@ import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; import static org.assertj.core.api.Assertions.assertThat; -import io.dropwizard.testing.junit5.DropwizardAppExtension; import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.Response.Status; import java.util.function.Consumer; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; +import org.apache.polaris.service.dropwizard.test.PolarisIntegrationTestFixture; +import org.apache.polaris.service.dropwizard.test.TestEnvironment; /** Common test utils for testing rate limiting */ public class TestUtil { @@ -33,20 +34,15 @@ public class TestUtil { * of the response. This is a relatively simple type of request that can be used for validating * whether the rate limiter intervenes. */ - public static Consumer constructRequestAsserter( - DropwizardAppExtension dropwizardAppExtension, - String userToken, - String realm) { + public static Consumer constructRequestAsserter( + TestEnvironment testEnv, PolarisIntegrationTestFixture fixture, String realm) { return (Response.Status status) -> { try (Response response = - dropwizardAppExtension - .client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", - dropwizardAppExtension.getLocalPort())) + fixture + .client + .target(String.format("%s/api/management/v1/principal-roles", testEnv.baseUri())) .request("application/json") - .header("Authorization", "Bearer " + userToken) + .header("Authorization", "Bearer " + fixture.adminToken) .header(REALM_PROPERTY_KEY, realm) .get()) { assertThat(response).returns(status.getStatusCode(), Response::getStatus); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TokenBucketRateLimiterTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TokenBucketTest.java similarity index 96% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TokenBucketRateLimiterTest.java rename to dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TokenBucketTest.java index 472b79aca..33a7451ea 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TokenBucketRateLimiterTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/ratelimiter/TokenBucketTest.java @@ -31,8 +31,8 @@ import org.junit.jupiter.api.Test; import org.threeten.extra.MutableClock; -/** Main unit test class for TokenBucketRateLimiter */ -public class TokenBucketRateLimiterTest { +/** Main unit test class for TokenBucket */ +public class TokenBucketTest { @Test void testBasic() { MutableClock clock = MutableClock.of(Instant.now(), ZoneOffset.UTC); @@ -70,7 +70,6 @@ void testConcurrent() throws InterruptedException { try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { for (int i = 0; i < numTasks; i++) { - int i_ = i; executor.submit( () -> { try { diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/ManifestFileCleanupTaskHandlerTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/ManifestFileCleanupTaskHandlerTest.java index b594b8e01..7e6487931 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/ManifestFileCleanupTaskHandlerTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/ManifestFileCleanupTaskHandlerTest.java @@ -22,6 +22,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatPredicate; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import java.io.IOException; import java.util.HashMap; import java.util.List; @@ -50,21 +52,16 @@ import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.AsyncTaskType; import org.apache.polaris.core.entity.TaskEntity; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.service.task.ManifestFileCleanupTaskHandler; import org.apache.polaris.service.task.TaskUtils; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +@QuarkusTest class ManifestFileCleanupTaskHandlerTest { - private InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory; - private RealmContext realmContext; + @Inject MetaStoreManagerFactory metaStoreManagerFactory; - @BeforeEach - void setUp() { - metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); - realmContext = () -> "realmName"; - } + private final RealmContext realmContext = () -> "realmName"; @Test public void testCleanupFileNotExists() throws IOException { diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/TableCleanupTaskHandlerTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/TableCleanupTaskHandlerTest.java index 1fc788c34..90188dc1e 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/TableCleanupTaskHandlerTest.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/task/TableCleanupTaskHandlerTest.java @@ -20,6 +20,8 @@ import static org.assertj.core.api.Assertions.assertThat; +import io.quarkus.test.junit.QuarkusTest; +import jakarta.inject.Inject; import java.io.IOException; import java.util.List; import java.util.UUID; @@ -42,25 +44,20 @@ import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.TableLikeEntity; import org.apache.polaris.core.entity.TaskEntity; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.service.task.ManifestFileCleanupTaskHandler; import org.apache.polaris.service.task.TableCleanupTaskHandler; import org.apache.polaris.service.task.TaskUtils; import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.slf4j.LoggerFactory; +@QuarkusTest class TableCleanupTaskHandlerTest { - private InMemoryPolarisMetaStoreManagerFactory metaStoreManagerFactory; - private RealmContext realmContext; + @Inject MetaStoreManagerFactory metaStoreManagerFactory; - @BeforeEach - void setUp() { - metaStoreManagerFactory = new InMemoryPolarisMetaStoreManagerFactory(); - realmContext = () -> "realmName"; - } + private final RealmContext realmContext = () -> "realmName"; @Test public void testTableCleanup() throws IOException { diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/DefaultTestEnvironmentResolver.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/DefaultTestEnvironmentResolver.java new file mode 100644 index 000000000..d5330d550 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/DefaultTestEnvironmentResolver.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.test; + +import org.junit.jupiter.api.extension.ExtensionContext; + +public class DefaultTestEnvironmentResolver implements TestEnvironmentResolver { + + private final int localPort = Integer.getInteger("quarkus.http.port"); + private final int localManagementPort = Integer.getInteger("quarkus.management.port"); + + /** Resolves the TestEnvironment to point to the local Quarkus Application instance. */ + @Override + public TestEnvironment resolveTestEnvironment(ExtensionContext extensionContext) { + return new TestEnvironment( + String.format("http://localhost:%d", localPort), + String.format("http://localhost:%d", localManagementPort)); + } +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/DropwizardTestEnvironmentResolver.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/DropwizardTestEnvironmentResolver.java deleted file mode 100644 index 0676fb20a..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/DropwizardTestEnvironmentResolver.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.test; - -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import jakarta.annotation.Nullable; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.util.Arrays; -import java.util.Optional; -import org.apache.polaris.core.entity.PolarisGrantRecord; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.extension.ParameterResolutionException; -import org.junit.platform.commons.util.ReflectionUtils; -import org.slf4j.LoggerFactory; - -public class DropwizardTestEnvironmentResolver implements TestEnvironmentResolver { - /** - * Resolves the TestEnvironment to point to the local Dropwizard instance. - * - * @param extensionContext - * @return - */ - @Override - public TestEnvironment resolveTestEnvironment(ExtensionContext extensionContext) { - try { - DropwizardAppExtension dropwizardAppExtension = findDropwizardExtension(extensionContext); - if (dropwizardAppExtension == null) { - throw new ParameterResolutionException("Could not find DropwizardAppExtension."); - } - return new TestEnvironment( - dropwizardAppExtension.client(), - String.format("http://localhost:%d", dropwizardAppExtension.getLocalPort())); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - } - - public static @Nullable DropwizardAppExtension findDropwizardExtension( - ExtensionContext extensionContext) throws IllegalAccessException { - Field dropwizardExtensionField = - findAnnotatedFields(extensionContext.getRequiredTestClass(), true); - if (dropwizardExtensionField == null) { - LoggerFactory.getLogger(PolarisGrantRecord.class) - .warn( - "Unable to find dropwizard extension field in test class {}", - extensionContext.getRequiredTestClass()); - return null; - } - DropwizardAppExtension appExtension = - (DropwizardAppExtension) ReflectionUtils.makeAccessible(dropwizardExtensionField).get(null); - return appExtension; - } - - private static Field findAnnotatedFields(Class testClass, boolean isStaticMember) { - final Optional set = - Arrays.stream(testClass.getDeclaredFields()) - .filter(m -> isStaticMember == Modifier.isStatic(m.getModifiers())) - .filter(m -> DropwizardAppExtension.class.isAssignableFrom(m.getType())) - .findFirst(); - if (set.isPresent()) { - return set.get(); - } - if (!testClass.getSuperclass().equals(Object.class)) { - return findAnnotatedFields(testClass.getSuperclass(), isStaticMember); - } - return null; - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisConnectionExtension.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisConnectionExtension.java deleted file mode 100644 index e11cc19b4..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisConnectionExtension.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.test; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.apache.polaris.service.dropwizard.test.DropwizardTestEnvironmentResolver.findDropwizardExtension; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import java.io.IOException; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.context.RealmContext; -import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.core.persistence.MetaStoreManagerFactory; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.service.context.CallContextResolver; -import org.apache.polaris.service.context.RealmContextResolver; -import org.apache.polaris.service.dropwizard.auth.TokenUtils; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; -import org.junit.jupiter.api.extension.AfterAllCallback; -import org.junit.jupiter.api.extension.BeforeAllCallback; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.extension.ExtensionContext.Namespace; -import org.junit.jupiter.api.extension.ParameterContext; -import org.junit.jupiter.api.extension.ParameterResolutionException; -import org.junit.jupiter.api.extension.ParameterResolver; - -public class PolarisConnectionExtension - implements BeforeAllCallback, AfterAllCallback, ParameterResolver { - - public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private MetaStoreManagerFactory metaStoreManagerFactory; - private DropwizardAppExtension dropwizardAppExtension; - - public record PolarisToken(String token) {} - - private static PolarisPrincipalSecrets adminSecrets; - private static String realm; - - @Override - public void beforeAll(ExtensionContext extensionContext) throws Exception { - dropwizardAppExtension = findDropwizardExtension(extensionContext); - if (dropwizardAppExtension == null) { - return; - } - - // Generate unique realm using test name for each test since the tests can run in parallel - realm = extensionContext.getRequiredTestClass().getName().replace('.', '_'); - extensionContext - .getStore(Namespace.create(extensionContext.getRequiredTestClass())) - .put(REALM_PROPERTY_KEY, realm); - - try { - PolarisApplicationConfig config = - (PolarisApplicationConfig) dropwizardAppExtension.getConfiguration(); - metaStoreManagerFactory = config.findService(MetaStoreManagerFactory.class); - if (!(metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory)) { - metaStoreManagerFactory.bootstrapRealms(List.of(realm)); - } - - URI testEnvUri = TestEnvironmentExtension.getEnv(extensionContext).baseUri(); - String path = testEnvUri.getPath(); - if (path.isEmpty()) { - path = "/"; - } - - RealmContext realmContext = - config - .findService(RealmContextResolver.class) - .resolveRealmContext( - String.format("%s://%s", testEnvUri.getScheme(), testEnvUri.getHost()), - "GET", - path, - Map.of(), - Map.of(REALM_PROPERTY_KEY, realm)); - CallContext ctx = - config - .findService(CallContextResolver.class) - .resolveCallContext(realmContext, "GET", path, Map.of(), Map.of()); - CallContext.setCurrentContext(ctx); - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager(ctx.getRealmContext()); - PolarisMetaStoreManager.EntityResult principal = - metaStoreManager.readEntityByName( - ctx.getPolarisCallContext(), - null, - PolarisEntityType.PRINCIPAL, - PolarisEntitySubType.NULL_SUBTYPE, - PolarisEntityConstants.getRootPrincipalName()); - - Map propertiesMap = readInternalProperties(principal); - adminSecrets = - metaStoreManager - .loadPrincipalSecrets(ctx.getPolarisCallContext(), propertiesMap.get("client_id")) - .getPrincipalSecrets(); - } finally { - CallContext.unsetCurrentContext(); - } - } - - @Override - public void afterAll(ExtensionContext context) { - if (!(metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory)) { - metaStoreManagerFactory.purgeRealms(List.of(realm)); - } - } - - public static void createTestDir(String realm) throws IOException { - // Set up the database location - Path testDir = Path.of("build/test_data/polaris/" + realm); - if (Files.exists(testDir)) { - if (Files.isDirectory(testDir)) { - Files.walk(testDir) - .sorted(Comparator.reverseOrder()) - .forEach( - path -> { - try { - Files.delete(path); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - - } else { - Files.delete(testDir); - } - } - Files.createDirectories(testDir); - } - - static PolarisPrincipalSecrets getAdminSecrets() { - return adminSecrets; - } - - @Override - public boolean supportsParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - return parameterContext - .getParameter() - .getType() - .equals(PolarisConnectionExtension.PolarisToken.class) - || parameterContext.getParameter().getType().equals(MetaStoreManagerFactory.class) - || parameterContext.getParameter().getType().equals(PolarisPrincipalSecrets.class) - || (parameterContext.getParameter().getType().equals(String.class) - && parameterContext.getParameter().isAnnotationPresent(PolarisRealm.class)); - } - - @Override - public Object resolveParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - if (parameterContext.getParameter().getType().equals(PolarisToken.class)) { - try { - TestEnvironment testEnv = TestEnvironmentExtension.getEnv(extensionContext); - String token = - TokenUtils.getTokenFromSecrets( - testEnv.apiClient(), - testEnv.baseUri().toString(), - adminSecrets.getPrincipalClientId(), - adminSecrets.getMainSecret(), - realm); - return new PolarisToken(token); - } catch (IllegalAccessException e) { - throw new ParameterResolutionException(e.getMessage()); - } - } else if (parameterContext.getParameter().getType().equals(String.class) - && parameterContext.getParameter().isAnnotationPresent(PolarisRealm.class)) { - return realm; - } else if (parameterContext.getParameter().getType().equals(PolarisPrincipalSecrets.class)) { - return adminSecrets; - } else { - return metaStoreManagerFactory; - } - } - - private static Map readInternalProperties( - PolarisMetaStoreManager.EntityResult principal) { - try { - return OBJECT_MAPPER.readValue( - principal.getEntity().getInternalProperties(), - new TypeReference>() {}); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisIntegrationTestFixture.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisIntegrationTestFixture.java new file mode 100644 index 000000000..a6ba12605 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisIntegrationTestFixture.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.test; + +import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientBuilder; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.List; +import java.util.Map; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; +import org.apache.polaris.core.admin.model.Principal; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisMetaStoreSession; +import org.apache.polaris.service.dropwizard.auth.TokenUtils; +import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; +import org.junit.jupiter.api.TestInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PolarisIntegrationTestFixture { + + public record SnowmanIdentifier(String principalName, String principalRoleName) {} + + public record SnowmanCredentials( + String clientId, String clientSecret, SnowmanIdentifier identifier) {} + + private static final Logger LOGGER = LoggerFactory.getLogger(PolarisIntegrationTestFixture.class); + + private final PolarisIntegrationTestHelper helper; + + public final String realm; + public final PolarisPrincipalSecrets adminSecrets; + public final SnowmanCredentials snowmanCredentials; + public final String adminToken; + public final String userToken; + public final Client client; + + private final URI baseUri; + + public PolarisIntegrationTestFixture( + PolarisIntegrationTestHelper helper, TestEnvironment testEnv, TestInfo testInfo) { + this.helper = helper; + this.client = ClientBuilder.newClient(); + this.baseUri = testEnv.baseUri(); + // Generate unique realm using test name for each test since the tests can run in parallel + realm = testInfo.getTestClass().orElseThrow().getName().replace('.', '_'); + adminSecrets = fetchAdminSecrets(); + adminToken = + TokenUtils.getTokenFromSecrets( + client, + baseUri, + adminSecrets.getPrincipalClientId(), + adminSecrets.getMainSecret(), + realm); + snowmanCredentials = createSnowmanCredentials(testEnv); + userToken = + TokenUtils.getTokenFromSecrets( + client, + baseUri, + snowmanCredentials.clientId(), + snowmanCredentials.clientSecret(), + realm); + } + + private PolarisPrincipalSecrets fetchAdminSecrets() { + if (!(helper.metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory)) { + helper.metaStoreManagerFactory.bootstrapRealms(List.of(realm)); + } + + RealmContext realmContext = + helper.realmContextResolver.resolveRealmContext( + baseUri.toString(), "GET", "/", Map.of(), Map.of(REALM_PROPERTY_KEY, realm)); + + PolarisMetaStoreSession metaStoreSession = + helper.metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); + PolarisCallContext polarisContext = + new PolarisCallContext( + metaStoreSession, helper.diagServices, helper.configurationStore, helper.clock); + try (CallContext ctx = CallContext.of(realmContext, polarisContext)) { + CallContext.setCurrentContext(ctx); + PolarisMetaStoreManager metaStoreManager = + helper.metaStoreManagerFactory.getOrCreateMetaStoreManager(ctx.getRealmContext()); + PolarisMetaStoreManager.EntityResult principal = + metaStoreManager.readEntityByName( + ctx.getPolarisCallContext(), + null, + PolarisEntityType.PRINCIPAL, + PolarisEntitySubType.NULL_SUBTYPE, + PolarisEntityConstants.getRootPrincipalName()); + + Map propertiesMap = readInternalProperties(principal); + return metaStoreManager + .loadPrincipalSecrets(ctx.getPolarisCallContext(), propertiesMap.get("client_id")) + .getPrincipalSecrets(); + } finally { + CallContext.unsetCurrentContext(); + } + } + + private SnowmanCredentials createSnowmanCredentials(TestEnvironment testEnv) { + + SnowmanIdentifier snowmanIdentifier = getSnowmanIdentifier(testEnv); + PrincipalRole principalRole = new PrincipalRole(snowmanIdentifier.principalRoleName()); + + try (Response createPrResponse = + client + .target(String.format("%s/api/management/v1/principal-roles", baseUri)) + .request("application/json") + .header("Authorization", "Bearer " + adminToken) + .header(REALM_PROPERTY_KEY, realm) + .post(Entity.json(principalRole))) { + assertThat(createPrResponse) + .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + } + + Principal principal = new Principal(snowmanIdentifier.principalName()); + SnowmanCredentials snowmanCredentials; + + try (Response createPResponse = + client + .target(String.format("%s/api/management/v1/principals", baseUri)) + .request("application/json") + .header("Authorization", "Bearer " + adminToken) // how is token getting used? + .header(REALM_PROPERTY_KEY, realm) + .post(Entity.json(principal))) { + assertThat(createPResponse) + .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + + PrincipalWithCredentials snowmanWithCredentials = + createPResponse.readEntity(PrincipalWithCredentials.class); + try (Response rotateResp = + client + .target( + String.format( + "%s/api/management/v1/principals/%s/rotate", baseUri, principal.getName())) + .request(MediaType.APPLICATION_JSON) + .header( + "Authorization", + "Bearer " + + TokenUtils.getTokenFromSecrets( + client, + baseUri, + snowmanWithCredentials.getCredentials().getClientId(), + snowmanWithCredentials.getCredentials().getClientSecret(), + realm)) + .header(REALM_PROPERTY_KEY, realm) + .post(Entity.json(snowmanWithCredentials))) { + + assertThat(rotateResp).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + + // Use the rotated credentials. + snowmanWithCredentials = rotateResp.readEntity(PrincipalWithCredentials.class); + } + snowmanCredentials = + new SnowmanCredentials( + snowmanWithCredentials.getCredentials().getClientId(), + snowmanWithCredentials.getCredentials().getClientSecret(), + snowmanIdentifier); + } + try (Response assignPrResponse = + client + .target( + String.format( + "%s/api/management/v1/principals/%s/principal-roles", + baseUri, principal.getName())) + .request("application/json") + .header("Authorization", "Bearer " + adminToken) // how is token getting used? + .header(REALM_PROPERTY_KEY, realm) + .put(Entity.json(new GrantPrincipalRoleRequest(principalRole)))) { + assertThat(assignPrResponse) + .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); + } + return snowmanCredentials; + } + + public void destroy() { + try { + if (realm != null) { + helper.metaStoreManagerFactory.purgeRealms(List.of(realm)); + } + } catch (Exception e) { + LOGGER.error("Failed to purge realm", e); + } finally { + if (client != null) { + try { + client.close(); + } catch (Exception e) { + LOGGER.error("Failed to close client", e); + } + } + } + } + + private Map readInternalProperties( + PolarisMetaStoreManager.EntityResult principal) { + try { + return helper.objectMapper.readValue( + principal.getEntity().getInternalProperties(), new TypeReference<>() {}); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private static SnowmanIdentifier getSnowmanIdentifier(TestEnvironment testEnv) { + return new SnowmanIdentifier("snowman" + testEnv.testId(), "catalog-admin" + testEnv.testId()); + } +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisIntegrationTestHelper.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisIntegrationTestHelper.java new file mode 100644 index 000000000..df3808e2a --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/PolarisIntegrationTestHelper.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.test; + +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import java.time.Clock; +import org.apache.polaris.core.PolarisConfigurationStore; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.service.context.RealmContextResolver; +import org.junit.jupiter.api.TestInfo; + +@Singleton +public class PolarisIntegrationTestHelper { + + @Inject MetaStoreManagerFactory metaStoreManagerFactory; + @Inject RealmContextResolver realmContextResolver; + @Inject ObjectMapper objectMapper; + @Inject PolarisDiagnostics diagServices; + @Inject PolarisConfigurationStore configurationStore; + @Inject Clock clock; + + public PolarisIntegrationTestFixture createFixture(TestEnvironment testEnv, TestInfo testInfo) { + return new PolarisIntegrationTestFixture(this, testEnv, testInfo); + } +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/SnowmanCredentialsExtension.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/SnowmanCredentialsExtension.java deleted file mode 100644 index cbf8b0f51..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/SnowmanCredentialsExtension.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.test; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.assertj.core.api.Assertions.assertThat; - -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; -import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; -import org.apache.polaris.core.admin.model.Principal; -import org.apache.polaris.core.admin.model.PrincipalRole; -import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.service.dropwizard.auth.TokenUtils; -import org.junit.jupiter.api.extension.AfterAllCallback; -import org.junit.jupiter.api.extension.BeforeAllCallback; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.extension.ExtensionContext.Namespace; -import org.junit.jupiter.api.extension.ParameterContext; -import org.junit.jupiter.api.extension.ParameterResolutionException; -import org.junit.jupiter.api.extension.ParameterResolver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SnowmanCredentialsExtension - implements BeforeAllCallback, AfterAllCallback, ParameterResolver { - - private static final Logger LOGGER = LoggerFactory.getLogger(SnowmanCredentialsExtension.class); - private SnowmanCredentials snowmanCredentials; - - public record SnowmanIdentifier(String principalName, String principalRoleName) {} - - public record SnowmanCredentials( - String clientId, String clientSecret, SnowmanIdentifier identifier) {} - - @Override - public void beforeAll(ExtensionContext extensionContext) throws Exception { - PolarisPrincipalSecrets adminSecrets = PolarisConnectionExtension.getAdminSecrets(); - String realm = - extensionContext - .getStore(Namespace.create(extensionContext.getRequiredTestClass())) - .get(REALM_PROPERTY_KEY, String.class); - - if (adminSecrets == null) { - LOGGER - .atError() - .log( - "No admin secrets configured - you must also configure your test with PolarisConnectionExtension"); - return; - } - - TestEnvironment testEnv = TestEnvironmentExtension.getEnv(extensionContext); - String userToken = - TokenUtils.getTokenFromSecrets( - testEnv.apiClient(), - testEnv.baseUri().toString(), - adminSecrets.getPrincipalClientId(), - adminSecrets.getMainSecret(), - realm); - - SnowmanIdentifier snowmanIdentifier = getSnowmanIdentifier(testEnv); - PrincipalRole principalRole = new PrincipalRole(snowmanIdentifier.principalRoleName()); - try (Response createPrResponse = - testEnv - .apiClient() - .target(String.format("%s/api/management/v1/principal-roles", testEnv.baseUri())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(principalRole))) { - assertThat(createPrResponse) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - - Principal principal = new Principal(snowmanIdentifier.principalName()); - - try (Response createPResponse = - testEnv - .apiClient() - .target(String.format("%s/api/management/v1/principals", testEnv.baseUri())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) // how is token getting used? - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(principal))) { - assertThat(createPResponse) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - PrincipalWithCredentials snowmanWithCredentials = - createPResponse.readEntity(PrincipalWithCredentials.class); - try (Response rotateResp = - testEnv - .apiClient() - .target( - String.format( - "%s/api/management/v1/principals/%s/rotate", - testEnv.baseUri(), principal.getName())) - .request(MediaType.APPLICATION_JSON) - .header( - "Authorization", - "Bearer " - + TokenUtils.getTokenFromSecrets( - testEnv.apiClient(), - testEnv.baseUri().toString(), - snowmanWithCredentials.getCredentials().getClientId(), - snowmanWithCredentials.getCredentials().getClientSecret(), - realm)) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(snowmanWithCredentials))) { - - assertThat(rotateResp).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - - // Use the rotated credentials. - snowmanWithCredentials = rotateResp.readEntity(PrincipalWithCredentials.class); - } - snowmanCredentials = - new SnowmanCredentials( - snowmanWithCredentials.getCredentials().getClientId(), - snowmanWithCredentials.getCredentials().getClientSecret(), - snowmanIdentifier); - } - try (Response assignPrResponse = - testEnv - .apiClient() - .target( - String.format( - "%s/api/management/v1/principals/%s/principal-roles", - testEnv.baseUri(), principal.getName())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) // how is token getting used? - .header(REALM_PROPERTY_KEY, realm) - .put(Entity.json(new GrantPrincipalRoleRequest(principalRole)))) { - assertThat(assignPrResponse) - .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - @Override - public void afterAll(ExtensionContext extensionContext) throws Exception { - PolarisPrincipalSecrets adminSecrets = PolarisConnectionExtension.getAdminSecrets(); - String realm = - extensionContext - .getStore(Namespace.create(extensionContext.getRequiredTestClass())) - .get(REALM_PROPERTY_KEY, String.class); - - if (adminSecrets == null) { - LOGGER - .atError() - .log( - "No admin secrets configured - you must also configure your test with PolarisConnectionExtension"); - return; - } - - TestEnvironment testEnv = TestEnvironmentExtension.getEnv(extensionContext); - String userToken = - TokenUtils.getTokenFromSecrets( - testEnv.apiClient(), - testEnv.baseUri().toString(), - adminSecrets.getPrincipalClientId(), - adminSecrets.getMainSecret(), - realm); - - SnowmanIdentifier snowmanIdentifier = getSnowmanIdentifier(testEnv); - testEnv - .apiClient() - .target( - String.format( - "%s/api/management/v1/principal-roles/%s", - testEnv.baseUri(), snowmanIdentifier.principalRoleName())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .delete() - .close(); - - testEnv - .apiClient() - .target( - String.format( - "%s/api/management/v1/principals/%s", - testEnv.baseUri(), snowmanIdentifier.principalName())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .delete() - .close(); - } - - // FIXME - this would be better done with a Credentials-specific annotation processor so - // tests could declare which credentials they want (e.g., @TestCredentials("root") ) - // For now, snowman comes from here and root comes from PolarisConnectionExtension - - @Override - public boolean supportsParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - - return parameterContext.getParameter().getType() == SnowmanCredentials.class; - } - - @Override - public Object resolveParameter( - ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - return snowmanCredentials; - } - - private static SnowmanIdentifier getSnowmanIdentifier(TestEnvironment testEnv) { - return new SnowmanIdentifier("snowman" + testEnv.testId(), "catalog-admin" + testEnv.testId()); - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironment.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironment.java index 232a57b7e..7ce726217 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironment.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironment.java @@ -18,20 +18,47 @@ */ package org.apache.polaris.service.dropwizard.test; -import jakarta.ws.rs.client.Client; import java.net.URI; import java.util.UUID; -/** - * Defines the test environment that a test should run in. - * - * @param apiClient The HTTP client to use when making requests - * @param baseUri The base URL that requests should target, for example http://localhost:1234 - * @param testId An ID unique to this test. This can be used to prefix resource names, such as - * catalog names, to prevent collision. - */ -public record TestEnvironment(Client apiClient, URI baseUri, String testId) { - public TestEnvironment(Client apiClient, String baseUri) { - this(apiClient, URI.create(baseUri), UUID.randomUUID().toString().replace("-", "")); +/** Defines the test environment that a test should run in. */ +public class TestEnvironment { + + private final URI baseUri; + private final URI baseManagementUri; + private final String testId; + + public TestEnvironment(String baseUri, String baseManagementUri) { + this( + URI.create(baseUri), + URI.create(baseManagementUri), + UUID.randomUUID().toString().replace("-", "")); + } + + public TestEnvironment(URI baseUri, URI baseManagementUri, String testId) { + this.baseUri = baseUri; + this.baseManagementUri = baseManagementUri; + this.testId = testId; + } + + /** The base URL that requests should target, for example {@code http://localhost:8181} */ + public URI baseUri() { + return baseUri; + } + + /** + * The base URL that requests should target for the management interface, for example {@code + * http://localhost:8182} + */ + public URI baseManagementUri() { + return baseManagementUri; + } + + /** + * An ID unique to this test. This can be used to prefix resource names, such as catalog names, to + * prevent collision. + */ + public String testId() { + return testId; } } diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironmentExtension.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironmentExtension.java index 0131fbf3a..94d16e07b 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironmentExtension.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestEnvironmentExtension.java @@ -21,6 +21,7 @@ import java.lang.reflect.InvocationTargetException; import java.util.Optional; import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ExtensionContext.Namespace; import org.junit.jupiter.api.extension.ParameterContext; import org.junit.jupiter.api.extension.ParameterResolutionException; import org.junit.jupiter.api.extension.ParameterResolver; @@ -43,7 +44,7 @@ public static TestEnvironment getEnv(ExtensionContext extensionContext) throws IllegalAccessException { // This must be cached because the TestEnvironment has a randomly generated ID return extensionContext - .getStore(ExtensionContext.Namespace.create(extensionContext.getRequiredTestClass())) + .getStore(Namespace.create(extensionContext.getRequiredTestClass())) .getOrComputeIfAbsent( ENV_PROPERTY_KEY, (k) -> getTestEnvironmentResolver().resolveTestEnvironment(extensionContext), @@ -71,7 +72,7 @@ public Object resolveParameter( private static TestEnvironmentResolver getTestEnvironmentResolver() { String impl = Optional.ofNullable(System.getenv(ENV_TEST_ENVIRONMENT_RESOLVER_IMPL)) - .orElse(DropwizardTestEnvironmentResolver.class.getName()); + .orElse(DefaultTestEnvironmentResolver.class.getName()); try { return (TestEnvironmentResolver) (Class.forName(impl).getDeclaredConstructor().newInstance()); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestMetricsUtil.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestMetricsUtil.java index 52257c4ca..6458b84b0 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestMetricsUtil.java +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/test/TestMetricsUtil.java @@ -20,51 +20,36 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.micrometer.core.instrument.Tag; +import jakarta.ws.rs.client.Client; import jakarta.ws.rs.core.Response; -import java.util.Collection; -import java.util.List; -import org.apache.commons.lang3.StringUtils; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.stream.Collectors; +import org.hawkular.agent.prometheus.text.TextPrometheusMetricsProcessor; +import org.hawkular.agent.prometheus.types.MetricFamily; +import org.hawkular.agent.prometheus.walkers.CollectorPrometheusMetricsWalker; /** Utils for working with metrics in tests */ public class TestMetricsUtil { - private static final String SUFFIX_TOTAL = ".total"; - - /** Gets a total counter by calling the Prometheus metrics endpoint */ - public static double getTotalCounter( - DropwizardAppExtension dropwizardAppExtension, - String metricName, - Collection tags) { - - metricName += SUFFIX_TOTAL; - metricName = metricName.replace('.', '_').replace('-', '_'); - - // Example of a line from the metrics endpoint: - // polaris_TimedApi_count_realm_total{API_NAME="polaris.principals.getPrincipal",REALM_ID="org_apache_polaris_service_TimedApplicationEventListenerTest"} 1.0 - // This method assumes that tag ordering isn't guaranteed - List tagFilters = - tags.stream().map(tag -> String.format("%s=\"%s\"", tag.getKey(), tag.getValue())).toList(); + public static Map fetchMetrics(Client client, URI baseManagementUri) { Response response = - dropwizardAppExtension - .client() - .target( - String.format("http://localhost:%d/metrics", dropwizardAppExtension.getAdminPort())) - .request() - .get(); + client.target(String.format("%s/q/metrics", baseManagementUri)).request().get(); assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - String[] responseLines = response.readEntity(String.class).split("\n"); - for (String line : responseLines) { - int numTags = - StringUtils.countMatches(line, '='); // Assumes the tag values don't contain an '=' - if (line.startsWith(metricName) - && tagFilters.stream().allMatch(line::contains) - && numTags == tagFilters.size()) { - return Double.parseDouble(line.split(" ")[1]); - } - } - return 0; + String body = response.readEntity(String.class); + InputStream inputStream = new ByteArrayInputStream(body.getBytes(StandardCharsets.UTF_8)); + CollectorPrometheusMetricsWalker walker = new CollectorPrometheusMetricsWalker(); + new TextPrometheusMetricsProcessor(inputStream, walker).walk(); + return walker.getAllMetricFamilies().stream() + .collect( + Collectors.toMap( + MetricFamily::getName, + metricFamily -> metricFamily, + (mf1, mf2) -> { + throw new IllegalStateException("Duplicate metric family: " + mf1.getName()); + })); } } diff --git a/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml b/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml deleted file mode 100644 index 9d8f770e5..000000000 --- a/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml +++ /dev/null @@ -1,164 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -server: - # Maximum number of threads. - maxThreads: 200 - - # Minimum number of thread to keep alive. - minThreads: 10 - applicationConnectors: - # HTTP-specific options. - - type: http - - # The port on which the HTTP server listens for service requests. - port: 8181 - - adminConnectors: - - type: http - port: 8182 - - # The hostname of the interface to which the HTTP server socket wil be found. If omitted, the - # socket will listen on all interfaces. - #bindHost: localhost - - # ssl: - # keyStore: ./example.keystore - # keyStorePassword: example - # - # keyStoreType: JKS # (optional, JKS is default) - - # HTTP request log settings - requestLog: - appenders: - # Settings for logging to stdout. - - type: console - - # Settings for logging to a file. - - type: file - - # The file to which statements will be logged. - currentLogFilename: ./logs/request.log - - # When the log file rolls over, the file will be archived to requests-2012-03-15.log.gz, - # requests.log will be truncated, and new statements written to it. - archivedLogFilenamePattern: ./logs/requests-%d.log.gz - - # The maximum number of log files to archive. - archivedFileCount: 14 - - # Enable archiving if the request log entries go to the their own file - archive: true - -featureConfiguration: - ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: true - ALLOW_WILDCARD_LOCATION: true - ALLOW_SPECIFYING_FILE_IO_IMPL: true - SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION: true - ALLOW_OVERLAPPING_CATALOG_URLS: true - ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING: false - SUPPORTED_CATALOG_STORAGE_TYPES: - - FILE - - S3 - - GCS - - AZURE - -metaStoreManager: - type: in-memory - -io: - factoryType: default - -oauth2: - type: default - -authenticator: - class: org.apache.polaris.service.auth.DefaultPolarisAuthenticator - -tokenBroker: - type: symmetric-key - secret: polaris - -callContextResolver: - type: default - -realmContextResolver: - type: default - -defaultRealm: POLARIS - -cors: - allowed-origins: - - localhost - - # Logging settings. -logging: - - # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL. - level: INFO - - # Logger-specific levels. - loggers: - org.apache.polaris: DEBUG - # only used to warn about the tokens endpoint being deprecated - org.apache.iceberg.rest.RESTSessionCatalog: ERROR - - appenders: - - - type: console - # If true, write log statements to stdout. - # enabled: true - # Do not display log statements below this threshold to stdout. - threshold: ALL - # Custom Logback PatternLayout with threadname. - logFormat: "%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}] %c{30}: %m %kvp%n%ex" - - # Settings for logging to a file. - - type: file - # If true, write log statements to a file. - # enabled: true - # Do not write log statements below this threshold to the file. - threshold: ALL - # Custom Logback PatternLayout with threadname. - logFormat: "%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}] %c: %m %kvp%n%ex" - - # when using json logging, you must use a format like this, else the - # mdc section of the json log will be incorrect - # logFormat: "%-5p [%d{ISO8601} - %-6r] [%t] [%X] %c: %m%n%ex" - - # The file to which statements will be logged. - currentLogFilename: ./logs/iceberg-rest.log - # When the log file rolls over, the file will be archived to polaris-2012-03-15.log.gz, - # polaris.log will be truncated, and new statements written to it. - archivedLogFilenamePattern: ./logs/iceberg-rest-%d.log.gz - # The maximum number of log files to archive. - archivedFileCount: 14 - -# Limits the size of request bodies sent to Polaris. -1 means no limit. -maxRequestBodyBytes: 1000000 - -# Limits the request rate per realm. -rateLimiter: - type: realm-token-bucket - -# The token bucket factory to use when using the realm-token-bucket rate limiter. -tokenBucketFactory: - type: default - requestsPerSecond: 9999 - windowSeconds: 10 diff --git a/extension/persistence/eclipselink/build.gradle.kts b/extension/persistence/eclipselink/build.gradle.kts index 0c167f7ad..5a9db06af 100644 --- a/extension/persistence/eclipselink/build.gradle.kts +++ b/extension/persistence/eclipselink/build.gradle.kts @@ -23,6 +23,7 @@ fun isValidDep(dep: String): Boolean { } plugins { + alias(libs.plugins.quarkus) id("polaris-server") `java-library` } @@ -30,11 +31,11 @@ plugins { dependencies { implementation(project(":polaris-core")) implementation(project(":polaris-jpa-model")) + implementation(libs.eclipselink) - implementation(platform(libs.dropwizard.bom)) - implementation(libs.jakarta.inject.api) - implementation(libs.smallrye.common.annotation) - implementation("io.dropwizard:dropwizard-jackson") + implementation(enforcedPlatform(libs.quarkus.bom)) + implementation(libs.slf4j.api) + val eclipseLinkDeps: String? = project.findProperty("eclipseLinkDeps") as String? eclipseLinkDeps?.let { val dependenciesList = it.split(",") @@ -48,15 +49,25 @@ dependencies { } } + // only for @VisibleForTesting + compileOnly(libs.guava) + compileOnly(libs.jakarta.annotation.api) + compileOnly(libs.jakarta.enterprise.cdi.api) + compileOnly(libs.jakarta.inject.api) + compileOnly("io.smallrye.common:smallrye-common-annotation") // @Identifier + compileOnly("io.smallrye.config:smallrye-config-core") // @ConfigMapping + + compileOnly(platform(libs.jackson.bom)) + compileOnly("com.fasterxml.jackson.core:jackson-annotations") + compileOnly("com.fasterxml.jackson.core:jackson-core") testImplementation(libs.h2) testImplementation(testFixtures(project(":polaris-core"))) testImplementation(platform(libs.junit.bom)) - testImplementation("org.junit.jupiter:junit-jupiter") - testImplementation(libs.assertj.core) - testImplementation(libs.mockito.core) + testImplementation(libs.bundles.junit.testing) + testRuntimeOnly("org.junit.jupiter:junit-jupiter") testRuntimeOnly("org.junit.platform:junit-platform-launcher") } @@ -68,3 +79,7 @@ tasks.register("archiveConf") { } tasks.named("test") { dependsOn("archiveConf") } + +tasks.named("compileJava") { dependsOn("compileQuarkusGeneratedSourcesJava") } + +tasks.named("sourcesJar") { dependsOn("compileQuarkusGeneratedSourcesJava") } diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkConfiguration.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkConfiguration.java new file mode 100644 index 000000000..6410aa184 --- /dev/null +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkConfiguration.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.extension.persistence.impl.eclipselink; + +import io.smallrye.config.ConfigMapping; + +@ConfigMapping(prefix = "polaris.eclipselink") +public interface EclipseLinkConfiguration { + + String confFile(); + + String persistenceUnit(); +} diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java index 21e466b78..0b0592ab4 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java @@ -18,9 +18,9 @@ */ package org.apache.polaris.extension.persistence.impl.eclipselink; -import com.fasterxml.jackson.annotation.JsonProperty; import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nonnull; +import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.context.RealmContext; @@ -34,16 +34,13 @@ * using an EclipseLink based meta store to store and retrieve all Polaris metadata. It can be * configured through persistence.xml to use supported RDBMS as the meta store. */ +@ApplicationScoped @Identifier("eclipse-link") public class EclipseLinkPolarisMetaStoreManagerFactory extends LocalPolarisMetaStoreManagerFactory { - @JsonProperty("conf-file") - private String confFile; - @JsonProperty("persistence-unit") - private String persistenceUnitName; - - @Inject protected PolarisStorageIntegrationProvider storageIntegration; + @Inject EclipseLinkConfiguration eclipseLinkConfiguration; + @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Override protected PolarisEclipseLinkStore createBackingStore(@Nonnull PolarisDiagnostics diagnostics) { @@ -55,10 +52,10 @@ protected PolarisMetaStoreSession createMetaStoreSession( @Nonnull PolarisEclipseLinkStore store, @Nonnull RealmContext realmContext) { return new PolarisEclipseLinkMetaStoreSessionImpl( store, - storageIntegration, + storageIntegrationProvider, realmContext, - confFile, - persistenceUnitName, + eclipseLinkConfiguration.confFile(), + eclipseLinkConfiguration.persistenceUnit(), secretsGenerator(realmContext)); } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 900cb466a..20ed95d99 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -20,11 +20,13 @@ [versions] hadoop = "3.4.1" iceberg = "1.7.1" -dropwizard = "4.0.11" +junit = "5.11.4" +quarkus = "3.17.5" slf4j = "2.0.16" swagger = "1.6.14" [bundles] +junit-testing = ["assertj-core", "mockito-core", "mockito-junit-jupiter", "junit-jupiter-api", "junit-jupiter-params", "junit-platform-reporting"] [libraries] @@ -40,7 +42,6 @@ bouncycastle-bcprov = { module = "org.bouncycastle:bcprov-jdk18on", version = "1 caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version = "3.1.8" } commons-codec1 = { module = "commons-codec:commons-codec", version = "1.17.1" } commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.17.0" } -dropwizard-bom = { module = "io.dropwizard:dropwizard-bom", version.ref = "dropwizard" } eclipselink = { module = "org.eclipse.persistence:eclipselink", version = "4.0.4" } errorprone = { module = "com.google.errorprone:error_prone_core", version = "2.29.2" } google-cloud-storage-bom = { module = "com.google.cloud:google-cloud-storage-bom", version = "2.42.0" } @@ -49,25 +50,32 @@ h2 = { module = "com.h2database:h2", version = "2.3.232" } # Strict dnsjava downgrade due to https://github.com/dnsjava/dnsjava/issues/329 dnsjava = { module = "dnsjava:dnsjava", version = "3.5.3!!" } hadoop-client-api = { module = "org.apache.hadoop:hadoop-client-api", version.ref = "hadoop" } +hadoop-client-runtime = { module = "org.apache.hadoop:hadoop-client-runtime", version.ref = "hadoop" } hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop" } hadoop-hdfs-client = { module = "org.apache.hadoop:hadoop-hdfs-client", version.ref = "hadoop" } iceberg-bom = { module = "org.apache.iceberg:iceberg-bom", version.ref = "iceberg" } jackson-bom = { module = "com.fasterxml.jackson:jackson-bom", version = "2.17.2" } jakarta-annotation-api = { module = "jakarta.annotation:jakarta.annotation-api", version = "3.0.0" } +jakarta-enterprise-cdi-api = { module = "jakarta.enterprise:jakarta.enterprise.cdi-api", version = "4.1.0" } jakarta-inject-api = { module = "jakarta.inject:jakarta.inject-api", version = "2.0.1" } jakarta-persistence-api = { module = "jakarta.persistence:jakarta.persistence-api", version = "3.1.0" } jakarta-servlet-api = { module = "jakarta.servlet:jakarta.servlet-api", version = "6.1.0" } jakarta-validation-api = { module = "jakarta.validation:jakarta.validation-api", version = "3.0.2" } jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version = "3.1.0" } -javax-annotation-api = { module = "javax.annotation:javax.annotation-api", version = "1.3.2" } +javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version = "4.0.1" } jetbrains-annotations = { module = "org.jetbrains:annotations", version = "24.1.0" } -junit-bom = { module = "org.junit:junit-bom", version = "5.10.3" } +junit-bom = { module = "org.junit:junit-bom", version.ref = "junit" } +junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "junit" } +junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit" } +junit-platform-reporting = { module = "org.junit.platform:junit-platform-reporting", version = "1.10.3" } logback-core = { module = "ch.qos.logback:logback-core", version = "1.4.14" } micrometer-bom = { module = "io.micrometer:micrometer-bom", version = "1.13.6" } mockito-core = { module = "org.mockito:mockito-core", version = "5.11.0" } +mockito-junit-jupiter = { module = "org.mockito:mockito-junit-jupiter", version = "5.11.0" } opentelemetry-bom = { module = "io.opentelemetry:opentelemetry-bom", version = "1.38.0" } opentelemetry-semconv = { module = "io.opentelemetry.semconv:opentelemetry-semconv", version = "1.25.0-alpha" } prometheus-metrics-exporter-servlet-jakarta = { module = "io.prometheus:prometheus-metrics-exporter-servlet-jakarta", version = "1.3.0" } +quarkus-bom = { module = "io.quarkus.platform:quarkus-bom", version.ref = "quarkus" } s3mock-testcontainers = { module = "com.adobe.testing:s3mock-testcontainers", version = "3.11.0" } slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } smallrye-common-annotation = { module = "io.smallrye.common:smallrye-common-annotation", version = "2.9.0" } @@ -78,6 +86,8 @@ testcontainers-bom = { module = "org.testcontainers:testcontainers-bom", version threeten-extra = { module = "org.threeten:threeten-extra", version = "1.8.0" } [plugins] +jandex = { id = "org.kordamp.gradle.jandex", version = "2.1.0" } openapi-generator = { id = "org.openapi.generator", version = "7.6.0" } +quarkus = { id = "io.quarkus", version.ref = "quarkus" } rat = { id = "org.nosphere.apache.rat", version = "0.8.1" } spotless = { id = "com.diffplug.spotless", version = "6.25.0" } diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index bf7feec16..cc8ef1f78 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -23,7 +23,7 @@ polaris-api-iceberg-service=api/iceberg-service polaris-api-management-model=api/management-model polaris-api-management-service=api/management-service polaris-service-common=service/common -polaris-dropwizard-service=dropwizard/service +polaris-quarkus-service=dropwizard/service polaris-eclipselink=extension/persistence/eclipselink polaris-jpa-model=extension/persistence/jpa-model aggregated-license-report=aggregated-license-report diff --git a/polaris-core/build.gradle.kts b/polaris-core/build.gradle.kts index c855a1ef5..89ec15cb0 100644 --- a/polaris-core/build.gradle.kts +++ b/polaris-core/build.gradle.kts @@ -21,6 +21,7 @@ plugins { id("polaris-client") id("java-library") id("java-test-fixtures") + alias(libs.plugins.jandex) } dependencies { @@ -108,3 +109,5 @@ dependencies { compileOnly(libs.jakarta.annotation.api) } + +tasks.named("javadoc") { dependsOn("jandex") } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java b/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java index 2b64fe664..abe598714 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/PolarisCallContext.java @@ -58,6 +58,14 @@ public PolarisCallContext( this.clock = Clock.system(ZoneId.systemDefault()); } + public static PolarisCallContext copyOf(PolarisCallContext original) { + return new PolarisCallContext( + original.getMetaStore(), + original.getDiagServices(), + original.getConfigurationStore(), + original.getClock()); + } + public PolarisMetaStoreSession getMetaStore() { return metaStore; } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/context/CallContext.java b/polaris-core/src/main/java/org/apache/polaris/core/context/CallContext.java index d9962f9c3..71c457720 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/context/CallContext.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/context/CallContext.java @@ -100,8 +100,9 @@ public Map contextVariables() { * CallContext}. */ static CallContext copyOf(CallContext base) { - RealmContext realmContext = base.getRealmContext(); - PolarisCallContext polarisCallContext = base.getPolarisCallContext(); + String realmId = base.getRealmContext().getRealmIdentifier(); + RealmContext realmContext = () -> realmId; + PolarisCallContext polarisCallContext = PolarisCallContext.copyOf(base.getPolarisCallContext()); Map contextVariables = base.contextVariables().entrySet().stream() .filter(e -> !e.getKey().equals(CLOSEABLES)) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java index 1e2ca3006..6c9294418 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java @@ -34,6 +34,7 @@ import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,6 +49,7 @@ public abstract class LocalPolarisMetaStoreManagerFactory final Map metaStoreManagerMap = new HashMap<>(); final Map storageCredentialCacheMap = new HashMap<>(); + final Map entityCacheMap = new HashMap<>(); final Map backingStoreMap = new HashMap<>(); final Map> sessionSupplierMap = new HashMap<>(); protected final PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); @@ -156,6 +158,16 @@ public synchronized StorageCredentialCache getOrCreateStorageCredentialCache( return storageCredentialCacheMap.get(realmContext.getRealmIdentifier()); } + @Override + public synchronized EntityCache getOrCreateEntityCache(RealmContext realmContext) { + if (!entityCacheMap.containsKey(realmContext.getRealmIdentifier())) { + PolarisMetaStoreManager metaStoreManager = getOrCreateMetaStoreManager(realmContext); + entityCacheMap.put(realmContext.getRealmIdentifier(), new EntityCache(metaStoreManager)); + } + + return entityCacheMap.get(realmContext.getRealmIdentifier()); + } + /** * This method bootstraps service for a given realm: i.e. creates all the needed entities in the * metastore and creates a root service principal. After that we rotate the root principal diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java index 7cdb36c5a..5d4691a55 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java @@ -23,12 +23,10 @@ import java.util.function.Supplier; import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.storage.cache.StorageCredentialCache; -/** - * Configuration interface for configuring the {@link PolarisMetaStoreManager} via Dropwizard - * configuration - */ +/** Configuration interface for configuring the {@link PolarisMetaStoreManager}. */ public interface MetaStoreManagerFactory { PolarisMetaStoreManager getOrCreateMetaStoreManager(RealmContext realmContext); @@ -37,6 +35,8 @@ public interface MetaStoreManagerFactory { StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext); + EntityCache getOrCreateEntityCache(RealmContext realmContext); + Map bootstrapRealms(List realms); /** Purge all metadata for the realms provided */ diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java index 8836a171c..05efba7fb 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/cache/EntityCache.java @@ -23,20 +23,17 @@ import com.github.benmanes.caffeine.cache.RemovalListener; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; -import jakarta.inject.Inject; import java.util.AbstractMap; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.context.RealmScoped; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisGrantRecord; import org.apache.polaris.core.persistence.cache.PolarisRemoteCache.CachedEntryResult; /** The entity cache, can be private or shared */ -@RealmScoped public class EntityCache { // cache mode @@ -56,7 +53,6 @@ public class EntityCache { * * @param polarisRemoteCache the meta store manager implementation */ - @Inject public EntityCache(@Nonnull PolarisRemoteCache polarisRemoteCache) { // by name cache diff --git a/server-templates/api.mustache b/server-templates/api.mustache index 5979bf8b0..b817bc495 100644 --- a/server-templates/api.mustache +++ b/server-templates/api.mustache @@ -23,6 +23,7 @@ import {{import}}; {{/imports}} import io.micrometer.core.annotation.Timed; +import io.micrometer.core.aop.MeterTag; import java.util.Map; import java.util.List; @@ -107,7 +108,7 @@ public class {{classname}} { @Produces({ {{#produces}}"{{{mediaType}}}"{{^-last}}, {{/-last}}{{/produces}} }){{/hasProduces}}{{#hasAuthMethods}} {{#authMethods}}{{#isOAuth}}@RolesAllowed({ {{#scopes}}"{{scope}}"{{^-last}}, {{/-last}}{{/scopes}} }){{/isOAuth}}{{/authMethods}}{{/hasAuthMethods}} @Timed("{{metricsPrefix}}.{{baseName}}.{{nickname}}") - public Response {{nickname}}({{#isMultipart}}MultipartFormDataInput input,{{/isMultipart}}{{#allParams}}{{>queryParams}}{{>pathParams}}{{>headerParams}}{{>bodyParams}}{{^isMultipart}}{{>formParams}},{{/isMultipart}}{{#isMultipart}}{{^isFormParam}},{{/isFormParam}}{{/isMultipart}}{{/allParams}}@Context RealmContext realmContext,@Context SecurityContext securityContext) { + public Response {{nickname}}({{#isMultipart}}MultipartFormDataInput input,{{/isMultipart}}{{#allParams}}{{>queryParams}}{{>pathParams}}{{>headerParams}}{{>bodyParams}}{{^isMultipart}}{{>formParams}},{{/isMultipart}}{{#isMultipart}}{{^isFormParam}},{{/isFormParam}}{{/isMultipart}}{{/allParams}}@Context @MeterTag(key="realm_id",expression="realmIdentifier") RealmContext realmContext,@Context SecurityContext securityContext) { {{! Don't log form or header params in case there are secrets, e.g., OAuth tokens }} LOGGER.atDebug().setMessage("Invoking {{baseName}} with params") .addKeyValue("operation", "{{nickname}}"){{#allParams}}{{^isHeaderParam}}{{^isFormParam}} diff --git a/service/common/build.gradle.kts b/service/common/build.gradle.kts index e027641a8..d9aff4b19 100644 --- a/service/common/build.gradle.kts +++ b/service/common/build.gradle.kts @@ -17,7 +17,10 @@ * under the License. */ -plugins { id("polaris-server") } +plugins { + id("polaris-server") + alias(libs.plugins.jandex) +} dependencies { implementation(project(":polaris-core")) @@ -41,19 +44,21 @@ dependencies { exclude("com.sun.jersey", "jersey-core") exclude("com.sun.jersey", "jersey-server") exclude("com.sun.jersey", "jersey-servlet") + exclude("com.sun.jersey", "jersey-servlet") + exclude("io.dropwizard.metrics", "metrics-core") } implementation(libs.hadoop.hdfs.client) - compileOnly(libs.jakarta.annotation.api) - compileOnly(libs.jakarta.inject.api) - compileOnly(libs.jakarta.servlet.api) - compileOnly(libs.jakarta.validation.api) - compileOnly(libs.jakarta.ws.rs.api) + implementation(libs.jakarta.annotation.api) + implementation(libs.jakarta.enterprise.cdi.api) + implementation(libs.jakarta.inject.api) + implementation(libs.jakarta.servlet.api) + implementation(libs.jakarta.validation.api) + implementation(libs.jakarta.ws.rs.api) - compileOnly(libs.smallrye.common.annotation) + implementation(libs.smallrye.common.annotation) implementation(platform(libs.jackson.bom)) - implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-xml") implementation("com.fasterxml.jackson.core:jackson-annotations") implementation(libs.caffeine) @@ -82,3 +87,5 @@ dependencies { implementation(platform(libs.azuresdk.bom)) implementation("com.azure:azure-core") } + +tasks.named("javadoc") { dependsOn("jandex") } diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index 49b7a3bba..d20c53cee 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.admin; +import jakarta.enterprise.context.RequestScoped; import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; @@ -75,6 +76,7 @@ import org.slf4j.LoggerFactory; /** Concrete implementation of the Polaris API services */ +@RequestScoped public class PolarisServiceImpl implements PolarisCatalogsApiService, PolarisPrincipalsApiService, @@ -83,15 +85,20 @@ public class PolarisServiceImpl private final RealmEntityManagerFactory entityManagerFactory; private final PolarisAuthorizer polarisAuthorizer; private final MetaStoreManagerFactory metaStoreManagerFactory; + private final CallContext callContext; @Inject public PolarisServiceImpl( RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, - PolarisAuthorizer polarisAuthorizer) { + PolarisAuthorizer polarisAuthorizer, + CallContext callContext) { this.entityManagerFactory = entityManagerFactory; this.metaStoreManagerFactory = metaStoreManagerFactory; this.polarisAuthorizer = polarisAuthorizer; + this.callContext = callContext; + // FIXME: This is a hack to set the current context for downstream calls. + CallContext.setCurrentContext(callContext); } private PolarisAdminService newAdminService( @@ -107,12 +114,7 @@ private PolarisAdminService newAdminService( PolarisMetaStoreManager metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); return new PolarisAdminService( - // FIXME remove call to CallContext.getCurrentContext() - CallContext.getCurrentContext(), - entityManager, - metaStoreManager, - authenticatedPrincipal, - polarisAuthorizer); + callContext, entityManager, metaStoreManager, authenticatedPrincipal, polarisAuthorizer); } /** From PolarisCatalogsApiService */ @@ -130,7 +132,7 @@ public Response createCatalog( } private void validateStorageConfig(StorageConfigInfo storageConfigInfo) { - PolarisCallContext polarisCallContext = CallContext.getCurrentContext().getPolarisCallContext(); + PolarisCallContext polarisCallContext = callContext.getPolarisCallContext(); List allowedStorageTypes = polarisCallContext .getConfigurationStore() diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/AuthenticatorConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/auth/AuthenticatorConfiguration.java new file mode 100644 index 000000000..c0eed4790 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/auth/AuthenticatorConfiguration.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.auth; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.Optional; + +public interface AuthenticatorConfiguration { + + /** The configuration for the token broker factory. */ + TokenBrokerFactoryConfiguration tokenBrokerFactory(); + + interface TokenBrokerFactoryConfiguration { + + /** The maximum token duration. */ + Duration maxTokenGeneration(); + + /** Configuration for the rsa-key-pair token broker factory. */ + Optional rsaKeyPair(); + + /** Configuration for the symmetric-key token broker factory. */ + Optional symmetricKey(); + + interface RSAKeyPairConfiguration { + + /** The path to the public key file. */ + Path publicKeyFile(); + + /** The path to the private key file. */ + Path privateKeyFile(); + } + + interface SymmetricKeyConfiguration { + + /** + * The secret to use for both signing and verifying signatures. Either this option of {@link + * #file()} must be provided. + */ + Optional secret(); + + /** + * The file to read the secret from. Either this option of {@link #secret()} must be provided. + */ + Optional file(); + } + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java b/service/common/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java index 28dd9e0ef..f7845d464 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/BasePolarisAuthenticator.java @@ -18,14 +18,12 @@ */ package org.apache.polaris.service.auth; -import jakarta.inject.Inject; import java.util.Arrays; import java.util.HashSet; import java.util.Optional; import java.util.Set; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.iceberg.exceptions.NotAuthorizedException; -import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; @@ -51,27 +49,29 @@ public abstract class BasePolarisAuthenticator public static final String PRINCIPAL_ROLE_PREFIX = "PRINCIPAL_ROLE:"; private static final Logger LOGGER = LoggerFactory.getLogger(BasePolarisAuthenticator.class); - @Inject protected MetaStoreManagerFactory metaStoreManagerFactory; + protected final MetaStoreManagerFactory metaStoreManagerFactory; + protected final CallContext callContext; - public PolarisCallContext getCurrentPolarisContext() { - return CallContext.getCurrentContext().getPolarisCallContext(); + protected BasePolarisAuthenticator( + MetaStoreManagerFactory metaStoreManagerFactory, CallContext callContext) { + this.metaStoreManagerFactory = metaStoreManagerFactory; + this.callContext = callContext; } protected Optional getPrincipal(DecodedToken tokenInfo) { LOGGER.debug("Resolving principal for tokenInfo client_id={}", tokenInfo.getClientId()); - RealmContext realmContext = CallContext.getCurrentContext().getRealmContext(); PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); + metaStoreManagerFactory.getOrCreateMetaStoreManager(callContext.getRealmContext()); PolarisEntity principal; try { principal = tokenInfo.getPrincipalId() > 0 ? PolarisEntity.of( metaStoreManager.loadEntity( - getCurrentPolarisContext(), 0L, tokenInfo.getPrincipalId())) + callContext.getPolarisCallContext(), 0L, tokenInfo.getPrincipalId())) : PolarisEntity.of( metaStoreManager.readEntityByName( - getCurrentPolarisContext(), + callContext.getPolarisCallContext(), null, PolarisEntityType.PRINCIPAL, PolarisEntitySubType.NULL_SUBTYPE, @@ -108,9 +108,7 @@ protected Optional getPrincipal(DecodedToken toke AuthenticatedPolarisPrincipal authenticatedPrincipal = new AuthenticatedPolarisPrincipal(new PrincipalEntity(principal), activatedPrincipalRoles); LOGGER.debug("Populating authenticatedPrincipal into CallContext: {}", authenticatedPrincipal); - CallContext.getCurrentContext() - .contextVariables() - .put(CallContext.AUTHENTICATED_PRINCIPAL, authenticatedPrincipal); + callContext.contextVariables().put(CallContext.AUTHENTICATED_PRINCIPAL, authenticatedPrincipal); return Optional.of(authenticatedPrincipal); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java b/service/common/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java index 3edbd9c70..b12b54941 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/DefaultOAuth2ApiService.java @@ -21,6 +21,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.RequestScoped; import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; @@ -28,6 +29,7 @@ import org.apache.hadoop.hdfs.web.oauth2.OAuth2Constants; import org.apache.iceberg.rest.auth.OAuth2Properties; import org.apache.iceberg.rest.responses.OAuthTokenResponse; +import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.service.catalog.api.IcebergRestOAuth2ApiService; import org.apache.polaris.service.types.TokenType; @@ -38,13 +40,19 @@ * Default implementation of the {@link IcebergRestOAuth2ApiService} that generates a JWT token for * the client if the client secret matches. */ +@RequestScoped @Identifier("default") public class DefaultOAuth2ApiService implements IcebergRestOAuth2ApiService { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultOAuth2ApiService.class); - @Inject private TokenBrokerFactory tokenBrokerFactory; + private final TokenBrokerFactory tokenBrokerFactory; + private final CallContext callContext; - public DefaultOAuth2ApiService() {} + @Inject + public DefaultOAuth2ApiService(TokenBrokerFactory tokenBrokerFactory, CallContext callContext) { + this.tokenBrokerFactory = tokenBrokerFactory; + this.callContext = callContext; + } @Override public Response getToken( @@ -97,13 +105,18 @@ public Response getToken( // secret and treat it as a new token request if (clientId != null && clientSecret != null) { yield tokenBroker.generateFromClientSecrets( - clientId, clientSecret, OAuth2Constants.CLIENT_CREDENTIALS, scope); + clientId, + clientSecret, + OAuth2Constants.CLIENT_CREDENTIALS, + scope, + callContext.getPolarisCallContext()); } else { yield tokenBroker.generateFromToken(subjectTokenType, subjectToken, grantType, scope); } } case null -> - tokenBroker.generateFromClientSecrets(clientId, clientSecret, grantType, scope); + tokenBroker.generateFromClientSecrets( + clientId, clientSecret, grantType, scope, callContext.getPolarisCallContext()); }; if (tokenResponse == null) { return OAuthUtils.getResponseFromError(OAuthTokenErrorResponse.Error.unsupported_grant_type); diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java b/service/common/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java index eff07c2a5..900a4ddd3 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/DefaultPolarisAuthenticator.java @@ -19,19 +19,35 @@ package org.apache.polaris.service.auth; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.RequestScoped; import jakarta.inject.Inject; import java.util.Optional; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +@RequestScoped @Identifier("default") public class DefaultPolarisAuthenticator extends BasePolarisAuthenticator { - @Inject private TokenBrokerFactory tokenBrokerFactory; + + private final TokenBrokerFactory tokenBrokerFactory; + + public DefaultPolarisAuthenticator() { + this(null, null, null); + } + + @Inject + public DefaultPolarisAuthenticator( + MetaStoreManagerFactory metaStoreManagerFactory, + TokenBrokerFactory tokenBrokerFactory, + CallContext callContext) { + super(metaStoreManagerFactory, callContext); + this.tokenBrokerFactory = tokenBrokerFactory; + } @Override public Optional authenticate(String credentials) { - TokenBroker handler = - tokenBrokerFactory.apply(CallContext.getCurrentContext().getRealmContext()); + TokenBroker handler = tokenBrokerFactory.apply(callContext.getRealmContext()); DecodedToken decodedToken = handler.verify(credentials); return getPrincipal(decodedToken); } diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/JWTBroker.java b/service/common/src/main/java/org/apache/polaris/service/auth/JWTBroker.java index 74f6491ca..f1945e3c4 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/JWTBroker.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/JWTBroker.java @@ -29,6 +29,7 @@ import java.util.UUID; import org.apache.commons.lang3.StringUtils; import org.apache.iceberg.exceptions.NotAuthorizedException; +import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; @@ -119,7 +120,11 @@ public TokenResponse generateFromToken( @Override public TokenResponse generateFromClientSecrets( - String clientId, String clientSecret, String grantType, String scope) { + String clientId, + String clientSecret, + String grantType, + String scope, + PolarisCallContext polarisCallContext) { // Initial sanity checks TokenRequestValidator validator = new TokenRequestValidator(); Optional initialValidationResponse = @@ -129,7 +134,8 @@ public TokenResponse generateFromClientSecrets( } Optional principal = - TokenBroker.findPrincipalEntity(metaStoreManager, clientId, clientSecret); + TokenBroker.findPrincipalEntity( + metaStoreManager, clientId, clientSecret, polarisCallContext); if (principal.isEmpty()) { return new TokenResponse(OAuthTokenErrorResponse.Error.unauthorized_client); } diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPair.java b/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPair.java index 497877445..18f270238 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPair.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPair.java @@ -19,6 +19,7 @@ package org.apache.polaris.service.auth; import com.auth0.jwt.algorithms.Algorithm; +import java.nio.file.Path; import java.security.interfaces.RSAPrivateKey; import java.security.interfaces.RSAPublicKey; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -26,17 +27,19 @@ /** Generates a JWT using a Public/Private RSA Key */ public class JWTRSAKeyPair extends JWTBroker { - public JWTRSAKeyPair(PolarisMetaStoreManager metaStoreManager, int maxTokenGenerationInSeconds) { - super(metaStoreManager, maxTokenGenerationInSeconds); - } + private final KeyProvider keyProvider; - KeyProvider getKeyProvider() { - return new LocalRSAKeyProvider(); + public JWTRSAKeyPair( + PolarisMetaStoreManager metaStoreManager, + int maxTokenGenerationInSeconds, + Path publicKeyFile, + Path privateKeyFile) { + super(metaStoreManager, maxTokenGenerationInSeconds); + keyProvider = new LocalRSAKeyProvider(publicKeyFile, privateKeyFile); } @Override public Algorithm getAlgorithm() { - KeyProvider keyProvider = getKeyProvider(); return Algorithm.RSA256( (RSAPublicKey) keyProvider.getPublicKey(), (RSAPrivateKey) keyProvider.getPrivateKey()); } diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java b/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java index 4b11a4666..f0917fc60 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/JWTRSAKeyPairFactory.java @@ -19,24 +19,62 @@ package org.apache.polaris.service.auth; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.NoSuchAlgorithmException; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.service.auth.AuthenticatorConfiguration.TokenBrokerFactoryConfiguration; +import org.apache.polaris.service.auth.AuthenticatorConfiguration.TokenBrokerFactoryConfiguration.RSAKeyPairConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +@ApplicationScoped @Identifier("rsa-key-pair") public class JWTRSAKeyPairFactory implements TokenBrokerFactory { - private int maxTokenGenerationInSeconds = 3600; - @Inject private MetaStoreManagerFactory metaStoreManagerFactory; + private static final Logger LOGGER = LoggerFactory.getLogger(JWTRSAKeyPairFactory.class); - public void setMaxTokenGenerationInSeconds(int maxTokenGenerationInSeconds) { - this.maxTokenGenerationInSeconds = maxTokenGenerationInSeconds; + private final MetaStoreManagerFactory metaStoreManagerFactory; + private final TokenBrokerFactoryConfiguration tokenBrokerFactoryConfiguration; + private final RSAKeyPairConfiguration keyPairConfiguration; + + @Inject + public JWTRSAKeyPairFactory( + MetaStoreManagerFactory metaStoreManagerFactory, + AuthenticatorConfiguration authenticatorConfiguration) { + this.metaStoreManagerFactory = metaStoreManagerFactory; + this.tokenBrokerFactoryConfiguration = authenticatorConfiguration.tokenBrokerFactory(); + this.keyPairConfiguration = + tokenBrokerFactoryConfiguration.rsaKeyPair().orElseGet(this::generateKeyPair); } @Override public TokenBroker apply(RealmContext realmContext) { return new JWTRSAKeyPair( metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext), - maxTokenGenerationInSeconds); + (int) tokenBrokerFactoryConfiguration.maxTokenGeneration().toSeconds(), + keyPairConfiguration.publicKeyFile(), + keyPairConfiguration.privateKeyFile()); + } + + private RSAKeyPairConfiguration generateKeyPair() { + LOGGER.warn( + "No public and private key files were provided; these will be generated. " + + "This should not be done in production!"); + try { + Path privateFileLocation = Files.createTempFile("polaris-private", ".pem"); + Path publicFileLocation = Files.createTempFile("polaris-public", ".pem"); + PemUtils.generateKeyPair(privateFileLocation, publicFileLocation); + return new GeneratedKeyPair(privateFileLocation, publicFileLocation); + } catch (IOException | NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } } + + private record GeneratedKeyPair(Path privateKeyFile, Path publicKeyFile) + implements RSAKeyPairConfiguration {} } diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java b/service/common/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java index bb04b0482..48ae9f9c8 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/JWTSymmetricKeyFactory.java @@ -19,56 +19,60 @@ package org.apache.polaris.service.auth; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Paths; +import java.nio.file.Path; +import java.time.Duration; import java.util.function.Supplier; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.service.auth.AuthenticatorConfiguration.TokenBrokerFactoryConfiguration.SymmetricKeyConfiguration; +@ApplicationScoped @Identifier("symmetric-key") public class JWTSymmetricKeyFactory implements TokenBrokerFactory { - @Inject private MetaStoreManagerFactory metaStoreManagerFactory; - private int maxTokenGenerationInSeconds = 3600; - private String file; - private String secret; - @Override - public TokenBroker apply(RealmContext realmContext) { - if (file == null && secret == null) { + private final MetaStoreManagerFactory metaStoreManagerFactory; + private final Duration maxTokenGeneration; + private final Path file; + private final String secret; + + @Inject + public JWTSymmetricKeyFactory( + MetaStoreManagerFactory metaStoreManagerFactory, + AuthenticatorConfiguration authenticatorConfiguration) { + this.metaStoreManagerFactory = metaStoreManagerFactory; + this.maxTokenGeneration = authenticatorConfiguration.tokenBrokerFactory().maxTokenGeneration(); + SymmetricKeyConfiguration symmetricKeyConfiguration = + authenticatorConfiguration + .tokenBrokerFactory() + .symmetricKey() + .orElseThrow(() -> new IllegalStateException("Symmetric key configuration is missing")); + this.secret = symmetricKeyConfiguration.secret().orElse(null); + this.file = symmetricKeyConfiguration.file().orElse(null); + if (this.file == null && this.secret == null) { throw new IllegalStateException("Either file or secret must be set"); } + } + + @Override + public TokenBroker apply(RealmContext realmContext) { Supplier secretSupplier = secret != null ? () -> secret : readSecretFromDisk(); return new JWTSymmetricKeyBroker( metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext), - maxTokenGenerationInSeconds, + (int) maxTokenGeneration.toSeconds(), secretSupplier); } private Supplier readSecretFromDisk() { return () -> { try { - return Files.readString(Paths.get(file)); + return Files.readString(file); } catch (IOException e) { throw new RuntimeException("Failed to read secret from file: " + file, e); } }; } - - public void setMaxTokenGenerationInSeconds(int maxTokenGenerationInSeconds) { - this.maxTokenGenerationInSeconds = maxTokenGenerationInSeconds; - } - - public void setFile(String file) { - this.file = file; - } - - public void setSecret(String secret) { - this.secret = secret; - } - - public void setMetaStoreManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { - this.metaStoreManagerFactory = metaStoreManagerFactory; - } } diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java b/service/common/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java index 666af1f2a..a42736844 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/LocalRSAKeyProvider.java @@ -19,10 +19,9 @@ package org.apache.polaris.service.auth; import java.io.IOException; +import java.nio.file.Path; import java.security.PrivateKey; import java.security.PublicKey; -import org.apache.polaris.core.PolarisCallContext; -import org.apache.polaris.core.context.CallContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,19 +31,14 @@ */ public class LocalRSAKeyProvider implements KeyProvider { - private static final String LOCAL_PRIVATE_KEY_LOCATION_KEY = "LOCAL_PRIVATE_KEY_LOCATION_KEY"; - private static final String LOCAL_PUBLIC_KEY_LOCATION_KEY = "LOCAL_PUBLIC_LOCATION_KEY"; - private static final Logger LOGGER = LoggerFactory.getLogger(LocalRSAKeyProvider.class); - private String getLocation(String configKey) { - CallContext callContext = CallContext.getCurrentContext(); - PolarisCallContext pCtx = callContext.getPolarisCallContext(); - String fileLocation = pCtx.getConfigurationStore().getConfiguration(pCtx, configKey); - if (fileLocation == null) { - throw new RuntimeException("Cannot find location for key " + configKey); - } - return fileLocation; + private final Path publicKeyFileLocation; + private final Path privateKeyFileLocation; + + public LocalRSAKeyProvider(Path publicKeyFileLocation, Path privateKeyFileLocation) { + this.publicKeyFileLocation = publicKeyFileLocation; + this.privateKeyFileLocation = privateKeyFileLocation; } /** @@ -54,7 +48,6 @@ private String getLocation(String configKey) { */ @Override public PublicKey getPublicKey() { - final String publicKeyFileLocation = getLocation(LOCAL_PUBLIC_KEY_LOCATION_KEY); try { return PemUtils.readPublicKeyFromFile(publicKeyFileLocation, "RSA"); } catch (IOException e) { @@ -70,7 +63,6 @@ public PublicKey getPublicKey() { */ @Override public PrivateKey getPrivateKey() { - final String privateKeyFileLocation = getLocation(LOCAL_PRIVATE_KEY_LOCATION_KEY); try { return PemUtils.readPrivateKeyFromFile(privateKeyFileLocation, "RSA"); } catch (IOException e) { diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/NoneTokenBrokerFactory.java b/service/common/src/main/java/org/apache/polaris/service/auth/NoneTokenBrokerFactory.java new file mode 100644 index 000000000..311175176 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/auth/NoneTokenBrokerFactory.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.auth; + +import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.service.types.TokenType; + +/** Default {@link TokenBrokerFactory} that produces token brokers that do not do anything. */ +@ApplicationScoped +@Identifier("none") +public class NoneTokenBrokerFactory implements TokenBrokerFactory { + + public static final TokenBroker NONE_TOKEN_BROKER = + new TokenBroker() { + @Override + public boolean supportsGrantType(String grantType) { + return false; + } + + @Override + public boolean supportsRequestedTokenType(TokenType tokenType) { + return false; + } + + @Override + public TokenResponse generateFromClientSecrets( + String clientId, + String clientSecret, + String grantType, + String scope, + PolarisCallContext polarisCallContext) { + return null; + } + + @Override + public TokenResponse generateFromToken( + TokenType tokenType, String subjectToken, String grantType, String scope) { + return null; + } + + @Override + public DecodedToken verify(String token) { + return null; + } + }; + + @Override + public TokenBroker apply(RealmContext realmContext) { + return NONE_TOKEN_BROKER; + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/PemUtils.java b/service/common/src/main/java/org/apache/polaris/service/auth/PemUtils.java index 24c43a7f0..fd4a0be62 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/PemUtils.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/PemUtils.java @@ -20,11 +20,14 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import java.io.File; +import java.io.BufferedWriter; import java.io.FileNotFoundException; -import java.io.FileReader; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.KeyFactory; +import java.security.KeyPair; +import java.security.KeyPairGenerator; import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; import java.security.PublicKey; @@ -32,64 +35,81 @@ import java.security.spec.InvalidKeySpecException; import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.X509EncodedKeySpec; +import java.util.Base64; import org.bouncycastle.util.io.pem.PemObject; import org.bouncycastle.util.io.pem.PemReader; public class PemUtils { - private static byte[] parsePEMFile(File pemFile) throws IOException { - if (!pemFile.isFile() || !pemFile.exists()) { + private static byte[] parsePEMFile(Path pemPath) throws IOException { + if (!Files.isRegularFile(pemPath) || !Files.exists(pemPath)) { throw new FileNotFoundException( - String.format("The file '%s' doesn't exist.", pemFile.getAbsolutePath())); + String.format("The file '%s' doesn't exist.", pemPath.toAbsolutePath())); + } + try (PemReader reader = new PemReader(Files.newBufferedReader(pemPath, UTF_8))) { + PemObject pemObject = reader.readPemObject(); + return pemObject.getContent(); } - PemReader reader = new PemReader(new FileReader(pemFile, UTF_8)); - PemObject pemObject = reader.readPemObject(); - byte[] content = pemObject.getContent(); - reader.close(); - return content; } private static PublicKey getPublicKey(byte[] keyBytes, String algorithm) { - PublicKey publicKey = null; try { KeyFactory kf = KeyFactory.getInstance(algorithm); EncodedKeySpec keySpec = new X509EncodedKeySpec(keyBytes); - publicKey = kf.generatePublic(keySpec); + return kf.generatePublic(keySpec); } catch (NoSuchAlgorithmException e) { - System.out.println( - "Could not reconstruct the public key, the given algorithm could not be found."); + throw new RuntimeException( + "Could not reconstruct the public key, the given algorithm could not be found", e); } catch (InvalidKeySpecException e) { - System.out.println("Could not reconstruct the public key"); + throw new RuntimeException("Could not reconstruct the public key", e); } - - return publicKey; } private static PrivateKey getPrivateKey(byte[] keyBytes, String algorithm) { - PrivateKey privateKey = null; try { KeyFactory kf = KeyFactory.getInstance(algorithm); EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(keyBytes); - privateKey = kf.generatePrivate(keySpec); + return kf.generatePrivate(keySpec); } catch (NoSuchAlgorithmException e) { - System.out.println( - "Could not reconstruct the private key, the given algorithm could not be found."); + throw new RuntimeException( + "Could not reconstruct the private key, the given algorithm could not be found", e); } catch (InvalidKeySpecException e) { - System.out.println("Could not reconstruct the private key"); + throw new RuntimeException("Could not reconstruct the private key", e); } - - return privateKey; } - public static PublicKey readPublicKeyFromFile(String filepath, String algorithm) + public static PublicKey readPublicKeyFromFile(Path filepath, String algorithm) throws IOException { - byte[] bytes = PemUtils.parsePEMFile(new File(filepath)); + byte[] bytes = PemUtils.parsePEMFile(filepath); return PemUtils.getPublicKey(bytes, algorithm); } - public static PrivateKey readPrivateKeyFromFile(String filepath, String algorithm) + public static PrivateKey readPrivateKeyFromFile(Path filepath, String algorithm) throws IOException { - byte[] bytes = PemUtils.parsePEMFile(new File(filepath)); + byte[] bytes = PemUtils.parsePEMFile(filepath); return PemUtils.getPrivateKey(bytes, algorithm); } + + public static void generateKeyPair(Path privateFileLocation, Path publicFileLocation) + throws NoSuchAlgorithmException, IOException { + KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA"); + kpg.initialize(2048); + KeyPair kp = kpg.generateKeyPair(); + try (BufferedWriter writer = Files.newBufferedWriter(privateFileLocation, UTF_8)) { + writer.write("-----BEGIN PRIVATE KEY-----"); + writer.newLine(); + writer.write(Base64.getMimeEncoder().encodeToString(kp.getPrivate().getEncoded())); + writer.newLine(); + writer.write("-----END PRIVATE KEY-----"); + writer.newLine(); + } + try (BufferedWriter writer = Files.newBufferedWriter(publicFileLocation, UTF_8)) { + writer.write("-----BEGIN PUBLIC KEY-----"); + writer.newLine(); + writer.write(Base64.getMimeEncoder().encodeToString(kp.getPublic().getEncoded())); + writer.newLine(); + writer.write("-----END PUBLIC KEY-----"); + writer.newLine(); + } + } } diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java b/service/common/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java index 26e35f344..cd4b914aa 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/TestInlineBearerTokenPolarisAuthenticator.java @@ -20,15 +20,17 @@ import com.google.common.base.Splitter; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.RequestScoped; +import jakarta.inject.Inject; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; -import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,18 +47,27 @@ * This class does not expect a client to be either present or correct. Lookup is delegated to the * {@link PolarisMetaStoreManager} for the current realm. */ +@RequestScoped @Identifier("test") public class TestInlineBearerTokenPolarisAuthenticator extends BasePolarisAuthenticator { private static final Logger LOGGER = LoggerFactory.getLogger(TestInlineBearerTokenPolarisAuthenticator.class); + public TestInlineBearerTokenPolarisAuthenticator() { + this(null, null); + } + + @Inject + public TestInlineBearerTokenPolarisAuthenticator( + MetaStoreManagerFactory metaStoreManagerFactory, CallContext callContext) { + super(metaStoreManagerFactory, callContext); + } + @Override public Optional authenticate(String credentials) { Map properties = extractPrincipal(credentials); PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager( - CallContext.getCurrentContext().getRealmContext()); - PolarisCallContext callContext = CallContext.getCurrentContext().getPolarisCallContext(); + metaStoreManagerFactory.getOrCreateMetaStoreManager(callContext.getRealmContext()); String principal = properties.get("principal"); LOGGER.info("Checking for existence of principal {} in map {}", principal, properties); @@ -71,7 +82,9 @@ public Optional authenticate(String credentials) } PolarisPrincipalSecrets secrets = - metaStoreManager.loadPrincipalSecrets(callContext, principal).getPrincipalSecrets(); + metaStoreManager + .loadPrincipalSecrets(callContext.getPolarisCallContext(), principal) + .getPrincipalSecrets(); if (secrets == null) { // For test scenarios, if we're allowing short-circuiting into the bearer flow, there may // not be a clientId/clientSecret, and instead we'll let the BasePolarisAuthenticator diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java b/service/common/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java index b23530bc2..910defbfa 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/TestOAuth2ApiService.java @@ -19,6 +19,7 @@ package org.apache.polaris.service.auth; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.RequestScoped; import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; @@ -39,11 +40,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@RequestScoped @Identifier("test") public class TestOAuth2ApiService implements IcebergRestOAuth2ApiService { private static final Logger LOGGER = LoggerFactory.getLogger(TestOAuth2ApiService.class); - @Inject private MetaStoreManagerFactory metaStoreManagerFactory; + @Inject MetaStoreManagerFactory metaStoreManagerFactory; + @Inject CallContext callContext; @Override public Response getToken( @@ -80,8 +83,7 @@ public Response getToken( private String getPrincipalName(String clientId, RealmContext realmContext) { PolarisMetaStoreManager metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext); - // FIXME remove call to CallContext.getCurrentContext() - PolarisCallContext polarisCallContext = CallContext.getCurrentContext().getPolarisCallContext(); + PolarisCallContext polarisCallContext = callContext.getPolarisCallContext(); PrincipalSecretsResult secretsResult = metaStoreManager.loadPrincipalSecrets(polarisCallContext, clientId); if (secretsResult.isSuccess()) { diff --git a/service/common/src/main/java/org/apache/polaris/service/auth/TokenBroker.java b/service/common/src/main/java/org/apache/polaris/service/auth/TokenBroker.java index 190a21e54..2ace9bb92 100644 --- a/service/common/src/main/java/org/apache/polaris/service/auth/TokenBroker.java +++ b/service/common/src/main/java/org/apache/polaris/service/auth/TokenBroker.java @@ -22,7 +22,6 @@ import java.util.Optional; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; -import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -36,7 +35,11 @@ public interface TokenBroker { boolean supportsRequestedTokenType(TokenType tokenType); TokenResponse generateFromClientSecrets( - final String clientId, final String clientSecret, final String grantType, final String scope); + final String clientId, + final String clientSecret, + final String grantType, + final String scope, + PolarisCallContext polarisCallContext); TokenResponse generateFromToken( TokenType tokenType, String subjectToken, final String grantType, final String scope); @@ -44,9 +47,11 @@ TokenResponse generateFromToken( DecodedToken verify(String token); static @Nonnull Optional findPrincipalEntity( - PolarisMetaStoreManager metaStoreManager, String clientId, String clientSecret) { + PolarisMetaStoreManager metaStoreManager, + String clientId, + String clientSecret, + PolarisCallContext polarisCallContext) { // Validate the principal is present and secrets match - PolarisCallContext polarisCallContext = CallContext.getCurrentContext().getPolarisCallContext(); PrincipalSecretsResult principalSecrets = metaStoreManager.loadPrincipalSecrets(polarisCallContext, clientId); if (!principalSecrets.isSuccess()) { diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 483778ccd..176e78e3a 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import jakarta.enterprise.context.RequestScoped; import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; @@ -74,6 +75,7 @@ * org.apache.iceberg.rest.CatalogHandlers} after finding the appropriate {@link Catalog} for the * current {@link RealmContext}. */ +@RequestScoped public class IcebergCatalogAdapter implements IcebergRestCatalogApiService, IcebergRestConfigurationApiService { @@ -109,6 +111,7 @@ public class IcebergCatalogAdapter .add(Endpoint.create("POST", ResourcePaths.V1_TRANSACTIONS_COMMIT)) .build(); + private final CallContext callContext; private final CallContextCatalogFactory catalogFactory; private final MetaStoreManagerFactory metaStoreManagerFactory; private final RealmEntityManagerFactory entityManagerFactory; @@ -116,14 +119,18 @@ public class IcebergCatalogAdapter @Inject public IcebergCatalogAdapter( + CallContext callContext, CallContextCatalogFactory catalogFactory, RealmEntityManagerFactory entityManagerFactory, MetaStoreManagerFactory metaStoreManagerFactory, PolarisAuthorizer polarisAuthorizer) { + this.callContext = callContext; this.catalogFactory = catalogFactory; this.entityManagerFactory = entityManagerFactory; this.metaStoreManagerFactory = metaStoreManagerFactory; this.polarisAuthorizer = polarisAuthorizer; + // FIXME: This is a hack to set the current context for downstream calls. + CallContext.setCurrentContext(callContext); } private PolarisCatalogHandlerWrapper newHandlerWrapper( @@ -138,8 +145,7 @@ private PolarisCatalogHandlerWrapper newHandlerWrapper( entityManagerFactory.getOrCreateEntityManager(realmContext); return new PolarisCatalogHandlerWrapper( - // FIXME remove call to CallContext.getCurrentContext() - CallContext.getCurrentContext(), + callContext, entityManager, metaStoreManagerFactory.getOrCreateMetaStoreManager(realmContext), authenticatedPrincipal, @@ -540,8 +546,6 @@ public Response getConfig( if (warehouse == null) { throw new BadRequestException("Please specify a warehouse"); } - // FIXME remove call to CallContext.getCurrentContext() - CallContext callContext = CallContext.getCurrentContext(); Resolver resolver = entityManager.prepareResolver(callContext, authenticatedPrincipal, warehouse); ResolverStatus resolverStatus = resolver.resolveAll(); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java index b930fb35a..b962ec79d 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/DefaultFileIOFactory.java @@ -19,12 +19,14 @@ package org.apache.polaris.service.catalog.io; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.io.FileIO; /** A simple FileIOFactory implementation that defers all the work to the Iceberg SDK */ +@ApplicationScoped @Identifier("default") public class DefaultFileIOFactory implements FileIOFactory { @Override diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java index 64999bbf5..ca3c08511 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOFactory.java @@ -18,15 +18,10 @@ */ package org.apache.polaris.service.catalog.io; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import java.util.Map; import org.apache.iceberg.io.FileIO; /** Interface for providing a way to construct FileIO objects, such as for reading/writing S3. */ -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "factoryType") public interface FileIOFactory { FileIO loadFileIO(String impl, Map properties); } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java index 016afa54c..469587a39 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/WasbTranslatingFileIOFactory.java @@ -19,18 +19,19 @@ package org.apache.polaris.service.catalog.io; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.io.FileIO; /** A {@link FileIOFactory} that translates WASB paths to ABFS ones */ +@ApplicationScoped @Identifier("wasb") public class WasbTranslatingFileIOFactory implements FileIOFactory { @Override public FileIO loadFileIO(String ioImpl, Map properties) { - WasbTranslatingFileIO wrapped = - new WasbTranslatingFileIO(CatalogUtil.loadFileIO(ioImpl, properties, new Configuration())); - return wrapped; + return new WasbTranslatingFileIO( + CatalogUtil.loadFileIO(ioImpl, properties, new Configuration())); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java b/service/common/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java index 35a54a5f0..3db2fd998 100644 --- a/service/common/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java +++ b/service/common/src/main/java/org/apache/polaris/service/config/DefaultConfigurationStore.java @@ -18,33 +18,51 @@ */ package org.apache.polaris.service.config; +import com.fasterxml.jackson.databind.ObjectMapper; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.util.Map; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfigurationStore; import org.apache.polaris.core.context.CallContext; +@ApplicationScoped public class DefaultConfigurationStore implements PolarisConfigurationStore { - private final Map config; - private final Map> realmConfig; - public DefaultConfigurationStore(Map config) { - this.config = config; - this.realmConfig = Map.of(); + private final Map defaults; + private final Map> realmOverrides; + + // FIXME the whole PolarisConfigurationStore + PolarisConfiguration needs to be refactored + // to become a proper Quarkus configuration object + @Inject + public DefaultConfigurationStore( + ObjectMapper objectMapper, FeaturesConfiguration configurations) { + this( + configurations.parseDefaults(objectMapper), + configurations.parseRealmOverrides(objectMapper)); + } + + public DefaultConfigurationStore(Map defaults) { + this(defaults, Map.of()); } public DefaultConfigurationStore( - Map config, Map> realmConfig) { - this.config = config; - this.realmConfig = realmConfig; + Map defaults, Map> realmOverrides) { + this.defaults = Map.copyOf(defaults); + this.realmOverrides = Map.copyOf(realmOverrides); } - @SuppressWarnings("unchecked") @Override public @Nullable T getConfiguration(@Nonnull PolarisCallContext ctx, String configName) { String realm = CallContext.getCurrentContext().getRealmContext().getRealmIdentifier(); - return (T) - realmConfig.getOrDefault(realm, Map.of()).getOrDefault(configName, config.get(configName)); + @SuppressWarnings("unchecked") + T confgValue = + (T) + realmOverrides + .getOrDefault(realm, Map.of()) + .getOrDefault(configName, defaults.get(configName)); + return confgValue; } } diff --git a/service/common/src/main/java/org/apache/polaris/service/config/FeaturesConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/config/FeaturesConfiguration.java new file mode 100644 index 000000000..389cc079c --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/config/FeaturesConfiguration.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.config; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +// FIXME replace with static properties +public interface FeaturesConfiguration { + + interface RealmOverrides { + Map overrides(); + } + + Map defaults(); + + Map realmOverrides(); + + default Map parseDefaults(ObjectMapper objectMapper) { + return convertMap(objectMapper, defaults()); + } + + default Map> parseRealmOverrides(ObjectMapper objectMapper) { + Map> m = new HashMap<>(); + for (String realm : realmOverrides().keySet()) { + m.put(realm, convertMap(objectMapper, realmOverrides().get(realm).overrides())); + } + return m; + } + + private static Map convertMap( + ObjectMapper objectMapper, Map properties) { + Map m = new HashMap<>(); + for (String configName : properties.keySet()) { + String json = properties.get(configName); + try { + JsonNode node = objectMapper.readTree(json); + m.put(configName, configValue(node)); + } catch (JsonProcessingException e) { + throw new RuntimeException( + "Invalid JSON value for feature configuration: " + configName, e); + } + } + return m; + } + + private static Object configValue(JsonNode node) { + return switch (node.getNodeType()) { + case BOOLEAN -> node.asBoolean(); + case STRING -> node.asText(); + case NUMBER -> + switch (node.numberType()) { + case INT, LONG -> node.asLong(); + case FLOAT, DOUBLE -> node.asDouble(); + default -> + throw new IllegalArgumentException("Unsupported number type: " + node.numberType()); + }; + case ARRAY -> { + List list = new ArrayList<>(); + node.elements().forEachRemaining(n -> list.add(configValue(n))); + yield List.copyOf(list); + } + default -> + throw new IllegalArgumentException( + "Unsupported feature configuration JSON type: " + node.getNodeType()); + }; + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java index 620e67900..03799bd40 100644 --- a/service/common/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/config/RealmEntityManagerFactory.java @@ -18,37 +18,30 @@ */ package org.apache.polaris.service.config; +import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; -import jakarta.inject.Provider; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; -import org.apache.polaris.core.persistence.cache.EntityCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** Gets or creates PolarisEntityManager instances based on config values and RealmContext. */ +@ApplicationScoped public class RealmEntityManagerFactory { + private static final Logger LOGGER = LoggerFactory.getLogger(RealmEntityManagerFactory.class); + private final MetaStoreManagerFactory metaStoreManagerFactory; // Key: realmIdentifier private final Map cachedEntityManagers = new ConcurrentHashMap<>(); - private final Provider entityCache; - - // Subclasses for test injection. - protected RealmEntityManagerFactory() { - this.metaStoreManagerFactory = null; - this.entityCache = null; - } @Inject - public RealmEntityManagerFactory( - MetaStoreManagerFactory metaStoreManagerFactory, Provider entityCache) { + public RealmEntityManagerFactory(MetaStoreManagerFactory metaStoreManagerFactory) { this.metaStoreManagerFactory = metaStoreManagerFactory; - this.entityCache = entityCache; } public PolarisEntityManager getOrCreateEntityManager(RealmContext context) { @@ -63,7 +56,7 @@ public PolarisEntityManager getOrCreateEntityManager(RealmContext context) { return new PolarisEntityManager( metaStoreManagerFactory.getOrCreateMetaStoreManager(context), metaStoreManagerFactory.getOrCreateStorageCredentialCache(context), - entityCache.get()); + metaStoreManagerFactory.getOrCreateEntityCache(context)); }); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/config/TaskHandlerConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/config/TaskHandlerConfiguration.java deleted file mode 100644 index fee41e71b..000000000 --- a/service/common/src/main/java/org/apache/polaris/service/config/TaskHandlerConfiguration.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.config; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; - -public class TaskHandlerConfiguration { - private int poolSize = 10; - private boolean fixedSize = true; - private String threadNamePattern = "taskHandler-%d"; - - public void setPoolSize(int poolSize) { - this.poolSize = poolSize; - } - - public void setFixedSize(boolean fixedSize) { - this.fixedSize = fixedSize; - } - - public void setThreadNamePattern(String threadNamePattern) { - this.threadNamePattern = threadNamePattern; - } - - public ExecutorService executorService() { - return fixedSize - ? Executors.newFixedThreadPool(poolSize, threadFactory()) - : Executors.newCachedThreadPool(threadFactory()); - } - - private ThreadFactory threadFactory() { - return new ThreadFactoryBuilder().setNameFormat(threadNamePattern).setDaemon(true).build(); - } -} diff --git a/service/common/src/main/java/org/apache/polaris/service/context/ContextConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/context/ContextConfiguration.java new file mode 100644 index 000000000..cd72cf2e8 --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/context/ContextConfiguration.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.context; + +public interface ContextConfiguration { + + /** The configuration for the realm context resolver. */ + RealmContextResolverConfiguration realmContextResolver(); + + interface RealmContextResolverConfiguration { + + /** The default realm to use when no realm is specified. */ + String defaultRealm(); + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java b/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java index d43966867..6196bd0a9 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/DefaultCallContextResolver.java @@ -19,13 +19,12 @@ package org.apache.polaris.service.context; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.time.Clock; -import java.time.ZoneId; import java.util.Map; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisConfigurationStore; -import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; @@ -40,12 +39,15 @@ * *

Example: principal:data-engineer;password:test;realm:acct123 */ +@ApplicationScoped @Identifier("default") public class DefaultCallContextResolver implements CallContextResolver { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultCallContextResolver.class); - @Inject private MetaStoreManagerFactory metaStoreManagerFactory; - @Inject private PolarisConfigurationStore configurationStore; + @Inject MetaStoreManagerFactory metaStoreManagerFactory; + @Inject PolarisConfigurationStore configurationStore; + @Inject PolarisDiagnostics diagnostics; + @Inject Clock clock; @Override public CallContext resolveCallContext( @@ -63,15 +65,10 @@ public CallContext resolveCallContext( .addKeyValue("headers", headers) .log("Resolving CallContext"); - PolarisDiagnostics diagServices = new PolarisDefaultDiagServiceImpl(); PolarisMetaStoreSession metaStoreSession = metaStoreManagerFactory.getOrCreateSessionSupplier(realmContext).get(); PolarisCallContext polarisContext = - new PolarisCallContext( - metaStoreSession, - diagServices, - configurationStore, - Clock.system(ZoneId.systemDefault())); + new PolarisCallContext(metaStoreSession, diagnostics, configurationStore, clock); return CallContext.of(realmContext, polarisContext); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/context/DefaultRealmContextResolver.java b/service/common/src/main/java/org/apache/polaris/service/context/DefaultRealmContextResolver.java index 6e81a1204..f158af6f8 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/DefaultRealmContextResolver.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/DefaultRealmContextResolver.java @@ -20,6 +20,8 @@ import com.google.common.base.Splitter; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.util.HashMap; import java.util.Map; import org.apache.polaris.core.context.RealmContext; @@ -32,13 +34,19 @@ * *

Example: principal:data-engineer;password:test;realm:acct123 */ +@ApplicationScoped @Identifier("default") public class DefaultRealmContextResolver implements RealmContextResolver { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultRealmContextResolver.class); public static final String REALM_PROPERTY_KEY = "realm"; - private String defaultRealm = "default-realm"; + private final ContextConfiguration configuration; + + @Inject + public DefaultRealmContextResolver(ContextConfiguration configuration) { + this.configuration = configuration; + } @Override public RealmContext resolveRealmContext( @@ -56,7 +64,7 @@ public RealmContext resolveRealmContext( path, queryParams, headers); - final Map parsedProperties = parseBearerTokenAsKvPairs(headers); + Map parsedProperties = parseBearerTokenAsKvPairs(headers); if (!parsedProperties.containsKey(REALM_PROPERTY_KEY) && headers.containsKey(REALM_PROPERTY_KEY)) { @@ -65,20 +73,13 @@ public RealmContext resolveRealmContext( if (!parsedProperties.containsKey(REALM_PROPERTY_KEY)) { LOGGER.warn( - "Failed to parse {} from headers; using {}", REALM_PROPERTY_KEY, getDefaultRealm()); - parsedProperties.put(REALM_PROPERTY_KEY, getDefaultRealm()); + "Failed to parse {} from headers; using {}", + REALM_PROPERTY_KEY, + configuration.realmContextResolver().defaultRealm()); + parsedProperties.put(REALM_PROPERTY_KEY, configuration.realmContextResolver().defaultRealm()); } - return () -> parsedProperties.get(REALM_PROPERTY_KEY); - } - - @Override - public void setDefaultRealm(String defaultRealm) { - this.defaultRealm = defaultRealm; - } - - @Override - public String getDefaultRealm() { - return this.defaultRealm; + String realmId = parsedProperties.get(REALM_PROPERTY_KEY); + return () -> realmId; } /** diff --git a/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java b/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java index a72f71431..a23802431 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.context; +import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.nio.file.Paths; import java.util.HashMap; @@ -39,6 +40,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@ApplicationScoped public class PolarisCallContextCatalogFactory implements CallContextCatalogFactory { private static final Logger LOGGER = LoggerFactory.getLogger(PolarisCallContextCatalogFactory.class); diff --git a/service/common/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java b/service/common/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java index 66e0c99da..9f38df605 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/RealmContextResolver.java @@ -29,8 +29,4 @@ RealmContext resolveRealmContext( String path, Map queryParams, Map headers); - - void setDefaultRealm(String defaultRealm); - - String getDefaultRealm(); } diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java index 7714fc3df..3fd2e9de0 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java @@ -18,9 +18,9 @@ */ package org.apache.polaris.service.persistence; -import com.google.common.annotations.VisibleForTesting; import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nonnull; +import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.util.Collections; import java.util.HashSet; @@ -37,12 +37,24 @@ import org.apache.polaris.core.persistence.PolarisTreeMapStore; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; +@ApplicationScoped @Identifier("in-memory") public class InMemoryPolarisMetaStoreManagerFactory extends LocalPolarisMetaStoreManagerFactory { - @Inject protected PolarisStorageIntegrationProvider storageIntegration; - final Set bootstrappedRealms = new HashSet<>(); + private final PolarisStorageIntegrationProvider storageIntegration; + + public InMemoryPolarisMetaStoreManagerFactory() { + this(null); + } + + @Inject + public InMemoryPolarisMetaStoreManagerFactory( + PolarisStorageIntegrationProvider storageIntegration) { + this.storageIntegration = storageIntegration; + } + + private final Set bootstrappedRealms = new HashSet<>(); @Override protected PolarisTreeMapStore createBackingStore(@Nonnull PolarisDiagnostics diagnostics) { @@ -91,10 +103,4 @@ private void bootstrapRealmAndPrintCredentials(String realmId) { principalSecrets.getPrincipalSecrets().getMainSecret()); System.out.println(msg); } - - @VisibleForTesting - public void setStorageIntegrationProvider( - PolarisStorageIntegrationProvider storageIntegrationProvider) { - this.storageIntegration = storageIntegrationProvider; - } } diff --git a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/DefaultTokenBucketFactory.java b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/DefaultTokenBucketFactory.java index 8393e8f00..a1a9683e8 100644 --- a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/DefaultTokenBucketFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/DefaultTokenBucketFactory.java @@ -18,32 +18,35 @@ */ package org.apache.polaris.service.ratelimiter; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.time.Clock; +import java.time.Duration; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.polaris.core.context.RealmContext; +@ApplicationScoped @Identifier("default") public class DefaultTokenBucketFactory implements TokenBucketFactory { private final long requestsPerSecond; - private final long windowSeconds; + private final Duration window; private final Clock clock; private final Map perRealmBuckets = new ConcurrentHashMap<>(); - @JsonCreator - public DefaultTokenBucketFactory( - @JsonProperty("requestsPerSecond") long requestsPerSecond, - @JsonProperty("windowSeconds") long windowSeconds) { - this(requestsPerSecond, windowSeconds, Clock.systemUTC()); + @Inject + public DefaultTokenBucketFactory(RateLimiterConfiguration configuration, Clock clock) { + this( + configuration.tokenBucket().requestsPerSecond(), + configuration.tokenBucket().window(), + clock); } - public DefaultTokenBucketFactory(long requestsPerSecond, long windowSeconds, Clock clock) { + public DefaultTokenBucketFactory(long requestsPerSecond, Duration window, Clock clock) { this.requestsPerSecond = requestsPerSecond; - this.windowSeconds = windowSeconds; + this.window = window; this.clock = clock; } @@ -54,6 +57,8 @@ public TokenBucket getOrCreateTokenBucket(RealmContext realmContext) { realmId, k -> new TokenBucket( - requestsPerSecond, Math.multiplyExact(requestsPerSecond, windowSeconds), clock)); + requestsPerSecond, + Math.multiplyExact(requestsPerSecond, window.toSeconds()), + clock)); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java index b20c01693..4d0ff7893 100644 --- a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java +++ b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/NoOpRateLimiter.java @@ -19,9 +19,11 @@ package org.apache.polaris.service.ratelimiter; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.ApplicationScoped; /** Rate limiter that always allows the request */ @Identifier("no-op") +@ApplicationScoped public class NoOpRateLimiter implements RateLimiter { @Override public boolean canProceed() { diff --git a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisHealthCheck.java b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterConfiguration.java similarity index 73% rename from dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisHealthCheck.java rename to service/common/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterConfiguration.java index 32174bc87..521795ef0 100644 --- a/dropwizard/service/src/main/java/org/apache/polaris/service/dropwizard/PolarisHealthCheck.java +++ b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterConfiguration.java @@ -16,14 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard; +package org.apache.polaris.service.ratelimiter; -import com.codahale.metrics.health.HealthCheck; +import java.time.Duration; -/** Default {@link HealthCheck} implementation. */ -public class PolarisHealthCheck extends HealthCheck { - @Override - protected Result check() throws Exception { - return Result.healthy(); +public interface RateLimiterConfiguration { + + TokenBucketConfiguration tokenBucket(); + + interface TokenBucketConfiguration { + + long requestsPerSecond(); + + Duration window(); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java index d72def5fa..99a7d1ade 100644 --- a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java +++ b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RateLimiterFilter.java @@ -22,8 +22,8 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.ContainerRequestFilter; import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.ext.Provider; import java.io.IOException; -import javax.ws.rs.ext.Provider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java index 0d1deb88f..ee451f8cd 100644 --- a/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java +++ b/service/common/src/main/java/org/apache/polaris/service/ratelimiter/RealmTokenBucketRateLimiter.java @@ -18,19 +18,28 @@ */ package org.apache.polaris.service.ratelimiter; -import com.google.common.annotations.VisibleForTesting; import io.smallrye.common.annotation.Identifier; +import jakarta.enterprise.context.RequestScoped; import jakarta.inject.Inject; -import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.context.RealmContext; /** * Rate limiter that maps the request's realm identifier to its own TokenBucket, with its own * capacity. */ -@Identifier("realm-token-bucket") +@Identifier("default") +@RequestScoped public class RealmTokenBucketRateLimiter implements RateLimiter { - @Inject protected TokenBucketFactory tokenBucketFactory; + private final TokenBucketFactory tokenBucketFactory; + private final RealmContext realmContext; + + @Inject + public RealmTokenBucketRateLimiter( + TokenBucketFactory tokenBucketFactory, RealmContext realmContext) { + this.tokenBucketFactory = tokenBucketFactory; + this.realmContext = realmContext; + } /** * This signifies that a request is being made. That is, the rate limiter should count the request @@ -40,13 +49,6 @@ public class RealmTokenBucketRateLimiter implements RateLimiter { */ @Override public boolean canProceed() { - return tokenBucketFactory - .getOrCreateTokenBucket(CallContext.getCurrentContext().getRealmContext()) - .tryAcquire(); - } - - @VisibleForTesting - public void setTokenBucketFactory(TokenBucketFactory tokenBucketFactory) { - this.tokenBucketFactory = tokenBucketFactory; + return tokenBucketFactory.getOrCreateTokenBucket(realmContext).tryAcquire(); } } diff --git a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java index 98195c0e5..cde57535b 100644 --- a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java @@ -22,9 +22,10 @@ import com.google.auth.http.HttpTransportFactory; import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.ServiceOptions; -import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.util.EnumMap; import java.util.Map; import java.util.Set; @@ -40,12 +41,17 @@ import org.apache.polaris.core.storage.gcp.GcpCredentialsStorageIntegration; import software.amazon.awssdk.services.sts.StsClient; -@Identifier("default") +@ApplicationScoped public class PolarisStorageIntegrationProviderImpl implements PolarisStorageIntegrationProvider { private final Supplier stsClientSupplier; private final Supplier gcpCredsProvider; + @Inject + public PolarisStorageIntegrationProviderImpl(StorageConfiguration storageConfiguration) { + this(storageConfiguration.stsClientSupplier(), storageConfiguration.gcpCredentialsSupplier()); + } + public PolarisStorageIntegrationProviderImpl( Supplier stsClientSupplier, Supplier gcpCredsProvider) { this.stsClientSupplier = stsClientSupplier; diff --git a/service/common/src/main/java/org/apache/polaris/service/storage/StorageConfiguration.java b/service/common/src/main/java/org/apache/polaris/service/storage/StorageConfiguration.java new file mode 100644 index 000000000..711cf032e --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/storage/StorageConfiguration.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.storage; + +import com.google.auth.oauth2.AccessToken; +import com.google.auth.oauth2.GoogleCredentials; +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.*; +import java.util.function.Supplier; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; + +public interface StorageConfiguration { + + Duration DEFAULT_TOKEN_LIFESPAN = Duration.ofHours(1); + + /** + * The AWS access key to use for authentication. If not present, the default credentials provider + * chain will be used. + */ + Optional awsAccessKey(); + + /** + * The AWS secret key to use for authentication. If not present, the default credentials provider + * chain will be used. + */ + Optional awsSecretKey(); + + /** + * The GCP access token to use for authentication. If not present, the default credentials + * provider chain will be used. + */ + Optional gcpAccessToken(); + + /** + * The lifespan of the GCP access token. If not present, the {@linkplain #DEFAULT_TOKEN_LIFESPAN + * default token lifespan} will be used. + */ + Optional gcpAccessTokenLifespan(); + + default Supplier stsClientSupplier() { + return () -> { + StsClientBuilder stsClientBuilder = StsClient.builder(); + if (awsAccessKey().isPresent() && awsSecretKey().isPresent()) { + LoggerFactory.getLogger(StorageConfiguration.class) + .warn("Using hard-coded AWS credentials - this is not recommended for production"); + StaticCredentialsProvider awsCredentialsProvider = + StaticCredentialsProvider.create( + AwsBasicCredentials.create(awsAccessKey().get(), awsSecretKey().get())); + stsClientBuilder.credentialsProvider(awsCredentialsProvider); + } + return stsClientBuilder.build(); + }; + } + + default Supplier gcpCredentialsSupplier() { + return () -> { + if (gcpAccessToken().isEmpty()) { + try { + return GoogleCredentials.getApplicationDefault(); + } catch (IOException e) { + throw new RuntimeException("Failed to get GCP credentials", e); + } + } else { + AccessToken accessToken = + new AccessToken( + gcpAccessToken().get(), + new Date( + Instant.now() + .plus(gcpAccessTokenLifespan().orElse(DEFAULT_TOKEN_LIFESPAN)) + .toEpochMilli())); + return GoogleCredentials.create(accessToken); + } + }; + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java b/service/common/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java index 920e5e21d..cc084a4d2 100644 --- a/service/common/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/task/TaskExecutorImpl.java @@ -19,12 +19,13 @@ package org.apache.polaris.service.task; import jakarta.annotation.Nonnull; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisBaseEntity; @@ -42,15 +43,27 @@ */ public class TaskExecutorImpl implements TaskExecutor { private static final Logger LOGGER = LoggerFactory.getLogger(TaskExecutorImpl.class); - public static final long TASK_RETRY_DELAY = 1000; - private final ExecutorService executorService; + private static final long TASK_RETRY_DELAY = 1000; + + private final Executor executor; private final MetaStoreManagerFactory metaStoreManagerFactory; - private final List taskHandlers = new ArrayList<>(); + private final TaskFileIOSupplier fileIOSupplier; + private final List taskHandlers = new CopyOnWriteArrayList<>(); public TaskExecutorImpl( - ExecutorService executorService, MetaStoreManagerFactory metaStoreManagerFactory) { - this.executorService = executorService; + Executor executor, + MetaStoreManagerFactory metaStoreManagerFactory, + TaskFileIOSupplier fileIOSupplier) { + this.executor = executor; this.metaStoreManagerFactory = metaStoreManagerFactory; + this.fileIOSupplier = fileIOSupplier; + } + + public void init() { + addTaskHandler(new TableCleanupTaskHandler(this, metaStoreManagerFactory, fileIOSupplier)); + addTaskHandler( + new ManifestFileCleanupTaskHandler( + fileIOSupplier, Executors.newVirtualThreadPerTaskExecutor())); } /** @@ -68,69 +81,70 @@ public void addTaskHandler(TaskHandler taskHandler) { */ @Override public void addTaskHandlerContext(long taskEntityId, CallContext callContext) { + // Unfortunately CallContext is a request-scoped bean and must be cloned now, + // because its usage inside the TaskExecutor thread pool will outlive its + // lifespan, so the original CallContext will eventually be closed while + // the task is still running. + // Note: PolarisCallContext has request-scoped beans as well, and must be cloned. + // FIXME replace with context propagation? CallContext clone = CallContext.copyOf(callContext); tryHandleTask(taskEntityId, clone, null, 1); } private @Nonnull CompletableFuture tryHandleTask( - long taskEntityId, CallContext clone, Throwable e, int attempt) { + long taskEntityId, CallContext callContext, Throwable e, int attempt) { if (attempt > 3) { return CompletableFuture.failedFuture(e); } return CompletableFuture.runAsync( - () -> { - // set the call context INSIDE the async task - try (CallContext ctx = CallContext.setCurrentContext(CallContext.copyOf(clone))) { - PolarisMetaStoreManager metaStoreManager = - metaStoreManagerFactory.getOrCreateMetaStoreManager(ctx.getRealmContext()); - PolarisBaseEntity taskEntity = - metaStoreManager - .loadEntity(ctx.getPolarisCallContext(), 0L, taskEntityId) - .getEntity(); - if (!PolarisEntityType.TASK.equals(taskEntity.getType())) { - throw new IllegalArgumentException("Provided taskId must be a task entity type"); - } - TaskEntity task = TaskEntity.of(taskEntity); - Optional handlerOpt = - taskHandlers.stream().filter(th -> th.canHandleTask(task)).findFirst(); - if (handlerOpt.isEmpty()) { - LOGGER - .atWarn() - .addKeyValue("taskEntityId", taskEntityId) - .addKeyValue("taskType", task.getTaskType()) - .log("Unable to find handler for task type"); - return; - } - TaskHandler handler = handlerOpt.get(); - boolean success = handler.handleTask(task); - if (success) { - LOGGER - .atInfo() - .addKeyValue("taskEntityId", taskEntityId) - .addKeyValue("handlerClass", handler.getClass()) - .log("Task successfully handled"); - metaStoreManager.dropEntityIfExists( - ctx.getPolarisCallContext(), - null, - PolarisEntity.toCore(taskEntity), - Map.of(), - false); - } else { - LOGGER - .atWarn() - .addKeyValue("taskEntityId", taskEntityId) - .addKeyValue("taskEntityName", taskEntity.getName()) - .log("Unable to execute async task"); - } - } - }, - executorService) + () -> handleTask(taskEntityId, callContext, attempt), executor) .exceptionallyComposeAsync( (t) -> { LOGGER.warn("Failed to handle task entity id {}", taskEntityId, t); - return tryHandleTask(taskEntityId, clone, t, attempt + 1); + return tryHandleTask(taskEntityId, callContext, t, attempt + 1); }, CompletableFuture.delayedExecutor( - TASK_RETRY_DELAY * (long) attempt, TimeUnit.MILLISECONDS, executorService)); + TASK_RETRY_DELAY * (long) attempt, TimeUnit.MILLISECONDS, executor)); + } + + protected void handleTask(long taskEntityId, CallContext ctx, int attempt) { + // set the call context INSIDE the async task + CallContext.setCurrentContext(ctx); + LOGGER.info("Handling task entity id {}", taskEntityId); + PolarisMetaStoreManager metaStoreManager = + metaStoreManagerFactory.getOrCreateMetaStoreManager(ctx.getRealmContext()); + PolarisBaseEntity taskEntity = + metaStoreManager.loadEntity(ctx.getPolarisCallContext(), 0L, taskEntityId).getEntity(); + if (!PolarisEntityType.TASK.equals(taskEntity.getType())) { + throw new IllegalArgumentException("Provided taskId must be a task entity type"); + } + TaskEntity task = TaskEntity.of(taskEntity); + Optional handlerOpt = + taskHandlers.stream().filter(th -> th.canHandleTask(task)).findFirst(); + if (handlerOpt.isEmpty()) { + LOGGER + .atWarn() + .addKeyValue("taskEntityId", taskEntityId) + .addKeyValue("taskType", task.getTaskType()) + .log("Unable to find handler for task type"); + return; + } + TaskHandler handler = handlerOpt.get(); + boolean success = handler.handleTask(task); + if (success) { + LOGGER + .atInfo() + .addKeyValue("taskEntityId", taskEntityId) + .addKeyValue("handlerClass", handler.getClass()) + .log("Task successfully handled"); + metaStoreManager.dropEntityIfExists( + ctx.getPolarisCallContext(), null, PolarisEntity.toCore(taskEntity), Map.of(), false); + } else { + LOGGER + .atWarn() + .addKeyValue("taskEntityId", taskEntityId) + .addKeyValue("taskEntityName", taskEntity.getName()) + .log("Unable to execute async task"); + } } } diff --git a/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java b/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java index 75ceff8ec..e926e2d60 100644 --- a/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java +++ b/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java @@ -18,6 +18,8 @@ */ package org.apache.polaris.service.task; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -33,11 +35,13 @@ import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.service.catalog.io.FileIOFactory; +@ApplicationScoped public class TaskFileIOSupplier implements Function { private final MetaStoreManagerFactory metaStoreManagerFactory; private final FileIOFactory fileIOFactory; private final PolarisConfigurationStore configurationStore; + @Inject public TaskFileIOSupplier( MetaStoreManagerFactory metaStoreManagerFactory, FileIOFactory fileIOFactory, diff --git a/extension/persistence/eclipselink/src/main/resources/META-INF/hk2-locator/default b/service/common/src/main/java/org/apache/polaris/service/task/TaskHandlerConfiguration.java similarity index 76% rename from extension/persistence/eclipselink/src/main/resources/META-INF/hk2-locator/default rename to service/common/src/main/java/org/apache/polaris/service/task/TaskHandlerConfiguration.java index ccebc9f69..7b278c1dc 100644 --- a/extension/persistence/eclipselink/src/main/resources/META-INF/hk2-locator/default +++ b/service/common/src/main/java/org/apache/polaris/service/task/TaskHandlerConfiguration.java @@ -16,8 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -[org.apache.polaris.extension.persistence.impl.eclipselink.EclipseLinkPolarisMetaStoreManagerFactory]S -contract={org.apache.polaris.core.persistence.MetaStoreManagerFactory} -name=eclipse-link -qualifier={io.smallrye.common.annotation.Identifier} +package org.apache.polaris.service.task; +public interface TaskHandlerConfiguration { + + int maxConcurrentTasks(); + + int maxQueuedTasks(); +}