diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConnectorModule.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConnectorModule.java index 7fa004432b27..ce64420d331d 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConnectorModule.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConnectorModule.java @@ -21,6 +21,7 @@ import com.google.inject.Scopes; import com.google.inject.Singleton; import com.google.inject.multibindings.Multibinder; +import com.google.inject.multibindings.OptionalBinder; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.plugin.base.logging.FormatInterpolator; import io.trino.plugin.base.logging.SessionInterpolatedValues; @@ -90,6 +91,7 @@ protected void setup(Binder binder) optionsConfigurers.addBinding().to(CredentialsOptionsConfigurer.class).in(Scopes.SINGLETON); optionsConfigurers.addBinding().to(HeaderOptionsConfigurer.class).in(Scopes.SINGLETON); optionsConfigurers.addBinding().to(RetryOptionsConfigurer.class).in(Scopes.SINGLETON); + newOptionalBinder(binder, ProxyTransportFactory.class); install(conditionalModule( BigQueryConfig.class, @@ -97,6 +99,7 @@ protected void setup(Binder binder) proxyBinder -> { configBinder(proxyBinder).bindConfig(BigQueryProxyConfig.class); newSetBinder(proxyBinder, BigQueryOptionsConfigurer.class).addBinding().to(ProxyOptionsConfigurer.class).in(Scopes.SINGLETON); + newOptionalBinder(binder, ProxyTransportFactory.class).setDefault().to(ProxyTransportFactory.DefaultProxyTransportFactory.class).in(Scopes.SINGLETON); })); } @@ -164,10 +167,19 @@ protected void setup(Binder binder) .to(IdentityCacheMapping.SingletonIdentityCacheMapping.class) .in(Scopes.SINGLETON); - newOptionalBinder(binder, BigQueryCredentialsSupplier.class) + OptionalBinder credentialsSupplierBinder = newOptionalBinder(binder, BigQueryCredentialsSupplier.class); + credentialsSupplierBinder .setDefault() - .to(StaticBigQueryCredentialsSupplier.class) + .to(DefaultBigQueryCredentialsProvider.class) .in(Scopes.SINGLETON); + + StaticCredentialsConfig staticCredentialsConfig = buildConfigObject(StaticCredentialsConfig.class); + if (staticCredentialsConfig.getCredentialsFile().isPresent() || staticCredentialsConfig.getCredentialsKey().isPresent()) { + credentialsSupplierBinder + .setBinding() + .to(StaticBigQueryCredentialsSupplier.class) + .in(Scopes.SINGLETON); + } } } diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/DefaultBigQueryCredentialsProvider.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/DefaultBigQueryCredentialsProvider.java new file mode 100644 index 000000000000..f7443ad10319 --- /dev/null +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/DefaultBigQueryCredentialsProvider.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.bigquery; + +import com.google.auth.Credentials; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.inject.Inject; +import io.trino.spi.connector.ConnectorSession; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class DefaultBigQueryCredentialsProvider + implements BigQueryCredentialsSupplier +{ + private final Optional transportFactory; + + @Inject + public DefaultBigQueryCredentialsProvider(Optional transportFactory) + { + this.transportFactory = requireNonNull(transportFactory, "transportFactory is null"); + } + + @Override + public Optional getCredentials(ConnectorSession session) + { + return transportFactory.map(factory -> { + try { + return GoogleCredentials.getApplicationDefault(factory.getTransportOptions().getHttpTransportFactory()); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } +} diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ProxyOptionsConfigurer.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ProxyOptionsConfigurer.java index f3aef16c923e..2cbf4d4f3f21 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ProxyOptionsConfigurer.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ProxyOptionsConfigurer.java @@ -13,74 +13,35 @@ */ package io.trino.plugin.bigquery; -import com.google.api.client.http.apache.v2.ApacheHttpTransport; import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; -import com.google.cloud.TransportOptions; import com.google.cloud.bigquery.BigQueryOptions; -import com.google.cloud.http.HttpTransportOptions; import com.google.inject.Inject; -import io.grpc.HttpConnectProxiedSocketAddress; import io.grpc.ManagedChannelBuilder; -import io.grpc.ProxiedSocketAddress; import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; import io.grpc.netty.shaded.io.netty.handler.ssl.ApplicationProtocolConfig; import io.grpc.netty.shaded.io.netty.handler.ssl.IdentityCipherSuiteFilter; import io.grpc.netty.shaded.io.netty.handler.ssl.JdkSslContext; -import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorSession; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.client.HttpClient; -import org.apache.http.conn.routing.HttpRoutePlanner; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.ProxyAuthenticationStrategy; -import org.apache.http.impl.conn.DefaultProxyRoutePlanner; - -import javax.net.ssl.SSLContext; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.net.URI; -import java.security.GeneralSecurityException; -import java.util.Optional; import static com.google.common.base.Preconditions.checkState; import static io.grpc.netty.shaded.io.netty.handler.ssl.ClientAuth.OPTIONAL; -import static io.trino.plugin.base.ssl.SslUtils.createSSLContext; -import static io.trino.plugin.bigquery.BigQueryErrorCode.BIGQUERY_PROXY_SSL_INITIALIZATION_FAILED; import static java.util.Objects.requireNonNull; public class ProxyOptionsConfigurer implements BigQueryGrpcOptionsConfigurer { - private final TransportOptions transportOptions; - private final Optional sslContext; - private final URI proxyUri; - private final Optional proxyUsername; - private final Optional proxyPassword; + private final ProxyTransportFactory proxyTransportFactory; @Inject - public ProxyOptionsConfigurer(BigQueryProxyConfig proxyConfig) + public ProxyOptionsConfigurer(ProxyTransportFactory proxyTransportFactory) { - requireNonNull(proxyConfig, "proxyConfig is null"); - this.proxyUri = proxyConfig.getUri(); - this.proxyUsername = proxyConfig.getUsername(); - this.proxyPassword = proxyConfig.getPassword(); - - this.sslContext = buildSslContext(proxyConfig.getKeystorePath(), proxyConfig.getKeystorePassword(), proxyConfig.getTruststorePath(), proxyConfig.getTruststorePassword()); - this.transportOptions = buildTransportOptions(sslContext, proxyUri, proxyUsername, proxyPassword); + this.proxyTransportFactory = requireNonNull(proxyTransportFactory, "proxyTransportFactory is null"); } @Override public BigQueryOptions.Builder configure(BigQueryOptions.Builder builder, ConnectorSession session) { - return builder.setTransportOptions(transportOptions); + return builder.setTransportOptions(proxyTransportFactory.getTransportOptions()); } @Override @@ -93,7 +54,7 @@ private ManagedChannelBuilder configureChannel(ManagedChannelBuilder managedChan { checkState(managedChannelBuilder instanceof NettyChannelBuilder, "Expected ManagedChannelBuilder to be provider by Netty"); NettyChannelBuilder nettyChannelBuilder = (NettyChannelBuilder) managedChannelBuilder; - sslContext.ifPresent(context -> { + proxyTransportFactory.getSslContext().ifPresent(context -> { JdkSslContext jdkSslContext = new JdkSslContext(context, true, null, IdentityCipherSuiteFilter.INSTANCE, new ApplicationProtocolConfig( ApplicationProtocolConfig.Protocol.ALPN, ApplicationProtocolConfig.SelectorFailureBehavior.CHOOSE_MY_LAST_PROTOCOL, @@ -104,66 +65,6 @@ private ManagedChannelBuilder configureChannel(ManagedChannelBuilder managedChan .useTransportSecurity(); }); - return managedChannelBuilder.proxyDetector(this::createProxyDetector); - } - - private ProxiedSocketAddress createProxyDetector(SocketAddress socketAddress) - { - HttpConnectProxiedSocketAddress.Builder builder = HttpConnectProxiedSocketAddress.newBuilder() - .setProxyAddress(new InetSocketAddress(proxyUri.getHost(), proxyUri.getPort())) - .setTargetAddress((InetSocketAddress) socketAddress); - - proxyUsername.ifPresent(builder::setUsername); - proxyPassword.ifPresent(builder::setPassword); - - return builder.build(); - } - - private static TransportOptions buildTransportOptions(Optional sslContext, URI proxyUri, Optional proxyUser, Optional proxyPassword) - { - HttpHost proxyHost = new HttpHost(proxyUri.getHost(), proxyUri.getPort()); - HttpRoutePlanner httpRoutePlanner = new DefaultProxyRoutePlanner(proxyHost); - - HttpClientBuilder httpClientBuilder = ApacheHttpTransport.newDefaultHttpClientBuilder() - .setRoutePlanner(httpRoutePlanner); - - if (sslContext.isPresent()) { - SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext.get()); - httpClientBuilder.setSSLSocketFactory(sslSocketFactory); - } - - if (proxyUser.isPresent()) { - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials( - new AuthScope(proxyHost.getHostName(), proxyHost.getPort()), - new UsernamePasswordCredentials(proxyUser.get(), proxyPassword.orElse(""))); - - httpClientBuilder - .setProxyAuthenticationStrategy(ProxyAuthenticationStrategy.INSTANCE) - .setDefaultCredentialsProvider(credentialsProvider); - } - - HttpClient client = httpClientBuilder.build(); // TODO: close http client on catalog deregistration - return HttpTransportOptions.newBuilder() - .setHttpTransportFactory(() -> new ApacheHttpTransport(client)) - .build(); - } - - private static Optional buildSslContext( - Optional keyStorePath, - Optional keyStorePassword, - Optional trustStorePath, - Optional trustStorePassword) - { - if (keyStorePath.isEmpty() && trustStorePath.isEmpty()) { - return Optional.empty(); - } - - try { - return Optional.of(createSSLContext(keyStorePath, keyStorePassword, trustStorePath, trustStorePassword)); - } - catch (GeneralSecurityException | IOException e) { - throw new TrinoException(BIGQUERY_PROXY_SSL_INITIALIZATION_FAILED, e); - } + return managedChannelBuilder.proxyDetector(proxyTransportFactory::createProxyDetector); } } diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ProxyTransportFactory.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ProxyTransportFactory.java new file mode 100644 index 000000000000..2127d4beedab --- /dev/null +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ProxyTransportFactory.java @@ -0,0 +1,150 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.bigquery; + +import com.google.api.client.http.apache.v2.ApacheHttpTransport; +import com.google.cloud.http.HttpTransportOptions; +import com.google.inject.Inject; +import io.grpc.HttpConnectProxiedSocketAddress; +import io.grpc.ProxiedSocketAddress; +import io.trino.spi.TrinoException; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.HttpClient; +import org.apache.http.conn.routing.HttpRoutePlanner; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.ProxyAuthenticationStrategy; +import org.apache.http.impl.conn.DefaultProxyRoutePlanner; + +import javax.net.ssl.SSLContext; + +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.net.URI; +import java.security.GeneralSecurityException; +import java.util.Optional; + +import static io.trino.plugin.base.ssl.SslUtils.createSSLContext; +import static io.trino.plugin.bigquery.BigQueryErrorCode.BIGQUERY_PROXY_SSL_INITIALIZATION_FAILED; +import static java.util.Objects.requireNonNull; + +public interface ProxyTransportFactory +{ + HttpTransportOptions getTransportOptions(); + + Optional getSslContext(); + + ProxiedSocketAddress createProxyDetector(SocketAddress socketAddress); + + class DefaultProxyTransportFactory + implements ProxyTransportFactory + { + private final HttpTransportOptions transportOptions; + private final Optional sslContext; + private final URI proxyUri; + private final Optional proxyUsername; + private final Optional proxyPassword; + + @Inject + public DefaultProxyTransportFactory(BigQueryProxyConfig proxyConfig) + { + requireNonNull(proxyConfig, "proxyConfig is null"); + this.proxyUri = proxyConfig.getUri(); + this.proxyUsername = proxyConfig.getUsername(); + this.proxyPassword = proxyConfig.getPassword(); + + this.sslContext = buildSslContext(proxyConfig.getKeystorePath(), proxyConfig.getKeystorePassword(), proxyConfig.getTruststorePath(), proxyConfig.getTruststorePassword()); + this.transportOptions = buildTransportOptions(sslContext, proxyUri, proxyUsername, proxyPassword); + } + + @Override + public HttpTransportOptions getTransportOptions() + { + return transportOptions; + } + + @Override + public Optional getSslContext() + { + return sslContext; + } + + @Override + public ProxiedSocketAddress createProxyDetector(SocketAddress socketAddress) + { + HttpConnectProxiedSocketAddress.Builder builder = HttpConnectProxiedSocketAddress.newBuilder() + .setProxyAddress(new InetSocketAddress(proxyUri.getHost(), proxyUri.getPort())) + .setTargetAddress((InetSocketAddress) socketAddress); + + proxyUsername.ifPresent(builder::setUsername); + proxyPassword.ifPresent(builder::setPassword); + + return builder.build(); + } + + private static HttpTransportOptions buildTransportOptions(Optional sslContext, URI proxyUri, Optional proxyUser, Optional proxyPassword) + { + HttpHost proxyHost = new HttpHost(proxyUri.getHost(), proxyUri.getPort()); + HttpRoutePlanner httpRoutePlanner = new DefaultProxyRoutePlanner(proxyHost); + + HttpClientBuilder httpClientBuilder = ApacheHttpTransport.newDefaultHttpClientBuilder() + .setRoutePlanner(httpRoutePlanner); + + if (sslContext.isPresent()) { + SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext.get()); + httpClientBuilder.setSSLSocketFactory(sslSocketFactory); + } + + if (proxyUser.isPresent()) { + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials( + new AuthScope(proxyHost.getHostName(), proxyHost.getPort()), + new UsernamePasswordCredentials(proxyUser.get(), proxyPassword.orElse(""))); + + httpClientBuilder + .setProxyAuthenticationStrategy(ProxyAuthenticationStrategy.INSTANCE) + .setDefaultCredentialsProvider(credentialsProvider); + } + + HttpClient client = httpClientBuilder.build(); // TODO: close http client on catalog deregistration + return HttpTransportOptions.newBuilder() + .setHttpTransportFactory(() -> new ApacheHttpTransport(client)) + .build(); + } + + private static Optional buildSslContext( + Optional keyStorePath, + Optional keyStorePassword, + Optional trustStorePath, + Optional trustStorePassword) + { + if (keyStorePath.isEmpty() && trustStorePath.isEmpty()) { + return Optional.empty(); + } + + try { + return Optional.of(createSSLContext(keyStorePath, keyStorePassword, trustStorePath, trustStorePassword)); + } + catch (GeneralSecurityException | IOException e) { + throw new TrinoException(BIGQUERY_PROXY_SSL_INITIALIZATION_FAILED, e); + } + } + } +} diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/StaticBigQueryCredentialsSupplier.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/StaticBigQueryCredentialsSupplier.java index b0156eef7f41..c59fdf9395fc 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/StaticBigQueryCredentialsSupplier.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/StaticBigQueryCredentialsSupplier.java @@ -14,7 +14,9 @@ package io.trino.plugin.bigquery; import com.google.auth.Credentials; +import com.google.auth.http.HttpTransportFactory; import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.http.HttpTransportOptions; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.inject.Inject; @@ -22,7 +24,9 @@ import java.io.ByteArrayInputStream; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.io.UncheckedIOException; import java.util.Base64; import java.util.Optional; @@ -33,14 +37,17 @@ public class StaticBigQueryCredentialsSupplier private final Supplier> credentialsCreator; @Inject - public StaticBigQueryCredentialsSupplier(StaticCredentialsConfig config) + public StaticBigQueryCredentialsSupplier(StaticCredentialsConfig config, Optional proxyTransportFactory) { + Optional httpTransportFactory = proxyTransportFactory + .map(ProxyTransportFactory::getTransportOptions) + .map(HttpTransportOptions::getHttpTransportFactory); // lazy creation, cache once it's created Optional credentialsKey = config.getCredentialsKey() - .map(StaticBigQueryCredentialsSupplier::createCredentialsFromKey); + .map(key -> createCredentialsFromKey(httpTransportFactory, key)); Optional credentialsFile = config.getCredentialsFile() - .map(StaticBigQueryCredentialsSupplier::createCredentialsFromFile); + .map(keyFile -> createCredentialsFromFile(httpTransportFactory, keyFile)); this.credentialsCreator = Suppliers.memoize(() -> credentialsKey.or(() -> credentialsFile)); } @@ -51,23 +58,31 @@ public Optional getCredentials(ConnectorSession session) return credentialsCreator.get(); } - private static Credentials createCredentialsFromKey(String key) + private static Credentials createCredentialsFromKey(Optional httpTransportFactory, String key) + { + return createCredentialsFromStream(httpTransportFactory, new ByteArrayInputStream(Base64.getDecoder().decode(key))); + } + + private static Credentials createCredentialsFromFile(Optional httpTransportFactory, String file) { try { - return GoogleCredentials.fromStream(new ByteArrayInputStream(Base64.getDecoder().decode(key))); + return createCredentialsFromStream(httpTransportFactory, new FileInputStream(file)); } - catch (IOException e) { - throw new UncheckedIOException("Failed to create Credentials from key", e); + catch (FileNotFoundException e) { + throw new UncheckedIOException("Failed to create Credentials from file", e); } } - private static Credentials createCredentialsFromFile(String file) + private static Credentials createCredentialsFromStream(Optional httpTransportFactory, InputStream inputStream) { try { - return GoogleCredentials.fromStream(new FileInputStream(file)); + if (httpTransportFactory.isPresent()) { + return GoogleCredentials.fromStream(inputStream, httpTransportFactory.get()); + } + return GoogleCredentials.fromStream(inputStream); } catch (IOException e) { - throw new UncheckedIOException("Failed to create Credentials from file", e); + throw new UncheckedIOException("Failed to create Credentials from stream", e); } } } diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/StaticCredentialsConfig.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/StaticCredentialsConfig.java index 1d3925cd168c..e36243435009 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/StaticCredentialsConfig.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/StaticCredentialsConfig.java @@ -13,7 +13,6 @@ */ package io.trino.plugin.bigquery; -import com.google.auth.oauth2.GoogleCredentials; import io.airlift.configuration.Config; import io.airlift.configuration.ConfigDescription; import io.airlift.configuration.ConfigSecuritySensitive; @@ -21,7 +20,6 @@ import javax.validation.constraints.AssertTrue; -import java.io.IOException; import java.util.Optional; public class StaticCredentialsConfig @@ -29,23 +27,11 @@ public class StaticCredentialsConfig private Optional credentialsKey = Optional.empty(); private Optional credentialsFile = Optional.empty(); - @AssertTrue(message = "Exactly one of 'bigquery.credentials-key' or 'bigquery.credentials-file' must be specified, or the default GoogleCredentials could be created") + @AssertTrue(message = "Exactly one of 'bigquery.credentials-key' or 'bigquery.credentials-file' must be specified") public boolean isCredentialsConfigurationValid() { // only one of them (at most) should be present - if (credentialsKey.isPresent() && credentialsFile.isPresent()) { - return false; - } - // if no credentials were supplied, let's check if we can create the default ones - if (credentialsKey.isEmpty() && credentialsFile.isEmpty()) { - try { - GoogleCredentials.getApplicationDefault(); - } - catch (IOException e) { - return false; - } - } - return true; + return credentialsKey.isEmpty() || credentialsFile.isEmpty(); } public Optional getCredentialsKey()