Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Configure HttpTransport for proxied GoogleCredentials #17783

Merged
merged 3 commits into from
Jun 9, 2023
Merged

Configure HttpTransport for proxied GoogleCredentials #17783

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
74c0ba3
Extract ProxyTransportFactory for BigQuery
wendigo Jun 7, 2023
76c3587
Configure HttpTransport for proxied GoogleCredentials
wendigo Jun 7, 2023
2d9a83b
Drop configuration validation for default credentials
wendigo Jun 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConnectorModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.google.inject.Scopes;
import com.google.inject.Singleton;
import com.google.inject.multibindings.Multibinder;
import com.google.inject.multibindings.OptionalBinder;
import io.airlift.configuration.AbstractConfigurationAwareModule;
import io.trino.plugin.base.logging.FormatInterpolator;
import io.trino.plugin.base.logging.SessionInterpolatedValues;
Expand Down Expand Up @@ -90,13 +91,15 @@ protected void setup(Binder binder)
optionsConfigurers.addBinding().to(CredentialsOptionsConfigurer.class).in(Scopes.SINGLETON);
optionsConfigurers.addBinding().to(HeaderOptionsConfigurer.class).in(Scopes.SINGLETON);
optionsConfigurers.addBinding().to(RetryOptionsConfigurer.class).in(Scopes.SINGLETON);
newOptionalBinder(binder, ProxyTransportFactory.class);

install(conditionalModule(
BigQueryConfig.class,
BigQueryConfig::isProxyEnabled,
proxyBinder -> {
configBinder(proxyBinder).bindConfig(BigQueryProxyConfig.class);
newSetBinder(proxyBinder, BigQueryOptionsConfigurer.class).addBinding().to(ProxyOptionsConfigurer.class).in(Scopes.SINGLETON);
newOptionalBinder(binder, ProxyTransportFactory.class).setDefault().to(ProxyTransportFactory.DefaultProxyTransportFactory.class).in(Scopes.SINGLETON);
}));
}

Expand Down Expand Up @@ -164,10 +167,19 @@ protected void setup(Binder binder)
.to(IdentityCacheMapping.SingletonIdentityCacheMapping.class)
.in(Scopes.SINGLETON);

newOptionalBinder(binder, BigQueryCredentialsSupplier.class)
OptionalBinder<BigQueryCredentialsSupplier> credentialsSupplierBinder = newOptionalBinder(binder, BigQueryCredentialsSupplier.class);
credentialsSupplierBinder
.setDefault()
.to(StaticBigQueryCredentialsSupplier.class)
.to(DefaultBigQueryCredentialsProvider.class)
.in(Scopes.SINGLETON);

StaticCredentialsConfig staticCredentialsConfig = buildConfigObject(StaticCredentialsConfig.class);
if (staticCredentialsConfig.getCredentialsFile().isPresent() || staticCredentialsConfig.getCredentialsKey().isPresent()) {
credentialsSupplierBinder
.setBinding()
.to(StaticBigQueryCredentialsSupplier.class)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

static one also needs the proxied transport. The service account key still requires to go talk to oauth2.googleapis.com to get an access token, the service account key itself is not an access token.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Makes sense. Thanks for investigating that

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done. I can indeed see extra proxied calls

.in(Scopes.SINGLETON);
}
}
}

Expand Down
50 changes: 50 additions & 0 deletions ...o-bigquery/src/main/java/io/trino/plugin/bigquery/DefaultBigQueryCredentialsProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.plugin.bigquery;

import com.google.auth.Credentials;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.inject.Inject;
import io.trino.spi.connector.ConnectorSession;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Optional;

import static java.util.Objects.requireNonNull;

public class DefaultBigQueryCredentialsProvider
implements BigQueryCredentialsSupplier
{
private final Optional<ProxyTransportFactory> transportFactory;

@Inject
public DefaultBigQueryCredentialsProvider(Optional<ProxyTransportFactory> transportFactory)
{
this.transportFactory = requireNonNull(transportFactory, "transportFactory is null");
}

@Override
public Optional<Credentials> getCredentials(ConnectorSession session)
{
return transportFactory.map(factory -> {
try {
return GoogleCredentials.getApplicationDefault(factory.getTransportOptions().getHttpTransportFactory());
}
catch (IOException e) {
throw new UncheckedIOException(e);
}
});
}
}
111 changes: 6 additions & 105 deletions plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ProxyOptionsConfigurer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,74 +13,35 @@
*/
package io.trino.plugin.bigquery;

import com.google.api.client.http.apache.v2.ApacheHttpTransport;
import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
import com.google.cloud.TransportOptions;
import com.google.cloud.bigquery.BigQueryOptions;
import com.google.cloud.http.HttpTransportOptions;
import com.google.inject.Inject;
import io.grpc.HttpConnectProxiedSocketAddress;
import io.grpc.ManagedChannelBuilder;
import io.grpc.ProxiedSocketAddress;
import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
import io.grpc.netty.shaded.io.netty.handler.ssl.ApplicationProtocolConfig;
import io.grpc.netty.shaded.io.netty.handler.ssl.IdentityCipherSuiteFilter;
import io.grpc.netty.shaded.io.netty.handler.ssl.JdkSslContext;
import io.trino.spi.TrinoException;
import io.trino.spi.connector.ConnectorSession;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.conn.routing.HttpRoutePlanner;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.ProxyAuthenticationStrategy;
import org.apache.http.impl.conn.DefaultProxyRoutePlanner;

import javax.net.ssl.SSLContext;

import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.net.URI;
import java.security.GeneralSecurityException;
import java.util.Optional;

import static com.google.common.base.Preconditions.checkState;
import static io.grpc.netty.shaded.io.netty.handler.ssl.ClientAuth.OPTIONAL;
import static io.trino.plugin.base.ssl.SslUtils.createSSLContext;
import static io.trino.plugin.bigquery.BigQueryErrorCode.BIGQUERY_PROXY_SSL_INITIALIZATION_FAILED;
import static java.util.Objects.requireNonNull;

public class ProxyOptionsConfigurer
implements BigQueryGrpcOptionsConfigurer
{
private final TransportOptions transportOptions;
private final Optional<SSLContext> sslContext;
private final URI proxyUri;
private final Optional<String> proxyUsername;
private final Optional<String> proxyPassword;
private final ProxyTransportFactory proxyTransportFactory;

@Inject
public ProxyOptionsConfigurer(BigQueryProxyConfig proxyConfig)
public ProxyOptionsConfigurer(ProxyTransportFactory proxyTransportFactory)
{
requireNonNull(proxyConfig, "proxyConfig is null");
this.proxyUri = proxyConfig.getUri();
this.proxyUsername = proxyConfig.getUsername();
this.proxyPassword = proxyConfig.getPassword();

this.sslContext = buildSslContext(proxyConfig.getKeystorePath(), proxyConfig.getKeystorePassword(), proxyConfig.getTruststorePath(), proxyConfig.getTruststorePassword());
this.transportOptions = buildTransportOptions(sslContext, proxyUri, proxyUsername, proxyPassword);
this.proxyTransportFactory = requireNonNull(proxyTransportFactory, "proxyTransportFactory is null");
}

@Override
public BigQueryOptions.Builder configure(BigQueryOptions.Builder builder, ConnectorSession session)
{
return builder.setTransportOptions(transportOptions);
return builder.setTransportOptions(proxyTransportFactory.getTransportOptions());
}

@Override
Expand All @@ -93,7 +54,7 @@ private ManagedChannelBuilder configureChannel(ManagedChannelBuilder managedChan
{
checkState(managedChannelBuilder instanceof NettyChannelBuilder, "Expected ManagedChannelBuilder to be provider by Netty");
NettyChannelBuilder nettyChannelBuilder = (NettyChannelBuilder) managedChannelBuilder;
sslContext.ifPresent(context -> {
proxyTransportFactory.getSslContext().ifPresent(context -> {
JdkSslContext jdkSslContext = new JdkSslContext(context, true, null, IdentityCipherSuiteFilter.INSTANCE, new ApplicationProtocolConfig(
ApplicationProtocolConfig.Protocol.ALPN,
ApplicationProtocolConfig.SelectorFailureBehavior.CHOOSE_MY_LAST_PROTOCOL,
Expand All @@ -104,66 +65,6 @@ private ManagedChannelBuilder configureChannel(ManagedChannelBuilder managedChan
.useTransportSecurity();
});

return managedChannelBuilder.proxyDetector(this::createProxyDetector);
}

private ProxiedSocketAddress createProxyDetector(SocketAddress socketAddress)
{
HttpConnectProxiedSocketAddress.Builder builder = HttpConnectProxiedSocketAddress.newBuilder()
.setProxyAddress(new InetSocketAddress(proxyUri.getHost(), proxyUri.getPort()))
.setTargetAddress((InetSocketAddress) socketAddress);

proxyUsername.ifPresent(builder::setUsername);
proxyPassword.ifPresent(builder::setPassword);

return builder.build();
}

private static TransportOptions buildTransportOptions(Optional<SSLContext> sslContext, URI proxyUri, Optional<String> proxyUser, Optional<String> proxyPassword)
{
HttpHost proxyHost = new HttpHost(proxyUri.getHost(), proxyUri.getPort());
HttpRoutePlanner httpRoutePlanner = new DefaultProxyRoutePlanner(proxyHost);

HttpClientBuilder httpClientBuilder = ApacheHttpTransport.newDefaultHttpClientBuilder()
.setRoutePlanner(httpRoutePlanner);

if (sslContext.isPresent()) {
SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext.get());
httpClientBuilder.setSSLSocketFactory(sslSocketFactory);
}

if (proxyUser.isPresent()) {
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(
new AuthScope(proxyHost.getHostName(), proxyHost.getPort()),
new UsernamePasswordCredentials(proxyUser.get(), proxyPassword.orElse("")));

httpClientBuilder
.setProxyAuthenticationStrategy(ProxyAuthenticationStrategy.INSTANCE)
.setDefaultCredentialsProvider(credentialsProvider);
}

HttpClient client = httpClientBuilder.build(); // TODO: close http client on catalog deregistration
return HttpTransportOptions.newBuilder()
.setHttpTransportFactory(() -> new ApacheHttpTransport(client))
.build();
}

private static Optional<SSLContext> buildSslContext(
Optional<File> keyStorePath,
Optional<String> keyStorePassword,
Optional<File> trustStorePath,
Optional<String> trustStorePassword)
{
if (keyStorePath.isEmpty() && trustStorePath.isEmpty()) {
return Optional.empty();
}

try {
return Optional.of(createSSLContext(keyStorePath, keyStorePassword, trustStorePath, trustStorePassword));
}
catch (GeneralSecurityException | IOException e) {
throw new TrinoException(BIGQUERY_PROXY_SSL_INITIALIZATION_FAILED, e);
}
return managedChannelBuilder.proxyDetector(proxyTransportFactory::createProxyDetector);
}
}
Loading