Skip to content

Commit

Permalink
Now with separately initialized endpoint provider
Browse files Browse the repository at this point in the history
  • Loading branch information
pitrou committed Mar 14, 2024
1 parent 59fe9b7 commit 8eeef9d
Showing 1 changed file with 45 additions and 12 deletions.
57 changes: 45 additions & 12 deletions cpp/src/arrow/filesystem/s3fs.cc
Original file line number Diff line number Diff line change
Expand Up @@ -130,19 +130,17 @@
#include "arrow/util/task_group.h"
#include "arrow/util/thread_pool.h"

namespace arrow {

using internal::TaskGroup;
using internal::ToChars;
using internal::Uri;
using io::internal::SubmitIO;

namespace fs {
namespace arrow::fs {

using ::Aws::Client::AWSError;
using ::Aws::S3::S3Errors;
namespace S3Model = Aws::S3::Model;

using ::arrow::internal::TaskGroup;
using ::arrow::internal::ToChars;
using ::arrow::internal::Uri;
using ::arrow::io::internal::SubmitIO;

using internal::ConnectRetryStrategy;
using internal::DetectS3Backend;
using internal::ErrorToStatus;
Expand Down Expand Up @@ -915,6 +913,39 @@ Result<std::shared_ptr<S3ClientHolder>> GetClientHolder(
// -----------------------------------------------------------------------
// S3 client factory: build S3Client from S3Options

struct EndpointConfigKey {
explicit EndpointConfigKey(const S3Options& options)
: region(options.region),
scheme(options.scheme),
endpoint_override(options.endpoint_override),
force_virtual_addressing(options.force_virtual_addressing) {}

std::string region;
std::string scheme;
std::string endpoint_override;
bool force_virtual_addressing;

bool operator==(const EndpointConfigKey& other) {
return region == other.region && scheme == other.scheme &&
endpoint_override == other.endpoint_override &&
force_virtual_addressing == other.force_virtual_addressing;
}
};

} // namespace
} // namespace arrow::fs

template <>
struct std::hash<arrow::fs::EndpointConfigKey> {
std::size_t operator()(const arrow::fs::EndpointConfigKey& key) const noexcept {
auto h = std::hash<std::string>{};
return h(key.region) ^ h(key.scheme) ^ h(key.endpoint_override);
}
};

namespace arrow::fs {
namespace {

class ClientBuilder {
public:
explicit ClientBuilder(S3Options options) : options_(std::move(options)) {}
Expand Down Expand Up @@ -992,9 +1023,12 @@ class ClientBuilder {
}

auto endpoint_provider = std::make_shared<Aws::S3::S3EndpointProvider>();
endpoint_provider->InitBuiltInParameters(client_config_);

auto client =
std::make_shared<S3Client>(credentials_provider_, nullptr, client_config_);
client->accessEndpointProvider() = endpoint_provider;

auto client = std::make_shared<S3Client>(credentials_provider_, endpoint_provider,
client_config_);
client->s3_retry_strategy_ = options_.retry_strategy;
return GetClientHolder(std::move(client));
}
Expand Down Expand Up @@ -3091,5 +3125,4 @@ Result<std::string> ResolveS3BucketRegion(const std::string& bucket) {
return resolver->ResolveRegion(bucket);
}

} // namespace fs
} // namespace arrow
} // namespace arrow::fs

0 comments on commit 8eeef9d

Please sign in to comment.