Skip to content

Commit

Permalink
Changes from review
Browse files Browse the repository at this point in the history
  • Loading branch information
@shaunrd0
shaunrd0 committed Nov 3, 2023
1 parent 01cbede commit 03bddbf
Show file tree
Hide file tree
Showing 9 changed files with 252 additions and 150 deletions.
26 changes: 15 additions & 11 deletions test/src/unit-vfs.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -360,12 +360,11 @@ TEMPLATE_LIST_TEST_CASE(
"[vfs][ls_cb]",
TestBackends) {
TestType fs({10, 50});
if (!fs.supports_fs()) {
if (!fs.is_supported()) {
return;
}
fs.setup_test();

tiledb::sm::LsCallback cb =
tiledb::sm::LsCallbackCAPI cb =
[](const char* path, size_t path_len, uint64_t size, void* data) {
auto ls_objects = static_cast<VFSTest::LsObjects*>(data);
ls_objects->emplace_back(std::string(path, path_len), size);
Expand All @@ -383,13 +382,12 @@ TEMPLATE_LIST_TEST_CASE(
TEST_CASE("VFS: ls_recursive callback stops traversal", "[vfs][ls_recursive]") {
// Currently only S3 is supported for VFS::ls_recursive.
VFSTest vfs_test({10, 50}, "s3://");
if (!vfs_test.supports_fs()) {
if (!vfs_test.is_supported()) {
return;
}
vfs_test.setup_test();

size_t cb_count = GENERATE(1, 11, 50);
tiledb::sm::LsCallback cb =
tiledb::sm::LsCallbackCAPI cb =
[&cb_count](
const char* path, size_t path_len, uint64_t size, void* data) {
auto ls_objects = static_cast<VFSTest::LsObjects*>(data);
Expand All @@ -399,7 +397,6 @@ TEST_CASE("VFS: ls_recursive callback stops traversal", "[vfs][ls_recursive]") {
}
return 1;
};

vfs_test.test_ls_recursive(cb, cb_count);
}

Expand All @@ -408,24 +405,31 @@ TEST_CASE(
"[vfs][ls_recursive]") {
std::string prefix =
GENERATE("s3://", "hdfs://", "azure://", "gcs://", "file://", "mem://");
if (prefix == "file://") {
#ifdef _WIN32
prefix += tiledb::sm::Win::current_dir() + "/";
#else
prefix += tiledb::sm::Posix::current_dir() + "/";
#endif
}
VFSTest vfs_test({1}, prefix);
if (!vfs_test.supports_fs()) {
if (!vfs_test.is_supported()) {
return;
}
vfs_test.setup_test();
std::string backend = vfs_test.temp_dir_.backend_name();

auto cb = [](const char*, size_t, uint64_t, void*) { return 1; };
VFSTest::LsObjects data;
tiledb::sm::LsCallbackWrapper cb_wrapper(cb, &data);
// Currently only S3 is supported for VFS::ls_recursive.
if (vfs_test.temp_dir_.is_s3()) {
DYNAMIC_SECTION(backend << " supported backend should not throw") {
CHECK_NOTHROW(vfs_test.vfs_.ls_recursive(vfs_test.temp_dir_, cb, &data));
CHECK_NOTHROW(vfs_test.vfs_.ls_recursive(vfs_test.temp_dir_, cb_wrapper));
}
} else {
DYNAMIC_SECTION(backend << " unsupported backend should throw") {
CHECK_THROWS_WITH(
vfs_test.vfs_.ls_recursive(vfs_test.temp_dir_, cb, &data),
vfs_test.vfs_.ls_recursive(vfs_test.temp_dir_, cb_wrapper),
Catch::Matchers::ContainsSubstring(
"storage backend is not supported"));
}
Expand Down
2 changes: 1 addition & 1 deletion test/support/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ target_compile_definitions(tiledb_test_support_lib PRIVATE
target_link_libraries(tiledb_test_support_lib
PUBLIC
TILEDB_CORE_OBJECTS_ILIB
Catch2::Catch2
$<TARGET_OBJECTS:tdb_catch>
)

# tdb_catch object library
Expand Down
154 changes: 78 additions & 76 deletions test/support/src/vfs_helpers.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,37 +42,34 @@
// after tdb_catch.h.
#include "test/support/src/serialization_wrappers.h"

#include <test/support/tdb_catch.h>
#include <test/support/tdb_catch_prng.h>
#include "test/support/src/helpers.h"
#include "test/support/src/vfs_helpers.h"

namespace tiledb {
namespace test {

tiledb::sm::URI test_dir(const std::string& prefix) {
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<uint64_t> dist(0);
return tiledb::sm::URI(prefix + "tiledb-" + std::to_string(dist(gen)));
return tiledb::sm::URI(prefix + "tiledb-" + std::to_string(PRNG::get()()));
}

tiledb::sm::Config create_test_config() {
tiledb::sm::Config cfg;
if constexpr (!tiledb::test::aws_s3_config) {
// Set up connection to minio backend emulator.
REQUIRE_NOTHROW(cfg.set("vfs.s3.endpoint_override", "localhost:9999"));
REQUIRE_NOTHROW(cfg.set("vfs.s3.scheme", "https"));
REQUIRE_NOTHROW(cfg.set("vfs.s3.use_virtual_addressing", "false"));
REQUIRE_NOTHROW(cfg.set("vfs.s3.verify_ssl", "false"));
cfg.set("vfs.s3.endpoint_override", "localhost:9999").ok();
cfg.set("vfs.s3.scheme", "https").ok();
cfg.set("vfs.s3.use_virtual_addressing", "false").ok();
cfg.set("vfs.s3.verify_ssl", "false").ok();
}
REQUIRE_NOTHROW(
cfg.set("vfs.azure.storage_account_name", "devstoreaccount1"));
REQUIRE_NOTHROW(cfg.set(
"vfs.azure.storage_account_key",
"Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/"
"K1SZFPTOtr/KBHBeksoGMGw=="));
REQUIRE_NOTHROW(cfg.set(
"vfs.azure.blob_endpoint", "http://127.0.0.1:10000/devstoreaccount1"));
cfg.set("vfs.azure.storage_account_name", "devstoreaccount1").ok();
cfg.set(
"vfs.azure.storage_account_key",
"Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/"
"K1SZFPTOtr/KBHBeksoGMGw==")
.ok();
cfg.set("vfs.azure.blob_endpoint", "http://127.0.0.1:10000/devstoreaccount1")
.ok();
return cfg;
}

Expand Down Expand Up @@ -442,7 +439,7 @@ std::string TemporaryDirectoryFixture::create_temporary_array(
return array_uri;
}

VFSTest::VFSTest(
VFSTestBase::VFSTestBase(
const std::vector<size_t>& test_tree, const std::string& prefix)
: test_tree_(test_tree)
, compute_(4)
Expand All @@ -452,25 +449,13 @@ VFSTest::VFSTest(
&io_,
&compute_,
tiledb::test::create_test_config())
, temp_dir_(tiledb::test::test_dir(prefix)) {
if (prefix == "file://") {
#ifdef _WIN32
temp_dir_ =
tiledb::test::test_dir(prefix + tiledb::sm::Win::current_dir() + "/");
#else
temp_dir_ =
tiledb::test::test_dir(prefix + tiledb::sm::Posix::current_dir() + "/");
#endif
}

if (temp_dir_.is_file() || temp_dir_.is_memfs()) {
vfs_.create_dir(temp_dir_).ok();
} else if (vfs_.supports_uri_scheme(temp_dir_)) {
vfs_.create_bucket(temp_dir_).ok();
}
, prefix_(prefix)
, temp_dir_(tiledb::test::test_dir(prefix_))
, is_supported_(vfs_.supports_uri_scheme(temp_dir_)) {
// TODO: Throw when we can provide a list of supported filesystems to Catch2.
}

VFSTest::~VFSTest() {
VFSTestBase::~VFSTestBase() {
if (vfs_.supports_uri_scheme(temp_dir_)) {
bool is_dir = false;
vfs_.is_dir(temp_dir_, &is_dir).ok();
Expand All @@ -480,32 +465,17 @@ VFSTest::~VFSTest() {
}
}

void VFSTest::create_objects(
const sm::URI& uri, size_t count, const std::string& prefix) {
for (size_t i = 1; i <= count; i++) {
auto object_uri = uri.join_path(prefix + std::to_string(i));
vfs_.touch(object_uri).ok();
std::string data(i * 10, 'a');
vfs_.open_file(object_uri, sm::VFSMode::VFS_WRITE).ok();
vfs_.write(object_uri, data.data(), data.size()).ok();
vfs_.close_file(object_uri).ok();
expected_results_.emplace_back(object_uri.to_string(), data.size());
}
}

void VFSTest::setup_test() {
for (size_t i = 1; i <= test_tree_.size(); i++) {
sm::URI path = temp_dir_.join_path("subdir_" + std::to_string(i));
// VFS::create_dir is a no-op for S3.
vfs_.create_dir(path).ok();
create_objects(path, test_tree_[i - 1], "test_file_");
}
VFSTest::VFSTest(
const std::vector<size_t>& test_tree, const std::string& prefix)
: VFSTestBase(test_tree, prefix) {
setup_test();
}

void VFSTest::test_ls_recursive(
tiledb::sm::LsCallback cb, size_t expected_count) {
tiledb::sm::LsCallbackCAPI cb, size_t expected_count) {
LsObjects ls_objects;
CHECK_NOTHROW(vfs_.ls_recursive(temp_dir_, cb, &ls_objects));
tiledb::sm::LsCallbackWrapper wrapper(cb, &ls_objects);
CHECK_NOTHROW(vfs_.ls_recursive(temp_dir_, wrapper));

std::sort(expected_results_.begin(), expected_results_.end());
if (expected_count != 0) {
Expand All @@ -515,47 +485,79 @@ void VFSTest::test_ls_recursive(
CHECK(ls_objects == expected_results_);
}

S3Test::S3Test(const std::vector<size_t>& test_tree)
: VFSTest(test_tree, "s3://") {
}
void VFSTest::setup_test() {
if (!is_supported_) {
return;
}

void S3Test::create_objects(
[[maybe_unused]] const sm::URI& uri,
[[maybe_unused]] size_t count,
[[maybe_unused]] const std::string& prefix) {
#ifdef HAVE_S3
for (size_t i = 1; i <= count; i++) {
auto object_uri = uri.join_path(prefix + std::to_string(i));
vfs_.s3()->touch(object_uri).ok();
std::string data(i * 10, 'a');
vfs_.s3()->write(object_uri, data.data(), data.size()).ok();
vfs_.s3()->flush_object(object_uri).ok();
expected_results_.emplace_back(object_uri.to_string(), data.size());
if (temp_dir_.is_file() || temp_dir_.is_memfs()) {
vfs_.create_dir(temp_dir_).ok();
} else {
vfs_.create_bucket(temp_dir_).ok();
}
for (size_t i = 1; i <= test_tree_.size(); i++) {
sm::URI path = temp_dir_.join_path("subdir_" + std::to_string(i));
// VFS::create_dir is a no-op for S3.
vfs_.create_dir(path).ok();
for (size_t j = 1; j <= test_tree_[i - 1]; j++) {
auto object_uri = path.join_path("test_file_" + std::to_string(j));
vfs_.touch(object_uri).ok();
std::string data(j * 10, 'a');
vfs_.open_file(object_uri, sm::VFSMode::VFS_WRITE).ok();
vfs_.write(object_uri, data.data(), data.size()).ok();
vfs_.close_file(object_uri).ok();
expected_results_.emplace_back(object_uri.to_string(), data.size());
}
}
}

LocalFsTest::LocalFsTest(const std::vector<size_t>& test_tree)
: VFSTestBase(test_tree, "file://") {
#ifdef _WIN32
temp_dir_ =
tiledb::test::test_dir(prefix_ + tiledb::sm::Win::current_dir() + "/");
#else
temp_dir_ =
tiledb::test::test_dir(prefix_ + tiledb::sm::Posix::current_dir() + "/");
#endif
setup_test();
}

S3Test::S3Test(const std::vector<size_t>& test_tree)
: VFSTestBase(test_tree, "s3://") {
setup_test();
}

void S3Test::setup_test() {
#ifdef HAVE_S3
vfs_.s3().create_bucket(temp_dir_).ok();
for (size_t i = 1; i <= test_tree_.size(); i++) {
sm::URI path = temp_dir_.join_path("subdir_" + std::to_string(i));
// VFS::create_dir is a no-op for S3; Just create objects.
create_objects(path, test_tree_[i - 1], "test_file_");
for (size_t j = 1; j <= test_tree_[i - 1]; j++) {
auto object_uri = path.join_path("test_file_" + std::to_string(j));
vfs_.s3().touch(object_uri).ok();
std::string data(j * 10, 'a');
vfs_.s3().write(object_uri, data.data(), data.size()).ok();
vfs_.s3().flush_object(object_uri).ok();
expected_results_.emplace_back(object_uri.to_string(), data.size());
}
}
#endif
}

void S3Test::test_ls_cb(
[[maybe_unused]] tiledb::sm::LsCallback cb,
[[maybe_unused]] tiledb::sm::LsCallbackCAPI cb,
[[maybe_unused]] bool recursive) {
#ifdef HAVE_S3
LsObjects ls_objects;
tiledb::sm::LsCallbackWrapper wrapper(cb, &ls_objects);
// If testing with recursion use the root directory, otherwise use a subdir.
auto path = recursive ? temp_dir_ : temp_dir_.join_path("subdir_1");
if (recursive) {
CHECK_NOTHROW(vfs_.s3()->ls_cb(path, cb, &ls_objects, ""));
CHECK_NOTHROW(vfs_.s3().ls_cb(path, wrapper, ""));
} else {
CHECK_NOTHROW(vfs_.s3()->ls_cb(path, cb, &ls_objects));
CHECK_NOTHROW(vfs_.s3().ls_cb(path, wrapper));
}

if (!recursive) {
Expand Down
Loading

0 comments on commit 03bddbf

Please sign in to comment.