diff --git a/CMakeLists.txt b/CMakeLists.txt index a5e2f56a4f0c..bcc915f3048d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -219,7 +219,12 @@ else() elseif (CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo") add_compile_options(-DNDEBUG -O3 -g3 -ggdb3 -gdwarf-3) elseif (CMAKE_BUILD_TYPE MATCHES "Coverage") - add_compile_options(-DDEBUG -g3 -gdwarf-3 --coverage) + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + add_compile_options(-DDEBUG -O0 -g3 --coverage -fprofile-instr-generate -fcoverage-mapping) + add_link_options(--coverage -fprofile-instr-generate -fcoverage-mapping) + else() + add_compile_options(-DDEBUG -g3 -gdwarf-3 --coverage) + endif() endif() # Use -Wno-literal-suffix on Linux with C++ sources. @@ -341,6 +346,7 @@ list(APPEND TILEDB_C_API_RELATIVE_HEADERS "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/datatype/datatype_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/dimension/dimension_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/dimension_label/dimension_label_api_external.h" + "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/enumeration/enumeration_api_experimental.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/error/error_api_external.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filesystem/filesystem_api_enum.h" "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filesystem/filesystem_api_external.h" diff --git a/PJD_TODO.md b/PJD_TODO.md new file mode 100644 index 000000000000..61aa4f42c906 --- /dev/null +++ b/PJD_TODO.md @@ -0,0 +1,22 @@ +Chores Left +=== + +* Testing + * Array schema evolution after schema is read from disk + * Throw error if enumeration has longer length than the integer width of the attribute + * Require the attribute type to be integral when setting an enumeration + * Don't allow signed integer values for attributes with enumeratiojns? Does R need this? + * Require cell_val_num == 1 for attributes + * Don't forget to test drop and then add attribute with enumeration. + * ArraySchemaEvolution - serialization when adding the same enumeration to multiple attributes + +* Missing APIs + * array schema evolution C API + * QueryConditon C API + +* Miscellany + * Add Enumeration::dump(FILE* out) + * Document enumeration format changes in the storage format docs + +* Future TODO: + * Update schema consolidation and vaccuming for enumerations diff --git a/examples/cpp_api/enumerations.cc b/examples/cpp_api/enumerations.cc new file mode 100644 index 000000000000..845ace42544e --- /dev/null +++ b/examples/cpp_api/enumerations.cc @@ -0,0 +1,106 @@ +/** + * @file enumerations.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This example shows the use of enumerations in TileDB + */ + +#include +#include +#include + +using namespace tiledb; + +std::string array_bucket = "s3://tiledb-davisp"; +std::string create_uri = "tiledb://demo/s3://tiledb-davisp/enumerations"; +std::string array_uri = "tiledb://demo/enumerations"; + +void create_array(Config& cfg) { + Context ctx(cfg); + VFS vfs(ctx); + + if (vfs.is_bucket(array_bucket)) { + vfs.remove_bucket(array_bucket); + } + + std::cerr << "Creating bucket." << std::endl; + vfs.create_bucket(array_bucket); + if (!vfs.is_bucket(array_bucket)) { + throw std::runtime_error("Failed to create bucket."); + } + + std::cerr << "Bucket exists." << std::endl; + + ArraySchema schema(ctx, TILEDB_DENSE); + + auto dim = Dimension::create(ctx, "dim", {{1, 100}}); + auto dom = Domain(ctx); + dom.add_dimension(dim); + schema.set_domain(dom); + + auto attr = Attribute::create(ctx, "attr"); + + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx, values); + ArraySchemaExperimental::add_attribute(ctx, schema, attr, enmr); + + std::cerr << "Creating array" << std::endl; + Array::create(create_uri, schema); +} + +void write_array(Config& cfg) { + Context ctx(cfg); + + std::vector attr_data; + for (size_t i = 1; i <= 100; i++) { + attr_data.push_back(i % 4); + } + + Array array(ctx, array_uri, TILEDB_WRITE); + Query query(ctx, array, TILEDB_WRITE); + query.set_layout(TILEDB_ROW_MAJOR).set_data_buffer("attr", attr_data); + if (query.submit() != Query::Status::COMPLETE) { + throw std::runtime_error("Failed to write array."); + } + + array.close(); +} + +int main(int, char*[]) { + Config cfg; + cfg["rest.username"] = "demo"; + cfg["rest.password"] = "demodemodemodemo"; + cfg["rest.server_address"] = "http://localhost:8081"; + + try { + create_array(cfg); + write_array(cfg); + } catch (std::exception& exc) { + std::cerr << "XKCD: Exception: " << exc.what() << std::endl; + } +} diff --git a/format_spec/enumeration.md b/format_spec/enumeration.md new file mode 100644 index 000000000000..3382305f03d2 --- /dev/null +++ b/format_spec/enumeration.md @@ -0,0 +1,28 @@ +--- +title: Enumerations +--- + +## Main Structure + +``` +my_array # array folder + | ... + |_ schema # ArraySchema directory named `__schema` + |_ enumerations # Enumeration directory named `__enumerations` + | |_ enumeration # enumeration data with names `__t1_t2_uuid_v` + + +Enumeration data is stored in a subdirectory of the [array schema][./array_schema.md] +directory. Enumerations are stored using [Generic Tiles][./generic_tile.md]. +Data stored in the generic tile follows the current format as of version 19. + +| **Field** | **Type** | **Description** | +| :--- | :--- | :--- | +| Version number | `uint32_t` | Format version number of the generic tile | +| Datatype | `uint8_t` | The datatype of the enumeration values | +| Cell Val Num | `uint32_t` | The cell val num of the enumeration values | +| Ordered | `bool` | Whether the enumeration values should be considered ordered | +| Data Size | `uint64_t` | The number of bytes used to store the values | +| Data | `uint8_t` * Data Size | The data for the enumeration values | +| Offsets Size | `uint64_t` | The number of bytes used to store offsets if cell_var_num is TILEDB_VAR_NUM | +| Offsets | `uint8_t` * Offsets Size | The offsets data for the enumeration if cell_var_num is TILEDB_VAR_NUM | diff --git a/scripts/generate-coverage-report.py b/scripts/generate-coverage-report.py new file mode 100755 index 000000000000..1737ec156bee --- /dev/null +++ b/scripts/generate-coverage-report.py @@ -0,0 +1,553 @@ +#!/usr/bin/env python3 + +# Write comment here to read all of this with the knowledge that the line +# math is 1-index based. Its easy to forget that parsing the output of +# porcelain. + +import argparse as ap +import html +import os +import re +import subprocess as sp +import sys +import textwrap as tw +import time + +LLVM_PROFDATA = "/Library/Developer/CommandLineTools/usr/bin/llvm-profdata" +LLVM_COV = "/Library/Developer/CommandLineTools/usr/bin/llvm-cov" + +RANGE_RE = re.compile("@@\s-(\d+)(,(\d+))?\s+\+(?P\d+)(,(?P\d+))?\s+@@") + +# From: https://stackoverflow.com/a/14693789 +ANSI_CODE_RE = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') + +SOURCE_EXTENSIONS = set([ + ".h", + ".c", + ".c++", + ".cc", + ".cpp", + ".hpp" +]) + +def log(*args, **kwargs): + kwargs["file"] = sys.stderr + print(*args, **kwargs) + +def is_source_file(fname): + if fname is None: + return False + (_, ext) = os.path.splitext(fname) + return ext in SOURCE_EXTENSIONS + +def find_source_dir(): + if not os.path.exists("CMakeCache.txt"): + print("Error finding CMakeCache.txt") + print("Are you running in your build directory?") + exit(2) + with open("CMakeCache.txt") as handle: + for line in handle: + bits = line.split("=", 1) + if len(bits) != 2: + continue + if bits[0] == "CMAKE_HOME_DIRECTORY:INTERNAL": + return bits[1].strip() + print("Failed to find source directory in CMakeCache.txt") + exit(2) + +def generate_file_list(): + source_dir = find_source_dir() + cmd = [ + "git", + "-C", + source_dir, + "merge-base", + "origin/dev", + "HEAD" + ] + merge_base = sp.check_output(" ".join(cmd), shell=True).decode("utf-8") + log("Generating diff between ") + cmd = [ + "git", + "-C", + source_dir, + "--no-pager", + "diff", + "--unified=0", + merge_base.strip() + ] + output = sp.check_output(" ".join(cmd), shell=True).decode("utf-8") + ret = {} + fname = None + ranges = [] + for line in output.splitlines(): + if line.startswith("+++ b/"): + if is_source_file(fname): + ret[fname] = ranges + fname = line[len("+++ b/"):].strip() + ranges = [] + elif line.startswith("@@"): + match = RANGE_RE.search(line.strip()) + if match is None: + log("Failed to parse: '{}'".format(line.strip())) + exit(3) + add_start = match.group("start") + add_len = match.group("length") + if add_len is None: + add_len = 1 + else: + add_len = int(add_len) + if add_len == 0: + continue + source_marker = line.rsplit("@@")[-1].strip() + ranges.append((int(add_start), int(add_len), source_marker)) + if is_source_file(fname): + ret[fname] = ranges + return ret + +def update_profdata(): + profraw_files = [] + for path, dnames, fnames in os.walk("."): + for fname in fnames: + if os.path.splitext(fname)[1] == ".profraw": + profraw_files.append(os.path.join(path, fname)) + cmd = [ + LLVM_PROFDATA, + "merge", + "-sparse", + ] + profraw_files + [ + "-o", "default.profdata" + ] + sp.check_call(cmd) + +def generate_text_reports(file_info, targets): + file_list = file_info.keys() + file_list = list(sorted([os.path.join("..", fname) for fname in file_list])) + cmd = [ + LLVM_COV, + "show", + "--instr-profile=default.profdata", + "--Xdemangler=c++filt", + "--Xdemangler=--no-strip-underscore", + "--output-dir=.", + targets[0] + ] + for tgt in targets[1:]: + cmd.extend(["-object", tgt]) + cmd += file_list + sp.check_call(cmd) + +def load_text_reports(file_info): + ret = {} + prev_was_dashes = None + curr_fname = None + curr_template = None + curr_lines = None + for path, dnames, fnames in os.walk("coverage"): + for fname in fnames: + if os.path.splitext(fname)[1] != ".txt": + continue + if curr_fname is not None: + ret[curr_fname] = curr_lines + fname = os.path.join(path, fname) + curr_fname = fname + curr_lines = [] + with open(fname) as handle: + data = handle.read() + data = ANSI_CODE_RE.sub('', data) + for (idx, line) in enumerate(data.splitlines()): + if idx < 3: + continue + line = line.strip() + if not line: + continue + if prev_was_dashes and line.startswith("|"): + curr_template = line[1:].strip().rstrip(":") + continue + else: + curr_template = None + if all(c == "-" for c in line): + prev_was_dashes = True + continue + else: + prev_was_dashes = False + bits = line.strip().split("|", 2) + assert len(bits) == 3, line.strip() + " " + fname + line = int(bits[0]) + count = None + if bits[1].strip(): + count = bits[1] + if count.endswith("E"): + # I have no idea if this is correct, but the data seems to make + # sense that this is a bug where it should be E1 i.e., * 10^1 + count = int(float(count[:-1]) * 100) + elif count.endswith("k"): + count = int(float(count[:-1]) * 1000) + elif count.endswith("M"): + count = int(float(count[:-1]) * 1000000) + else: + count = int(bits[1].strip()) + curr_lines.append((line, count, curr_template, bits[2].rstrip())) + if curr_fname is not None: + ret[curr_fname] = curr_lines + return ret + +def coverage_key(fname): + base = os.path.join("coverage", find_source_dir().lstrip(os.sep)) + return os.path.join(base, fname) + ".txt" + +def fake_txt_report(fname): + abs_path = os.path.join(find_source_dir(), fname) + ret = [] + with open(abs_path) as handle: + for (idx, line) in enumerate(handle): + ret.append((idx + 1, None, None, line.rstrip())) + return ret + +def render_html(file_info, txt_reports): + print(tw.dedent("""\ + + + + + + + + + """.format(css=css()))) + render_toc(file_info, txt_reports) + for fname in sorted(file_info.keys()): + if coverage_key(fname) not in txt_reports: + continue + render_file(fname, file_info[fname], txt_reports[coverage_key(fname)]) + for fname in sorted(file_info.keys()): + if coverage_key(fname) in txt_reports: + continue + render_file(fname, file_info[fname], fake_txt_report(fname)) + print(tw.dedent("""\ + + + """)) + +def render_toc(file_info, txt_reports): + print('

Covered Files

') + print(' ') + print(' ') + print(' ') + print(' ') + print(' ') + print(' ') + print(' ') + print(' ') + total_lines = 0 + total_executable = 0 + total_executed = 0 + for fname in file_info: + if coverage_key(fname) not in txt_reports: + continue + (lines, executable, executed, coverage, row_class) = calculate_coverage(fname, file_info, txt_reports) + total_lines += int(lines) + if executable != "": + total_executable += int(executable) + if executed != "": + total_executed += int(executed) + anchor = fname.replace(os.sep, "-") + print(' '.format(row_class)) + print(' '.format(anchor=anchor, fname=fname)) + print(' '.format(lines)) + print(' '.format(executable)) + print(' '.format(executed)) + print(' '.format(coverage)) + print(' ') + # Print out a totals row + (lines, executable, executed, coverage, row_class) = classify_coverage(total_lines, total_executable, total_executed); + print(' '.format(row_class)) + print(' ') + print(' '.format(lines)) + print(' '.format(executable)) + print(' '.format(executed)) + print(' '.format(coverage)) + print(' ') + print('
FilenameAdditionsExecutableExecutedCovered
{fname}{}{}{}{}
Total Coverage{}{}{}{}
') + print('

No Data Collected

') + print(' ') + print(' ') + print(' ') + print(' ') + print(' ') + for fname in file_info: + if coverage_key(fname) in txt_reports: + continue + anchor = fname.replace(os.sep, "-") + changes = 0 + for region in file_info[fname]: + changes += region[1] + print(' ') + print(' '.format(anchor, fname)) + print(' '.format(changes)) + print(' ') + print('
FilenameAdditions
{}{}
') + +def calculate_coverage(fname, file_info, txt_reports): + regions = list(sorted(file_info[fname])) + # No regions mean we're picking up a file that we only removed lines from + if not regions: + return "0", "0", "0", "100.00%", "covered-file-good" + lines = 0 + executable = 0 + executed = 0 + txt_report = txt_reports[coverage_key(fname)] + for line in txt_report: + if not regions: + break + if line[0] < regions[0][0]: + continue + if line[0] <= regions[0][0] + regions[0][1] - 1: + lines += 1 + if line[1] is None: + continue + executable += 1 + if line[1] > 0: + executed += 1 + if line[0] >= regions[0][0] + regions[0][1] - 1: + regions.pop(0) + return classify_coverage(lines, executable, executed) + +def classify_coverage(lines, executable, executed): + # None of our additions in this file were marked as executable so we + # mark it 100% to avoid confusion that there are any more tests to add + if executable == 0: + return lines, "0", "0", "100.00%", "covered-file-good" + coverage = 100.0 * float(executed) / float(executable) + if coverage < 20.0: + return lines, executable, executed, "{:.2f}%".format(coverage), "covered-file-warn" + if coverage < 80.0: + return lines, executable, executed, "{:.2f}%".format(coverage), "covered-file" + else: + return lines, executable, executed, "{:.2f}%".format(coverage), "covered-file-good" + +def render_file(fname, file_info, txt_report): + anchor = fname.replace(os.sep, "-") + source_args = {"anchor": anchor, "fname": fname} + # Should already be sorted but why rely on that assumption? + changes = list(sorted(file_info)) + regions = add_context(file_info, 3) + print(" ") + print(' ') + print(' ') + print(' ') + print(' ') + print(' ') + print(' ') + print(' ') + print(' ') + print(' ') + for line in txt_report: + # if regions: + # print(regions) + # print(line) + if not regions: + break + if line[0] < regions[0][0]: + continue + if regions and line[0] == regions[0][0]: + print(' ') + print(' ') + print(' ') + change_value = "" + if changes and line[0] >= changes[0][0] and line[0] < changes[0][0] + changes[0][1]: + change_value = "+" + if line[1] is None: + row_class = "uncovered-line" + elif line[1] == 0: + row_class = "unexecuted-line" + else: + row_class = "executed-line" + else: + row_class = "uncovered-line" + print(' '.format(row_class)) + print(' '.format(line=line[0])) + if line[1] is None: + print(' ') + else: + print(' '.format(count=line[1])) + print(' '.format(change_value)) + print(' '.format(code=html.escape(line[3]))) + print(' ') + # Update changes and regions if we're on the last line of either + if changes and line[0] >= changes[0][0] + changes[0][1] - 1: + changes.pop(0) + if regions and line[0] >= regions[0][0] + regions[0][1] - 1: + regions.pop(0) + print("
') + print('
') + print(' 
{fname}
'.format(**source_args)) + print('
') + print(' 
Line
Count
Source
') + print('
') + print(' 
{}
'.format(html.escape(regions[0][2]))) + print('
') + print(' 
{line}
{count}
{}
{code}
") + +def add_context(regions, num_lines): + ret = [] + for region in regions: + new_start = max(0, region[0] - num_lines) + new_len = region[1] + 2 * num_lines + if ret and new_start <= ret[-1][0] + ret[-1][1]: + end_line = region[0] + region[1] + num_lines + ret[-1][1] = end_line - ret[-1][0] + else: + ret.append([new_start, new_len, region[2]]) + return list(sorted(tuple(row) for row in ret)) + +def parse_args(argv): + parser = ap.ArgumentParser( + prog=argv[0], + description='Generate code coverage reports') + parser.add_argument("target", metavar="EXECUTABLE", nargs='+', + help = "One or more paths to executables used to generate profraw data"); + return parser.parse_args() + +def main(): + args = parse_args(sys.argv) + file_info = generate_file_list() + update_profdata() + generate_text_reports(file_info, args.target) + txt_reports = load_text_reports(file_info) + render_html(file_info, txt_reports) + +def css(): + # This CSS was cribbed from the llvm-cov output + return tw.dedent(""" + body { + font-family: monospace; + } + pre { + margin-top: 0px !important; + margin-bottom: 0px !important; + } + table { + border-collapse: collapse; + border: 1px solid gray; + margin: auto; + margin-bottom: 2em; + } + th { + text-align: left; + } + td { + vertical-align: top; + padding: 2px 8px; + border-collapse: collapse; + border-right: solid 1px #eee; + border-left: solid 1px #eee; + text-align: left; + } + td pre { + display: inline-block; + } + td:first-child { + border-left: none; + } + td:last-child { + border-right: none; + } + tr:hover { + background-color: #f0f0f0; + } + .lines { + text-align: right; + } + .executable { + text-align: right; + } + .executed { + text-align: right; + } + .coverage { + text-align: right; + } + .source-file { + padding: 5px 10px; + border-bottom: 1px; + background-color: aliceblue; + line-height: 35px; + } + .source-marker { + padding: 5px 10px; + border-bottom: 1px; + background-color: azure; + line-height: 20px; + } + .changed { + text-align: center; + } + .uncovered-file { + background: palegoldenrod; + } + .covered-file { + } + .covered-file-warn { + background: lightpink; + } + .covered-file-good { + background: palegreen; + } + .line-number { + text-align: right; + } + .line-count { + text-align: right; + } + .uncovered-line { + + } + .unexecuted-line { + background: lightpink; + } + .executed-line { + + } + /* CSS OVERRIDES */ + body { + max-width: 1024px; + padding: 0 16px; + margin: 0 auto; + } + table { + border: none; + background-color: #f9fafb; + border-radius: 6px; + overflow: hidden; + width: 100%; + box-shadow: 0 1px 3px rgba(0,0,0,.16); + } + th { + color: #444; + font-size: 14px; + line-height: 1.5; + padding: 4px; + } + tr { + line-height: 1.5; + transition: background 0.3s ease-in-out; + color: #444; + } + a { + text-decoration: none; + color: rgb(0, 112, 240); + border-color: rgb(0, 112, 240); + border-width: 0; + } + a:hover { + border-bottom-width: 1px; + border-style: solid; + } + /* END OF CSS OVERRIDES */ + """) + +if __name__ == "__main__": + main() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6930e294ac4c..50b5783b46ad 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -139,6 +139,7 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-capi-dense_neg.cc src/unit-capi-dense_vector.cc src/unit-capi-enum_values.cc + src/unit-capi-enumerations.cc src/unit-capi-error.cc src/unit-capi-filestore.cc src/unit-capi-fill_values.cc @@ -176,12 +177,15 @@ set(TILEDB_UNIT_TEST_SOURCES src/unit-dimension.cc src/unit-duplicates.cc src/unit-empty-var-length.cc + src/unit-enumerations.cc + src/unit-enum-helpers.cc src/unit-filter-buffer.cc src/unit-filter-pipeline.cc src/unit-global-order.cc src/unit-gcs.cc src/unit-gs.cc src/unit-hdfs-filesystem.cc + src/unit-misc-util-safe-integral-casts.cc src/unit-ordered-dim-label-reader.cc src/unit-tile-metadata.cc src/unit-tile-metadata-generator.cc @@ -217,6 +221,7 @@ if (TILEDB_CPP_API) src/unit-cppapi-deletes.cc src/unit-cppapi-dense-qc-coords-mode.cc src/unit-cppapi-time.cc + src/unit-cppapi-enumerations.cc src/unit-cppapi-fill_values.cc src/unit-cppapi-filter.cc src/unit-cppapi-float-scaling-filter.cc diff --git a/test/src/unit-capi-enumerations.cc b/test/src/unit-capi-enumerations.cc new file mode 100644 index 000000000000..4fbe93a4fc88 --- /dev/null +++ b/test/src/unit-capi-enumerations.cc @@ -0,0 +1,120 @@ +/** + * @file unit-capi-enumerations.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2017-2021 TileDB Inc. + * @copyright Copyright (c) 2016 MIT and Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests for the various Enumerations C API errors. + */ + +#include +#include "tiledb/sm/c_api/tiledb.h" +#include "tiledb/sm/c_api/tiledb_experimental.h" + +#include + +TEST_CASE( + "C API: Test invalid attribute for tiledb_attribute_set_enumeration_name", + "[enumeration][capi][error]") { + tiledb_ctx_t* ctx; + + auto rc = tiledb_ctx_alloc(nullptr, &ctx); + CHECK(rc == TILEDB_OK); + + rc = tiledb_attribute_set_enumeration_name(ctx, nullptr, "enmr_name"); + REQUIRE(rc == TILEDB_ERR); +} + +TEST_CASE( + "C API: Test invalid attribute for tiledb_attribute_get_enumeration_name", + "[enumeration][capi][error]") { + tiledb_ctx_t* ctx; + + auto rc = tiledb_ctx_alloc(nullptr, &ctx); + CHECK(rc == TILEDB_OK); + + tiledb_string_handle_t* name; + rc = tiledb_attribute_get_enumeration_name(ctx, nullptr, &name); + REQUIRE(rc == TILEDB_ERR); +} + +TEST_CASE( + "C API: Test invalid array schema for tiledb_array_schema_add_enumeration", + "[enumeration][capi][error]") { + tiledb_ctx_t* ctx; + + auto rc = tiledb_ctx_alloc(nullptr, &ctx); + CHECK(rc == TILEDB_OK); + + tiledb_enumeration_t* enmr; + uint32_t values[5] = {1, 2, 3, 4, 5}; + rc = tiledb_enumeration_alloc( + ctx, + "an_enumeration", + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enmr); + REQUIRE(rc == TILEDB_OK); + + rc = tiledb_array_schema_add_enumeration(ctx, nullptr, enmr); + REQUIRE(rc == TILEDB_ERR); +} + +TEST_CASE( + "C API: Test invalid array for tiledb_array_get_enumeration", + "[enumeration][capi][error]") { + tiledb_ctx_t* ctx; + + auto rc = tiledb_ctx_alloc(nullptr, &ctx); + CHECK(rc == TILEDB_OK); + + tiledb_enumeration_t* enmr; + rc = tiledb_array_get_enumeration(ctx, nullptr, "an_enumeration", &enmr); + REQUIRE(rc == TILEDB_ERR); +} + +TEST_CASE( + "C API: Test invalid enumeration name for tiledb_array_get_enumeration", + "[enumeration][capi][error]") { + tiledb_ctx_t* ctx; + + auto rc = tiledb_ctx_alloc(nullptr, &ctx); + CHECK(rc == TILEDB_OK); + + tiledb_array_t* array; + rc = tiledb_array_alloc(ctx, "array_uri", &array); + REQUIRE(rc == TILEDB_OK); + + tiledb_enumeration_t* enmr; + rc = tiledb_array_get_enumeration(ctx, array, nullptr, &enmr); + REQUIRE(rc == TILEDB_ERR); +} diff --git a/test/src/unit-cppapi-deletes.cc b/test/src/unit-cppapi-deletes.cc index f7a941d771ac..d7446bc48ee6 100644 --- a/test/src/unit-cppapi-deletes.cc +++ b/test/src/unit-cppapi-deletes.cc @@ -109,6 +109,7 @@ struct DeletesFx { void remove_sparse_array(); void remove_array(const std::string& array_name); bool is_array(const std::string& array_name); + std::vector list_schemas(const std::string& array_name); }; DeletesFx::DeletesFx() @@ -488,6 +489,29 @@ bool DeletesFx::is_array(const std::string& array_name) { return vfs_.is_dir(array_name); } +std::vector DeletesFx::list_schemas( + const std::string& array_name) { + auto& enum_dir = tiledb::sm::constants::array_enumerations_dir_name; + auto schemas = + vfs_.ls(array_name + "/" + tiledb::sm::constants::array_schema_dir_name); + + auto it = schemas.begin(); + while (it != schemas.end()) { + if ((*it).size() < enum_dir.size()) { + continue; + } + if ((*it).substr((*it).size() - enum_dir.size()) == enum_dir) { + break; + } + ++it; + } + if (it != schemas.end()) { + schemas.erase(it); + } + + return schemas; +} + TEST_CASE_METHOD( DeletesFx, "CPP API: Test writing delete condition", @@ -1949,8 +1973,7 @@ TEST_CASE_METHOD( // Check write CHECK(tiledb::test::num_commits(SPARSE_ARRAY_NAME) == 4); CHECK(tiledb::test::num_fragments(SPARSE_ARRAY_NAME) == 4); - auto schemas = - vfs_.ls(array_name + "/" + tiledb::sm::constants::array_schema_dir_name); + auto schemas = list_schemas(array_name); CHECK(schemas.size() == 1); auto meta = vfs_.ls( array_name + "/" + tiledb::sm::constants::array_metadata_dir_name); @@ -1984,8 +2007,7 @@ TEST_CASE_METHOD( // Check working directory after delete REQUIRE(vfs_.is_file(extraneous_file_path)); CHECK(tiledb::test::num_fragments(SPARSE_ARRAY_NAME) == 0); - schemas = - vfs_.ls(array_name + "/" + tiledb::sm::constants::array_schema_dir_name); + schemas = list_schemas(array_name); CHECK(schemas.size() == 0); meta = vfs_.ls( array_name + "/" + tiledb::sm::constants::array_metadata_dir_name); @@ -2042,8 +2064,7 @@ TEST_CASE_METHOD( vfs_.touch(extraneous_file_path); // Check write - auto schemas = - vfs_.ls(array_name + "/" + tiledb::sm::constants::array_schema_dir_name); + auto schemas = list_schemas(array_name); CHECK(schemas.size() == 1); auto uris = vfs_.ls(array_name); bool ok_exists = false; @@ -2072,8 +2093,7 @@ TEST_CASE_METHOD( CHECK(!tiledb::sm::utils::parse::starts_with(uri, ok_prefix)); } REQUIRE(vfs_.is_file(extraneous_file_path)); - schemas = - vfs_.ls(array_name + "/" + tiledb::sm::constants::array_schema_dir_name); + schemas = list_schemas(array_name); CHECK(schemas.size() == 0); remove_sparse_array(); @@ -2259,11 +2279,9 @@ TEST_CASE_METHOD( auto group_meta_dir = vfs_.ls(GROUP_NAME + tiledb::sm::constants::group_metadata_dir_name); CHECK(group_meta_dir.size() == 1); - auto array_schema = - vfs_.ls(array_path + tiledb::sm::constants::array_schema_dir_name); + auto array_schema = list_schemas(array_path); CHECK(array_schema.size() == 1); - auto array2_schema = - vfs_.ls(array2_path + tiledb::sm::constants::array_schema_dir_name); + auto array2_schema = list_schemas(array2_path); CHECK(array2_schema.size() == 1); // Recursively delete group in modify exclusive mode @@ -2283,11 +2301,9 @@ TEST_CASE_METHOD( auto group2_detail_dir = vfs_.ls(group2_path + tiledb::sm::constants::group_detail_dir_name); CHECK(group2_detail_dir.size() == 0); - array_schema = - vfs_.ls(array_path + tiledb::sm::constants::array_schema_dir_name); + array_schema = list_schemas(array_path); CHECK(array_schema.size() == 0); - array2_schema = - vfs_.ls(array2_path + tiledb::sm::constants::array_schema_dir_name); + array2_schema = list_schemas(array2_path); CHECK(array2_schema.size() == 0); // Clean up diff --git a/test/src/unit-cppapi-enumerations.cc b/test/src/unit-cppapi-enumerations.cc new file mode 100644 index 000000000000..451ea540a095 --- /dev/null +++ b/test/src/unit-cppapi-enumerations.cc @@ -0,0 +1,325 @@ +/** + * @file unit-cppapi-enumeration.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the C++ API for filter related functions. + */ + +#include +#include "tiledb/sm/cpp_api/tiledb" +#include "tiledb/sm/cpp_api/tiledb_experimental" + +using namespace tiledb; + +struct CPPEnumerationFx { + CPPEnumerationFx(); + ~CPPEnumerationFx(); + + void create_array(); + void rm_array(); + + std::string uri_; + Context ctx_; + VFS vfs_; +}; + +const static std::string enmr_name = "an_enumeration"; + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Create Fixed Size", + "[enumeration][create][fixed-size]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + REQUIRE(enmr.ptr() != nullptr); + REQUIRE(enmr.name() == enmr_name); + REQUIRE(enmr.type() == TILEDB_INT32); + REQUIRE(enmr.cell_val_num() == 1); + REQUIRE(enmr.ordered() == false); + + auto data = enmr.as_vector(); + REQUIRE(data == values); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration API - Create Variable Size", + "[enumeration][create][var-size]") { + std::vector values = {"fee", "fi", "fo", "fum"}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + REQUIRE(enmr.ptr() != nullptr); + REQUIRE(enmr.name() == enmr_name); + REQUIRE(enmr.type() == TILEDB_STRING_ASCII); + REQUIRE(enmr.cell_val_num() == TILEDB_VAR_NUM); + REQUIRE(enmr.ordered() == false); + + auto data = enmr.as_vector(); + REQUIRE(data == values); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Add Enumeration to Schema", + "[enumeration][add-attribute]") { + ArraySchema schema(ctx_, TILEDB_DENSE); + + auto dim = Dimension::create(ctx_, "dim", {{-100, 100}}); + auto dom = Domain(ctx_); + dom.add_dimension(dim); + schema.set_domain(dom); + + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx_, enmr_name, values); + ArraySchemaExperimental::add_enumeration(ctx_, schema, enmr); + + auto attr = Attribute::create(ctx_, "attr"); + AttributeExperimental::set_enumeration_name(ctx_, attr, enmr_name); + schema.add_attribute(attr); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumerations From Disk - Array::get_enumeration", + "[enumeration][array-get-enumeration]") { + create_array(); + auto array = Array(ctx_, uri_, TILEDB_READ); + auto enmr = ArrayExperimental::get_enumeration(ctx_, array, "an_enumeration"); + REQUIRE(enmr.ptr() != nullptr); + REQUIRE(enmr.name() == "an_enumeration"); + REQUIRE(enmr.type() == TILEDB_STRING_ASCII); + REQUIRE(enmr.cell_val_num() == TILEDB_VAR_NUM); + REQUIRE(enmr.ordered() == false); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumerations From Disk - Attribute::get_enumeration_name", + "[enumeration][attr-get-enumeration-name]") { + create_array(); + auto schema = Array::load_schema(ctx_, uri_); + + auto attr1 = schema.attribute("attr1"); + auto enmr_name1 = AttributeExperimental::get_enumeration_name(ctx_, attr1); + REQUIRE(enmr_name1.has_value() == true); + + auto attr2 = schema.attribute("attr2"); + auto enmr_name2 = AttributeExperimental::get_enumeration_name(ctx_, attr2); + REQUIRE(enmr_name2.has_value() == false); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration Query - Basic", + "[enumeration][query][basic]") { + // Basic smoke test. Check that a simple query condition applied against + // an array returns sane results. + create_array(); + + // Check attr1 == "fred" when attr1 is an enumeration. + QueryCondition qc(ctx_); + qc.init("attr1", "fred", 4, TILEDB_EQ); + + // Execute the query condition against the array + std::vector dim(5); + std::vector attr1(5); + + auto array = Array(ctx_, uri_, TILEDB_READ); + Query query(ctx_, array); + query.add_range("dim", 1, 5) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("dim", dim) + .set_data_buffer("attr1", attr1) + .set_condition(qc); + REQUIRE(query.submit() == Query::Status::COMPLETE); + query.finalize(); + + // attr1 == "fred" in position 0 and position 4. + std::vector dim_expect = {1, 2, 3, 4, 5}; + std::vector attr1_expect = {0, INT32_MIN, INT32_MIN, INT32_MIN, 0}; + + REQUIRE(dim == dim_expect); + REQUIRE(attr1 == attr1_expect); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration Query - Negation", + "[enumeration][query][negation]") { + // Another basic query test, the only twist here is that we're checking + // that query condition negation works as expected. + create_array(); + + // Create a query condition attr1 == "fred" and then negate it so that + // we can verify rewriting a negated query works. + QueryCondition qc1(ctx_); + qc1.init("attr1", "fred", 4, TILEDB_EQ); + auto qc2 = qc1.negate(); + + // Execute a read query against the created array + std::vector dim(5); + std::vector attr1(5); + + auto array = Array(ctx_, uri_, TILEDB_READ); + Query query(ctx_, array); + query.add_range("dim", 1, 5) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("dim", dim) + .set_data_buffer("attr1", attr1) + .set_condition(qc2); + REQUIRE(query.submit() == Query::Status::COMPLETE); + query.finalize(); + + // attr1 == "fred" in positions 0 and 4 so those values should not match + // and return the default fill values. + std::vector dim_expect = {1, 2, 3, 4, 5}; + std::vector attr1_expect = {INT32_MIN, 1, 2, 1, INT32_MIN}; + + REQUIRE(dim == dim_expect); + REQUIRE(attr1 == attr1_expect); +} + +TEST_CASE_METHOD( + CPPEnumerationFx, + "CPP: Enumeration Query - Combination", + "[enumeration][query][combination]") { + // Same test as before except using multi-condition query condtions + create_array(); + + // Create query condition: attr1 == "fred" OR attr2 = 3.0f + QueryCondition qc1(ctx_); + qc1.init("attr1", "fred", 4, TILEDB_EQ); + + QueryCondition qc2(ctx_); + float val = 3.0f; + qc2.init("attr2", &val, sizeof(float), TILEDB_EQ); + + QueryCondition qc3 = qc1.combine(qc2, TILEDB_OR); + + // Execute a query with the query condition + std::vector dim(5); + std::vector attr1(5); + std::vector attr2(5); + + auto array = Array(ctx_, uri_, TILEDB_READ); + Query query(ctx_, array); + query.add_range("dim", 1, 5) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("dim", dim) + .set_data_buffer("attr1", attr1) + .set_data_buffer("attr2", attr2) + .set_condition(qc3); + REQUIRE(query.submit() == Query::Status::COMPLETE); + query.finalize(); + + // Check the results match the expected results. attr1 == "fred" in + // positions 0 and 4, while attr2 == 3.0f in position 2. + std::vector dim_expect = {1, 2, 3, 4, 5}; + std::vector attr1_expect = {0, INT32_MIN, 2, INT32_MIN, 0}; + std::vector attr2_expect = { + 1.0f, std::nanf(""), 3.0f, std::nanf(""), 5.0f}; + + REQUIRE(dim == dim_expect); + REQUIRE(attr1 == attr1_expect); + + // NaN != NaN so we have to loop over the attr2 results to special case + // `isnan(v1) == isnan(v2)` when NaN is encountered the expect vector. + for (size_t i = 0; i < attr2_expect.size(); i++) { + if (std::isnan(attr2_expect[i])) { + REQUIRE(std::isnan(attr2[i])); + } else { + REQUIRE(attr2[i] == attr2_expect[i]); + } + } +} + +CPPEnumerationFx::CPPEnumerationFx() + : uri_("enumeration_test_array") + , vfs_(ctx_) { + rm_array(); +} + +CPPEnumerationFx::~CPPEnumerationFx() { + rm_array(); +} + +void CPPEnumerationFx::create_array() { + // Create a simple array for testing. This ends up with just + // five elements in the array. dim is an int32_t dimension, + // attr1 is an enumeration with string values and int32_t + // attribute values. attr2 is a float attribute. + // + // The array data is summarized as below, however, pay attention to the + // fact that attr1 is storing integral index values instead of the + // raw string data. + // + // dim = {1, 2, 3, 4, 5} + // attr1 = {"fred", "wilma", "barney", "wilma", "fred"} + // attr2 = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f} + ArraySchema schema(ctx_, TILEDB_DENSE); + + auto dim = Dimension::create(ctx_, "dim", {{-100, 100}}); + auto dom = Domain(ctx_); + dom.add_dimension(dim); + schema.set_domain(dom); + + // The list of string values in the attr1 enumeration + std::vector values = {"fred", "wilma", "barney", "pebbles"}; + auto enmr = Enumeration::create(ctx_, "an_enumeration", values); + ArraySchemaExperimental::add_enumeration(ctx_, schema, enmr); + + auto attr1 = Attribute::create(ctx_, "attr1"); + AttributeExperimental::set_enumeration_name(ctx_, attr1, "an_enumeration"); + schema.add_attribute(attr1); + + auto attr2 = Attribute::create(ctx_, "attr2"); + schema.add_attribute(attr2); + + Array::create(uri_, schema); + + // Attribute data + std::vector attr1_values = {0, 1, 2, 1, 0}; + std::vector attr2_values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f}; + + Array array(ctx_, uri_, TILEDB_WRITE); + Subarray subarray(ctx_, array); + subarray.set_subarray({1, 5}); + Query query(ctx_, array); + query.set_subarray(subarray) + .set_layout(TILEDB_ROW_MAJOR) + .set_data_buffer("attr1", attr1_values) + .set_data_buffer("attr2", attr2_values); + CHECK_NOTHROW(query.submit()); + query.finalize(); + array.close(); +} + +void CPPEnumerationFx::rm_array() { + if (vfs_.is_dir(uri_)) { + vfs_.remove_dir(uri_); + } +} diff --git a/test/src/unit-cppapi-schema-evolution.cc b/test/src/unit-cppapi-schema-evolution.cc index 73d847725c6a..a291bfc488e5 100644 --- a/test/src/unit-cppapi-schema-evolution.cc +++ b/test/src/unit-cppapi-schema-evolution.cc @@ -31,8 +31,15 @@ */ #include +#include "tiledb/sm/array_schema/array_schema.h" +#include "tiledb/sm/array_schema/array_schema_evolution.h" +#include "tiledb/sm/array_schema/attribute.h" +#include "tiledb/sm/array_schema/dimension.h" +#include "tiledb/sm/array_schema/domain.h" #include "tiledb/sm/cpp_api/tiledb" #include "tiledb/sm/cpp_api/tiledb_experimental" +#include "tiledb/sm/enums/array_type.h" +#include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/misc/constants.h" #include @@ -776,3 +783,31 @@ TEST_CASE( vfs.remove_dir(array_uri); } } + +TEST_CASE( + "SchemaEvolution Error Handling Tests", + "[cppapi][schema][evolution][errors]") { + auto ase = make_shared(HERE()); + REQUIRE_THROWS(ase->evolve_schema(nullptr)); + REQUIRE_THROWS(ase->add_attribute(nullptr)); + + auto attr = make_shared( + HERE(), "attr", tiledb::sm::Datatype::STRING_ASCII); + ase->add_attribute(attr.get()); + REQUIRE_THROWS(ase->add_attribute(attr.get())); + + ase->set_timestamp_range(std::make_pair(1, 1)); + + auto schema = make_shared( + HERE(), tiledb::sm::ArrayType::SPARSE); + auto dim = make_shared( + HERE(), "dim1", tiledb::sm::Datatype::INT32); + int range[2] = {0, 1000}; + throw_if_not_ok(dim->set_domain(range)); + + auto dom = make_shared(HERE()); + throw_if_not_ok(dom->add_dimension(dim)); + throw_if_not_ok(schema->set_domain(dom)); + + CHECK_NOTHROW(ase->evolve_schema(schema)); +} diff --git a/test/src/unit-enum-helpers.cc b/test/src/unit-enum-helpers.cc new file mode 100644 index 000000000000..5bc06c8af9f5 --- /dev/null +++ b/test/src/unit-enum-helpers.cc @@ -0,0 +1,62 @@ +/** + * @file unit-enums.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests for enum conversions + */ + +#include + +#include "test/support/tdb_catch.h" +#include "tiledb/sm/enums/datatype.h" + +using namespace tiledb::sm; + +TEST_CASE("Test datatype_max_integral_value", "[enums][datatype][max-integral-value]") { + std::vector> test_data = { + {Datatype::BOOL, 255}, + {Datatype::INT8, 127}, + {Datatype::INT16, 32767}, + {Datatype::INT32, 2147483647}, + {Datatype::INT64, 9223372036854775807}, + + {Datatype::UINT8, 255}, + {Datatype::UINT16, 65535}, + {Datatype::UINT32, 4294967295}, + {Datatype::UINT64, 18446744073709551615u} + }; + + for (auto& [dtype, max_size] : test_data) { + std::cerr << "DTYPE: " << datatype_str(dtype) << std::endl; + REQUIRE(datatype_max_integral_value(dtype) == max_size); + } + + REQUIRE_THROWS(datatype_max_integral_value(Datatype::BLOB)); + REQUIRE_THROWS(datatype_max_integral_value(Datatype::FLOAT64)); + REQUIRE_THROWS(datatype_max_integral_value(Datatype::STRING_ASCII)); +} diff --git a/test/src/unit-enumerations.cc b/test/src/unit-enumerations.cc new file mode 100644 index 000000000000..b8a5dc417013 --- /dev/null +++ b/test/src/unit-enumerations.cc @@ -0,0 +1,1754 @@ +/** + * @file unit-enumerations.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the C++ API for enumeration related functions. + */ + +#include + +#include "test/support/tdb_catch.h" +#include "tiledb/sm/array/array.h" +#include "tiledb/sm/array/array_directory.h" +#include "tiledb/sm/array_schema/array_schema.h" +#include "tiledb/sm/array_schema/array_schema_evolution.h" +#include "tiledb/sm/array_schema/attribute.h" +#include "tiledb/sm/array_schema/dimension.h" +#include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/enumeration.h" +#include "tiledb/sm/buffer/buffer.h" +#include "tiledb/sm/buffer/buffer_list.h" +#include "tiledb/sm/config/config.h" +#include "tiledb/sm/enums/array_type.h" +#include "tiledb/sm/enums/encryption_type.h" +#include "tiledb/sm/enums/layout.h" +#include "tiledb/sm/query/query.h" +#include "tiledb/sm/query/query_condition.h" +#include "tiledb/sm/storage_manager/context.h" + +#ifdef TILEDB_SERIALIZATION +#include "tiledb/sm/enums/serialization_type.h" +#include "tiledb/sm/serialization/array_schema.h" +#include "tiledb/sm/serialization/array_schema_evolution.h" +#include "tiledb/sm/serialization/query.h" +#endif + +using namespace tiledb::sm; + +#define ENUM_NAME_PTR "an_enumeration" +const static std::string default_enmr_name = ENUM_NAME_PTR; + +struct EnumerationFx { + EnumerationFx(); + ~EnumerationFx(); + + template + shared_ptr create_enumeration( + const std::vector& values, + bool ordered = false, + Datatype type = static_cast(255), + std::string enmr_name = default_enmr_name); + + template + void check_enumeration( + shared_ptr enmr, + const std::string& name, + const std::vector& values, + Datatype data_type, + uint32_t cell_val_num, + bool ordered); + + template + void check_storage_serialization(const std::vector& values); + + template + void check_storage_deserialization(const std::vector& values); + + storage_size_t calculate_serialized_size(shared_ptr enmr); + WriterTile serialize_to_tile(shared_ptr enmr); + + template + std::vector as_vector(shared_ptr enmr); + + shared_ptr create_schema(); + + void create_array(); + shared_ptr get_array(QueryType type); + shared_ptr get_array_directory(); + shared_ptr get_array_schema_latest(); + + // Serialization helpers + ArraySchema ser_des_array_schema( + shared_ptr schema, + bool client_side, + SerializationType stype); + + shared_ptr ser_des_array_schema_evolution( + ArraySchemaEvolution* ase, bool client_side, SerializationType stype); + + void ser_des_query( + Query* q_in, Query* q_out, bool client_side, SerializationType stype); + + template + bool vec_cmp(std::vector v1, std::vector v2); + + void flatten_buffer_list(BufferList& blist, Buffer& buf); + + void rm_array(); + + URI uri_; + Config cfg_; + Context ctx_; + EncryptionKey enc_key_; +}; + +template +QueryCondition create_qc( + const char* field_name, T condition_value, const QueryConditionOp& op); + +/* ********************************* */ +/* Testing Enumeration */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "Basic Fixed Size Enumeration Creation", + "[enumeration][basic][fixed]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + check_enumeration( + enmr, default_enmr_name, values, Datatype::UINT32, 1, false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Basic Variable Size Enumeration Creation", + "[enumeration][basic][fixed]") { + std::vector values = {"foo", "bar", "baz", "bingo", "bango"}; + auto enmr = create_enumeration(values); + check_enumeration( + enmr, + default_enmr_name, + values, + Datatype::STRING_ASCII, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Basic Variable Size With Empty Value Enumeration Creation", + "[enumeration][basic][fixed]") { + std::vector values = {"foo", "bar", "", "bingo", "bango"}; + auto enmr = create_enumeration(values); + check_enumeration( + enmr, + default_enmr_name, + values, + Datatype::STRING_ASCII, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation with Ordered", + "[enumeration][basic][var-size][ordered]") { + std::vector values = {"foo", "bar", "baz", "bingo", "bango"}; + auto enmr = create_enumeration(values, true); + check_enumeration( + enmr, + default_enmr_name, + values, + Datatype::STRING_ASCII, + constants::var_num, + true); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation with Datatype", + "[enumeration][basic][var-size][custom-datatype]") { + std::vector values = {"foo", "bar", "baz", "bingo", "bango"}; + auto enmr = create_enumeration(values, false, Datatype::STRING_UTF8); + check_enumeration( + enmr, + default_enmr_name, + values, + Datatype::STRING_UTF8, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation with Multi-Cell Val Num", + "[enumeration][basic][fixed][multi-cell-val-num]") { + std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + auto enmr = Enumeration::create( + default_enmr_name, + Datatype::INT32, + 2, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 0); + check_enumeration(enmr, default_enmr_name, values, Datatype::INT32, 2, false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid empty name - std::string", + "[enumeration][error][invalid-name]") { + std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + REQUIRE_THROWS(Enumeration::create( + std::string(), + Datatype::INT32, + 2, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid empty name - char*", + "[enumeration][error][invalid-name]") { + std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + REQUIRE_THROWS(Enumeration::create( + "", + Datatype::INT32, + 2, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid name with slash", + "[enumeration][error][invalid-name]") { + std::vector values = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + REQUIRE_THROWS(Enumeration::create( + "an/enumeration", + Datatype::INT32, + 2, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid cell val num", + "[enumeration][error][invalid-cell-val-num]") { + std::vector values = {1, 2, 3}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 0, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - No data pointer", + "[enumeration][error][data-nullptr]") { + std::vector values = {1, 2, 3}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + nullptr, + values.size() * sizeof(int), + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Zero data size", + "[enumeration][error][data-zero-size]") { + std::vector values = {1, 2, 3}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + values.data(), + 0, + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - No offsets pointer", + "[enumeration][error][offsets-nullptr]") { + auto data = "foobarbazbam"; + std::vector offsets = {0, 3, 6, 9}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + data, + strlen(data), + nullptr, + offsets.size() * sizeof(uint64_t))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - No offsets size", + "[enumeration][error][offsets-zero-size]") { + auto data = "foobarbazbam"; + std::vector offsets = {0, 3, 6, 9}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + data, + strlen(data), + offsets.data(), + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Offsets not required, pointer provided", + "[enumeration][error][offsets-not-required]") { + std::vector values = {0, 1, 2, 3, 4}; + std::vector offsets = {0, 3, 6, 9}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + values.data(), + values.size() * sizeof(int), + offsets.data(), + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Offsets not required, size provided", + "[enumeration][error][offsets-not-required]") { + std::vector values = {0, 1, 2, 3, 4}; + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + values.data(), + values.size() * sizeof(int), + nullptr, + 100)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid offsests size provided", + "[enumeration][error][offsets-invalid-size]") { + auto data = "foobarbazbam"; + std::vector offsets = {0, 3, 6, 9}; + // Passing 3 for the offsets size is incorrect because the offsets size has + // to be a multiple of `sizeof(uint64_t)` + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + data, + strlen(data), + offsets.data(), + 3)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Offsets to data beyond provided data size", + "[enumeration][error][invalid-offset-data]") { + auto data = "foobarbazbam"; + std::vector offsets = {0, 3, 6, 100}; + // The last offset is larger than data_size + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::STRING_ASCII, + constants::var_num, + false, + data, + strlen(data), + offsets.data(), + offsets.size() * sizeof(uint64_t))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Invalid data size", + "[enumeration][error][invalid-data-size]") { + std::vector values = {1, 2, 3, 4, 5}; + // Passing 3 for the data size is invalid as its not a multiple of + // sizeof(int) + REQUIRE_THROWS(Enumeration::create( + default_enmr_name, + Datatype::INT32, + 1, + false, + values.data(), + 3, + nullptr, + 0)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Repeated fixed sized values", + "[enumeration][error][repeated-values]") { + std::vector values = {1, 2, 3, 3, 4, 5}; + REQUIRE_THROWS(create_enumeration(values)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Repeated variable sized values", + "[enumeration][error][repeated-values]") { + std::vector values = {"foo", "bar", "bang", "bar"}; + REQUIRE_THROWS(create_enumeration(values)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Creation Error - Repeated empty variable sized values", + "[enumeration][error][repeated-values]") { + std::vector values = {"foo", "", "bang", ""}; + REQUIRE_THROWS(create_enumeration(values)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Serialization - Fixed Size", + "[enumeration][serialization][fixed-size]") { + std::vector values = {1, 2, 3, 4, 5}; + check_storage_serialization(values); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Serialization - Variable Size", + "[enumeration][serialization][var-size]") { + std::vector values = {"foo", "bar", "baz", "bam", "cap"}; + check_storage_serialization(values); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Deserialization - Fixed Size", + "[enumeration][deserialization][fixed-size]") { + std::vector values = {1, 2, 3, 4, 5}; + check_storage_deserialization(values); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Deserialization - Variable Size", + "[enumeration][deserialization][var-size]") { + std::vector values = {"foo", "bar", "baz", "bam", "cap"}; + check_storage_deserialization(values); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration Deserialization Error - Invalid version", + "[enumeration][deserialization][error][invalid-version]") { + std::vector values = {"foo", "bar", "baz", "bam", "cap"}; + auto enmr = create_enumeration(values); + auto tile = serialize_to_tile(enmr); + + REQUIRE(tile.size() > 4); + auto data = tile.data(); + memset(data, 1, 4); + + Deserializer deserializer(tile.data(), tile.size()); + REQUIRE_THROWS(Enumeration::deserialize(deserializer)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration index_of - Fixed Size", + "[enumeration][index-of][fixed-size]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + + for (uint64_t i = 0; i < values.size(); i++) { + int tmp = values[i]; + UntypedDatumView udv(&tmp, sizeof(int)); + REQUIRE(enmr->index_of(udv) == i); + } +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration index_of - Fixed Size Missing", + "[enumeration][index-of][fixed-size]") { + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + + int zero = 0; + UntypedDatumView udv_zero(&zero, sizeof(int)); + REQUIRE(enmr->index_of(udv_zero) == constants::enumeration_missing_value); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration index_of - Variable Size", + "[enumeration][index-of][var-size]") { + std::vector values = {"foo", "bar", "baz", "bang", "ohai"}; + auto enmr = create_enumeration(values); + + for (uint64_t i = 0; i < values.size(); i++) { + UntypedDatumView udv(values[i].data(), values[i].size()); + REQUIRE(enmr->index_of(udv) == i); + } +} + +TEST_CASE_METHOD( + EnumerationFx, + "Enumeration index_of - Variable Size Missing", + "[enumeration][index-of][var-size]") { + std::vector values = {"foo", "bar", "baz", "bang", "ohai"}; + auto enmr = create_enumeration(values); + + UntypedDatumView udv_empty("", 0); + REQUIRE(enmr->index_of(udv_empty) == constants::enumeration_missing_value); +} + +/* ********************************* */ +/* Testing Attribute */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "Attribute set_enumeration_name - Error - Empty Name", + "[enumeration][attribute][error]") { + auto attr = make_shared(HERE(), "foo", Datatype::INT8); + REQUIRE_THROWS(attr->set_enumeration_name("")); +} + +/* ********************************* */ +/* Testing Array */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Get Enumeration", + "[enumeration][array][get-enumeration]") { + create_array(); + auto array = get_array(QueryType::READ); + auto enmr = array->get_enumeration("test_enmr"); + REQUIRE(enmr != nullptr); + + std::vector values = {"ant", "bat", "cat", "dog", "emu"}; + check_enumeration( + enmr, + "test_enmr", + values, + Datatype::STRING_ASCII, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Get Enumeration Repeated", + "[enumeration][array][get-enumeration]") { + create_array(); + auto array = get_array(QueryType::READ); + auto enmr1 = array->get_enumeration("test_enmr"); + auto enmr2 = array->get_enumeration("test_enmr"); + REQUIRE(enmr1 == enmr2); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Get Non-existEnumeration No Attribute", + "[enumeration][array][no-attribute]") { + create_array(); + auto array = get_array(QueryType::READ); + REQUIRE_THROWS(array->get_enumeration("foo")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Array - Get Enumeration Error - Not Open", + "[enumeration][array][error][not-open]") { + auto array = make_shared(HERE(), uri_, ctx_.storage_manager()); + REQUIRE_THROWS(array->get_enumeration("foo")); +} + +/* ********************************* */ +/* Testing ArrayDirectory */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "ArrayDirectory - Load Enumeration", + "[enumeration][array-directory][load-enumeration]") { + create_array(); + + auto schema = get_array_schema_latest(); + auto ad = get_array_directory(); + auto enmr_name = schema->attribute("attr1")->get_enumeration_name(); + REQUIRE(enmr_name.has_value()); + + auto enmr = ad->load_enumeration(schema, enmr_name.value(), enc_key_); + std::vector values = {"ant", "bat", "cat", "dog", "emu"}; + check_enumeration( + enmr, + "test_enmr", + values, + Datatype::STRING_ASCII, + constants::var_num, + false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArrayDirectory - Load Enumeration Error", + "[enumeration][array-directory][error]") { + create_array(); + + auto schema = get_array_schema_latest(); + auto ad = get_array_directory(); + + // Check that this function throws an exception when attempting to load + // an unknown enumeration + REQUIRE_THROWS(ad->load_enumeration(schema, "not_an_enumeration", enc_key_)); +} + +/* ********************************* */ +/* Testing ArraySchema */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Add Enumeration - Enumeration nullptr Error", + "[enumeration][array-schema][error]") { + auto schema = make_shared(HERE()); + REQUIRE_THROWS(schema->add_enumeration(nullptr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Add Basic Enumeration", + "[enumeration][array-schema][basic]") { + auto schema = make_shared(HERE()); + + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + schema->add_enumeration(enmr); + + auto attr = make_shared(HERE(), "foo", Datatype::INT8); + attr->set_enumeration_name(enmr->name()); + CHECK_NOTHROW(schema->add_attribute(attr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Get Enumeration", + "[enumeration][array-schema][get]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + + std::vector values = {1, 2, 3, 4, 5}; + auto enmr1 = create_enumeration(values); + schema->add_enumeration(enmr1); + + auto enmr2 = schema->get_enumeration(default_enmr_name); + check_enumeration(enmr2, enmr1->name(), values, Datatype::INT32, 1, false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Get Missing Enumeration Error", + "[enumeration][array-schema][error]") { + auto schema = make_shared(HERE(), ArrayType::SPARSE); + REQUIRE_THROWS(schema->get_enumeration("not_an_enumeration")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Add Enumeration with Existing Enumeration of same Name", + "[enumeration][array-schema][eror]") { + auto schema = make_shared(HERE(), ArrayType::SPARSE); + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + + schema->add_enumeration(enmr); + REQUIRE_THROWS(schema->add_enumeration(enmr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Add Attribute with Missing Enumeration Error", + "[enumeration][array-schema][eror]") { + auto schema = make_shared(HERE(), ArrayType::SPARSE); + auto attr = make_shared(HERE(), "an_attr", Datatype::INT32); + attr->set_enumeration_name("not_an_enumeration"); + REQUIRE(!schema->add_attribute(attr).ok()); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Get All Enumeration Names Empty", + "[enumeration][array-schema][get-all][empty]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + auto enmr_names = schema->get_enumeration_names(); + REQUIRE(enmr_names.size() == 0); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Get All Enumeration Names", + "[enumeration][array-schema][get-all]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + + std::vector values = {1.0, 1.1, 1.2, 1.3, 1.4}; + auto enmr1 = create_enumeration(values); + schema->add_enumeration(enmr1); + + auto enmr_names = schema->get_enumeration_names(); + REQUIRE(enmr_names.size() == 1); + + auto enmr2 = schema->get_enumeration(enmr_names[0]); + REQUIRE(enmr2 == enmr1); + check_enumeration( + enmr2, default_enmr_name, values, Datatype::FLOAT32, 1, false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Attribute with Invalid Datatype", + "[enumeration][array-schema][error][bad-attr-datatype]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + schema->add_enumeration(enmr); + + auto attr = make_shared(HERE(), "ohai", Datatype::FLOAT32); + attr->set_enumeration_name(default_enmr_name); + REQUIRE(schema->add_attribute(attr).ok() == false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Attribute with Invalid Cell Val Num", + "[enumeration][array-schema][error][bad-attr-cell-val-num]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values); + schema->add_enumeration(enmr); + + auto attr = make_shared(HERE(), "ohai", Datatype::INT32, 2, DataOrder::UNORDERED_DATA); + attr->set_enumeration_name(default_enmr_name); + REQUIRE(schema->add_attribute(attr).ok() == false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Store nullptr Enumeration Error", + "[enumeration][array-schema][error][store-nullptr-enumeration]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + REQUIRE_THROWS(schema->store_enumeration(nullptr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Store Enumeration Error", + "[enumeration][array-schema][error][store-unknown-enumeration]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = + create_enumeration(values, false, Datatype::INT32, "unknown_enmr"); + REQUIRE_THROWS(schema->store_enumeration(enmr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Store Enumeration Error - Already Loaded", + "[enumeration][array-schema][error][store-loaded-enumeration]") { + auto schema = make_shared(HERE(), ArrayType::DENSE); + + std::vector values = {0, 1, 2, 100000000}; + auto enmr = create_enumeration(values); + schema->add_enumeration(enmr); + + // This is an error because there's already an enumeration stored in the + // ArraySchema::enumeration_map_. When deserializing schemas from disk the + // entries in ArraySchema::enumeration_map_ are (std::string, nullptr). + REQUIRE_THROWS(schema->store_enumeration(enmr)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Attribute Get Enumeration Name From Attribute", + "[enumeration][array-schema][has-enumeration]") { + auto schema = make_shared(HERE(), ArrayType::SPARSE); + + std::vector values = {"a", "spot", "of", "tea", "perhaps?"}; + auto enmr = create_enumeration(values); + schema->add_enumeration(enmr); + + REQUIRE(schema->get_enumeration(enmr->name()) == enmr); + REQUIRE(schema->get_enumeration_path_name(enmr->name()) == enmr->path_name()); + + auto attr = make_shared(HERE(), "ohai", Datatype::INT64); + attr->set_enumeration_name(default_enmr_name); + throw_if_not_ok(schema->add_attribute(attr)); + + REQUIRE(schema->attribute("ohai")->get_enumeration_name().has_value()); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Schema Copy Constructor", + "[enumeration][array-schema][copy-ctor]") { + auto schema = create_schema(); + + // Check that the schema is valid and that we can copy it using the + // copy constructor. + CHECK_NOTHROW(schema->check()); + CHECK_NOTHROW(make_shared(HERE(), *(schema.get()))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchema - Mismatched path name error", + "[enumeration][array-schema][error]") { + create_array(); + auto schema = get_array_schema_latest(); + + // Creating a new Enumeration will give it a different UUID path name. + std::vector values = {1, 2, 3, 4, 5}; + auto enmr = create_enumeration(values, false, Datatype::INT32, "test_enmr"); + + // Storing an enumeration with a known name but different path is an error + REQUIRE_THROWS(schema->store_enumeration(enmr)); +} + +/* ********************************* */ +/* Testing ArraySchemaEvolution */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Simple No Enumeration", + "[enumeration][array-schema-evolution][simple]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + auto ase = make_shared(HERE()); + auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); + ase->add_attribute(attr3.get()); + CHECK_NOTHROW(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Simple With Enumeration", + "[enumeration][array-schema-evolution][simple]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr = create_enumeration(values); + ase->add_enumeration(enmr); + + auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); + attr3->set_enumeration_name(default_enmr_name); + ase->add_attribute(attr3.get()); + + ase->drop_attribute("attr2"); + + CHECK_NOTHROW(ase->evolve_schema(orig_schema)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Drop Attribute After Add", + "[enumeration][array-schema-evolution][drop-add]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr = create_enumeration(values); + ase->add_enumeration(enmr); + + auto attr3 = make_shared(HERE(), "attr3", Datatype::UINT32); + attr3->set_enumeration_name(default_enmr_name); + ase->add_attribute(attr3.get()); + + CHECK_NOTHROW(ase->drop_attribute("attr3")); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration Attribute Names to Add", + "[enumeration][array-schema-evolution][simple]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr = create_enumeration(values); + ase->add_enumeration(enmr); + + auto enmr_names = ase->enumeration_names_to_add(); + REQUIRE(enmr_names.size() == 1); + REQUIRE(enmr_names[0] == enmr->name()); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to Add", + "[enumeration][array-schema-evolution][enmr-to-add]") { + create_array(); + auto orig_schema = get_array_schema_latest(); + auto ase = make_shared(HERE()); + + std::vector values{0, 1, 2, 3, 4, 1000}; + auto enmr1 = create_enumeration(values); + ase->add_enumeration(enmr1); + + auto enmr2 = ase->enumeration_to_add(enmr1->name()); + REQUIRE(enmr2 == enmr1); +} + +TEST_CASE_METHOD( + EnumerationFx, + "ArraySchemaEvolution - Enumeration to Add - Missing Name", + "[enumeration][array-schema-evolution][missing-name]") { + create_array(); + auto ase = make_shared(HERE()); + REQUIRE(ase->enumeration_to_add("foo") == nullptr); +} + +/* ********************************* */ +/* Testing QueryCondition */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite Enumeration Value", + "[enumeration][query-condition][rewrite-enumeration-value]") { + create_array(); + auto array = get_array(QueryType::READ); + auto schema = array->array_schema_latest_ptr(); + + // This is normally invoked by the query class when not being tested. It's + // required here so that the enumeration's data is loaded. + array->get_enumeration("test_enmr"); + + // Assert that the enumerations were loaded + auto enmr_names = schema->get_loaded_enumeration_names(); + REQUIRE(enmr_names.size() == 1); + REQUIRE(enmr_names[0] == "test_enmr"); + + // Create two copies of the same query condition for assertions + auto qc1 = create_qc("attr1", std::string("cat"), QueryConditionOp::EQ); + auto qc2 = qc1; + + qc2.rewrite_enumeration_conditions(*(schema.get())); + + // Assert that the rewritten tree matches in the right places while also + // different to verify the assertion of having been rewritten. + auto& tree1 = qc1.ast(); + auto& tree2 = qc2.ast(); + + REQUIRE(tree1->is_expr() == false); + REQUIRE(tree1->get_field_name() == "attr1"); + + REQUIRE(tree2->is_expr() == tree1->is_expr()); + REQUIRE(tree2->get_field_name() == tree1->get_field_name()); + + auto udv1 = tree1->get_condition_value_view(); + auto udv2 = tree2->get_condition_value_view(); + REQUIRE(udv2.size() != udv1.size()); + REQUIRE(udv2.content() != udv1.content()); + REQUIRE(udv2.value_as() == 2); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Skip enumeration rewrite", + "[enumeration][query-condition][skip-rewrite]") { + create_array(); + auto schema = get_array_schema_latest(); + + // Almost exactly the same test as before, except this time we call + // `set_use_enumeration(false)` before rewriting and assert that the + // resulting rewritten query tree matches exactly since no enumeration + // rewriting has taken place. + + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + qc1.set_use_enumeration(false); + auto qc2 = qc1; + + qc2.rewrite_enumeration_conditions(*(schema.get())); + + auto& tree1 = qc1.ast(); + auto& tree2 = qc2.ast(); + + // Check that both trees match exactly + + REQUIRE(tree1->is_expr() == false); + REQUIRE(tree1->get_field_name() == "attr1"); + + REQUIRE(tree2->is_expr() == tree1->is_expr()); + REQUIRE(tree2->get_field_name() == tree1->get_field_name()); + + auto udv1 = tree1->get_condition_value_view(); + auto udv2 = tree1->get_condition_value_view(); + REQUIRE(udv2.size() == udv1.size()); + REQUIRE(udv2.content() == udv1.content()); + REQUIRE(udv2.value_as() == 2); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite - No failure on unknown attribute", + "[enumeration][query-condition]") { + create_array(); + auto schema = get_array_schema_latest(); + + auto qc1 = create_qc("not_an_attr", (int)2, QueryConditionOp::EQ); + qc1.rewrite_enumeration_conditions(*(schema.get())); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite - Enumeration Not Loaded", + "[enumeration][query-condition][error]") { + create_array(); + auto schema = get_array_schema_latest(); + + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + REQUIRE_THROWS(qc1.rewrite_enumeration_conditions(*(schema.get()))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite - Inequality on Unordered Enumeration", + "[enumeration][query-condition][error]") { + // If an enumeration isn't marked as ordered, then it should throw an error + // when attempting to use an inequality operator on the attribute. + create_array(); + auto array = get_array(QueryType::READ); + auto schema = get_array_schema_latest(); + + // This is normally invoked by the query class when not being tested. + array->get_enumeration("test_enmr"); + + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::LT); + REQUIRE_THROWS(qc1.rewrite_enumeration_conditions(*(schema.get()))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - Rewrite Empty QC - Coverage", + "[enumeration][query-condition][coverage]") { + // Check that qc.rewrite_enumeration_conditions doesn't throw on an empty QC + create_array(); + auto schema = get_array_schema_latest(); + + QueryCondition qc; + CHECK_NOTHROW(qc.rewrite_enumeration_conditions(*(schema.get()))); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - use_enumeration - Check Accessor for Coverage", + "[enumeration][query-condition][coverage]") { + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::LT); + auto& node = qc1.ast(); + + REQUIRE(node->use_enumeration() == true); + qc1.set_use_enumeration(false); + REQUIRE(node->use_enumeration() == false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - set_use_enumeration - Affects Children", + "[enumeration][query-condition][set_use_enumeration]") { + // Check that set_use_enumeration is applied to the entire tree if applied + // to an expression node. + + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + auto qc2 = create_qc("attr2", 3.0f, QueryConditionOp::LT); + QueryCondition qc3; + throw_if_not_ok(qc1.combine(qc2, QueryConditionCombinationOp::AND, &qc3)); + + auto& tree1 = qc3.ast(); + for (auto& child : tree1->get_children()) { + REQUIRE(child->use_enumeration() == true); + } + + qc3.set_use_enumeration(false); + + auto& tree2 = qc3.ast(); + for (auto& child : tree2->get_children()) { + REQUIRE(child->use_enumeration() == false); + } +} + +TEST_CASE_METHOD( + EnumerationFx, + "QueryCondition - use_enumeration - Error on ASTNodeExpr", + "[enumeration][query-condition][error]") { + // Check that an ASTNodeExpr throws correctly when calling + // `use_enumeration()`. + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + auto qc2 = create_qc("attr2", 3.0f, QueryConditionOp::LT); + QueryCondition qc3; + throw_if_not_ok(qc1.combine(qc2, QueryConditionCombinationOp::AND, &qc3)); + auto& node = qc3.ast(); + + REQUIRE_THROWS(node->use_enumeration()); +} + +/* ********************************* */ +/* Testing Cap'N Proto Serialization */ +/* ********************************* */ + +#ifdef TILEDB_SERIALIZATION + +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic New ArraySchema Serialization", + "[enumeration][capnp][basic][initialized-in-ram") { + auto client_side = GENERATE(true, false); + auto ser_type = GENERATE(SerializationType::CAPNP, SerializationType::JSON); + + auto schema1 = create_schema(); + auto schema2 = ser_des_array_schema(schema1, client_side, ser_type); + + auto all_names1 = schema1->get_enumeration_names(); + auto all_names2 = schema2.get_enumeration_names(); + REQUIRE(vec_cmp(all_names1, all_names2)); + + auto loaded_names1 = schema1->get_loaded_enumeration_names(); + auto loaded_names2 = schema2.get_loaded_enumeration_names(); + REQUIRE(vec_cmp(loaded_names1, loaded_names2)); + + // This is a new schema in RAM, so the loaded names should be the same + // as all names. + REQUIRE(vec_cmp(all_names1, loaded_names1)); + REQUIRE(vec_cmp(all_names2, loaded_names2)); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic Exisitng ArraySchema Serialization", + "[enumeration][capnp][basic][deserialized-from-disk]") { + create_array(); + + auto client_side = GENERATE(true, false); + auto ser_type = GENERATE(SerializationType::CAPNP, SerializationType::JSON); + + auto schema1 = get_array_schema_latest(); + auto schema2 = ser_des_array_schema(schema1, client_side, ser_type); + + auto all_names1 = schema1->get_enumeration_names(); + auto all_names2 = schema2.get_enumeration_names(); + REQUIRE(vec_cmp(all_names1, all_names2)); + + // This schema was deserialized from disk without any enumerations loaded + // so both of these should be empty. + auto loaded_names1 = schema1->get_loaded_enumeration_names(); + auto loaded_names2 = schema2.get_loaded_enumeration_names(); + + REQUIRE(loaded_names1.empty()); + REQUIRE(loaded_names2.empty()); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic ArraySchemaEvolution Serialization", + "[enumeration][capnp][basic][array-schema-evolution]") { + auto client_side = GENERATE(true, false); + auto ser_type = GENERATE(SerializationType::CAPNP, SerializationType::JSON); + + std::vector values1 = {1, 2, 3, 4, 5}; + auto enmr1 = create_enumeration(values1, false, Datatype::INT32, "enmr1"); + + std::vector values2 = {1.0, 2.0, 3.0, 4.0, 5.0}; + auto enmr2 = create_enumeration(values2, true, Datatype::FLOAT64, "enmr2"); + + auto attr = make_shared(HERE(), "ohai", Datatype::INT64); + attr->set_enumeration_name("enmr2"); + + ArraySchemaEvolution ase1; + ase1.add_attribute(attr.get()); + ase1.add_enumeration(enmr1); + ase1.add_enumeration(enmr2); + ase1.drop_attribute("some_attr"); + + auto ase2 = ser_des_array_schema_evolution(&ase1, client_side, ser_type); + + auto enmrs_to_add1 = ase1.enumeration_names_to_add(); + auto enmrs_to_add2 = ase2->enumeration_names_to_add(); + REQUIRE(vec_cmp(enmrs_to_add1, enmrs_to_add2)); + + for (auto& name : enmrs_to_add1) { + REQUIRE(ase1.enumeration_to_add(name) != nullptr); + REQUIRE(ase2->enumeration_to_add(name) != nullptr); + REQUIRE(ase1.enumeration_to_add(name) != ase2->enumeration_to_add(name)); + } +} + +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic Backwards Compatible Query Serialization", + "[enumeration][capnp][basic][query-condition-old]") { + auto client_side = GENERATE(true, false); + + // Query does not support serialization to JSON + auto ser_type = SerializationType::CAPNP; + + create_array(); + auto array = get_array(QueryType::READ); + + // This is normally invoked by the query class when not being tested. + array->get_enumeration("test_enmr"); + + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + qc1.set_use_enumeration(false); + + Query q1(ctx_.storage_manager(), array); + throw_if_not_ok(q1.set_condition(qc1)); + + Query q2(ctx_.storage_manager(), array); + ser_des_query(&q1, &q2, client_side, ser_type); + + auto qc2 = q2.condition(); + REQUIRE(qc2.has_value()); + + auto& node = qc2.value().ast(); + REQUIRE(node->use_enumeration() == false); +} + +TEST_CASE_METHOD( + EnumerationFx, + "Cap'N Proto - Basic New Query Serialization", + "[enumeration][capnp][basic][query-condition-new]") { + auto client_side = GENERATE(true, false); + + // Query does not support serialization to JSON + auto ser_type = SerializationType::CAPNP; + + create_array(); + auto array = get_array(QueryType::READ); + + // This is normally invoked by the query class when not being tested. + array->get_enumeration("test_enmr"); + + auto qc1 = create_qc("attr1", (int)2, QueryConditionOp::EQ); + qc1.set_use_enumeration(false); + + auto qc2 = create_qc("attr2", std::string("foo"), QueryConditionOp::NE); + QueryCondition qc3; + + throw_if_not_ok(qc1.combine(qc2, QueryConditionCombinationOp::OR, &qc3)); + + Query q1(ctx_.storage_manager(), array); + throw_if_not_ok(q1.set_condition(qc3)); + + Query q2(ctx_.storage_manager(), array); + ser_des_query(&q1, &q2, client_side, ser_type); + + auto qc4 = q2.condition(); + REQUIRE(qc4.has_value()); + + auto& node1 = qc4.value().ast()->get_children()[0]; + auto& node2 = qc4.value().ast()->get_children()[1]; + REQUIRE(node1->use_enumeration() == false); + REQUIRE(node2->use_enumeration() == true); +} + +#endif // ifdef TILEDB_SERIALIZATIONs + +/* ********************************* */ +/* VERIFY SUPPORT CODE */ +/* ********************************* */ + +TEST_CASE_METHOD( + EnumerationFx, + "Check EnumerationFx::vec_cmp", + "[enumeration][test-code][verify]") { + std::vector v1 = {1, 2, 3, 4, 5}; + std::vector v2 = {5, 3, 4, 2, 1}; + + REQUIRE(vec_cmp(v1, v2)); + + std::vector v3 = {}; + REQUIRE(!vec_cmp(v1, v3)); + + std::vector v4 = {1, 2}; + REQUIRE(!vec_cmp(v1, v4)); + + std::vector v5 = {3, 4, 5, 6, 7}; + REQUIRE(!vec_cmp(v1, v5)); +} + +/* ********************************* */ +/* TEST SUPPORT CODE */ +/* ********************************* */ + +struct TypeParams { + TypeParams(Datatype type, uint32_t cell_val_num) + : type_(type) + , cell_val_num_(cell_val_num) { + } + + template + static TypeParams get(const std::vector>&) { + return TypeParams(Datatype::STRING_ASCII, constants::var_num); + } + + static TypeParams get(const std::vector&) { + return TypeParams(Datatype::INT32, 1); + } + + static TypeParams get(const std::vector&) { + return TypeParams(Datatype::UINT32, 1); + } + + static TypeParams get(const std::vector&) { + return TypeParams(Datatype::UINT64, 1); + } + + static TypeParams get(const std::vector&) { + return TypeParams(Datatype::FLOAT32, 1); + } + + static TypeParams get(const std::vector&) { + return TypeParams(Datatype::FLOAT64, 1); + } + + Datatype type_; + uint32_t cell_val_num_; +}; + +EnumerationFx::EnumerationFx() + : uri_("enumeration_test_array") + , ctx_(cfg_) { + rm_array(); + throw_if_not_ok(enc_key_.set_key(EncryptionType::NO_ENCRYPTION, nullptr, 0)); +} + +EnumerationFx::~EnumerationFx() { + rm_array(); +} + +template +shared_ptr EnumerationFx::create_enumeration( + const std::vector& values, + bool ordered, + Datatype type, + std::string name) { + TypeParams tp = TypeParams::get(values); + + if (type != static_cast(255)) { + tp.type_ = type; + } + + if constexpr (std::is_pod_v) { + return Enumeration::create( + name, + tp.type_, + tp.cell_val_num_, + ordered, + values.data(), + values.size() * sizeof(T), + nullptr, + 0); + } else { + uint64_t total_size = 0; + for (auto v : values) { + total_size += v.size(); + } + + uint8_t data[total_size]; + std::vector offsets; + offsets.reserve(values.size()); + uint64_t curr_offset = 0; + + for (auto v : values) { + std::memcpy(data + curr_offset, v.data(), v.size()); + offsets.push_back(curr_offset); + curr_offset += v.size(); + } + + return Enumeration::create( + name, + tp.type_, + tp.cell_val_num_, + ordered, + data, + total_size, + offsets.data(), + offsets.size() * sizeof(uint64_t)); + } +} + +template +void EnumerationFx::check_enumeration( + shared_ptr enmr, + const std::string& name, + const std::vector& values, + Datatype data_type, + uint32_t cell_val_num, + bool ordered) { + REQUIRE(enmr->name() == name); + REQUIRE(!enmr->path_name().empty()); + REQUIRE(enmr->type() == data_type); + REQUIRE(enmr->cell_val_num() == cell_val_num); + REQUIRE(enmr->ordered() == ordered); + + std::vector data = as_vector(enmr); + REQUIRE(data == values); +} + +template +void EnumerationFx::check_storage_serialization(const std::vector& values) { + auto enmr = create_enumeration(values); + auto tile = serialize_to_tile(enmr); + REQUIRE(tile.size() == calculate_serialized_size(enmr)); +} + +template +void EnumerationFx::check_storage_deserialization( + const std::vector& values) { + auto enmr = create_enumeration(values); + auto tile = serialize_to_tile(enmr); + + Deserializer deserializer(tile.data(), tile.size()); + auto deserialized = Enumeration::deserialize(deserializer); + + REQUIRE(deserialized->name() == enmr->name()); + REQUIRE(deserialized->path_name().empty() == false); + REQUIRE(deserialized->type() == enmr->type()); + REQUIRE(deserialized->cell_val_num() == enmr->cell_val_num()); + REQUIRE(deserialized->ordered() == enmr->ordered()); + REQUIRE(deserialized->cell_size() == enmr->cell_size()); + REQUIRE(deserialized->var_size() == enmr->var_size()); + + auto orig_dspan = enmr->data(); + auto des_dspan = deserialized->data(); + REQUIRE(des_dspan.size() == orig_dspan.size()); + REQUIRE(memcmp(des_dspan.data(), orig_dspan.data(), orig_dspan.size()) == 0); + + if (enmr->var_size()) { + auto orig_ospan = enmr->offsets(); + auto des_ospan = deserialized->offsets(); + REQUIRE(orig_ospan.size() == des_ospan.size()); + REQUIRE( + memcmp(des_ospan.data(), orig_ospan.data(), orig_ospan.size()) == 0); + } +} + +storage_size_t EnumerationFx::calculate_serialized_size( + shared_ptr enmr) { + // Size is the sum of the following sizes: + storage_size_t num_bytes = 0; + + // uint32_t - version + num_bytes += sizeof(uint32_t); + + // uint32_t - name length + num_bytes += sizeof(uint32_t); + + // name.size() bytes + num_bytes += enmr->name().size(); + + // uint32_t - path_name length; + num_bytes += sizeof(uint32_t); + + // path_name.size() bytes + num_bytes += enmr->path_name().size(); + + // uint8_t - data type + num_bytes += sizeof(uint8_t); + + // uint32_t - cell_val_num + num_bytes += sizeof(uint32_t); + + // bool - ordered + num_bytes += sizeof(bool); + + // uint64_t - data.size() + // data.size() bytes + auto dspan = enmr->data(); + num_bytes += sizeof(uint64_t); + num_bytes += dspan.size(); + + // if var_sized: + if (enmr->var_size()) { + auto ospan = enmr->offsets(); + num_bytes += sizeof(uint64_t); + num_bytes += ospan.size(); + } + + return num_bytes; +} + +WriterTile EnumerationFx::serialize_to_tile( + shared_ptr enmr) { + SizeComputationSerializer size_serializer; + enmr->serialize(size_serializer); + + WriterTile tile{WriterTile::from_generic(size_serializer.size())}; + Serializer serializer(tile.data(), tile.size()); + enmr->serialize(serializer); + + return tile; +} + +template +std::vector EnumerationFx::as_vector(shared_ptr enmr) { + std::vector ret; + + if constexpr (std::is_pod_v) { + auto dspan = enmr->data(); + + const T* elems = reinterpret_cast(dspan.data()); + size_t count = dspan.size() / sizeof(T); + + ret.reserve(count); + for (size_t i = 0; i < count; i++) { + ret.push_back(elems[i]); + } + } else { + auto dspan = enmr->data(); + auto ospan = enmr->offsets(); + + auto str_data = reinterpret_cast(dspan.data()); + auto elems = reinterpret_cast(ospan.data()); + size_t count = ospan.size() / sizeof(uint64_t); + + ret.reserve(count); + for (size_t i = 0; i < count; i++) { + uint64_t len; + if (i + 1 < count) { + len = elems[i + 1] - elems[i]; + } else { + len = dspan.size() - elems[i]; + } + ret.emplace_back(str_data + elems[i], len); + } + } + + return ret; +} + +shared_ptr EnumerationFx::create_schema() { + // Create a schema to serialize + auto schema = make_shared(HERE(), ArrayType::SPARSE); + + auto dim = make_shared(HERE(), "dim1", Datatype::INT32); + int range[2] = {0, 1000}; + throw_if_not_ok(dim->set_domain(range)); + + auto dom = make_shared(HERE()); + throw_if_not_ok(dom->add_dimension(dim)); + throw_if_not_ok(schema->set_domain(dom)); + + std::vector values = {"ant", "bat", "cat", "dog", "emu"}; + auto enmr = + create_enumeration(values, false, Datatype::STRING_ASCII, "test_enmr"); + schema->add_enumeration(enmr); + + auto attr1 = make_shared(HERE(), "attr1", Datatype::INT32); + attr1->set_enumeration_name("test_enmr"); + throw_if_not_ok(schema->add_attribute(attr1)); + + auto attr2 = make_shared(HERE(), "attr2", Datatype::STRING_ASCII); + throw_if_not_ok(schema->add_attribute(attr2)); + + return schema; +} + +void EnumerationFx::create_array() { + auto schema = create_schema(); + throw_if_not_ok(ctx_.storage_manager()->array_create(uri_, schema, enc_key_)); +} + +shared_ptr EnumerationFx::get_array(QueryType type) { + auto array = make_shared(HERE(), uri_, ctx_.storage_manager()); + throw_if_not_ok(array->open(type, EncryptionType::NO_ENCRYPTION, nullptr, 0)); + return array; +} + +shared_ptr EnumerationFx::get_array_directory() { + return make_shared( + HERE(), ctx_.resources(), uri_, 0, UINT64_MAX, ArrayDirectoryMode::READ); +} + +shared_ptr EnumerationFx::get_array_schema_latest() { + return get_array_directory()->load_array_schema_latest(enc_key_); +} + +#ifdef TILEDB_SERIALIZATION + +ArraySchema EnumerationFx::ser_des_array_schema( + shared_ptr schema, + bool client_side, + SerializationType stype) { + Buffer buf; + throw_if_not_ok(serialization::array_schema_serialize( + *(schema.get()), stype, &buf, client_side)); + return serialization::array_schema_deserialize(stype, buf); +} + +shared_ptr EnumerationFx::ser_des_array_schema_evolution( + ArraySchemaEvolution* ase, bool client_side, SerializationType stype) { + Buffer buf; + throw_if_not_ok(serialization::array_schema_evolution_serialize( + ase, stype, &buf, client_side)); + + ArraySchemaEvolution* ret; + throw_if_not_ok( + serialization::array_schema_evolution_deserialize(&ret, stype, buf)); + + return shared_ptr(ret); +} + +void EnumerationFx::ser_des_query( + Query* q_in, Query* q_out, bool client_side, SerializationType stype) { + Buffer buf; + BufferList blist; + + throw_if_not_ok( + serialization::query_serialize(q_in, stype, client_side, &blist)); + + flatten_buffer_list(blist, buf); + + throw_if_not_ok(serialization::query_deserialize( + buf, + stype, + client_side, + nullptr, + q_out, + &(ctx_.resources().compute_tp()))); +} + +#else // No TILEDB_SERIALIZATION + +ArraySchema EnumerationFx::ser_des_array_schema( + shared_ptr, bool, SerializationType) { + throw std::logic_error("Serialization not enabled."); +} + +shared_ptr EnumerationFx::ser_des_array_schema_evolution( + ArraySchemaEvolution*, bool, SerializationType) { + throw std::logic_error("Serialization not enabled."); +} + +void EnumerationFx::ser_des_query(Query*, Query*, bool, SerializationType) { + throw std::logic_error("Serialization not enabled."); +} + +#endif // TILEDB_SERIALIZATION + +template +bool EnumerationFx::vec_cmp(std::vector v1, std::vector v2) { + std::sort(v1.begin(), v1.end()); + std::sort(v2.begin(), v2.end()); + + if (v1.size() != v2.size()) { + return false; + } + + for (size_t i = 0; i < v1.size(); i++) { + if (v1[i] != v2[i]) { + return false; + } + } + + return true; +} + +void EnumerationFx::flatten_buffer_list(BufferList& blist, Buffer& buf) { + const auto nbytes = blist.total_size(); + throw_if_not_ok(buf.realloc(nbytes)); + + blist.reset_offset(); + throw_if_not_ok(blist.read(buf.data(), nbytes)); + buf.set_size(nbytes); +} + +void EnumerationFx::rm_array() { + bool is_dir; + throw_if_not_ok(ctx_.resources().vfs().is_dir(uri_, &is_dir)); + if (is_dir) { + throw_if_not_ok(ctx_.resources().vfs().remove_dir(uri_)); + } +} + +template +QueryCondition create_qc( + const char* field_name, T condition_value, const QueryConditionOp& op) { + QueryCondition ret; + + if constexpr (std::is_pod_v) { + throw_if_not_ok(ret.init(field_name, &condition_value, sizeof(T), op)); + } else { + throw_if_not_ok(ret.init( + field_name, condition_value.data(), condition_value.size(), op)); + } + + return ret; +} diff --git a/test/src/unit-misc-util-safe-integral-casts.cc b/test/src/unit-misc-util-safe-integral-casts.cc new file mode 100644 index 000000000000..b16d286eb8a4 --- /dev/null +++ b/test/src/unit-misc-util-safe-integral-casts.cc @@ -0,0 +1,517 @@ +/** + * @file unit-misc-util-safe-integal-casts.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * Tests the C++ API for enumeration related functions. + */ + +#include +#include + +#include "test/support/tdb_catch.h" +#include "tiledb/sm/misc/utils.h" + +// These tests are for the safe_integral_cast and safe_itegral_cast_to_datatype +// functions found in `tiledb/sm/misc/utils.h`. These casts are named "safe" +// because they only allow casts that result in the same semantic value in the +// target type. This means that some casts that would be allowed by static_cast +// or similar facilties are rejected. Most notably, casts between types with +// different signed-ness are more thoroughly checked for correctness. +// +// There are two main tests in this file, one for safe_integral_cast and one +// for safe_integral_cast_to_datatype. These tests work by generating a +// comprehensive sample of test data to exhaustively check that casts between +// all supported integral datatypes are covered. Generating tests in this +// manner allows us to assert whether a cast should succeed vs one that should +// fail. +// +// Both tests have the same general outline. First, pass a callback to the +// run_test function which is responsible for generating a TestCase value +// which is nothing more than a type width (i.e., sizeof(int32_t)) and a +// set of random bytes in length 1 to byte_width. This ensures that we're +// able to test casst that should succeed (i.e., 5 stored in an int32_t is +// castable to a uint8_t) and casts that should fail (i.e., 300 stored in an +// int32_t is not castable to either int8_t or uint8_t). +// +// The callback passed to `run_test` is then responsible for enumerating all +// of the test types for cast checks. Both tests have the same naming pattern +// of: +// +// 1. check_$function_name - Based on TestCase.byte_width, pick the source +// datatype for the test +// 2. dispatch_$function_name - Attempt to convert the source test data to all +// target types. +// 3. run_$functon_name - Attempt the cast for the given TestData, source, and +// target types. These use the should_cast_succeed helper to know whether +// a given cast should succeed or fail. +// +// Beyond the main test layouts, there are two important helper functions +// that will be useful for understanding. +// +// First, `should_cast_succeed` which takes the source type and value, and a +// target type and returns a boolean on whether a cast should succeed. This +// is the crux of these tests as this logic is the concise list of rules that +// dictate when a cast is safe or not. +// +// The second function is `calculate_width` which is called by +// `should_cast_succeed`. This calculates the number of bits and bytes required +// to hold a given value and informs `should_cast_succeed` when a cast is +// representable in a target type. If the code in this function looks a little +// weird, you'll want to review 2's complement representation on why it counts +// things as it does. + +using namespace tiledb::sm; + +/* ********************************* */ +/* CONSTANTS */ +/* ********************************* */ + +// Widths of integral types int8_t to int64_t +constexpr int TYPE_WIDTHS[] = {1, 2, 4, 8}; +constexpr int NUM_TYPE_WIDTHS = 4; + +// Width of widest type (i.e., sizeof(int64_t)) +constexpr int MAX_WIDTH = 8; + +// An aribtrary number of iterations to run per test +constexpr int RANDOM_ITERATIONS = 500; + +// A POD struct representing a test case +struct TestCase { + // The width of the type, i.e., sizeof(int8_t), sizeof(uint32_t), etc, this + // is obviously restricted to the values 1, 2, 4, 8. + uint8_t type_width; + + // Storage for the generated value's data. LSB is data[0] + uint8_t data[MAX_WIDTH]; +}; + +/* ********************************* */ +/* TEST CALLBACKS */ +/* ********************************* */ + +// Test implementation for util::datatype::safe_integral_cast +void check_safe_integral_cast(const TestCase& tc); + +// Test implementation for util::datatype::safe_integral_cast_to_datatype +void check_safe_integral_cast_to_datatype(const TestCase& tc); + +/* ********************************* */ +/* TEST HELPER FUNCTIONS */ +/* ********************************* */ + +// Test driver function +template +void run_test(TestFunc func); + +// Apply the test function against each source type based on tc.type_width +template +void dispatch(TestFunc func, const TestCase& tc); + +// Return a boolean indicating whether the given cast should succeed or fail +template +bool should_cast_succeed(Source src); + +template +bool should_cast_succeed(Source src, Datatype dtype); + +// Calculate the width of a value in both bits and bytes. +template +std::tuple calculate_width(Type val); + +// Fills in a TestCase's data, byte_width, and msb_set members +void generate_case(TestCase& tc, uint8_t gen_byte_width, bool set_msb); + +// Convert a uint8_t[] to integral type T +template +T buf_to_value(const uint8_t* buffer); + +/* ********************************* */ +/* safe_integral_cast */ +/* ********************************* */ + +TEST_CASE("util::datatype::safe_integral_cast", "[safe-integral-casts]") { + run_test(check_safe_integral_cast); +} + +template +void dispatch_safe_integral_cast(const TestCase& tc); + +template +void run_safe_integral_cast(const TestCase&, Source src); + +void check_safe_integral_cast(const TestCase& tc) { + switch (tc.type_width) { + case 1: + dispatch_safe_integral_cast(tc); + dispatch_safe_integral_cast(tc); + return; + case 2: + dispatch_safe_integral_cast(tc); + dispatch_safe_integral_cast(tc); + return; + case 4: + dispatch_safe_integral_cast(tc); + dispatch_safe_integral_cast(tc); + return; + case 8: + dispatch_safe_integral_cast(tc); + dispatch_safe_integral_cast(tc); + return; + default: + throw std::logic_error("Invalid type_width"); + } +} + +template +void dispatch_safe_integral_cast(const TestCase& tc) { + Source val = buf_to_value(tc.data); + + run_safe_integral_cast(tc, val); + run_safe_integral_cast(tc, val); + + run_safe_integral_cast(tc, val); + run_safe_integral_cast(tc, val); + + run_safe_integral_cast(tc, val); + run_safe_integral_cast(tc, val); + + run_safe_integral_cast(tc, val); + run_safe_integral_cast(tc, val); +} + +template +void run_safe_integral_cast(const TestCase&, Source src) { + if (should_cast_succeed(src)) { + auto tgt = utils::datatype::safe_integral_cast(src); + REQUIRE(src == utils::datatype::safe_integral_cast(tgt)); + } else { + REQUIRE_THROWS(utils::datatype::safe_integral_cast(src)); + } +} + +/* ********************************* */ +/* safe_integral_cast_to_datatype */ +/* ********************************* */ + +TEST_CASE( + "util::datatype::safe_integral_cast_to_datatype", "[safe-integral-casts]") { + run_test(check_safe_integral_cast_to_datatype); +} + +TEST_CASE( + "util::datatype::safe_integral_cast_to_datatype bad type", + "[safe-integral-casts][error]") { + ByteVecValue bvv; + REQUIRE_THROWS( + utils::datatype::safe_integral_cast_to_datatype(5, Datatype::BLOB, bvv)); + REQUIRE_THROWS(utils::datatype::safe_integral_cast_to_datatype( + 5, Datatype::STRING_ASCII, bvv)); +} + +template +void dispatch_safe_integral_cast_to_datatype(const TestCase& tc); + +template +void run_safe_integral_cast_to_datatype( + const TestCase&, Source src, Datatype dt); + +void check_safe_integral_cast_to_datatype(const TestCase& tc) { + switch (tc.type_width) { + case 1: + dispatch_safe_integral_cast_to_datatype(tc); + dispatch_safe_integral_cast_to_datatype(tc); + return; + case 2: + dispatch_safe_integral_cast_to_datatype(tc); + dispatch_safe_integral_cast_to_datatype(tc); + return; + case 4: + dispatch_safe_integral_cast_to_datatype(tc); + dispatch_safe_integral_cast_to_datatype(tc); + return; + case 8: + dispatch_safe_integral_cast_to_datatype(tc); + dispatch_safe_integral_cast_to_datatype(tc); + return; + default: + throw std::logic_error("Invalid type_width"); + } +} + +template +void dispatch_safe_integral_cast_to_datatype(const TestCase& tc) { + Source val = buf_to_value(tc.data); + + std::vector datatypes = { + Datatype::BOOL, + Datatype::INT8, + Datatype::UINT8, + Datatype::INT16, + Datatype::UINT16, + Datatype::INT32, + Datatype::UINT32, + Datatype::INT64, + Datatype::UINT64}; + + for (auto dt : datatypes) { + run_safe_integral_cast_to_datatype(tc, val, dt); + } +} + +template +void run_safe_integral_cast_to_datatype( + const TestCase&, Source src, Datatype dtype) { + ByteVecValue dest; + if (should_cast_succeed(src, dtype)) { + utils::datatype::safe_integral_cast_to_datatype(src, dtype, dest); + switch (dtype) { + case Datatype::BOOL: + REQUIRE(dest.rvalue_as() == (uint8_t)src); + return; + case Datatype::INT8: + REQUIRE(dest.rvalue_as() == (int8_t)src); + return; + case Datatype::UINT8: + REQUIRE(dest.rvalue_as() == (uint8_t)src); + return; + case Datatype::INT16: + REQUIRE(dest.rvalue_as() == (int16_t)src); + return; + case Datatype::UINT16: + REQUIRE(dest.rvalue_as() == (uint16_t)src); + return; + case Datatype::INT32: + REQUIRE(dest.rvalue_as() == (int32_t)src); + return; + case Datatype::UINT32: + REQUIRE(dest.rvalue_as() == (uint32_t)src); + return; + case Datatype::INT64: + REQUIRE(dest.rvalue_as() == (int64_t)src); + return; + case Datatype::UINT64: + REQUIRE(dest.rvalue_as() == (uint64_t)src); + return; + default: + throw std::logic_error("Invalid datatype for test"); + } + } else { + REQUIRE_THROWS(utils::datatype::safe_integral_cast_to_datatype( + src, dtype, dest)); + } +} + +/* ********************************* */ +/* TEST HELPER IMPLEMENTATIONS */ +/* ********************************* */ + +template +void run_test(TestFunc tfunc) { + TestCase tc; + for (uint8_t i = 0; i < NUM_TYPE_WIDTHS; i++) { + tc.type_width = TYPE_WIDTHS[i]; + + // Always check no bits set edge case + memset(tc.data, 0, MAX_WIDTH); + tfunc(tc); + + for (uint8_t gen_width = 1; gen_width <= tc.type_width; gen_width++) { + for (int i = 0; i < RANDOM_ITERATIONS; i++) { + generate_case(tc, gen_width, false); + tfunc(tc); + + generate_case(tc, gen_width, true); + tfunc(tc); + } + } + + // Always check all bits set edge case + memset(tc.data, 255, MAX_WIDTH); + tfunc(tc); + } +} + +template +bool should_cast_succeed(Source src) { + auto [bit_width, byte_width] = calculate_width(src); + bool tgt_signed = std::numeric_limits::is_signed; + uint8_t tgt_size = sizeof(Target); + + // Obviously, if we have more bytes in src than the Target type can hold, + // the cast will fail. + if (byte_width > tgt_size) { + return false; + } + + // Unsigned types can't hold negative values, so if src is negative + // and Target is not signed, we must fail. + if (src < 0 && !tgt_signed) { + return false; + } + + // When converting from a positive value to a signed type we have to make + // sure that the signed type has more than bit_width + 1 bits available so + // that the resulting value doesn't become negative. + if ((src > 0 && tgt_signed) && (bit_width >= (tgt_size * 8))) { + return false; + } + + // When converting to a negative value to a signed type, we have to make + // sure the target has enough bits to represent the value. + if ((src < 0 && tgt_signed) && (bit_width >= (tgt_size * 8))) { + return false; + } + + // Otherwise, the cast should succeed + return true; +} + +template +bool should_cast_succeed(Source src, Datatype dtype) { + switch (dtype) { + case Datatype::BOOL: + return should_cast_succeed(src); + case Datatype::INT8: + return should_cast_succeed(src); + case Datatype::UINT8: + return should_cast_succeed(src); + case Datatype::INT16: + return should_cast_succeed(src); + case Datatype::UINT16: + return should_cast_succeed(src); + case Datatype::INT32: + return should_cast_succeed(src); + case Datatype::UINT32: + return should_cast_succeed(src); + case Datatype::INT64: + return should_cast_succeed(src); + case Datatype::UINT64: + return should_cast_succeed(src); + default: + throw std::logic_error("Invalid datatype for test"); + } +} + +template +std::tuple calculate_width(Type val) { + // Calculate the number of bits required to hold src. For positive values, + // this is the largest numbered bit set to 1, for negative values its the + // largest numbered bit set to 0. + + uint8_t bit_width = 0; + if (val > 0) { + // For positive values, we find the highest set bit by counting the number + // of right shifts required before the value is zero. + uint8_t i = 0; + while (val > 0) { + val >>= 1; + i += 1; + } + bit_width = i; + } else if (val < 0) { + // For negative values, we find the highest unset bit by counting the + // number of left shifts before the value becomes greater than or equal + // to zero. + uint8_t i = 0; + while (val < 0) { + val <<= 1; + i += 1; + } + bit_width = sizeof(Type) * 8 - i; + } + + uint8_t byte_width = bit_width / 8; + if (bit_width % 8 != 0) { + byte_width += 1; + } + + return {bit_width, byte_width}; +} + +// A simple function for generating random uint64_t values +uint64_t generate_value() { + static std::random_device rand_dev; + static std::mt19937_64 generator(rand_dev()); + static std::uniform_int_distribution dist; + return dist(generator); +} + +// Generate a TestCase with the given gen_width number of random bytes. The +// only curisoity is the set_msb value which forces the most significant bit +// of the most significant byte to 0 or 1. Forcing the msb to 1 allows us to +// assert that we're testing two important cases. First, casting negative +// values between different types widths, and second, checking that large +// unsigned values of a given type that aren't representable in the signed +// type of the same byte width. +void generate_case(TestCase& tc, uint8_t gen_width, bool set_msb) { + // Reset TestCase state + memset(tc.data, 0, MAX_WIDTH); + + // Convince GCC that we'll not write beyond MAX_WIDTH bytes into tc.data + if (gen_width > MAX_WIDTH) { + throw std::logic_error("Invalid gen_width in test case"); + } + + // A standard bit shift approach for converting integral values into + // an array of uint8_t values. Its important to note that the << and >> + // operators are endian-ness agnostic. So this just masks everything but + // the least-significant-byte, converts that to a uint8_t and stores it + // in the array. Then drop the least-significant byte of the random value + // by shifting 8 bits to the right. + // + // TestCase.byte_width is calculated here since its perfectly valid for + // a generated value to not have any bits set in any given byte which can + // cause expected failures to unexpected succeed if its the MSB that is + // all zeroes. + uint64_t rand_value = generate_value(); + for (int i = 0; i < gen_width; i++) { + tc.data[i] = static_cast(rand_value & 0xFF); + rand_value >>= 8; + } + + // If set_msb is false, set the gen_width's byte msb to 0, else set to 1 + if (!set_msb) { + tc.data[gen_width - 1] &= 0x7F; + } else { + tc.data[gen_width - 1] |= 0x80; + } +} + +// Convert a uint8_t buffer to the given type T. This works using the standard +// bit twiddling approach to progressive bitwise-or one byte at a time in a +// most-to-least significant byte order. +template +T buf_to_value(const uint8_t* buffer) { + T ret = 0; + for (int8_t i = sizeof(T) - 1; i >= 0; i--) { + T new_byte = (T)buffer[i]; + ret = (ret << 8) | new_byte; + } + return ret; +} diff --git a/tiledb/CMakeLists.txt b/tiledb/CMakeLists.txt index c23ee2c0bc1f..2fabec080c0b 100644 --- a/tiledb/CMakeLists.txt +++ b/tiledb/CMakeLists.txt @@ -84,10 +84,12 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/tiledb_experimental ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_deprecated.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_schema.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_schema_evolution.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/array_schema_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/attribute.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/attribute_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/config.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/consolidation_plan_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/context.h @@ -96,6 +98,7 @@ if (TILEDB_CPP_API) ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/dimension.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/dimension_label_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/domain.h + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/enumeration_experimental.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/error.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/exception.h ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/cpp_api/filter.h @@ -147,6 +150,7 @@ set(TILEDB_CORE_SOURCES ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/dimension.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/dimension_label.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/domain.cc + ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/array_schema/enumeration.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/buffer/buffer.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/buffer/buffer_list.cc ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/c_api/api_argument_validator.cc @@ -862,14 +866,22 @@ if(TILEDB_SERIALIZATION) ${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization/${TILEDB_SERIALIZATION_GENERATED_SUBDIR}) # We only need to override the include path and PATH env for EP builds + # note: We run the compiler under `cmake -E env ...` because we need to run capnp executable + # the capnp driver needs to be able to find the plugin executables (eg capnpc-c++). + # For system installed capnp binaries we rely on the user to have their PATH + # include the directory with CapNProto plugin binaries. if(TILEDB_CAPNP_EP_BUILT) - list(APPEND CAPNP_COMPILE_COMMAND "${CMAKE_COMMAND}" -E env PATH="${TILEDB_EP_BASE}/install/bin" "${CAPNP_EXECUTABLE}" compile -I "${TILEDB_EP_BASE}/install/include" -oc++:"${TILEDB_SERIALIZATION_GENERATED_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization/tiledb-rest.capnp" --src-prefix="${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization") + list(APPEND CAPNP_COMPILE_COMMAND "${CMAKE_COMMAND}" -E env PATH="${TILEDB_EP_BASE}/install/bin" "${CAPNP_EXECUTABLE}" compile -I "${TILEDB_EP_BASE}/install/include") + elseif(TILEDB_VCPKG) + set(CAPNP_PLUGIN_DIR $) + list(APPEND CAPNP_COMPILE_COMMAND "${CMAKE_COMMAND}" -E env PATH="${CAPNP_PLUGIN_DIR}" "${CAPNP_EXECUTABLE}" compile "-I${CAPNP_INCLUDE_DIRECTORY}") else() - list(APPEND CAPNP_COMPILE_COMMAND "${CAPNP_EXECUTABLE}" compile -oc++:"${TILEDB_SERIALIZATION_GENERATED_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization/tiledb-rest.capnp" --src-prefix "${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization") + list(APPEND CAPNP_COMPILE_COMMAND "${CAPNP_EXECUTABLE}" compile) endif() - # note: run the compiler under `cmake -E env ...` because we need to run capnp executable - # the capnp driver needs to be able to find the plugin executables (eg capnpc-c++) + # Add the rest of the capnp compile command + list(APPEND CAPNP_COMPILE_COMMAND -oc++:"${TILEDB_SERIALIZATION_GENERATED_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization/tiledb-rest.capnp" --src-prefix="${CMAKE_CURRENT_SOURCE_DIR}/sm/serialization") + add_custom_target( update-serialization ${CMAKE_COMMAND} -E echo "CAPNP_COMPILE_COMMAND: '${CAPNP_COMPILE_COMMAND}'" diff --git a/tiledb/api/c_api/CMakeLists.txt b/tiledb/api/c_api/CMakeLists.txt index 655bc3402249..a1d2cdf7e30d 100644 --- a/tiledb/api/c_api/CMakeLists.txt +++ b/tiledb/api/c_api/CMakeLists.txt @@ -76,6 +76,9 @@ add_subdirectory(data_order) # `dimension_label`: depends on `context`, `datatype`, `data_order` add_subdirectory(dimension_label) +# `enumeration`: depends on `buffer`, `constants`, and `context` +add_subdirectory(enumeration) + # `filesystem`: no dependencies add_subdirectory(filesystem) diff --git a/tiledb/api/c_api/dimension_label/CMakeLists.txt b/tiledb/api/c_api/dimension_label/CMakeLists.txt index 9cfe7b01e707..e8f9d0d214ba 100644 --- a/tiledb/api/c_api/dimension_label/CMakeLists.txt +++ b/tiledb/api/c_api/dimension_label/CMakeLists.txt @@ -46,6 +46,7 @@ target_link_libraries(capi_dimension_label_stub PUBLIC array_schema $(type); + + bool is_ordered = false; + if (ordered != 0) { + is_ordered = true; + } + + try { + *enumeration = tiledb_enumeration_handle_t::make_handle( + std::string(name), + datatype, + cell_val_num, + is_ordered, + data, + data_size, + offsets, + offsets_size); + } catch (...) { + *enumeration = nullptr; + throw; + } + + return TILEDB_OK; +} + +void tiledb_enumeration_free(tiledb_enumeration_t** enumeration) { + ensure_output_pointer_is_valid(enumeration); + ensure_enumeration_is_valid(*enumeration); + tiledb_enumeration_handle_t::break_handle(*enumeration); +} + +capi_return_t tiledb_enumeration_get_name( + tiledb_enumeration_t* enumeration, tiledb_string_handle_t** name) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(name); + *name = tiledb_string_handle_t::make_handle(enumeration->name()); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_get_type( + tiledb_enumeration_t* enumeration, tiledb_datatype_t* type) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(type); + *type = static_cast(enumeration->type()); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_get_cell_val_num( + tiledb_enumeration_t* enumeration, uint32_t* cell_val_num) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(cell_val_num); + *cell_val_num = enumeration->cell_val_num(); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_get_ordered( + tiledb_enumeration_t* enumeration, int* ordered) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(ordered); + *ordered = static_cast(enumeration->ordered()); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_get_data( + tiledb_enumeration_t* enumeration, const void** data, uint64_t* data_size) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(data); + ensure_output_pointer_is_valid(data_size); + auto dspan = enumeration->data(); + *data = dspan.data(); + *data_size = dspan.size(); + return TILEDB_OK; +} + +capi_return_t tiledb_enumeration_get_offsets( + tiledb_enumeration_t* enumeration, + const void** offsets, + uint64_t* offsets_size) { + ensure_enumeration_is_valid(enumeration); + ensure_output_pointer_is_valid(offsets); + ensure_output_pointer_is_valid(offsets_size); + auto ospan = enumeration->offsets(); + *offsets = ospan.data(); + *offsets_size = ospan.size(); + return TILEDB_OK; +} + +} // namespace tiledb::api + +using tiledb::api::api_entry_context; +using tiledb::api::api_entry_void; + +capi_return_t tiledb_enumeration_alloc( + tiledb_ctx_t* ctx, + const char* name, + tiledb_datatype_t type, + uint32_t cell_val_num, + int ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size, + tiledb_enumeration_t** enumeration) noexcept { + return api_entry_context( + ctx, + name, + type, + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size, + enumeration); +} + +void tiledb_enumeration_free(tiledb_enumeration_t** enumeration) noexcept { + return api_entry_void(enumeration); +} + +capi_return_t tiledb_enumeration_get_name( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + tiledb_string_handle_t** name) noexcept { + return api_entry_context( + ctx, enumeration, name); +} + +capi_return_t tiledb_enumeration_get_type( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + tiledb_datatype_t* type) noexcept { + return api_entry_context( + ctx, enumeration, type); +} + +capi_return_t tiledb_enumeration_get_cell_val_num( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + uint32_t* cell_val_num) noexcept { + return api_entry_context( + ctx, enumeration, cell_val_num); +} + +capi_return_t tiledb_enumeration_get_ordered( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + int* ordered) noexcept { + return api_entry_context( + ctx, enumeration, ordered); +} + +capi_return_t tiledb_enumeration_get_data( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + const void** data, + uint64_t* data_size) noexcept { + return api_entry_context( + ctx, enumeration, data, data_size); +} + +capi_return_t tiledb_enumeration_get_offsets( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + const void** offsets, + uint64_t* offsets_size) noexcept { + return api_entry_context( + ctx, enumeration, offsets, offsets_size); +} diff --git a/tiledb/api/c_api/enumeration/enumeration_api_experimental.h b/tiledb/api/c_api/enumeration/enumeration_api_experimental.h new file mode 100644 index 000000000000..fd5ff27271d9 --- /dev/null +++ b/tiledb/api/c_api/enumeration/enumeration_api_experimental.h @@ -0,0 +1,257 @@ +/** + * @file tiledb/api/c_api/enumeration/enumeration_api_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the enumeration section of the C API for TileDB. + */ + +#ifndef TILEDB_CAPI_ENUMERATION_EXPERIMENTAL_H +#define TILEDB_CAPI_ENUMERATION_EXPERIMENTAL_H + +#include "../api_external_common.h" +#include "../context/context_api_external.h" +#include "../datatype/datatype_api_external.h" +#include "../string/string_api_external.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** A TileDB dimension. */ +typedef struct tiledb_enumeration_handle_t tiledb_enumeration_t; + +/** + * Creates an Enumeration. + * + * **Example:** + * + * @code{.c} + * tiledb_enumeration_t* enumeration; + * void* data = get_data(); + * uint64_t data_size = get_data_size(); + * tiledb_enumeration_alloc( + * ctx, + * TILEDB_INT64, + * cell_val_num, + * FALSE, + * data, + * data_size, + * nullptr, + * 0, + * &enumeration); + * @endcode + * + * @param ctx The TileDB context. + * @param name The name of the enumeration. + * @param type The enumeration type. + * @param cell_val_num The number of values per enumeration value. + * @param ordered Whether this enumeration should be considered as ordered. + * @param data A pointer to the enumeration value data. + * @param data_size The length of the data buffer provided. + * @param offsets A pointer to the offsets buffer if cell_vall_num + * is TILEDB_VAR_NUM. + * @param offsets_size The length of the offsets buffer, zero if no offsets. + * @param enumeration The newly allocated enumeration. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_alloc( + tiledb_ctx_t* ctx, + const char* name, + tiledb_datatype_t type, + uint32_t cell_val_num, + int ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size, + tiledb_enumeration_t** enumeration) TILEDB_NOEXCEPT; + +/** + * Destroys a TileDB enumeration, freeing associated memory. + * + * **Example:** + * + * @code{.c} + * tiledb_enumeration_free(&enumeration); + * @endcode + * + * @param enumeration The enumeration to be destroyed. + */ +TILEDB_EXPORT void tiledb_enumeration_free(tiledb_enumeration_t** enumeration) + TILEDB_NOEXCEPT; + +/** + * Return the datatype of the enumeration values + * + * **Example:** + * + * @code{.c} + * tiledb_string_handle_t* name_strh; + * tiledb_enumeration_get_type(ctx, enumeration, &name_strh); + * const char* name; + * size_t name_len; + * tiledb_string_view(str_handle, &name, &name_len); + * @endcode + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param name The name of the enumeration. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_name( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + tiledb_string_handle_t** name) TILEDB_NOEXCEPT; + +/** + * Return the datatype of the enumeration values + * + * **Example:** + * + * @code{.c} + * tiledb_datatype_t type; + * tiledb_enumeration_get_type(ctx, enumeration, &type); + * @endcode + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param type The data type of the enumeration. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_type( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + tiledb_datatype_t* type) TILEDB_NOEXCEPT; + +/** + * Return the cell value number of the enumeration values + * + * **Example:** + * + * @code{.c} + * uint32_t cell_val_num; + * tiledb_enumeration_get_cell_val_num(ctx, enumeration, &cell_val_num); + * @endcode + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param type The cell value number of the enumeration. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_cell_val_num( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + uint32_t* cell_val_num) TILEDB_NOEXCEPT; + +/** + * Return whether the enumeration values should be considered ordered. + * + * **Example:** + * + * @code{.c} + * int ordered; + * tiledb_enumeration_get_ordered(ctx, enumeration, &ordered); + * @endcode + * + * The cell values are considered if the value in ordered after `TILEDB_OK` + * is returned is non-zero. I.e., this is standard `int` as `bool` behavior. + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param ordered A boolean value indicating whether the values are ordered. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_ordered( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + int* ordered) TILEDB_NOEXCEPT; + +/** + * Return a pointer to and size of the enumerations underlying value data + * + * **Example:** + * + * @code{.c} + * void* data = NULL; + * uint64_t data_size = 0; + * tiledb_enumeration_get_data(ctx, enumeration, &data, &data_size); + * @endcode + * + * The pointer returned from this function references internal data managed + * by the enumeration. As such, clients should not attempt to free it, or + * access it beyond the lifetime of the enumeration instance. + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param data The returned pointer to this enumeration value buffer. + * @param data_size The length of the buffer pointed to by data. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_data( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + const void** data, + uint64_t* data_size) TILEDB_NOEXCEPT; + +/** + * Return a pointer to and size of the enumerations underlying value offsets + * + * **Example:** + * + * @code{.c} + * void* offsets = NULL; + * uint64_t offsets_size = 0; + * tiledb_enumeration_get_offsets(ctx, enumeration, &offsets, &offsets_size); + * @endcode + * + * The pointer returned from this function references internal data managed + * by the enumeration. As such, clients should not attempt to free it, or + * access it beyond the lifetime of the enumeration instance. + * + * If the enumeration values are var sized (i.e., cell_var_num is + * TILEDB_VAR_NUM) the offsets buffer will contain a `uint64_t` value for the + * starting offset of each underlying enumeration value. Note that the number + * of offsets is calculated as `offsets_size / sizeof(uint64_t)`. + * + * @param ctx The TileDB context. + * @param enumeration The enumeration. + * @param data The returned pointer to this enumeration offsets buffer. + * @param data_size The length of the buffer pointed to by offsets. + * @return `TILEDB_OK` or `TILEDB_ERR`. + */ +TILEDB_EXPORT capi_return_t tiledb_enumeration_get_offsets( + tiledb_ctx_t* ctx, + tiledb_enumeration_t* enumeration, + const void** offsets, + uint64_t* offsets_size) TILEDB_NOEXCEPT; + +#ifdef __cplusplus +} +#endif + +#endif // TILEDB_CAPI_ENUMERATION_EXPERIMENTAL_H diff --git a/tiledb/api/c_api/enumeration/enumeration_api_internal.h b/tiledb/api/c_api/enumeration/enumeration_api_internal.h new file mode 100644 index 000000000000..10a9bfe627a4 --- /dev/null +++ b/tiledb/api/c_api/enumeration/enumeration_api_internal.h @@ -0,0 +1,132 @@ +/** + * @file tiledb/api/c_api/enumeration/enumeration_api_internal.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the enumeration section of the C API for TileDB. + */ + +#ifndef TILEDB_CAPI_ENUMERATION_INTERNAL_H +#define TILEDB_CAPI_ENUMERATION_INTERNAL_H + +#include "enumeration_api_experimental.h" +#include "tiledb/api/c_api_support/handle/handle.h" +#include "tiledb/common/common.h" +#include "tiledb/sm/array_schema/enumeration.h" + +/** + * Handle `struct` for API enumeration objects. + */ +struct tiledb_enumeration_handle_t + : public tiledb::api::CAPIHandle { + private: + shared_ptr enumeration_; + + public: + /** + * Type name + */ + static constexpr std::string_view object_type_name{"enumeration"}; + + template + tiledb_enumeration_handle_t(Arg&&... arg) + : enumeration_( + tiledb::sm::Enumeration::create(std::forward(arg)...)) { + } + + /** + * Constructor from `shared_ptr` copies the shared pointer. + */ + explicit tiledb_enumeration_handle_t( + shared_ptr& e) + : enumeration_(e) { + } + + /** + * Copy the underlying enumeration object. + */ + [[nodiscard]] shared_ptr copy() { + return enumeration_; + } + + /** + * Return the name of the enumeration. + */ + [[nodiscard]] inline const std::string& name() const { + return enumeration_->name(); + } + + /** + * Return the data type of the enumeration values. + */ + [[nodiscard]] inline tiledb_datatype_t type() const { + return static_cast(enumeration_->type()); + } + + /** + * Return the cell_val_num of the enumeration values + */ + [[nodiscard]] inline uint32_t cell_val_num() const { + return enumeration_->cell_val_num(); + } + + /** + * Return a bool indicating whether the enumeration values are ordered. + */ + [[nodiscard]] inline bool ordered() const { + return enumeration_->ordered(); + } + + /** + * Return a pointer and size tuple for the underlying data type. + */ + [[nodiscard]] inline const span data() const { + return enumeration_->data(); + } + + /** + * Return a pointer and size tuple for the underlying offsets buffer. + */ + [[nodiscard]] inline const span offsets() const { + return enumeration_->offsets(); + } +}; + +namespace tiledb::api { + +/** + * Returns after successfully validating an error. Throws otherwise. + * + * @param dim A possibly-valid enumeration handle. + */ +inline void ensure_enumeration_is_valid(const tiledb_enumeration_handle_t* e) { + ensure_handle_is_valid(e); +} + +} // namespace tiledb::api + +#endif // TILEDB_CAPI_ENUMERATION_INTERNAL_H diff --git a/tiledb/api/c_api/enumeration/test/CMakeLists.txt b/tiledb/api/c_api/enumeration/test/CMakeLists.txt new file mode 100644 index 000000000000..7ace61d484d8 --- /dev/null +++ b/tiledb/api/c_api/enumeration/test/CMakeLists.txt @@ -0,0 +1,32 @@ +# +# tiledb/api/c_api/enumeration/test/CMakeLists.txt +# +# The MIT License +# +# Copyright (c) 2023 TileDB, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# + +include(unit_test) + +commence(unit_test capi_enumeration) + this_target_sources(unit_capi_enumeration.cc) + this_target_object_libraries(capi_enumeration_stub) +conclude(unit_test) diff --git a/tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc b/tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc new file mode 100644 index 000000000000..0b7139553f2a --- /dev/null +++ b/tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc @@ -0,0 +1,39 @@ +/** + * @file tiledb/api/c_api/enumeration/test/compile_capi_enumeration_stub_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "../enumeration_api_internal.h" +#include "tiledb/sm/enums/datatype.h" + +int main() { + try { + tiledb_enumeration_handle_t e{ + "fooo", tiledb::sm::Datatype::INT32, 1, 0, nullptr, 0, nullptr, 0}; + } catch (...) { + } + return 0; +} diff --git a/tiledb/api/c_api/enumeration/test/unit_capi_enumeration.cc b/tiledb/api/c_api/enumeration/test/unit_capi_enumeration.cc new file mode 100644 index 000000000000..a185f5e4d7ad --- /dev/null +++ b/tiledb/api/c_api/enumeration/test/unit_capi_enumeration.cc @@ -0,0 +1,471 @@ +/** + * @file tiledb/api/c_api/enumeration/test/unit_capi_enumeration.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#define CATCH_CONFIG_MAIN +#include +#include "../enumeration_api_experimental.h" +#include "tiledb/api/c_api_test_support/testsupport_capi_context.h" +#include "tiledb/sm/misc/constants.h" + +// TILEDB_VAR_NUM is defined in tiledb.h which we can't use without linking +// against the entire libtiledb. Instead we'll just declare our own. +#ifndef TILEDB_VAR_NUM +#define TILEDB_VAR_NUM tiledb::sm::constants::var_num +#endif + +using namespace tiledb::api::test_support; + +struct FixedSizeEnumeration { + FixedSizeEnumeration() { + uint32_t values[5] = {1, 2, 3, 4, 5}; + auto rc = tiledb_enumeration_alloc( + ctx_.context, + "an_enumeration", + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration_); + REQUIRE(rc == TILEDB_OK); + } + + ~FixedSizeEnumeration() { + tiledb_enumeration_free(&enumeration_); + } + + ordinary_context ctx_; + tiledb_enumeration_t* enumeration_; +}; + +struct VarSizeEnumeration { + VarSizeEnumeration() { + const char* values = "foobarbazbingobango"; + uint64_t offsets[5] = {0, 3, 6, 9, 14}; + auto rc = tiledb_enumeration_alloc( + ctx_.context, + "an_enumeration", + TILEDB_STRING_UTF8, + TILEDB_VAR_NUM, + 0, + values, + strlen(values), + offsets, + 5 * sizeof(uint64_t), + &enumeration_); + REQUIRE(rc == TILEDB_OK); + } + + ~VarSizeEnumeration() { + tiledb_enumeration_free(&enumeration_); + } + + ordinary_context ctx_; + tiledb_enumeration_t* enumeration_; +}; + +TEST_CASE( + "C API: tiledb_enumeration_alloc argument validation", + "[capi][enumeration]") { + ordinary_context ctx{}; + tiledb_enumeration_t* enumeration = nullptr; + + int32_t values[5] = {1, 2, 3, 4, 5}; + const char* data = "foobarbazbingobango"; + uint64_t offsets[5] = {0, 3, 6, 9, 14}; + + SECTION("success - fixed size") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_OK); + tiledb_enumeration_free(&enumeration); + } + + SECTION("success - var size") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_STRING_ASCII, + TILEDB_VAR_NUM, + 0, + (void*)data, + strlen(data), + offsets, + sizeof(uint64_t) * 5, + &enumeration); + REQUIRE(rc == TILEDB_OK); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - null context") { + auto rc = tiledb_enumeration_alloc( + nullptr, + "an_enumeration", + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - invalid name") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + nullptr, + TILEDB_UINT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - invalid datatype") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + (tiledb_datatype_t)255, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - data nullptr") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_INT32, + 1, + 0, + nullptr, + sizeof(uint32_t) * 5, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - data_size == 0") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_INT32, + 1, + 0, + values, + 0, + nullptr, + 0, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - enumeration nullptr") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_INT32, + 1, + 0, + values, + sizeof(uint32_t) * 5, + nullptr, + 0, + nullptr); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - offsets nullptr") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_STRING_ASCII, + TILEDB_VAR_NUM, + 0, + (void*)data, + strlen(data), + nullptr, + sizeof(uint64_t) * 5, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } + + SECTION("failure - offsets_size == 0") { + auto rc = tiledb_enumeration_alloc( + ctx.context, + "an_enumeration", + TILEDB_STRING_ASCII, + TILEDB_VAR_NUM, + 0, + (void*)data, + strlen(data), + offsets, + 0, + &enumeration); + REQUIRE(rc == TILEDB_ERR); + tiledb_enumeration_free(&enumeration); + } +} + +TEST_CASE( + "C API: tiledb_enumeration_free argument validation", + "[capi][enumeration]") { + REQUIRE_NOTHROW(tiledb_enumeration_free(nullptr)); +} + +TEST_CASE( + "C API: tiledb_enumeration_get_type argument validation", + "[capi][enumeration]") { + FixedSizeEnumeration fe; + VarSizeEnumeration ve; + tiledb_datatype_t dt; + + SECTION("success") { + auto rc = + tiledb_enumeration_get_type(fe.ctx_.context, fe.enumeration_, &dt); + REQUIRE(rc == TILEDB_OK); + REQUIRE(dt == TILEDB_UINT32); + + rc = tiledb_enumeration_get_type(ve.ctx_.context, ve.enumeration_, &dt); + REQUIRE(rc == TILEDB_OK); + REQUIRE(dt == TILEDB_STRING_UTF8); + } + + SECTION("failure - invalid context") { + auto rc = tiledb_enumeration_get_type(nullptr, fe.enumeration_, &dt); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + } + + SECTION("failure - invalid enumeration") { + auto rc = tiledb_enumeration_get_type(fe.ctx_.context, nullptr, &dt); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid type pointer") { + auto rc = + tiledb_enumeration_get_type(fe.ctx_.context, fe.enumeration_, nullptr); + REQUIRE(rc == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_enumeration_get_cell_val_num argument validation", + "[capi][enumeration]") { + FixedSizeEnumeration fe; + VarSizeEnumeration ve; + uint32_t cvn; + + SECTION("success") { + auto rc = tiledb_enumeration_get_cell_val_num( + fe.ctx_.context, fe.enumeration_, &cvn); + REQUIRE(rc == TILEDB_OK); + REQUIRE(cvn == 1); + + rc = tiledb_enumeration_get_cell_val_num( + ve.ctx_.context, ve.enumeration_, &cvn); + REQUIRE(rc == TILEDB_OK); + REQUIRE(cvn == TILEDB_VAR_NUM); + } + + SECTION("failure - invalid context") { + auto rc = + tiledb_enumeration_get_cell_val_num(nullptr, fe.enumeration_, &cvn); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + } + + SECTION("failure - invalid enumeration") { + auto rc = + tiledb_enumeration_get_cell_val_num(fe.ctx_.context, nullptr, &cvn); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid cell_val_num pointer") { + auto rc = tiledb_enumeration_get_cell_val_num( + fe.ctx_.context, fe.enumeration_, nullptr); + REQUIRE(rc == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_enumeration_get_ordered argument validation", + "[capi][enumeration]") { + FixedSizeEnumeration fe; + VarSizeEnumeration ve; + int o; + + SECTION("success") { + auto rc = + tiledb_enumeration_get_ordered(fe.ctx_.context, fe.enumeration_, &o); + REQUIRE(rc == TILEDB_OK); + REQUIRE(!o); + + rc = tiledb_enumeration_get_ordered(ve.ctx_.context, ve.enumeration_, &o); + REQUIRE(rc == TILEDB_OK); + REQUIRE(!o); + } + + SECTION("failure - invalid context") { + auto rc = tiledb_enumeration_get_ordered(nullptr, fe.enumeration_, &o); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + } + + SECTION("failure - invalid enumeration") { + auto rc = tiledb_enumeration_get_ordered(fe.ctx_.context, nullptr, &o); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid ordered pointer") { + auto rc = tiledb_enumeration_get_ordered( + fe.ctx_.context, fe.enumeration_, nullptr); + REQUIRE(rc == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_enumeration_get_data argument validation", + "[capi][enumeration]") { + FixedSizeEnumeration fe; + VarSizeEnumeration ve; + const void* d; + uint64_t ds; + + uint32_t fixed_expect[5] = {1, 2, 3, 4, 5}; + const char* var_expect = "foobarbazbingobango"; + + SECTION("success") { + auto rc = + tiledb_enumeration_get_data(fe.ctx_.context, fe.enumeration_, &d, &ds); + REQUIRE(rc == TILEDB_OK); + REQUIRE(std::memcmp(fixed_expect, d, sizeof(uint32_t) * 5) == 0); + REQUIRE(ds == sizeof(uint32_t) * 5); + + rc = tiledb_enumeration_get_data(ve.ctx_.context, ve.enumeration_, &d, &ds); + REQUIRE(rc == TILEDB_OK); + REQUIRE(std::memcmp(var_expect, d, strlen(var_expect)) == 0); + REQUIRE(ds == strlen(var_expect)); + } + + SECTION("failure - invalid context") { + auto rc = tiledb_enumeration_get_data(nullptr, fe.enumeration_, &d, &ds); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + } + + SECTION("failure - invalid enumeration") { + auto rc = tiledb_enumeration_get_data(fe.ctx_.context, nullptr, &d, &ds); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid data pointer") { + auto rc = tiledb_enumeration_get_data( + fe.ctx_.context, fe.enumeration_, nullptr, &ds); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid data size pointer") { + auto rc = tiledb_enumeration_get_data( + fe.ctx_.context, fe.enumeration_, &d, nullptr); + REQUIRE(rc == TILEDB_ERR); + } +} + +TEST_CASE( + "C API: tiledb_enumeration_get_offsets argument validation", + "[capi][enumeration]") { + FixedSizeEnumeration fe; + VarSizeEnumeration ve; + const void* o; + uint64_t os; + + uint64_t var_expect[5] = {0, 3, 6, 9, 14}; + + SECTION("success") { + auto rc = tiledb_enumeration_get_offsets( + fe.ctx_.context, fe.enumeration_, &o, &os); + REQUIRE(rc == TILEDB_OK); + REQUIRE(o == nullptr); + REQUIRE(os == 0); + + rc = tiledb_enumeration_get_offsets( + ve.ctx_.context, ve.enumeration_, &o, &os); + REQUIRE(rc == TILEDB_OK); + REQUIRE(std::memcmp(var_expect, o, sizeof(uint64_t) * 5) == 0); + REQUIRE(os == sizeof(uint64_t) * 5); + } + + SECTION("failure - invalid context") { + auto rc = tiledb_enumeration_get_offsets(nullptr, fe.enumeration_, &o, &os); + REQUIRE(rc == TILEDB_INVALID_CONTEXT); + } + + SECTION("failure - invalid enumeration") { + auto rc = tiledb_enumeration_get_offsets(fe.ctx_.context, nullptr, &o, &os); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid offsets pointer") { + auto rc = tiledb_enumeration_get_offsets( + fe.ctx_.context, fe.enumeration_, nullptr, &os); + REQUIRE(rc == TILEDB_ERR); + } + + SECTION("failure - invalid offsets size pointer") { + auto rc = tiledb_enumeration_get_offsets( + fe.ctx_.context, fe.enumeration_, &o, nullptr); + REQUIRE(rc == TILEDB_ERR); + } +} diff --git a/tiledb/sm/array/array.cc b/tiledb/sm/array/array.cc index 43ee7ef43279..294c338e02a0 100644 --- a/tiledb/sm/array/array.cc +++ b/tiledb/sm/array/array.cc @@ -582,6 +582,20 @@ void Array::delete_fragments_list( } } +shared_ptr Array::get_enumeration( + const std::string& enumeration_name) { + if (!is_open_) { + throw ArrayStatusException("Cannot get enumeration; Array is not open"); + } + + if (array_schema_latest_->is_enumeration_loaded(enumeration_name)) { + return array_schema_latest_->get_enumeration(enumeration_name); + } + + return array_dir_.load_enumeration( + array_schema_latest_, enumeration_name, get_encryption_key()); +} + bool Array::is_empty() const { return fragment_metadata_.empty(); } diff --git a/tiledb/sm/array/array.h b/tiledb/sm/array/array.h index 92f2fccf14f4..ad7676a6ad9b 100644 --- a/tiledb/sm/array/array.h +++ b/tiledb/sm/array/array.h @@ -257,6 +257,19 @@ class Array { return fragment_metadata_; } + /** + * Get the enumeration for the given name. + * + * This function retrieves the enumeration for the given name. If the + * corresponding enumeration has not been loaded from storage it is + * loaded before this function returns. + * + * @param enumeration_name The name of the enumeration. + * @return shared_ptr or nullptr on failure. + */ + shared_ptr get_enumeration( + const std::string& enumeration_name); + /** * Returns `true` if the array is empty at the time it is opened. * The funciton returns `false` if the array is not open. diff --git a/tiledb/sm/array/array_directory.cc b/tiledb/sm/array/array_directory.cc index 35984fffb0ca..c105e7838348 100644 --- a/tiledb/sm/array/array_directory.cc +++ b/tiledb/sm/array/array_directory.cc @@ -33,6 +33,7 @@ #include "tiledb/sm/array/array_directory.h" #include "tiledb/common/logger.h" #include "tiledb/common/stdx_string.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/filesystem/vfs.h" #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/parallel_functions.h" @@ -185,6 +186,29 @@ ArrayDirectory::load_all_array_schemas( return array_schemas; } +shared_ptr ArrayDirectory::load_enumeration( + shared_ptr schema, + const std::string& enumeration_name, + const EncryptionKey& encryption_key) const { + auto timer_se = resources_.get().stats().start_timer("sm_load_enumeration"); + + auto path_name = schema->get_enumeration_path_name(enumeration_name); + + auto enmr_uri = uri_.join_path(constants::array_schema_dir_name) + .join_path(constants::array_enumerations_dir_name) + .join_path(path_name); + + auto&& tile = GenericTileIO::load(resources_, enmr_uri, 0, encryption_key); + resources_.get().stats().add_counter("read_enumeration_size", tile.size()); + + Deserializer deserializer(tile.data(), tile.size()); + auto enum_ptr = Enumeration::deserialize(deserializer); + + schema->store_enumeration(enum_ptr); + + return enum_ptr; +} + const URI& ArrayDirectory::uri() const { return uri_; } @@ -1140,10 +1164,12 @@ Status ArrayDirectory::compute_array_schema_uris( if (!array_schema_dir_uris.empty()) { array_schema_uris_.reserve( array_schema_uris_.size() + array_schema_dir_uris.size()); - std::copy( - array_schema_dir_uris.begin(), - array_schema_dir_uris.end(), - std::back_inserter(array_schema_uris_)); + for (auto& uri : array_schema_dir_uris) { + if (uri.last_path_part() == constants::array_enumerations_dir_name) { + continue; + } + array_schema_uris_.push_back(uri); + } } return Status::Ok(); diff --git a/tiledb/sm/array/array_directory.h b/tiledb/sm/array/array_directory.h index c4e245314f46..b0e479e297ce 100644 --- a/tiledb/sm/array/array_directory.h +++ b/tiledb/sm/array/array_directory.h @@ -385,6 +385,19 @@ class ArrayDirectory { std::unordered_map> load_all_array_schemas(const EncryptionKey& encryption_key) const; + /** + * Load an enumeration from schema with the given name. + * + * @param schema The ArraySchema that references the enumeration name. + * @param enumeration_name The name of the enumeration to load. + * @param encryption_key The encryption key to use. + * @return shared_ptr The loaded enumeration. + */ + shared_ptr load_enumeration( + shared_ptr schema, + const std::string& enumeration_name, + const EncryptionKey& encryption_key) const; + /** Returns the array URI. */ const URI& uri() const; diff --git a/tiledb/sm/array/test/CMakeLists.txt b/tiledb/sm/array/test/CMakeLists.txt index 3f06e3922eeb..62b222de11ea 100644 --- a/tiledb/sm/array/test/CMakeLists.txt +++ b/tiledb/sm/array/test/CMakeLists.txt @@ -28,7 +28,7 @@ include(unit_test) commence(unit_test array) this_target_sources(main.cc unit_array_directory.cc) - this_target_link_libraries(array) + this_target_link_libraries(array context_resources) conclude(unit_test) commence(unit_test consistency) diff --git a/tiledb/sm/array_schema/CMakeLists.txt b/tiledb/sm/array_schema/CMakeLists.txt index f9616f3b7092..bff3c61bb334 100644 --- a/tiledb/sm/array_schema/CMakeLists.txt +++ b/tiledb/sm/array_schema/CMakeLists.txt @@ -32,7 +32,14 @@ include(object_library) # commence(object_library attribute) this_target_sources(attribute.cc) - this_target_object_libraries(baseline buffer constants filter_pipeline range stringx) + this_target_object_libraries( + baseline + buffer + constants + filter_pipeline + range + stringx + uuid) conclude(object_library) # @@ -51,13 +58,27 @@ commence(object_library domain) this_target_object_libraries(datum dimension math) conclude(object_library) +# +# `enumeration` object library +# +commence(object_library enumeration) + this_target_sources(enumeration.cc) + this_target_object_libraries(buffer constants uuid) +conclude(object_library) + # # `array_schema` object library # commence(object_library array_schema) this_target_sources(array_schema.cc dimension_label.cc) this_target_object_libraries( - attribute domain time uri_format uuid vfs) + attribute domain enumeration time uri_format uuid vfs) conclude(object_library) +# This is linked outside the object_library scope because ContextResources +# is recompiled as part of the capi_context_stub. Including context_resources +# here like this prevents many headaches revolving around duplicate symbols +# when linking executables. +target_link_libraries(compile_array_schema PRIVATE context_resources generic_tile_io) + add_test_subdirectory() diff --git a/tiledb/sm/array_schema/array_schema.cc b/tiledb/sm/array_schema/array_schema.cc index 656f034f9466..121a5fce9439 100644 --- a/tiledb/sm/array_schema/array_schema.cc +++ b/tiledb/sm/array_schema/array_schema.cc @@ -39,6 +39,7 @@ #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/dimension_label.h" #include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/compressor.h" @@ -50,6 +51,9 @@ #include "tiledb/sm/filter/webp_filter.h" #include "tiledb/sm/misc/hilbert.h" #include "tiledb/sm/misc/tdb_time.h" +#include "tiledb/sm/misc/utils.h" +#include "tiledb/sm/tile/generic_tile_io.h" +#include "tiledb/storage_format/uri/generate_uri.h" #include "tiledb/storage_format/uri/parse_uri.h" #include @@ -120,6 +124,45 @@ ArraySchema::ArraySchema( uint64_t capacity, std::vector> attributes, std::vector> dim_label_refs, + std::unordered_map enumeration_path_map, + FilterPipeline cell_var_offsets_filters, + FilterPipeline cell_validity_filters, + FilterPipeline coords_filters) + : ArraySchema( + uri, + version, + timestamp_range, + name, + array_type, + allows_dups, + domain, + cell_order, + tile_order, + capacity, + attributes, + dim_label_refs, + {}, + enumeration_path_map, + cell_var_offsets_filters, + cell_validity_filters, + coords_filters) { +} + +ArraySchema::ArraySchema( + URI uri, + uint32_t version, + std::pair timestamp_range, + std::string name, + ArrayType array_type, + bool allows_dups, + shared_ptr domain, + Layout cell_order, + Layout tile_order, + uint64_t capacity, + std::vector> attributes, + std::vector> dim_label_refs, + std::vector> enumerations, + std::unordered_map enumeration_path_map, FilterPipeline cell_var_offsets_filters, FilterPipeline cell_validity_filters, FilterPipeline coords_filters) @@ -135,6 +178,7 @@ ArraySchema::ArraySchema( , capacity_(capacity) , attributes_(attributes) , dimension_labels_(dim_label_refs) + , enumeration_path_map_(enumeration_path_map) , cell_var_offsets_filters_(cell_var_offsets_filters) , cell_validity_filters_(cell_validity_filters) , coords_filters_(coords_filters) { @@ -160,6 +204,15 @@ ArraySchema::ArraySchema( dimension_label_map_[label->name()] = label.get(); } + for (auto& [enmr_name, enmr_uri] : enumeration_path_map_) { + (void)enmr_uri; + enumeration_map_[enmr_name] = nullptr; + } + + for (const auto& enmr : enumerations) { + enumeration_map_[enmr->name()] = enmr; + } + // Check array schema is valid. st = check_double_delta_compressor(coords_filters_); if (!st.ok()) { @@ -195,9 +248,13 @@ ArraySchema::ArraySchema(const ArraySchema& array_schema) { throw_if_not_ok(set_domain(array_schema.domain_)); + enumeration_map_ = array_schema.enumeration_map_; + enumeration_path_map_ = array_schema.enumeration_path_map_; + attribute_map_.clear(); - for (auto attr : array_schema.attributes_) + for (auto attr : array_schema.attributes_) { throw_if_not_ok(add_attribute(attr, false)); + } // Create dimension label map for (const auto& label : array_schema.dimension_labels_) { @@ -384,6 +441,29 @@ void ArraySchema::check_webp_filter() const { } } +void ArraySchema::check_enumerations() const { + for (auto& attr : attributes_) { + auto enmr_name = attr->get_enumeration_name(); + if (!enmr_name.has_value()) { + continue; + } + + auto enmr = get_enumeration(enmr_name.value()); + if (enmr == nullptr) { + throw ArraySchemaStatusException("Cannot add attribute referencing " + "enumeration '" + enmr_name.value() + + "' as the enumeration has not been loaded."); + } + + if (datatype_max_integral_value(attr->type()) < enmr->elem_count()) { + throw ArraySchemaStatusException("Unable to use enumeration '" + + enmr_name.value() + "' for attribute '" + attr->name() + + "' because the attribute's type is not large enough to represent " + "all enumeration values."); + } + } +} + Status ArraySchema::check() const { if (domain_ == nullptr) return LOG_STATUS( @@ -423,6 +503,7 @@ Status ArraySchema::check() const { RETURN_NOT_OK(check_string_compressor(coords_filters())); check_attribute_dimension_label_names(); check_webp_filter(); + check_enumerations(); // Check for ordered attributes on sparse arrays and arrays with more than one // dimension. @@ -768,6 +849,18 @@ void ArraySchema::serialize(Serializer& serializer) const { for (auto& label : dimension_labels_) { label->serialize(serializer, version); } + + // Write Enumeration path map + auto enmr_num = utils::datatype::safe_integral_cast( + enumeration_map_.size()); + + serializer.write(enmr_num); + for (auto& [enmr_name, enmr_uri] : enumeration_path_map_) { + serializer.write(enmr_name.size()); + serializer.write(enmr_name.data(), enmr_name.size()); + serializer.write(enmr_uri.size()); + serializer.write(enmr_uri.data(), enmr_uri.size()); + } } Layout ArraySchema::tile_order() const { @@ -844,6 +937,34 @@ Status ArraySchema::add_attribute( return LOG_STATUS(Status_ArraySchemaError(msg)); } + auto enmr_name = attr->get_enumeration_name(); + if (enmr_name.has_value()) { + // The referenced enumeration must exist when the attribut is added + auto iter = enumeration_map_.find(enmr_name.value()); + if (iter == enumeration_map_.end()) { + std::string msg = + "Cannot add attribute; Attribute refers to an " + "unknown enumeration named '" + + enmr_name.value() + "'."; + return LOG_STATUS(Status_ArraySchemaError(msg)); + } + + // This attribute must have an integral datatype to support Enumerations + if (!datatype_is_integer(attr->type()) || attr->type() == Datatype::BLOB) { + std::string msg = "Unable to use enumeration with attribute '" + + attr->name() + "', attribute must have an integral data type, " + "not " + datatype_str(attr->type()); + return LOG_STATUS(Status_ArraySchemaError(msg)); + } + + // The attribute must have a cell_val_num of 1 + if (attr->cell_val_num() != 1) { + std::string msg = "Attributes with enumerations must have a cell_val_num " + "of 1."; + return LOG_STATUS(Status_ArraySchemaError(msg)); + } + } + // Create new attribute and potentially set a default name attributes_.emplace_back(attr); attribute_map_[attr->name()] = attr.get(); @@ -955,6 +1076,114 @@ Status ArraySchema::drop_attribute(const std::string& attr_name) { return Status::Ok(); } +void ArraySchema::add_enumeration(shared_ptr enmr) { + if (enmr == nullptr) { + throw ArraySchemaStatusException( + "Error adding enumeration. Enumeration " + "must not be nullptr."); + } + + if (enumeration_map_.find(enmr->name()) != enumeration_map_.end()) { + throw ArraySchemaStatusException( + "Error adding enumeration. Enumeration with name '" + enmr->name() + + "' already exists in this ArraySchema."); + } + + enumeration_map_[enmr->name()] = enmr; + enumeration_path_map_[enmr->name()] = enmr->path_name(); +} + +void ArraySchema::store_enumeration(shared_ptr enmr) { + if (enmr == nullptr) { + throw ArraySchemaStatusException( + "Error storing enumeration. Enumeration must not be nullptr."); + } + + auto name_iter = enumeration_map_.find(enmr->name()); + if (name_iter == enumeration_map_.end()) { + throw ArraySchemaStatusException( + "Error storing enumeration. Unknown enumeration name '" + enmr->name() + + "'."); + } + + if (name_iter->second != nullptr) { + throw ArraySchemaStatusException( + "Error storing enumeration. Enumeration named '" + enmr->name() + + "' has already been stored."); + } + + auto path_iter = enumeration_path_map_.find(enmr->name()); + if (path_iter == enumeration_path_map_.end()) { + throw ArraySchemaStatusException( + "Error storing enumeration. Missing path name map entry."); + } + + if (path_iter->second != enmr->path_name()) { + throw ArraySchemaStatusException( + "Error storing enumeration. Path name mismatch for enumeration " + "named '" + + enmr->name() + "'."); + } + + name_iter->second = enmr; +} + +std::vector ArraySchema::get_enumeration_names() const { + std::vector enmr_names; + for (auto& entry : enumeration_path_map_) { + enmr_names.emplace_back(entry.first); + } + return enmr_names; +} + +std::vector ArraySchema::get_loaded_enumeration_names() const { + std::vector enmr_names; + for (auto& entry : enumeration_map_) { + if (entry.second != nullptr) { + enmr_names.emplace_back(entry.first); + } + } + return enmr_names; +} + +bool ArraySchema::is_enumeration_loaded( + const std::string& enumeration_name) const { + auto iter = enumeration_map_.find(enumeration_name); + + if (iter == enumeration_map_.end()) { + throw ArraySchemaStatusException( + "Unable to check if unknown enumeration is loaded. No enumeration " + "named '" + + enumeration_name + "'."); + } + + return iter->second != nullptr; +} + +shared_ptr ArraySchema::get_enumeration( + const std::string& enmr_name) const { + auto iter = enumeration_map_.find(enmr_name); + if (iter == enumeration_map_.end()) { + throw ArraySchemaStatusException( + "Unable to get enumeration. Unknown enumeration named '" + enmr_name + + "'."); + } + + return iter->second; +} + +const std::string& ArraySchema::get_enumeration_path_name( + const std::string& enmr_name) const { + auto iter = enumeration_path_map_.find(enmr_name); + if (iter == enumeration_path_map_.end()) { + throw ArraySchemaStatusException( + "Unable to get enumeration path name. Unknown enumeration named '" + + enmr_name + "'."); + } + + return iter->second; +} + // #TODO Add security validation on incoming URI ArraySchema ArraySchema::deserialize( Deserializer& deserializer, const URI& uri) { @@ -1039,6 +1268,23 @@ ArraySchema ArraySchema::deserialize( } } + // Load enumeration name to path map + std::unordered_map enumeration_path_map; + if (version >= constants::enumerations_min_format_version) { + uint32_t enmr_num = deserializer.read(); + for (uint32_t i = 0; i < enmr_num; i++) { + auto enmr_name_size = deserializer.read(); + std::string enmr_name( + deserializer.get_ptr(enmr_name_size), enmr_name_size); + + auto enmr_path_size = deserializer.read(); + std::string enmr_path_name( + deserializer.get_ptr(enmr_path_size), enmr_path_size); + + enumeration_path_map[enmr_name] = enmr_path_name; + } + } + // Validate if (cell_order == Layout::HILBERT && domain->dim_num() > Hilbert::HC_MAX_DIM) { @@ -1081,6 +1327,7 @@ ArraySchema ArraySchema::deserialize( capacity, attributes, dimension_labels, + enumeration_path_map, cell_var_filters, cell_validity_filters, coords_filters); @@ -1280,6 +1527,7 @@ const std::string& ArraySchema::name() const { /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ + void ArraySchema::check_attribute_dimension_label_names() const { std::set names; // Check attribute and dimension names are unique. diff --git a/tiledb/sm/array_schema/array_schema.h b/tiledb/sm/array_schema/array_schema.h index d4a8bc64e15a..99edc664623a 100644 --- a/tiledb/sm/array_schema/array_schema.h +++ b/tiledb/sm/array_schema/array_schema.h @@ -43,6 +43,7 @@ #include "tiledb/sm/misc/constants.h" #include "tiledb/sm/misc/hilbert.h" #include "tiledb/sm/misc/uuid.h" +#include "tiledb/sm/storage_manager/context_resources.h" using namespace tiledb::common; @@ -55,6 +56,7 @@ class ConstBuffer; class Dimension; class DimensionLabel; class Domain; +class Enumeration; enum class ArrayType : uint8_t; enum class Compressor : uint8_t; @@ -108,6 +110,8 @@ class ArraySchema { * @param tile_order The tile order. * @param capacity The tile capacity for the case of sparse fragments. * @param attributes The array attributes. + * @param dimension_labels The array dimension labels. + * @param enumeration_path_map The array enumeration path map * @param cell_var_offsets_filters * The filter pipeline run on offset tiles for var-length attributes. * @param cell_validity_filters @@ -127,6 +131,47 @@ class ArraySchema { uint64_t capacity, std::vector> attributes, std::vector> dimension_labels, + std::unordered_map enumeration_path_map, + FilterPipeline cell_var_offsets_filters, + FilterPipeline cell_validity_filters, + FilterPipeline coords_filters); + + /** Constructor. + * @param uri The URI of the array schema file. + * @param version The format version of this array schema. + * @param timestamp_range The timestamp the array schema was written. + * @param name The file name of the schema in timestamp_timestamp_uuid format. + * @param array_type The array type. + * @param allows_dups True if the (sparse) array allows coordinate duplicates. + * @param domain The array domain. + * @param cell_order The cell order. + * @param tile_order The tile order. + * @param capacity The tile capacity for the case of sparse fragments. + * @param attributes The array attributes. + * @param dimension_labels The array dimension labels. + * @param enumerations The array enumerations + * @param enumeration_path_map The array enumeration path map + * @param cell_var_offsets_filters + * The filter pipeline run on offset tiles for var-length attributes. + * @param cell_validity_filters + * The filter pipeline run on validity tiles for nullable attributes. + * @param coords_filters The filter pipeline run on coordinate tiles. + **/ + ArraySchema( + URI uri, + uint32_t version, + std::pair timestamp_range, + std::string name, + ArrayType array_type, + bool allows_dups, + shared_ptr domain, + Layout cell_order, + Layout tile_order, + uint64_t capacity, + std::vector> attributes, + std::vector> dimension_labels, + std::vector> enumerations, + std::unordered_map enumeration_path_map, FilterPipeline cell_var_offsets_filters, FilterPipeline cell_validity_filters, FilterPipeline coords_filters); @@ -363,6 +408,63 @@ class ArraySchema { */ Status drop_attribute(const std::string& attr_name); + /** + * Add an Enumeration to this ArraySchema. + * + * @param enmr The enumeration to add. + */ + void add_enumeration(shared_ptr enmr); + + /** + * Store a known enumeration on this ArraySchema after the schema + * was loaded. This allows for only incuring the cost of loading an + * enumeration when it is needed. An exception is thrown f the + * Enumeration is unknown to this ArraySchema. + * + * @param enmr The Enumeration to store. + */ + void store_enumeration(shared_ptr enmr); + + /** + * Get a vector of Enumeration names. + * + * @return A vector of enumeration names. + */ + std::vector get_enumeration_names() const; + + /** + * Get a vector of loaded Enumeration names. + * + * @return A vector of loaded enumeration names. + */ + std::vector get_loaded_enumeration_names() const; + + /** + * Check if a given enumeration has already been loaded. + * + * @param enumeration_name The name of the enumeration to check + * @return bool Whether the enumeration has been loaded or not. + */ + bool is_enumeration_loaded(const std::string& enumeration_name) const; + + /** + * Get an Enumeration by name. Throws if the attribute is unknown. + * + * @param enmr_name The name of the Enumeration. + * @return shared_ptr + */ + shared_ptr get_enumeration( + const std::string& enmr_name) const; + + /** + * Get an Enumeration's object name. Throws if the attribute is unknown. + * + * @param enmr_name The name of the Enumeration. + * @return The path name of the enumeration on disk + */ + const std::string& get_enumeration_path_name( + const std::string& enmr_name) const; + /** * It assigns values to the members of the object from the input buffer. * @@ -546,6 +648,13 @@ class ArraySchema { /** A map from the dimension label names to the label schemas. */ std::unordered_map dimension_label_map_; + /** A map of Enumeration names to Enumeration pointers. */ + std::unordered_map> + enumeration_map_; + + /** A map of Enumeration names to Enumeration URIs */ + std::unordered_map enumeration_path_map_; + /** The filter pipeline run on offset tiles for var-length attributes. */ FilterPipeline cell_var_offsets_filters_; @@ -591,6 +700,9 @@ class ArraySchema { void check_webp_filter() const; + // Check whether attributes referencing enumerations are valid. + void check_enumerations() const; + /** Clears all members. Use with caution! */ void clear(); }; diff --git a/tiledb/sm/array_schema/array_schema_evolution.cc b/tiledb/sm/array_schema/array_schema_evolution.cc index bf285733c71e..3f5fd628abcf 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.cc +++ b/tiledb/sm/array_schema/array_schema_evolution.cc @@ -35,10 +35,12 @@ #include "tiledb/common/common.h" #include "tiledb/common/heap_memory.h" #include "tiledb/common/logger.h" +#include "tiledb/common/status.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/buffer/buffer.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/compressor.h" @@ -58,6 +60,14 @@ using namespace tiledb::common; namespace tiledb { namespace sm { +/** Class for locally generated exceptions. */ +class ArraySchemaEvolutionException : public StatusException { + public: + explicit ArraySchemaEvolutionException(const std::string& msg) + : StatusException("ArraySchemaEvolution", msg) { + } +}; + /* ****************************** */ /* CONSTRUCTORS & DESTRUCTORS */ /* ****************************** */ @@ -65,6 +75,17 @@ namespace sm { ArraySchemaEvolution::ArraySchemaEvolution() { } +ArraySchemaEvolution::ArraySchemaEvolution( + std::unordered_map> attrs_to_add, + std::unordered_map> enmrs_to_add, + std::unordered_set attrs_to_drop, + std::pair timestamp_range) + : attributes_to_add_map_(attrs_to_add) + , enumerations_to_add_map_(enmrs_to_add) + , attributes_to_drop_(attrs_to_drop) + , timestamp_range_(timestamp_range) { +} + ArraySchemaEvolution::~ArraySchemaEvolution() { clear(); } @@ -73,30 +94,33 @@ ArraySchemaEvolution::~ArraySchemaEvolution() { /* API */ /* ****************************** */ -tuple>> -ArraySchemaEvolution::evolve_schema( +shared_ptr ArraySchemaEvolution::evolve_schema( const shared_ptr& orig_schema) { std::lock_guard lock(mtx_); if (orig_schema == nullptr) { - return { - LOG_STATUS(Status_ArraySchemaEvolutionError( - "Cannot evolve schema; Input array schema is null")), - nullopt}; + throw ArraySchemaEvolutionException( + "Cannot evolve schema; Input array schema is null"); } auto schema = make_shared(HERE(), *(orig_schema.get())); + // Add enumerations. Must be done before attributes so that any attributes + // referencing enumerations won't fail to be added. + for (auto& enmr : enumerations_to_add_map_) { + schema->add_enumeration(enmr.second); + } + // Add attributes. for (auto& attr : attributes_to_add_map_) { - RETURN_NOT_OK_TUPLE(schema->add_attribute(attr.second, false), nullopt); + throw_if_not_ok(schema->add_attribute(attr.second, false)); } // Drop attributes. for (auto& attr_name : attributes_to_drop_) { bool has_attr = false; - RETURN_NOT_OK_TUPLE(schema->has_attribute(attr_name, &has_attr), nullopt); + throw_if_not_ok(schema->has_attribute(attr_name, &has_attr)); if (has_attr) { - RETURN_NOT_OK_TUPLE(schema->drop_attribute(attr_name), nullopt); + throw_if_not_ok(schema->drop_attribute(attr_name)); } } @@ -104,28 +128,27 @@ ArraySchemaEvolution::evolve_schema( // Set timestamp, if specified if (std::get<0>(timestamp_range_) != 0) { - RETURN_NOT_OK_TUPLE( - schema.get()->set_timestamp_range(timestamp_range_), nullopt); - RETURN_NOT_OK_TUPLE(schema->generate_uri(timestamp_range_), nullopt); + throw_if_not_ok(schema.get()->set_timestamp_range(timestamp_range_)); + throw_if_not_ok(schema->generate_uri(timestamp_range_)); } else { // Generate new schema URI - RETURN_NOT_OK_TUPLE(schema->generate_uri(), nullopt); + throw_if_not_ok(schema->generate_uri()); } - return {Status::Ok(), schema}; + return schema; } -Status ArraySchemaEvolution::add_attribute(const Attribute* attr) { +void ArraySchemaEvolution::add_attribute(const Attribute* attr) { std::lock_guard lock(mtx_); // Sanity check if (attr == nullptr) - return LOG_STATUS(Status_ArraySchemaEvolutionError( - "Cannot add attribute; Input attribute is null")); + throw ArraySchemaEvolutionException( + "Cannot add attribute; Input attribute is null"); if (attributes_to_add_map_.find(attr->name()) != attributes_to_add_map_.end()) { - return LOG_STATUS(Status_ArraySchemaEvolutionError( - "Cannot add attribute; Input attribute name is already there")); + throw ArraySchemaEvolutionException( + "Cannot add attribute; Input attribute name is already there"); } // Create new attribute and potentially set a default name @@ -134,8 +157,6 @@ Status ArraySchemaEvolution::add_attribute(const Attribute* attr) { if (attributes_to_drop_.find(attr->name()) != attributes_to_drop_.end()) { attributes_to_drop_.erase(attr->name()); } - - return Status::Ok(); } std::vector ArraySchemaEvolution::attribute_names_to_add() const { @@ -157,15 +178,47 @@ const Attribute* ArraySchemaEvolution::attribute_to_add( return (it == attributes_to_add_map_.end()) ? nullptr : it->second.get(); } -Status ArraySchemaEvolution::drop_attribute(const std::string& attribute_name) { +void ArraySchemaEvolution::add_enumeration(shared_ptr enmr) { + std::lock_guard lock(mtx_); + enumerations_to_add_map_[enmr->name()] = enmr; +} + +std::vector ArraySchemaEvolution::enumeration_names_to_add() + const { + std::lock_guard lock(mtx_); + std::vector names; + names.reserve(enumerations_to_add_map_.size()); + for (auto elem : enumerations_to_add_map_) { + names.push_back(elem.first); + } + + return names; +} + +shared_ptr ArraySchemaEvolution::enumeration_to_add( + const std::string& name) const { + std::lock_guard lock(mtx_); + auto it = enumerations_to_add_map_.find(name); + + if (it == enumerations_to_add_map_.end()) { + return nullptr; + } + + return it->second; +} + +void ArraySchemaEvolution::drop_attribute(const std::string& attribute_name) { std::lock_guard lock(mtx_); attributes_to_drop_.insert(attribute_name); - if (attributes_to_add_map_.find(attribute_name) != - attributes_to_add_map_.end()) { + auto ait = attributes_to_add_map_.find(attribute_name); + if (ait != attributes_to_add_map_.end()) { // Reset the pointer and erase it - attributes_to_add_map_.erase(attribute_name); + attributes_to_add_map_.erase(ait); + } + auto eit = enumerations_to_add_map_.find(attribute_name); + if (eit != enumerations_to_add_map_.end()) { + enumerations_to_add_map_.erase(eit); } - return Status::Ok(); } std::vector ArraySchemaEvolution::attribute_names_to_drop() const { @@ -179,7 +232,7 @@ std::vector ArraySchemaEvolution::attribute_names_to_drop() const { return names; } -Status ArraySchemaEvolution::set_timestamp_range( +void ArraySchemaEvolution::set_timestamp_range( const std::pair& timestamp_range) { if (timestamp_range.first != timestamp_range.second) { throw std::runtime_error(std::string( @@ -188,7 +241,6 @@ Status ArraySchemaEvolution::set_timestamp_range( std::to_string(timestamp_range.second) + " are not equal!")); } timestamp_range_ = timestamp_range; - return Status::Ok(); } std::pair ArraySchemaEvolution::timestamp_range() const { @@ -202,6 +254,7 @@ std::pair ArraySchemaEvolution::timestamp_range() const { void ArraySchemaEvolution::clear() { attributes_to_add_map_.clear(); + enumerations_to_add_map_.clear(); attributes_to_drop_.clear(); timestamp_range_ = {0, 0}; } diff --git a/tiledb/sm/array_schema/array_schema_evolution.h b/tiledb/sm/array_schema/array_schema_evolution.h index 6ba64b754519..cdf3227207aa 100644 --- a/tiledb/sm/array_schema/array_schema_evolution.h +++ b/tiledb/sm/array_schema/array_schema_evolution.h @@ -38,7 +38,6 @@ #include #include "tiledb/common/common.h" -#include "tiledb/common/status.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/filter/filter_pipeline.h" #include "tiledb/sm/misc/constants.h" @@ -55,6 +54,7 @@ class Buffer; class ConstBuffer; class Dimension; class Domain; +class Enumeration; class ArraySchema; enum class ArrayType : uint8_t; @@ -72,6 +72,19 @@ class ArraySchemaEvolution { /** Constructor. */ ArraySchemaEvolution(); + /** Constructor. + * @param attrs_to_add Attributes to add to the schema. + * @param enmrs_to_add Enumerations to add to the schema. + * @param attrs_to_drop Attributes to remove from the schema. + * @param timestamp_range Timestamp range to use for the new schema. + */ + ArraySchemaEvolution( + std::unordered_map> attrs_to_add, + std::unordered_map> + enmrs_to_add, + std::unordered_set attrs_to_drop, + std::pair timestamp_range); + /** Destructor. */ ~ArraySchemaEvolution(); @@ -79,16 +92,15 @@ class ArraySchemaEvolution { /* API */ /* ********************************* */ - tuple>> evolve_schema( + shared_ptr evolve_schema( const shared_ptr& orig_schema); /** * Adds an attribute, copying the input. * * @param attr The attribute to be added - * @return Status */ - Status add_attribute(const Attribute* attr); + void add_attribute(const Attribute* attr); /** Returns the names of attributes to add. */ std::vector attribute_names_to_add() const; @@ -99,19 +111,35 @@ class ArraySchemaEvolution { */ const Attribute* attribute_to_add(const std::string& name) const; + /** + * Adds an enumeration + * + * @param enmr The enumeration to add + */ + void add_enumeration(shared_ptr enmr); + + /** Returns the names of the enumerations to add. */ + std::vector enumeration_names_to_add() const; + + /** + * Returns a constant pointer to the selected enumeration or nullptr if + * it does not exist. + */ + shared_ptr enumeration_to_add( + const std::string& name) const; + /** * Drops an attribute. * - * @param attr The attribute to be dropped - * @return Status + * @param attr The attribute to be dropped. */ - Status drop_attribute(const std::string& attribute_name); + void drop_attribute(const std::string& attribute_name); /** Returns the names of attributes to drop. */ std::vector attribute_names_to_drop() const; /** Set a timestamp range for the array schema evolution */ - Status set_timestamp_range( + void set_timestamp_range( const std::pair& timestamp_range); /** Returns the timestamp range. */ @@ -126,6 +154,10 @@ class ArraySchemaEvolution { /** It maps each attribute name to the corresponding attribute object. */ std::unordered_map> attributes_to_add_map_; + /** Enumerations to add with any attribute. */ + std::unordered_map> + enumerations_to_add_map_; + /** The names of array attributes to be dropped. */ std::unordered_set attributes_to_drop_; @@ -151,4 +183,4 @@ class ArraySchemaEvolution { } // namespace sm } // namespace tiledb -#endif // TILEDB_SCHEMA_EVOLUTION_H \ No newline at end of file +#endif // TILEDB_SCHEMA_EVOLUTION_H diff --git a/tiledb/sm/array_schema/attribute.cc b/tiledb/sm/array_schema/attribute.cc index 8c6132a5c925..fc23b8d3bab1 100644 --- a/tiledb/sm/array_schema/attribute.cc +++ b/tiledb/sm/array_schema/attribute.cc @@ -38,6 +38,7 @@ #include "tiledb/sm/enums/filter_type.h" #include "tiledb/sm/filter/compression_filter.h" #include "tiledb/sm/misc/parse_argument.h" +#include "tiledb/sm/misc/uuid.h" #include "tiledb/type/range/range.h" #include @@ -115,7 +116,8 @@ Attribute::Attribute( const FilterPipeline& filter_pipeline, const ByteVecValue& fill_value, uint8_t fill_value_validity, - DataOrder order) + DataOrder order, + std::optional enumeration_name) : cell_val_num_(cell_val_num) , nullable_(nullable) , filters_(filter_pipeline) @@ -123,7 +125,8 @@ Attribute::Attribute( , type_(type) , fill_value_(fill_value) , fill_value_validity_(fill_value_validity) - , order_(order) { + , order_(order) + , enumeration_name_(enumeration_name) { } Attribute::Attribute(const Attribute* attr) { @@ -136,6 +139,7 @@ Attribute::Attribute(const Attribute* attr) { fill_value_ = attr->fill_value_; fill_value_validity_ = attr->fill_value_validity_; order_ = attr->order_; + enumeration_name_ = attr->enumeration_name_; } Attribute::~Attribute() = default; @@ -204,6 +208,17 @@ Attribute Attribute::deserialize( order = data_order_from_int(deserializer.read()); } + std::optional enmr_name; + if (version >= 19) { + auto enmr_name_length = deserializer.read(); + if (enmr_name_length > 0) { + std::string enmr_name_value; + enmr_name_value.resize(enmr_name_length); + deserializer.read(enmr_name_value.data(), enmr_name_length); + enmr_name = enmr_name_value; + } + } + return Attribute( name, datatype, @@ -212,7 +227,8 @@ Attribute Attribute::deserialize( filterpipeline, fill_value, fill_value_validity, - order); + order, + enmr_name); } void Attribute::dump(FILE* out) const { @@ -299,6 +315,17 @@ void Attribute::serialize( if (version >= 17) { serializer.write(static_cast(order_)); } + + // Write enumeration URI + if (version >= 19) { + if (enumeration_name_.has_value()) { + serializer.write(enumeration_name_.value().size()); + serializer.write( + enumeration_name_.value().data(), enumeration_name_.value().size()); + } else { + serializer.write(0); + } + } } void Attribute::set_cell_val_num(unsigned int cell_val_num) { @@ -480,6 +507,19 @@ DataOrder Attribute::order() const { return order_; } +void Attribute::set_enumeration_name(std::optional enmr_name) { + if (enmr_name.has_value() && enmr_name.value().empty()) { + throw AttributeStatusException( + "Invalid enumeration name, name must " + "not be empty."); + } + enumeration_name_ = enmr_name; +} + +std::optional Attribute::get_enumeration_name() const { + return enumeration_name_; +} + /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ diff --git a/tiledb/sm/array_schema/attribute.h b/tiledb/sm/array_schema/attribute.h index 7e3e7a4dd302..368836646bc4 100644 --- a/tiledb/sm/array_schema/attribute.h +++ b/tiledb/sm/array_schema/attribute.h @@ -51,6 +51,7 @@ namespace sm { class Buffer; class ConstBuffer; +class Enumeration; enum class Compressor : uint8_t; enum class Datatype : uint8_t; @@ -110,7 +111,8 @@ class Attribute { const FilterPipeline& filter_pipeline, const ByteVecValue& fill_value, uint8_t fill_value_validity, - DataOrder order = DataOrder::UNORDERED_DATA); + DataOrder order = DataOrder::UNORDERED_DATA, + std::optional enumeration_name = nullopt); /** * Constructor. It clones the input attribute. @@ -242,6 +244,12 @@ class Attribute { static ByteVecValue default_fill_value( Datatype datatype, uint32_t cell_val_num); + /** Set an enumeration for this attribute. */ + void set_enumeration_name(std::optional enmr_name); + + /** Get the enumeration for this attribute. */ + std::optional get_enumeration_name() const; + private: /* ********************************* */ /* PRIVATE ATTRIBUTES */ @@ -271,8 +279,11 @@ class Attribute { /** The required order of the data stored in the attribute. */ DataOrder order_; + /** The name of the Enumeration to use for this attribute. */ + std::optional enumeration_name_; + /* ********************************* */ - /* PRIVATE ATTRIBUTES */ + /* PRIVATE METHODS */ /* ********************************* */ /** Sets the default fill value. */ diff --git a/tiledb/sm/array_schema/enumeration.cc b/tiledb/sm/array_schema/enumeration.cc new file mode 100644 index 000000000000..d8ce3b762660 --- /dev/null +++ b/tiledb/sm/array_schema/enumeration.cc @@ -0,0 +1,243 @@ +/** + * @file enumeration.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file implements class Enumeration. + */ + +#include + +#include "tiledb/sm/misc/uuid.h" + +#include "enumeration.h" + +namespace tiledb::sm { + +/** Class for locally generated status exceptions. */ +class EnumerationException : public StatusException { + public: + explicit EnumerationException(const std::string& msg) + : StatusException("Enumeration", msg) { + } +}; + +Enumeration::Enumeration( + const std::string& name, + const std::string& path_name, + Datatype type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) + : name_(name) + , path_name_(path_name) + , type_(type) + , cell_val_num_(cell_val_num) + , ordered_(ordered) + , data_(data_size) + , offsets_(offsets_size) { + ensure_datatype_is_valid(type); + + if (name.empty()) { + throw EnumerationException("Enumeration name must not be empty"); + } + + if (path_name_.empty()) { + std::string tmp_uuid; + throw_if_not_ok(uuid::generate_uuid(&tmp_uuid, false)); + path_name_ = + "__" + tmp_uuid + "_" + std::to_string(constants::enumerations_version); + } + + if (name.find("/") != std::string::npos) { + throw EnumerationException( + "Enumeration name must not contain path separators"); + } + + if (cell_val_num == 0) { + throw EnumerationException("Invalid cell_val_num in Enumeration"); + } + + if (data == nullptr || data_size == 0) { + throw EnumerationException("No attribute value data supplied."); + } + + if (var_size() && (offsets == nullptr || offsets_size == 0)) { + throw EnumerationException( + "Variable length datatype defined but offsets are not present"); + } else if (!var_size() && (offsets != nullptr || offsets_size > 0)) { + throw EnumerationException( + "Fixed length datatype defined but offsets are present"); + } + + if (var_size()) { + if (offsets_size % sizeof(uint64_t) != 0) { + throw EnumerationException( + "Invalid offsets size is not a multiple of sizeof(uint64_t)"); + } + auto offset_values = static_cast(offsets); + uint64_t last_offset = (offsets_size / sizeof(uint64_t)) - 1; + if (offset_values[last_offset] > data_size) { + throw EnumerationException( + "Provided data buffer size is too small for the provided offsets."); + } + } else { + if (data_size % cell_size() != 0) { + throw EnumerationException( + "Invalid data size is not a multiple of the cell size."); + } + } + + throw_if_not_ok(data_.write(data, 0, data_size)); + + if (offsets_size > 0) { + throw_if_not_ok(offsets_.write(offsets, 0, offsets_size)); + } + + generate_value_map(); +} + +shared_ptr Enumeration::deserialize( + Deserializer& deserializer) { + auto disk_version = deserializer.read(); + if (disk_version > constants::enumerations_version) { + throw EnumerationException( + "Invalid Enumeration version '" + std::to_string(disk_version) + + "' is newer than supported enumeration version '" + + std::to_string(constants::enumerations_version) + "'"); + } + + auto name_size = deserializer.read(); + std::string name(deserializer.get_ptr(name_size), name_size); + + auto path_name_size = deserializer.read(); + std::string path_name( + deserializer.get_ptr(path_name_size), path_name_size); + + auto type = deserializer.read(); + auto cell_val_num = deserializer.read(); + auto ordered = deserializer.read(); + + auto data_size = deserializer.read(); + const void* data = deserializer.get_ptr(data_size); + + uint64_t offsets_size = 0; + const void* offsets = nullptr; + + if (cell_val_num == constants::var_num) { + offsets_size = deserializer.read(); + offsets = deserializer.get_ptr(offsets_size); + } + + return create( + name, + path_name, + static_cast(type), + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size); +} + +void Enumeration::serialize(Serializer& serializer) const { + serializer.write(constants::enumerations_version); + + serializer.write(name_.size()); + serializer.write(name_.data(), name_.size()); + + serializer.write(path_name_.size()); + serializer.write(path_name_.data(), path_name_.size()); + + serializer.write(static_cast(type_)); + serializer.write(cell_val_num_); + serializer.write(ordered_); + serializer.write(data_.size()); + serializer.write(data_.data(), data_.size()); + + if (var_size()) { + serializer.write(offsets_.size()); + serializer.write(offsets_.data(), offsets_.size()); + } else { + assert(cell_val_num_ < constants::var_num); + assert(offsets_.size() == 0); + } +} + +uint64_t Enumeration::index_of(UntypedDatumView value) const { + std::string_view value_view( + static_cast(value.content()), value.size()); + + auto iter = value_map_.find(value_view); + if (iter == value_map_.end()) { + return constants::enumeration_missing_value; + } + + return iter->second; +} + +void Enumeration::generate_value_map() { + auto char_data = data_.data_as(); + if (var_size()) { + auto offsets = offsets_.data_as(); + uint64_t num_offsets = offsets_.size() / sizeof(uint64_t); + + for (uint64_t i = 0; i < num_offsets; i++) { + uint64_t length = 0; + if (i < num_offsets - 1) { + length = offsets[i + 1] - offsets[i]; + } else { + length = data_.size() - offsets[i]; + } + + auto sv = std::string_view(char_data + offsets[i], length); + add_value_to_map(sv, i); + } + } else { + uint64_t i = 0; + auto stride = cell_size(); + while (i * stride < data_.size()) { + auto sv = std::string_view(char_data + i * stride, stride); + add_value_to_map(sv, i); + i += 1; + } + } +} + +void Enumeration::add_value_to_map(std::string_view& sv, uint64_t index) { + if (value_map_.find(sv) != value_map_.end()) { + throw EnumerationException( + "Invalid duplicated value in enumeration '" + std::string(sv) + "'"); + } + value_map_[sv] = index; +} + +} // namespace tiledb::sm diff --git a/tiledb/sm/array_schema/enumeration.h b/tiledb/sm/array_schema/enumeration.h new file mode 100644 index 000000000000..e0f231944d41 --- /dev/null +++ b/tiledb/sm/array_schema/enumeration.h @@ -0,0 +1,338 @@ +/** + * @file enumeration.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file defines class Enumeration. + */ + +#ifndef TILEDB_ENUMERATION_H +#define TILEDB_ENUMERATION_H + +#include + +#include "tiledb/common/common.h" +#include "tiledb/common/types/untyped_datum.h" +#include "tiledb/sm/buffer/buffer.h" +#include "tiledb/sm/enums/datatype.h" +#include "tiledb/storage_format/serialization/serializers.h" + +namespace tiledb::sm { + +/** Defines an array enumeration */ +class Enumeration { + public: + /* ********************************* */ + /* CONSTRUCTORS & DESTRUCTORS */ + /* ********************************* */ + + /** No default constructor. Use the create factory method. */ + Enumeration() = delete; + + DISABLE_COPY(Enumeration); + DISABLE_MOVE(Enumeration); + + /** Destructor. */ + ~Enumeration() = default; + + /* ********************************* */ + /* OPERATORS */ + /* ********************************* */ + + DISABLE_COPY_ASSIGN(Enumeration); + DISABLE_MOVE_ASSIGN(Enumeration); + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Create a new Enumeration + * + * @param name The name of this Enumeration as referenced by attributes. + * @param type The datatype of the enumeration values. + * @param cell_val_num The cell_val_num of the enumeration. + * @param ordered Whether the enumeration should be considered as ordered. + * If false, prevents inequality operators in QueryConditons from + * being used with this enumeration. + * @param data A pointer to the enumerations values. + * @param data_size The length of the buffer pointed to by data. + * @param offsets If cell_var_num is constants::var_num a pointer to the + * offsets buffer. Must be null if cell_var_num is not var_num. + * @param offsets_size The size of the buffer pointed to by offsets. Must be + * zero of cell_var_num is not var_num. + * @return shared_ptr The created enumeration. + */ + static shared_ptr create( + const std::string& name, + Datatype type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) { + return create( + name, + "", + type, + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size); + } + + static shared_ptr create( + const std::string& name, + const std::string& path_name, + Datatype type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) { + struct EnableMakeShared : public Enumeration { + EnableMakeShared( + const std::string& name, + const std::string& path_name, + Datatype type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) + : Enumeration( + name, + path_name, + type, + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size) { + } + }; + return make_shared( + HERE(), + name, + path_name, + type, + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size); + } + + /** + * Deserialize an enumeration + * + * @param deserializer The deserializer to deserialize from. + * @return A new Enumeration. + */ + static shared_ptr deserialize(Deserializer& deserializer); + + /** + * Serializes the enumeration into a buffer. + * + * @param serializer The object the array schema is serialized into. + */ + void serialize(Serializer& serializer) const; + + /** + * The name of this Enumeration referenced by attributes. + * + * @return The name of this Enumeration. + */ + const std::string& name() const { + return name_; + } + + /** + * The path name for this Enumeration on disk. + * + * @return The path name of this Enumeration. + */ + const std::string& path_name() const { + return path_name_; + } + + /** + * The type of the enumeration values + * + * @return Datatype The datatype of the enumeration values. + */ + Datatype type() const { + return type_; + } + + /** + * The cell_val_num of the enumeration + * + * @return uint32_t The cell_val_num of the enumeration. + */ + uint32_t cell_val_num() const { + return cell_val_num_; + } + + /** + * Get the cell_size of the enumeration + * + * This returns constants::var_size when cell_val_num is var_num, otherwise + * it returns the cell_val_num * datatype_size(type()) + * + * @return uint64_t The cell size of this enumeration. + */ + uint64_t cell_size() const { + if (var_size()) { + return constants::var_size; + } + + return cell_val_num_ * datatype_size(type_); + } + + uint64_t elem_count() const { + if (var_size()) { + return offsets_.size() / sizeof(uint64_t); + } else { + return data_.size() / cell_size(); + } + } + + /** + * Returns whether this enumeration is variable sized. + * + * @return bool Whether this enumerations is variable sized. + */ + bool var_size() const { + return cell_val_num_ == constants::var_num; + } + + /** + * Returns whether this enumeration is considered ordered or not. + * + * If an enumeration is not ordered then inequality operators in + * QueryConditions against this Enumeration are disabled. + * + * @return bool Whether this enumeration is considered ordered. + */ + bool ordered() const { + return ordered_; + } + + /** + * Returns the data buffer + * + * @return tuple A pointer to the data buffer and the + size of the buffer pointed to. + */ + span data() const { + return {static_cast(data_.data()), data_.size()}; + } + + /** + * Returns the offsets buffer + * + * @return tuple A pointer to the offsets buffer and + * the size of the buffer pointed to. + */ + span offsets() const { + return {static_cast(offsets_.data()), offsets_.size()}; + } + + /** + * Return the index of a value in the enumeration + * + * @param data An UntypedDatumView of the value to look up. + * @return uint64_t The index of the value represented by data or + * constants::missing_enumeration_value if not found. + */ + uint64_t index_of(UntypedDatumView data) const; + + private: + /* ********************************* */ + /* PRIVATE CONSTRUCTORS */ + /* ********************************* */ + + Enumeration( + const std::string& name, + const std::string& path_name, + Datatype type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size); + + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** The name of this Enumeration stored in the enumerations directory. */ + std::string name_; + + /** The path name of this Enumeration as stored on disk. */ + std::string path_name_; + + /** The type of enumerated values */ + Datatype type_; + + /** Number of values per enumeration value */ + uint32_t cell_val_num_; + + /** A flag which enables or disables inequality comparisons */ + bool ordered_; + + /** The list of enumeration values */ + Buffer data_; + + /** The offsets of each enumeration value if var sized. */ + Buffer offsets_; + + /** Map of values to indices */ + std::unordered_map value_map_; + + /* ********************************* */ + /* PRIVATE METHODS */ + /* ********************************* */ + + /** Populate the value_map_ */ + void generate_value_map(); + + /** Add a value to value_map_ */ + void add_value_to_map(std::string_view& sv, uint64_t index); +}; + +} // namespace tiledb::sm + +#endif // TILEDB_DOMAIN_H diff --git a/tiledb/sm/array_schema/test/compile_enumeration_main.cc b/tiledb/sm/array_schema/test/compile_enumeration_main.cc new file mode 100644 index 000000000000..d35fa1cca83c --- /dev/null +++ b/tiledb/sm/array_schema/test/compile_enumeration_main.cc @@ -0,0 +1,39 @@ +/** + * @file compile_enumeration_main.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "tiledb/sm/array_schema/enumeration.h" +#include "tiledb/sm/enums/datatype.h" + +int main(int, char*[]) { + try { + tiledb::sm::Enumeration::create( + "foo", tiledb::sm::Datatype::INT32, 1, false, nullptr, 0, nullptr, 0); + } catch (...) { + } + return 0; +} diff --git a/tiledb/sm/c_api/tiledb.cc b/tiledb/sm/c_api/tiledb.cc index 5ce8f5e7caab..77d090e43a89 100644 --- a/tiledb/sm/c_api/tiledb.cc +++ b/tiledb/sm/c_api/tiledb.cc @@ -40,6 +40,7 @@ #include "tiledb/api/c_api/buffer_list/buffer_list_api_internal.h" #include "tiledb/api/c_api/config/config_api_internal.h" #include "tiledb/api/c_api/dimension/dimension_api_internal.h" +#include "tiledb/api/c_api/enumeration/enumeration_api_internal.h" #include "tiledb/api/c_api/error/error_api_internal.h" #include "tiledb/api/c_api/filter_list/filter_list_api_internal.h" #include "tiledb/api/c_api/string/string_api_internal.h" @@ -299,6 +300,36 @@ int32_t tiledb_attribute_set_cell_val_num( return TILEDB_OK; } +capi_return_t tiledb_attribute_set_enumeration_name( + tiledb_ctx_t* ctx, tiledb_attribute_t* attr, const char* enumeration_name) { + if (sanity_check(ctx, attr) == TILEDB_ERR) { + return TILEDB_ERR; + } + attr->attr_->set_enumeration_name(enumeration_name); + return TILEDB_OK; +} + +capi_return_t tiledb_attribute_get_enumeration_name( + tiledb_ctx_t* ctx, + tiledb_attribute_t* attr, + tiledb_string_handle_t** name) { + if (sanity_check(ctx, attr) == TILEDB_ERR) { + return TILEDB_ERR; + } + + ensure_output_pointer_is_valid(name); + + auto enmr_name = attr->attr_->get_enumeration_name(); + if (!enmr_name.has_value()) { + *name = nullptr; + return TILEDB_OK; + } + + *name = tiledb_string_handle_t::make_handle(enmr_name.value()); + + return TILEDB_OK; +} + int32_t tiledb_attribute_get_name( tiledb_ctx_t* ctx, const tiledb_attribute_t* attr, const char** name) { if (sanity_check(ctx) == TILEDB_ERR || sanity_check(ctx, attr) == TILEDB_ERR) @@ -725,6 +756,21 @@ int32_t tiledb_array_schema_timestamp_range( return TILEDB_OK; } +TILEDB_EXPORT int32_t tiledb_array_schema_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_enumeration_t* enumeration) { + if (sanity_check(ctx, array_schema) == TILEDB_ERR) { + return TILEDB_ERR; + } + + api::ensure_enumeration_is_valid(enumeration); + + array_schema->array_schema_->add_enumeration(enumeration->copy()); + + return TILEDB_OK; +} + int32_t tiledb_array_schema_set_coords_filter_list( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, @@ -1270,12 +1316,8 @@ int32_t tiledb_array_schema_evolution_add_attribute( sanity_check(ctx, attr) == TILEDB_ERR) return TILEDB_ERR; - throw_if_not_ok( - array_schema_evolution->array_schema_evolution_->add_attribute( - attr->attr_)); - return TILEDB_OK; + array_schema_evolution->array_schema_evolution_->add_attribute(attr->attr_); - // Success return TILEDB_OK; } @@ -1287,9 +1329,8 @@ int32_t tiledb_array_schema_evolution_drop_attribute( sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) return TILEDB_ERR; - throw_if_not_ok( - array_schema_evolution->array_schema_evolution_->drop_attribute( - attribute_name)); + array_schema_evolution->array_schema_evolution_->drop_attribute( + attribute_name); return TILEDB_OK; } @@ -1302,12 +1343,8 @@ int32_t tiledb_array_schema_evolution_set_timestamp_range( sanity_check(ctx, array_schema_evolution) == TILEDB_ERR) return TILEDB_ERR; - throw_if_not_ok( - array_schema_evolution->array_schema_evolution_->set_timestamp_range( - {lo, hi})); - return TILEDB_OK; - - // Success + array_schema_evolution->array_schema_evolution_->set_timestamp_range( + {lo, hi}); return TILEDB_OK; } @@ -3411,6 +3448,26 @@ int32_t tiledb_array_evolve( return TILEDB_OK; } +int32_t tiledb_array_get_enumeration( + tiledb_ctx_t* ctx, + const tiledb_array_t* array, + const char* attr_name, + tiledb_enumeration_t** enumeration) { + if (sanity_check(ctx, array) == TILEDB_ERR) { + return TILEDB_ERR; + } + + if (attr_name == nullptr) { + throw api::CAPIStatusException("'attr_name' must not be null"); + } + + api::ensure_output_pointer_is_valid(enumeration); + auto ptr = array->array_->get_enumeration(attr_name); + *enumeration = tiledb_enumeration_handle_t::make_handle(ptr); + + return TILEDB_OK; +} + int32_t tiledb_array_upgrade_version( tiledb_ctx_t* ctx, const char* array_uri, tiledb_config_t* config) { // Sanity Checks @@ -5540,6 +5597,22 @@ int32_t tiledb_attribute_set_cell_val_num( ctx, attr, cell_val_num); } +capi_return_t tiledb_attribute_set_enumeration_name( + tiledb_ctx_t* ctx, + tiledb_attribute_t* attr, + const char* enumeration_name) noexcept { + return api_entry( + ctx, attr, enumeration_name); +} + +capi_return_t tiledb_attribute_get_enumeration_name( + tiledb_ctx_t* ctx, + tiledb_attribute_t* attr, + tiledb_string_handle_t** name) noexcept { + return api_entry( + ctx, attr, name); +} + int32_t tiledb_attribute_get_name( tiledb_ctx_t* ctx, const tiledb_attribute_t* attr, @@ -5780,6 +5853,14 @@ int32_t tiledb_array_schema_timestamp_range( ctx, array_schema, lo, hi); } +int32_t tiledb_array_schema_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_enumeration_t* enumeration) noexcept { + return api_entry( + ctx, array_schema, enumeration); +} + int32_t tiledb_array_schema_set_coords_filter_list( tiledb_ctx_t* ctx, tiledb_array_schema_t* array_schema, @@ -6956,6 +7037,15 @@ int32_t tiledb_array_evolve( ctx, array_uri, array_schema_evolution); } +int32_t tiledb_array_get_enumeration( + tiledb_ctx_t* ctx, + const tiledb_array_t* array, + const char* attr_name, + tiledb_enumeration_t** enumeration) noexcept { + return api_entry( + ctx, array, attr_name, enumeration); +} + int32_t tiledb_array_upgrade_version( tiledb_ctx_t* ctx, const char* array_uri, diff --git a/tiledb/sm/c_api/tiledb_experimental.h b/tiledb/sm/c_api/tiledb_experimental.h index 8316351700e4..cddb407fb359 100644 --- a/tiledb/sm/c_api/tiledb_experimental.h +++ b/tiledb/sm/c_api/tiledb_experimental.h @@ -41,6 +41,7 @@ /* * API sections */ +#include "tiledb/api/c_api/enumeration/enumeration_api_experimental.h" #include "tiledb/api/c_api/group/group_api_external_experimental.h" #include "tiledb/api/c_api/query_plan/query_plan_api_external_experimental.h" #include "tiledb_dimension_label_experimental.h" @@ -210,6 +211,79 @@ TILEDB_EXPORT int32_t tiledb_array_schema_timestamp_range( uint64_t* lo, uint64_t* hi) TILEDB_NOEXCEPT; +/** + * Adds an enumeration to an array schema. + * + * **Example:** + * + * @code{.c} + * tiledb_enumeration_t* enumeration; + * tiledb_enumeration_alloc( + * ctx, + * "enumeration_name", + * TILEDB_INT64, + * 1, + * FALSE, + * data, + * data_size, + * nullptr, + * 0, + * &enumeration); + * tiledb_array_schema_add_enumeration(ctx, array_schema, enumeration); + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema The array schema. + * @param enumeration The enumeration to add with the attribute + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_array_schema_add_enumeration( + tiledb_ctx_t* ctx, + tiledb_array_schema_t* array_schema, + tiledb_enumeration_t* enumeration) TILEDB_NOEXCEPT; + +/* ********************************* */ +/* ATTRIBUTE ENUMERATIONS */ +/* ********************************* */ + +/** + * Set the enumeration name on an attribute. + * + * **Example:** + * + * @code{.c} + * tiledb_attribute_set_enumeration_name(ctx, attr, "enumeration_name"); + * @endcode + * + * @param ctx The TileDB context. + * @param attr The target attribute. + * @param enumeration_name The name of the enumeration to use for the attribute. + * @return `TILEDB_OK` for success, and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_attribute_set_enumeration_name( + tiledb_ctx_t* ctx, + tiledb_attribute_t* attr, + const char* enumeration_name) TILEDB_NOEXCEPT; + +/** + * Get the attribute's enumeration name if it has one. + * + * **Example:** + * + * @code{.c} + * tiledb_attribute_get_enumeration_name(ctx, attr, &); + * @endcode + * + * @param ctx The TileDB context. + * @param attr The target attribute. + * @param enumeration A bool indicating if the attribute has an enumeration + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT capi_return_t tiledb_attribute_get_enumeration_name( + tiledb_ctx_t* ctx, + tiledb_attribute_t* attribute, + tiledb_string_handle_t** name) TILEDB_NOEXCEPT; + /* ********************************* */ /* ARRAY */ /* ********************************* */ @@ -269,6 +343,33 @@ TILEDB_EXPORT int32_t tiledb_array_evolve( const char* array_uri, tiledb_array_schema_evolution_t* array_schema_evolution) TILEDB_NOEXCEPT; +/** + * Retrieves an attribute's enumeration given the attribute name (key). + * + * **Example:** + * + * The following retrieves the first attribute in the schema. + * + * @code{.c} + * tiledb_attribute_t* attr; + * tiledb_array_schema_get_enumeration( + * ctx, array_schema, "attr_0", &enumeration); + * // Make sure to delete the retrieved attribute in the end. + * @endcode + * + * @param ctx The TileDB context. + * @param array_schema The array schema. + * @param name The name (key) of the attribute from which to + * retrieve the enumeration. + * @param enumeration The enumeration object to retrieve. + * @return `TILEDB_OK` for success and `TILEDB_ERR` for error. + */ +TILEDB_EXPORT int32_t tiledb_array_get_enumeration( + tiledb_ctx_t* ctx, + const tiledb_array_t* array, + const char* name, + tiledb_enumeration_t** enumeration) TILEDB_NOEXCEPT; + /** * Upgrades an array to the latest format version. * diff --git a/tiledb/sm/cpp_api/array_experimental.h b/tiledb/sm/cpp_api/array_experimental.h new file mode 100644 index 000000000000..3b5b7636c590 --- /dev/null +++ b/tiledb/sm/cpp_api/array_experimental.h @@ -0,0 +1,64 @@ +/** + * @file array_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the experimental C++ API for arrays. + */ + +#ifndef TILEDB_CPP_API_ARRAY_EXPERIMENTAL_H +#define TILEDB_CPP_API_ARRAY_EXPERIMENTAL_H + +#include "array.h" +#include "enumeration_experimental.h" +#include "tiledb_experimental.h" + +#include + +namespace tiledb { +class ArrayExperimental { + public: + /** + * Get the Enumeration from the attribute with name `attr_name`. + * + * @param ctx The context to use. + * @param array The array containing the attribute. + * @param attr_name The name of the attribute to get the enumeration from. + * @return Enumeration The enumeration. + */ + static Enumeration get_enumeration( + const Context& ctx, const Array& array, const std::string& attr_name) { + tiledb_enumeration_t* enmr; + ctx.handle_error(tiledb_array_get_enumeration( + ctx.ptr().get(), array.ptr().get(), attr_name.c_str(), &enmr)); + return Enumeration(ctx, enmr); + } +}; + +} // namespace tiledb + +#endif diff --git a/tiledb/sm/cpp_api/array_schema_experimental.h b/tiledb/sm/cpp_api/array_schema_experimental.h index e712860126a6..4f6a746bd3f0 100644 --- a/tiledb/sm/cpp_api/array_schema_experimental.h +++ b/tiledb/sm/cpp_api/array_schema_experimental.h @@ -35,6 +35,7 @@ #include "array_schema.h" #include "dimension_label_experimental.h" +#include "enumeration_experimental.h" #include "filter_list.h" #include "tiledb_experimental.h" @@ -154,6 +155,21 @@ class ArraySchemaExperimental { ctx.ptr().get(), array_schema.ptr().get(), name.c_str(), &dim_label)); return DimensionLabel(ctx, dim_label); } + + /** + * Add an enumeration to the array schema. + * + * @param ctx TileDB context. + * @param array_schema Target array schema. + * @param enmr The enumeration to add. + */ + static void add_enumeration( + const Context& ctx, + const ArraySchema& array_schema, + const Enumeration& enmr) { + ctx.handle_error(tiledb_array_schema_add_enumeration( + ctx.ptr().get(), array_schema.ptr().get(), enmr.ptr().get())); + } }; } // namespace tiledb diff --git a/tiledb/sm/cpp_api/attribute_experimental.h b/tiledb/sm/cpp_api/attribute_experimental.h new file mode 100644 index 000000000000..5190633ccc1a --- /dev/null +++ b/tiledb/sm/cpp_api/attribute_experimental.h @@ -0,0 +1,94 @@ +/** + * @file attribute_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the experimental C++ API for attributes. + */ + +#ifndef TILEDB_CPP_API_ATTRIBUTE_EXPERIMENTAL_H +#define TILEDB_CPP_API_ATTRIBUTE_EXPERIMENTAL_H + +#include "attribute.h" +#include "context.h" +#include "enumeration_experimental.h" + +#include + +namespace tiledb { +class AttributeExperimental { + public: + /** + * Return whether an attribute has an enumeration + * + * @param ctx TileDB context. + * @param attribute Target attribute. + * @param enumeration_name The name of the enumeration to use. + */ + static void set_enumeration_name( + const Context& ctx, + Attribute& attribute, + const std::string& enumeration_name) { + ctx.handle_error(tiledb_attribute_set_enumeration_name( + ctx.ptr().get(), attribute.ptr().get(), enumeration_name.c_str())); + } + + /** + * Get the attribute's enumeration name. + * + * @param ctx TileDB context. + * @param attr Target attribute. + * @return std::optional The enumeration name if one exists. + */ + static std::optional get_enumeration_name( + const Context& ctx, Attribute& attribute) { + // Get the enumeration name as a string handle + tiledb_string_handle_t* enmr_name; + tiledb_ctx_t* c_ctx = ctx.ptr().get(); + ctx.handle_error(tiledb_attribute_get_enumeration_name( + c_ctx, attribute.ptr().get(), &enmr_name)); + + if (enmr_name == nullptr) { + return std::nullopt; + } + + // Convert string handle to a std::string + const char* name_ptr; + size_t name_len; + ctx.handle_error(tiledb_string_view(enmr_name, &name_ptr, &name_len)); + std::string ret(name_ptr, name_len); + + // Release the string handle + ctx.handle_error(tiledb_string_free(&enmr_name)); + + return ret; + } +}; + +} // namespace tiledb + +#endif diff --git a/tiledb/sm/cpp_api/deleter.h b/tiledb/sm/cpp_api/deleter.h index c72d8ef71a1f..61f9a5d2c92f 100644 --- a/tiledb/sm/cpp_api/deleter.h +++ b/tiledb/sm/cpp_api/deleter.h @@ -111,6 +111,10 @@ class Deleter { tiledb_domain_free(&p); } + void operator()(tiledb_enumeration_t* p) const { + tiledb_enumeration_free(&p); + } + void operator()(tiledb_vfs_t* p) const { tiledb_vfs_free(&p); } diff --git a/tiledb/sm/cpp_api/enumeration_experimental.h b/tiledb/sm/cpp_api/enumeration_experimental.h new file mode 100644 index 000000000000..52ac47504333 --- /dev/null +++ b/tiledb/sm/cpp_api/enumeration_experimental.h @@ -0,0 +1,355 @@ +/** + * @file enumeration_experimental.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2023 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file declares the C++ API for the TileDB Enumeration class. + */ + +#ifndef TILEDB_CPP_API_ENUMERATION_EXPERIMENTAL_H +#define TILEDB_CPP_API_ENUMERATION_EXPERIMENTAL_H + +#include "context.h" +#include "tiledb.h" +#include "tiledb_experimental.h" + +namespace tiledb { + +class Enumeration { + public: + /** + * Create an enumeration by wrapping a pointer allocated by the C API + * + * @param ctx The Context to use. + * @param enmr The tiledb_enumeration_t allocated by the C API. + */ + Enumeration(const Context& ctx, tiledb_enumeration_t* enmr) + : ctx_(ctx) { + enumeration_ = std::shared_ptr(enmr, deleter_); + } + + Enumeration(const Enumeration&) = default; + Enumeration(Enumeration&&) = default; + Enumeration& operator=(const Enumeration&) = default; + Enumeration& operator=(Enumeration&&) = default; + + /** Destructor. */ + ~Enumeration() = default; + + /* ********************************* */ + /* API */ + /* ********************************* */ + + /** Get the C TileDB context object. + * + * @return shared_ptr + */ + std::shared_ptr ptr() const { + return enumeration_; + } + + /** + * Get the name of the enumeration + * + * @return std::string The name of the enumeration. + */ + std::string name() const { + // Get the enumeration name as a string handle + tiledb_string_handle_t* enmr_name; + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + ctx_.get().handle_error( + tiledb_enumeration_get_name(c_ctx, enumeration_.get(), &enmr_name)); + + // Convert string handle to a std::string + const char* name_ptr; + size_t name_len; + ctx_.get().handle_error( + tiledb_string_view(enmr_name, &name_ptr, &name_len)); + std::string ret(name_ptr, name_len); + + // Release the string handle + ctx_.get().handle_error(tiledb_string_free(&enmr_name)); + + return ret; + } + + /** + * Get the type of the enumeration + * + * @return tiledb_datatype_t The datatype of the enumeration values. + */ + tiledb_datatype_t type() const { + tiledb_datatype_t ret; + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + ctx_.get().handle_error( + tiledb_enumeration_get_type(c_ctx, enumeration_.get(), &ret)); + return ret; + } + + /** + * Get the cell_val_num of the enumeration + * + * @return uint32_t The cell_val_num of the enumeration. + */ + uint32_t cell_val_num() const { + uint32_t ret; + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + ctx_.get().handle_error( + tiledb_enumeration_get_cell_val_num(c_ctx, enumeration_.get(), &ret)); + return ret; + } + + /** + * Get whether this enumeration is considered ordered. + * + * If an enumeration is not considered ordered inequality operators are + * disabled in QueryConditions applied against the enumeration values. + * + * @return bool Whether the enumeration is considered ordered. + */ + bool ordered() const { + int is_ordered; + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + ctx_.get().handle_error( + tiledb_enumeration_get_ordered(c_ctx, enumeration_.get(), &is_ordered)); + return is_ordered ? true : false; + } + + /** + * Convert the enumeration values into a vector + * + * @param vec The vector that will store the values of the enumeration. + */ + template + std::vector, T>> + as_vector() { + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + + const void* data; + uint64_t data_size; + ctx_.get().handle_error(tiledb_enumeration_get_data( + c_ctx, enumeration_.get(), &data, &data_size)); + + const T* elems = static_cast(data); + size_t count = data_size / sizeof(T); + + std::vector ret; + ret.reserve(count); + for (size_t i = 0; i < count; i++) { + ret.push_back(elems[i]); + } + + return ret; + } + + /** + * Convert the enumeration values into a vector of std::string values + * + * @param vec The vector used to return the string data. + */ + template + std::vector, T>> + as_vector() { + tiledb_ctx_t* c_ctx = ctx_.get().ptr().get(); + + const void* data; + uint64_t data_size; + ctx_.get().handle_error(tiledb_enumeration_get_data( + c_ctx, enumeration_.get(), &data, &data_size)); + + const void* offsets; + uint64_t offsets_size; + ctx_.get().handle_error(tiledb_enumeration_get_offsets( + c_ctx, enumeration_.get(), &offsets, &offsets_size)); + + const char* str_data = static_cast(data); + const uint64_t* elems = static_cast(offsets); + size_t count = offsets_size / sizeof(uint64_t); + + std::vector ret; + ret.reserve(count); + for (size_t i = 0; i < count; i++) { + uint64_t len; + if (i + 1 < count) { + len = elems[i + 1] - elems[i]; + } else { + len = data_size - elems[i]; + } + ret.emplace_back(str_data + elems[i], len); + } + + return ret; + } + + /* ********************************* */ + /* STATIC FUNCTIONS */ + /* ********************************* */ + + /** + * Create an enumeration from a vector of trivial values (i.e., int's or other + * integral or floating point values) + * + * @param ctx The context to use. + * @param values The list of values to use for this enumeration. + * @param ordered Whether or not to consider this enumeration ordered. + * @param type The datatype of the enumeration values. This is automatically + * deduced if not provided. + */ + template * = nullptr> + static Enumeration create( + const Context& ctx, + const std::string& name, + std::vector& values, + bool ordered = false, + tiledb_datatype_t type = static_cast(255)) { + using DataT = impl::TypeHandler; + + if (type == static_cast(255)) { + type = DataT::tiledb_type; + } + + return create( + ctx, + name, + type, + DataT::tiledb_num, + ordered, + values.data(), + values.size() * sizeof(T), + nullptr, + 0); + } + + /** + * Create an enumeration from a vector of strings + * + * @param ctx The context to use. + * @param values The vector of values for the enumeration. + * @param ordered Whether to consider the enumerationv alues as ordered. + * @param type The datatype of the enumeration values. This is automatically + * deduced if not provided. However, this can be used to override the + * deduced type if need be. For instance, TILEDB_STRING_ASCII is the + * default type for strings but TILEDB_STRING_UTF8 can be specified. + */ + template * = nullptr> + static Enumeration create( + const Context& ctx, + const std::string& name, + std::vector>& values, + bool ordered = false, + tiledb_datatype_t type = static_cast(255)) { + using DataT = impl::TypeHandler; + static_assert( + DataT::tiledb_num == 1, "Enumeration string values cannot be compound"); + + if (type == static_cast(255)) { + type = DataT::tiledb_type; + } + + uint64_t total_size = 0; + for (auto v : values) { + total_size += v.size() * sizeof(T); + } + + uint8_t data[total_size]; + std::vector offsets; + offsets.reserve(values.size()); + uint64_t curr_offset = 0; + + for (auto v : values) { + std::memcpy(data + curr_offset, v.data(), v.size() * sizeof(T)); + offsets.push_back(curr_offset); + curr_offset += v.size() * sizeof(T); + } + + return create( + ctx, + name, + type, + TILEDB_VAR_NUM, + ordered, + data, + total_size, + offsets.data(), + offsets.size() * sizeof(uint64_t)); + } + + /** + * Create an enumeration + * + * @param ctx The context to use. + * @param type The datatype of the enumeration values. + * @param cell_val_num The cell_val_num of the enumeration. + * @param ordered Whether this enumeration should be considered ordered. + * @param data A pointer to a buffer of values for this enumeration. + * @param data_size The size of the buffer pointed to by data. + * @param offsets A pointer to the offsets buffer if required. + * @param offsets_size The size of the buffer pointed to by offsets. + */ + static Enumeration create( + const Context& ctx, + const std::string& name, + tiledb_datatype_t type, + uint32_t cell_val_num, + bool ordered, + const void* data, + uint64_t data_size, + const void* offsets, + uint64_t offsets_size) { + tiledb_enumeration_t* enumeration; + ctx.handle_error(tiledb_enumeration_alloc( + ctx.ptr().get(), + name.c_str(), + type, + cell_val_num, + ordered, + data, + data_size, + offsets, + offsets_size, + &enumeration)); + return Enumeration(ctx, enumeration); + } + + private: + /* ********************************* */ + /* PRIVATE ATTRIBUTES */ + /* ********************************* */ + + /** The TileDB context. */ + std::reference_wrapper ctx_; + + /** Deleter wrapper. */ + impl::Deleter deleter_; + + /** Pointer to the TileDB C Enumeration object. */ + std::shared_ptr enumeration_; +}; + +} // namespace tiledb + +#endif // TILEDB_CPP_API_ENUMERATION_EXPERIMENTAL_H diff --git a/tiledb/sm/cpp_api/tiledb_experimental b/tiledb/sm/cpp_api/tiledb_experimental index df23d84ae518..0e213c4f0ec1 100644 --- a/tiledb/sm/cpp_api/tiledb_experimental +++ b/tiledb/sm/cpp_api/tiledb_experimental @@ -33,9 +33,12 @@ #ifndef TILEDB_EXPERIMENTAL_CPP_H #define TILEDB_EXPERIMENTAL_CPP_H +#include "array_experimental.h" #include "array_schema_evolution.h" #include "array_schema_experimental.h" +#include "attribute_experimental.h" #include "dimension_label_experimental.h" +#include "enumeration_experimental.h" #include "consolidation_plan_experimental.h" #include "group_experimental.h" #include "query_experimental.h" diff --git a/tiledb/sm/enums/datatype.h b/tiledb/sm/enums/datatype.h index c2050d6d674b..9c14c153939b 100644 --- a/tiledb/sm/enums/datatype.h +++ b/tiledb/sm/enums/datatype.h @@ -332,6 +332,39 @@ inline bool datatype_is_integer(Datatype type) { type == Datatype::INT64 || type == Datatype::UINT64); } +inline uint64_t datatype_max_integral_value(Datatype type) { + // There's a lack of symmetry between this function and datatype_is_integer + // because the latter considers Datatype::BLOB an integer even though the + // C++ docs specifically state that it is not an arithmetic type. + // + // From https://en.cppreference.com/w/cpp/types/byte + // "it is not a character type and is not an arithmetic type" + // + switch (type) { + case Datatype::BOOL: + return static_cast(std::numeric_limits::max()); + case Datatype::INT8: + return static_cast(std::numeric_limits::max()); + case Datatype::UINT8: + return static_cast(std::numeric_limits::max()); + case Datatype::INT16: + return static_cast(std::numeric_limits::max()); + case Datatype::UINT16: + return static_cast(std::numeric_limits::max()); + case Datatype::INT32: + return static_cast(std::numeric_limits::max()); + case Datatype::UINT32: + return static_cast(std::numeric_limits::max()); + case Datatype::INT64: + return static_cast(std::numeric_limits::max()); + case Datatype::UINT64: + return static_cast(std::numeric_limits::max()); + default: + throw std::runtime_error( + "Datatype (" + datatype_str(type) + ") is not integral."); + } +} + /** Returns true if the input datatype is a real type. */ inline bool datatype_is_real(Datatype type) { return (type == Datatype::FLOAT32 || type == Datatype::FLOAT64); diff --git a/tiledb/sm/misc/constants.cc b/tiledb/sm/misc/constants.cc index 17e95b511ee9..936c368d542c 100644 --- a/tiledb/sm/misc/constants.cc +++ b/tiledb/sm/misc/constants.cc @@ -93,6 +93,9 @@ const std::string fragment_metadata_filename = "__fragment_metadata.tdb"; /** The array dimension labels directory name. */ const std::string array_dimension_labels_dir_name = "__labels"; +/** The array enumerations directory name. */ +const std::string array_enumerations_dir_name = "__enumerations"; + /** The default tile capacity. */ const uint64_t capacity = 10000; @@ -213,6 +216,9 @@ const uint32_t empty_ucs4 = 0; /** The special value for an empty ANY. */ const uint8_t empty_any = 0; +/** The return value for missing entries in an Enumeration */ +const uint64_t enumeration_missing_value = std::numeric_limits::max(); + /** The file suffix used in TileDB. */ const std::string file_suffix = ".tdb"; @@ -662,6 +668,12 @@ const format_version_t deletes_min_version = 16; /** The lowest version supported for updates. */ const format_version_t updates_min_version = 16; +/** The lowest version supported format version for enumerations. */ +const format_version_t enumerations_min_format_version = 19; + +/** The current enumerations version. */ +const format_version_t enumerations_version = 0; + /** The maximum size of a tile chunk (unit of compression) in bytes. */ const uint64_t max_tile_chunk_size = 64 * 1024; diff --git a/tiledb/sm/misc/constants.h b/tiledb/sm/misc/constants.h index d2796f86fe58..5b690dfc3fbf 100644 --- a/tiledb/sm/misc/constants.h +++ b/tiledb/sm/misc/constants.h @@ -82,6 +82,9 @@ extern const std::string array_commits_dir_name; /** The array dimension labels directory name. */ extern const std::string array_dimension_labels_dir_name; +/** The array enumerations directory name. */ +extern const std::string array_enumerations_dir_name; + /** The default tile capacity. */ extern const uint64_t capacity; @@ -199,6 +202,9 @@ extern const uint32_t empty_ucs4; /** The special value for an empty ANY. */ extern const uint8_t empty_any; +/** The return value for missing entries in an Enumeration */ +extern const uint64_t enumeration_missing_value; + /** The file suffix used in TileDB. */ extern const std::string file_suffix; @@ -638,6 +644,12 @@ extern const format_version_t deletes_min_version; /** The lowest version supported for updates. */ extern const format_version_t updates_min_version; +/** The lowest version supported for enumerations. */ +extern const format_version_t enumerations_min_format_version; + +/** The current Enumerations version. */ +extern const format_version_t enumerations_version; + /** The maximum size of a tile chunk (unit of compression) in bytes. */ extern const uint64_t max_tile_chunk_size; @@ -721,7 +733,6 @@ extern const uint64_t s3_min_multipart_part_size; * global order writes intermediate chunks */ extern const std::string s3_multipart_buffering_dirname; - } // namespace constants } // namespace sm diff --git a/tiledb/sm/misc/utils.cc b/tiledb/sm/misc/utils.cc index a3ac5a6aa967..3124e4e0af1a 100644 --- a/tiledb/sm/misc/utils.cc +++ b/tiledb/sm/misc/utils.cc @@ -33,6 +33,7 @@ #include "tiledb/sm/misc/utils.h" #include "tiledb/common/logger.h" +#include "tiledb/common/unreachable.h" #include "tiledb/sm/enums/datatype.h" #include "tiledb/sm/filesystem/uri.h" #include "tiledb/sm/misc/constants.h" diff --git a/tiledb/sm/misc/utils.h b/tiledb/sm/misc/utils.h index 48918e08ad29..0749938725e2 100644 --- a/tiledb/sm/misc/utils.h +++ b/tiledb/sm/misc/utils.h @@ -43,6 +43,9 @@ #include "constants.h" #include "tiledb/common/logger_public.h" #include "tiledb/common/status.h" +#include "tiledb/common/unreachable.h" +#include "tiledb/sm/enums/datatype.h" +#include "tiledb/sm/misc/types.h" namespace tiledb { namespace sm { @@ -52,7 +55,6 @@ using namespace tiledb::common; class Posix; class URI; -enum class Datatype : uint8_t; enum class SerializationType : uint8_t; namespace utils { @@ -72,6 +74,105 @@ namespace datatype { template Status check_template_type_to_datatype(Datatype datatype); +/** + * Safely convert integral values between different bit widths checking for + * invalid conversions. The basic checks are just to make sure that the + * conversion is roundtrip-able without an intermediate change in sign. + * + * This would likely be significantly faster if I were to take the time and + * write a cross platform interface for '__builtin_clzll'. However, the current + * needs don't need the absolute speed as this is currently only used once + * per Enumeration attribute per query (as opposed, once per query condition + * comparison or something crazy). + * + * The algorithm here might seem a bit odd here when we could instead be + * using std::numeric_limits::{min,max}() which is what I started out + * using. However, when Target and Source don't agree on signed-ness the + * compiler generates warnings about comparing differently signed values. + * + * @param src The value to convert. + * @return The converted value. + */ +template +Target safe_integral_cast(Source src) { + using SourceLimits = std::numeric_limits; + using TargetLimits = std::numeric_limits; + + static_assert(SourceLimits::is_integer, "Source is not an integral type"); + static_assert(TargetLimits::is_integer, "Target is not an integral type"); + + Target ret = static_cast(src); + + // If it can't round trip, its an invalid cast. Note that the converse + // is not true as a sign could have changed for types of the same bit width + // but different signed-ness. + if (static_cast(ret) != src) { + throw std::invalid_argument("Invalid integral cast: Roundtrip failed"); + } + + // If the sign changed + if ((src < 0 && ret >= 0) || (src >= 0 && ret < 0)) { + throw std::invalid_argument("Invalid integral cast: Sign changed"); + } + + return ret; +} + +/** + * Use `safe_integral_cast` to convert an integral value into a specific + * Datatype stored in a ByteVecValue. + * + * @param value The value to convert. + * @param type The datatype to convert the value to. + * @param dest A ByteVecValue to store the converted value in. + */ +template +void safe_integral_cast_to_datatype( + Source value, Datatype type, ByteVecValue& dest) { + if (!datatype_is_integer(type)) { + throw std::invalid_argument("Datatype must be integral"); + } + + if (type == Datatype::BLOB) { + throw std::invalid_argument( + "Datatype::BLOB not supported in integral conversion"); + } + + switch (type) { + case Datatype::BOOL: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::INT8: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::UINT8: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::INT16: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::UINT16: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::INT32: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::UINT32: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::INT64: + dest.assign_as(safe_integral_cast(value)); + return; + case Datatype::UINT64: + dest.assign_as(safe_integral_cast(value)); + return; + default: + throw std::logic_error("Definitions of integral types are mismatched."); + } + + ::stdx::unreachable(); +} + } // namespace datatype /* ********************************* */ diff --git a/tiledb/sm/query/ast/CMakeLists.txt b/tiledb/sm/query/ast/CMakeLists.txt index 4568075fbeb7..c46c57fd7244 100644 --- a/tiledb/sm/query/ast/CMakeLists.txt +++ b/tiledb/sm/query/ast/CMakeLists.txt @@ -39,7 +39,7 @@ list(APPEND SOURCES # commence(object_library query_ast) this_target_sources(${SOURCES}) - this_target_object_libraries(array_schema baseline) + this_target_object_libraries(array_schema generic_tile_io baseline) conclude(object_library) add_test_subdirectory() diff --git a/tiledb/sm/query/ast/query_ast.cc b/tiledb/sm/query/ast/query_ast.cc index ddec36e555ea..07e5db228bb0 100644 --- a/tiledb/sm/query/ast/query_ast.cc +++ b/tiledb/sm/query/ast/query_ast.cc @@ -31,6 +31,9 @@ */ #include "query_ast.h" +#include "tiledb/common/unreachable.h" +#include "tiledb/sm/array_schema/enumeration.h" +#include "tiledb/sm/misc/utils.h" using namespace tiledb::common; @@ -60,10 +63,65 @@ void ASTNodeVal::get_field_names( field_name_set.insert(field_name_); } +void ASTNodeVal::get_enumeration_field_names( + std::unordered_set& field_name_set) const { + if (use_enumeration_) { + field_name_set.insert(field_name_); + } +} + bool ASTNodeVal::is_backwards_compatible() const { return true; } +void ASTNodeVal::rewrite_enumeration_conditions( + const ArraySchema& array_schema) { + // This is called by the Query class before applying a query condition. This + // works by looking up each related enumeration and translating the + // condition's value to reflect the underlying index value. I.e., if the + // query condition was created with `my_attr = "foo"`, and `my_attr` is an + // attribute with an enumeration, this will replace the condition's value + // with the index value returned by `Enumeration::index_of()`. + + if (!use_enumeration_) { + return; + } + + if (!array_schema.is_attr(field_name_)) { + return; + } + + auto attr = array_schema.attribute(field_name_); + auto enmr_name = attr->get_enumeration_name(); + if (!enmr_name.has_value()) { + return; + } + + auto enumeration = array_schema.get_enumeration(enmr_name.value()); + if (!enumeration) { + throw std::logic_error( + "Missing requried enumeration for field '" + field_name_ + "'"); + } + + if (!enumeration->ordered() && + (op_ != QueryConditionOp::EQ && op_ != QueryConditionOp::NE)) { + throw std::logic_error( + "Cannot apply an inequality operator against an unordered Enumeration"); + } + + auto idx = enumeration->index_of(condition_value_view_); + auto val_size = datatype_size(attr->type()); + + condition_value_data_ = ByteVecValue(val_size); + utils::datatype::safe_integral_cast_to_datatype( + idx, attr->type(), condition_value_data_); + + condition_value_view_ = + UntypedDatumView(condition_value_data_.data(), val_size); + + use_enumeration_ = false; +} + Status ASTNodeVal::check_node_validity(const ArraySchema& array_schema) const { const uint64_t condition_value_size = condition_value_data_.size(); @@ -186,6 +244,14 @@ const QueryConditionCombinationOp& ASTNodeVal::get_combination_op() const { "value node."); } +bool ASTNodeVal::use_enumeration() const { + return use_enumeration_; +} + +void ASTNodeVal::set_use_enumeration(bool use_enumeration) { + use_enumeration_ = use_enumeration; +} + bool ASTNodeExpr::is_expr() const { return true; } @@ -205,6 +271,13 @@ void ASTNodeExpr::get_field_names( } } +void ASTNodeExpr::get_enumeration_field_names( + std::unordered_set& field_name_set) const { + for (const auto& child : nodes_) { + child->get_enumeration_field_names(field_name_set); + } +} + bool ASTNodeExpr::is_backwards_compatible() const { if (combination_op_ != QueryConditionCombinationOp::AND) { return false; @@ -217,6 +290,13 @@ bool ASTNodeExpr::is_backwards_compatible() const { return true; } +void ASTNodeExpr::rewrite_enumeration_conditions( + const ArraySchema& array_schema) { + for (auto& child : nodes_) { + child->rewrite_enumeration_conditions(array_schema); + } +} + Status ASTNodeExpr::check_node_validity(const ArraySchema& array_schema) const { // If the node is a compound expression node, ensure there are at least // two children in the node and then run a check on each child node. @@ -275,9 +355,22 @@ const QueryConditionOp& ASTNodeExpr::get_op() const { const std::vector>& ASTNodeExpr::get_children() const { return nodes_; } + const QueryConditionCombinationOp& ASTNodeExpr::get_combination_op() const { return combination_op_; } +bool ASTNodeExpr::use_enumeration() const { + throw std::runtime_error( + "ASTNodeExpr::use_enumeration: Cannot get enumeration status from " + "an AST expression node."); +} + +void ASTNodeExpr::set_use_enumeration(bool use_enumeration) { + for (auto& child : nodes_) { + child->set_use_enumeration(use_enumeration); + } +} + } // namespace sm } // namespace tiledb diff --git a/tiledb/sm/query/ast/query_ast.h b/tiledb/sm/query/ast/query_ast.h index 30a08884d905..2a7f35e23898 100644 --- a/tiledb/sm/query/ast/query_ast.h +++ b/tiledb/sm/query/ast/query_ast.h @@ -91,11 +91,19 @@ class ASTNode { * @brief Gets the set of field names from all the value nodes in the ASTNode. * * @param field_name_set The set variable the function populates. - * @return std::unordered_set& Set of the field names in the - * node. */ virtual void get_field_names( std::unordered_set& field_name_set) const = 0; + + /** + * @brief Gets the set of field names from all the value nodes that reference + * an enumerated field. + * + * @param field_name_set The set variable the function populates. + */ + virtual void get_enumeration_field_names( + std::unordered_set& field_name_set) const = 0; + /** * @brief Returns true if the AST is previously supported by previous versions * of TileDB. This means that the AST should only have AND combination ops, @@ -106,6 +114,15 @@ class ASTNode { */ virtual bool is_backwards_compatible() const = 0; + /** + * @brief Update an node value condition values that refer to enumerated + * attributes. + * + * @param array_schema The array schema with all relevant enumerations loaded. + */ + virtual void rewrite_enumeration_conditions( + const ArraySchema& array_schema) = 0; + /** * @brief Checks whether the node is valid based on the array schema of the * array that the query condition that contains this AST node will execute a @@ -176,6 +193,26 @@ class ASTNode { */ virtual const QueryConditionCombinationOp& get_combination_op() const = 0; + /** + * @brief Return whether this node's condition should be applied against + * the attributes enumerated values or the underlying index data if + * applicable for a given attribute. + * + * @return bool If true, apply this condition against the enumerated values. + */ + virtual bool use_enumeration() const = 0; + + /** + * @brief By default, a query condition is applied against an enumeration's + * values. This can be disabled to apply a given condition against the + * underlying integer data stored for the attribute by passing `false` to + * this method. + * + * @param use_enumeration A bool indicating whether this condition should be + * applied against the enumerations values or not. + */ + virtual void set_use_enumeration(bool use_enumeration) = 0; + /** * @brief Default virtual destructor. */ @@ -196,12 +233,15 @@ class ASTNodeVal : public ASTNode { * @param condition_value_size The byte size of condition_value. * @param op The relational operation between the value of the field * and `condition_value`. + * @param use_enumeration Whether or not to use an associated + * enumeration at query time. */ ASTNodeVal( const std::string& field_name, const void* const condition_value, const uint64_t condition_value_size, - const QueryConditionOp op) + const QueryConditionOp op, + bool use_enumeration = true) : field_name_(field_name) , condition_value_data_(condition_value_size) , condition_value_view_( @@ -209,7 +249,8 @@ class ASTNodeVal : public ASTNode { (void*)"" : condition_value_data_.data()), condition_value_data_.size()) - , op_(op) { + , op_(op) + , use_enumeration_(use_enumeration) { if (condition_value_view_.size() != 0) { memcpy( condition_value_data_.data(), condition_value, condition_value_size); @@ -228,7 +269,8 @@ class ASTNodeVal : public ASTNode { (void*)"" : condition_value_data_.data()), condition_value_data_.size()) - , op_(rhs.op_) { + , op_(rhs.op_) + , use_enumeration_(rhs.use_enumeration_) { } /** @@ -243,7 +285,8 @@ class ASTNodeVal : public ASTNode { (void*)"" : condition_value_data_.data()), condition_value_data_.size()) - , op_(negate_query_condition_op(rhs.op_)) { + , op_(negate_query_condition_op(rhs.op_)) + , use_enumeration_(rhs.use_enumeration_) { } /** @@ -284,12 +327,19 @@ class ASTNodeVal : public ASTNode { * @brief Gets the set of field names from all the value nodes in the ASTNode. * * @param field_name_set The set variable the function populates. - * @return std::unordered_set& Set of the field names in the - * node. */ void get_field_names( std::unordered_set& field_name_set) const override; + /** + * @brief Gets the set of field names from all the value nodes that reference + * an enumerated field. + * + * @param field_name_set The set variable the function populates. + */ + void get_enumeration_field_names( + std::unordered_set& field_name_set) const override; + /** * @brief Returns true if the AST is previously supported by previous versions * of TileDB. This means that the AST should only have AND combination ops, @@ -300,6 +350,17 @@ class ASTNodeVal : public ASTNode { */ bool is_backwards_compatible() const override; + /** + * Update any node value condition values that refer to enumerated attributes. + * For any value condition on an attribute that has an Enumeration, the + * user specified value is passed to the attribute's enumeration `index_of` + * method to replace the user provided value with the Enumeration's value + * index. + * + * @param array_schema The array schema with all relevant enumerations loaded. + */ + void rewrite_enumeration_conditions(const ArraySchema& array_schema) override; + /** * @brief Checks whether the node is valid based on the array schema of the * array that the query condition that contains this AST node will execute a @@ -370,6 +431,26 @@ class ASTNodeVal : public ASTNode { */ const QueryConditionCombinationOp& get_combination_op() const override; + /** + * @brief Return whether this node's condition should be applied against + * the attributes enumerated values or the underlying index data if + * applicable for a given attribute. + * + * @return bool If true, apply this condition against the enumerated values. + */ + bool use_enumeration() const override; + + /** + * @brief By default, a query condition is applied against an enumeration's + * values. This can be disabled to apply a given condition against the + * underlying integer data stored for the attribute by passing `false` to + * this method. + * + * @param use_enumeration A bool indicating whether this condition should be + * applied against the enumerations values or not. + */ + void set_use_enumeration(bool use_enumeration) override; + private: /** The attribute name. */ std::string field_name_; @@ -382,6 +463,9 @@ class ASTNodeVal : public ASTNode { /** The comparison operator. */ QueryConditionOp op_; + + /** Whether this condiiton applies to enumerated values if applicable */ + bool use_enumeration_; }; /** @@ -461,12 +545,19 @@ class ASTNodeExpr : public ASTNode { * @brief Gets the set of field names from all the value nodes in the ASTNode. * * @param field_name_set The set variable the function populates. - * @return std::unordered_set& Set of the field names in the - * node. */ void get_field_names( std::unordered_set& field_name_set) const override; + /** + * @brief Gets the set of field names from all the value nodes that reference + * an enumerated field. + * + * @param field_name_set The set variable the function populates. + */ + void get_enumeration_field_names( + std::unordered_set& field_name_set) const override; + /** * @brief Returns true if the AST is previously supported by previous versions * of TileDB. This means that the AST should only have AND combination ops, @@ -477,6 +568,14 @@ class ASTNodeExpr : public ASTNode { */ bool is_backwards_compatible() const override; + /** + * @brief Update an node value condition values that refer to enumerated + * attributes. + * + * @param array_schema The array schema with all relevant enumerations loaded. + */ + void rewrite_enumeration_conditions(const ArraySchema& array_schema) override; + /** * @brief Checks whether the node is valid based on the array schema of the * array that the query condition that contains this AST node will execute a @@ -547,6 +646,31 @@ class ASTNodeExpr : public ASTNode { */ const QueryConditionCombinationOp& get_combination_op() const override; + /** + * @brief Return whether this node's condition should be applied against + * the attributes enumerated values or the underlying index data if + * applicable for a given attribute. + * + * This method always throws when called on an expression node. + * + * @return bool If true, apply this condition against the enumerated values. + */ + bool use_enumeration() const override; + + /** + * @brief By default, a query condition is applied against an enumeration's + * values. This can be disabled to apply a given condition against the + * underlying integer data stored for the attribute by passing `false` to + * this method. + * + * When called on an expression node this value is recursively applied + * against all value nodes in the AST. + * + * @param use_enumeration A bool indicating whether this condition should be + * applied against the enumerations values or not. + */ + void set_use_enumeration(bool use_enumeration) override; + private: /** The node list **/ std::vector> nodes_; @@ -558,4 +682,4 @@ class ASTNodeExpr : public ASTNode { } // namespace sm } // namespace tiledb -#endif // TILEDB_QUERY_AST_H \ No newline at end of file +#endif // TILEDB_QUERY_AST_H diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 6332354a0414..97c9b86b006b 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -455,13 +455,13 @@ Status Query::finalize() { return Status::Ok(); } - if (array_->is_remote()) { + if (array_->is_remote() && type_ == QueryType::WRITE) { auto rest_client = storage_manager_->rest_client(); if (rest_client == nullptr) return logger_->status(Status_QueryError( "Error in query finalize; remote array with no rest client.")); - if (type_ == QueryType::WRITE && layout_ == Layout::GLOBAL_ORDER) { + if (layout_ == Layout::GLOBAL_ORDER) { return logger_->status(Status_QueryError( "Error in query finalize; remote global order writes are only " "allowed to call submit_and_finalize to submit the last tile")); @@ -844,6 +844,37 @@ Status Query::process() { } } + if (condition_.has_value()) { + auto& names = condition_->enumeration_field_names(); + std::unordered_set deduped_enmr_names; + for (auto name : names) { + auto attr = array_schema_->attribute(name); + if (attr == nullptr) { + continue; + } + auto enmr_name = attr->get_enumeration_name(); + if (enmr_name.has_value()) { + deduped_enmr_names.insert(enmr_name.value()); + } + } + std::vector enmr_names; + enmr_names.reserve(deduped_enmr_names.size()); + for (auto& enmr_name : deduped_enmr_names) { + enmr_names.emplace_back(enmr_name); + } + + throw_if_not_ok(parallel_for( + storage_manager_->compute_tp(), + 0, + enmr_names.size(), + [&](const uint64_t i) { + array_->get_enumeration(enmr_names[i]); + return Status::Ok(); + })); + + condition_->rewrite_enumeration_conditions(array_schema()); + } + // Update query status. status_ = QueryStatus::INPROGRESS; @@ -1236,7 +1267,7 @@ Status Query::check_set_fixed_buffer(const std::string& name) { } void Query::set_config(const Config& config) { - if (status_ != QueryStatus::UNINITIALIZED) { + if (!remote_query_ && status_ != QueryStatus::UNINITIALIZED) { throw QueryStatusException( "[set_config] Cannot set config after initialization."); } diff --git a/tiledb/sm/query/query_condition.cc b/tiledb/sm/query/query_condition.cc index a8c78c8ff409..b93f78e21af3 100644 --- a/tiledb/sm/query/query_condition.cc +++ b/tiledb/sm/query/query_condition.cc @@ -123,6 +123,15 @@ Status QueryCondition::init( return Status::Ok(); } +void QueryCondition::rewrite_enumeration_conditions( + const ArraySchema& array_schema) { + if (!tree_) { + return; + } + + tree_->rewrite_enumeration_conditions(array_schema); +} + Status QueryCondition::check(const ArraySchema& array_schema) const { if (!tree_) { return Status::Ok(); @@ -144,6 +153,7 @@ Status QueryCondition::combine( } combined_cond->field_names_.clear(); + combined_cond->enumeration_field_names_.clear(); combined_cond->tree_ = this->tree_->combine(rhs.tree_, combination_op); return Status::Ok(); } @@ -158,6 +168,7 @@ Status QueryCondition::negate( } combined_cond->field_names_.clear(); + combined_cond->enumeration_field_names_.clear(); combined_cond->tree_ = this->tree_->get_negated_tree(); return Status::Ok(); } @@ -174,6 +185,15 @@ std::unordered_set& QueryCondition::field_names() const { return field_names_; } +std::unordered_set& QueryCondition::enumeration_field_names() + const { + if (enumeration_field_names_.empty() && tree_ != nullptr) { + tree_->get_enumeration_field_names(enumeration_field_names_); + } + + return enumeration_field_names_; +} + uint64_t QueryCondition::condition_timestamp() const { if (condition_marker_.empty()) { return 0; @@ -188,6 +208,10 @@ uint64_t QueryCondition::condition_timestamp() const { return timestamps.first; } +void QueryCondition::set_use_enumeration(bool use_enumeration) { + tree_->set_use_enumeration(use_enumeration); +} + /** Full template specialization for `char*` and `QueryConditionOp::LT`. */ template <> struct QueryCondition::BinaryCmpNullChecks { diff --git a/tiledb/sm/query/query_condition.h b/tiledb/sm/query/query_condition.h index 717f047ceab0..b0c16154ce3d 100644 --- a/tiledb/sm/query/query_condition.h +++ b/tiledb/sm/query/query_condition.h @@ -112,6 +112,15 @@ class QueryCondition { uint64_t condition_value_size, const QueryConditionOp& op); + /** + * Translate any query conditions against enumerated attributes to the + * underlying attribute type. + * + * @param array_schema The current array schema with all required enumerations + * loaded. + */ + void rewrite_enumeration_conditions(const ArraySchema& array_schema); + /** * Verifies that the current state contains supported comparison * operations. Currently, we support the following: @@ -160,6 +169,12 @@ class QueryCondition { */ std::unordered_set& field_names() const; + /** + * Returns a set of all unique field names that reference an enumerated + * attribute condition in the AST representing the query condition. + */ + std::unordered_set& enumeration_field_names() const; + /** * Returns the timestamp for this condition. */ @@ -245,6 +260,17 @@ class QueryCondition { */ uint64_t condition_index() const; + /** + * By default, a query condition is applied against the enumerated values + * of an attribute. Setting use_enumeration to false prevents the translation + * and applies this query condition directly against the underlying integral + * attribute data. + * + * @param use_enumeration A bool indicating whether to use the enumeration + * values. + */ + void set_use_enumeration(bool use_enumeration); + private: /* ********************************* */ /* PRIVATE DATATYPES */ @@ -294,6 +320,9 @@ class QueryCondition { /** Caches all field names in the value nodes of the AST. */ mutable std::unordered_set field_names_; + /** Caches all field names that references enumerations in the AST. */ + mutable std::unordered_set enumeration_field_names_; + /* ********************************* */ /* PRIVATE METHODS */ /* ********************************* */ diff --git a/tiledb/sm/serialization/array_schema.cc b/tiledb/sm/serialization/array_schema.cc index 6d41ffaaa728..89b79d943080 100644 --- a/tiledb/sm/serialization/array_schema.cc +++ b/tiledb/sm/serialization/array_schema.cc @@ -42,6 +42,7 @@ #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/dimension_label.h" #include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/compressor.h" #include "tiledb/sm/enums/data_order.h" @@ -350,6 +351,11 @@ void attribute_to_capnp( const auto& filters = attribute->filters(); auto filter_pipeline_builder = attribute_builder->initFilterPipeline(); throw_if_not_ok(filter_pipeline_to_capnp(&filters, &filter_pipeline_builder)); + + auto enmr_name = attribute->get_enumeration_name(); + if (enmr_name.has_value()) { + attribute_builder->setEnumerationName(enmr_name.value()); + } } shared_ptr attribute_from_capnp( @@ -397,6 +403,11 @@ shared_ptr attribute_from_capnp( datatype, attribute_reader.getCellValNum()); } + std::optional enmr_name; + if (attribute_reader.hasEnumerationName()) { + enmr_name = attribute_reader.getEnumerationName(); + } + return tiledb::common::make_shared( HERE(), attribute_reader.getName(), @@ -406,7 +417,8 @@ shared_ptr attribute_from_capnp( *(filters.get()), fill_value_vec, fill_value_validity, - data_order); + data_order, + enmr_name); } Status dimension_to_capnp( @@ -761,6 +773,61 @@ shared_ptr dimension_label_from_capnp( is_relative); } +void enumeration_to_capnp( + shared_ptr enumeration, + capnp::Enumeration::Builder& enmr_builder) { + enmr_builder.setName(enumeration->name()); + enmr_builder.setType(datatype_str(enumeration->type())); + enmr_builder.setCellValNum(enumeration->cell_val_num()); + enmr_builder.setOrdered(enumeration->ordered()); + + auto dspan = enumeration->data(); + enmr_builder.setData(::kj::arrayPtr(dspan.data(), dspan.size())); + + if (enumeration->var_size()) { + auto ospan = enumeration->offsets(); + enmr_builder.setOffsets(::kj::arrayPtr(ospan.data(), ospan.size())); + } +} + +shared_ptr enumeration_from_capnp( + const capnp::Enumeration::Reader& reader) { + auto name = reader.getName(); + auto path_name = reader.getPathName(); + Datatype datatype = Datatype::ANY; + throw_if_not_ok(datatype_enum(reader.getType(), &datatype)); + + if (!reader.hasData()) { + throw SerializationStatusException( + "[Deserialization::enumeration_from_capnp] Deserialization of " + "Enumeration is missing its data buffer."); + } + + auto data_reader = reader.getData().asBytes(); + auto data = data_reader.begin(); + auto data_size = data_reader.size(); + + const void* offsets = nullptr; + uint64_t offsets_size = 0; + + if (reader.hasOffsets()) { + auto offsets_reader = reader.getOffsets().asBytes(); + offsets = offsets_reader.begin(); + offsets_size = offsets_reader.size(); + } + + return Enumeration::create( + name, + path_name, + datatype, + reader.getCellValNum(), + reader.getOrdered(), + data, + data_size, + offsets, + offsets_size); +} + Status array_schema_to_capnp( const ArraySchema& array_schema, capnp::ArraySchema::Builder* array_schema_builder, @@ -807,9 +874,9 @@ Status array_schema_to_capnp( // Attributes const unsigned num_attrs = array_schema.attribute_num(); - auto attributes_buidler = array_schema_builder->initAttributes(num_attrs); + auto attribute_builders = array_schema_builder->initAttributes(num_attrs); for (size_t i = 0; i < num_attrs; i++) { - auto attribute_builder = attributes_buidler[i]; + auto attribute_builder = attribute_builders[i]; attribute_to_capnp(array_schema.attribute(i), &attribute_builder); } @@ -831,6 +898,27 @@ Status array_schema_to_capnp( } } + // Loaded enumerations + auto loaded_enmr_names = array_schema.get_loaded_enumeration_names(); + const unsigned num_loaded_enmrs = loaded_enmr_names.size(); + auto enmr_builders = array_schema_builder->initEnumerations(num_loaded_enmrs); + for (size_t i = 0; i < num_loaded_enmrs; i++) { + auto enmr = array_schema.get_enumeration(loaded_enmr_names[i]); + auto builder = enmr_builders[i]; + enumeration_to_capnp(enmr, builder); + } + + // Enumeration path map + auto enmr_names = array_schema.get_enumeration_names(); + const unsigned num_enmr_names = enmr_names.size(); + auto enmr_path_map_builders = + array_schema_builder->initEnumerationPathMap(num_enmr_names); + for (size_t i = 0; i < num_enmr_names; i++) { + auto enmr_path_name = array_schema.get_enumeration_path_name(enmr_names[i]); + enmr_path_map_builders[i].setKey(enmr_names[i]); + enmr_path_map_builders[i].setValue(enmr_path_name); + } + return Status::Ok(); } @@ -997,6 +1085,39 @@ ArraySchema array_schema_from_capnp( } } + // Loaded enumerations + std::vector> enumerations; + if (schema_reader.hasEnumerations()) { + auto enmr_readers = schema_reader.getEnumerations(); + enumerations.reserve(enmr_readers.size()); + try { + for (auto&& enmr_reader : enmr_readers) { + enumerations.emplace_back(enumeration_from_capnp(enmr_reader)); + } + } catch (const std::exception& e) { + std::throw_with_nested(std::runtime_error( + "[Deserialization::array_schema_from_capnp] Cannot deserialize " + "enumerations")); + } + } + + // Enumeration path map + std::unordered_map enmr_path_map; + if (schema_reader.hasEnumerationPathMap()) { + auto enmr_path_map_readers = schema_reader.getEnumerationPathMap(); + try { + for (auto&& kv_reader : enmr_path_map_readers) { + auto enmr_name = kv_reader.getKey(); + auto enmr_path_name = kv_reader.getValue(); + enmr_path_map[enmr_name] = enmr_path_name; + } + } catch (const std::exception& e) { + std::throw_with_nested(std::runtime_error( + "[Deserialization::array_schema_from_capnp] Cannot deserialize " + "enumeration path map")); + } + } + // Set the range if we have two values // #TODO Add security validation std::pair timestamp_range; @@ -1026,6 +1147,8 @@ ArraySchema array_schema_from_capnp( capacity, attributes, dimension_labels, + enumerations, + enmr_path_map, cell_var_offsets_filters, cell_validity_filters, coords_filters); diff --git a/tiledb/sm/serialization/array_schema.h b/tiledb/sm/serialization/array_schema.h index b926c3990a41..918d006abdb4 100644 --- a/tiledb/sm/serialization/array_schema.h +++ b/tiledb/sm/serialization/array_schema.h @@ -119,6 +119,24 @@ void dimension_label_to_capnp( shared_ptr dimension_label_from_capnp( const capnp::DimensionLabel::Reader& reader); +/** + * Serialize an Enumeration do cap'n proto object + * + * @param enumeration Enumeration to serialize. + * @param builder Cap'n proto class. + */ +void enumeration_to_capnp( + shared_ptr, capnp::Enumeration::Builder& enm_builder); + +/** + * Deserialize a dimension label from a cap'n proto object + * + * @param reader Cap'n proto reader object + * @return A new Enumeration + */ +shared_ptr enumeration_from_capnp( + const capnp::Enumeration::Reader& reader); + #endif // TILEDB_SERIALIZATION /** diff --git a/tiledb/sm/serialization/array_schema_evolution.cc b/tiledb/sm/serialization/array_schema_evolution.cc index cb06262467c4..6cebb70b8a02 100644 --- a/tiledb/sm/serialization/array_schema_evolution.cc +++ b/tiledb/sm/serialization/array_schema_evolution.cc @@ -42,6 +42,7 @@ #include "tiledb/sm/array_schema/attribute.h" #include "tiledb/sm/array_schema/dimension.h" #include "tiledb/sm/array_schema/domain.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/enums/array_type.h" #include "tiledb/sm/enums/compressor.h" #include "tiledb/sm/enums/datatype.h" @@ -102,6 +103,20 @@ Status array_schema_evolution_to_capnp( attribute_to_capnp(attr_to_add, &attribute_builder); } + auto enmr_names_to_add = array_schema_evolution->enumeration_names_to_add(); + auto num_enmrs = enmr_names_to_add.size(); + + if (num_enmrs > 0) { + auto enmrs_to_add_builder = + array_schema_evolution_builder->initEnumerationsToAdd(num_enmrs); + for (size_t i = 0; i < num_enmrs; i++) { + auto enmr = + array_schema_evolution->enumeration_to_add(enmr_names_to_add[i]); + auto builder = enmrs_to_add_builder[i]; + enumeration_to_capnp(enmr, builder); + } + } + auto timestamp_builder = array_schema_evolution_builder->initTimestampRange(2); const auto& timestamp_range = array_schema_evolution->timestamp_range(); @@ -111,34 +126,46 @@ Status array_schema_evolution_to_capnp( return Status::Ok(); } -Status array_schema_evolution_from_capnp( - const capnp::ArraySchemaEvolution::Reader& evolution_reader, - tdb_unique_ptr* array_schema_evolution) { - array_schema_evolution->reset(tdb_new(ArraySchemaEvolution)); - // Set attributes to drop +tdb_unique_ptr array_schema_evolution_from_capnp( + const capnp::ArraySchemaEvolution::Reader& evolution_reader) { + // Create attributes to add + std::unordered_map> attrs_to_add; + auto attrs_to_add_reader = evolution_reader.getAttributesToAdd(); + for (auto attr_reader : attrs_to_add_reader) { + auto attr = attribute_from_capnp(attr_reader); + attrs_to_add[attr->name()] = attr; + } + + // Create enumerations to add + std::unordered_map> enmrs_to_add; + auto enmrs_to_add_reader = evolution_reader.getEnumerationsToAdd(); + for (auto enmr_reader : enmrs_to_add_reader) { + auto enmr = enumeration_from_capnp(enmr_reader); + enmrs_to_add[enmr->name()] = enmr; + } + + // Create attributes to drop + std::unordered_set attrs_to_drop; auto attributes_to_drop_reader = evolution_reader.getAttributesToDrop(); for (auto attr_reader : attributes_to_drop_reader) { std::string attr_name = std::string(attr_reader.cStr()); - RETURN_NOT_OK((*array_schema_evolution)->drop_attribute(attr_name)); - } - - // Set attributes to add - auto attributes_to_add_reader = evolution_reader.getAttributesToAdd(); - for (auto attr_reader : attributes_to_add_reader) { - auto attr = attribute_from_capnp(attr_reader); - RETURN_NOT_OK((*array_schema_evolution)->add_attribute(attr.get())); + attrs_to_drop.insert(attr_name); } + std::pair ts_range; // Set the range if we have two values if (evolution_reader.hasTimestampRange() && evolution_reader.getTimestampRange().size() >= 2) { const auto& timestamp_range = evolution_reader.getTimestampRange(); - throw_if_not_ok((*array_schema_evolution) - ->set_timestamp_range(std::make_pair( - timestamp_range[0], timestamp_range[1]))); + ts_range = std::make_pair(timestamp_range[0], timestamp_range[1]); } - return Status::Ok(); + return tdb_unique_ptr(tdb_new( + ArraySchemaEvolution, + attrs_to_add, + enmrs_to_add, + attrs_to_drop, + ts_range)); } Status array_schema_evolution_serialize( @@ -216,8 +243,8 @@ Status array_schema_evolution_deserialize( array_schema_evolution_builder); capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = array_schema_evolution_builder.asReader(); - RETURN_NOT_OK(array_schema_evolution_from_capnp( - array_schema_evolution_reader, &decoded_array_schema_evolution)); + decoded_array_schema_evolution = + array_schema_evolution_from_capnp(array_schema_evolution_reader); break; } case SerializationType::CAPNP: { @@ -228,8 +255,8 @@ Status array_schema_evolution_deserialize( serialized_buffer.size() / sizeof(::capnp::word))); capnp::ArraySchemaEvolution::Reader array_schema_evolution_reader = reader.getRoot(); - RETURN_NOT_OK(array_schema_evolution_from_capnp( - array_schema_evolution_reader, &decoded_array_schema_evolution)); + decoded_array_schema_evolution = + array_schema_evolution_from_capnp(array_schema_evolution_reader); break; } default: { diff --git a/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ index 08df30190b05..38693a34d0a8 100644 --- a/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ @@ -635,17 +635,17 @@ const ::capnp::_::RawSchema s_facceeafd4472c68 = { 1, 2, i_facceeafd4472c68, nullptr, nullptr, { &s_facceeafd4472c68, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<274> b_d71de32f98e296fe = { +static const ::capnp::_::AlignedData<315> b_d71de32f98e296fe = { { 0, 0, 0, 0, 5, 0, 6, 0, 254, 150, 226, 152, 47, 227, 29, 215, 18, 0, 0, 0, 1, 0, 2, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 13, 0, 7, 0, 0, 0, 0, 0, + 15, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 242, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 79, 3, 0, 0, + 29, 0, 0, 0, 191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -653,112 +653,126 @@ static const ::capnp::_::AlignedData<274> b_d71de32f98e296fe = { 112, 58, 65, 114, 114, 97, 121, 83, 99, 104, 101, 109, 97, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 60, 0, 0, 0, 3, 0, 4, 0, + 68, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 149, 1, 0, 0, 82, 0, 0, 0, + 205, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 148, 1, 0, 0, 3, 0, 1, 0, - 160, 1, 0, 0, 2, 0, 1, 0, + 204, 1, 0, 0, 3, 0, 1, 0, + 216, 1, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 157, 1, 0, 0, 90, 0, 0, 0, + 213, 1, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 156, 1, 0, 0, 3, 0, 1, 0, - 184, 1, 0, 0, 2, 0, 1, 0, + 212, 1, 0, 0, 3, 0, 1, 0, + 240, 1, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 181, 1, 0, 0, 74, 0, 0, 0, + 237, 1, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 180, 1, 0, 0, 3, 0, 1, 0, - 192, 1, 0, 0, 2, 0, 1, 0, + 236, 1, 0, 0, 3, 0, 1, 0, + 248, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 189, 1, 0, 0, 82, 0, 0, 0, + 245, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 188, 1, 0, 0, 3, 0, 1, 0, - 200, 1, 0, 0, 2, 0, 1, 0, + 244, 1, 0, 0, 3, 0, 1, 0, + 0, 2, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 197, 1, 0, 0, 170, 0, 0, 0, + 253, 1, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 200, 1, 0, 0, 3, 0, 1, 0, - 212, 1, 0, 0, 2, 0, 1, 0, + 0, 2, 0, 0, 3, 0, 1, 0, + 12, 2, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 1, 0, 0, 58, 0, 0, 0, + 9, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 204, 1, 0, 0, 3, 0, 1, 0, - 216, 1, 0, 0, 2, 0, 1, 0, + 4, 2, 0, 0, 3, 0, 1, 0, + 16, 2, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 213, 1, 0, 0, 170, 0, 0, 0, + 13, 2, 0, 0, 170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 216, 1, 0, 0, 3, 0, 1, 0, - 228, 1, 0, 0, 2, 0, 1, 0, + 16, 2, 0, 0, 3, 0, 1, 0, + 28, 2, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 225, 1, 0, 0, 82, 0, 0, 0, + 25, 2, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 224, 1, 0, 0, 3, 0, 1, 0, - 236, 1, 0, 0, 2, 0, 1, 0, + 24, 2, 0, 0, 3, 0, 1, 0, + 36, 2, 0, 0, 2, 0, 1, 0, 8, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 233, 1, 0, 0, 34, 0, 0, 0, + 33, 2, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 228, 1, 0, 0, 3, 0, 1, 0, - 240, 1, 0, 0, 2, 0, 1, 0, + 28, 2, 0, 0, 3, 0, 1, 0, + 40, 2, 0, 0, 2, 0, 1, 0, 9, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 237, 1, 0, 0, 66, 0, 0, 0, + 37, 2, 0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 1, 0, 0, 3, 0, 1, 0, - 4, 2, 0, 0, 2, 0, 1, 0, + 32, 2, 0, 0, 3, 0, 1, 0, + 60, 2, 0, 0, 2, 0, 1, 0, 10, 0, 0, 0, 64, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 2, 0, 0, 138, 0, 0, 0, + 57, 2, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 4, 2, 0, 0, 3, 0, 1, 0, - 16, 2, 0, 0, 2, 0, 1, 0, + 60, 2, 0, 0, 3, 0, 1, 0, + 72, 2, 0, 0, 2, 0, 1, 0, 11, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 13, 2, 0, 0, 186, 0, 0, 0, + 69, 2, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 16, 2, 0, 0, 3, 0, 1, 0, - 28, 2, 0, 0, 2, 0, 1, 0, + 72, 2, 0, 0, 3, 0, 1, 0, + 84, 2, 0, 0, 2, 0, 1, 0, 12, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 2, 0, 0, 42, 0, 0, 0, + 81, 2, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 20, 2, 0, 0, 3, 0, 1, 0, - 32, 2, 0, 0, 2, 0, 1, 0, + 76, 2, 0, 0, 3, 0, 1, 0, + 88, 2, 0, 0, 2, 0, 1, 0, 13, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 2, 0, 0, 122, 0, 0, 0, + 85, 2, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 28, 2, 0, 0, 3, 0, 1, 0, - 56, 2, 0, 0, 2, 0, 1, 0, + 84, 2, 0, 0, 3, 0, 1, 0, + 112, 2, 0, 0, 2, 0, 1, 0, 14, 0, 0, 0, 12, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 2, 0, 0, 130, 0, 0, 0, + 109, 2, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 52, 2, 0, 0, 3, 0, 1, 0, - 80, 2, 0, 0, 2, 0, 1, 0, + 108, 2, 0, 0, 3, 0, 1, 0, + 136, 2, 0, 0, 2, 0, 1, 0, + 15, 0, 0, 0, 13, 0, 0, 0, + 0, 0, 1, 0, 15, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 2, 0, 0, 106, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 132, 2, 0, 0, 3, 0, 1, 0, + 160, 2, 0, 0, 2, 0, 1, 0, + 16, 0, 0, 0, 14, 0, 0, 0, + 0, 0, 1, 0, 16, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 157, 2, 0, 0, 154, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 160, 2, 0, 0, 3, 0, 1, 0, + 188, 2, 0, 0, 2, 0, 1, 0, 97, 114, 114, 97, 121, 84, 121, 112, 101, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, @@ -907,6 +921,33 @@ static const ::capnp::_::AlignedData<274> b_d71de32f98e296fe = { 222, 209, 12, 209, 98, 141, 255, 206, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 80, 97, 116, 104, 77, + 97, 112, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 151, 188, 17, 242, 43, 223, 218, 227, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -917,13 +958,15 @@ static const ::capnp::_::RawSchema* const d_d71de32f98e296fe[] = { &s_92ad78f56de3d76a, &s_bc4583f733eac4f5, &s_ceff8d62d10cd1de, + &s_d00b2f19cc21b9b4, &s_de030f447664754c, + &s_e3dadf2bf211bc97, }; -static const uint16_t m_d71de32f98e296fe[] = {10, 0, 1, 2, 3, 4, 14, 5, 12, 6, 7, 13, 8, 11, 9}; -static const uint16_t i_d71de32f98e296fe[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}; +static const uint16_t m_d71de32f98e296fe[] = {10, 0, 1, 2, 3, 4, 14, 5, 16, 15, 12, 6, 7, 13, 8, 11, 9}; +static const uint16_t i_d71de32f98e296fe[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; const ::capnp::_::RawSchema s_d71de32f98e296fe = { - 0xd71de32f98e296fe, b_d71de32f98e296fe.words, 274, d_d71de32f98e296fe, m_d71de32f98e296fe, - 4, 15, i_d71de32f98e296fe, nullptr, nullptr, { &s_d71de32f98e296fe, nullptr, nullptr, 0, 0, nullptr } + 0xd71de32f98e296fe, b_d71de32f98e296fe.words, 315, d_d71de32f98e296fe, m_d71de32f98e296fe, + 6, 17, i_d71de32f98e296fe, nullptr, nullptr, { &s_d71de32f98e296fe, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<174> b_ceff8d62d10cd1de = { @@ -1114,17 +1157,17 @@ const ::capnp::_::RawSchema s_ceff8d62d10cd1de = { 1, 10, i_ceff8d62d10cd1de, nullptr, nullptr, { &s_ceff8d62d10cd1de, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { +static const ::capnp::_::AlignedData<101> b_a1b81d67548230d4 = { { 0, 0, 0, 0, 5, 0, 6, 0, 212, 48, 130, 84, 103, 29, 184, 161, 18, 0, 0, 0, 1, 0, 0, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 3, 0, 7, 0, 0, 0, 0, 0, + 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 175, 0, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -1133,28 +1176,35 @@ static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { 99, 104, 101, 109, 97, 69, 118, 111, 108, 117, 116, 105, 111, 110, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 12, 0, 0, 0, 3, 0, 4, 0, + 16, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 69, 0, 0, 0, 138, 0, 0, 0, + 97, 0, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 0, 0, 0, 3, 0, 1, 0, - 100, 0, 0, 0, 2, 0, 1, 0, + 100, 0, 0, 0, 3, 0, 1, 0, + 128, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 97, 0, 0, 0, 130, 0, 0, 0, + 125, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 96, 0, 0, 0, 3, 0, 1, 0, - 124, 0, 0, 0, 2, 0, 1, 0, + 124, 0, 0, 0, 3, 0, 1, 0, + 152, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 121, 0, 0, 0, 122, 0, 0, 0, + 149, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 120, 0, 0, 0, 3, 0, 1, 0, - 148, 0, 0, 0, 2, 0, 1, 0, + 148, 0, 0, 0, 3, 0, 1, 0, + 176, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 173, 0, 0, 0, 146, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 176, 0, 0, 0, 3, 0, 1, 0, + 204, 0, 0, 0, 2, 0, 1, 0, 97, 116, 116, 114, 105, 98, 117, 116, 101, 115, 84, 111, 68, 114, 111, 112, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1192,6 +1242,20 @@ static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 115, 84, 111, 65, 100, + 100, 0, 0, 0, 0, 0, 0, 0, + 14, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 0, 1, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -1200,25 +1264,26 @@ static const ::capnp::_::AlignedData<80> b_a1b81d67548230d4 = { #if !CAPNP_LITE static const ::capnp::_::RawSchema* const d_a1b81d67548230d4[] = { &s_92ad78f56de3d76a, + &s_d00b2f19cc21b9b4, }; -static const uint16_t m_a1b81d67548230d4[] = {1, 0, 2}; -static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2}; +static const uint16_t m_a1b81d67548230d4[] = {1, 0, 3, 2}; +static const uint16_t i_a1b81d67548230d4[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_a1b81d67548230d4 = { - 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 80, d_a1b81d67548230d4, m_a1b81d67548230d4, - 1, 3, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr } + 0xa1b81d67548230d4, b_a1b81d67548230d4.words, 101, d_a1b81d67548230d4, m_a1b81d67548230d4, + 2, 4, i_a1b81d67548230d4, nullptr, nullptr, { &s_a1b81d67548230d4, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { +static const ::capnp::_::AlignedData<160> b_92ad78f56de3d76a = { { 0, 0, 0, 0, 5, 0, 6, 0, 106, 215, 227, 109, 245, 120, 173, 146, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 5, 0, 7, 0, 0, 0, 0, 0, + 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 226, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 199, 1, 0, 0, + 29, 0, 0, 0, 255, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -1226,63 +1291,70 @@ static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { 112, 58, 65, 116, 116, 114, 105, 98, 117, 116, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 32, 0, 0, 0, 3, 0, 4, 0, + 36, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 209, 0, 0, 0, 90, 0, 0, 0, + 237, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 208, 0, 0, 0, 3, 0, 1, 0, - 220, 0, 0, 0, 2, 0, 1, 0, + 236, 0, 0, 0, 3, 0, 1, 0, + 248, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 217, 0, 0, 0, 42, 0, 0, 0, + 245, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 212, 0, 0, 0, 3, 0, 1, 0, - 224, 0, 0, 0, 2, 0, 1, 0, + 240, 0, 0, 0, 3, 0, 1, 0, + 252, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 221, 0, 0, 0, 42, 0, 0, 0, + 249, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 216, 0, 0, 0, 3, 0, 1, 0, - 228, 0, 0, 0, 2, 0, 1, 0, + 244, 0, 0, 0, 3, 0, 1, 0, + 0, 1, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 225, 0, 0, 0, 122, 0, 0, 0, + 253, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 224, 0, 0, 0, 3, 0, 1, 0, - 236, 0, 0, 0, 2, 0, 1, 0, + 252, 0, 0, 0, 3, 0, 1, 0, + 8, 1, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 233, 0, 0, 0, 82, 0, 0, 0, + 5, 1, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 0, 0, 0, 3, 0, 1, 0, - 244, 0, 0, 0, 2, 0, 1, 0, + 4, 1, 0, 0, 3, 0, 1, 0, + 16, 1, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 32, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 241, 0, 0, 0, 74, 0, 0, 0, + 13, 1, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 240, 0, 0, 0, 3, 0, 1, 0, - 252, 0, 0, 0, 2, 0, 1, 0, + 12, 1, 0, 0, 3, 0, 1, 0, + 24, 1, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 33, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 249, 0, 0, 0, 146, 0, 0, 0, + 21, 1, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 252, 0, 0, 0, 3, 0, 1, 0, - 8, 1, 0, 0, 2, 0, 1, 0, + 24, 1, 0, 0, 3, 0, 1, 0, + 36, 1, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 5, 1, 0, 0, 50, 0, 0, 0, + 33, 1, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 0, 0, 3, 0, 1, 0, - 12, 1, 0, 0, 2, 0, 1, 0, + 28, 1, 0, 0, 3, 0, 1, 0, + 40, 1, 0, 0, 2, 0, 1, 0, + 8, 0, 0, 0, 5, 0, 0, 0, + 0, 0, 1, 0, 8, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 37, 1, 0, 0, 130, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 36, 1, 0, 0, 3, 0, 1, 0, + 48, 1, 0, 0, 2, 0, 1, 0, 99, 101, 108, 108, 86, 97, 108, 78, 117, 109, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, @@ -1346,6 +1418,15 @@ static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 114, 100, 101, 114, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 101, 110, 117, 109, 101, 114, 97, 116, + 105, 111, 110, 78, 97, 109, 101, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1359,11 +1440,147 @@ static const ::capnp::_::AlignedData<144> b_92ad78f56de3d76a = { static const ::capnp::_::RawSchema* const d_92ad78f56de3d76a[] = { &s_bc4583f733eac4f5, }; -static const uint16_t m_92ad78f56de3d76a[] = {0, 4, 6, 3, 1, 5, 7, 2}; -static const uint16_t i_92ad78f56de3d76a[] = {0, 1, 2, 3, 4, 5, 6, 7}; +static const uint16_t m_92ad78f56de3d76a[] = {0, 8, 4, 6, 3, 1, 5, 7, 2}; +static const uint16_t i_92ad78f56de3d76a[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; const ::capnp::_::RawSchema s_92ad78f56de3d76a = { - 0x92ad78f56de3d76a, b_92ad78f56de3d76a.words, 144, d_92ad78f56de3d76a, m_92ad78f56de3d76a, - 1, 8, i_92ad78f56de3d76a, nullptr, nullptr, { &s_92ad78f56de3d76a, nullptr, nullptr, 0, 0, nullptr } + 0x92ad78f56de3d76a, b_92ad78f56de3d76a.words, 160, d_92ad78f56de3d76a, m_92ad78f56de3d76a, + 1, 9, i_92ad78f56de3d76a, nullptr, nullptr, { &s_92ad78f56de3d76a, nullptr, nullptr, 0, 0, nullptr } +}; +#endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<125> b_d00b2f19cc21b9b4 = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 180, 185, 33, 204, 25, 47, 11, 208, + 18, 0, 0, 0, 1, 0, 1, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 5, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 242, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 143, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 69, 110, 117, 109, 101, 114, + 97, 116, 105, 111, 110, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 28, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 181, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 176, 0, 0, 0, 3, 0, 1, 0, + 188, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 185, 0, 0, 0, 74, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 184, 0, 0, 0, 3, 0, 1, 0, + 196, 0, 0, 0, 2, 0, 1, 0, + 2, 0, 0, 0, 2, 0, 0, 0, + 0, 0, 1, 0, 2, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 193, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 188, 0, 0, 0, 3, 0, 1, 0, + 200, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 197, 0, 0, 0, 90, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 196, 0, 0, 0, 3, 0, 1, 0, + 208, 0, 0, 0, 2, 0, 1, 0, + 4, 0, 0, 0, 32, 0, 0, 0, + 0, 0, 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 205, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 200, 0, 0, 0, 3, 0, 1, 0, + 212, 0, 0, 0, 2, 0, 1, 0, + 5, 0, 0, 0, 3, 0, 0, 0, + 0, 0, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 209, 0, 0, 0, 42, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 216, 0, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 213, 0, 0, 0, 66, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 208, 0, 0, 0, 3, 0, 1, 0, + 220, 0, 0, 0, 2, 0, 1, 0, + 110, 97, 109, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 97, 116, 104, 78, 97, 109, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 121, 112, 101, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 99, 101, 108, 108, 86, 97, 108, 78, + 117, 109, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 114, 100, 101, 114, 101, 100, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 97, 116, 97, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 111, 102, 102, 115, 101, 116, 115, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 13, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_d00b2f19cc21b9b4 = b_d00b2f19cc21b9b4.words; +#if !CAPNP_LITE +static const uint16_t m_d00b2f19cc21b9b4[] = {3, 5, 0, 6, 4, 1, 2}; +static const uint16_t i_d00b2f19cc21b9b4[] = {0, 1, 2, 3, 4, 5, 6}; +const ::capnp::_::RawSchema s_d00b2f19cc21b9b4 = { + 0xd00b2f19cc21b9b4, b_d00b2f19cc21b9b4.words, 125, nullptr, m_d00b2f19cc21b9b4, + 0, 7, i_d00b2f19cc21b9b4, nullptr, nullptr, { &s_d00b2f19cc21b9b4, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<143> b_d20a578112fa92a2 = { @@ -4271,17 +4488,17 @@ const ::capnp::_::RawSchema s_cbe1e7c13508aa2c = { 1, 4, i_cbe1e7c13508aa2c, nullptr, nullptr, { &s_cbe1e7c13508aa2c, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<65> b_dac6a7f675c57409 = { +static const ::capnp::_::AlignedData<81> b_dac6a7f675c57409 = { { 0, 0, 0, 0, 5, 0, 6, 0, 9, 116, 197, 117, 246, 167, 198, 218, - 18, 0, 0, 0, 1, 0, 0, 0, + 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 18, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 175, 0, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -4290,28 +4507,35 @@ static const ::capnp::_::AlignedData<65> b_dac6a7f675c57409 = { 105, 111, 110, 67, 108, 97, 117, 115, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 12, 0, 0, 0, 3, 0, 4, 0, + 16, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 69, 0, 0, 0, 82, 0, 0, 0, + 97, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 68, 0, 0, 0, 3, 0, 1, 0, - 80, 0, 0, 0, 2, 0, 1, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 77, 0, 0, 0, 50, 0, 0, 0, + 105, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 0, 0, 0, 3, 0, 1, 0, - 84, 0, 0, 0, 2, 0, 1, 0, + 100, 0, 0, 0, 3, 0, 1, 0, + 112, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 81, 0, 0, 0, 26, 0, 0, 0, + 109, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 76, 0, 0, 0, 3, 0, 1, 0, - 88, 0, 0, 0, 2, 0, 1, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 116, 0, 0, 0, 2, 0, 1, 0, + 3, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 3, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 113, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 112, 0, 0, 0, 3, 0, 1, 0, + 124, 0, 0, 0, 2, 0, 1, 0, 102, 105, 101, 108, 100, 78, 97, 109, 101, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, @@ -4335,19 +4559,28 @@ static const ::capnp::_::AlignedData<65> b_dac6a7f675c57409 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 115, 101, 69, 110, 117, 109, 101, + 114, 97, 116, 105, 111, 110, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; ::capnp::word const* const bp_dac6a7f675c57409 = b_dac6a7f675c57409.words; #if !CAPNP_LITE -static const uint16_t m_dac6a7f675c57409[] = {0, 2, 1}; -static const uint16_t i_dac6a7f675c57409[] = {0, 1, 2}; +static const uint16_t m_dac6a7f675c57409[] = {0, 2, 3, 1}; +static const uint16_t i_dac6a7f675c57409[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_dac6a7f675c57409 = { - 0xdac6a7f675c57409, b_dac6a7f675c57409.words, 65, nullptr, m_dac6a7f675c57409, - 0, 3, i_dac6a7f675c57409, nullptr, nullptr, { &s_dac6a7f675c57409, nullptr, nullptr, 0, 0, nullptr } + 0xdac6a7f675c57409, b_dac6a7f675c57409.words, 81, nullptr, m_dac6a7f675c57409, + 0, 4, i_dac6a7f675c57409, nullptr, nullptr, { &s_dac6a7f675c57409, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { +static const ::capnp::_::AlignedData<132> b_afc739d5c01e6496 = { { 0, 0, 0, 0, 5, 0, 6, 0, 150, 100, 30, 192, 213, 57, 199, 175, 18, 0, 0, 0, 1, 0, 1, 0, @@ -4357,7 +4590,7 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { 21, 0, 0, 0, 210, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 0, 0, 0, 87, 1, 0, 0, + 29, 0, 0, 0, 143, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -4365,49 +4598,56 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { 112, 58, 65, 83, 84, 78, 111, 100, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 24, 0, 0, 0, 3, 0, 4, 0, + 28, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 153, 0, 0, 0, 106, 0, 0, 0, + 181, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 152, 0, 0, 0, 3, 0, 1, 0, - 164, 0, 0, 0, 2, 0, 1, 0, + 180, 0, 0, 0, 3, 0, 1, 0, + 192, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 161, 0, 0, 0, 82, 0, 0, 0, + 189, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 160, 0, 0, 0, 3, 0, 1, 0, - 172, 0, 0, 0, 2, 0, 1, 0, + 188, 0, 0, 0, 3, 0, 1, 0, + 200, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 169, 0, 0, 0, 50, 0, 0, 0, + 197, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 164, 0, 0, 0, 3, 0, 1, 0, - 176, 0, 0, 0, 2, 0, 1, 0, + 192, 0, 0, 0, 3, 0, 1, 0, + 204, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 173, 0, 0, 0, 26, 0, 0, 0, + 201, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 168, 0, 0, 0, 3, 0, 1, 0, - 180, 0, 0, 0, 2, 0, 1, 0, + 196, 0, 0, 0, 3, 0, 1, 0, + 208, 0, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 177, 0, 0, 0, 74, 0, 0, 0, + 205, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 176, 0, 0, 0, 3, 0, 1, 0, - 204, 0, 0, 0, 2, 0, 1, 0, + 204, 0, 0, 0, 3, 0, 1, 0, + 232, 0, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 201, 0, 0, 0, 114, 0, 0, 0, + 229, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 200, 0, 0, 0, 3, 0, 1, 0, - 212, 0, 0, 0, 2, 0, 1, 0, + 228, 0, 0, 0, 3, 0, 1, 0, + 240, 0, 0, 0, 2, 0, 1, 0, + 6, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 237, 0, 0, 0, 122, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 236, 0, 0, 0, 3, 0, 1, 0, + 248, 0, 0, 0, 2, 0, 1, 0, 105, 115, 69, 120, 112, 114, 101, 115, 115, 105, 111, 110, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, @@ -4462,6 +4702,15 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 117, 115, 101, 69, 110, 117, 109, 101, + 114, 97, 116, 105, 111, 110, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } }; @@ -4470,11 +4719,11 @@ static const ::capnp::_::AlignedData<116> b_afc739d5c01e6496 = { static const ::capnp::_::RawSchema* const d_afc739d5c01e6496[] = { &s_afc739d5c01e6496, }; -static const uint16_t m_afc739d5c01e6496[] = {4, 5, 1, 0, 3, 2}; -static const uint16_t i_afc739d5c01e6496[] = {0, 1, 2, 3, 4, 5}; +static const uint16_t m_afc739d5c01e6496[] = {4, 5, 1, 0, 3, 6, 2}; +static const uint16_t i_afc739d5c01e6496[] = {0, 1, 2, 3, 4, 5, 6}; const ::capnp::_::RawSchema s_afc739d5c01e6496 = { - 0xafc739d5c01e6496, b_afc739d5c01e6496.words, 116, d_afc739d5c01e6496, m_afc739d5c01e6496, - 1, 6, i_afc739d5c01e6496, nullptr, nullptr, { &s_afc739d5c01e6496, nullptr, nullptr, 0, 0, nullptr } + 0xafc739d5c01e6496, b_afc739d5c01e6496.words, 132, d_afc739d5c01e6496, m_afc739d5c01e6496, + 1, 7, i_afc739d5c01e6496, nullptr, nullptr, { &s_afc739d5c01e6496, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<73> b_eaf57cb9871fc06f = { @@ -9010,6 +9259,14 @@ constexpr ::capnp::Kind Attribute::_capnpPrivate::kind; constexpr ::capnp::_::RawSchema const* Attribute::_capnpPrivate::schema; #endif // !CAPNP_LITE +// Enumeration +constexpr uint16_t Enumeration::_capnpPrivate::dataWordSize; +constexpr uint16_t Enumeration::_capnpPrivate::pointerCount; +#if !CAPNP_LITE +constexpr ::capnp::Kind Enumeration::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* Enumeration::_capnpPrivate::schema; +#endif // !CAPNP_LITE + // AttributeBufferHeader constexpr uint16_t AttributeBufferHeader::_capnpPrivate::dataWordSize; constexpr uint16_t AttributeBufferHeader::_capnpPrivate::pointerCount; diff --git a/tiledb/sm/serialization/posix/tiledb-rest.capnp.h b/tiledb/sm/serialization/posix/tiledb-rest.capnp.h index 260cbaa58fb9..8cbda7df089a 100644 --- a/tiledb/sm/serialization/posix/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/posix/tiledb-rest.capnp.h @@ -23,6 +23,7 @@ CAPNP_DECLARE_SCHEMA(d71de32f98e296fe); CAPNP_DECLARE_SCHEMA(ceff8d62d10cd1de); CAPNP_DECLARE_SCHEMA(a1b81d67548230d4); CAPNP_DECLARE_SCHEMA(92ad78f56de3d76a); +CAPNP_DECLARE_SCHEMA(d00b2f19cc21b9b4); CAPNP_DECLARE_SCHEMA(d20a578112fa92a2); CAPNP_DECLARE_SCHEMA(95e26a84d32d8223); CAPNP_DECLARE_SCHEMA(a2a652536db09fa0); @@ -196,7 +197,7 @@ struct ArraySchema { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(d71de32f98e296fe, 2, 13) + CAPNP_DECLARE_STRUCT_HEADER(d71de32f98e296fe, 2, 15) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -230,7 +231,7 @@ struct ArraySchemaEvolution { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(a1b81d67548230d4, 0, 3) + CAPNP_DECLARE_STRUCT_HEADER(a1b81d67548230d4, 0, 4) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -247,7 +248,24 @@ struct Attribute { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(92ad78f56de3d76a, 1, 5) + CAPNP_DECLARE_STRUCT_HEADER(92ad78f56de3d76a, 1, 6) +#if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { + return &schema->defaultBrand; + } +#endif // !CAPNP_LITE + }; +}; + +struct Enumeration { + Enumeration() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(d00b2f19cc21b9b4, 1, 5) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -820,7 +838,7 @@ struct ConditionClause { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(dac6a7f675c57409, 0, 3) + CAPNP_DECLARE_STRUCT_HEADER(dac6a7f675c57409, 1, 3) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -2441,6 +2459,18 @@ class ArraySchema::Reader { ::capnp::Kind::STRUCT>::Reader getDimensionLabels() const; + inline bool hasEnumerations() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader + getEnumerations() const; + + inline bool hasEnumerationPathMap() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Reader + getEnumerationPathMap() const; + private: ::capnp::_::StructReader _reader; template @@ -2640,6 +2670,48 @@ class ArraySchema::Builder { ::capnp::Kind::STRUCT>> disownDimensionLabels(); + inline bool hasEnumerations(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + getEnumerations(); + inline void setEnumerations(::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + initEnumerations(unsigned int size); + inline void adoptEnumerations( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> + disownEnumerations(); + + inline bool hasEnumerationPathMap(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Builder + getEnumerationPathMap(); + inline void setEnumerationPathMap(::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Builder + initEnumerationPathMap(unsigned int size); + inline void adoptEnumerationPathMap( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>> + disownEnumerationPathMap(); + private: ::capnp::_::StructBuilder _builder; template @@ -2883,6 +2955,12 @@ class ArraySchemaEvolution::Reader { inline ::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>::Reader getTimestampRange() const; + inline bool hasEnumerationsToAdd() const; + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader + getEnumerationsToAdd() const; + private: ::capnp::_::StructReader _reader; template @@ -2972,6 +3050,28 @@ class ArraySchemaEvolution::Builder { inline ::capnp::Orphan<::capnp::List<::uint64_t, ::capnp::Kind::PRIMITIVE>> disownTimestampRange(); + inline bool hasEnumerationsToAdd(); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + getEnumerationsToAdd(); + inline void setEnumerationsToAdd( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value); + inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder + initEnumerationsToAdd(unsigned int size); + inline void adoptEnumerationsToAdd( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value); + inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> + disownEnumerationsToAdd(); + private: ::capnp::_::StructBuilder _builder; template @@ -3042,6 +3142,9 @@ class Attribute::Reader { inline bool hasOrder() const; inline ::capnp::Text::Reader getOrder() const; + inline bool hasEnumerationName() const; + inline ::capnp::Text::Reader getEnumerationName() const; + private: ::capnp::_::StructReader _reader; template @@ -3131,6 +3234,13 @@ class Attribute::Builder { inline void adoptOrder(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownOrder(); + inline bool hasEnumerationName(); + inline ::capnp::Text::Builder getEnumerationName(); + inline void setEnumerationName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initEnumerationName(unsigned int size); + inline void adoptEnumerationName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownEnumerationName(); + private: ::capnp::_::StructBuilder _builder; template @@ -3163,6 +3273,153 @@ class Attribute::Pipeline { }; #endif // !CAPNP_LITE +class Enumeration::Reader { + public: + typedef Enumeration Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base) + : _reader(base) { + } + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasName() const; + inline ::capnp::Text::Reader getName() const; + + inline bool hasPathName() const; + inline ::capnp::Text::Reader getPathName() const; + + inline bool hasType() const; + inline ::capnp::Text::Reader getType() const; + + inline ::uint32_t getCellValNum() const; + + inline bool getOrdered() const; + + inline bool hasData() const; + inline ::capnp::Data::Reader getData() const; + + inline bool hasOffsets() const; + inline ::capnp::Data::Reader getOffsets() const; + + private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class Enumeration::Builder { + public: + typedef Enumeration Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) { + } + inline explicit Builder(::capnp::_::StructBuilder base) + : _builder(base) { + } + inline operator Reader() const { + return Reader(_builder.asReader()); + } + inline Reader asReader() const { + return *this; + } + + inline ::capnp::MessageSize totalSize() const { + return asReader().totalSize(); + } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return asReader().toString(); + } +#endif // !CAPNP_LITE + + inline bool hasName(); + inline ::capnp::Text::Builder getName(); + inline void setName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initName(unsigned int size); + inline void adoptName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownName(); + + inline bool hasPathName(); + inline ::capnp::Text::Builder getPathName(); + inline void setPathName(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initPathName(unsigned int size); + inline void adoptPathName(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownPathName(); + + inline bool hasType(); + inline ::capnp::Text::Builder getType(); + inline void setType(::capnp::Text::Reader value); + inline ::capnp::Text::Builder initType(unsigned int size); + inline void adoptType(::capnp::Orphan<::capnp::Text>&& value); + inline ::capnp::Orphan<::capnp::Text> disownType(); + + inline ::uint32_t getCellValNum(); + inline void setCellValNum(::uint32_t value); + + inline bool getOrdered(); + inline void setOrdered(bool value); + + inline bool hasData(); + inline ::capnp::Data::Builder getData(); + inline void setData(::capnp::Data::Reader value); + inline ::capnp::Data::Builder initData(unsigned int size); + inline void adoptData(::capnp::Orphan<::capnp::Data>&& value); + inline ::capnp::Orphan<::capnp::Data> disownData(); + + inline bool hasOffsets(); + inline ::capnp::Data::Builder getOffsets(); + inline void setOffsets(::capnp::Data::Reader value); + inline ::capnp::Data::Builder initOffsets(unsigned int size); + inline void adoptOffsets(::capnp::Orphan<::capnp::Data>&& value); + inline ::capnp::Orphan<::capnp::Data> disownOffsets(); + + private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class Enumeration::Pipeline { + public: + typedef Enumeration Pipelines; + + inline Pipeline(decltype(nullptr)) + : _typeless(nullptr) { + } + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) { + } + + private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + class AttributeBufferHeader::Reader { public: typedef AttributeBufferHeader Reads; @@ -7193,6 +7450,8 @@ class ConditionClause::Reader { inline bool hasOp() const; inline ::capnp::Text::Reader getOp() const; + inline bool getUseEnumeration() const; + private: ::capnp::_::StructReader _reader; template @@ -7253,6 +7512,9 @@ class ConditionClause::Builder { inline void adoptOp(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownOp(); + inline bool getUseEnumeration(); + inline void setUseEnumeration(bool value); + private: ::capnp::_::StructBuilder _builder; template @@ -7321,6 +7583,8 @@ class ASTNode::Reader { inline bool hasCombinationOp() const; inline ::capnp::Text::Reader getCombinationOp() const; + inline bool getUseEnumeration() const; + private: ::capnp::_::StructReader _reader; template @@ -7411,6 +7675,9 @@ class ASTNode::Builder { inline void adoptCombinationOp(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownCombinationOp(); + inline bool getUseEnumeration(); + inline void setUseEnumeration(bool value); + private: ::capnp::_::StructBuilder _builder; template @@ -15475,58 +15742,203 @@ ArraySchema::Builder::disownDimensionLabels() { ::capnp::POINTERS)); } -inline ::uint32_t DimensionLabel::Reader::getDimensionId() const { - return _reader.getDataField<::uint32_t>( - ::capnp::bounded<0>() * ::capnp::ELEMENTS); -} - -inline ::uint32_t DimensionLabel::Builder::getDimensionId() { - return _builder.getDataField<::uint32_t>( - ::capnp::bounded<0>() * ::capnp::ELEMENTS); -} -inline void DimensionLabel::Builder::setDimensionId(::uint32_t value) { - _builder.setDataField<::uint32_t>( - ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); -} - -inline bool DimensionLabel::Reader::hasName() const { - return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) +inline bool ArraySchema::Reader::hasEnumerations() const { + return !_reader.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS) .isNull(); } -inline bool DimensionLabel::Builder::hasName() { - return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) +inline bool ArraySchema::Builder::hasEnumerations() { + return !_builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS) .isNull(); } -inline ::capnp::Text::Reader DimensionLabel::Reader::getName() const { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); -} -inline ::capnp::Text::Builder DimensionLabel::Builder::getName() { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader +ArraySchema::Reader::getEnumerations() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<13>() * + ::capnp::POINTERS)); } -inline void DimensionLabel::Builder::setName(::capnp::Text::Reader value) { - ::capnp::_::PointerHelpers<::capnp::Text>::set( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - value); +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchema::Builder::getEnumerations() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<13>() * + ::capnp::POINTERS)); } -inline ::capnp::Text::Builder DimensionLabel::Builder::initName( - unsigned int size) { - return ::capnp::_::PointerHelpers<::capnp::Text>::init( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - size); +inline void ArraySchema::Builder::setEnumerations( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS), + value); } -inline void DimensionLabel::Builder::adoptName( - ::capnp::Orphan<::capnp::Text>&& value) { - ::capnp::_::PointerHelpers<::capnp::Text>::adopt( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), - kj::mv(value)); +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchema::Builder::initEnumerations(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS), + size); } -inline ::capnp::Orphan<::capnp::Text> DimensionLabel::Builder::disownName() { - return ::capnp::_::PointerHelpers<::capnp::Text>::disown( - _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +inline void ArraySchema::Builder::adoptEnumerations( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<13>() * ::capnp::POINTERS), + kj::mv(value)); } - +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> +ArraySchema::Builder::disownEnumerations() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<13>() * + ::capnp::POINTERS)); +} + +inline bool ArraySchema::Reader::hasEnumerationPathMap() const { + return !_reader.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchema::Builder::hasEnumerationPathMap() { + return !_builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp:: + List<::tiledb::sm::serialization::capnp::KV, ::capnp::Kind::STRUCT>::Reader + ArraySchema::Reader::getEnumerationPathMap() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<14>() * + ::capnp::POINTERS)); +} +inline ::capnp:: + List<::tiledb::sm::serialization::capnp::KV, ::capnp::Kind::STRUCT>::Builder + ArraySchema::Builder::getEnumerationPathMap() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<14>() * + ::capnp::POINTERS)); +} +inline void ArraySchema::Builder::setEnumerationPathMap( + ::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS), + value); +} +inline ::capnp:: + List<::tiledb::sm::serialization::capnp::KV, ::capnp::Kind::STRUCT>::Builder + ArraySchema::Builder::initEnumerationPathMap(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS), + size); +} +inline void ArraySchema::Builder::adoptEnumerationPathMap( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<14>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>> +ArraySchema::Builder::disownEnumerationPathMap() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::KV, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<14>() * + ::capnp::POINTERS)); +} + +inline ::uint32_t DimensionLabel::Reader::getDimensionId() const { + return _reader.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint32_t DimensionLabel::Builder::getDimensionId() { + return _builder.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void DimensionLabel::Builder::setDimensionId(::uint32_t value) { + _builder.setDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool DimensionLabel::Reader::hasName() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool DimensionLabel::Builder::hasName() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader DimensionLabel::Reader::getName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder DimensionLabel::Builder::getName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void DimensionLabel::Builder::setName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder DimensionLabel::Builder::initName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + size); +} +inline void DimensionLabel::Builder::adoptName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> DimensionLabel::Builder::disownName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} + inline bool DimensionLabel::Reader::hasUri() const { return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) .isNull(); @@ -15962,6 +16374,80 @@ ArraySchemaEvolution::Builder::disownTimestampRange() { _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); } +inline bool ArraySchemaEvolution::Reader::hasEnumerationsToAdd() const { + return !_reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline bool ArraySchemaEvolution::Builder::hasEnumerationsToAdd() { + return !_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader +ArraySchemaEvolution::Reader::getEnumerationsToAdd() const { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_reader + .getPointerField( + ::capnp::bounded<3>() * + ::capnp::POINTERS)); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchemaEvolution::Builder::getEnumerationsToAdd() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::get(_builder + .getPointerField( + ::capnp::bounded<3>() * + ::capnp::POINTERS)); +} +inline void ArraySchemaEvolution::Builder::setEnumerationsToAdd( + ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Reader value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + set(_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + value); +} +inline ::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>::Builder +ArraySchemaEvolution::Builder::initEnumerationsToAdd(unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + init( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + size); +} +inline void ArraySchemaEvolution::Builder::adoptEnumerationsToAdd( + ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>&& value) { + ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>:: + adopt( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>> +ArraySchemaEvolution::Builder::disownEnumerationsToAdd() { + return ::capnp::_::PointerHelpers<::capnp::List< + ::tiledb::sm::serialization::capnp::Enumeration, + ::capnp::Kind::STRUCT>>::disown(_builder + .getPointerField( + ::capnp::bounded<3>() * + ::capnp::POINTERS)); +} + inline ::uint32_t Attribute::Reader::getCellValNum() const { return _reader.getDataField<::uint32_t>( ::capnp::bounded<0>() * ::capnp::ELEMENTS); @@ -16207,6 +16693,263 @@ inline ::capnp::Orphan<::capnp::Text> Attribute::Builder::disownOrder() { _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); } +inline bool Attribute::Reader::hasEnumerationName() const { + return !_reader.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Attribute::Builder::hasEnumerationName() { + return !_builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Attribute::Reader::getEnumerationName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Attribute::Builder::getEnumerationName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS)); +} +inline void Attribute::Builder::setEnumerationName( + ::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Attribute::Builder::initEnumerationName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + size); +} +inline void Attribute::Builder::adoptEnumerationName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> +Attribute::Builder::disownEnumerationName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<5>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasName() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasName() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Enumeration::Reader::getName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Enumeration::Builder::getName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Enumeration::Builder::initName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Enumeration::Builder::disownName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasPathName() const { + return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasPathName() { + return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Enumeration::Reader::getPathName() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Enumeration::Builder::getPathName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setPathName(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Enumeration::Builder::initPathName( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptPathName( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Enumeration::Builder::disownPathName() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasType() const { + return !_reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasType() { + return !_builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Text::Reader Enumeration::Reader::getType() const { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline ::capnp::Text::Builder Enumeration::Builder::getType() { + return ::capnp::_::PointerHelpers<::capnp::Text>::get( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setType(::capnp::Text::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Text>::set( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Text::Builder Enumeration::Builder::initType( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Text>::init( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptType( + ::capnp::Orphan<::capnp::Text>&& value) { + ::capnp::_::PointerHelpers<::capnp::Text>::adopt( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Text> Enumeration::Builder::disownType() { + return ::capnp::_::PointerHelpers<::capnp::Text>::disown( + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); +} + +inline ::uint32_t Enumeration::Reader::getCellValNum() const { + return _reader.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline ::uint32_t Enumeration::Builder::getCellValNum() { + return _builder.getDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void Enumeration::Builder::setCellValNum(::uint32_t value) { + _builder.setDataField<::uint32_t>( + ::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + +inline bool Enumeration::Reader::getOrdered() const { + return _reader.getDataField(::capnp::bounded<32>() * ::capnp::ELEMENTS); +} + +inline bool Enumeration::Builder::getOrdered() { + return _builder.getDataField( + ::capnp::bounded<32>() * ::capnp::ELEMENTS); +} +inline void Enumeration::Builder::setOrdered(bool value) { + _builder.setDataField( + ::capnp::bounded<32>() * ::capnp::ELEMENTS, value); +} + +inline bool Enumeration::Reader::hasData() const { + return !_reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasData() { + return !_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Data::Reader Enumeration::Reader::getData() const { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline ::capnp::Data::Builder Enumeration::Builder::getData() { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setData(::capnp::Data::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Data>::set( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Data::Builder Enumeration::Builder::initData( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Data>::init( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptData( + ::capnp::Orphan<::capnp::Data>&& value) { + ::capnp::_::PointerHelpers<::capnp::Data>::adopt( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Data> Enumeration::Builder::disownData() { + return ::capnp::_::PointerHelpers<::capnp::Data>::disown( + _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); +} + +inline bool Enumeration::Reader::hasOffsets() const { + return !_reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Enumeration::Builder::hasOffsets() { + return !_builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + .isNull(); +} +inline ::capnp::Data::Reader Enumeration::Reader::getOffsets() const { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline ::capnp::Data::Builder Enumeration::Builder::getOffsets() { + return ::capnp::_::PointerHelpers<::capnp::Data>::get( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} +inline void Enumeration::Builder::setOffsets(::capnp::Data::Reader value) { + ::capnp::_::PointerHelpers<::capnp::Data>::set( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + value); +} +inline ::capnp::Data::Builder Enumeration::Builder::initOffsets( + unsigned int size) { + return ::capnp::_::PointerHelpers<::capnp::Data>::init( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + size); +} +inline void Enumeration::Builder::adoptOffsets( + ::capnp::Orphan<::capnp::Data>&& value) { + ::capnp::_::PointerHelpers<::capnp::Data>::adopt( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::capnp::Data> Enumeration::Builder::disownOffsets() { + return ::capnp::_::PointerHelpers<::capnp::Data>::disown( + _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); +} + inline bool AttributeBufferHeader::Reader::hasName() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); @@ -20752,6 +21495,17 @@ inline ::capnp::Orphan<::capnp::Text> ConditionClause::Builder::disownOp() { _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); } +inline bool ConditionClause::Reader::getUseEnumeration() const { + return _reader.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); +} + +inline bool ConditionClause::Builder::getUseEnumeration() { + return _builder.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); +} +inline void ConditionClause::Builder::setUseEnumeration(bool value) { + _builder.setDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS, value); +} + inline bool ASTNode::Reader::getIsExpression() const { return _reader.getDataField(::capnp::bounded<0>() * ::capnp::ELEMENTS); } @@ -20986,6 +21740,17 @@ inline ::capnp::Orphan<::capnp::Text> ASTNode::Builder::disownCombinationOp() { _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); } +inline bool ASTNode::Reader::getUseEnumeration() const { + return _reader.getDataField(::capnp::bounded<1>() * ::capnp::ELEMENTS); +} + +inline bool ASTNode::Builder::getUseEnumeration() { + return _builder.getDataField(::capnp::bounded<1>() * ::capnp::ELEMENTS); +} +inline void ASTNode::Builder::setUseEnumeration(bool value) { + _builder.setDataField(::capnp::bounded<1>() * ::capnp::ELEMENTS, value); +} + inline bool Condition::Reader::hasClauses() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 0b0d5f822f88..ac7e0434b7c9 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -776,6 +776,9 @@ static Status condition_ast_to_capnp( const std::string op_str = query_condition_op_str(node->get_op()); ensure_qc_op_string_is_valid(op_str); ast_builder->setOp(op_str); + + // Store whether this expression should skip the enumeration lookup. + ast_builder->setUseEnumeration(node->use_enumeration()); } else { // Store the boolean expression tag. ast_builder->setIsExpression(true); @@ -823,6 +826,8 @@ static void clause_to_capnp( const std::string op_str = query_condition_op_str(node->get_op()); ensure_qc_op_string_is_valid(op_str); clause_builder->setOp(op_str); + + clause_builder->setUseEnumeration(node->use_enumeration()); } Status condition_to_capnp( @@ -996,8 +1001,10 @@ tdb_unique_ptr condition_ast_from_capnp( } ensure_qc_op_is_valid(op); + auto use_enumeration = ast_reader.getUseEnumeration(); + return tdb_unique_ptr( - tdb_new(ASTNodeVal, field_name, data, size, op)); + tdb_new(ASTNodeVal, field_name, data, size, op, use_enumeration)); } // Getting and validating the query condition combination operator. @@ -1049,8 +1056,10 @@ Status condition_from_capnp( } ensure_qc_op_is_valid(op); + bool use_enumeration = clause.getUseEnumeration(); + ast_nodes.push_back(tdb_unique_ptr( - tdb_new(ASTNodeVal, field_name, data, size, op))); + tdb_new(ASTNodeVal, field_name, data, size, op, use_enumeration))); } // Constructing the tree from the list of AST nodes. diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index 1bcfb60f3bcb..c8491ade3829 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -122,6 +122,12 @@ struct ArraySchema { dimensionLabels @14 :List(DimensionLabel); # Dimension labels of the array + + enumerations @15: List(Enumeration); + # Enumerations of the array + + enumerationPathMap @16: List(KV); + # Enumeration name to path map } struct DimensionLabel { @@ -167,6 +173,9 @@ struct ArraySchemaEvolution { timestampRange @2 :List(UInt64); # Timestamp range of array schema + + enumerationsToAdd @3 :List(Enumeration); + # Enumerations to be added } struct Attribute { @@ -194,6 +203,33 @@ struct Attribute { order @7 :Text; # The prescribed order of the data stored in the attribute + + enumerationName @8 :Text; + # Name of the enumeration for this attribute, if it has one +} + +struct Enumeration { +# Enumeration of values for use by Attributes + name @0 :Text; + # Enumeration name + + pathName @1 :Text; + # Enumeration path name + + type @2 :Text; + # Type of the Enumeration values + + cellValNum @3 :UInt32; + # Enumeration number of values per cell + + ordered @4 :Bool; + # Whether the enumeration is considered orderable + + data @5 :Data; + # The contents of the enumeration values + + offsets @6 :Data; + # The contents of the enumeration offsets buffer } struct AttributeBufferHeader { @@ -534,6 +570,9 @@ struct ConditionClause { op @2 :Text; # The comparison operation + + useEnumeration @3 :Bool; + # Whether or not to use the associated attribute's Enumeration } struct ASTNode { @@ -557,6 +596,9 @@ struct ASTNode { combinationOp @5 :Text; # The combination logical operator + + useEnumeration @6 :Bool; + # Whether or not to use the associated attribute's Enumeration } struct Condition { diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 8759e89d67ef..c74a034e323b 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -46,6 +46,7 @@ #include "tiledb/sm/array/array_directory.h" #include "tiledb/sm/array_schema/array_schema.h" #include "tiledb/sm/array_schema/array_schema_evolution.h" +#include "tiledb/sm/array_schema/enumeration.h" #include "tiledb/sm/consolidator/consolidator.h" #include "tiledb/sm/consolidator/fragment_consolidator.h" #include "tiledb/sm/enums/array_type.h" @@ -612,6 +613,11 @@ Status StorageManagerCanonical::array_create( array_uri.join_path(constants::array_schema_dir_name); RETURN_NOT_OK(vfs()->create_dir(array_schema_dir_uri)); + // Create the enumerations directory inside the array schema directory + URI array_enumerations_uri = + array_schema_dir_uri.join_path(constants::array_enumerations_dir_name); + RETURN_NOT_OK(vfs()->create_dir(array_enumerations_uri)); + // Create commit directory URI array_commit_uri = array_uri.join_path(constants::array_commits_dir_name); RETURN_NOT_OK(vfs()->create_dir(array_commit_uri)); @@ -717,12 +723,16 @@ Status StorageManager::array_evolve_schema( auto&& array_schema = array_dir.load_array_schema_latest(encryption_key); + // Load all enumerations for validation after evolution. + for (auto& name : array_schema->get_enumeration_names()) { + array_dir.load_enumeration(array_schema, name, encryption_key); + } + // Evolve schema - auto&& [st1, array_schema_evolved] = - schema_evolution->evolve_schema(array_schema); - RETURN_NOT_OK(st1); + auto array_schema_evolved = schema_evolution->evolve_schema(array_schema); + RETURN_NOT_OK(array_schema_evolved->check()); - Status st = store_array_schema(array_schema_evolved.value(), encryption_key); + Status st = store_array_schema(array_schema_evolved, encryption_key); if (!st.ok()) { logger_->status_no_return_value(st); return logger_->status(Status_StorageManagerError( @@ -1674,11 +1684,46 @@ Status StorageManagerCanonical::store_array_schema( URI array_schema_dir_uri = array_schema->array_uri().join_path(constants::array_schema_dir_name); RETURN_NOT_OK(vfs()->is_dir(array_schema_dir_uri, &schema_dir_exists)); + if (!schema_dir_exists) RETURN_NOT_OK(vfs()->create_dir(array_schema_dir_uri)); RETURN_NOT_OK(store_data_to_generic_tile(tile, schema_uri, encryption_key)); + // Create the `__enumerations` directory under `__schema` if it doesn't + // exist. This might happen if someone tries to add an enumeration to an + // array created before version 19. + bool enumerations_dir_exists = false; + URI array_enumerations_dir_uri = + array_schema_dir_uri.join_path(constants::array_enumerations_dir_name); + RETURN_NOT_OK( + vfs()->is_dir(array_enumerations_dir_uri, &enumerations_dir_exists)); + + if (!enumerations_dir_exists) { + RETURN_NOT_OK(vfs()->create_dir(array_enumerations_dir_uri)); + } + + // Serialize all enumerations into the `__enumerations` directory + for (auto& enmr_name : array_schema->get_loaded_enumeration_names()) { + auto enmr = array_schema->get_enumeration(enmr_name); + if (enmr == nullptr) { + return logger_->status(Status_StorageManagerError( + "Error serializing enumeration; Loaded enumeration is null")); + } + + SizeComputationSerializer enumeration_size_serializer; + enmr->serialize(enumeration_size_serializer); + + WriterTile tile{ + WriterTile::from_generic(enumeration_size_serializer.size())}; + Serializer serializer(tile.data(), tile.size()); + enmr->serialize(serializer); + + auto abs_enmr_uri = array_enumerations_dir_uri.join_path(enmr->path_name()); + RETURN_NOT_OK( + store_data_to_generic_tile(tile, abs_enmr_uri, encryption_key)); + } + return Status::Ok(); } diff --git a/tiledb/sm/tile/CMakeLists.txt b/tiledb/sm/tile/CMakeLists.txt index c23f54bec7ff..f52ea816f9b0 100644 --- a/tiledb/sm/tile/CMakeLists.txt +++ b/tiledb/sm/tile/CMakeLists.txt @@ -44,7 +44,6 @@ commence(object_library generic_tile_io) baseline buffer constants - context_resources tiledb_crypto filter_pipeline tile @@ -52,4 +51,10 @@ commence(object_library generic_tile_io) ) conclude(object_library) +# This is linked outside the object_library scope because ContextResources +# is recompiled as part of the capi_context_stub. Including context_resources +# here like this prevents many headaches revolving around duplicate symbols +# when linking executables. +target_link_libraries(compile_generic_tile_io PRIVATE context_resources) + add_test_subdirectory() diff --git a/tiledb/storage_format/uri/CMakeLists.txt b/tiledb/storage_format/uri/CMakeLists.txt index 6923c26548d7..4dbd4dd3f319 100644 --- a/tiledb/storage_format/uri/CMakeLists.txt +++ b/tiledb/storage_format/uri/CMakeLists.txt @@ -30,6 +30,6 @@ include(object_library) # `uri_format` object library # commence(object_library uri_format) - this_target_sources(parse_uri.cc) + this_target_sources(parse_uri.cc generate_uri.cc) this_target_object_libraries(baseline time uuid vfs) conclude(object_library)