Add Enumerated Data Types

CMakeLists.txt

@@ -227,7 +227,10 @@ else()
     "$<$<CONFIG:Release,RelWithDebInfo>:-O3>"
     "$<$<CONFIG:RelWithDebInfo>:-g3;-ggdb3;-gdwarf-3>")
   if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-    add_compile_options("$<$<CONFIG:Debug>:-fstandalone-debug>")
+    add_compile_options(
+      "$<$<CONFIG:Debug>:-fstandalone-debug>"
+      "$<$<CONFIG:Coverage>:-fprofile-instr-generate;-fcoverage-mapping>")
+    add_link_options("$<$<CONFIG:Coverage>:--coverage;-fprofile-instr-generate;-fcoverage-mapping>")
   endif()
 
   # Disable newer Apple Clang warnings about unqualified calls to std::move
@@ -345,6 +348,7 @@ list(APPEND TILEDB_C_API_RELATIVE_HEADERS
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/dimension/dimension_api_external.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/dimension_label/dimension_label_api_external.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/domain/domain_api_external.h"
+    "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/enumeration/enumeration_api_experimental.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/error/error_api_external.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filesystem/filesystem_api_enum.h"
     "${CMAKE_SOURCE_DIR}/tiledb/api/c_api/filesystem/filesystem_api_external.h"

examples/cpp_api/enumerations.cc

@@ -0,0 +1,160 @@
+/**
+ * @file enumerations.cc
+ *
+ * @section LICENSE
+ *
+ * The MIT License
+ *
+ * @copyright Copyright (c) 2023 TileDB, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ * @section DESCRIPTION
+ *
+ * When run, this program will create a simple 2D sparse array with an
+ * enumeration and then use a query condition to select data based on the
+ * enumerations values.
+ */
+
+#include <iostream>
+#include <tiledb/tiledb>
+#include <tiledb/tiledb_experimental>
+
+using namespace tiledb;
+
+// Name of array.
+std::string array_name("enumerations_example_array");
+
+void create_array() {
+  Context ctx;
+
+  // First some standard boiler plate for creating an array. Nothing here
+  // is important or required for Enumerations.
+  ArraySchema schema(ctx, TILEDB_SPARSE);
+  schema.set_order({{TILEDB_ROW_MAJOR, TILEDB_ROW_MAJOR}});
+
+  auto dim1 = Dimension::create<int>(ctx, "rows", {{1, 4}}, 4);
+  auto dim2 = Dimension::create<int>(ctx, "cols", {{1, 4}}, 4);
+
+  Domain dom(ctx);
+  dom.add_dimensions(dim1, dim2);
+  schema.set_domain(dom);
+
+  // The most basic enumeration only requires a name and a vector of values
+  // to use as lookups. Enumeration values can be any supported TileDB type
+  // although they are most commonly strings.
+  std::vector<std::string> values = {"red", "green", "blue"};
+  auto enmr = Enumeration::create(ctx, "colors", values);
+
+  // To use an enumeration with an attribute, we just set the enumeration
+  // name on the attribute before adding it to the schema. Attributes that
+  // use an enumeration are required to have an integral type that is wide
+  // enough to index the entire enumeration. For instance, an enumeration with
+  // 256 values can fit in a uint8_t type, but at 257 values, the attribute
+  // would require a type of int16_t at a minimum.
+  auto attr = Attribute::create<uint8_t>(ctx, "attr");
+  AttributeExperimental::set_enumeration_name(ctx, attr, "colors");
+
+  // The enumeration must be added to the schema before any attribute that
+  // references the enumeration so that the requirements of the attribute
+  // can be accurately checked.
+  ArraySchemaExperimental::add_enumeration(ctx, schema, enmr);
+
+  // Finally, we add the attribute as per normal.
+  schema.add_attribute(attr);
+
+  Array::create(array_name, schema);
+}
+
+void write_array() {
+  Context ctx;
+
+  std::vector<int> row_data = {1, 2, 2};
+  std::vector<int> col_data = {1, 4, 3};
+
+  // Attribute data for an enumeration is just numeric indices into the
+  // list of enumeration values.
+  std::vector<uint8_t> attr_data = {2, 1, 1};
+
+  // Open the array for writing and create the query.
+  Array array(ctx, array_name, TILEDB_WRITE);
+  Query query(ctx, array);
+  query.set_layout(TILEDB_UNORDERED)
+      .set_data_buffer("rows", row_data)
+      .set_data_buffer("cols", col_data)
+      .set_data_buffer("attr", attr_data);
+
+  // Write and close the array
+  query.submit();
+  array.close();
+}
+
+void read_array() {
+  Context ctx;
+
+  // This is all standard boiler plate for reading from an array. The
+  // section below will demonstrate using a QueryCondition to select
+  // rows based on the enumeration.
+  Array array(ctx, array_name, TILEDB_READ);
+  Subarray subarray(ctx, array);
+  subarray.add_range(0, 1, 4).add_range(1, 1, 4);
+
+  std::vector<int> row_data(16);
+  std::vector<int> col_data(16);
+  std::vector<uint8_t> attr_data(16);
+
+  Query query(ctx, array, TILEDB_READ);
+  query.set_subarray(subarray)
+      .set_layout(TILEDB_ROW_MAJOR)
+      .set_data_buffer("rows", row_data)
+      .set_data_buffer("cols", col_data)
+      .set_data_buffer("attr", attr_data);
+
+  // Query conditions apply against the enumeration's values instead of the
+  // integral data. Thus, we can select values here using color names instead
+  // of the integer indices.
+  QueryCondition qc(ctx);
+  qc.init("attr", "green", 5, TILEDB_EQ);
+  query.set_condition(qc);
+
+  // Finally, submit and display the query results
+  query.submit();
+  array.close();
+
+  // Print out the results.
+  auto result_num = (int)query.result_buffer_elements()["attr"].second;
+  for (int i = 0; i < result_num; i++) {
+    int r = row_data[i];
+    int c = col_data[i];
+    int a = attr_data[i];
+    std::cout << "Cell (" << r << ", " << c << ") has attr " << a << "\n";
+  }
+}
+
+int main() {
+  Context ctx;
+
+  if (Object::object(ctx, array_name).type() != Object::Type::Array) {
+    create_array();
+    write_array();
+  }
+
+  read_array();
+  return 0;
+}

format_spec/FORMAT_SPEC.md

@@ -4,7 +4,7 @@ title: Format Specification
 
 **Notes:**
 
-* The current TileDB format version number is **18** (`uint32_t`).
+* The current TileDB format version number is **20** (`uint32_t`).
 * Data written by TileDB and referenced in this document is **little-endian**
   with the following exceptions:
 

format_spec/enumeration.md

@@ -0,0 +1,27 @@
+---
+title: Enumerations
+---
+
+## Main Structure
+
+```
+my_array                                    # array folder
+   |  ...
+   |_ schema                      # ArraySchema directory named `__schema`
+         |_ enumerations          # Enumeration directory named `__enumerations`
+               |_ enumeration     # enumeration data with names `__uuid_v`
+
+
+Enumeration data is stored in a subdirectory of the [array schema][./array_schema.md]
+directory. Enumerations are stored using [Generic Tiles][./generic_tile.md].
+
+| **Field** | **Type** | **Description** |
+| :--- | :--- | :--- |
+| Version number | `uint32_t` | Enumerations version number |
+| Datatype | `uint8_t` | The datatype of the enumeration values |
+| Cell Val Num | `uint32_t` | The cell val num of the enumeration values |
+| Ordered | `bool` | Whether the enumeration values should be considered ordered |
+| Data Size | `uint64_t` | The number of bytes used to store the values |
+| Data | `uint8_t` * Data Size | The data for the enumeration values |
+| Offsets Size | `uint64_t` | The number of bytes used to store offsets if cell_var_num is TILEDB_VAR_NUM |
+| Offsets | `uint8_t` * Offsets Size | The offsets data for the enumeration if cell_var_num is TILEDB_VAR_NUM |

test/CMakeLists.txt

@@ -134,6 +134,7 @@ set(TILEDB_UNIT_TEST_SOURCES
   src/unit-capi-dense_neg.cc
   src/unit-capi-dense_vector.cc
   src/unit-capi-enum_values.cc
+  src/unit-capi-enumerations.cc
   src/unit-capi-error.cc
   src/unit-capi-filestore.cc
   src/unit-capi-fill_values.cc
@@ -172,6 +173,8 @@ set(TILEDB_UNIT_TEST_SOURCES
   src/unit-dimension.cc
   src/unit-duplicates.cc
   src/unit-empty-var-length.cc
+  src/unit-enumerations.cc
+  src/unit-enum-helpers.cc
   src/unit-filter-buffer.cc
   src/unit-filter-pipeline.cc
   src/unit-global-order.cc
@@ -213,6 +216,7 @@ if (TILEDB_CPP_API)
     src/unit-cppapi-deletes.cc
     src/unit-cppapi-dense-qc-coords-mode.cc
     src/unit-cppapi-time.cc
+    src/unit-cppapi-enumerations.cc
     src/unit-cppapi-fill_values.cc
     src/unit-cppapi-filter.cc
     src/unit-cppapi-float-scaling-filter.cc

test/src/unit-capi-array_schema.cc

@@ -959,7 +959,10 @@ int ArraySchemaFx::get_schema_file_struct(const char* path, void* data) {
   int rc = tiledb_vfs_is_dir(ctx, vfs, path, &is_dir);
   CHECK(rc == TILEDB_OK);
 
-  data_struct->path = path;
+  if (!is_dir) {
+    data_struct->path = path;
+  }
+
   return 1;
 }
 

test/src/unit-capi-enumerations.cc

@@ -0,0 +1,119 @@
+/**
+ * @file unit-capi-enumerations.cc
+ *
+ * @section LICENSE
+ *
+ * The MIT License
+ *
+ * @copyright Copyright (c) 2023 TileDB Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ * @section DESCRIPTION
+ *
+ * Tests for the various Enumerations C API errors.
+ */
+
+#include <test/support/tdb_catch.h>
+#include "tiledb/sm/c_api/tiledb.h"
+#include "tiledb/sm/c_api/tiledb_experimental.h"
+
+#include <iostream>
+
+TEST_CASE(
+    "C API: Test invalid attribute for tiledb_attribute_set_enumeration_name",
+    "[enumeration][capi][error]") {
+  tiledb_ctx_t* ctx;
+
+  auto rc = tiledb_ctx_alloc(nullptr, &ctx);
+  CHECK(rc == TILEDB_OK);
+
+  rc = tiledb_attribute_set_enumeration_name(ctx, nullptr, "enmr_name");
+  REQUIRE(rc == TILEDB_ERR);
+}
+
+TEST_CASE(
+    "C API: Test invalid attribute for tiledb_attribute_get_enumeration_name",
+    "[enumeration][capi][error]") {
+  tiledb_ctx_t* ctx;
+
+  auto rc = tiledb_ctx_alloc(nullptr, &ctx);
+  CHECK(rc == TILEDB_OK);
+
+  tiledb_string_t* name;
+  rc = tiledb_attribute_get_enumeration_name(ctx, nullptr, &name);
+  REQUIRE(rc == TILEDB_ERR);
+}
+
+TEST_CASE(
+    "C API: Test invalid array schema for tiledb_array_schema_add_enumeration",
+    "[enumeration][capi][error]") {
+  tiledb_ctx_t* ctx;
+
+  auto rc = tiledb_ctx_alloc(nullptr, &ctx);
+  CHECK(rc == TILEDB_OK);
+
+  tiledb_enumeration_t* enmr;
+  uint32_t values[5] = {1, 2, 3, 4, 5};
+  rc = tiledb_enumeration_alloc(
+      ctx,
+      "an_enumeration",
+      TILEDB_UINT32,
+      1,
+      0,
+      values,
+      sizeof(uint32_t) * 5,
+      nullptr,
+      0,
+      &enmr);
+  REQUIRE(rc == TILEDB_OK);
+
+  rc = tiledb_array_schema_add_enumeration(ctx, nullptr, enmr);
+  REQUIRE(rc == TILEDB_ERR);
+}
+
+TEST_CASE(
+    "C API: Test invalid array for tiledb_array_get_enumeration",
+    "[enumeration][capi][error]") {
+  tiledb_ctx_t* ctx;
+
+  auto rc = tiledb_ctx_alloc(nullptr, &ctx);
+  CHECK(rc == TILEDB_OK);
+
+  tiledb_enumeration_t* enmr;
+  rc = tiledb_array_get_enumeration(ctx, nullptr, "an_enumeration", &enmr);
+  REQUIRE(rc == TILEDB_ERR);
+}
+
+TEST_CASE(
+    "C API: Test invalid enumeration name for tiledb_array_get_enumeration",
+    "[enumeration][capi][error]") {
+  tiledb_ctx_t* ctx;
+
+  auto rc = tiledb_ctx_alloc(nullptr, &ctx);
+  CHECK(rc == TILEDB_OK);
+
+  tiledb_array_t* array;
+  rc = tiledb_array_alloc(ctx, "array_uri", &array);
+  REQUIRE(rc == TILEDB_OK);
+
+  tiledb_enumeration_t* enmr;
+  rc = tiledb_array_get_enumeration(ctx, array, nullptr, &enmr);
+  REQUIRE(rc == TILEDB_ERR);
+}