Add mvcc support for catalog (#3301)

* catalog mvcc; bump extension version

* a bunch of fixes: 32 bit; java api tests; storage driver; clang-tidy; exception messages, etc.

* storage version

* fix extension exception test expected messages

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.15)
 
-project(Kuzu VERSION 0.4.1 LANGUAGES CXX C)
+project(Kuzu VERSION 0.4.1.1 LANGUAGES CXX C)
 
 find_package(Threads REQUIRED)
 
@@ -219,7 +219,7 @@ add_subdirectory(third_party)
 if(${BUILD_KUZU})
 add_definitions(-DKUZU_ROOT_DIRECTORY="${PROJECT_SOURCE_DIR}")
 add_definitions(-DKUZU_CMAKE_VERSION="${CMAKE_PROJECT_VERSION}")
-add_definitions(-DKUZU_EXTENSION_VERSION="0.3.1")
+add_definitions(-DKUZU_EXTENSION_VERSION="0.3.2")
 
 include_directories(src/include)
 

extension/duckdb/src/duckdb_catalog.cpp

@@ -41,9 +41,10 @@ static std::string getQuery(const binder::BoundCreateTableInfo& info) {
         extraInfo->schemaName, info.tableName);
 }
 
+// TODO(Ziyi): Do u need to pass in `dbPath` and `catalogName` here? Why not use the private field?
 void DuckDBCatalog::createForeignTable(duckdb::Connection& con, const std::string& tableName,
     const std::string& dbPath, const std::string& catalogName) {
-    auto tableID = assignNextTableID();
+    auto tableID = tables->assignNextOID();
     auto info = bindCreateTableInfo(con, tableName, dbPath, catalogName);
     if (info == nullptr) {
         return;
@@ -58,12 +59,12 @@ void DuckDBCatalog::createForeignTable(duckdb::Connection& con, const std::strin
     }
     DuckDBScanBindData bindData(getQuery(*info), std::move(columnTypes), std::move(columnNames),
         std::bind(&DuckDBCatalog::getConnection, this, dbPath));
-    auto tableEntry = std::make_unique<catalog::DuckDBTableCatalogEntry>(info->tableName, tableID,
-        getScanFunction(std::move(bindData)));
+    auto tableEntry = std::make_unique<catalog::DuckDBTableCatalogEntry>(tables.get(),
+        info->tableName, tableID, getScanFunction(std::move(bindData)));
     for (auto& propertyInfo : extraInfo->propertyInfos) {
         tableEntry->addProperty(propertyInfo.name, propertyInfo.type.copy());
     }
-    tables->createEntry(std::move(tableEntry));
+    tables->createEntry(&transaction::DUMMY_WRITE_TRANSACTION, std::move(tableEntry));
 }
 
 static bool getTableInfo(duckdb::Connection& con, const std::string& tableName,
@@ -91,6 +92,7 @@ static bool getTableInfo(duckdb::Connection& con, const std::string& tableName,
     return true;
 }
 
+// TODO(Ziyi): Do u need to pass in `catalogName` here? Why not use the private field?
 bool DuckDBCatalog::bindPropertyInfos(duckdb::Connection& con, const std::string& tableName,
     const std::string& catalogName, std::vector<binder::PropertyInfo>& propertyInfos) {
     std::vector<common::LogicalType> columnTypes;
@@ -106,6 +108,7 @@ bool DuckDBCatalog::bindPropertyInfos(duckdb::Connection& con, const std::string
     return true;
 }
 
+// TODO(Ziyi): Do u need to pass in `dbPath` and `catalogName` here? Why not use the private field?
 std::unique_ptr<binder::BoundCreateTableInfo> DuckDBCatalog::bindCreateTableInfo(
     duckdb::Connection& con, const std::string& tableName, const std::string& dbPath,
     const std::string& catalogName) {
@@ -122,6 +125,7 @@ std::string DuckDBCatalog::getDefaultSchemaName() const {
     return "main";
 }
 
+// TODO(Ziyi): Do u need to pass in `dbPath` here? Why not use the private field?
 std::pair<duckdb::DuckDB, duckdb::Connection> DuckDBCatalog::getConnection(
     const std::string& dbPath) const {
     try {

extension/duckdb/src/duckdb_table_catalog_entry.cpp

@@ -1,19 +1,28 @@
 #include "duckdb_table_catalog_entry.h"
 
+#include "binder/ddl/bound_create_table_info.h"
+
 namespace kuzu {
 namespace catalog {
 
-DuckDBTableCatalogEntry::DuckDBTableCatalogEntry(std::string name, common::table_id_t tableID,
-    function::TableFunction scanFunction)
-    : TableCatalogEntry{CatalogEntryType::FOREIGN_TABLE_ENTRY, std::move(name), tableID},
+DuckDBTableCatalogEntry::DuckDBTableCatalogEntry(CatalogSet* set, std::string name,
+    common::table_id_t tableID, function::TableFunction scanFunction)
+    : TableCatalogEntry{set, CatalogEntryType::FOREIGN_TABLE_ENTRY, std::move(name), tableID},
       scanFunction{std::move(scanFunction)} {}
 
 common::TableType DuckDBTableCatalogEntry::getTableType() const {
     return common::TableType::FOREIGN;
 }
 
-std::unique_ptr<CatalogEntry> DuckDBTableCatalogEntry::copy() const {
-    return std::make_unique<DuckDBTableCatalogEntry>(*this);
+std::unique_ptr<TableCatalogEntry> DuckDBTableCatalogEntry::copy() const {
+    auto other = std::make_unique<DuckDBTableCatalogEntry>(set, name, tableID, scanFunction);
+    other->copyFrom(*this);
+    return other;
+}
+
+std::unique_ptr<binder::BoundExtraCreateCatalogEntryInfo>
+DuckDBTableCatalogEntry::getBoundExtraCreateInfo(transaction::Transaction*) const {
+    KU_UNREACHABLE;
 }
 
 } // namespace catalog

extension/duckdb/src/include/duckdb_table_catalog_entry.h

@@ -11,7 +11,7 @@ class DuckDBTableCatalogEntry final : public TableCatalogEntry {
     //===--------------------------------------------------------------------===//
     // constructors
     //===--------------------------------------------------------------------===//
-    DuckDBTableCatalogEntry(std::string name, common::table_id_t tableID,
+    DuckDBTableCatalogEntry(CatalogSet* set, std::string name, common::table_id_t tableID,
         function::TableFunction scanFunction);
 
     //===--------------------------------------------------------------------===//
@@ -23,7 +23,11 @@ class DuckDBTableCatalogEntry final : public TableCatalogEntry {
     //===--------------------------------------------------------------------===//
     // serialization & deserialization
     //===--------------------------------------------------------------------===//
-    std::unique_ptr<CatalogEntry> copy() const override;
+    std::unique_ptr<TableCatalogEntry> copy() const override;
+
+private:
+    std::unique_ptr<binder::BoundExtraCreateCatalogEntryInfo> getBoundExtraCreateInfo(
+        transaction::Transaction* transaction) const override;
 
 private:
     function::TableFunction scanFunction;

extension/duckdb/test/test_files/duckdb.test

@@ -37,7 +37,7 @@ The 😂😃🧘🏻‍♂️🌍🌦️🍞🚗 movie|2544| the movie is very v
 49992|50|31.582059|False|2056-05-02||[62,24,94]|[LpQO8OT3x45a]|[[268,281,166],[144,16,126,208,298],[22,287]]|{ID: 936, "name": sGPSafxMAhKiP}
 -STATEMENT LOAD FROM tinysnb.person1 RETURN *;
 ---- error
-Catalog exception: Table: person1 does not exist.
+Catalog exception: person1 does not exist in catalog.
 -STATEMENT DETACH tinysnb;
 ---- 1
 Detached database successfully.
@@ -116,7 +116,7 @@ Used database successfully.
 -LOG WrongTableName
 -STATEMENT CALL TABLE_INFO('tinysnb.personxx') RETURN *;
 ---- error
-Catalog exception: Table: personxx does not exist.
+Catalog exception: personxx does not exist in catalog.
 -LOG WrongCatalogName
 -STATEMENT CALL TABLE_INFO('tinysnbxx.person') RETURN *;
 ---- error

extension/postgres/test/test_files/postgres.test

@@ -29,7 +29,7 @@ Sóló cón tu párejâ|126|this is a very very good movie|{rating: 5.300000, st
 The 😂😃🧘🏻‍♂️🌍🌦️🍞🚗 movie|2544|the movie is very very good|{rating: 7.000000, stars: 10, "views": 982, "release": 2018-11-13 13:33:11, release_ns: 2018-11-13 13:33:11, release_ms: 2018-11-13 13:33:11, release_sec: 2018-11-13 13:33:11, release_tz: 2018-11-13 05:33:11+00, film: 2014-09-12, u8: 12, u16: 120, u32: 55, u64: 1, hugedata: -1844674407370954899456.000000}|\x5CxAB\x5CxCD|[{"key": audience1, "value": 33}]
 -STATEMENT LOAD FROM tinysnb.person1 RETURN *;
 ---- error
-Catalog exception: Table: person1 does not exist.
+Catalog exception: person1 does not exist in catalog.
 -STATEMENT LOAD FROM tinysnb1.person RETURN *;
 ---- error
 Binder exception: No database named tinysnb1 has been attached.

src/binder/bind/bind_ddl.cpp

@@ -11,7 +11,6 @@
 #include "common/exception/message.h"
 #include "common/string_format.h"
 #include "common/types/types.h"
-#include "main/client_context.h"
 #include "parser/ddl/alter.h"
 #include "parser/ddl/create_table.h"
 #include "parser/ddl/create_table_info.h"

src/binder/bind/bind_graph_pattern.cpp

@@ -575,7 +575,7 @@ std::shared_ptr<NodeExpression> Binder::createQueryNode(const std::string& parse
 
 void Binder::bindQueryNodeProperties(NodeExpression& node) {
     auto tableSchemas =
-        clientContext->getCatalog()->getTableSchemas(clientContext->getTx(), node.getTableIDs());
+        clientContext->getCatalog()->getTableEntries(clientContext->getTx(), node.getTableIDs());
     auto propertyNames = getPropertyNames(tableSchemas);
     for (auto& propertyName : propertyNames) {
         auto property = createPropertyExpression(propertyName, node.getUniqueName(),
@@ -591,19 +591,19 @@ std::vector<table_id_t> Binder::bindTableIDs(const std::vector<std::string>& tab
     std::unordered_set<common::table_id_t> tableIDSet;
     if (tableNames.empty()) { // Rewrite empty table names as all tables.
         if (nodePattern) {    // Fill all node table schemas to node pattern.
-            if (!catalog->containsNodeTable(tx)) {
-                throw BinderException("No node table exists in database.");
-            }
             for (auto tableID : catalog->getNodeTableIDs(tx)) {
                 tableIDSet.insert(tableID);
             }
-        } else { // Fill all rel table schemas to rel pattern.
-            if (!catalog->containsRelTable(tx)) {
-                throw BinderException("No rel table exists in database.");
+            if (tableIDSet.empty()) {
+                throw BinderException("No node table exists in database.");
             }
+        } else { // Fill all rel table schemas to rel pattern.
             for (auto tableID : catalog->getRelTableIDs(tx)) {
                 tableIDSet.insert(tableID);
             }
+            if (tableIDSet.empty()) {
+                throw BinderException("No rel table exists in database.");
+            }
         }
     } else {
         for (auto& tableName : tableNames) {

src/binder/bind/bind_standalone_call.cpp

@@ -2,6 +2,7 @@
 #include "binder/bound_standalone_call.h"
 #include "binder/expression/expression_util.h"
 #include "binder/expression_visitor.h"
+#include "common/cast.h"
 #include "common/exception/binder.h"
 #include "extension/extension.h"
 #include "main/db_config.h"

src/binder/bind/read/bind_load_from.cpp

@@ -26,8 +26,8 @@ static TableFunction getObjectScanFunc(const ObjectScanSource* objectSource,
     }
     auto tableName = objectSource->objectNames[1];
     auto attachedCatalog = attachedDB->getCatalog();
-    auto tableID = attachedCatalog->getTableID(tableName);
-    auto entry = attachedCatalog->getTableCatalogEntry(tableID);
+    auto tableID = attachedCatalog->getTableID(clientContext->getTx(), tableName);
+    auto entry = attachedCatalog->getTableCatalogEntry(clientContext->getTx(), tableID);
     auto tableEntry = ku_dynamic_cast<CatalogEntry*, TableCatalogEntry*>(entry);
     return tableEntry->getScanFunction();
 }
@@ -58,6 +58,7 @@ std::unique_ptr<BoundReadingClause> Binder::bindLoadFrom(const ReadingClause& re
                 throw BinderException(ExceptionMessage::variableNotInScope(objectName));
             }
         } else if (objectSource->objectNames.size() == 2) {
+            // Bind external database table
             scanFunction = getObjectScanFunc(objectSource, clientContext);
             auto bindInput = function::TableFuncBindInput();
             bindData = scanFunction.bindFunc(clientContext, &bindInput);

src/catalog/CMakeLists.txt

@@ -3,7 +3,6 @@ add_subdirectory(catalog_entry)
 add_library(kuzu_catalog
         OBJECT
         catalog.cpp
-        catalog_content.cpp
         catalog_set.cpp
         property.cpp)