diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml index 0d5ff02938e..2bf79ac460f 100644 --- a/.github/workflows/ci-workflow.yml +++ b/.github/workflows/ci-workflow.yml @@ -46,6 +46,15 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Update PostgreSQL host + working-directory: extension/postgres_scanner/test/test_files + env: + FNAME: postgres_scanner.test + FIND: "localhost" + run: | + node -e 'fs=require("fs");fs.readFile(process.env.FNAME,"utf8",(err,data)=>{if(err!=null)throw err;fs.writeFile(process.env.FNAME,data.replaceAll(process.env.FIND,process.env.PG_HOST),"utf8",e=>{if(e!=null)throw e;});});' + cat postgres_scanner.test + - name: Ensure Python dependencies run: | pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu @@ -179,6 +188,15 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Update PostgreSQL host + working-directory: extension/postgres_scanner/test/test_files + env: + FNAME: postgres_scanner.test + FIND: "localhost" + run: | + node -e 'fs=require("fs");fs.readFile(process.env.FNAME,"utf8",(err,data)=>{if(err!=null)throw err;fs.writeFile(process.env.FNAME,data.replaceAll(process.env.FIND,process.env.PG_HOST),"utf8",e=>{if(e!=null)throw e;});});' + cat postgres_scanner.test + - name: Ensure Python dependencies run: | pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu @@ -225,10 +243,20 @@ jobs: AWS_S3_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} + PG_HOST: ${{ secrets.PG_HOST }} RUN_ID: "$(hostname)-$([Math]::Floor((Get-Date).TimeOfDay.TotalSeconds))" steps: - uses: actions/checkout@v3 + - name: Update PostgreSQL host + working-directory: extension/postgres_scanner/test/test_files + env: + FNAME: postgres_scanner.test + FIND: "localhost" + run: | + node -e 'fs=require("fs");fs.readFile(process.env.FNAME,"utf8",(err,data)=>{if(err!=null)throw err;fs.writeFile(process.env.FNAME,data.replaceAll(process.env.FIND,process.env.PG_HOST),"utf8",e=>{if(e!=null)throw e;});});' + cat postgres_scanner.test + - name: Ensure Python dependencies run: | pip install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu @@ -430,10 +458,20 @@ jobs: AWS_S3_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} + PG_HOST: ${{ secrets.PG_HOST }} RUN_ID: "$(hostname)-$(date +%s)" steps: - uses: actions/checkout@v3 + - name: Update PostgreSQL host + working-directory: extension/postgres_scanner/test/test_files + env: + FNAME: postgres_scanner.test + FIND: "localhost" + run: | + node -e 'fs=require("fs");fs.readFile(process.env.FNAME,"utf8",(err,data)=>{if(err!=null)throw err;fs.writeFile(process.env.FNAME,data.replaceAll(process.env.FIND,process.env.PG_HOST),"utf8",e=>{if(e!=null)throw e;});});' + cat postgres_scanner.test + - name: Ensure Python dependencies run: | pip3 install torch~=2.0.0 --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/Makefile b/Makefile index e00fbcb71f8..1eeae88d6a6 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ allconfig: $(call config-cmake-release, \ -DBUILD_BENCHMARK=TRUE \ -DBUILD_EXAMPLES=TRUE \ - -DBUILD_EXTENSIONS="httpfs;duckdb_scanner" \ + -DBUILD_EXTENSIONS="httpfs;duckdb_scanner;postgres_scanner" \ -DBUILD_JAVA=TRUE \ -DBUILD_NODEJS=TRUE \ -DBUILD_PYTHON=TRUE \ @@ -79,7 +79,7 @@ alldebug: $(call run-cmake-debug, \ -DBUILD_BENCHMARK=TRUE \ -DBUILD_EXAMPLES=TRUE \ - -DBUILD_EXTENSIONS="httpfs;duckdb_scanner" \ + -DBUILD_EXTENSIONS="httpfs;duckdb_scanner;postgres_scanner" \ -DBUILD_JAVA=TRUE \ -DBUILD_NODEJS=TRUE \ -DBUILD_PYTHON=TRUE \ @@ -156,7 +156,7 @@ example: extension-test: $(call run-cmake-release, \ - -DBUILD_EXTENSIONS="httpfs;duckdb_scanner" \ + -DBUILD_EXTENSIONS="httpfs;duckdb_scanner;postgres_scanner" \ -DBUILD_EXTENSION_TESTS=TRUE \ ) ctest --test-dir build/release/extension --output-on-failure -j ${TEST_JOBS} @@ -164,13 +164,13 @@ extension-test: extension-debug: $(call run-cmake-debug, \ - -DBUILD_EXTENSIONS="httpfs;duckdb_scanner" \ + -DBUILD_EXTENSIONS="httpfs;duckdb_scanner;postgres_scanner" \ -DBUILD_KUZU=FALSE \ ) extension-release: $(call run-cmake-release, \ - -DBUILD_EXTENSIONS="httpfs;duckdb_scanner" \ + -DBUILD_EXTENSIONS="httpfs;duckdb_scanner;postgres_scanner" \ -DBUILD_KUZU=FALSE \ ) diff --git a/extension/CMakeLists.txt b/extension/CMakeLists.txt index 53043bc5896..0f362842346 100644 --- a/extension/CMakeLists.txt +++ b/extension/CMakeLists.txt @@ -10,6 +10,10 @@ if ("duckdb_scanner" IN_LIST BUILD_EXTENSIONS) endif() endif() +if ("postgres_scanner" IN_LIST BUILD_EXTENSIONS) + add_subdirectory(postgres_scanner) +endif() + if (${BUILD_EXTENSION_TESTS}) add_definitions(-DTEST_FILES_DIR="extension") add_subdirectory(${CMAKE_SOURCE_DIR}/test/gtest ${CMAKE_CURRENT_BINARY_DIR}/test/gtest EXCLUDE_FROM_ALL) diff --git a/extension/duckdb_scanner/CMakeLists.txt b/extension/duckdb_scanner/CMakeLists.txt index 8adca714d24..0d1279f5996 100644 --- a/extension/duckdb_scanner/CMakeLists.txt +++ b/extension/duckdb_scanner/CMakeLists.txt @@ -14,12 +14,6 @@ add_library(duckdb_scanner src/duckdb_catalog.cpp src/duckdb_table_catalog_entry.cpp) -set_target_properties(duckdb_scanner PROPERTIES - OUTPUT_NAME duckdb_scanner - PREFIX "lib" - SUFFIX ".kuzu_extension" -) - set_target_properties(duckdb_scanner PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" diff --git a/extension/duckdb_scanner/src/duckdb_catalog.cpp b/extension/duckdb_scanner/src/duckdb_catalog.cpp index 1bf5cb263fc..8fa2d4919d9 100644 --- a/extension/duckdb_scanner/src/duckdb_catalog.cpp +++ b/extension/duckdb_scanner/src/duckdb_catalog.cpp @@ -1,28 +1,127 @@ #include "duckdb_catalog.h" +#include "common/exception/binder.h" +#include "duckdb_type_converter.h" + namespace kuzu { namespace duckdb_scanner { -common::table_id_t DuckDBCatalogContent::createForeignTable( - const binder::BoundCreateTableInfo& info) { - auto tableID = assignNextTableID(); +void DuckDBCatalogContent::init( + const std::string& dbPath, const std::string& catalogName, main::ClientContext* context) { + auto con = getConnection(dbPath); + auto query = common::stringFormat( + "select table_name from information_schema.tables where table_catalog = '{}' and " + "table_schema = '{}';", + catalogName, getDefaultSchemaName()); + auto result = con.Query(query); + std::unique_ptr resultChunk; + try { + resultChunk = result->Fetch(); + } catch (std::exception& e) { throw common::BinderException(e.what()); } + if (resultChunk->size() == 0) { + return; + } + common::ValueVector tableNamesVector{ + *common::LogicalType::STRING(), context->getMemoryManager()}; + duckdb_scanner::duckdb_conversion_func_t conversionFunc; + duckdb_scanner::getDuckDBVectorConversionFunc(common::PhysicalTypeID::STRING, conversionFunc); + conversionFunc(resultChunk->data[0], tableNamesVector, resultChunk->size()); + for (auto i = 0u; i < resultChunk->size(); i++) { + auto tableName = tableNamesVector.getValue(i).getAsString(); + createForeignTable(con, tableName, dbPath, catalogName); + } +} + +static std::string getQuery(const binder::BoundCreateTableInfo& info) { auto extraInfo = common::ku_dynamic_cast(info.extraInfo.get()); + return common::stringFormat( + "SELECT * FROM {}.{}.{}", extraInfo->catalogName, extraInfo->schemaName, info.tableName); +} + +void DuckDBCatalogContent::createForeignTable(duckdb::Connection& con, const std::string& tableName, + const std::string& dbPath, const std::string& catalogName) { + auto tableID = assignNextTableID(); + auto info = bindCreateTableInfo(con, tableName, dbPath, catalogName); + if (info == nullptr) { + return; + } + auto extraInfo = common::ku_dynamic_cast(info->extraInfo.get()); std::vector columnTypes; std::vector columnNames; for (auto& propertyInfo : extraInfo->propertyInfos) { columnNames.push_back(propertyInfo.name); columnTypes.push_back(propertyInfo.type); } - DuckDBScanBindData bindData(common::stringFormat("SELECT * FROM {}", info.tableName), - extraInfo->dbPath, std::move(columnTypes), std::move(columnNames)); + DuckDBScanBindData bindData(getQuery(*info), std::move(columnTypes), std::move(columnNames), + std::bind(&DuckDBCatalogContent::getConnection, this, dbPath)); auto tableEntry = std::make_unique( - info.tableName, tableID, getScanFunction(std::move(bindData))); + info->tableName, tableID, getScanFunction(std::move(bindData))); for (auto& propertyInfo : extraInfo->propertyInfos) { tableEntry->addProperty(propertyInfo.name, propertyInfo.type.copy()); } tables->createEntry(std::move(tableEntry)); - return tableID; +} + +static bool getTableInfo(duckdb::Connection& con, const std::string& tableName, + const std::string& schemaName, const std::string& catalogName, + std::vector& columnTypes, std::vector& columnNames) { + auto query = + common::stringFormat("select data_type,column_name from information_schema.columns where " + "table_name = '{}' and table_schema = '{}' and table_catalog = '{}';", + tableName, schemaName, catalogName); + auto result = con.Query(query); + if (result->RowCount() == 0) { + return false; + } + columnTypes.reserve(result->RowCount()); + columnNames.reserve(result->RowCount()); + for (auto i = 0u; i < result->RowCount(); i++) { + try { + columnTypes.push_back(DuckDBTypeConverter::convertDuckDBType( + result->GetValue(0, i).GetValue())); + } catch (common::BinderException& e) { return false; } + columnNames.push_back(result->GetValue(1, i).GetValue()); + } + return true; +} + +bool DuckDBCatalogContent::bindPropertyInfos(duckdb::Connection& con, const std::string& tableName, + const std::string& catalogName, std::vector& propertyInfos) { + std::vector columnTypes; + std::vector columnNames; + if (!getTableInfo( + con, tableName, getDefaultSchemaName(), catalogName, columnTypes, columnNames)) { + return false; + } + for (auto i = 0u; i < columnNames.size(); i++) { + auto propertyInfo = binder::PropertyInfo(columnNames[i], columnTypes[i]); + propertyInfos.push_back(std::move(propertyInfo)); + } + return true; +} + +std::unique_ptr DuckDBCatalogContent::bindCreateTableInfo( + duckdb::Connection& con, const std::string& tableName, const std::string& dbPath, + const std::string& catalogName) { + std::vector propertyInfos; + if (!bindPropertyInfos(con, tableName, catalogName, propertyInfos)) { + return nullptr; + } + return std::make_unique(common::TableType::FOREIGN, tableName, + std::make_unique( + dbPath, catalogName, getDefaultSchemaName(), std::move(propertyInfos))); +} + +std::string DuckDBCatalogContent::getDefaultSchemaName() const { + return "main"; +} + +duckdb::Connection DuckDBCatalogContent::getConnection(const std::string& dbPath) const { + duckdb::DuckDB db(dbPath); + duckdb::Connection con(db); + return con; } } // namespace duckdb_scanner diff --git a/extension/duckdb_scanner/src/duckdb_scan.cpp b/extension/duckdb_scanner/src/duckdb_scan.cpp index 95b0aa7ee9b..403d81216dd 100644 --- a/extension/duckdb_scanner/src/duckdb_scan.cpp +++ b/extension/duckdb_scanner/src/duckdb_scan.cpp @@ -1,6 +1,5 @@ #include "duckdb_scan.h" -#include "common/exception/binder.h" #include "common/types/types.h" #include "function/table/bind_input.h" @@ -13,10 +12,11 @@ namespace duckdb_scanner { void getDuckDBVectorConversionFunc( PhysicalTypeID physicalTypeID, duckdb_conversion_func_t& conversion_func); -DuckDBScanBindData::DuckDBScanBindData(std::string query, std::string dbPath, - std::vector columnTypes, std::vector columnNames) +DuckDBScanBindData::DuckDBScanBindData(std::string query, + std::vector columnTypes, std::vector columnNames, + init_duckdb_conn_t initDuckDBConn) : TableFuncBindData{std::move(columnTypes), std::move(columnNames)}, query{std::move(query)}, - dbPath{std::move(dbPath)} { + initDuckDBConn{std::move(initDuckDBConn)} { conversionFunctions.resize(this->columnTypes.size()); for (auto i = 0u; i < this->columnTypes.size(); i++) { getDuckDBVectorConversionFunc( @@ -25,7 +25,7 @@ DuckDBScanBindData::DuckDBScanBindData(std::string query, std::string dbPath, } std::unique_ptr DuckDBScanBindData::copy() const { - return std::make_unique(query, dbPath, columnTypes, columnNames); + return std::make_unique(query, columnTypes, columnNames, initDuckDBConn); } DuckDBScanSharedState::DuckDBScanSharedState(std::unique_ptr queryResult) @@ -52,9 +52,12 @@ struct DuckDBScanFunction { std::unique_ptr DuckDBScanFunction::initSharedState( function::TableFunctionInitInput& input) { auto scanBindData = reinterpret_cast(input.bindData); - auto db = duckdb::DuckDB(scanBindData->dbPath); - auto conn = duckdb::Connection(db); + auto conn = scanBindData->initDuckDBConn(); auto result = conn.SendQuery(scanBindData->query); + if (result->HasError()) { + throw common::RuntimeException( + common::stringFormat("Failed to execute query: {} in duckdb.", result->GetError())); + } return std::make_unique(std::move(result)); } diff --git a/extension/duckdb_scanner/src/duckdb_storage.cpp b/extension/duckdb_scanner/src/duckdb_storage.cpp index b3413d63834..642eab9c09f 100644 --- a/extension/duckdb_scanner/src/duckdb_storage.cpp +++ b/extension/duckdb_scanner/src/duckdb_storage.cpp @@ -10,73 +10,6 @@ namespace kuzu { namespace duckdb_scanner { -static bool getTableInfo(duckdb::Connection& con, std::string tableName, - std::vector& columnTypes, std::vector& columnNames) { - auto result = con.Query( - common::stringFormat("select data_type,column_name from information_schema.columns where " - "table_name = '{}' and table_schema='main';", - tableName)); - if (result->RowCount() == 0) { - return false; - } - columnTypes.reserve(result->RowCount()); - columnNames.reserve(result->RowCount()); - for (auto i = 0u; i < result->RowCount(); i++) { - try { - columnTypes.push_back(DuckDBTypeConverter::convertDuckDBType( - result->GetValue(0, i).GetValue())); - } catch (common::BinderException& e) { return false; } - columnNames.push_back(result->GetValue(1, i).GetValue()); - } - return true; -} - -std::unique_ptr getCreateTableInfo( - duckdb::Connection& con, std::string tableName, std::string dbPath) { - std::vector columnTypes; - std::vector columnNames; - if (!getTableInfo(con, tableName, columnTypes, columnNames)) { - return nullptr; - } - std::vector propertyInfos; - for (auto i = 0u; i < columnNames.size(); i++) { - auto propertyInfo = binder::PropertyInfo(columnNames[i], columnTypes[i]); - propertyInfos.push_back(std::move(propertyInfo)); - } - return std::make_unique(common::TableType::FOREIGN, tableName, - std::make_unique( - std::move(dbPath), std::move(propertyInfos))); -} - -std::unique_ptr createCatalog( - std::string dbPath, main::ClientContext* context) { - duckdb::DuckDB db(dbPath); - duckdb::Connection con(db); - auto query = "select table_name from information_schema.tables where table_schema = 'main';"; - auto result = con.SendQuery(query); - std::unique_ptr resultChunk; - try { - resultChunk = result->Fetch(); - } catch (std::exception& e) { return 0; } - if (resultChunk == nullptr) { - return 0; - } - auto tableNamesVector = std::make_unique( - common::LogicalTypeID::STRING, context->getMemoryManager()); - duckdb_conversion_func_t conversionFunc; - getDuckDBVectorConversionFunc(common::PhysicalTypeID::STRING, conversionFunc); - conversionFunc(resultChunk->data[0], *tableNamesVector, resultChunk->size()); - std::unique_ptr catalogContent = std::make_unique(); - for (auto i = 0u; i < resultChunk->size(); i++) { - auto tableName = tableNamesVector->getValue(i).getAsString(); - auto createTableInfo = getCreateTableInfo(con, tableName, dbPath); - if (createTableInfo != nullptr) { - catalogContent->createForeignTable(*createTableInfo); - } - } - return catalogContent; -} - std::unique_ptr attachDuckDB( std::string dbName, std::string dbPath, main::ClientContext* clientContext) { if (dbName == "") { @@ -87,17 +20,16 @@ std::unique_ptr attachDuckDB( dbName = dbPath; } } - return std::make_unique(dbName, createCatalog(dbPath, clientContext)); + auto duckdbCatalog = std::make_unique(); + duckdbCatalog->init(dbPath, dbName, clientContext); + return std::make_unique(dbName, std::move(duckdbCatalog)); } DuckDBStorageExtension::DuckDBStorageExtension() : StorageExtension{attachDuckDB} {} bool DuckDBStorageExtension::canHandleDB(std::string dbType) const { common::StringUtils::toUpper(dbType); - if (dbType == "DUCKDB") { - return true; - } - return false; + return dbType == "DUCKDB"; } } // namespace duckdb_scanner diff --git a/extension/duckdb_scanner/src/include/duckdb_catalog.h b/extension/duckdb_scanner/src/include/duckdb_catalog.h index 8ba6c0a3dd7..407e33e92e1 100644 --- a/extension/duckdb_scanner/src/include/duckdb_catalog.h +++ b/extension/duckdb_scanner/src/include/duckdb_catalog.h @@ -8,14 +8,18 @@ namespace kuzu { namespace duckdb_scanner { -struct BoundExtraCreateDuckDBTableInfo final : public binder::BoundExtraCreateTableInfo { +struct BoundExtraCreateDuckDBTableInfo : public binder::BoundExtraCreateTableInfo { std::string dbPath; + std::string catalogName; + std::string schemaName; - BoundExtraCreateDuckDBTableInfo( - std::string dbPath, std::vector propertyInfos) - : BoundExtraCreateTableInfo{std::move(propertyInfos)}, dbPath{std::move(dbPath)} {} + BoundExtraCreateDuckDBTableInfo(std::string dbPath, std::string catalogName, + std::string schemaName, std::vector propertyInfos) + : BoundExtraCreateTableInfo{std::move(propertyInfos)}, dbPath{std::move(dbPath)}, + catalogName{std::move(catalogName)}, schemaName{std::move(schemaName)} {} BoundExtraCreateDuckDBTableInfo(const BoundExtraCreateDuckDBTableInfo& other) - : BoundExtraCreateTableInfo{copyVector(other.propertyInfos)}, dbPath{other.dbPath} {} + : BoundExtraCreateTableInfo{copyVector(other.propertyInfos)}, dbPath{other.dbPath}, + catalogName{other.catalogName}, schemaName{other.schemaName} {} std::unique_ptr copy() const override { return std::make_unique(*this); @@ -26,7 +30,25 @@ class DuckDBCatalogContent : public catalog::CatalogContent { public: DuckDBCatalogContent() : catalog::CatalogContent{nullptr /* vfs */} {} - common::table_id_t createForeignTable(const binder::BoundCreateTableInfo& info); + virtual void init( + const std::string& dbPath, const std::string& catalogName, main::ClientContext* context); + +protected: + bool bindPropertyInfos(duckdb::Connection& con, const std::string& tableName, + const std::string& catalogName, std::vector& propertyInfos); + +private: + virtual std::unique_ptr bindCreateTableInfo( + duckdb::Connection& con, const std::string& tableName, const std::string& dbPath, + const std::string& catalogName); + + virtual std::string getDefaultSchemaName() const; + + virtual duckdb::Connection getConnection(const std::string& dbPath) const; + +private: + void createForeignTable(duckdb::Connection& con, const std::string& tableName, + const std::string& dbPath, const std::string& catalogName); }; } // namespace duckdb_scanner diff --git a/extension/duckdb_scanner/src/include/duckdb_scan.h b/extension/duckdb_scanner/src/include/duckdb_scan.h index a783dfeb453..fe422737da5 100644 --- a/extension/duckdb_scanner/src/include/duckdb_scan.h +++ b/extension/duckdb_scanner/src/include/duckdb_scan.h @@ -18,16 +18,17 @@ namespace duckdb_scanner { using duckdb_conversion_func_t = std::function; +using init_duckdb_conn_t = std::function; struct DuckDBScanBindData : public function::TableFuncBindData { - explicit DuckDBScanBindData(std::string query, std::string dbPath, - std::vector columnTypes, std::vector columnNames); + explicit DuckDBScanBindData(std::string query, std::vector columnTypes, + std::vector columnNames, init_duckdb_conn_t initDuckDBConn); std::unique_ptr copy() const override; std::string query; - std::string dbPath; std::vector conversionFunctions; + init_duckdb_conn_t initDuckDBConn; }; struct DuckDBScanSharedState : public function::TableFuncSharedState { diff --git a/extension/duckdb_scanner/src/include/duckdb_scanner_extension.h b/extension/duckdb_scanner/src/include/duckdb_scanner_extension.h index cd34256028e..2911768f0cc 100644 --- a/extension/duckdb_scanner/src/include/duckdb_scanner_extension.h +++ b/extension/duckdb_scanner/src/include/duckdb_scanner_extension.h @@ -6,7 +6,7 @@ namespace kuzu { namespace duckdb_scanner { -class DuckDBScannerExtension : public extension::Extension { +class DuckDBScannerExtension final : public extension::Extension { public: static void load(main::ClientContext* context); }; diff --git a/extension/duckdb_scanner/src/include/duckdb_storage.h b/extension/duckdb_scanner/src/include/duckdb_storage.h index 1d19662beba..aed030ba93f 100644 --- a/extension/duckdb_scanner/src/include/duckdb_storage.h +++ b/extension/duckdb_scanner/src/include/duckdb_storage.h @@ -6,7 +6,7 @@ namespace kuzu { namespace duckdb_scanner { -class DuckDBStorageExtension : public storage::StorageExtension { +class DuckDBStorageExtension final : public storage::StorageExtension { public: DuckDBStorageExtension(); diff --git a/extension/postgres_scanner/CMakeLists.txt b/extension/postgres_scanner/CMakeLists.txt new file mode 100644 index 00000000000..a8d3e658755 --- /dev/null +++ b/extension/postgres_scanner/CMakeLists.txt @@ -0,0 +1,58 @@ +find_package(DuckDB REQUIRED) + +add_library(postgres_scanner + SHARED + ../duckdb_scanner/src/duckdb_scan.cpp + ../duckdb_scanner/src/duckdb_catalog.cpp + ../duckdb_scanner/src/duckdb_table_catalog_entry.cpp + ../duckdb_scanner/src/duckdb_type_converter.cpp + src/postgres_scanner_extension.cpp + src/postgres_storage.cpp + src/postgres_catalog.cpp) + +include_directories( + src/include + ../duckdb_scanner/src/include + ${DuckDB_INCLUDE_DIRS} + ${PROJECT_SOURCE_DIR}/src/include) + +set_target_properties(postgres_scanner PROPERTIES + OUTPUT_NAME postgres_scanner + PREFIX "lib" + SUFFIX ".kuzu_extension" +) + +set_target_properties(postgres_scanner + PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build" +) + +target_link_libraries(postgres_scanner + PRIVATE + ${DuckDB_LIBRARIES}) + +if (WIN32) + # On windows, there is no dynamic lookup available, so it's not + # possible to generically look for symbols on library load. There are + # two main alternatives to statically linking kuzu, neither of which is + # appealing: + # 1. Link against the shared library. This works well assuming + # the DLL is locatable, but this assumption isn't valid for users + # of kuzu_shell.exe. + # 2. Link against the executable (kuzu_shell.exe). This is + # strange but works well for kuzu_shell.exe. However, it forces + # users who are embedding kuzu in their application to recompile + # the extension _and_ export the symbols for the extension to + # locate on load. + # We choose the simplest option. Windows isn't known + # for its small libraries anyways... + # Future work could make it possible to embed extension into kuzu, + # which would help fix this problem. + target_link_libraries(postgres_scanner PRIVATE kuzu) +endif() + +if (APPLE) + set_target_properties(postgres_scanner PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") +endif () diff --git a/extension/postgres_scanner/src/include/postgres_catalog.h b/extension/postgres_scanner/src/include/postgres_catalog.h new file mode 100644 index 00000000000..9437d9d31d6 --- /dev/null +++ b/extension/postgres_scanner/src/include/postgres_catalog.h @@ -0,0 +1,50 @@ +#pragma once + +#include "duckdb_catalog.h" + +namespace kuzu { +namespace postgres_scanner { + +struct BoundExtraCreatePostgresTableInfo final + : public duckdb_scanner::BoundExtraCreateDuckDBTableInfo { + std::string pgConnectionStr; + + BoundExtraCreatePostgresTableInfo(std::string pgConnectionStr, std::string dbPath, + std::string catalogName, std::string schemaName, + std::vector propertyInfos) + : BoundExtraCreateDuckDBTableInfo{std::move(dbPath), std::move(catalogName), + std::move(schemaName), std::move(propertyInfos)}, + pgConnectionStr{std::move(pgConnectionStr)} {} + BoundExtraCreatePostgresTableInfo(const BoundExtraCreatePostgresTableInfo& other) + : BoundExtraCreateDuckDBTableInfo{other.dbPath, other.catalogName, other.schemaName, + copyVector(other.propertyInfos)}, + pgConnectionStr{other.pgConnectionStr} {} + + std::unique_ptr copy() const override { + return std::make_unique(*this); + } +}; + +class PostgresCatalogContent final : public duckdb_scanner::DuckDBCatalogContent { +public: + PostgresCatalogContent() : duckdb_scanner::DuckDBCatalogContent{} {} + + void init(const std::string& dbPath, const std::string& catalogName, + main::ClientContext* context) override; + +private: + std::unique_ptr bindCreateTableInfo(duckdb::Connection& con, + const std::string& tableName, const std::string& dbPath, + const std::string& /*catalogName*/) override; + + std::string getDefaultSchemaName() const override; + + duckdb::Connection getConnection(const std::string& dbPath) const override; + +private: + static constexpr char DEFAULT_CATALOG_NAME[] = "pg"; + static constexpr char DEFAULT_SCHEMA_NAME[] = "public"; +}; + +} // namespace postgres_scanner +} // namespace kuzu diff --git a/extension/postgres_scanner/src/include/postgres_scanner_extension.h b/extension/postgres_scanner/src/include/postgres_scanner_extension.h new file mode 100644 index 00000000000..aae2733d5ed --- /dev/null +++ b/extension/postgres_scanner/src/include/postgres_scanner_extension.h @@ -0,0 +1,15 @@ +#pragma once + +#include "extension/extension.h" +#include "main/database.h" + +namespace kuzu { +namespace postgres_scanner { + +class PostgresScannerExtension final : public extension::Extension { +public: + static void load(main::ClientContext* context); +}; + +} // namespace postgres_scanner +} // namespace kuzu diff --git a/extension/postgres_scanner/src/include/postgres_storage.h b/extension/postgres_scanner/src/include/postgres_storage.h new file mode 100644 index 00000000000..bf0f14b0556 --- /dev/null +++ b/extension/postgres_scanner/src/include/postgres_storage.h @@ -0,0 +1,17 @@ +#pragma once + +#include "common/string_utils.h" +#include "storage/storage_extension.h" + +namespace kuzu { +namespace postgres_scanner { + +class PostgresStorageExtension final : public storage::StorageExtension { +public: + PostgresStorageExtension(); + + bool canHandleDB(std::string dbType) const override; +}; + +} // namespace postgres_scanner +} // namespace kuzu diff --git a/extension/postgres_scanner/src/postgres_catalog.cpp b/extension/postgres_scanner/src/postgres_catalog.cpp new file mode 100644 index 00000000000..91f559080b5 --- /dev/null +++ b/extension/postgres_scanner/src/postgres_catalog.cpp @@ -0,0 +1,46 @@ +#include "postgres_catalog.h" + +#include "common/exception/binder.h" +#include "postgres_storage.h" + +namespace kuzu { +namespace postgres_scanner { + +void PostgresCatalogContent::init( + const std::string& dbPath, const std::string& /*catalogName*/, main::ClientContext* context) { + duckdb_scanner::DuckDBCatalogContent::init(dbPath, DEFAULT_CATALOG_NAME, context); +} + +std::string PostgresCatalogContent::getDefaultSchemaName() const { + return DEFAULT_SCHEMA_NAME; +} + +std::unique_ptr PostgresCatalogContent::bindCreateTableInfo( + duckdb::Connection& con, const std::string& tableName, const std::string& dbPath, + const std::string& /*catalogName*/) { + std::vector propertyInfos; + if (!bindPropertyInfos(con, tableName, DEFAULT_CATALOG_NAME, propertyInfos)) { + return nullptr; + } + auto extraCreatePostgresTableInfo = std::make_unique(dbPath, + "" /* dbPath */, DEFAULT_CATALOG_NAME, getDefaultSchemaName(), std::move(propertyInfos)); + return std::make_unique( + common::TableType::FOREIGN, tableName, std::move(extraCreatePostgresTableInfo)); +} + +duckdb::Connection PostgresCatalogContent::getConnection(const std::string& dbPath) const { + duckdb::DuckDB db(nullptr); + duckdb::Connection con(db); + con.Query("install postgres;"); + con.Query("load postgres;"); + auto result = con.Query( + common::stringFormat("attach '{}' as {} (TYPE postgres);", dbPath, DEFAULT_CATALOG_NAME)); + if (result->HasError()) { + throw common::BinderException(common::stringFormat( + "Failed to attach postgres database due to: {}", result->GetError())); + } + return con; +} + +} // namespace postgres_scanner +} // namespace kuzu diff --git a/extension/postgres_scanner/src/postgres_scanner_extension.cpp b/extension/postgres_scanner/src/postgres_scanner_extension.cpp new file mode 100644 index 00000000000..d5ac7337075 --- /dev/null +++ b/extension/postgres_scanner/src/postgres_scanner_extension.cpp @@ -0,0 +1,27 @@ +#include "postgres_scanner_extension.h" + +#include "postgres_storage.h" + +namespace kuzu { +namespace postgres_scanner { + +void PostgresScannerExtension::load(main::ClientContext* context) { + auto db = context->getDatabase(); + db->registerStorageExtension("postgres", std::make_unique()); +} + +} // namespace postgres_scanner +} // namespace kuzu + +extern "C" { +// Because we link against the static library on windows, we implicitly inherit KUZU_STATIC_DEFINE, +// which cancels out any exporting, so we can't use KUZU_API. +#if defined(_WIN32) +#define INIT_EXPORT __declspec(dllexport) +#else +#define INIT_EXPORT __attribute__((visibility("default"))) +#endif +INIT_EXPORT void init(kuzu::main::ClientContext* context) { + kuzu::postgres_scanner::PostgresScannerExtension::load(context); +} +} diff --git a/extension/postgres_scanner/src/postgres_storage.cpp b/extension/postgres_scanner/src/postgres_storage.cpp new file mode 100644 index 00000000000..e31118a4172 --- /dev/null +++ b/extension/postgres_scanner/src/postgres_storage.cpp @@ -0,0 +1,40 @@ +#include "postgres_storage.h" + +#include + +#include "catalog/catalog_entry/table_catalog_entry.h" +#include "duckdb_type_converter.h" +#include "postgres_catalog.h" + +namespace kuzu { +namespace postgres_scanner { + +std::string extractDBName(const std::string& connectionInfo) { + std::regex pattern("dbname=([^ ]+)"); + std::smatch match; + if (std::regex_search(connectionInfo, match, pattern)) { + return match.str(1); + } + throw common::RuntimeException{"Invalid postgresql connection string."}; +} + +std::unique_ptr attachPostgres( + std::string dbName, std::string dbPath, main::ClientContext* clientContext) { + auto catalogName = extractDBName(dbPath); + if (dbName == "") { + dbName = catalogName; + } + auto postgresCatalog = std::make_unique(); + postgresCatalog->init(dbPath, catalogName, clientContext); + return std::make_unique(dbName, std::move(postgresCatalog)); +} + +PostgresStorageExtension::PostgresStorageExtension() : StorageExtension{attachPostgres} {} + +bool PostgresStorageExtension::canHandleDB(std::string dbType) const { + common::StringUtils::toUpper(dbType); + return dbType == "POSTGRES"; +} + +} // namespace postgres_scanner +} // namespace kuzu diff --git a/extension/postgres_scanner/test/test_files/create_test_db.sql b/extension/postgres_scanner/test/test_files/create_test_db.sql new file mode 100644 index 00000000000..e2ecbc11aff --- /dev/null +++ b/extension/postgres_scanner/test/test_files/create_test_db.sql @@ -0,0 +1,241 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 14.10 (Homebrew) +-- Dumped by pg_dump version 14.10 (Homebrew) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: audience_type; Type: TYPE; Schema: public; Owner: ci +-- + +CREATE TYPE public.audience_type AS ( + key character varying, + value bigint +); + + +ALTER TYPE public.audience_type OWNER TO ci; + +-- +-- Name: description_type; Type: TYPE; Schema: public; Owner: ci +-- + +CREATE TYPE public.description_type AS ( + rating double precision, + stars bigint, + views bigint, + release timestamp without time zone, + release_ns timestamp without time zone, + release_ms timestamp without time zone, + release_sec timestamp without time zone, + release_tz timestamp with time zone, + film date, + u8 smallint, + u16 smallint, + u32 integer, + u64 bigint, + hugedata numeric +); + + +ALTER TYPE public.description_type OWNER TO ci; + +-- +-- Name: mood; Type: TYPE; Schema: public; Owner: ci +-- + +CREATE TYPE public.mood AS ENUM ( + 'sad', + 'ok', + 'happy' +); + + +ALTER TYPE public.mood OWNER TO ci; + +-- +-- Name: state_type; Type: TYPE; Schema: public; Owner: ci +-- + +CREATE TYPE public.state_type AS ( + revenue smallint, + location character varying[] +); + + +ALTER TYPE public.state_type OWNER TO ci; + +-- +-- Name: stock_type; Type: TYPE; Schema: public; Owner: ci +-- + +CREATE TYPE public.stock_type AS ( + price bigint[], + volume bigint +); + + +ALTER TYPE public.stock_type OWNER TO ci; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: movies; Type: TABLE; Schema: public; Owner: ci +-- + +CREATE TABLE public.movies ( + name character varying NOT NULL, + length integer, + note character varying, + description public.description_type, + content bytea, + audience public.audience_type[] +); + + +ALTER TABLE public.movies OWNER TO ci; + +-- +-- Name: organisation; Type: TABLE; Schema: public; Owner: ci +-- + +CREATE TABLE public.organisation ( + id bigint NOT NULL, + name character varying, + orgcode bigint, + mark double precision, + score bigint, + history interval, + licensevalidinterval interval, + rating double precision, + state public.state_type, + stock public.stock_type, + info character varying +); + + +ALTER TABLE public.organisation OWNER TO ci; + +-- +-- Name: person; Type: TABLE; Schema: public; Owner: ci +-- + +CREATE TABLE public.person ( + id bigint NOT NULL, + fname character varying, + gender bigint, + isstudent boolean, + isworker boolean, + age bigint, + eyesight double precision, + birthdate date, + registertime timestamp without time zone, + lastjobduration interval, + workedhours bigint[], + usednames character varying[], + height double precision, + u uuid +); + + +ALTER TABLE public.person OWNER TO ci; + +-- +-- Name: persontest; Type: TABLE; Schema: public; Owner: ci +-- + +CREATE TABLE public.persontest ( + id integer +); + + +ALTER TABLE public.persontest OWNER TO ci; + +-- +-- Data for Name: movies; Type: TABLE DATA; Schema: public; Owner: ci +-- + +COPY public.movies (name, length, note, description, content, audience) FROM stdin; +Sóló cón tu párejâ 126 this is a very very good movie (5.3,2,152,"2011-08-20 11:25:30","2011-08-20 11:25:30","2011-08-20 11:25:30","2011-08-20 11:25:30","2011-08-20 11:25:30+08",2012-05-11,220,20,1,180,1844674407370955161811111111) \\x5c7841415c784142696e746572657374696e675c783042 {"(audience1,52)","(audience53,42)"} +The 😂😃🧘🏻‍♂️🌍🌦️🍞🚗 movie 2544 the movie is very very good (7,10,982,"2018-11-13 13:33:11","2018-11-13 13:33:11","2018-11-13 13:33:11","2018-11-13 13:33:11","2018-11-13 13:33:11+08",2014-09-12,12,120,55,1,-1844674407370955161511) \\x5c7841425c784344 {"(audience1,33)"} +Roma 298 the movie is very interesting and funny (1223,100,10003,"2011-02-11 16:44:22","2011-02-11 16:44:22","2011-02-11 16:44:22","2011-02-11 16:44:22","2011-02-11 16:44:22+08",2013-02-22,1,15,200,4,-15) \\x707572652061736369692063686172616374657273 {} +\. + + +-- +-- Data for Name: organisation; Type: TABLE DATA; Schema: public; Owner: ci +-- + +COPY public.organisation (id, name, orgcode, mark, score, history, licensevalidinterval, rating, state, stock, info) FROM stdin; +1 ABFsUni 325 3.7 -2 10 years 5 mons 13:00:00.000024 3 years 5 days 1 (138,"{toronto,""montr,eal""}") ("{96,56}",1000) 3.12 +4 CsWork 934 4.1 -100 2 years 4 days 10:00:00 26 years 52 days 48:00:00 0.78 (152,"{""vanco,uver north area""}") ("{15,78,671}",432) abcd +6 DEsWork 824 4.1 7 2 years 04:34:00.000022 82:00:00.1 0.52 (558,"{""very long city name"",""new york""}") ({22},99) 2023-12-15 +\. + + +-- +-- Data for Name: person; Type: TABLE DATA; Schema: public; Owner: ci +-- + +COPY public.person (id, fname, gender, isstudent, isworker, age, eyesight, birthdate, registertime, lastjobduration, workedhours, usednames, height, u) FROM stdin; +0 Alice 1 t f 35 5 1900-01-01 2011-08-20 11:25:30 3 years 2 days 13:02:00 {10,5} {Aida} 1.731 a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 +2 Bob 2 t f 30 5.1 1900-01-01 2008-11-03 15:25:30.000526 10 years 5 mons 13:00:00.000024 {12,8} {Bobby} 0.99 a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12 +3 Carol 1 f t 45 5 1940-06-22 1911-08-20 02:32:21 48:24:11 {4,5} {Carmen,Fred} 1 a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a13 +5 Dan 2 f t 20 4.8 1950-07-23 2031-11-30 12:25:30 10 years 5 mons 13:00:00.000024 {1,9} {Wolfeschlegelstein,Daniel} 1.3 a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a14 +7 Elizabeth 1 f t 20 4.7 1980-10-26 1976-12-23 11:21:42 48:24:11 {2} {Ein} 1.463 a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a15 +8 Farooq 2 t f 25 4.5 1980-10-26 1972-07-31 13:22:30.678559 00:18:00.024 {3,4,5,6,7} {Fesdwe} 1.51 a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a16 +9 Greg 2 f f 40 4.9 1980-10-26 1976-12-23 11:21:42 10 years 5 mons 13:00:00.000024 {1} {Grad} 1.6 a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a17 +10 Hubert Blaine Wolfeschlegelsteinhausenbergerdorff 2 f t 83 4.9 1990-11-27 2023-02-21 13:25:30 3 years 2 days 13:02:00 {10,11,12,3,4,5,6,7} {Ad,De,Hi,Kye,Orlan} 1.323 a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a18 +\. + + +-- +-- Data for Name: persontest; Type: TABLE DATA; Schema: public; Owner: ci +-- + +COPY public.persontest (id) FROM stdin; +\. + + +-- +-- Name: movies movies_pkey; Type: CONSTRAINT; Schema: public; Owner: ci +-- + +ALTER TABLE ONLY public.movies + ADD CONSTRAINT movies_pkey PRIMARY KEY (name); + + +-- +-- Name: organisation organisation_pkey; Type: CONSTRAINT; Schema: public; Owner: ci +-- + +ALTER TABLE ONLY public.organisation + ADD CONSTRAINT organisation_pkey PRIMARY KEY (id); + + +-- +-- Name: person person_pkey; Type: CONSTRAINT; Schema: public; Owner: ci +-- + +ALTER TABLE ONLY public.person + ADD CONSTRAINT person_pkey PRIMARY KEY (id); + + +-- +-- PostgreSQL database dump complete +-- + diff --git a/extension/postgres_scanner/test/test_files/postgres_scanner.test b/extension/postgres_scanner/test/test_files/postgres_scanner.test new file mode 100644 index 00000000000..bd8fc2fc8f2 --- /dev/null +++ b/extension/postgres_scanner/test/test_files/postgres_scanner.test @@ -0,0 +1,49 @@ +-GROUP PostgresScanner +-DATASET CSV empty + +-- + +-CASE ScanPostgresTable +-STATEMENT load extension "${KUZU_ROOT_DIRECTORY}/extension/postgres_scanner/build/libpostgres_scanner.kuzu_extension" +---- ok +-STATEMENT ATTACH 'dbname=pgscan user=ci host=localhost' as tinysnb (dbtype 'POSTGRES'); +---- ok +-STATEMENT LOAD FROM tinysnb_person RETURN *; +---- 8 +0|Alice|1|True|False|35|5.000000|1900-01-01|2011-08-20 11:25:30|3 years 2 days 13:02:00|[10,5]|[Aida]|1.731000|a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 +10|Hubert Blaine Wolfeschlegelsteinhausenbergerdorff|2|False|True|83|4.900000|1990-11-27|2023-02-21 13:25:30|3 years 2 days 13:02:00|[10,11,12,3,4,5,6,7]|[Ad,De,Hi,Kye,Orlan]|1.323000|a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a18 +2|Bob|2|True|False|30|5.100000|1900-01-01|2008-11-03 15:25:30.000526|10 years 5 months 13:00:00.000024|[12,8]|[Bobby]|0.990000|a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12 +3|Carol|1|False|True|45|5.000000|1940-06-22|1911-08-20 02:32:21|48:24:11|[4,5]|[Carmen,Fred]|1.000000|a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a13 +5|Dan|2|False|True|20|4.800000|1950-07-23|2031-11-30 12:25:30|10 years 5 months 13:00:00.000024|[1,9]|[Wolfeschlegelstein,Daniel]|1.300000|a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a14 +7|Elizabeth|1|False|True|20|4.700000|1980-10-26|1976-12-23 11:21:42|48:24:11|[2]|[Ein]|1.463000|a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a15 +8|Farooq|2|True|False|25|4.500000|1980-10-26|1972-07-31 13:22:30.678559|00:18:00.024|[3,4,5,6,7]|[Fesdwe]|1.510000|a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a16 +9|Greg|2|False|False|40|4.900000|1980-10-26|1976-12-23 11:21:42|10 years 5 months 13:00:00.000024|[1]|[Grad]|1.600000|a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a17 +-STATEMENT LOAD FROM tinysnb_organisation RETURN *; +---- 3 +1|ABFsUni|325|3.700000|-2|10 years 5 months 13:00:00.000024|3 years 5 days|1.000000|{revenue: 138, "location": [toronto,montr,eal]}|{price: [96,56], volume: 1000}|3.12 +4|CsWork|934|4.100000|-100|2 years 4 days 10:00:00|26 years 52 days 48:00:00|0.780000|{revenue: 152, "location": [vanco,uver north area]}|{price: [15,78,671], volume: 432}|abcd +6|DEsWork|824|4.100000|7|2 years 04:34:00.000022|82:00:00.1|0.520000|{revenue: 558, "location": [very long city name,new york]}|{price: [22], volume: 99}|2023-12-15 +-STATEMENT LOAD FROM tinysnb_movies RETURN *; +---- 3 +Roma|298|the movie is very interesting and funny|{rating: 1223.000000, stars: 100, "views": 10003, "release": 2011-02-11 16:44:22, release_ns: 2011-02-11 16:44:22, release_ms: 2011-02-11 16:44:22, release_sec: 2011-02-11 16:44:22, release_tz: 2011-02-11 08:44:22+00, film: 2013-02-22, u8: 1, u16: 15, u32: 200, u64: 4, hugedata: -15.000000}|pure ascii characters|[] +Sóló cón tu párejâ|126|this is a very very good movie|{rating: 5.300000, stars: 2, "views": 152, "release": 2011-08-20 11:25:30, release_ns: 2011-08-20 11:25:30, release_ms: 2011-08-20 11:25:30, release_sec: 2011-08-20 11:25:30, release_tz: 2011-08-20 03:25:30+00, film: 2012-05-11, u8: 220, u16: 20, u32: 1, u64: 180, hugedata: 1844674407370955161600000000.000000}|\x5CxAA\x5CxABinteresting\x5Cx0B|[{"key": audience1, "value": 52},{"key": audience53, "value": 42}] +The 😂😃🧘🏻‍♂️🌍🌦️🍞🚗 movie|2544|the movie is very very good|{rating: 7.000000, stars: 10, "views": 982, "release": 2018-11-13 13:33:11, release_ns: 2018-11-13 13:33:11, release_ms: 2018-11-13 13:33:11, release_sec: 2018-11-13 13:33:11, release_tz: 2018-11-13 05:33:11+00, film: 2014-09-12, u8: 12, u16: 120, u32: 55, u64: 1, hugedata: -1844674407370954899456.000000}|\x5CxAB\x5CxCD|[{"key": audience1, "value": 33}] +-STATEMENT LOAD FROM tinysnb_person1 RETURN *; +---- error +Catalog exception: Table: person1 does not exist. +-STATEMENT LOAD FROM tinysnb1_person RETURN *; +---- error +Binder exception: No database named tinysnb1 has been attached. +-STATEMENT ATTACH 'dbname=pgscan user=ci host=localhost' (dbtype 'POSTGRES'); +---- ok +-STATEMENT LOAD FROM pgscan_movies RETURN count(*); +---- 1 +3 +-STATEMENT LOAD FROM pgscan_movies where length > 2500 RETURN name; +---- 1 +The 😂😃🧘🏻‍♂️🌍🌦️🍞🚗 movie +-LOG IncorrectConnectionStr +-STATEMENT ATTACH 'dbname=test2132131 user=ci host=127.0.0.1' as tinysnb (dbtype 'POSTGRES'); +---- error +Binder exception: Failed to attach postgres database due to: IO Error: Unable to connect to Postgres at dbname=test2132131 user=ci host=127.0.0.1: connection to server at "127.0.0.1", port 5432 failed: Connection refused + Is the server running on that host and accepting TCP/IP connections? diff --git a/src/include/catalog/catalog_content.h b/src/include/catalog/catalog_content.h index a1ffe6ad0bf..10c45e302c1 100644 --- a/src/include/catalog/catalog_content.h +++ b/src/include/catalog/catalog_content.h @@ -20,6 +20,8 @@ class CatalogContent { public: KUZU_API explicit CatalogContent(common::VirtualFileSystem* vfs); + virtual ~CatalogContent() = default; + CatalogContent(const std::string& directory, common::VirtualFileSystem* vfs); CatalogContent(std::unique_ptr tables,