diff --git a/src/duckdb/extension/parquet/parquet_extension.cpp b/src/duckdb/extension/parquet/parquet_extension.cpp index dd61f971..6a978d9d 100644 --- a/src/duckdb/extension/parquet/parquet_extension.cpp +++ b/src/duckdb/extension/parquet/parquet_extension.cpp @@ -651,7 +651,13 @@ class ParquetScanFunction { bool require_extra_columns = result->multi_file_reader_state && result->multi_file_reader_state->RequiresExtraColumns(); if (input.CanRemoveFilterColumns() || require_extra_columns) { - result->projection_ids = input.projection_ids; + if (!input.projection_ids.empty()) { + result->projection_ids = input.projection_ids; + } else { + result->projection_ids.resize(input.column_ids.size()); + iota(begin(result->projection_ids), end(result->projection_ids), 0); + } + const auto table_types = bind_data.types; for (const auto &col_idx : input.column_ids) { if (IsRowIdColumnId(col_idx)) { diff --git a/src/duckdb/src/catalog/dependency_manager.cpp b/src/duckdb/src/catalog/dependency_manager.cpp index 66042b4d..51c60d3a 100644 --- a/src/duckdb/src/catalog/dependency_manager.cpp +++ b/src/duckdb/src/catalog/dependency_manager.cpp @@ -513,6 +513,10 @@ void DependencyManager::AlterObject(CatalogTransaction transaction, CatalogEntry disallow_alter = false; break; } + case AlterTableType::ADD_COLUMN: { + disallow_alter = false; + break; + } default: break; } diff --git a/src/duckdb/src/common/enum_util.cpp b/src/duckdb/src/common/enum_util.cpp index 6ee1ec51..33b64469 100644 --- a/src/duckdb/src/common/enum_util.cpp +++ b/src/duckdb/src/common/enum_util.cpp @@ -18,6 +18,7 @@ #include "duckdb/common/enums/catalog_lookup_behavior.hpp" #include "duckdb/common/enums/catalog_type.hpp" #include "duckdb/common/enums/compression_type.hpp" +#include "duckdb/common/enums/copy_overwrite_mode.hpp" #include "duckdb/common/enums/cte_materialize.hpp" #include "duckdb/common/enums/date_part_specifier.hpp" #include "duckdb/common/enums/debug_initialize.hpp" @@ -1306,6 +1307,34 @@ ConstraintType EnumUtil::FromString(const char *value) { throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value)); } +template<> +const char* EnumUtil::ToChars(CopyOverwriteMode value) { + switch(value) { + case CopyOverwriteMode::COPY_ERROR_ON_CONFLICT: + return "COPY_ERROR_ON_CONFLICT"; + case CopyOverwriteMode::COPY_OVERWRITE: + return "COPY_OVERWRITE"; + case CopyOverwriteMode::COPY_OVERWRITE_OR_IGNORE: + return "COPY_OVERWRITE_OR_IGNORE"; + default: + throw NotImplementedException(StringUtil::Format("Enum value: '%d' not implemented", value)); + } +} + +template<> +CopyOverwriteMode EnumUtil::FromString(const char *value) { + if (StringUtil::Equals(value, "COPY_ERROR_ON_CONFLICT")) { + return CopyOverwriteMode::COPY_ERROR_ON_CONFLICT; + } + if (StringUtil::Equals(value, "COPY_OVERWRITE")) { + return CopyOverwriteMode::COPY_OVERWRITE; + } + if (StringUtil::Equals(value, "COPY_OVERWRITE_OR_IGNORE")) { + return CopyOverwriteMode::COPY_OVERWRITE_OR_IGNORE; + } + throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value)); +} + template<> const char* EnumUtil::ToChars(DataFileType value) { switch(value) { diff --git a/src/duckdb/src/common/error_data.cpp b/src/duckdb/src/common/error_data.cpp index 07457419..c7262cab 100644 --- a/src/duckdb/src/common/error_data.cpp +++ b/src/duckdb/src/common/error_data.cpp @@ -1,6 +1,6 @@ #include "duckdb/common/error_data.hpp" -#include "duckdb/common/exception.hpp" +#include "duckdb/common/exception.hpp" #include "duckdb/common/string_util.hpp" #include "duckdb/common/to_string.hpp" #include "duckdb/common/types.hpp" @@ -50,7 +50,10 @@ ErrorData::ErrorData(const string &message) : initialized(true), type(ExceptionT const string &ErrorData::Message() { if (final_message.empty()) { - final_message = Exception::ExceptionTypeToString(type) + " Error: " + raw_message; + if (type != ExceptionType::UNKNOWN_TYPE) { + final_message = Exception::ExceptionTypeToString(type) + " "; + } + final_message += "Error: " + raw_message; if (type == ExceptionType::INTERNAL) { final_message += "\nThis error signals an assertion failure within DuckDB. This usually occurs due to " "unexpected conditions or errors in the program's logic.\nFor more information, see " diff --git a/src/duckdb/src/common/local_file_system.cpp b/src/duckdb/src/common/local_file_system.cpp index cb15b5d6..e650d79b 100644 --- a/src/duckdb/src/common/local_file_system.cpp +++ b/src/duckdb/src/common/local_file_system.cpp @@ -485,9 +485,14 @@ int64_t LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_byte bool LocalFileSystem::Trim(FileHandle &handle, idx_t offset_bytes, idx_t length_bytes) { #if defined(__linux__) + // FALLOC_FL_PUNCH_HOLE requires glibc 2.18 or up +#if __GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ < 18) + return false; +#else int fd = handle.Cast().fd; int res = fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, offset_bytes, length_bytes); return res == 0; +#endif #else return false; #endif diff --git a/src/duckdb/src/common/multi_file_reader.cpp b/src/duckdb/src/common/multi_file_reader.cpp index 7e08b7f8..fec6664c 100644 --- a/src/duckdb/src/common/multi_file_reader.cpp +++ b/src/duckdb/src/common/multi_file_reader.cpp @@ -14,6 +14,9 @@ namespace duckdb { +MultiFileReaderGlobalState::~MultiFileReaderGlobalState() { +} + MultiFileReader::~MultiFileReader() { } diff --git a/src/duckdb/src/common/printer.cpp b/src/duckdb/src/common/printer.cpp index 2c24069c..0c704b74 100644 --- a/src/duckdb/src/common/printer.cpp +++ b/src/duckdb/src/common/printer.cpp @@ -62,7 +62,7 @@ idx_t Printer::TerminalWidth() { #ifndef DUCKDB_DISABLE_PRINT #ifdef DUCKDB_WINDOWS CONSOLE_SCREEN_BUFFER_INFO csbi; - int columns, rows; + int rows; GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi); rows = csbi.srWindow.Right - csbi.srWindow.Left + 1; diff --git a/src/duckdb/src/common/string_util.cpp b/src/duckdb/src/common/string_util.cpp index 5cdf95a6..6177129c 100644 --- a/src/duckdb/src/common/string_util.cpp +++ b/src/duckdb/src/common/string_util.cpp @@ -246,7 +246,7 @@ bool StringUtil::CIEquals(const string &l1, const string &l2) { bool StringUtil::CILessThan(const string &s1, const string &s2) { const auto charmap = UpperFun::ASCII_TO_UPPER_MAP; - unsigned char u1, u2; + unsigned char u1 {}, u2 {}; idx_t length = MinValue(s1.length(), s2.length()); length += s1.length() != s2.length(); diff --git a/src/duckdb/src/core_functions/aggregate/holistic/approximate_quantile.cpp b/src/duckdb/src/core_functions/aggregate/holistic/approximate_quantile.cpp index e356caec..edda3458 100644 --- a/src/duckdb/src/core_functions/aggregate/holistic/approximate_quantile.cpp +++ b/src/duckdb/src/core_functions/aggregate/holistic/approximate_quantile.cpp @@ -181,16 +181,25 @@ unique_ptr BindApproxQuantile(ClientContext &context, AggregateFun throw BinderException("APPROXIMATE QUANTILE can only take constant quantile parameters"); } Value quantile_val = ExpressionExecutor::EvaluateScalar(context, *arguments[1]); + if (quantile_val.IsNull()) { + throw BinderException("APPROXIMATE QUANTILE parameter list cannot be NULL"); + } vector quantiles; - if (quantile_val.type().id() != LogicalTypeId::LIST) { - quantiles.push_back(CheckApproxQuantile(quantile_val)); - } else if (quantile_val.IsNull()) { - throw BinderException("APPROXIMATE QUANTILE parameter list cannot be NULL"); - } else { + switch (quantile_val.type().id()) { + case LogicalTypeId::LIST: for (const auto &element_val : ListValue::GetChildren(quantile_val)) { quantiles.push_back(CheckApproxQuantile(element_val)); } + break; + case LogicalTypeId::ARRAY: + for (const auto &element_val : ArrayValue::GetChildren(quantile_val)) { + quantiles.push_back(CheckApproxQuantile(element_val)); + } + break; + default: + quantiles.push_back(CheckApproxQuantile(quantile_val)); + break; } // remove the quantile argument so we can use the unary aggregate diff --git a/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp b/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp index 84446a6c..e0150700 100644 --- a/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp +++ b/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp @@ -1509,12 +1509,20 @@ unique_ptr BindQuantile(ClientContext &context, AggregateFunction throw BinderException("QUANTILE argument must not be NULL"); } vector quantiles; - if (quantile_val.type().id() != LogicalTypeId::LIST) { - quantiles.push_back(CheckQuantile(quantile_val)); - } else { + switch (quantile_val.type().id()) { + case LogicalTypeId::LIST: for (const auto &element_val : ListValue::GetChildren(quantile_val)) { quantiles.push_back(CheckQuantile(element_val)); } + break; + case LogicalTypeId::ARRAY: + for (const auto &element_val : ArrayValue::GetChildren(quantile_val)) { + quantiles.push_back(CheckQuantile(element_val)); + } + break; + default: + quantiles.push_back(CheckQuantile(quantile_val)); + break; } Function::EraseArgument(function, arguments, arguments.size() - 1); diff --git a/src/duckdb/src/execution/operator/helper/physical_load.cpp b/src/duckdb/src/execution/operator/helper/physical_load.cpp index 1c28406a..e20b5af0 100644 --- a/src/duckdb/src/execution/operator/helper/physical_load.cpp +++ b/src/duckdb/src/execution/operator/helper/physical_load.cpp @@ -17,14 +17,14 @@ static void InstallFromRepository(ClientContext &context, const LoadInfo &info) } ExtensionHelper::InstallExtension(context, info.filename, info.load_type == LoadType::FORCE_INSTALL, repository, - info.version); + true, info.version); } SourceResultType PhysicalLoad::GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const { if (info->load_type == LoadType::INSTALL || info->load_type == LoadType::FORCE_INSTALL) { if (info->repository.empty()) { ExtensionHelper::InstallExtension(context.client, info->filename, - info->load_type == LoadType::FORCE_INSTALL, nullptr, info->version); + info->load_type == LoadType::FORCE_INSTALL, nullptr, true, info->version); } else { InstallFromRepository(context.client, *info); } diff --git a/src/duckdb/src/execution/operator/persistent/physical_copy_database.cpp b/src/duckdb/src/execution/operator/persistent/physical_copy_database.cpp index d0d83130..e66881be 100644 --- a/src/duckdb/src/execution/operator/persistent/physical_copy_database.cpp +++ b/src/duckdb/src/execution/operator/persistent/physical_copy_database.cpp @@ -49,8 +49,10 @@ SourceResultType PhysicalCopyDatabase::GetData(ExecutionContext &context, DataCh catalog.CreateTable(context.client, *bound_info); break; } + case CatalogType::INDEX_ENTRY: default: - throw InternalException("Entry type not supported in PhysicalCopyDatabase"); + throw NotImplementedException("Entry type %s not supported in PhysicalCopyDatabase", + CatalogTypeToString(create_info->type)); } } return SourceResultType::FINISHED; diff --git a/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp b/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp index 9205067b..2280b770 100644 --- a/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp +++ b/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp @@ -228,12 +228,16 @@ unique_ptr PhysicalCopyToFile::GetLocalSinkState(ExecutionContex return std::move(res); } -void CheckDirectory(FileSystem &fs, const string &file_path, bool overwrite) { - if (fs.IsRemoteFile(file_path) && overwrite) { - // we only remove files for local file systems - // as remote file systems (e.g. S3) do not support RemoveFile +void CheckDirectory(FileSystem &fs, const string &file_path, CopyOverwriteMode overwrite_mode) { + if (overwrite_mode == CopyOverwriteMode::COPY_OVERWRITE_OR_IGNORE) { + // with overwrite or ignore we fully ignore the presence of any files instead of erasing them return; } + if (fs.IsRemoteFile(file_path) && overwrite_mode == CopyOverwriteMode::COPY_OVERWRITE) { + // we can only remove files for local file systems currently + // as remote file systems (e.g. S3) do not support RemoveFile + throw NotImplementedException("OVERWRITE is not supported for remote file systems"); + } vector file_list; vector directory_list; directory_list.push_back(file_path); @@ -251,13 +255,12 @@ void CheckDirectory(FileSystem &fs, const string &file_path, bool overwrite) { if (file_list.empty()) { return; } - if (overwrite) { + if (overwrite_mode == CopyOverwriteMode::COPY_OVERWRITE) { for (auto &file : file_list) { fs.RemoveFile(file); } } else { - throw IOException("Directory \"%s\" is not empty! Enable OVERWRITE_OR_IGNORE option to force writing", - file_path); + throw IOException("Directory \"%s\" is not empty! Enable OVERWRITE option to overwrite files", file_path); } } @@ -272,11 +275,11 @@ unique_ptr PhysicalCopyToFile::GetGlobalSinkState(ClientContext throw IOException("Cannot write to \"%s\" - it exists and is a file, not a directory!", file_path); } else { // for local files we can remove the file if OVERWRITE_OR_IGNORE is enabled - if (overwrite_or_ignore) { + if (overwrite_mode == CopyOverwriteMode::COPY_OVERWRITE) { fs.RemoveFile(file_path); } else { throw IOException("Cannot write to \"%s\" - it exists and is a file, not a directory! Enable " - "OVERWRITE_OR_IGNORE option to force writing", + "OVERWRITE option to overwrite the file", file_path); } } @@ -285,7 +288,7 @@ unique_ptr PhysicalCopyToFile::GetGlobalSinkState(ClientContext if (!fs.DirectoryExists(file_path)) { fs.CreateDirectory(file_path); } else { - CheckDirectory(fs, file_path, overwrite_or_ignore); + CheckDirectory(fs, file_path, overwrite_mode); } auto state = make_uniq(nullptr); diff --git a/src/duckdb/src/execution/physical_plan/plan_copy_to_file.cpp b/src/duckdb/src/execution/physical_plan/plan_copy_to_file.cpp index cd7f19cd..c3194e25 100644 --- a/src/duckdb/src/execution/physical_plan/plan_copy_to_file.cpp +++ b/src/duckdb/src/execution/physical_plan/plan_copy_to_file.cpp @@ -17,7 +17,7 @@ unique_ptr PhysicalPlanGenerator::CreatePlan(LogicalCopyToFile op.file_path = fs.JoinPath(path, "tmp_" + base); } if (op.per_thread_output || op.file_size_bytes.IsValid() || op.partition_output || !op.partition_columns.empty() || - op.overwrite_or_ignore) { + op.overwrite_mode != CopyOverwriteMode::COPY_ERROR_ON_CONFLICT) { // hive-partitioning/per-thread output does not care about insertion order, and does not support batch indexes preserve_insertion_order = false; supports_batch_index = false; @@ -42,7 +42,7 @@ unique_ptr PhysicalPlanGenerator::CreatePlan(LogicalCopyToFile auto copy = make_uniq(op.types, op.function, std::move(op.bind_data), op.estimated_cardinality); copy->file_path = op.file_path; copy->use_tmp_file = op.use_tmp_file; - copy->overwrite_or_ignore = op.overwrite_or_ignore; + copy->overwrite_mode = op.overwrite_mode; copy->filename_pattern = op.filename_pattern; copy->file_extension = op.file_extension; copy->per_thread_output = op.per_thread_output; diff --git a/src/duckdb/src/function/table/system/duckdb_extensions.cpp b/src/duckdb/src/function/table/system/duckdb_extensions.cpp index 8fe90803..3c7f2396 100644 --- a/src/duckdb/src/function/table/system/duckdb_extensions.cpp +++ b/src/duckdb/src/function/table/system/duckdb_extensions.cpp @@ -125,7 +125,7 @@ unique_ptr DuckDBExtensionsInit(ClientContext &context if (entry == installed_extensions.end()) { installed_extensions[info.name] = std::move(info); } else { - if (!entry->second.loaded) { + if (entry->second.install_mode != ExtensionInstallMode::STATICALLY_LINKED) { entry->second.file_path = info.file_path; entry->second.install_mode = info.install_mode; entry->second.installed_from = info.installed_from; @@ -144,13 +144,12 @@ unique_ptr DuckDBExtensionsInit(ClientContext &context auto &ext_info = e.second; auto entry = installed_extensions.find(ext_name); if (entry == installed_extensions.end() || !entry->second.installed) { - ExtensionInformation info; + ExtensionInformation &info = installed_extensions[ext_name]; info.name = ext_name; info.loaded = true; info.extension_version = ext_info.version; info.installed = ext_info.mode == ExtensionInstallMode::STATICALLY_LINKED; info.install_mode = ext_info.mode; - installed_extensions[ext_name] = std::move(info); } else { entry->second.loaded = true; entry->second.extension_version = ext_info.version; diff --git a/src/duckdb/src/function/table/version/pragma_version.cpp b/src/duckdb/src/function/table/version/pragma_version.cpp index e4a43ea0..70242dd5 100644 --- a/src/duckdb/src/function/table/version/pragma_version.cpp +++ b/src/duckdb/src/function/table/version/pragma_version.cpp @@ -1,17 +1,17 @@ #ifndef DUCKDB_PATCH_VERSION -#define DUCKDB_PATCH_VERSION "3" +#define DUCKDB_PATCH_VERSION "0" #endif #ifndef DUCKDB_MINOR_VERSION -#define DUCKDB_MINOR_VERSION 10 +#define DUCKDB_MINOR_VERSION 0 #endif #ifndef DUCKDB_MAJOR_VERSION -#define DUCKDB_MAJOR_VERSION 0 +#define DUCKDB_MAJOR_VERSION 1 #endif #ifndef DUCKDB_VERSION -#define DUCKDB_VERSION "v0.10.3" +#define DUCKDB_VERSION "v1.0.0" #endif #ifndef DUCKDB_SOURCE_ID -#define DUCKDB_SOURCE_ID "70fd6a8a24" +#define DUCKDB_SOURCE_ID "1f98600c2c" #endif #include "duckdb/function/table/system_functions.hpp" #include "duckdb/main/database.hpp" diff --git a/src/duckdb/src/include/duckdb.h b/src/duckdb/src/include/duckdb.h index 2a7d2420..395befc0 100644 --- a/src/duckdb/src/include/duckdb.h +++ b/src/duckdb/src/include/duckdb.h @@ -35,6 +35,13 @@ #endif #endif +//! In the future, we are planning to move extension functions to a separate header. For now you can set the define +//! below to remove the functions that are planned to be moved out of this header. +// #define DUCKDB_NO_EXTENSION_FUNCTIONS + +//! Set the define below to remove all functions that are deprecated or planned to be deprecated +// #define DUCKDB_API_NO_DEPRECATED + //! API versions //! If no explicit API version is defined, the latest API version is used. //! Note that using older API versions (i.e. not using DUCKDB_API_LATEST) is deprecated. @@ -433,6 +440,7 @@ typedef struct _duckdb_value { // Table function types //===--------------------------------------------------------------------===// +#ifndef DUCKDB_NO_EXTENSION_FUNCTIONS //! A table function. Must be destroyed with `duckdb_destroy_table_function`. typedef void *duckdb_table_function; @@ -463,6 +471,7 @@ typedef void *duckdb_replacement_scan_info; //! A replacement scan function that can be added to a database. typedef void (*duckdb_replacement_callback_t)(duckdb_replacement_scan_info info, const char *table_name, void *data); +#endif //===--------------------------------------------------------------------===// // Arrow-related types @@ -709,13 +718,17 @@ Returns the number of columns present in a the result object. */ DUCKDB_API idx_t duckdb_column_count(duckdb_result *result); +#ifndef DUCKDB_API_NO_DEPRECATED /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Returns the number of rows present in the result object. * result: The result object. * returns: The number of rows present in the result object. */ DUCKDB_API idx_t duckdb_row_count(duckdb_result *result); +#endif /*! Returns the number of rows changed by the query stored in the result. This is relevant only for INSERT/UPDATE/DELETE @@ -726,6 +739,7 @@ queries. For other queries the rows_changed will be 0. */ DUCKDB_API idx_t duckdb_rows_changed(duckdb_result *result); +#ifndef DUCKDB_API_NO_DEPRECATED /*! **DEPRECATED**: Prefer using `duckdb_result_get_chunk` instead. @@ -769,6 +783,7 @@ if (nullmask[row]) { * returns: The nullmask of the specified column. */ DUCKDB_API bool *duckdb_nullmask_data(duckdb_result *result, idx_t col); +#endif /*! Returns the error message contained within the result. The error is only set if `duckdb_query` returns `DuckDBError`. @@ -783,8 +798,10 @@ DUCKDB_API const char *duckdb_result_error(duckdb_result *result); //===--------------------------------------------------------------------===// // Result Functions //===--------------------------------------------------------------------===// - +#ifndef DUCKDB_API_NO_DEPRECATED /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Fetches a data chunk from the duckdb_result. This function should be called repeatedly until the result is exhausted. The result must be destroyed with `duckdb_destroy_data_chunk`. @@ -804,6 +821,8 @@ Use `duckdb_result_chunk_count` to figure out how many chunks there are in the r DUCKDB_API duckdb_data_chunk duckdb_result_get_chunk(duckdb_result result, idx_t chunk_index); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Checks if the type of the internal result is StreamQueryResult. * result: The result object to check. @@ -812,12 +831,15 @@ Checks if the type of the internal result is StreamQueryResult. DUCKDB_API bool duckdb_result_is_streaming(duckdb_result result); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Returns the number of data chunks present in the result. * result: The result object * returns: Number of data chunks present in the result. */ DUCKDB_API idx_t duckdb_result_chunk_count(duckdb_result result); +#endif /*! Returns the return_type of the given result, or DUCKDB_RETURN_TYPE_INVALID on error @@ -827,6 +849,7 @@ Returns the return_type of the given result, or DUCKDB_RETURN_TYPE_INVALID on er */ DUCKDB_API duckdb_result_type duckdb_result_return_type(duckdb_result result); +#ifndef DUCKDB_API_NO_DEPRECATED //===--------------------------------------------------------------------===// // Safe fetch functions //===--------------------------------------------------------------------===// @@ -837,91 +860,127 @@ DUCKDB_API duckdb_result_type duckdb_result_return_type(duckdb_result result); // For fast access of values prefer using `duckdb_result_get_chunk` /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The boolean value at the specified location, or false if the value cannot be converted. */ DUCKDB_API bool duckdb_value_boolean(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The int8_t value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API int8_t duckdb_value_int8(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The int16_t value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API int16_t duckdb_value_int16(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The int32_t value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API int32_t duckdb_value_int32(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The int64_t value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API int64_t duckdb_value_int64(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The duckdb_hugeint value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API duckdb_hugeint duckdb_value_hugeint(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The duckdb_uhugeint value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API duckdb_uhugeint duckdb_value_uhugeint(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The duckdb_decimal value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API duckdb_decimal duckdb_value_decimal(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The uint8_t value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API uint8_t duckdb_value_uint8(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The uint16_t value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API uint16_t duckdb_value_uint16(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The uint32_t value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API uint32_t duckdb_value_uint32(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The uint64_t value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API uint64_t duckdb_value_uint64(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The float value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API float duckdb_value_float(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The double value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API double duckdb_value_double(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The duckdb_date value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API duckdb_date duckdb_value_date(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The duckdb_time value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API duckdb_time duckdb_value_time(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The duckdb_timestamp value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API duckdb_timestamp duckdb_value_timestamp(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The duckdb_interval value at the specified location, or 0 if the value cannot be converted. */ DUCKDB_API duckdb_interval duckdb_value_interval(duckdb_result *result, idx_t col, idx_t row); @@ -934,6 +993,8 @@ converted. The result must be freed with `duckdb_free`. DUCKDB_API char *duckdb_value_varchar(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The string value at the specified location. Attempts to cast the result value to string. * No support for nested types, and for other complex types. * The resulting field "string.data" must be freed with `duckdb_free.` @@ -961,15 +1022,20 @@ The result must NOT be freed. DUCKDB_API duckdb_string duckdb_value_string_internal(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: The duckdb_blob value at the specified location. Returns a blob with blob.data set to nullptr if the value cannot be converted. The resulting field "blob.data" must be freed with `duckdb_free.` */ DUCKDB_API duckdb_blob duckdb_value_blob(duckdb_result *result, idx_t col, idx_t row); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + * returns: Returns true if the value at the specified index is NULL, and false otherwise. */ DUCKDB_API bool duckdb_value_is_null(duckdb_result *result, idx_t col, idx_t row); +#endif //===--------------------------------------------------------------------===// // Helpers @@ -1405,7 +1471,10 @@ Note that the result must be freed with `duckdb_destroy_result`. DUCKDB_API duckdb_state duckdb_execute_prepared(duckdb_prepared_statement prepared_statement, duckdb_result *out_result); +#ifndef DUCKDB_API_NO_DEPRECATED /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Executes the prepared statement with the given bound parameters, and returns an optionally-streaming query result. To determine if the resulting query was in fact streamed, use `duckdb_result_is_streaming` @@ -1420,6 +1489,7 @@ Note that the result must be freed with `duckdb_destroy_result`. */ DUCKDB_API duckdb_state duckdb_execute_prepared_streaming(duckdb_prepared_statement prepared_statement, duckdb_result *out_result); +#endif //===--------------------------------------------------------------------===// // Extract Statements @@ -1492,8 +1562,10 @@ Note that after calling `duckdb_pending_prepared`, the pending result should alw */ DUCKDB_API duckdb_state duckdb_pending_prepared(duckdb_prepared_statement prepared_statement, duckdb_pending_result *out_result); - +#ifndef DUCKDB_API_NO_DEPRECATED /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Executes the prepared statement with the given bound parameters, and returns a pending result. This pending result will create a streaming duckdb_result when executed. The pending result represents an intermediate structure for a query that is not yet fully executed. @@ -1507,6 +1579,7 @@ Note that after calling `duckdb_pending_prepared_streaming`, the pending result */ DUCKDB_API duckdb_state duckdb_pending_prepared_streaming(duckdb_prepared_statement prepared_statement, duckdb_pending_result *out_result); +#endif /*! Closes the pending result and de-allocates all memory allocated for the result. @@ -2170,6 +2243,7 @@ Equivalent to `duckdb_validity_set_row_validity` with valid set to true. */ DUCKDB_API void duckdb_validity_set_row_valid(uint64_t *validity, idx_t row); +#ifndef DUCKDB_NO_EXTENSION_FUNCTIONS //===--------------------------------------------------------------------===// // Table Functions //===--------------------------------------------------------------------===// @@ -2516,6 +2590,7 @@ Report that an error has occurred while executing the replacement scan. * error: The error message */ DUCKDB_API void duckdb_replacement_scan_set_error(duckdb_replacement_scan_info info, const char *error); +#endif //===--------------------------------------------------------------------===// // Appender @@ -2743,11 +2818,14 @@ If the append is successful, DuckDBSuccess is returned. */ DUCKDB_API duckdb_state duckdb_append_data_chunk(duckdb_appender appender, duckdb_data_chunk chunk); +#ifndef DUCKDB_API_NO_DEPRECATED //===--------------------------------------------------------------------===// // Arrow Interface //===--------------------------------------------------------------------===// /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Executes a SQL query within a connection and stores the full (materialized) result in an arrow structure. If the query fails to execute, DuckDBError is returned and the error message can be retrieved by calling `duckdb_query_arrow_error`. @@ -2763,6 +2841,8 @@ query fails, otherwise the error stored within the result will not be freed corr DUCKDB_API duckdb_state duckdb_query_arrow(duckdb_connection connection, const char *query, duckdb_arrow *out_result); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Fetch the internal arrow schema from the arrow result. Remember to call release on the respective ArrowSchema object. @@ -2773,6 +2853,8 @@ ArrowSchema object. DUCKDB_API duckdb_state duckdb_query_arrow_schema(duckdb_arrow result, duckdb_arrow_schema *out_schema); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Fetch the internal arrow schema from the prepared statement. Remember to call release on the respective ArrowSchema object. @@ -2783,6 +2865,8 @@ ArrowSchema object. DUCKDB_API duckdb_state duckdb_prepared_arrow_schema(duckdb_prepared_statement prepared, duckdb_arrow_schema *out_schema); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Convert a data chunk into an arrow struct array. Remember to call release on the respective ArrowArray object. @@ -2793,6 +2877,8 @@ ArrowArray object. DUCKDB_API void duckdb_result_arrow_array(duckdb_result result, duckdb_data_chunk chunk, duckdb_arrow_array *out_array); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Fetch an internal arrow struct array from the arrow result. Remember to call release on the respective ArrowArray object. @@ -2806,6 +2892,8 @@ So consume the out_array before calling this function again. DUCKDB_API duckdb_state duckdb_query_arrow_array(duckdb_arrow result, duckdb_arrow_array *out_array); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Returns the number of columns present in the arrow result object. * result: The result object. @@ -2814,6 +2902,8 @@ Returns the number of columns present in the arrow result object. DUCKDB_API idx_t duckdb_arrow_column_count(duckdb_arrow result); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Returns the number of rows present in the arrow result object. * result: The result object. @@ -2822,6 +2912,8 @@ Returns the number of rows present in the arrow result object. DUCKDB_API idx_t duckdb_arrow_row_count(duckdb_arrow result); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Returns the number of rows changed by the query stored in the arrow result. This is relevant only for INSERT/UPDATE/DELETE queries. For other queries the rows_changed will be 0. @@ -2831,7 +2923,9 @@ INSERT/UPDATE/DELETE queries. For other queries the rows_changed will be 0. DUCKDB_API idx_t duckdb_arrow_rows_changed(duckdb_arrow result); /*! -Returns the error message contained within the result. The error is only set if `duckdb_query_arrow` returns +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + + Returns the error message contained within the result. The error is only set if `duckdb_query_arrow` returns `DuckDBError`. The error message should not be freed. It will be de-allocated when `duckdb_destroy_arrow` is called. @@ -2842,6 +2936,8 @@ The error message should not be freed. It will be de-allocated when `duckdb_dest DUCKDB_API const char *duckdb_query_arrow_error(duckdb_arrow result); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Closes the result and de-allocates all memory allocated for the arrow result. * result: The result to destroy. @@ -2849,6 +2945,8 @@ Closes the result and de-allocates all memory allocated for the arrow result. DUCKDB_API void duckdb_destroy_arrow(duckdb_arrow *result); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Releases the arrow array stream and de-allocates its memory. * stream: The arrow array stream to destroy. @@ -2856,6 +2954,8 @@ Releases the arrow array stream and de-allocates its memory. DUCKDB_API void duckdb_destroy_arrow_stream(duckdb_arrow_stream *stream_p); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Executes the prepared statement with the given bound parameters, and returns an arrow query result. Note that after running `duckdb_execute_prepared_arrow`, `duckdb_destroy_arrow` must be called on the result object. @@ -2867,6 +2967,8 @@ DUCKDB_API duckdb_state duckdb_execute_prepared_arrow(duckdb_prepared_statement duckdb_arrow *out_result); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Scans the Arrow stream and creates a view with the given name. * connection: The connection on which to execute the scan. @@ -2878,6 +2980,8 @@ DUCKDB_API duckdb_state duckdb_arrow_scan(duckdb_connection connection, const ch duckdb_arrow_stream arrow); /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Scans the Arrow array and creates a view with the given name. Note that after running `duckdb_arrow_array_scan`, `duckdb_destroy_arrow_stream` must be called on the out stream. @@ -2891,7 +2995,9 @@ Note that after running `duckdb_arrow_array_scan`, `duckdb_destroy_arrow_stream` DUCKDB_API duckdb_state duckdb_arrow_array_scan(duckdb_connection connection, const char *table_name, duckdb_arrow_schema arrow_schema, duckdb_arrow_array arrow_array, duckdb_arrow_stream *out_stream); +#endif +#ifndef DUCKDB_NO_EXTENSION_FUNCTIONS //===--------------------------------------------------------------------===// // Threading Information //===--------------------------------------------------------------------===// @@ -2972,12 +3078,15 @@ Returns true if the execution of the current query is finished. * con: The connection on which to check */ DUCKDB_API bool duckdb_execution_is_finished(duckdb_connection con); +#endif //===--------------------------------------------------------------------===// // Streaming Result Interface //===--------------------------------------------------------------------===// - +#ifndef DUCKDB_API_NO_DEPRECATED /*! +**DEPRECATION NOTICE**: This method is scheduled for removal in a future release. + Fetches a data chunk from the (streaming) duckdb_result. This function should be called repeatedly until the result is exhausted. @@ -2994,6 +3103,19 @@ It is not known beforehand how many chunks will be returned by this result. * returns: The resulting data chunk. Returns `NULL` if the result has an error. */ DUCKDB_API duckdb_data_chunk duckdb_stream_fetch_chunk(duckdb_result result); +#endif + +/*! +Fetches a data chunk from a duckdb_result. This function should be called repeatedly until the result is exhausted. + +The result must be destroyed with `duckdb_destroy_data_chunk`. + +It is not known beforehand how many chunks will be returned by this result. + +* result: The result object to fetch the data chunk from. +* returns: The resulting data chunk. Returns `NULL` if the result has an error. +*/ +DUCKDB_API duckdb_data_chunk duckdb_fetch_chunk(duckdb_result result); #ifdef __cplusplus } diff --git a/src/duckdb/src/include/duckdb/common/enable_shared_from_this.ipp b/src/duckdb/src/include/duckdb/common/enable_shared_from_this_ipp.hpp similarity index 100% rename from src/duckdb/src/include/duckdb/common/enable_shared_from_this.ipp rename to src/duckdb/src/include/duckdb/common/enable_shared_from_this_ipp.hpp diff --git a/src/duckdb/src/include/duckdb/common/enum_util.hpp b/src/duckdb/src/include/duckdb/common/enum_util.hpp index fea98310..6ca09798 100644 --- a/src/duckdb/src/include/duckdb/common/enum_util.hpp +++ b/src/duckdb/src/include/duckdb/common/enum_util.hpp @@ -98,6 +98,8 @@ enum class ConflictManagerMode : uint8_t; enum class ConstraintType : uint8_t; +enum class CopyOverwriteMode : uint8_t; + enum class DataFileType : uint8_t; enum class DatePartSpecifier : uint8_t; @@ -434,6 +436,9 @@ const char* EnumUtil::ToChars(ConflictManagerMode value); template<> const char* EnumUtil::ToChars(ConstraintType value); +template<> +const char* EnumUtil::ToChars(CopyOverwriteMode value); + template<> const char* EnumUtil::ToChars(DataFileType value); @@ -888,6 +893,9 @@ ConflictManagerMode EnumUtil::FromString(const char *value) template<> ConstraintType EnumUtil::FromString(const char *value); +template<> +CopyOverwriteMode EnumUtil::FromString(const char *value); + template<> DataFileType EnumUtil::FromString(const char *value); diff --git a/src/duckdb/src/include/duckdb/common/enums/copy_overwrite_mode.hpp b/src/duckdb/src/include/duckdb/common/enums/copy_overwrite_mode.hpp new file mode 100644 index 00000000..595a867e --- /dev/null +++ b/src/duckdb/src/include/duckdb/common/enums/copy_overwrite_mode.hpp @@ -0,0 +1,18 @@ +//===----------------------------------------------------------------------===// +// DuckDB +// +// duckdb/common/enums/copy_overwrite_mode.hpp +// +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "duckdb/common/constants.hpp" +#include "duckdb/common/vector.hpp" + +namespace duckdb { + +enum class CopyOverwriteMode : uint8_t { COPY_ERROR_ON_CONFLICT = 0, COPY_OVERWRITE = 1, COPY_OVERWRITE_OR_IGNORE = 2 }; + +} // namespace duckdb diff --git a/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp b/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp index 98217540..5992904c 100644 --- a/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +++ b/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp @@ -52,6 +52,7 @@ struct MultiFileReaderBindData { struct MultiFileReaderGlobalState { MultiFileReaderGlobalState(vector extra_columns_p, optional_ptr file_list_p) : extra_columns(std::move(extra_columns_p)), file_list(file_list_p) {}; + virtual ~MultiFileReaderGlobalState(); //! extra columns that will be produced during scanning const vector extra_columns; diff --git a/src/duckdb/src/include/duckdb/common/shared_ptr.hpp b/src/duckdb/src/include/duckdb/common/shared_ptr.hpp index 6d0910ca..eff60e3d 100644 --- a/src/duckdb/src/include/duckdb/common/shared_ptr.hpp +++ b/src/duckdb/src/include/duckdb/common/shared_ptr.hpp @@ -37,9 +37,9 @@ struct compatible_with_t : std::is_convertible {}; // NOLINT: invalid } // namespace duckdb -#include "duckdb/common/shared_ptr.ipp" -#include "duckdb/common/weak_ptr.ipp" -#include "duckdb/common/enable_shared_from_this.ipp" +#include "duckdb/common/shared_ptr_ipp.hpp" +#include "duckdb/common/weak_ptr_ipp.hpp" +#include "duckdb/common/enable_shared_from_this_ipp.hpp" namespace duckdb { diff --git a/src/duckdb/src/include/duckdb/common/shared_ptr.ipp b/src/duckdb/src/include/duckdb/common/shared_ptr_ipp.hpp similarity index 100% rename from src/duckdb/src/include/duckdb/common/shared_ptr.ipp rename to src/duckdb/src/include/duckdb/common/shared_ptr_ipp.hpp diff --git a/src/duckdb/src/include/duckdb/common/weak_ptr.ipp b/src/duckdb/src/include/duckdb/common/weak_ptr_ipp.hpp similarity index 100% rename from src/duckdb/src/include/duckdb/common/weak_ptr.ipp rename to src/duckdb/src/include/duckdb/common/weak_ptr_ipp.hpp diff --git a/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp b/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp index a405ee5c..980d5dc9 100644 --- a/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp +++ b/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp @@ -13,6 +13,7 @@ #include "duckdb/execution/physical_operator.hpp" #include "duckdb/function/copy_function.hpp" #include "duckdb/parser/parsed_data/copy_info.hpp" +#include "duckdb/common/enums/copy_overwrite_mode.hpp" namespace duckdb { @@ -31,7 +32,7 @@ class PhysicalCopyToFile : public PhysicalOperator { bool use_tmp_file; FilenamePattern filename_pattern; string file_extension; - bool overwrite_or_ignore; + CopyOverwriteMode overwrite_mode; bool parallel; bool per_thread_output; optional_idx file_size_bytes; diff --git a/src/duckdb/src/include/duckdb/main/attached_database.hpp b/src/duckdb/src/include/duckdb/main/attached_database.hpp index a02a3f10..6c6ffa40 100644 --- a/src/duckdb/src/include/duckdb/main/attached_database.hpp +++ b/src/duckdb/src/include/duckdb/main/attached_database.hpp @@ -54,6 +54,11 @@ class AttachedDatabase : public CatalogEntry { DatabaseInstance &GetDatabase() { return db; } + + optional_ptr GetStorageExtension() { + return storage_extension; + } + const string &GetName() const { return name; } @@ -74,6 +79,7 @@ class AttachedDatabase : public CatalogEntry { unique_ptr transaction_manager; AttachedDatabaseType type; optional_ptr parent_catalog; + optional_ptr storage_extension; bool is_initial_database = false; bool is_closed = false; }; diff --git a/src/duckdb/src/include/duckdb/main/config.hpp b/src/duckdb/src/include/duckdb/main/config.hpp index fe47d933..35928b98 100644 --- a/src/duckdb/src/include/duckdb/main/config.hpp +++ b/src/duckdb/src/include/duckdb/main/config.hpp @@ -208,6 +208,10 @@ struct DBConfigOptions { bool allow_extensions_metadata_mismatch = false; //! Enable emitting FSST Vectors bool enable_fsst_vectors = false; + //! Enable VIEWs to create dependencies + bool enable_view_dependencies = false; + //! Enable macros to create dependencies + bool enable_macro_dependencies = false; //! Start transactions immediately in all attached databases - instead of lazily when a database is referenced bool immediate_transaction_mode = false; //! Debug setting - how to initialize blocks in the storage layer when allocating diff --git a/src/duckdb/src/include/duckdb/main/extension_helper.hpp b/src/duckdb/src/include/duckdb/main/extension_helper.hpp index c88f95b7..7438e774 100644 --- a/src/duckdb/src/include/duckdb/main/extension_helper.hpp +++ b/src/duckdb/src/include/duckdb/main/extension_helper.hpp @@ -88,10 +88,12 @@ class ExtensionHelper { static unique_ptr InstallExtension(ClientContext &context, const string &extension, bool force_install, optional_ptr repository = nullptr, + bool throw_on_origin_mismatch = false, const string &version = ""); static unique_ptr InstallExtension(DBConfig &config, FileSystem &fs, const string &extension, bool force_install, optional_ptr repository = nullptr, + bool throw_on_origin_mismatch = false, const string &version = ""); //! Load an extension static void LoadExternalExtension(ClientContext &context, const string &extension); @@ -213,12 +215,10 @@ class ExtensionHelper { static bool CreateSuggestions(const string &extension_name, string &message); private: - static unique_ptr InstallExtensionInternal(DBConfig &config, FileSystem &fs, - const string &local_path, const string &extension, - bool force_install, const string &version, - optional_ptr repository, - optional_ptr http_logger = nullptr, - optional_ptr context = nullptr); + static unique_ptr InstallExtensionInternal( + DBConfig &config, FileSystem &fs, const string &local_path, const string &extension, bool force_install, + bool throw_on_origin_mismatch, const string &version, optional_ptr repository, + optional_ptr http_logger = nullptr, optional_ptr context = nullptr); static const vector PathComponents(); static string DefaultExtensionFolder(FileSystem &fs); static bool AllowAutoInstall(const string &extension); diff --git a/src/duckdb/src/include/duckdb/main/settings.hpp b/src/duckdb/src/include/duckdb/main/settings.hpp index 10d9fe30..a333cd92 100644 --- a/src/duckdb/src/include/duckdb/main/settings.hpp +++ b/src/duckdb/src/include/duckdb/main/settings.hpp @@ -206,6 +206,26 @@ struct EnableExternalAccessSetting { static Value GetSetting(const ClientContext &context); }; +struct EnableMacrosDependencies { + static constexpr const char *Name = "enable_macro_dependencies"; + static constexpr const char *Description = + "Enable created MACROs to create dependencies on the referenced objects (such as tables)"; + static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN; + static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value ¶meter); + static void ResetGlobal(DatabaseInstance *db, DBConfig &config); + static Value GetSetting(const ClientContext &context); +}; + +struct EnableViewDependencies { + static constexpr const char *Name = "enable_view_dependencies"; + static constexpr const char *Description = + "Enable created VIEWs to create dependencies on the referenced objects (such as tables)"; + static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN; + static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value ¶meter); + static void ResetGlobal(DatabaseInstance *db, DBConfig &config); + static Value GetSetting(const ClientContext &context); +}; + struct EnableFSSTVectors { static constexpr const char *Name = "enable_fsst_vectors"; static constexpr const char *Description = diff --git a/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp b/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp index 4cb62ca3..8731730c 100644 --- a/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +++ b/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp @@ -51,16 +51,16 @@ class LogicalComparisonJoin : public LogicalJoin { vector conditions, vector> arbitrary_expressions); - static void ExtractJoinConditions(ClientContext &context, JoinType type, unique_ptr &left_child, - unique_ptr &right_child, unique_ptr condition, - vector &conditions, + static void ExtractJoinConditions(ClientContext &context, JoinType type, JoinRefType ref_type, + unique_ptr &left_child, unique_ptr &right_child, + unique_ptr condition, vector &conditions, vector> &arbitrary_expressions); - static void ExtractJoinConditions(ClientContext &context, JoinType type, unique_ptr &left_child, - unique_ptr &right_child, + static void ExtractJoinConditions(ClientContext &context, JoinType type, JoinRefType ref_type, + unique_ptr &left_child, unique_ptr &right_child, vector> &expressions, vector &conditions, vector> &arbitrary_expressions); - static void ExtractJoinConditions(ClientContext &context, JoinType type, unique_ptr &left_child, - unique_ptr &right_child, + static void ExtractJoinConditions(ClientContext &context, JoinType type, JoinRefType ref_type, + unique_ptr &left_child, unique_ptr &right_child, const unordered_set &left_bindings, const unordered_set &right_bindings, vector> &expressions, vector &conditions, diff --git a/src/duckdb/src/include/duckdb/planner/operator/logical_copy_to_file.hpp b/src/duckdb/src/include/duckdb/planner/operator/logical_copy_to_file.hpp index ea79b940..bc95ae5a 100644 --- a/src/duckdb/src/include/duckdb/planner/operator/logical_copy_to_file.hpp +++ b/src/duckdb/src/include/duckdb/planner/operator/logical_copy_to_file.hpp @@ -13,6 +13,7 @@ #include "duckdb/common/optional_idx.hpp" #include "duckdb/function/copy_function.hpp" #include "duckdb/planner/logical_operator.hpp" +#include "duckdb/common/enums/copy_overwrite_mode.hpp" namespace duckdb { @@ -33,7 +34,7 @@ class LogicalCopyToFile : public LogicalOperator { bool use_tmp_file; FilenamePattern filename_pattern; string file_extension; - bool overwrite_or_ignore; + CopyOverwriteMode overwrite_mode; bool per_thread_output; optional_idx file_size_bytes; diff --git a/src/duckdb/src/include/duckdb/storage/storage_extension.hpp b/src/duckdb/src/include/duckdb/storage/storage_extension.hpp index 369dd71a..eece0b09 100644 --- a/src/duckdb/src/include/duckdb/storage/storage_extension.hpp +++ b/src/duckdb/src/include/duckdb/storage/storage_extension.hpp @@ -11,6 +11,7 @@ #include "duckdb/common/common.hpp" #include "duckdb/common/enums/access_mode.hpp" #include "duckdb/parser/tableref/table_function_ref.hpp" +#include "duckdb/storage/storage_manager.hpp" namespace duckdb { class AttachedDatabase; @@ -40,6 +41,12 @@ class StorageExtension { virtual ~StorageExtension() { } + + virtual void OnCheckpointStart(AttachedDatabase &db, CheckpointOptions checkpoint_options) { + } + + virtual void OnCheckpointEnd(AttachedDatabase &db, CheckpointOptions checkpoint_options) { + } }; } // namespace duckdb diff --git a/src/duckdb/src/main/attached_database.cpp b/src/duckdb/src/main/attached_database.cpp index b854760d..4655ae07 100644 --- a/src/duckdb/src/main/attached_database.cpp +++ b/src/duckdb/src/main/attached_database.cpp @@ -41,14 +41,15 @@ AttachedDatabase::AttachedDatabase(DatabaseInstance &db, Catalog &catalog_p, str internal = true; } -AttachedDatabase::AttachedDatabase(DatabaseInstance &db, Catalog &catalog_p, StorageExtension &storage_extension, +AttachedDatabase::AttachedDatabase(DatabaseInstance &db, Catalog &catalog_p, StorageExtension &storage_extension_p, ClientContext &context, string name_p, const AttachInfo &info, AccessMode access_mode) - : CatalogEntry(CatalogType::DATABASE_ENTRY, catalog_p, std::move(name_p)), db(db), parent_catalog(&catalog_p) { + : CatalogEntry(CatalogType::DATABASE_ENTRY, catalog_p, std::move(name_p)), db(db), parent_catalog(&catalog_p), + storage_extension(&storage_extension_p) { type = access_mode == AccessMode::READ_ONLY ? AttachedDatabaseType::READ_ONLY_DATABASE : AttachedDatabaseType::READ_WRITE_DATABASE; - catalog = - storage_extension.attach(storage_extension.storage_info.get(), context, *this, name, *info.Copy(), access_mode); + catalog = storage_extension->attach(storage_extension->storage_info.get(), context, *this, name, *info.Copy(), + access_mode); if (!catalog) { throw InternalException("AttachedDatabase - attach function did not return a catalog"); } @@ -57,7 +58,7 @@ AttachedDatabase::AttachedDatabase(DatabaseInstance &db, Catalog &catalog_p, Sto storage = make_uniq(*this, info.path, access_mode == AccessMode::READ_ONLY); } transaction_manager = - storage_extension.create_transaction_manager(storage_extension.storage_info.get(), *this, *catalog); + storage_extension->create_transaction_manager(storage_extension->storage_info.get(), *this, *catalog); if (!transaction_manager) { throw InternalException( "AttachedDatabase - create_transaction_manager function did not return a transaction manager"); @@ -82,7 +83,7 @@ bool AttachedDatabase::IsReadOnly() const { } bool AttachedDatabase::NameIsReserved(const string &name) { - return name == DEFAULT_SCHEMA || name == TEMP_CATALOG; + return name == DEFAULT_SCHEMA || name == TEMP_CATALOG || name == SYSTEM_CATALOG; } string AttachedDatabase::ExtractDatabaseName(const string &dbpath, FileSystem &fs) { diff --git a/src/duckdb/src/main/capi/stream-c.cpp b/src/duckdb/src/main/capi/stream-c.cpp index 8fcc42fb..5b950599 100644 --- a/src/duckdb/src/main/capi/stream-c.cpp +++ b/src/duckdb/src/main/capi/stream-c.cpp @@ -7,19 +7,28 @@ duckdb_data_chunk duckdb_stream_fetch_chunk(duckdb_result result) { return nullptr; } auto &result_data = *((duckdb::DuckDBResultData *)result.internal_data); - if (result_data.result_set_type == duckdb::CAPIResultSetType::CAPI_RESULT_TYPE_DEPRECATED) { - return nullptr; - } if (result_data.result->type != duckdb::QueryResultType::STREAM_RESULT) { // We can only fetch from a StreamQueryResult return nullptr; } - result_data.result_set_type = duckdb::CAPIResultSetType::CAPI_RESULT_TYPE_STREAMING; - auto &streaming = (duckdb::StreamQueryResult &)*result_data.result; - if (!streaming.IsOpen()) { + return duckdb_fetch_chunk(result); +} + +duckdb_data_chunk duckdb_fetch_chunk(duckdb_result result) { + if (!result.internal_data) { return nullptr; } + auto &result_data = *((duckdb::DuckDBResultData *)result.internal_data); + if (result_data.result_set_type == duckdb::CAPIResultSetType::CAPI_RESULT_TYPE_DEPRECATED) { + return nullptr; + } + result_data.result_set_type = duckdb::CAPIResultSetType::CAPI_RESULT_TYPE_STREAMING; + auto &result_instance = (duckdb::QueryResult &)*result_data.result; // FetchRaw ? Do we care about flattening them? - auto chunk = streaming.Fetch(); - return reinterpret_cast(chunk.release()); + try { + auto chunk = result_instance.Fetch(); + return reinterpret_cast(chunk.release()); + } catch (std::exception &e) { + return nullptr; + } } diff --git a/src/duckdb/src/main/capi/table_function-c.cpp b/src/duckdb/src/main/capi/table_function-c.cpp index a6916e8e..a3799a46 100644 --- a/src/duckdb/src/main/capi/table_function-c.cpp +++ b/src/duckdb/src/main/capi/table_function-c.cpp @@ -284,13 +284,17 @@ duckdb_state duckdb_register_table_function(duckdb_connection connection, duckdb if (tf->name.empty() || !info->bind || !info->init || !info->function) { return DuckDBError; } - con->context->RunFunctionInTransaction([&]() { - auto &catalog = duckdb::Catalog::GetSystemCatalog(*con->context); - duckdb::CreateTableFunctionInfo tf_info(*tf); - - // create the function in the catalog - catalog.CreateTableFunction(*con->context, tf_info); - }); + try { + con->context->RunFunctionInTransaction([&]() { + auto &catalog = duckdb::Catalog::GetSystemCatalog(*con->context); + duckdb::CreateTableFunctionInfo tf_info(*tf); + + // create the function in the catalog + catalog.CreateTableFunction(*con->context, tf_info); + }); + } catch (...) { // LCOV_EXCL_START + return DuckDBError; + } // LCOV_EXCL_STOP return DuckDBSuccess; } diff --git a/src/duckdb/src/main/config.cpp b/src/duckdb/src/main/config.cpp index 86bfacb4..17a503a0 100644 --- a/src/duckdb/src/main/config.cpp +++ b/src/duckdb/src/main/config.cpp @@ -94,6 +94,8 @@ static const ConfigurationOption internal_options[] = { DUCKDB_GLOBAL(ForceBitpackingModeSetting), DUCKDB_LOCAL(HomeDirectorySetting), DUCKDB_LOCAL(LogQueryPathSetting), + DUCKDB_GLOBAL(EnableMacrosDependencies), + DUCKDB_GLOBAL(EnableViewDependencies), DUCKDB_GLOBAL(LockConfigurationSetting), DUCKDB_GLOBAL(ImmediateTransactionModeSetting), DUCKDB_LOCAL(IntegerDivisionSetting), diff --git a/src/duckdb/src/main/extension/extension_helper.cpp b/src/duckdb/src/main/extension/extension_helper.cpp index 3954e232..37b3cd1a 100644 --- a/src/duckdb/src/main/extension/extension_helper.cpp +++ b/src/duckdb/src/main/extension/extension_helper.cpp @@ -62,10 +62,6 @@ #include "icu_extension.hpp" #endif -#if DUCKDB_EXTENSION_EXCEL_LINKED -#include "excel_extension.hpp" -#endif - #if DUCKDB_EXTENSION_PARQUET_LINKED #include "parquet_extension.hpp" #endif @@ -126,6 +122,8 @@ static const DefaultExtension internal_extensions[] = { {"arrow", "A zero-copy data integration between Apache Arrow and DuckDB", false}, {"azure", "Adds a filesystem abstraction for Azure blob storage to DuckDB", false}, {"iceberg", "Adds support for Apache Iceberg", false}, + {"vss", "Adds indexing support to accelerate Vector Similarity Search", false}, + {"delta", "Adds support for Delta Lake", false}, {nullptr, nullptr, false}}; idx_t ExtensionHelper::DefaultExtensionCount() { @@ -213,7 +211,7 @@ bool ExtensionHelper::TryAutoLoadExtension(ClientContext &context, const string if (dbconfig.options.autoinstall_known_extensions) { auto &config = DBConfig::GetConfig(context); auto autoinstall_repo = ExtensionRepository::GetRepositoryByUrl(config.options.autoinstall_extension_repo); - ExtensionHelper::InstallExtension(context, extension_name, false, autoinstall_repo); + ExtensionHelper::InstallExtension(context, extension_name, false, autoinstall_repo, false); } ExtensionHelper::LoadExternalExtension(context, extension_name); return true; @@ -323,26 +321,6 @@ vector ExtensionHelper::UpdateExtensions(DatabaseInstance }); #endif - for (const auto &extension : db.LoadedExtensions()) { - if (seen_extensions.find(extension) != seen_extensions.end()) { - const auto &loaded_extension_data = db.LoadedExtensionsData(); - const auto &loaded_install_info = loaded_extension_data.find(extension); - - ExtensionUpdateResult statically_loaded_ext_result; - - if (loaded_install_info == loaded_extension_data.end()) { - statically_loaded_ext_result.tag = ExtensionUpdateResultTag::UNKNOWN; - } else if (loaded_install_info->second.mode == ExtensionInstallMode::STATICALLY_LINKED) { - statically_loaded_ext_result.tag = ExtensionUpdateResultTag::STATICALLY_LOADED; - statically_loaded_ext_result.installed_version = loaded_install_info->second.version; - } else { - statically_loaded_ext_result.tag = ExtensionUpdateResultTag::UNKNOWN; - } - - result.push_back(std::move(statically_loaded_ext_result)); - } - } - return result; } diff --git a/src/duckdb/src/main/extension/extension_install.cpp b/src/duckdb/src/main/extension/extension_install.cpp index d188cb5d..8a278832 100644 --- a/src/duckdb/src/main/extension/extension_install.cpp +++ b/src/duckdb/src/main/extension/extension_install.cpp @@ -136,18 +136,21 @@ bool ExtensionHelper::CreateSuggestions(const string &extension_name, string &me unique_ptr ExtensionHelper::InstallExtension(DBConfig &config, FileSystem &fs, const string &extension, bool force_install, optional_ptr repository, + bool throw_on_origin_mismatch, const string &version) { #ifdef WASM_LOADABLE_EXTENSIONS // Install is currently a no-op return nullptr; #endif string local_path = ExtensionDirectory(config, fs); - return InstallExtensionInternal(config, fs, local_path, extension, force_install, version, repository); + return InstallExtensionInternal(config, fs, local_path, extension, force_install, throw_on_origin_mismatch, version, + repository); } unique_ptr ExtensionHelper::InstallExtension(ClientContext &context, const string &extension, bool force_install, optional_ptr repository, + bool throw_on_origin_mismatch, const string &version) { #ifdef WASM_LOADABLE_EXTENSIONS // Install is currently a no-op @@ -158,8 +161,8 @@ unique_ptr ExtensionHelper::InstallExtension(ClientContext string local_path = ExtensionDirectory(context); optional_ptr http_logger = ClientConfig::GetConfig(context).enable_http_logging ? context.client_data->http_logger.get() : nullptr; - return InstallExtensionInternal(db_config, fs, local_path, extension, force_install, version, repository, - http_logger, context); + return InstallExtensionInternal(db_config, fs, local_path, extension, force_install, throw_on_origin_mismatch, + version, repository, http_logger, context); } unsafe_unique_array ReadExtensionFileFromDisk(FileSystem &fs, const string &path, idx_t &file_size) { @@ -445,8 +448,8 @@ static void ThrowErrorOnMismatchingExtensionOrigin(FileSystem &fs, const string unique_ptr ExtensionHelper::InstallExtensionInternal(DBConfig &config, FileSystem &fs, const string &local_path, - const string &extension, bool force_install, const string &version, - optional_ptr repository, + const string &extension, bool force_install, bool throw_on_origin_mismatch, + const string &version, optional_ptr repository, optional_ptr http_logger, optional_ptr context) { #ifdef DUCKDB_DISABLE_EXTENSION_LOAD throw PermissionException("Installing external extensions is disabled through a compile time flag"); @@ -461,7 +464,8 @@ ExtensionHelper::InstallExtensionInternal(DBConfig &config, FileSystem &fs, cons if (fs.FileExists(local_extension_path) && !force_install) { // File exists: throw error if origin mismatches - if (!config.options.allow_extensions_metadata_mismatch && fs.FileExists(local_extension_path + ".info")) { + if (throw_on_origin_mismatch && !config.options.allow_extensions_metadata_mismatch && + fs.FileExists(local_extension_path + ".info")) { ThrowErrorOnMismatchingExtensionOrigin(fs, local_extension_path, extension_name, extension, repository); } diff --git a/src/duckdb/src/main/settings/settings.cpp b/src/duckdb/src/main/settings/settings.cpp index 6db7a8c9..7217af5a 100644 --- a/src/duckdb/src/main/settings/settings.cpp +++ b/src/duckdb/src/main/settings/settings.cpp @@ -437,6 +437,38 @@ Value EnableExternalAccessSetting::GetSetting(const ClientContext &context) { return Value::BOOLEAN(config.options.enable_external_access); } +//===--------------------------------------------------------------------===// +// Enable Macro Dependencies +//===--------------------------------------------------------------------===// +void EnableMacrosDependencies::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { + config.options.enable_macro_dependencies = input.GetValue(); +} + +void EnableMacrosDependencies::ResetGlobal(DatabaseInstance *db, DBConfig &config) { + config.options.enable_macro_dependencies = DBConfig().options.enable_macro_dependencies; +} + +Value EnableMacrosDependencies::GetSetting(const ClientContext &context) { + auto &config = DBConfig::GetConfig(context); + return Value::BOOLEAN(config.options.enable_macro_dependencies); +} + +//===--------------------------------------------------------------------===// +// Enable View Dependencies +//===--------------------------------------------------------------------===// +void EnableViewDependencies::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) { + config.options.enable_view_dependencies = input.GetValue(); +} + +void EnableViewDependencies::ResetGlobal(DatabaseInstance *db, DBConfig &config) { + config.options.enable_view_dependencies = DBConfig().options.enable_view_dependencies; +} + +Value EnableViewDependencies::GetSetting(const ClientContext &context) { + auto &config = DBConfig::GetConfig(context); + return Value::BOOLEAN(config.options.enable_view_dependencies); +} + //===--------------------------------------------------------------------===// // Enable FSST Vectors //===--------------------------------------------------------------------===// diff --git a/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp b/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp index e65f4b4f..4d9938f1 100644 --- a/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp +++ b/src/duckdb/src/optimizer/pushdown/pushdown_cross_product.cpp @@ -52,9 +52,9 @@ unique_ptr FilterPushdown::PushdownCrossProduct(unique_ptr conditions; vector> arbitrary_expressions; const auto join_type = JoinType::INNER; - LogicalComparisonJoin::ExtractJoinConditions(GetContext(), join_type, op->children[0], op->children[1], - left_bindings, right_bindings, join_expressions, conditions, - arbitrary_expressions); + LogicalComparisonJoin::ExtractJoinConditions(GetContext(), join_type, join_ref_type, op->children[0], + op->children[1], left_bindings, right_bindings, join_expressions, + conditions, arbitrary_expressions); // create the join from the join conditions return LogicalComparisonJoin::CreateJoin(GetContext(), join_type, join_ref_type, std::move(op->children[0]), std::move(op->children[1]), std::move(conditions), diff --git a/src/duckdb/src/optimizer/rule/move_constants.cpp b/src/duckdb/src/optimizer/rule/move_constants.cpp index 636265ff..29404164 100644 --- a/src/duckdb/src/optimizer/rule/move_constants.cpp +++ b/src/duckdb/src/optimizer/rule/move_constants.cpp @@ -41,6 +41,10 @@ unique_ptr MoveConstantsRule::Apply(LogicalOperator &op, vectorreturn_type.IsIntegral()); if (inner_constant.value.IsNull() || outer_constant.value.IsNull()) { + if (comparison.type == ExpressionType::COMPARE_DISTINCT_FROM || + comparison.type == ExpressionType::COMPARE_NOT_DISTINCT_FROM) { + return nullptr; + } return make_uniq(Value(comparison.return_type)); } auto &constant_type = outer_constant.return_type; diff --git a/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp b/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp index 471c6296..af70d7e0 100644 --- a/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +++ b/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp @@ -417,8 +417,8 @@ unique_ptr Binder::PlanLateralJoin(unique_ptr vector> arbitrary_expressions; if (condition) { // extract join conditions, if there are any - LogicalComparisonJoin::ExtractJoinConditions(context, join_type, left, right, std::move(condition), conditions, - arbitrary_expressions); + LogicalComparisonJoin::ExtractJoinConditions(context, join_type, JoinRefType::REGULAR, left, right, + std::move(condition), conditions, arbitrary_expressions); } auto perform_delim = PerformDuplicateElimination(*this, correlated); diff --git a/src/duckdb/src/planner/binder/statement/bind_copy.cpp b/src/duckdb/src/planner/binder/statement/bind_copy.cpp index 1e3928f0..7db1db81 100644 --- a/src/duckdb/src/planner/binder/statement/bind_copy.cpp +++ b/src/duckdb/src/planner/binder/statement/bind_copy.cpp @@ -55,12 +55,13 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) { } bool use_tmp_file = true; - bool overwrite_or_ignore = false; + CopyOverwriteMode overwrite_mode = CopyOverwriteMode::COPY_ERROR_ON_CONFLICT; FilenamePattern filename_pattern; bool user_set_use_tmp_file = false; bool per_thread_output = false; optional_idx file_size_bytes; vector partition_cols; + bool seen_overwrite_mode = false; CopyFunctionBindInput bind_input(*stmt.info); @@ -74,8 +75,20 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) { if (loption == "use_tmp_file") { use_tmp_file = GetBooleanArg(context, option.second); user_set_use_tmp_file = true; - } else if (loption == "overwrite_or_ignore") { - overwrite_or_ignore = GetBooleanArg(context, option.second); + } else if (loption == "overwrite_or_ignore" || loption == "overwrite") { + if (seen_overwrite_mode) { + throw BinderException("Can only set one of OVERWRITE_OR_IGNORE or OVERWRITE"); + } + seen_overwrite_mode = true; + + auto boolean = GetBooleanArg(context, option.second); + if (boolean) { + if (loption == "overwrite_or_ignore") { + overwrite_mode = CopyOverwriteMode::COPY_OVERWRITE_OR_IGNORE; + } else if (loption == "overwrite") { + overwrite_mode = CopyOverwriteMode::COPY_OVERWRITE; + } + } } else if (loption == "filename_pattern") { if (option.second.empty()) { throw IOException("FILENAME_PATTERN cannot be empty"); @@ -146,7 +159,7 @@ BoundStatement Binder::BindCopyTo(CopyStatement &stmt) { auto copy = make_uniq(copy_function.function, std::move(function_data), std::move(stmt.info)); copy->file_path = file_path; copy->use_tmp_file = use_tmp_file; - copy->overwrite_or_ignore = overwrite_or_ignore; + copy->overwrite_mode = overwrite_mode; copy->filename_pattern = filename_pattern; copy->file_extension = bind_input.file_extension; copy->per_thread_output = per_thread_output; diff --git a/src/duckdb/src/planner/binder/statement/bind_create.cpp b/src/duckdb/src/planner/binder/statement/bind_create.cpp index ece53e6d..b8c5c7bc 100644 --- a/src/duckdb/src/planner/binder/statement/bind_create.cpp +++ b/src/duckdb/src/planner/binder/statement/bind_create.cpp @@ -145,13 +145,19 @@ void Binder::BindCreateViewInfo(CreateViewInfo &base) { auto view_binder = Binder::CreateBinder(context); auto &dependencies = base.dependencies; auto &catalog = Catalog::GetCatalog(context, base.catalog); - view_binder->SetCatalogLookupCallback([&dependencies, &catalog](CatalogEntry &entry) { - if (&catalog != &entry.ParentCatalog()) { - // Don't register dependencies between catalogs - return; - } - dependencies.AddDependency(entry); - }); + + auto &db_config = DBConfig::GetConfig(context); + auto should_create_dependencies = db_config.options.enable_view_dependencies; + + if (should_create_dependencies) { + view_binder->SetCatalogLookupCallback([&dependencies, &catalog](CatalogEntry &entry) { + if (&catalog != &entry.ParentCatalog()) { + // Don't register dependencies between catalogs + return; + } + dependencies.AddDependency(entry); + }); + } view_binder->can_contain_nulls = true; auto copy = base.query->Copy(); @@ -204,14 +210,19 @@ SchemaCatalogEntry &Binder::BindCreateFunctionInfo(CreateInfo &info) { SelectBinder binder(*this, context, sel_node, group_info); auto &dependencies = base.dependencies; auto &catalog = Catalog::GetCatalog(context, info.catalog); - binder.SetCatalogLookupCallback([&dependencies, &catalog](CatalogEntry &entry) { - if (&catalog != &entry.ParentCatalog()) { - // Don't register any cross-catalog dependencies - return; - } - // Register any catalog entry required to bind the macro function - dependencies.AddDependency(entry); - }); + auto &db_config = DBConfig::GetConfig(context); + auto should_create_dependencies = db_config.options.enable_macro_dependencies; + + if (should_create_dependencies) { + binder.SetCatalogLookupCallback([&dependencies, &catalog](CatalogEntry &entry) { + if (&catalog != &entry.ParentCatalog()) { + // Don't register any cross-catalog dependencies + return; + } + // Register any catalog entry required to bind the macro function + dependencies.AddDependency(entry); + }); + } error = binder.Bind(expression, 0, false); if (error.HasError()) { error.Throw(); diff --git a/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp b/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp index bf447eaa..df1afca1 100644 --- a/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +++ b/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp @@ -20,6 +20,26 @@ namespace duckdb { +//! Only use conditions that are valid for the join ref type +static bool IsJoinTypeCondition(const JoinRefType ref_type, const ExpressionType expr_type) { + switch (ref_type) { + case JoinRefType::ASOF: + switch (expr_type) { + case ExpressionType::COMPARE_EQUAL: + case ExpressionType::COMPARE_NOT_DISTINCT_FROM: + case ExpressionType::COMPARE_GREATERTHANOREQUALTO: + case ExpressionType::COMPARE_GREATERTHAN: + case ExpressionType::COMPARE_LESSTHANOREQUALTO: + case ExpressionType::COMPARE_LESSTHAN: + return true; + default: + return false; + } + default: + return true; + } +} + //! Create a JoinCondition from a comparison static bool CreateJoinCondition(Expression &expr, const unordered_set &left_bindings, const unordered_set &right_bindings, vector &conditions) { @@ -47,7 +67,7 @@ static bool CreateJoinCondition(Expression &expr, const unordered_set &le } void LogicalComparisonJoin::ExtractJoinConditions( - ClientContext &context, JoinType type, unique_ptr &left_child, + ClientContext &context, JoinType type, JoinRefType ref_type, unique_ptr &left_child, unique_ptr &right_child, const unordered_set &left_bindings, const unordered_set &right_bindings, vector> &expressions, vector &conditions, vector> &arbitrary_expressions) { @@ -90,7 +110,8 @@ void LogicalComparisonJoin::ExtractJoinConditions( { // comparison, check if we can create a comparison JoinCondition - if (CreateJoinCondition(*expr, left_bindings, right_bindings, conditions)) { + if (IsJoinTypeCondition(ref_type, expr->type) && + CreateJoinCondition(*expr, left_bindings, right_bindings, conditions)) { // successfully created the join condition continue; } @@ -99,7 +120,7 @@ void LogicalComparisonJoin::ExtractJoinConditions( } } -void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type, +void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type, JoinRefType ref_type, unique_ptr &left_child, unique_ptr &right_child, vector> &expressions, @@ -108,11 +129,11 @@ void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinTy unordered_set left_bindings, right_bindings; LogicalJoin::GetTableReferences(*left_child, left_bindings); LogicalJoin::GetTableReferences(*right_child, right_bindings); - return ExtractJoinConditions(context, type, left_child, right_child, left_bindings, right_bindings, expressions, - conditions, arbitrary_expressions); + return ExtractJoinConditions(context, type, ref_type, left_child, right_child, left_bindings, right_bindings, + expressions, conditions, arbitrary_expressions); } -void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type, +void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinType type, JoinRefType ref_type, unique_ptr &left_child, unique_ptr &right_child, unique_ptr condition, vector &conditions, @@ -121,7 +142,7 @@ void LogicalComparisonJoin::ExtractJoinConditions(ClientContext &context, JoinTy vector> expressions; expressions.push_back(std::move(condition)); LogicalFilter::SplitPredicates(expressions); - return ExtractJoinConditions(context, type, left_child, right_child, expressions, conditions, + return ExtractJoinConditions(context, type, ref_type, left_child, right_child, expressions, conditions, arbitrary_expressions); } @@ -135,9 +156,6 @@ unique_ptr LogicalComparisonJoin::CreateJoin(ClientContext &con bool need_to_consider_arbitrary_expressions = true; switch (reftype) { case JoinRefType::ASOF: { - if (!arbitrary_expressions.empty()) { - throw BinderException("Invalid ASOF JOIN condition"); - } need_to_consider_arbitrary_expressions = false; auto asof_idx = conditions.size(); for (size_t c = 0; c < conditions.size(); ++c) { @@ -249,7 +267,7 @@ unique_ptr LogicalComparisonJoin::CreateJoin(ClientContext &con unique_ptr condition) { vector conditions; vector> arbitrary_expressions; - LogicalComparisonJoin::ExtractJoinConditions(context, type, left_child, right_child, std::move(condition), + LogicalComparisonJoin::ExtractJoinConditions(context, type, reftype, left_child, right_child, std::move(condition), conditions, arbitrary_expressions); return LogicalComparisonJoin::CreateJoin(context, type, reftype, std::move(left_child), std::move(right_child), std::move(conditions), std::move(arbitrary_expressions)); diff --git a/src/duckdb/src/planner/operator/logical_copy_to_file.cpp b/src/duckdb/src/planner/operator/logical_copy_to_file.cpp index 67f684d8..6ed72e79 100644 --- a/src/duckdb/src/planner/operator/logical_copy_to_file.cpp +++ b/src/duckdb/src/planner/operator/logical_copy_to_file.cpp @@ -13,7 +13,7 @@ void LogicalCopyToFile::Serialize(Serializer &serializer) const { serializer.WriteProperty(200, "file_path", file_path); serializer.WriteProperty(201, "use_tmp_file", use_tmp_file); serializer.WriteProperty(202, "filename_pattern", filename_pattern); - serializer.WriteProperty(203, "overwrite_or_ignore", overwrite_or_ignore); + serializer.WriteProperty(203, "overwrite_or_ignore", overwrite_mode); serializer.WriteProperty(204, "per_thread_output", per_thread_output); serializer.WriteProperty(205, "partition_output", partition_output); serializer.WriteProperty(206, "partition_columns", partition_columns); @@ -39,7 +39,7 @@ unique_ptr LogicalCopyToFile::Deserialize(Deserializer &deseria auto file_path = deserializer.ReadProperty(200, "file_path"); auto use_tmp_file = deserializer.ReadProperty(201, "use_tmp_file"); auto filename_pattern = deserializer.ReadProperty(202, "filename_pattern"); - auto overwrite_or_ignore = deserializer.ReadProperty(203, "overwrite_or_ignore"); + auto overwrite_mode = deserializer.ReadProperty(203, "overwrite_mode"); auto per_thread_output = deserializer.ReadProperty(204, "per_thread_output"); auto partition_output = deserializer.ReadProperty(205, "partition_output"); auto partition_columns = deserializer.ReadProperty>(206, "partition_columns"); @@ -86,7 +86,7 @@ unique_ptr LogicalCopyToFile::Deserialize(Deserializer &deseria result->use_tmp_file = use_tmp_file; result->filename_pattern = filename_pattern; result->file_extension = file_extension; - result->overwrite_or_ignore = overwrite_or_ignore; + result->overwrite_mode = overwrite_mode; result->per_thread_output = per_thread_output; result->partition_output = partition_output; result->partition_columns = partition_columns; diff --git a/src/duckdb/src/storage/storage_manager.cpp b/src/duckdb/src/storage/storage_manager.cpp index 903cc9fb..b2f6cd44 100644 --- a/src/duckdb/src/storage/storage_manager.cpp +++ b/src/duckdb/src/storage/storage_manager.cpp @@ -1,18 +1,20 @@ #include "duckdb/storage/storage_manager.hpp" -#include "duckdb/storage/checkpoint_manager.hpp" -#include "duckdb/storage/in_memory_block_manager.hpp" -#include "duckdb/storage/single_file_block_manager.hpp" -#include "duckdb/storage/object_cache.hpp" #include "duckdb/catalog/catalog.hpp" #include "duckdb/common/file_system.hpp" -#include "duckdb/main/database.hpp" -#include "duckdb/main/client_context.hpp" -#include "duckdb/function/function.hpp" -#include "duckdb/transaction/transaction_manager.hpp" #include "duckdb/common/serializer/buffered_file_reader.hpp" +#include "duckdb/function/function.hpp" #include "duckdb/main/attached_database.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/main/database.hpp" #include "duckdb/main/database_manager.hpp" +#include "duckdb/storage/checkpoint_manager.hpp" +#include "duckdb/storage/in_memory_block_manager.hpp" +#include "duckdb/storage/object_cache.hpp" +#include "duckdb/storage/single_file_block_manager.hpp" +#include "duckdb/transaction/transaction_manager.hpp" + +#include "duckdb/storage/storage_extension.hpp" namespace duckdb { @@ -283,6 +285,9 @@ void SingleFileStorageManager::CreateCheckpoint(CheckpointOptions options) { if (InMemory() || read_only || !load_complete) { return; } + if (db.GetStorageExtension()) { + db.GetStorageExtension()->OnCheckpointStart(db, options); + } auto &config = DBConfig::Get(db); if (GetWALSize() > 0 || config.options.force_checkpoint || options.action == CheckpointAction::FORCE_CHECKPOINT) { // we only need to checkpoint if there is anything in the WAL @@ -297,6 +302,10 @@ void SingleFileStorageManager::CreateCheckpoint(CheckpointOptions options) { if (options.wal_action == CheckpointWALAction::DELETE_WAL) { ResetWAL(); } + + if (db.GetStorageExtension()) { + db.GetStorageExtension()->OnCheckpointEnd(db, options); + } } DatabaseSize SingleFileStorageManager::GetDatabaseSize() {