Merge branch 'main' into Expose_DB_memory_address

.github/actions/manylinux_2014_setup/action.yml

@@ -108,4 +108,4 @@ runs:
       uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases
       with:
         key: ${{ github.job }}
-        save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb-java' }}
+        save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb-java' }}

.github/workflows/Java.yml

@@ -26,7 +26,7 @@ jobs:
     name: Java Linux (amd64)
     runs-on: ubuntu-latest
     container:
-      image: quay.io/pypa/manylinux2014_x86_64
+      image: quay.io/pypa/manylinux_2_28_x86_64
     env:
       GEN: ninja
       ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
@@ -71,7 +71,7 @@ jobs:
     name: Java Linux (aarch64)
     runs-on: ubuntu-latest
     container:
-      image: ubuntu:18.04
+      image: ubuntu:20.04
     needs: java-linux-amd64
     env:
       GEN: ninja
@@ -239,8 +239,8 @@ jobs:
       - name: Combine JARs
         shell: bash
         run: |
-          mkdir -p ~/.gnupg
-          echo "pinentry-mode loopback" > ~/.gnupg/gpg.conf
+          mkdir -m  700 -p ~/.gnupg
+          echo -e "passphrase asdf\npinentry-mode loopback\nno-tty\nbatch\n" > ~/.gnupg/gpg.conf
           if [[ "$GITHUB_REF" =~ ^(refs/heads/main|refs/tags/v.+)$ && "$GITHUB_REPOSITORY" = "duckdb/duckdb-java" ]] ; then
           export XML='
             <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
@@ -256,7 +256,7 @@ jobs:
               </servers>
             </settings> '
             mkdir ~/.m2
-            echo $XML | sed "s/PASSWORD/${{ secrets.MAVEN_PASSWORD }}/" > ~/.m2/settings.xml
+            echo $XML | sed "s|PASSWORD|${{ secrets.MAVEN_PASSWORD }}|" > ~/.m2/settings.xml
             echo "${{ secrets.MAVEN_PGP_PK }}" | base64 -d > maven_pgp_key
             gpg --batch --import maven_pgp_key
             python scripts/jdbc_maven_deploy.py ${{ github.ref_name }} jdbc-artifacts .
@@ -276,7 +276,7 @@ jobs:
     runs-on: ubuntu-20.04
     if: ${{ inputs.skip_tests != 'true' }}
     needs: java-linux-amd64
-    container: quay.io/pypa/manylinux2014_x86_64
+    container: quay.io/pypa/manylinux_2_28_x86_64
     env:
       BUILD_JDBC: 1
       GEN: ninja

scripts/jdbc_maven_deploy.py

@@ -115,8 +115,8 @@ def exec(cmd):
         <version>1.6.14</version>
         <extensions>true</extensions>
         <configuration>
-         <serverId>ossrh</serverId>
-         <nexusUrl>https://oss.sonatype.org/</nexusUrl>
+          <serverId>ossrh</serverId>
+          <nexusUrl>https://oss.sonatype.org/</nexusUrl>
        </configuration>
      </plugin>
    </plugins>
@@ -179,7 +179,7 @@ def exec(cmd):
     shutil.copyfile(jar, os.path.join(results_dir, os.path.basename(jar)))
 
 print("JARs created, uploading (this can take a while!)")
-deploy_cmd_prefix = 'mvn gpg:sign-and-deploy-file -Durl=%s -DrepositoryId=ossrh' % deploy_url
+deploy_cmd_prefix = 'mvn --no-transfer-progress gpg:sign-and-deploy-file -Durl=%s -DrepositoryId=ossrh' % deploy_url
 exec("%s -DpomFile=%s -Dfile=%s" % (deploy_cmd_prefix, pom, binary_jar))
 exec("%s -Dclassifier=sources -DpomFile=%s -Dfile=%s" % (deploy_cmd_prefix, pom, sources_jar))
 exec("%s -Dclassifier=javadoc -DpomFile=%s -Dfile=%s" % (deploy_cmd_prefix, pom, javadoc_jar))

src/duckdb/src/core_functions/aggregate/algebraic/avg.cpp → src/duckdb/extension/core_functions/aggregate/algebraic/avg.cpp

@@ -1,5 +1,5 @@
-#include "duckdb/core_functions/aggregate/algebraic_functions.hpp"
-#include "duckdb/core_functions/aggregate/sum_helpers.hpp"
+#include "core_functions/aggregate/algebraic_functions.hpp"
+#include "core_functions/aggregate/sum_helpers.hpp"
 #include "duckdb/common/types/hugeint.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/function/function_set.hpp"

src/duckdb/src/core_functions/aggregate/algebraic/corr.cpp → src/duckdb/extension/core_functions/aggregate/algebraic/corr.cpp

@@ -1,7 +1,7 @@
-#include "duckdb/core_functions/aggregate/algebraic_functions.hpp"
-#include "duckdb/core_functions/aggregate/algebraic/covar.hpp"
-#include "duckdb/core_functions/aggregate/algebraic/stddev.hpp"
-#include "duckdb/core_functions/aggregate/algebraic/corr.hpp"
+#include "core_functions/aggregate/algebraic_functions.hpp"
+#include "core_functions/aggregate/algebraic/covar.hpp"
+#include "core_functions/aggregate/algebraic/stddev.hpp"
+#include "core_functions/aggregate/algebraic/corr.hpp"
 #include "duckdb/function/function_set.hpp"
 
 namespace duckdb {

src/duckdb/src/core_functions/aggregate/algebraic/covar.cpp → src/duckdb/extension/core_functions/aggregate/algebraic/covar.cpp

@@ -1,6 +1,6 @@
-#include "duckdb/core_functions/aggregate/algebraic_functions.hpp"
+#include "core_functions/aggregate/algebraic_functions.hpp"
 #include "duckdb/common/types/null_value.hpp"
-#include "duckdb/core_functions/aggregate/algebraic/covar.hpp"
+#include "core_functions/aggregate/algebraic/covar.hpp"
 
 namespace duckdb {
 

src/duckdb/src/core_functions/aggregate/algebraic/stddev.cpp → src/duckdb/extension/core_functions/aggregate/algebraic/stddev.cpp

@@ -1,7 +1,7 @@
-#include "duckdb/core_functions/aggregate/algebraic_functions.hpp"
+#include "core_functions/aggregate/algebraic_functions.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/function/function_set.hpp"
-#include "duckdb/core_functions/aggregate/algebraic/stddev.hpp"
+#include "core_functions/aggregate/algebraic/stddev.hpp"
 #include <cmath>
 
 namespace duckdb {

src/duckdb/src/core_functions/aggregate/distributive/approx_count.cpp → src/duckdb/extension/core_functions/aggregate/distributive/approx_count.cpp

@@ -1,7 +1,7 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/types/hash.hpp"
 #include "duckdb/common/types/hyperloglog.hpp"
-#include "duckdb/core_functions/aggregate/distributive_functions.hpp"
+#include "core_functions/aggregate/distributive_functions.hpp"
 #include "duckdb/function/function_set.hpp"
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
 #include "hyperloglog.hpp"

src/duckdb/src/core_functions/aggregate/distributive/arg_min_max.cpp → src/duckdb/extension/core_functions/aggregate/distributive/arg_min_max.cpp

@@ -1,14 +1,14 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/operator/comparison_operators.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
-#include "duckdb/core_functions/aggregate/distributive_functions.hpp"
+#include "core_functions/aggregate/distributive_functions.hpp"
 #include "duckdb/function/cast/cast_function_set.hpp"
 #include "duckdb/function/function_set.hpp"
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
 #include "duckdb/planner/expression/bound_comparison_expression.hpp"
 #include "duckdb/planner/expression_binder.hpp"
-#include "duckdb/core_functions/create_sort_key.hpp"
-#include "duckdb/core_functions/aggregate/minmax_n_helpers.hpp"
+#include "duckdb/function/create_sort_key.hpp"
+#include "duckdb/function/aggregate/minmax_n_helpers.hpp"
 
 namespace duckdb {
 
@@ -192,7 +192,7 @@ struct GenericArgMinMaxState {
 
 	static void PrepareData(Vector &by, idx_t count, Vector &extra_state, UnifiedVectorFormat &result) {
 		OrderModifiers modifiers(ORDER_TYPE, OrderByNullType::NULLS_LAST);
-		CreateSortKeyHelpers::CreateSortKey(by, count, modifiers, extra_state);
+		CreateSortKeyHelpers::CreateSortKeyWithValidity(by, extra_state, modifiers, count);
 		extra_state.ToUnifiedFormat(count, result);
 	}
 };
@@ -285,7 +285,6 @@ struct VectorArgMinMaxBase : ArgMinMaxBase<COMPARATOR, IGNORE_NULL> {
 			target.arg_null = source.arg_null;
 			if (!target.arg_null) {
 				STATE::template AssignValue<typename STATE::ARG_TYPE>(target.arg, source.arg);
-				;
 			}
 			target.is_initialized = true;
 		}
@@ -303,21 +302,43 @@ struct VectorArgMinMaxBase : ArgMinMaxBase<COMPARATOR, IGNORE_NULL> {
 
 	static unique_ptr<FunctionData> Bind(ClientContext &context, AggregateFunction &function,
 	                                     vector<unique_ptr<Expression>> &arguments) {
+		if (arguments[1]->return_type.InternalType() == PhysicalType::VARCHAR) {
+			ExpressionBinder::PushCollation(context, arguments[1], arguments[1]->return_type);
+		}
 		function.arguments[0] = arguments[0]->return_type;
 		function.return_type = arguments[0]->return_type;
 		return nullptr;
 	}
 };
 
+template <class OP>
+AggregateFunction GetGenericArgMinMaxFunction() {
+	using STATE = ArgMinMaxState<string_t, string_t>;
+	return AggregateFunction(
+	    {LogicalType::ANY, LogicalType::ANY}, LogicalType::ANY, AggregateFunction::StateSize<STATE>,
+	    AggregateFunction::StateInitialize<STATE, OP, AggregateDestructorType::LEGACY>, OP::template Update<STATE>,
+	    AggregateFunction::StateCombine<STATE, OP>, AggregateFunction::StateVoidFinalize<STATE, OP>, nullptr, OP::Bind,
+	    AggregateFunction::StateDestroy<STATE, OP>);
+}
+
 template <class OP, class ARG_TYPE, class BY_TYPE>
 AggregateFunction GetVectorArgMinMaxFunctionInternal(const LogicalType &by_type, const LogicalType &type) {
+#ifndef DUCKDB_SMALLER_BINARY
 	using STATE = ArgMinMaxState<ARG_TYPE, BY_TYPE>;
-	return AggregateFunction(
-	    {type, by_type}, type, AggregateFunction::StateSize<STATE>, AggregateFunction::StateInitialize<STATE, OP>,
-	    OP::template Update<STATE>, AggregateFunction::StateCombine<STATE, OP>,
-	    AggregateFunction::StateVoidFinalize<STATE, OP>, nullptr, OP::Bind, AggregateFunction::StateDestroy<STATE, OP>);
+	return AggregateFunction({type, by_type}, type, AggregateFunction::StateSize<STATE>,
+	                         AggregateFunction::StateInitialize<STATE, OP, AggregateDestructorType::LEGACY>,
+	                         OP::template Update<STATE>, AggregateFunction::StateCombine<STATE, OP>,
+	                         AggregateFunction::StateVoidFinalize<STATE, OP>, nullptr, OP::Bind,
+	                         AggregateFunction::StateDestroy<STATE, OP>);
+#else
+	auto function = GetGenericArgMinMaxFunction<OP>();
+	function.arguments = {type, by_type};
+	function.return_type = type;
+	return function;
+#endif
 }
 
+#ifndef DUCKDB_SMALLER_BINARY
 template <class OP, class ARG_TYPE>
 AggregateFunction GetVectorArgMinMaxFunctionBy(const LogicalType &by_type, const LogicalType &type) {
 	switch (by_type.InternalType()) {
@@ -335,6 +356,7 @@ AggregateFunction GetVectorArgMinMaxFunctionBy(const LogicalType &by_type, const
 		throw InternalException("Unimplemented arg_min/arg_max aggregate");
 	}
 }
+#endif
 
 static const vector<LogicalType> ArgMaxByTypes() {
 	vector<LogicalType> types = {LogicalType::INTEGER,   LogicalType::BIGINT,       LogicalType::HUGEINT,
@@ -347,21 +369,34 @@ template <class OP, class ARG_TYPE>
 void AddVectorArgMinMaxFunctionBy(AggregateFunctionSet &fun, const LogicalType &type) {
 	auto by_types = ArgMaxByTypes();
 	for (const auto &by_type : by_types) {
+#ifndef DUCKDB_SMALLER_BINARY
 		fun.AddFunction(GetVectorArgMinMaxFunctionBy<OP, ARG_TYPE>(by_type, type));
+#else
+		fun.AddFunction(GetVectorArgMinMaxFunctionInternal<OP, string_t, string_t>(by_type, type));
+#endif
 	}
 }
 
 template <class OP, class ARG_TYPE, class BY_TYPE>
 AggregateFunction GetArgMinMaxFunctionInternal(const LogicalType &by_type, const LogicalType &type) {
+#ifndef DUCKDB_SMALLER_BINARY
 	using STATE = ArgMinMaxState<ARG_TYPE, BY_TYPE>;
-	auto function = AggregateFunction::BinaryAggregate<STATE, ARG_TYPE, BY_TYPE, ARG_TYPE, OP>(type, by_type, type);
+	auto function =
+	    AggregateFunction::BinaryAggregate<STATE, ARG_TYPE, BY_TYPE, ARG_TYPE, OP, AggregateDestructorType::LEGACY>(
+	        type, by_type, type);
 	if (type.InternalType() == PhysicalType::VARCHAR || by_type.InternalType() == PhysicalType::VARCHAR) {
 		function.destructor = AggregateFunction::StateDestroy<STATE, OP>;
 	}
 	function.bind = OP::Bind;
+#else
+	auto function = GetGenericArgMinMaxFunction<OP>();
+	function.arguments = {type, by_type};
+	function.return_type = type;
+#endif
 	return function;
 }
 
+#ifndef DUCKDB_SMALLER_BINARY
 template <class OP, class ARG_TYPE>
 AggregateFunction GetArgMinMaxFunctionBy(const LogicalType &by_type, const LogicalType &type) {
 	switch (by_type.InternalType()) {
@@ -379,18 +414,24 @@ AggregateFunction GetArgMinMaxFunctionBy(const LogicalType &by_type, const Logic
 		throw InternalException("Unimplemented arg_min/arg_max by aggregate");
 	}
 }
+#endif
 
 template <class OP, class ARG_TYPE>
 void AddArgMinMaxFunctionBy(AggregateFunctionSet &fun, const LogicalType &type) {
 	auto by_types = ArgMaxByTypes();
 	for (const auto &by_type : by_types) {
+#ifndef DUCKDB_SMALLER_BINARY
 		fun.AddFunction(GetArgMinMaxFunctionBy<OP, ARG_TYPE>(by_type, type));
+#else
+		fun.AddFunction(GetArgMinMaxFunctionInternal<OP, string_t, string_t>(by_type, type));
+#endif
 	}
 }
 
 template <class OP>
 static AggregateFunction GetDecimalArgMinMaxFunction(const LogicalType &by_type, const LogicalType &type) {
 	D_ASSERT(type.id() == LogicalTypeId::DECIMAL);
+#ifndef DUCKDB_SMALLER_BINARY
 	switch (type.InternalType()) {
 	case PhysicalType::INT16:
 		return GetArgMinMaxFunctionBy<OP, int16_t>(by_type, type);
@@ -401,6 +442,9 @@ static AggregateFunction GetDecimalArgMinMaxFunction(const LogicalType &by_type,
 	default:
 		return GetArgMinMaxFunctionBy<OP, hugeint_t>(by_type, type);
 	}
+#else
+	return GetArgMinMaxFunctionInternal<OP, string_t, string_t>(by_type, type);
+#endif
 }
 
 template <class OP>
@@ -449,17 +493,19 @@ void AddDecimalArgMinMaxFunctionBy(AggregateFunctionSet &fun, const LogicalType
 
 template <class OP>
 void AddGenericArgMinMaxFunction(AggregateFunctionSet &fun) {
-	using STATE = ArgMinMaxState<string_t, string_t>;
-	fun.AddFunction(
-	    AggregateFunction({LogicalType::ANY, LogicalType::ANY}, LogicalType::ANY, AggregateFunction::StateSize<STATE>,
-	                      AggregateFunction::StateInitialize<STATE, OP>, OP::template Update<STATE>,
-	                      AggregateFunction::StateCombine<STATE, OP>, AggregateFunction::StateVoidFinalize<STATE, OP>,
-	                      nullptr, OP::Bind, AggregateFunction::StateDestroy<STATE, OP>));
+	fun.AddFunction(GetGenericArgMinMaxFunction<OP>());
 }
 
 template <class COMPARATOR, bool IGNORE_NULL, OrderType ORDER_TYPE>
 static void AddArgMinMaxFunctions(AggregateFunctionSet &fun) {
+	using GENERIC_VECTOR_OP = VectorArgMinMaxBase<LessThan, IGNORE_NULL, ORDER_TYPE, GenericArgMinMaxState<ORDER_TYPE>>;
+#ifndef DUCKDB_SMALLER_BINARY
 	using OP = ArgMinMaxBase<COMPARATOR, IGNORE_NULL>;
+	using VECTOR_OP = VectorArgMinMaxBase<COMPARATOR, IGNORE_NULL, ORDER_TYPE>;
+#else
+	using OP = GENERIC_VECTOR_OP;
+	using VECTOR_OP = GENERIC_VECTOR_OP;
+#endif
 	AddArgMinMaxFunctionBy<OP, int32_t>(fun, LogicalType::INTEGER);
 	AddArgMinMaxFunctionBy<OP, int64_t>(fun, LogicalType::BIGINT);
 	AddArgMinMaxFunctionBy<OP, double>(fun, LogicalType::DOUBLE);
@@ -474,11 +520,9 @@ static void AddArgMinMaxFunctions(AggregateFunctionSet &fun) {
 		AddDecimalArgMinMaxFunctionBy<OP>(fun, by_type);
 	}
 
-	using VECTOR_OP = VectorArgMinMaxBase<COMPARATOR, IGNORE_NULL, ORDER_TYPE>;
 	AddVectorArgMinMaxFunctionBy<VECTOR_OP, string_t>(fun, LogicalType::ANY);
 
 	// we always use LessThan when using sort keys because the ORDER_TYPE takes care of selecting the lowest or highest
-	using GENERIC_VECTOR_OP = VectorArgMinMaxBase<LessThan, IGNORE_NULL, ORDER_TYPE, GenericArgMinMaxState<ORDER_TYPE>>;
 	AddGenericArgMinMaxFunction<GENERIC_VECTOR_OP>(fun);
 }
 
@@ -577,7 +621,7 @@ static void SpecializeArgMinMaxNFunction(AggregateFunction &function) {
 	using OP = MinMaxNOperation;
 
 	function.state_size = AggregateFunction::StateSize<STATE>;
-	function.initialize = AggregateFunction::StateInitialize<STATE, OP>;
+	function.initialize = AggregateFunction::StateInitialize<STATE, OP, AggregateDestructorType::LEGACY>;
 	function.combine = AggregateFunction::StateCombine<STATE, OP>;
 	function.destructor = AggregateFunction::StateDestroy<STATE, OP>;
 
@@ -588,6 +632,7 @@ static void SpecializeArgMinMaxNFunction(AggregateFunction &function) {
 template <class VAL_TYPE, class COMPARATOR>
 static void SpecializeArgMinMaxNFunction(PhysicalType arg_type, AggregateFunction &function) {
 	switch (arg_type) {
+#ifndef DUCKDB_SMALLER_BINARY
 	case PhysicalType::VARCHAR:
 		SpecializeArgMinMaxNFunction<VAL_TYPE, MinMaxStringValue, COMPARATOR>(function);
 		break;
@@ -603,6 +648,7 @@ static void SpecializeArgMinMaxNFunction(PhysicalType arg_type, AggregateFunctio
 	case PhysicalType::DOUBLE:
 		SpecializeArgMinMaxNFunction<VAL_TYPE, MinMaxFixedValue<double>, COMPARATOR>(function);
 		break;
+#endif
 	default:
 		SpecializeArgMinMaxNFunction<VAL_TYPE, MinMaxFallbackValue, COMPARATOR>(function);
 		break;
@@ -612,6 +658,7 @@ static void SpecializeArgMinMaxNFunction(PhysicalType arg_type, AggregateFunctio
 template <class COMPARATOR>
 static void SpecializeArgMinMaxNFunction(PhysicalType val_type, PhysicalType arg_type, AggregateFunction &function) {
 	switch (val_type) {
+#ifndef DUCKDB_SMALLER_BINARY
 	case PhysicalType::VARCHAR:
 		SpecializeArgMinMaxNFunction<MinMaxStringValue, COMPARATOR>(arg_type, function);
 		break;
@@ -627,6 +674,7 @@ static void SpecializeArgMinMaxNFunction(PhysicalType val_type, PhysicalType arg
 	case PhysicalType::DOUBLE:
 		SpecializeArgMinMaxNFunction<MinMaxFixedValue<double>, COMPARATOR>(arg_type, function);
 		break;
+#endif
 	default:
 		SpecializeArgMinMaxNFunction<MinMaxFallbackValue, COMPARATOR>(arg_type, function);
 		break;

src/duckdb/src/core_functions/aggregate/distributive/bitagg.cpp → src/duckdb/extension/core_functions/aggregate/distributive/bitagg.cpp

@@ -1,4 +1,4 @@
-#include "duckdb/core_functions/aggregate/distributive_functions.hpp"
+#include "core_functions/aggregate/distributive_functions.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"