From 434800837a520a380443f1baede1902ebb2f1041 Mon Sep 17 00:00:00 2001 From: Nick Lanham Date: Thu, 7 Nov 2024 12:31:44 -0800 Subject: [PATCH 1/2] initial thing, works but sets on every call --- src/uc_api.cpp | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/uc_api.cpp b/src/uc_api.cpp index 0ddcc85..23359df 100644 --- a/src/uc_api.cpp +++ b/src/uc_api.cpp @@ -2,6 +2,7 @@ #include "storage/uc_catalog.hpp" #include "yyjson.hpp" #include +#include namespace duckdb { @@ -10,6 +11,36 @@ static size_t GetRequestWriteCallback(void *contents, size_t size, size_t nmemb, return size * nmemb; } +// we statically compile in libcurl, which means the cert file location of the build machine is the +// place curl will look. But not every distro has this file in the same location, so we search a +// number of common locations and use the first one we find. +static string certFileLocations[] = { + // Arch, Debian-based, Gentoo + "/etc/ssl/certs/ca-certificates.crt", + // RedHat 7 based + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", + // Redhat 6 based + "/etc/pki/tls/certs/ca-bundle.crt", + // OpenSUSE + "/etc/ssl/ca-bundle.pem", + // Alpine + "/etc/ssl/cert.pem" +}; + +// Look through the the above locations and if one of the files exists, set that as the location +// curl should use. Returns true if a file was set, false if no matching file was found +static bool SetCurlCAFileInfo(CURL* curl) { + for (string& caFile : certFileLocations) { + struct stat buf; + if (stat(caFile.c_str(), &buf) == 0) { + // file exists + curl_easy_setopt(curl, CURLOPT_CAINFO, caFile.c_str()); + return true; + } + } + return false; +} + static string GetRequest(const string &url, const string &token = "") { CURL *curl; CURLcode res; @@ -24,6 +55,7 @@ static string GetRequest(const string &url, const string &token = "") { curl_easy_setopt(curl, CURLOPT_XOAUTH2_BEARER, token.c_str()); curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BEARER); } + SetCurlCAFileInfo(curl); // todo: log something if this returns false res = curl_easy_perform(curl); curl_easy_cleanup(curl); From 1e7437fba7978f5b817bf5aedf3297a651db4a02 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Tue, 12 Nov 2024 11:15:57 +0100 Subject: [PATCH 2/2] select cert file once on initialization --- src/include/uc_api.hpp | 3 +++ src/uc_api.cpp | 36 ++++++++++++++++++++++++------------ src/uc_catalog_extension.cpp | 5 ++++- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/include/uc_api.hpp b/src/include/uc_api.hpp index 4be9dcd..e76f9ec 100644 --- a/src/include/uc_api.hpp +++ b/src/include/uc_api.hpp @@ -50,6 +50,9 @@ struct UCAPITableCredentials { class UCAPI { public: + //! WARNING: not thread-safe. To be called once on extension initialization + static void InitializeCurl(); + static UCAPITableCredentials GetTableCredentials(const string &table_id, UCCredentials credentials); static vector GetCatalogs(const string &catalog, UCCredentials credentials); static vector GetTables(const string &catalog, const string &schema, UCCredentials credentials); diff --git a/src/uc_api.cpp b/src/uc_api.cpp index 23359df..bc3beb7 100644 --- a/src/uc_api.cpp +++ b/src/uc_api.cpp @@ -6,6 +6,9 @@ namespace duckdb { +//! We use a global here to store the path that is selected on the UCAPI::InitializeCurl call +static string SELECTED_CURL_CERT_PATH = ""; + static size_t GetRequestWriteCallback(void *contents, size_t size, size_t nmemb, void *userp) { ((std::string *)userp)->append((char *)contents, size * nmemb); return size * nmemb; @@ -27,18 +30,23 @@ static string certFileLocations[] = { "/etc/ssl/cert.pem" }; -// Look through the the above locations and if one of the files exists, set that as the location -// curl should use. Returns true if a file was set, false if no matching file was found +// Look through the the above locations and if one of the files exists, set that as the location curl should use. +static bool SelectCurlCertPath() { + for (string& caFile : certFileLocations) { + struct stat buf; + if (stat(caFile.c_str(), &buf) == 0) { + SELECTED_CURL_CERT_PATH = caFile; + } + } + return false; +} + static bool SetCurlCAFileInfo(CURL* curl) { - for (string& caFile : certFileLocations) { - struct stat buf; - if (stat(caFile.c_str(), &buf) == 0) { - // file exists - curl_easy_setopt(curl, CURLOPT_CAINFO, caFile.c_str()); - return true; - } - } - return false; + if (!SELECTED_CURL_CERT_PATH.empty()) { + curl_easy_setopt(curl, CURLOPT_CAINFO, SELECTED_CURL_CERT_PATH.c_str()); + return true; + } + return false; } static string GetRequest(const string &url, const string &token = "") { @@ -55,7 +63,7 @@ static string GetRequest(const string &url, const string &token = "") { curl_easy_setopt(curl, CURLOPT_XOAUTH2_BEARER, token.c_str()); curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BEARER); } - SetCurlCAFileInfo(curl); // todo: log something if this returns false + SetCurlCAFileInfo(curl); // todo: log something if this returns false res = curl_easy_perform(curl); curl_easy_cleanup(curl); @@ -135,6 +143,10 @@ static string GetCredentialsRequest(const string &url, const string &table_id, c throw InternalException("Failed to initialize curl"); } +void UCAPI::InitializeCurl() { + SelectCurlCertPath(); +} + //# list catalogs // echo "List of catalogs" // curl --request GET "https://${DATABRICKS_HOST}/api/2.1/unity-catalog/catalogs" \ diff --git a/src/uc_catalog_extension.cpp b/src/uc_catalog_extension.cpp index 6fcfe8e..fd2e060 100644 --- a/src/uc_catalog_extension.cpp +++ b/src/uc_catalog_extension.cpp @@ -10,9 +10,10 @@ #include "duckdb/common/string_util.hpp" #include "duckdb/function/scalar_function.hpp" #include "duckdb/main/extension_util.hpp" -#include +#include "duckdb/parser/parsed_data/create_scalar_function_info.hpp" #include "duckdb/parser/parsed_data/attach_info.hpp" #include "duckdb/storage/storage_extension.hpp" +#include "uc_api.hpp" namespace duckdb { @@ -129,6 +130,8 @@ class UCCatalogStorageExtension : public StorageExtension { }; static void LoadInternal(DatabaseInstance &instance) { + UCAPI::InitializeCurl(); + SecretType secret_type; secret_type.name = "uc"; secret_type.deserializer = KeyValueSecret::Deserialize;