Merge eabb683 into 84442b6

ydb/core/external_sources/s3/ut/docker-compose.yml

@@ -0,0 +1,31 @@
+version: '3.4'
+services:
+  # MinIO object storage
+  minio:
+    hostname: minio
+    image: 'minio/minio@sha256:1a3debf2408bde1f33b49cd70af245eb2173c5897a2e6bf99d7934005cd14537'
+    container_name: minio
+    ports:
+      - '9000'
+      - '9001'
+    environment:
+      MINIO_ACCESS_KEY: minio
+      MINIO_SECRET_KEY: minio123
+    command: server /data --console-address ":9001"
+
+  # This job creates the "datalake" bucket on Minio
+  mc-job:
+    image: 'minio/mc@sha256:03e4ea06fe42f94c078613554c34eeb2c7045e79a4b0d875a3c977bf27a8befb'
+    container_name: mc-job
+    volumes:
+      - ./test.json:/test.json
+    entrypoint: |
+      /bin/bash -c "
+      sleep 5;
+      /usr/bin/mc config --quiet host add myminio http://minio:9000 minio minio123;
+      /usr/bin/mc mb --quiet myminio/datalake
+      /usr/bin/mc put /test.json myminio/datalake/a/test.json
+      /usr/bin/mc put /test.json myminio/datalake/b/year=2023/month=01/day=03/test.json
+      "
+    depends_on:
+      - minio

ydb/core/external_sources/s3/ut/s3_aws_credentials_ut.cpp

@@ -0,0 +1,129 @@
+#include <ydb/core/kqp/ut/common/kqp_ut_common.h>
+#include <ydb/core/kqp/ut/federated_query/common/common.h>
+#include <ydb/core/kqp/federated_query/kqp_federated_query_helpers.h>
+#include <ydb/library/yql/utils/log/log.h>
+#include <ydb/public/sdk/cpp/client/draft/ydb_scripting.h>
+#include <ydb/public/sdk/cpp/client/ydb_operation/operation.h>
+#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
+#include <ydb/public/sdk/cpp/client/ydb_types/operation/operation.h>
+#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/generic/strbuf.h>
+#include <util/generic/string.h>
+#include <util/system/env.h>
+
+#include <fmt/format.h>
+
+namespace NKikimr::NKqp {
+
+using namespace NYdb;
+using namespace NYdb::NQuery;
+using namespace NKikimr::NKqp::NFederatedQueryTest;
+using namespace fmt::literals;
+
+TString Exec(const TString& cmd) {
+    std::array<char, 128> buffer;
+    TString result;
+    std::unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd.c_str(), "r"), pclose);
+    if (!pipe) {
+        throw std::runtime_error("popen() failed!");
+    }
+    while (fgets(buffer.data(), static_cast<int>(buffer.size()), pipe.get()) != nullptr) {
+        result += buffer.data();
+    }
+    return result;
+}
+
+TString GetExternalPort(const TString& service, const TString& port) {
+    auto dockerComposeBin = BinaryPath("library/recipes/docker_compose/bin/docker-compose");
+    auto composeFileYml = ArcadiaSourceRoot() + "/ydb/core/external_sources/s3/ut/docker-compose.yml";
+    auto result = StringSplitter(Exec(dockerComposeBin + " -f " + composeFileYml + " port " + service + " " + port)).Split(':').ToList<TString>();
+    return result ? Strip(result.back()) : TString{};
+}
+
+Y_UNIT_TEST_SUITE(S3AwsCredentials) {
+    Y_UNIT_TEST(ExecuteScriptWithEqSymbol) {
+        const TString externalDataSourceName = "/Root/external_data_source";
+        auto kikimr = MakeKikimrRunner(true);
+        auto tc = kikimr->GetTableClient();
+        auto session = tc.CreateSession().GetValueSync().GetSession();
+        const TString query = fmt::format(R"(
+            CREATE OBJECT id (TYPE SECRET) WITH (value=`minio`);
+            CREATE OBJECT key (TYPE SECRET) WITH (value=`minio123`);
+            CREATE EXTERNAL DATA SOURCE `{external_source}` WITH (
+                SOURCE_TYPE="ObjectStorage",
+                LOCATION="{location}",
+                AUTH_METHOD="AWS",
+                AWS_ACCESS_KEY_ID_SECRET_NAME="id",
+                AWS_SECRET_ACCESS_KEY_SECRET_NAME="key",
+                AWS_REGION="ru-central-1"
+            );)",
+            "external_source"_a = externalDataSourceName,
+            "location"_a = "localhost:" + GetExternalPort("minio", "9000") + "/datalake/"
+            );
+        auto result = session.ExecuteSchemeQuery(query).GetValueSync();
+        UNIT_ASSERT_C(result.GetStatus() == NYdb::EStatus::SUCCESS, result.GetIssues().ToString());
+        auto db = kikimr->GetQueryClient();
+        {
+            auto scriptExecutionOperation = db.ExecuteScript(fmt::format(R"(
+                SELECT * FROM `{external_source}`.`/a/` WITH (
+                    format="json_each_row",
+                    schema(
+                        key Utf8 NOT NULL,
+                        value Utf8 NOT NULL
+                    )
+                )
+            )", "external_source"_a = externalDataSourceName)).ExtractValueSync();
+            UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString());
+            UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId);
+
+            NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver());
+            UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString());
+            TFetchScriptResultsResult results = db.FetchScriptResults(scriptExecutionOperation.Id(), 0).ExtractValueSync();
+            UNIT_ASSERT_C(results.IsSuccess(), results.GetIssues().ToString());
+
+            TResultSetParser resultSet(results.ExtractResultSet());
+            UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnsCount(), 2);
+            UNIT_ASSERT_VALUES_EQUAL(resultSet.RowsCount(), 2);
+            UNIT_ASSERT(resultSet.TryNextRow());
+            UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(0).GetUtf8(), "1");
+            UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(1).GetUtf8(), "trololo");
+            UNIT_ASSERT(resultSet.TryNextRow());
+            UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(0).GetUtf8(), "2");
+            UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(1).GetUtf8(), "hello world");
+        }
+
+        {
+            auto scriptExecutionOperation = db.ExecuteScript(fmt::format(R"(
+                SELECT * FROM `{external_source}`.`/b/year=2023/` WITH (
+                    format="json_each_row",
+                    schema(
+                        key Utf8 NOT NULL,
+                        value Utf8 NOT NULL
+                    )
+                )
+            )", "external_source"_a = externalDataSourceName)).ExtractValueSync();
+            UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString());
+            UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId);
+
+            NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver());
+            UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString());
+            TFetchScriptResultsResult results = db.FetchScriptResults(scriptExecutionOperation.Id(), 0).ExtractValueSync();
+            UNIT_ASSERT_C(results.IsSuccess(), results.GetIssues().ToString());
+
+            TResultSetParser resultSet(results.ExtractResultSet());
+            UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnsCount(), 2);
+            UNIT_ASSERT_VALUES_EQUAL(resultSet.RowsCount(), 2);
+            UNIT_ASSERT(resultSet.TryNextRow());
+            UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(0).GetUtf8(), "1");
+            UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(1).GetUtf8(), "trololo");
+            UNIT_ASSERT(resultSet.TryNextRow());
+            UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(0).GetUtf8(), "2");
+            UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(1).GetUtf8(), "hello world");
+        }
+    }
+}
+
+} // namespace NKikimr::NKqp

ydb/core/external_sources/s3/ut/test.json

@@ -0,0 +1,2 @@
+{"key": "1", "value": "trololo"}
+{"key": "2", "value": "hello world"}

ydb/core/external_sources/s3/ut/ya.make

@@ -0,0 +1,68 @@
+UNITTEST_FOR(ydb/core/external_sources/s3)
+
+NO_CHECK_IMPORTS()
+
+DATA(arcadia/ydb/core/external_sources/s3/ut/docker-compose.yml)
+ENV(COMPOSE_PROJECT_NAME=s3)
+
+IF (AUTOCHECK) 
+    # Temporarily disable these tests due to infrastructure incompatibility
+    SKIP_TEST("DEVTOOLSUPPORT-44637")
+
+    # Split tests to chunks only when they're running on different machines with distbuild,
+    # otherwise this directive will slow down local test execution.
+    # Look through DEVTOOLSSUPPORT-39642 for more information.
+    FORK_SUBTESTS()
+
+    # TAG and REQUIREMENTS are copied from: https://docs.yandex-team.ru/devtools/test/environment#docker-compose
+    TAG(
+        ya:external
+        ya:force_sandbox
+        ya:fat
+    )
+
+    REQUIREMENTS(
+        cpu:all
+        container:4467981730
+        dns:dns64
+    )
+ENDIF()
+
+INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc)
+
+IF (OPENSOURCE)
+    # Including of docker_compose/recipe.inc automatically converts these tests into LARGE, 
+    # which makes it impossible to run them during precommit checks on Github CI. 
+    # Next several lines forces these tests to be MEDIUM. To see discussion, visit YDBOPS-8928.
+    SIZE(MEDIUM)
+    SET(TEST_TAGS_VALUE)
+    SET(TEST_REQUIREMENTS_VALUE)
+
+    # This requirement forces tests to be launched consequently,
+    # otherwise CI system would be overloaded due to simultaneous launch of many Docker containers.
+    # See DEVTOOLSSUPPORT-44103, YA-1759 for details.
+    TAG(ya:not_autocheck)
+    REQUIREMENTS(cpu:all)
+ENDIF()
+
+SRCS(
+    s3_aws_credentials_ut.cpp
+)
+
+PEERDIR(
+    library/cpp/testing/unittest
+    library/cpp/testing/common
+    ydb/core/kqp/ut/common
+    ydb/core/kqp/ut/federated_query/common
+    ydb/library/yql/sql/pg_dummy
+    ydb/public/sdk/cpp/client/ydb_types/operation
+    ydb/library/actors/core
+)
+
+DEPENDS(
+    library/recipes/docker_compose/bin
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()

ydb/core/external_sources/s3/ya.make

@@ -0,0 +1,3 @@
+RECURSE_FOR_TESTS(
+    ut
+)

ydb/library/yql/providers/common/http_gateway/ut/ya.make

@@ -3,6 +3,7 @@ UNITTEST_FOR(ydb/library/yql/providers/common/http_gateway)
 FORK_SUBTESTS()
 
 SRCS(
+    yql_aws_signature_ut.cpp
     yql_dns_gateway_ut.cpp
 )
 

ydb/library/yql/providers/common/http_gateway/ya.make

@@ -1,19 +1,22 @@
 LIBRARY()
 
 SRCS(
-    yql_http_gateway.cpp
+    yql_aws_signature.cpp
     yql_http_default_retry_policy.cpp
+    yql_http_gateway.cpp
 )
 
 PEERDIR(
     contrib/libs/curl
-    ydb/library/actors/prof
     library/cpp/monlib/dynamic_counters
     library/cpp/retry
+    ydb/library/actors/http
+    ydb/library/actors/prof
+    ydb/library/actors/protos
     ydb/library/yql/providers/common/proto
     ydb/library/yql/public/issue
-    ydb/library/yql/utils/log
     ydb/library/yql/utils
+    ydb/library/yql/utils/log
 )
 
 YQL_LAST_ABI_VERSION()