diff --git a/ydb/core/external_sources/s3/ut/docker-compose.yml b/ydb/core/external_sources/s3/ut/docker-compose.yml new file mode 100644 index 000000000000..9c983d87b300 --- /dev/null +++ b/ydb/core/external_sources/s3/ut/docker-compose.yml @@ -0,0 +1,31 @@ +version: '3.4' +services: + # MinIO object storage + minio: + hostname: minio + image: 'minio/minio@sha256:1a3debf2408bde1f33b49cd70af245eb2173c5897a2e6bf99d7934005cd14537' + container_name: minio + ports: + - '9000' + - '9001' + environment: + MINIO_ACCESS_KEY: minio + MINIO_SECRET_KEY: minio123 + command: server /data --console-address ":9001" + + # This job creates the "datalake" bucket on Minio + mc-job: + image: 'minio/mc@sha256:03e4ea06fe42f94c078613554c34eeb2c7045e79a4b0d875a3c977bf27a8befb' + container_name: mc-job + volumes: + - ./test.json:/test.json + entrypoint: | + /bin/bash -c " + sleep 5; + /usr/bin/mc config --quiet host add myminio http://minio:9000 minio minio123; + /usr/bin/mc mb --quiet myminio/datalake + /usr/bin/mc put /test.json myminio/datalake/a/test.json + /usr/bin/mc put /test.json myminio/datalake/b/year=2023/month=01/day=03/test.json + " + depends_on: + - minio diff --git a/ydb/core/external_sources/s3/ut/s3_aws_credentials_ut.cpp b/ydb/core/external_sources/s3/ut/s3_aws_credentials_ut.cpp new file mode 100644 index 000000000000..4eb9da765afa --- /dev/null +++ b/ydb/core/external_sources/s3/ut/s3_aws_credentials_ut.cpp @@ -0,0 +1,129 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include + +namespace NKikimr::NKqp { + +using namespace NYdb; +using namespace NYdb::NQuery; +using namespace NKikimr::NKqp::NFederatedQueryTest; +using namespace fmt::literals; + +TString Exec(const TString& cmd) { + std::array buffer; + TString result; + std::unique_ptr pipe(popen(cmd.c_str(), "r"), pclose); + if (!pipe) { + throw std::runtime_error("popen() failed!"); + } + while (fgets(buffer.data(), static_cast(buffer.size()), pipe.get()) != nullptr) { + result += buffer.data(); + } + return result; +} + +TString GetExternalPort(const TString& service, const TString& port) { + auto dockerComposeBin = BinaryPath("library/recipes/docker_compose/bin/docker-compose"); + auto composeFileYml = ArcadiaSourceRoot() + "/ydb/core/external_sources/s3/ut/docker-compose.yml"; + auto result = StringSplitter(Exec(dockerComposeBin + " -f " + composeFileYml + " port " + service + " " + port)).Split(':').ToList(); + return result ? Strip(result.back()) : TString{}; +} + +Y_UNIT_TEST_SUITE(S3AwsCredentials) { + Y_UNIT_TEST(ExecuteScriptWithEqSymbol) { + const TString externalDataSourceName = "/Root/external_data_source"; + auto kikimr = MakeKikimrRunner(true); + auto tc = kikimr->GetTableClient(); + auto session = tc.CreateSession().GetValueSync().GetSession(); + const TString query = fmt::format(R"( + CREATE OBJECT id (TYPE SECRET) WITH (value=`minio`); + CREATE OBJECT key (TYPE SECRET) WITH (value=`minio123`); + CREATE EXTERNAL DATA SOURCE `{external_source}` WITH ( + SOURCE_TYPE="ObjectStorage", + LOCATION="{location}", + AUTH_METHOD="AWS", + AWS_ACCESS_KEY_ID_SECRET_NAME="id", + AWS_SECRET_ACCESS_KEY_SECRET_NAME="key", + AWS_REGION="ru-central-1" + );)", + "external_source"_a = externalDataSourceName, + "location"_a = "localhost:" + GetExternalPort("minio", "9000") + "/datalake/" + ); + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_C(result.GetStatus() == NYdb::EStatus::SUCCESS, result.GetIssues().ToString()); + auto db = kikimr->GetQueryClient(); + { + auto scriptExecutionOperation = db.ExecuteScript(fmt::format(R"( + SELECT * FROM `{external_source}`.`/a/` WITH ( + format="json_each_row", + schema( + key Utf8 NOT NULL, + value Utf8 NOT NULL + ) + ) + )", "external_source"_a = externalDataSourceName)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); + UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + + NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); + UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); + TFetchScriptResultsResult results = db.FetchScriptResults(scriptExecutionOperation.Id(), 0).ExtractValueSync(); + UNIT_ASSERT_C(results.IsSuccess(), results.GetIssues().ToString()); + + TResultSetParser resultSet(results.ExtractResultSet()); + UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnsCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(resultSet.RowsCount(), 2); + UNIT_ASSERT(resultSet.TryNextRow()); + UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(0).GetUtf8(), "1"); + UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(1).GetUtf8(), "trololo"); + UNIT_ASSERT(resultSet.TryNextRow()); + UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(0).GetUtf8(), "2"); + UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(1).GetUtf8(), "hello world"); + } + + { + auto scriptExecutionOperation = db.ExecuteScript(fmt::format(R"( + SELECT * FROM `{external_source}`.`/b/year=2023/` WITH ( + format="json_each_row", + schema( + key Utf8 NOT NULL, + value Utf8 NOT NULL + ) + ) + )", "external_source"_a = externalDataSourceName)).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString()); + UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId); + + NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver()); + UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString()); + TFetchScriptResultsResult results = db.FetchScriptResults(scriptExecutionOperation.Id(), 0).ExtractValueSync(); + UNIT_ASSERT_C(results.IsSuccess(), results.GetIssues().ToString()); + + TResultSetParser resultSet(results.ExtractResultSet()); + UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnsCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(resultSet.RowsCount(), 2); + UNIT_ASSERT(resultSet.TryNextRow()); + UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(0).GetUtf8(), "1"); + UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(1).GetUtf8(), "trololo"); + UNIT_ASSERT(resultSet.TryNextRow()); + UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(0).GetUtf8(), "2"); + UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(1).GetUtf8(), "hello world"); + } + } +} + +} // namespace NKikimr::NKqp diff --git a/ydb/core/external_sources/s3/ut/test.json b/ydb/core/external_sources/s3/ut/test.json new file mode 100644 index 000000000000..49f79f74d9e1 --- /dev/null +++ b/ydb/core/external_sources/s3/ut/test.json @@ -0,0 +1,2 @@ +{"key": "1", "value": "trololo"} +{"key": "2", "value": "hello world"} diff --git a/ydb/core/external_sources/s3/ut/ya.make b/ydb/core/external_sources/s3/ut/ya.make new file mode 100644 index 000000000000..da4d33248657 --- /dev/null +++ b/ydb/core/external_sources/s3/ut/ya.make @@ -0,0 +1,68 @@ +UNITTEST_FOR(ydb/core/external_sources/s3) + +NO_CHECK_IMPORTS() + +DATA(arcadia/ydb/core/external_sources/s3/ut/docker-compose.yml) +ENV(COMPOSE_PROJECT_NAME=s3) + +IF (AUTOCHECK) + # Temporarily disable these tests due to infrastructure incompatibility + SKIP_TEST("DEVTOOLSUPPORT-44637") + + # Split tests to chunks only when they're running on different machines with distbuild, + # otherwise this directive will slow down local test execution. + # Look through DEVTOOLSSUPPORT-39642 for more information. + FORK_SUBTESTS() + + # TAG and REQUIREMENTS are copied from: https://docs.yandex-team.ru/devtools/test/environment#docker-compose + TAG( + ya:external + ya:force_sandbox + ya:fat + ) + + REQUIREMENTS( + cpu:all + container:4467981730 + dns:dns64 + ) +ENDIF() + +INCLUDE(${ARCADIA_ROOT}/library/recipes/docker_compose/recipe.inc) + +IF (OPENSOURCE) + # Including of docker_compose/recipe.inc automatically converts these tests into LARGE, + # which makes it impossible to run them during precommit checks on Github CI. + # Next several lines forces these tests to be MEDIUM. To see discussion, visit YDBOPS-8928. + SIZE(MEDIUM) + SET(TEST_TAGS_VALUE) + SET(TEST_REQUIREMENTS_VALUE) + + # This requirement forces tests to be launched consequently, + # otherwise CI system would be overloaded due to simultaneous launch of many Docker containers. + # See DEVTOOLSSUPPORT-44103, YA-1759 for details. + TAG(ya:not_autocheck) + REQUIREMENTS(cpu:all) +ENDIF() + +SRCS( + s3_aws_credentials_ut.cpp +) + +PEERDIR( + library/cpp/testing/unittest + library/cpp/testing/common + ydb/core/kqp/ut/common + ydb/core/kqp/ut/federated_query/common + ydb/library/yql/sql/pg_dummy + ydb/public/sdk/cpp/client/ydb_types/operation + ydb/library/actors/core +) + +DEPENDS( + library/recipes/docker_compose/bin +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/external_sources/s3/ya.make b/ydb/core/external_sources/s3/ya.make new file mode 100644 index 000000000000..618af85292dd --- /dev/null +++ b/ydb/core/external_sources/s3/ya.make @@ -0,0 +1,3 @@ +RECURSE_FOR_TESTS( + ut +) diff --git a/ydb/library/yql/providers/common/http_gateway/ut/ya.make b/ydb/library/yql/providers/common/http_gateway/ut/ya.make index 4d2a4bead4ae..f79c3babdb4b 100644 --- a/ydb/library/yql/providers/common/http_gateway/ut/ya.make +++ b/ydb/library/yql/providers/common/http_gateway/ut/ya.make @@ -3,6 +3,7 @@ UNITTEST_FOR(ydb/library/yql/providers/common/http_gateway) FORK_SUBTESTS() SRCS( + yql_aws_signature_ut.cpp yql_dns_gateway_ut.cpp ) diff --git a/ydb/library/yql/providers/common/http_gateway/ya.make b/ydb/library/yql/providers/common/http_gateway/ya.make index 4393f3fb4143..50c972e72739 100644 --- a/ydb/library/yql/providers/common/http_gateway/ya.make +++ b/ydb/library/yql/providers/common/http_gateway/ya.make @@ -1,19 +1,22 @@ LIBRARY() SRCS( - yql_http_gateway.cpp + yql_aws_signature.cpp yql_http_default_retry_policy.cpp + yql_http_gateway.cpp ) PEERDIR( contrib/libs/curl - ydb/library/actors/prof library/cpp/monlib/dynamic_counters library/cpp/retry + ydb/library/actors/http + ydb/library/actors/prof + ydb/library/actors/protos ydb/library/yql/providers/common/proto ydb/library/yql/public/issue - ydb/library/yql/utils/log ydb/library/yql/utils + ydb/library/yql/utils/log ) YQL_LAST_ABI_VERSION() diff --git a/ydb/library/yql/providers/common/http_gateway/yql_aws_signature.cpp b/ydb/library/yql/providers/common/http_gateway/yql_aws_signature.cpp new file mode 100644 index 000000000000..739c12f0dd94 --- /dev/null +++ b/ydb/library/yql/providers/common/http_gateway/yql_aws_signature.cpp @@ -0,0 +1,195 @@ +#include "yql_aws_signature.h" + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace NYql { + +TAwsSignature::TAwsSignature(const TString& method, const TString& url, const TString& contentType, const TString& payload, const TString& awsSigV4, const TString& userPwd, const TInstant& currentTime) + : Method(method) + , Url(url) + , ContentType(contentType) + , Payload(payload) + , CurrentTime(currentTime) +{ + const TVector secrets = StringSplitter(userPwd).Split(':'); + if (secrets.size() == 2) { + AccessKey = secrets[0]; + AccessSecret = secrets[1]; + } + + const TVector providers = StringSplitter(awsSigV4).Split(':'); + if (providers.size() == 4) { + Region = providers[2]; + Service = providers[3]; + } + + TStringBuf scheme; + TStringBuf host; + TStringBuf uri; + NHttp::CrackURL(Url, scheme, host, uri); + + Host = host; + Uri = uri; + + TStringBuf path; + TStringBuf cgi; + TStringBuf{Uri}.Split('?', path, cgi); + + Uri = path; + Cgi = cgi; + + PrepareCgiParameters(); +} + +TString TAwsSignature::GetAuthorization() const { + return TStringBuilder{} + << "AWS4-HMAC-SHA256 Credential=" << AccessKey << "/" << GetDate() << "/" << Region << "/" << Service << "/aws4_request, " + << "SignedHeaders=" << GetListSignedHeaders() + << ", Signature=" << CalcSignature(); +} + +TString TAwsSignature::GetXAmzContentSha256() const { + return HashSHA256(TStringBuilder{} << "\"" << Payload << "\""); +} + +TString TAwsSignature::GetAmzDate() const { + return CurrentTime.FormatLocalTime("%Y%m%dT%H%M%SZ"); +} + +TString TAwsSignature::GetContentType() const { + return ContentType; +} + +TString TAwsSignature::GetListSignedHeaders() const { + return ContentType ? "content-type;host;x-amz-content-sha256;x-amz-date" : "host;x-amz-content-sha256;x-amz-date"; +} + +TString TAwsSignature::SignHeaders() const { + return TStringBuilder{} << (ContentType ? (TStringBuilder{} << "content-type:" << GetContentType() << Endl) : TString{}) + << "host:" << Host << Endl + << "x-amz-content-sha256:" << GetXAmzContentSha256() << Endl + << "x-amz-date:" << GetAmzDate(); +} + +TString TAwsSignature::CreateHttpCanonicalRequest() const { + return TStringBuilder{} << Method << Endl + << UriEncode(Uri) << Endl + << Cgi << Endl + << SignHeaders() << Endl << Endl + << GetListSignedHeaders() << Endl + << GetXAmzContentSha256(); +} + +TString TAwsSignature::CreateStringToSign() const { + auto canonical = CreateHttpCanonicalRequest(); + return TStringBuilder{} << "AWS4-HMAC-SHA256" << Endl + << GetAmzDate() << Endl + << GetDate() << "/" << Region << "/" << Service << "/" << "aws4_request" << Endl + << HashSHA256(canonical); +} + +TString TAwsSignature::CalcSignature() const { + const auto dateKey = HmacSHA256(TString::Join("AWS4", AccessSecret), GetDate()); + const auto dateRegionKey = HmacSHA256(dateKey, Region); + const auto dateRegionServiceKey = HmacSHA256(dateRegionKey, Service); + const auto signingKey = HmacSHA256(dateRegionServiceKey, "aws4_request"); + const auto signatureHmac = HmacSHA256(signingKey, CreateStringToSign()); + return to_lower(HexEncode(signatureHmac.data(), signatureHmac.size())); +} + +TString TAwsSignature::GetDate() const { + return CurrentTime.FormatLocalTime("%Y%m%d"); +} + +TString TAwsSignature::HmacSHA256(TStringBuf key, TStringBuf data) { + unsigned char hash[SHA256_DIGEST_LENGTH]; + ui32 hl = SHA256_DIGEST_LENGTH; + const auto* res = HMAC(EVP_sha256(), key.data(), key.size(), reinterpret_cast(data.data()), data.size(), hash, &hl); + Y_ENSURE(res); + Y_ENSURE(hl == SHA256_DIGEST_LENGTH); + return TString{reinterpret_cast(res), hl}; +} + +TString TAwsSignature::HashSHA256(TStringBuf data) { + TMemoryInput stream{data}; + SHA256_CTX hasher; + SHA256_Init(&hasher); + char buf[4096]; + size_t read = 0; + while ((read = stream.Read(buf, sizeof(buf))) != 0) { + SHA256_Update(&hasher, buf, read); + } + unsigned char hash[SHA256_DIGEST_LENGTH]; + SHA256_Final(hash, &hasher); + return to_lower(HexEncode(hash, SHA256_DIGEST_LENGTH)); +} + +TString TAwsSignature::UriEncode(const TStringBuf input, bool encodeSlash) { + TStringStream result; + for (const char ch : input) { + if ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' || + ch == '-' || ch == '~' || ch == '.') { + result << ch; + } else if (ch == '/') { + if (encodeSlash) { + result << "%2F"; + } else { + result << ch; + } + } else { + result << "%" << HexEncode(&ch, 1); + } + } + return result.Str(); +} + +void TAwsSignature::PrepareCgiParameters() { + TCgiParameters cgi(Cgi); + TMap> sortedCgi; + + for (const auto& [key, value] : cgi) { + sortedCgi[key].push_back(value); + } + + for (auto& pair : sortedCgi) { + ::Sort(pair.second.begin(), pair.second.end()); + } + + if (sortedCgi.size()) { + TStringStream canonicalCgi; + + auto printSingleParam = [&canonicalCgi](const TString& key, const TVector& values) { + auto it = values.begin(); + canonicalCgi << UriEncode(key, true) << "=" << UriEncode(*it, true); + while (++it != values.end()) { + canonicalCgi << "&" << UriEncode(key, true) << "=" << UriEncode(*it, true); + } + + }; + + auto it = sortedCgi.begin(); + printSingleParam(it->first, it->second); + while (++it != sortedCgi.end()) { + canonicalCgi << "&"; + printSingleParam(it->first, it->second); + } + + Cgi = canonicalCgi.Str(); + } +} + +} diff --git a/ydb/library/yql/providers/common/http_gateway/yql_aws_signature.h b/ydb/library/yql/providers/common/http_gateway/yql_aws_signature.h new file mode 100644 index 000000000000..7a5032814b5b --- /dev/null +++ b/ydb/library/yql/providers/common/http_gateway/yql_aws_signature.h @@ -0,0 +1,54 @@ +#include +#include + +namespace NYql { + +struct TAwsSignature { +public: + TAwsSignature(const TString& method, const TString& url, const TString& contentType, const TString& payload, const TString& awsSigV4, const TString& userPwd, const TInstant& currentTime = TInstant::Now()); + + TString GetAuthorization() const; + + TString GetXAmzContentSha256() const; + + TString GetAmzDate() const; + + TString GetContentType() const; + +private: + TString GetListSignedHeaders() const; + + TString SignHeaders() const; + + TString CreateHttpCanonicalRequest() const; + + TString CreateStringToSign() const; + + TString CalcSignature() const; + + TString GetDate() const; + + static TString HmacSHA256(TStringBuf key, TStringBuf data); + + static TString HashSHA256(TStringBuf data); + + static TString UriEncode(const TStringBuf input, bool encodeSlash = false); + + void PrepareCgiParameters(); + +private: + TString Host; + TString Uri; + TString Cgi; + TString Method; + TString AccessKey; + TString AccessSecret; + TString Region; + TString Service; + TString Url; + TString ContentType; + TString Payload; + TInstant CurrentTime; +}; + +} diff --git a/ydb/library/yql/providers/common/http_gateway/yql_aws_signature_ut.cpp b/ydb/library/yql/providers/common/http_gateway/yql_aws_signature_ut.cpp new file mode 100644 index 000000000000..7a4489ffcee1 --- /dev/null +++ b/ydb/library/yql/providers/common/http_gateway/yql_aws_signature_ut.cpp @@ -0,0 +1,72 @@ +#include "yql_aws_signature.h" + +#include +#include + +namespace NYql { + +Y_UNIT_TEST_SUITE(TAwsSignature) { + Y_UNIT_TEST(Sign) { + NYql::TAwsSignature signature("GET", "http://os.com/my-bucket/year=2024/day=03/", "application/json", {}, "key", "pwd"); + UNIT_ASSERT_VALUES_EQUAL(signature.GetContentType(), "application/json"); + UNIT_ASSERT_VALUES_EQUAL(signature.GetXAmzContentSha256(), "12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126"); + UNIT_ASSERT_STRING_CONTAINS(signature.GetAuthorization(), "SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature="); + UNIT_ASSERT_STRING_CONTAINS(signature.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/"); + UNIT_ASSERT_STRING_CONTAINS(signature.GetAuthorization(), "///aws4_request"); + UNIT_ASSERT_STRING_CONTAINS(signature.GetAuthorization(), "Signature="); + UNIT_ASSERT_VALUES_UNEQUAL(signature.GetAmzDate(), ""); + } + + Y_UNIT_TEST(SignCmp) { + NYql::TAwsSignature signature1("GET", "http://os.com/my-bucket/year=2024/day=03/", "application/json", {}, "key", "pwd"); + NYql::TAwsSignature signature2("GET", "http://os.com/", "application/json", {}, "key", "pwd"); + UNIT_ASSERT_VALUES_EQUAL(signature1.GetContentType(), signature2.GetContentType()); + UNIT_ASSERT_VALUES_EQUAL(signature1.GetXAmzContentSha256(), signature2.GetXAmzContentSha256()); + UNIT_ASSERT_VALUES_UNEQUAL(signature1.GetAuthorization(), signature2.GetAuthorization()); + } + + Y_UNIT_TEST(SignPayload) { + NYql::TAwsSignature signature1("GET", "http://os.com/my-bucket/year=2024/day=03/", "application/json", {}, "key", "pwd"); + NYql::TAwsSignature signature2("GET", "http://os.com/", "application/json", "test", "key", "pwd"); + UNIT_ASSERT_VALUES_EQUAL(signature1.GetContentType(), signature2.GetContentType()); + UNIT_ASSERT_VALUES_UNEQUAL(signature1.GetXAmzContentSha256(), signature2.GetXAmzContentSha256()); + UNIT_ASSERT_VALUES_UNEQUAL(signature1.GetAuthorization(), signature2.GetAuthorization()); + } + + Y_UNIT_TEST(SignWithTime) { + auto time = TInstant::FromValue(30); + NYql::TAwsSignature signature1("GET", "http://os.com/my-bucket/year=2024/day=03/", "application/json", {}, "key", "pwd", time); + NYql::TAwsSignature signature2("GET", "http://os.com/", "application/json", "", "key", "pwd", time); + NYql::TAwsSignature signature3("GET", "http://os.com/my-bucket/year=2024/day=03/", "application/json", "", "key2", "pwd", time); + NYql::TAwsSignature signature4("POST", "http://os.com/my-bucket/year=2024/day=03/", "application/json", "", "key2", "pwd", time); + static const TString CONTENT_TYPE = "application/json"; + static const TString X_AMZ_CONTENT_SHA_256 = "12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126"; + static const TString X_AMX_DATE = "19700101T000000Z"; + UNIT_ASSERT_VALUES_EQUAL(signature1.GetContentType(), CONTENT_TYPE); + UNIT_ASSERT_VALUES_EQUAL(signature2.GetContentType(), CONTENT_TYPE); + UNIT_ASSERT_VALUES_EQUAL(signature3.GetContentType(), CONTENT_TYPE); + UNIT_ASSERT_VALUES_EQUAL(signature4.GetContentType(), CONTENT_TYPE); + UNIT_ASSERT_VALUES_EQUAL(signature1.GetXAmzContentSha256(), X_AMZ_CONTENT_SHA_256); + UNIT_ASSERT_VALUES_EQUAL(signature2.GetXAmzContentSha256(), X_AMZ_CONTENT_SHA_256); + UNIT_ASSERT_VALUES_EQUAL(signature3.GetXAmzContentSha256(), X_AMZ_CONTENT_SHA_256); + UNIT_ASSERT_VALUES_EQUAL(signature4.GetXAmzContentSha256(), X_AMZ_CONTENT_SHA_256); + UNIT_ASSERT_VALUES_EQUAL(signature1.GetAmzDate(), X_AMX_DATE); + UNIT_ASSERT_VALUES_EQUAL(signature2.GetAmzDate(), X_AMX_DATE); + UNIT_ASSERT_VALUES_EQUAL(signature3.GetAmzDate(), X_AMX_DATE); + UNIT_ASSERT_VALUES_EQUAL(signature4.GetAmzDate(), X_AMX_DATE); + UNIT_ASSERT_VALUES_EQUAL(signature1.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=fba1374eb117fe9d00fc04bb1b709a3ea6f152232ac1f7dc49117a505f7e9f3f"); + UNIT_ASSERT_VALUES_EQUAL(signature2.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=39b47595c16b5d7b8256fd00e3d17e2157c5050c293306c995ae6980f11c689f"); + UNIT_ASSERT_VALUES_EQUAL(signature3.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=fba1374eb117fe9d00fc04bb1b709a3ea6f152232ac1f7dc49117a505f7e9f3f"); + UNIT_ASSERT_VALUES_EQUAL(signature4.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=a495d0ada1156d7278a56fb754a728d3b9470725208ad422bc299d6d6f793a8b"); + } + + Y_UNIT_TEST(SignWithCanonization) { + auto time = TInstant::FromValue(30); + NYql::TAwsSignature signature1("GET", "http://os.com/path?a=3&b=7", "application/json", {}, "key", "pwd", time); + NYql::TAwsSignature signature2("GET", "http://os.com/path?b=7&a=3", "application/json", {}, "key", "pwd", time); + UNIT_ASSERT_VALUES_EQUAL(signature1.GetContentType(), signature2.GetContentType()); + UNIT_ASSERT_VALUES_EQUAL(signature1.GetAmzDate(), signature2.GetAmzDate()); + UNIT_ASSERT_VALUES_EQUAL(signature1.GetAuthorization(), signature2.GetAuthorization()); + } +} // Y_UNIT_TEST_SUITE(TAwsSignature) +} // namespace NYql diff --git a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp index ef5c2ce231e1..9d793cd403da 100644 --- a/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp +++ b/ydb/library/yql/providers/common/http_gateway/yql_http_gateway.cpp @@ -1,5 +1,6 @@ -#include "yql_http_gateway.h" +#include "yql_aws_signature.h" #include "yql_dns_gateway.h" +#include "yql_http_gateway.h" #include #include @@ -99,7 +100,8 @@ class TEasyCurl { size_t sizeLimit = 0, size_t bodySize = 0, const TCurlInitConfig& config = TCurlInitConfig(), - TDNSGateway<>::TDNSConstCurlListPtr dnsCache = nullptr) + TDNSGateway<>::TDNSConstCurlListPtr dnsCache = nullptr, + TString data = {}) : Headers(std::move(headers)) , Method(method) , Offset(offset) @@ -111,7 +113,8 @@ class TEasyCurl { , Config(config) , ErrorBuffer(static_cast(CURL_ERROR_SIZE), '\0') , DnsCache(dnsCache) - , Url(url) { + , Url(url) + , Data(std::move(data)) { InitHandles(); Counter->Inc(); } @@ -164,12 +167,41 @@ class TEasyCurl { curl_easy_setopt(Handle, CURLOPT_LOW_SPEED_LIMIT, Config.LowSpeedLimit); curl_easy_setopt(Handle, CURLOPT_ERRORBUFFER, ErrorBuffer.data()); - if (Headers.Options.AwsSigV4) { - curl_easy_setopt(Handle, CURLOPT_AWS_SIGV4, Headers.Options.AwsSigV4.c_str()); - } + if (Headers.Options.CurlSignature) { + if (Headers.Options.AwsSigV4) { + curl_easy_setopt(Handle, CURLOPT_AWS_SIGV4, Headers.Options.AwsSigV4.c_str()); + } + + if (Headers.Options.UserPwd) { + curl_easy_setopt(Handle, CURLOPT_USERPWD, Headers.Options.UserPwd.c_str()); + } + } else if (Headers.Options.AwsSigV4 || Headers.Options.UserPwd) { + TString method; + switch (Method) { + case EMethod::GET: + method = "GET"; + break; + case EMethod::POST: + method = "POST"; + break; + case EMethod::PUT: + method = "PUT"; + break; + case EMethod::DELETE: + method = "DELETE"; + break; + } - if (Headers.Options.UserPwd) { - curl_easy_setopt(Handle, CURLOPT_USERPWD, Headers.Options.UserPwd.c_str()); + TString contentType; + for (const auto& field: Headers.Fields) { + if (field.StartsWith("Content-Type:")) { + contentType = field.substr(strlen("Content-Type:")); + } + } + TAwsSignature signature(method, Url, contentType, Data, Headers.Options.AwsSigV4, Headers.Options.UserPwd); + Headers.Fields.push_back(TStringBuilder{} << "Authorization: " << signature.GetAuthorization()); + Headers.Fields.push_back(TStringBuilder{} << "x-amz-content-sha256: " << signature.GetXAmzContentSha256()); + Headers.Fields.push_back(TStringBuilder{} << "x-amz-date: " << signature.GetAmzDate()); } if (DnsCache != nullptr) { @@ -285,6 +317,7 @@ class TEasyCurl { TDNSGateway<>::TDNSConstCurlListPtr DnsCache; public: TString Url; + const TString Data; }; class TEasyCurlBuffer : public TEasyCurl { @@ -317,8 +350,8 @@ class TEasyCurlBuffer : public TEasyCurl { sizeLimit, data.size(), std::move(config), - std::move(dnsCache)) - , Data(std::move(data)) + std::move(dnsCache), + std::move(data)) , Input(Data) , Output(Buffer) , HeaderOutput(Header) @@ -422,7 +455,6 @@ class TEasyCurlBuffer : public TEasyCurl { return Input.Read(buffer, size * nmemb); } - const TString Data; TString Buffer, Header; TStringInput Input; TStringOutput Output, HeaderOutput; diff --git a/ydb/library/yql/providers/common/http_gateway/yql_http_header.h b/ydb/library/yql/providers/common/http_gateway/yql_http_header.h index d4f9a4892d25..1baa051e9556 100644 --- a/ydb/library/yql/providers/common/http_gateway/yql_http_header.h +++ b/ydb/library/yql/providers/common/http_gateway/yql_http_header.h @@ -10,6 +10,7 @@ struct THttpHeader { struct TOptions { TString UserPwd; TString AwsSigV4; + bool CurlSignature = false; }; TSmallVec Fields;