YQ-2302: read_max_bytes bypasses file_size_limit #4117
Conversation
alchizhevsky
commented
Apr 25, 2024
alchizhevsky
commented
- Bugfix
|
⚪
|
|
⚪ |
|
⚪
|
|
⚪
|
|
⚪
|
|
⚪
|
|
⚪
|
|
⚪
|
|
⚪
|
|
⚪
|
|
⚪
|
| @@ -3516,9 +3526,13 @@ std::pair<NYql::NDq::IDqComputeActorAsyncInput*, IActor*> CreateS3ReadActor( | |||
|
|
|||
| #undef SET_FLAG | |||
| #undef SUPPORTED_FLAGS | |||
| ui64 sizeLimit = std::numeric_limits<ui64>::max(); | |||
| if (const auto it = settings.find("sizeLimit"); settings.cend() != it) | |||
There was a problem hiding this comment.
Тело if заключай в фигурные скобки
| @@ -485,12 +485,18 @@ class TS3DqIntegration: public TDqIntegrationBase { | |||
|
|
|||
| YQL_CLOG(DEBUG, ProviderS3) << " hasDirectories=" << hasDirectories << ", consumersCount=" << consumersCount; | |||
|
|
|||
| ui64 readLimit = std::numeric_limits<ui64>::max(); | |||
| if (srcDesc.MutableSettings()->find("sizeLimit") != srcDesc.MutableSettings()->cend()) { | |||
| readLimit = FromString<ui64>(srcDesc.MutableSettings()->at("sizeLimit")); | |||
There was a problem hiding this comment.
Перейди на итератор, чтобы поиск ключа делался один раз, а не два
|
⚪
|
|
⚪
|
| objectPath.SetPathIndex(CurrentDirectoryPathIndex); | ||
| Objects.emplace_back(std::move(objectPath)); | ||
| ObjectsTotalSize += object.Size; | ||
| ObjectsTotalSize += std::min(object.Size, ReadLimit); |
There was a problem hiding this comment.
копипаста выражения. или в переменную положи или вытащи через objectPath.GetSize();
| @@ -245,13 +252,13 @@ class TS3LogicalOptProposalTransformer : public TOptimizeTransformerBase { | |||
| UnpackPathsList(packed, isTextEncoded, paths); | |||
|
|
|||
| for (auto& entry : paths) { | |||
| if (entry.Size > fileSizeLimit) { | |||
| if (std::min(entry.Size, userSizeLimit) > fileSizeLimit) { | |||
There was a problem hiding this comment.
постарайся избавиться от копирования выражения с std::min
ydb/tests/fq/s3/test_size_limit.py
Outdated
| result = result_set.rows[0].items[0].bytes_value | ||
| assert result == info.encode()[:limit] | ||
| if kikimr_params.param['raw'] < min(len(info), limit): | ||
| client.wait_query_status(query_id, fq.QueryMeta.FAILED) |
There was a problem hiding this comment.
сделай один вызов wait_query_status. и expected_status посчитай на основании выражения
|
⚪
|
|
⚪
|
