Merge tag '24.3.14' into stream-nb-24-3-merge-24.3.14

.github/actions/s3cmd/action.yml

@@ -53,7 +53,7 @@ runs:
           debug)
             folder+="-debug"
             ;;
-          release-*)
+          release*)
             folder+="-${BUILD_PRESET/release-/}"
             ;;
           *)

.github/actions/test_ya/action.yml

@@ -103,7 +103,7 @@ runs:
           debug)
             TESTMO_SOURCE="ya-${ARCH}-debug"
             ;;
-          release-*)
+          release*)
             TESTMO_SOURCE="ya-${ARCH}-${BUILD_PRESET/release-/}"
             ;;
           *)
@@ -180,6 +180,9 @@ runs:
           relwithdebinfo)
             params+=(--build "relwithdebinfo")
             ;;
+          release)
+            params+=(--build "release")
+            ;;
           release-asan)
             params+=(
               --build "release" --sanitize="address"

.github/config/muted_ya.txt

@@ -8,6 +8,7 @@ ydb/core/external_sources *
 ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient
 ydb/core/keyvalue/ut_trace TKeyValueTracingTest.*
 ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDataSourceMetadata
+ydb/core/kqp/ut/join KqpJoinOrder.Chain65Nodes
 ydb/core/kqp/ut/olap KqpOlapBlobsSharing.*
 ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL
 ydb/core/kqp/ut/pg KqpPg.CreateIndex
@@ -17,6 +18,7 @@ ydb/core/kqp/ut/tx KqpSnapshotRead.ReadOnlyTxWithIndexCommitsOnConcurrentWrite+w
 ydb/core/kqp/ut/tx KqpSinkTx.InvalidateOnError
 ydb/core/kqp/ut/query KqpLimits.QueryReplySize
 ydb/core/kqp/ut/query KqpQuery.QueryTimeout
+ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpReplace+HasSecondaryIndex
 ydb/core/kqp/ut/scan KqpRequestContext.TraceIdInErrorMessage
 ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns
 ydb/core/kqp/ut/scheme KqpScheme.AlterAsyncReplication
@@ -26,9 +28,9 @@ ydb/core/kqp/ut/scheme [15/50]*
 ydb/core/kqp/ut/scheme [44/50]*
 ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect
 ydb/core/kqp/ut/service KqpQueryService.QueryOnClosedSession
+ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpUpdate
 ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad
 ydb/core/kqp/ut/service [38/50]*
-ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpUpdate
 ydb/core/kqp/ut/service KqpQueryService.TableSink_OltpReplace+HasSecondaryIndex
 ydb/core/persqueue/ut [37/40] chunk chunk
 ydb/core/persqueue/ut [38/40] chunk chunk

.github/workflows/nightly_build.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        build_preset: ["relwithdebinfo", "release-asan"]
+        build_preset: ["release", "relwithdebinfo", "release-asan"]
     runs-on: [ self-hosted, auto-provisioned, "${{ format('build-preset-{0}', matrix.build_preset) }}" ]
     name: Build and test ${{ matrix.build_preset }}
     steps:

.github/workflows/pr_check.yml

@@ -203,7 +203,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        build_preset: ["relwithdebinfo", "release-asan", "release-clang14"]
+        build_preset: ["relwithdebinfo", "release-asan"]
     runs-on: [ self-hosted, auto-provisioned, "${{ format('build-preset-{0}', matrix.build_preset) }}" ]
     name: Build and test ${{ matrix.build_preset }}
     steps:
@@ -247,7 +247,7 @@ jobs:
             https://api.github.com/repos/${{github.repository}}/commits/${{github.event.pull_request.head.sha}}/status | \
             jq -cr '.statuses | .[] | select(.state=="success") | select(.context | startswith("build_")) | .context' | \
             wc -l )
-          if [[ $successbuilds == "3" ]];then
+          if [[ $successbuilds == "2" ]];then
             integrated_status="success"
           else
             integrated_status="failure"

.gitignore

@@ -7,6 +7,8 @@
 !*/
 # Unignore all files inside canondata dir
 !**/canondata/**
+# Allow docker files
+!Dockerfile
 
 /canonization_show_res.log
 
@@ -26,6 +28,9 @@ __pycache__/
 *.pb.h
 *.pb.cc
 
+# Other generated
+*.fbs.h
+
 # MacOS specific
 .DS_Store
 
@@ -40,6 +45,7 @@ __pycache__/
 .idea/
 .vscode/
 .clangd
+.antlr/
 
 # KDevelop IDE
 *.kdev4
@@ -79,3 +85,6 @@ list_result.log
 bin/config.json
 
 .vs/
+
+# handy for local junk, which is not intended to appear in the repo
+junk/

library/cpp/threading/future/core/future-inl.h

@@ -116,6 +116,9 @@ namespace NThreading {
             bool HasException() const {
                 return AtomicGet(State) == ExceptionSet;
             }
+            bool IsReady() const {
+                return AtomicGet(State) != NotReady;
+            }
 
             const T& GetValue(TDuration timeout = TDuration::Zero()) const {
                 AccessValue(timeout, ValueRead);
@@ -297,6 +300,9 @@ namespace NThreading {
             bool HasException() const {
                 return AtomicGet(State) == ExceptionSet;
             }
+            bool IsReady() const {
+                return AtomicGet(State) != NotReady;
+            }
 
             void GetValue(TDuration timeout = TDuration::Zero()) const {
                 TAtomicBase state = AtomicGet(State);
@@ -583,6 +589,10 @@ namespace NThreading {
     inline bool TFuture<T>::HasException() const {
         return State && State->HasException();
     }
+    template <typename T>
+    inline bool TFuture<T>::IsReady() const {
+        return State && State->IsReady();
+    }
 
     template <typename T>
     inline void TFuture<T>::Wait() const {
@@ -688,6 +698,9 @@ namespace NThreading {
     inline bool TFuture<void>::HasException() const {
         return State && State->HasException();
     }
+    inline bool TFuture<void>::IsReady() const {
+        return State && State->IsReady();
+    }
 
     inline void TFuture<void>::Wait() const {
         EnsureInitialized();
@@ -823,6 +836,11 @@ namespace NThreading {
         return State && State->HasException();
     }
 
+    template <typename T>
+    inline bool TPromise<T>::IsReady() const {
+        return State && State->IsReady();
+    }
+
     template <typename T>
     inline void TPromise<T>::SetException(const TString& e) {
         EnsureInitialized();
@@ -904,6 +922,10 @@ namespace NThreading {
         return State && State->HasException();
     }
 
+    inline bool TPromise<void>::IsReady() const {
+        return State && State->IsReady();
+    }
+
     inline void TPromise<void>::SetException(const TString& e) {
         EnsureInitialized();
         State->SetException(std::make_exception_ptr(yexception() << e));

library/cpp/threading/future/core/future.h

@@ -98,6 +98,12 @@ namespace NThreading {
         void TryRethrow() const;
         bool HasException() const;
 
+        // returns true if exception or value was set.
+        //   allows to check readiness without locking cheker-thread
+        //   NOTE: returns true even if value was extracted from promise
+        //   good replace for HasValue() || HasException()
+        bool IsReady() const;
+
         void Wait() const;
         bool Wait(TDuration timeout) const;
         bool Wait(TInstant deadline) const;
@@ -153,6 +159,11 @@ namespace NThreading {
         void TryRethrow() const;
         bool HasException() const;
 
+        // returns true if exception or value was set.
+        //   allows to check readiness without locking cheker-thread
+        //   good replace for HasValue() || HasException()
+        bool IsReady() const;
+
         void Wait() const;
         bool Wait(TDuration timeout) const;
         bool Wait(TInstant deadline) const;
@@ -216,6 +227,12 @@ namespace NThreading {
 
         void TryRethrow() const;
         bool HasException() const;
+
+        // returns true if exception or value was set.
+        //   allows to check readiness without locking cheker-thread
+        //   NOTE: returns true even if value was extracted from promise
+        //   good replace for HasValue() || HasException()
+        bool IsReady() const;
         void SetException(const TString& e);
         void SetException(std::exception_ptr e);
         bool TrySetException(std::exception_ptr e);
@@ -256,6 +273,11 @@ namespace NThreading {
 
         void TryRethrow() const;
         bool HasException() const;
+
+        // returns true if exception or value was set.
+        //   allows to check readiness without locking cheker-thread
+        //   good replace for HasValue() || HasException()
+        bool IsReady() const;
         void SetException(const TString& e);
         void SetException(std::exception_ptr e);
         bool TrySetException(std::exception_ptr e);

library/cpp/threading/future/future_ut.cpp

@@ -105,6 +105,7 @@ namespace {
 
             future = MakeFuture(345);
             UNIT_ASSERT(future.HasValue());
+            UNIT_ASSERT(future.IsReady());
             UNIT_ASSERT_EQUAL(future.GetValue(), 345);
         }
 
@@ -115,6 +116,7 @@ namespace {
 
             TFuture<void> future = promise.GetFuture();
             UNIT_ASSERT(future.HasValue());
+            UNIT_ASSERT(future.IsReady());
 
             future = MakeFuture();
             UNIT_ASSERT(future.HasValue());
@@ -523,6 +525,7 @@ namespace {
         {
             auto future1 = MakeErrorFuture<void>(std::make_exception_ptr(TFutureException()));
             UNIT_ASSERT(future1.HasException());
+            UNIT_ASSERT(future1.IsReady());
             UNIT_CHECK_GENERATED_EXCEPTION(future1.GetValue(), TFutureException);
 
             auto future2 = MakeErrorFuture<int>(std::make_exception_ptr(TFutureException()));
@@ -563,6 +566,7 @@ namespace {
             promise2.SetException("foo-exception");
             wait.Wait();
             UNIT_ASSERT(future2.HasException());
+            UNIT_ASSERT(!future1.IsReady());
             UNIT_ASSERT(!future1.HasValue() && !future1.HasException());
         }
 

ydb/core/audit/audit_log_impl.cpp

@@ -73,7 +73,7 @@ void WriteLog(const TString& log, const TVector<THolder<TLogBackend>>& logBacken
                 log.length()
             ));
         } catch (const yexception& e) {
-            LOG_W("WriteLog: unable to write audit log (error: " << e.what() << ")");
+            LOG_E("WriteLog: unable to write audit log (error: " << e.what() << ")");
         }
     }
 }
@@ -91,6 +91,27 @@ TString GetJsonLog(const TEvAuditLog::TEvWriteAuditLog::TPtr& ev) {
     return ss.Str();
 }
 
+TString GetJsonLogCompatibleLog(const TEvAuditLog::TEvWriteAuditLog::TPtr& ev) {
+    const auto* msg = ev->Get();
+    TStringStream ss;
+    NJsonWriter::TBuf json(NJsonWriter::HEM_DONT_ESCAPE_HTML, &ss);
+    {
+        auto obj = json.BeginObject();
+        obj
+            .WriteKey("@timestamp")
+            .WriteString(msg->Time.ToString().data())
+            .WriteKey("@log_type")
+            .WriteString("audit");
+
+        for (auto& [k, v] : msg->Parts) {
+            obj.WriteKey(k).WriteString(v);
+        }
+        json.EndObject();
+    }
+    ss << Endl;
+    return ss.Str();
+}
+
 TString GetTxtLog(const TEvAuditLog::TEvWriteAuditLog::TPtr& ev) {
     const auto* msg = ev->Get();
     TStringStream ss;
@@ -146,6 +167,9 @@ class TAuditLogActor final : public TActor<TAuditLogActor> {
                 case NKikimrConfig::TAuditConfig::TXT:
                     WriteLog(GetTxtLog(ev), logBackends.second);
                     break;
+                case NKikimrConfig::TAuditConfig::JSON_LOG_COMPATIBLE:
+                    WriteLog(GetJsonLogCompatibleLog(ev), logBackends.second);
+                    break;
                 default:
                     WriteLog(GetJsonLog(ev), logBackends.second);
                     break;

ydb/core/audit/ya.make

@@ -13,8 +13,4 @@ PEERDIR(
     ydb/core/base
 )
 
-RESOURCE(
-    ydb/core/kqp/kqp_default_settings.txt kqp_default_settings.txt
-)
-
 END()

ydb/core/base/blobstorage.h

@@ -488,6 +488,8 @@ struct TEvBlobStorage {
         EvInplacePatch,
         EvAssimilate,
 
+        EvGetQueuesInfo,     // for debugging purposes
+
         //
         EvPutResult = EvPut + 512,                              /// 268 632 576
         EvGetResult,
@@ -502,6 +504,8 @@ struct TEvBlobStorage {
         EvInplacePatchResult,
         EvAssimilateResult,
 
+        EvQueuesInfo,  // for debugging purposes
+
         // proxy <-> vdisk interface
         EvVPut = EvPut + 2 * 512,                               /// 268 633 088
         EvVGet,
@@ -871,6 +875,7 @@ struct TEvBlobStorage {
         EvRunActor = EvPut + 15 * 512,
         EvVMockCtlRequest,
         EvVMockCtlResponse,
+        EvDelayedMessageWrapper,
 
         // incremental huge blob keeper
         EvIncrHugeInit = EvPut + 17 * 512,

ydb/core/base/ticket_parser.h

@@ -154,14 +154,24 @@ namespace NKikimr {
 
         struct TError {
             TString Message;
+            TString LogMessage;
             bool Retryable = true;
 
             bool empty() const {
-                return Message.empty();
+                return Message.empty() && LogMessage.empty();
+            }
+
+            bool HasMessage() const {
+                return !Message.empty();
+            }
+
+            bool HasLogMessage() const {
+                return !LogMessage.empty();
             }
 
             void clear() {
                 Message.clear();
+                LogMessage.clear();
                 Retryable = true;
             }