Merge branch 'ydb-platform:main' into YDBOPS-11054

.github/config/muted_ya.txt

@@ -255,4 +255,5 @@ ydb/tests/sql/large test_workload_manager.py.TestWorkloadManager.test_resource_p
 ydb/tests/sql/large test_workload_manager.py.TestWorkloadManager.test_resource_pool_queue_resource_weight[1-True]
 ydb/tests/sql/large test_workload_manager.py.TestWorkloadManager.test_resource_pool_queue_size_limit
 ydb/tests/stress/log/tests test_workload.py.TestYdbLogWorkload.test[column]
+ydb/tests/stress/kv/tests test_workload.py.TestYdbKvWorkload.test[column]
 ydb/tools/stress_tool/ut TDeviceTestTool.PDiskTestLogWrite

build/conf/python.conf

@@ -477,8 +477,8 @@ module PY_ANY_MODULE: DLL_UNIT {
 
     # -bundle
     when ($DARWIN == "yes") {
-        when ($CLANG_VER == "16") {
-            # TODO(pg): also check 17 and beyond
+        when ($CLANG_VER == "16" || $CLANG_VER == "18") {
+            # TODO(pg): also check 19 and beyond
         }
         otherwise {
             LDFLAGS+=-flat_namespace

build/conf/ts/node_modules.conf

@@ -36,7 +36,7 @@ _PREPARE_NO_DEPS_CMD=$TOUCH_UNIT \
 
 module _PREPARE_DEPS_BASE: _BARE_UNIT {
     .CMD=_PREPARE_DEPS_CMD
-    .IGNORED=SRCS FILES TS_FILES COPY_FILE
+    .IGNORED=SRCS FILES TS_FILES COPY_FILE RUN_JAVASCRIPT_AFTER_BUILD
     # Propagates peers to related modules
     .PEERDIR_POLICY=as_build_from
     .NODE_TYPE=Bundle

build/conf/ts/ts.conf

@@ -1,6 +1,7 @@
 NODEJS_ROOT=
 NODEJS_BIN=$NODEJS_ROOT/node
 
+_TS_REQUIRED_MISSING=~~required~~
 TS_TRACE=no
 TS_LOG=no
 TS_LOCAL_CLI=no
@@ -49,18 +50,23 @@ when($AUTOCHECK == "yes") {
   NOTS_TOOL_BUILD_REQUIREMENTS__NO_UID__=${hide;requirements:"cpu:4"}
 }
 
+
+_RUN_JAVASCRIPT_AFTER_BUILD_ARGS=
+_RUN_JAVASCRIPT_AFTER_BUILD_INPUTS=
+
 # Arguments for builders' commands, passed after the command
+# TODO FBP-1978 (remove output.tar output)
 NOTS_TOOL_COMMON_BUILDER_ARGS=\
-  --output-file ${output:TS_OUTPUT_FILE} ${hide;output:TS_OUTPUT_FILE_UUID} \
+  --output-file ${output:TARGET} ${hide;output:"output.tar"} ${hide;output:"output.tar.uuid"} \
   --tsconfigs $TS_CONFIG_PATH \
   --vcs-info "${VCS_INFO_FILE}" \
   $NOTS_TOOL_BUILD_REQUIREMENTS__NO_UID__ \
-  $NOTS_TOOL_BUILD_ENV
+  $NOTS_TOOL_BUILD_ENV \
+  $_RUN_JAVASCRIPT_AFTER_BUILD_ARGS \
+  $_RENDER_RUN_JAVASCRIPT_AFTER_BUILD_INPUTS($_RUN_JAVASCRIPT_AFTER_BUILD_INPUTS)
 
 ERM_PACKAGES_PATH=devtools/frontend_build_platform/erm/erm-packages.json
 
-TS_OUTPUT_FILE=output.tar
-TS_OUTPUT_FILE_UUID=output.tar.uuid
 TS_EXCLUDE_DIR_GLOB=(.idea|.vscode|node_modules)/**/*
 TS_COMMON_OUTDIR_GLOB=(build|dist|bundle|\${join=|:WEBPACK_OUTPUT_DIR}|$TS_NEXT_OUTPUT_DIR|$VITE_OUTPUT_DIR)/**/*
 
@@ -95,8 +101,8 @@ module _TS_BASE_UNIT: _BARE_UNIT {
     SET(MODULE_LANG TS)
     # TS should peer to TS
     SET(PEERDIR_TAGS TS TS_PROTO TS_PROTO_FROM_SCHEMA)
-    # .fake tells builder to not materialize it in results
-    SET(MODULE_SUFFIX .ts.fake)
+
+    SET(MODULE_SUFFIX .output.tar)
 
     # We read erm-packages.json during configuration, so we have to include it to configuration cache key
     SET_APPEND(_MAKEFILE_INCLUDE_LIKE_DEPS ${ARCADIA_ROOT}/$ERM_PACKAGES_PATH)
@@ -206,12 +212,27 @@ macro TS_FILES_GLOB(Glob...) {
 ### Then file will be copied to DESTINATION folder preserving file structure.
 ### Copied file becomes output of TS_PACKAGE
 ### Documentation: https://docs.yandex-team.ru/frontend-in-arcadia/references/TS_PACKAGE#ts-large-files
-macro TS_LARGE_FILES(DESTINATION="~~required~~", FILES...) {
+macro TS_LARGE_FILES(DESTINATION="$_TS_REQUIRED_MISSING", FILES...) {
     LARGE_FILES($FILES)
     # TODO: FBP-1795
     _TS_LARGE_FILES($DESTINATION $FILES)
 }
 
+
+macro RUN_JAVASCRIPT_AFTER_BUILD(JS_SCRIPT, OUTDIR="", IN{input}[], ARGS...) {
+  # --after-build-args uses `arg=val` format because args can start with --, so value will be parsed as another key.
+  # With `arg=val` format parser knows that after `=` is a value of the --after-build-args.
+  # For --after-build-outdir it is used to handle possible empty string in OUTDIR
+  SET(_RUN_JAVASCRIPT_AFTER_BUILD_ARGS --after-build-js $JS_SCRIPT  "--after-build-args=${join="<~~~>":ARGS}" "--after-build-outdir=$OUTDIR")
+  SET(_RUN_JAVASCRIPT_AFTER_BUILD_INPUTS ${IN})
+  _RUN_JAVASCRIPT_AFTER_BUILD_ADD_JS_SCRIPT_AS_INPUT($JS_SCRIPT)
+}
+
+macro _RENDER_RUN_JAVASCRIPT_AFTER_BUILD_INPUTS(IN{input}[]) {
+  .CMD=${hide;input:IN}
+}
+
+
 @import "${CONF_ROOT}/conf/ts/node_modules.conf"
 @import "${CONF_ROOT}/conf/ts/ts_next.conf"
 @import "${CONF_ROOT}/conf/ts/ts_package.conf"

build/conf/ts/ts_next.conf

@@ -30,8 +30,7 @@ macro TS_NEXT_EXPERIMENTAL_COMPILE() {
     SET(TS_NEXT_COMMAND experimental-compile)
 }
 
-TS_NEXT_CMD=$TOUCH_UNIT \
-    && $_TS_FILES_COPY_CMD \
+TS_NEXT_CMD=$_TS_FILES_COPY_CMD \
     && $ADD_VCS_INFO_FILE_CMD \
     && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-next $NOTS_TOOL_COMMON_BUILDER_ARGS \
       --ts-next-command ${TS_NEXT_COMMAND} \
@@ -46,7 +45,7 @@ TS_NEXT_CMD=$TOUCH_UNIT \
 ###
 ### NextJS app, built with `next build`. Requires sources to be under /src folder.
 ### /pages and /app on the root level ar not supported.
-### Build results are output.tar.
+### Build results are <module_name>.output.tar.
 ###
 ### Documentation: https://docs.yandex-team.ru/frontend-in-arcadia/references/TS_NEXT
 ###

build/conf/ts/ts_package.conf

@@ -32,6 +32,8 @@ multimodule TS_PACKAGE {
         .CMD=TS_PACK
         .EPILOGUE=_TS_PACKAGE_EPILOGUE
         .ALLOWED=TS_FILES TS_FILES_GLOB
+        # TODO: FBP-2029
+        .RESTRICTED=RUN_JAVASCRIPT_AFTER_BUILD
         .ALIASES=FILES=TS_FILES SRCS=TS_FILES
         .PEERDIRSELF=TS_PREPARE_DEPS
 
@@ -45,6 +47,9 @@ multimodule TS_PACKAGE {
     }
 
     module TS_PREPARE_DEPS: _PREPARE_DEPS_BASE {
-
+        # TODO: FBP-2029
+        # You need to restricts macro in all submodules of multimodule
+        # otherwise .RESTRICTED will behave as .IGNORED
+        .RESTRICTED=RUN_JAVASCRIPT_AFTER_BUILD
     }
 }

build/conf/ts/ts_tsc.conf

@@ -1,5 +1,4 @@
-TS_TSC_CMD=$TOUCH_UNIT \
-    && $_TS_FILES_COPY_CMD \
+TS_TSC_CMD=$_TS_FILES_COPY_CMD \
     && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-tsc $NOTS_TOOL_COMMON_BUILDER_ARGS \
     $_NODE_MODULES_INOUTS ${hide:PEERS} \
     ${hide;input:"package.json"} ${TS_CONFIG_FILES} $_AS_HIDDEN_INPUTS(IN $TS_INPUT_FILES) \

build/conf/ts/ts_vite.conf

@@ -33,8 +33,7 @@ macro VITE_OUTPUT(DirName) {
 }
 
 
-TS_VITE_CMD=$TOUCH_UNIT \
-    && $_TS_FILES_COPY_CMD \
+TS_VITE_CMD=$_TS_FILES_COPY_CMD \
     && $ADD_VCS_INFO_FILE_CMD \
     && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-vite $NOTS_TOOL_COMMON_BUILDER_ARGS \
       --bundler-config-path ${input:VITE_CONFIG_PATH} \
@@ -47,7 +46,7 @@ TS_VITE_CMD=$TOUCH_UNIT \
 ### @usage: TS_VITE([name])
 ###
 ### The Vite bundle, bundles JavaScript code.
-### Build results are packed as `output.tar`.
+### Build results are packed as `<module_name>.output.tar`.
 ###
 ### Documentation: https://docs.yandex-team.ru/frontend-in-arcadia/references/TS_VITE
 ###

build/conf/ts/ts_webpack.conf

@@ -30,8 +30,7 @@ macro WEBPACK_OUTPUT(FirstDirName, DirNames...) {
     SET(WEBPACK_OUTPUT_DIR $FirstDirName $DirNames)
 }
 
-TS_WEBPACK_CMD=$TOUCH_UNIT \
-    && $_TS_FILES_COPY_CMD \
+TS_WEBPACK_CMD=$_TS_FILES_COPY_CMD \
     && $ADD_VCS_INFO_FILE_CMD \
     && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-webpack $NOTS_TOOL_COMMON_BUILDER_ARGS \
       --bundler-config-path ${input:WEBPACK_CONFIG_PATH} \
@@ -44,7 +43,7 @@ TS_WEBPACK_CMD=$TOUCH_UNIT \
 ### @usage: TS_WEBPACK([name])
 ###
 ### The Webpack bundle, bundles JavaScript code.
-### Build results are packed as `output.tar`.
+### Build results are packed as `<module_name>.output.tar`.
 ###
 ### Documentation: https://docs.yandex-team.ru/frontend-in-arcadia/references/TS_WEBPACK
 ###

build/mapping.conf.json

@@ -491,6 +491,7 @@
         "7854653248": "https://devtools-registry.s3.yandex.net/7854653248",
         "7903404398": "https://devtools-registry.s3.yandex.net/7903404398",
         "7914217459": "https://devtools-registry.s3.yandex.net/7914217459",
+        "7948644946": "https://devtools-registry.s3.yandex.net/7948644946",
         "5486731632": "https://devtools-registry.s3.yandex.net/5486731632",
         "5514350352": "https://devtools-registry.s3.yandex.net/5514350352",
         "5514360398": "https://devtools-registry.s3.yandex.net/5514360398",
@@ -1702,6 +1703,7 @@
         "7854653248": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
         "7903404398": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
         "7914217459": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
+        "7948644946": "devtools/ya/test/programs/test_tool/bin/test_tool for linux",
         "5486731632": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",
         "5514350352": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",
         "5514360398": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux",

build/platform/test_tool/host.ya.make.inc

@@ -1,12 +1,12 @@
 IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7914244334)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7948640341)
 ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7914243379)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7948639335)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7914245698)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7948643524)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7914242140)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7948638449)
 ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7914244981)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7948641537)
 
 ENDIF()

build/platform/test_tool/host_os.ya.make.inc

@@ -1,12 +1,12 @@
 IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7914215842)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7948642784)
 ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7914215284)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7948641644)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7914217459)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7948644946)
 ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7914214265)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7948640691)
 ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64)
-    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7914216562)
+    DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:7948644187)
 
 ENDIF()

build/plugins/lib/nots/package_manager/base/constants.py

@@ -1,9 +1,8 @@
 BUILD_DIRNAME = "build"
 BUNDLE_DIRNAME = "bundle"
-NODE_MODULES_BUNDLE_FILENAME = "node_modules.tar"
 NODE_MODULES_DIRNAME = "node_modules"
 NODE_MODULES_WORKSPACE_BUNDLE_FILENAME = "workspace_node_modules.tar"
 NPM_REGISTRY_URL = "http://npm.yandex-team.ru"
-OUTPUT_TAR_FILENAME = "output.tar"
+OUTPUT_TAR_UUID_FILENAME = "output.tar.uuid"
 PACKAGE_JSON_FILENAME = "package.json"
 PNPM_LOCKFILE_FILENAME = "pnpm-lock.yaml"

build/plugins/nots.py

@@ -985,3 +985,12 @@ def on_depends_on_mod(unit: NotsUnitType) -> None:
     if unit.get("_TS_TEST_DEPENDS_ON_BUILD"):
         for_mod_path = unit.get("TS_TEST_FOR_PATH")
         unit.ondepends([for_mod_path])
+
+
+@_with_report_configure_error
+def on_run_javascript_after_build_add_js_script_as_input(unit: NotsUnitType, js_script: str) -> None:
+    js_script = os.path.normpath(js_script)
+    if js_script.startswith("node_modules/"):
+        return
+
+    __set_append(unit, "_RUN_JAVASCRIPT_AFTER_BUILD_INPUTS", js_script)