merge to stable-23-3 (#452)

* issue-324: DescribeData: 1. BlobOffsets should be in bytes, not blocks 2. returning FileSize 3. added a ut that checks that we don't return data outside of the aligned superrange that contains the requested range (#399)

* issue #324: storing NProto::TFileStore in session state to use it in the two stage read implementation in TServiceActor (#400)

* issue #324: DescribeData: offset calculation shouldn't use TBlock::BlockIndex since this block might be not initialized (#402)

* NBSNEBIUS-101: use vhost-side reads WIP. Issue: #95 (#394)

* [Draft] NBSNEBIUS-101: use vhost-side reads

* WIP: service_actor_readdata

* add describe data test

* add describe data test

* add ut + fix readblob implementation

* enable TwoStageReadEnabled feature-flag

* fix ut + trigger large tests

---------

Co-authored-by: Maxim Deb Natkh <debnatkh@yandex.ru>

* issue-324: proper EvGet error handling, not outputting user data to logs, dependencies cleanup, code cleanup (#419)

* issue-324: proper EvGet error handling, not outputting user data to logs, dependencies cleanup, code cleanup

* issue-324: proper EvGet error handling, not outputting user data to logs, dependencies cleanup, code cleanup - forgot libs/storage/model/ut

* issue-324: proper EvGet error handling, not outputting user data to logs, dependencies cleanup, code cleanup - forgot libs/storage/model/public.h

* issue-324: proper EvGet error handling, not outputting user data to logs, dependencies cleanup, code cleanup - fixed event-log lib

* issue-324: TServiceActor: ReadData fallback in case DescribeData or EvGet fail (#428)

* issue-324: TServiceActor: ReadData fallback in case DescribeData or EvGet fail

* issue-324: TServiceActor: ReadData fallback in case DescribeData or EvGet fail

* issue-324: TServiceActor: ReadData fallback in case DescribeData or EvGet fail - discarding EvGetResults after switching to the ReadData fallback

* issue-324: TServiceActor: ReadData fallback in case DescribeData or EvGet fail - cleanup

* issue-324: added ut and loadtest for the TwoStageRead feature, fixed a crash in TReadDataActor::HandleReadBlobResponse (#441)

* issue-324: added ut and loadtest for the TwoStageRead feature, fixed a crash in TReadDataActor::HandleReadBlobResponse

* issue-324: added ut and loadtest for the TwoStageRead feature, fixed a crash in TReadDataActor::HandleReadBlobResponse - forgot nfs-storage.txt

* updated CMakeLists after cherry-pick

* updated CMakeLists after cherry-pick - forgot to add new CMakeLists

* contrib/ydb -> ydb

* contrib/ydb -> ydb

---------

Co-authored-by: Maxim Deb Natkh <debnatkh@gmail.com>
Co-authored-by: Maxim Deb Natkh <debnatkh@yandex.ru>

cloud/filestore/libs/storage/api/service.h

@@ -17,7 +17,7 @@ namespace NCloud::NFileStore::NStorage {
 #define FILESTORE_SERVICE_REQUESTS(xxx, ...)                                   \
 // FILESTORE_SERVICE_REQUESTS
 
-#define FILESTORE_SERVICE_REQUESTS_FWD(xxx, ...)                               \
+#define FILESTORE_SERVICE_REQUESTS_HANDLE(xxx, ...)                            \
     xxx(AddClusterNode,                     __VA_ARGS__)                       \
     xxx(RemoveClusterNode,                  __VA_ARGS__)                       \
     xxx(ListClusterNodes,                   __VA_ARGS__)                       \
@@ -55,9 +55,17 @@ namespace NCloud::NFileStore::NStorage {
     xxx(ReleaseLock,                        __VA_ARGS__)                       \
     xxx(TestLock,                           __VA_ARGS__)                       \
                                                                                \
-    xxx(ReadData,                           __VA_ARGS__)                       \
     xxx(WriteData,                          __VA_ARGS__)                       \
     xxx(AllocateData,                       __VA_ARGS__)                       \
+// FILESTORE_SERVICE_REQUESTS_HANDLE
+
+#define FILESTORE_SERVICE_REQUESTS_NO_HANDLE(xxx, ...)                         \
+    xxx(ReadData,                           __VA_ARGS__)                       \
+// FILESTORE_SERVICE_REQUESTS_NO_HANDLE
+
+#define FILESTORE_SERVICE_REQUESTS_FWD(xxx, ...)                               \
+    FILESTORE_SERVICE_REQUESTS_NO_HANDLE(xxx,   __VA_ARGS__)                   \
+    FILESTORE_SERVICE_REQUESTS_HANDLE(xxx,      __VA_ARGS__)                   \
 // FILESTORE_SERVICE_REQUESTS_FWD
 
 ////////////////////////////////////////////////////////////////////////////////

cloud/filestore/libs/storage/core/CMakeLists.linux-x86_64.txt

@@ -36,5 +36,4 @@ target_sources(filestore-libs-storage-core PRIVATE
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/core/tablet.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/core/tablet_counters.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/core/tablet_schema.cpp
-  ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/core/utils.cpp
 )

cloud/filestore/libs/storage/core/ya.make

@@ -9,7 +9,6 @@ SRCS(
     tablet.cpp
     tablet_counters.cpp
     tablet_schema.cpp
-    utils.cpp
 )
 
 PEERDIR(

cloud/filestore/libs/storage/model/CMakeLists.linux-x86_64.txt

@@ -5,6 +5,7 @@
 # ya.make files by maintainers. Any complex modifications which can't be ported back to the
 # original buildsystem will not be accepted.
 
+add_subdirectory(ut)
 
 get_built_tool_path(
   TOOL_enum_parser_bin
@@ -21,7 +22,10 @@ target_link_libraries(filestore-libs-storage-model PUBLIC
   tools-enum_parser-enum_serialization_runtime
 )
 target_sources(filestore-libs-storage-model PRIVATE
+  ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/model/block_buffer.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/model/channel_data_kind.cpp
+  ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/model/range.cpp
+  ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/model/utils.cpp
 )
 generate_enum_serilization(filestore-libs-storage-model
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/model/channel_data_kind.h

cloud/filestore/libs/storage/tablet/model/block_buffer.cpp → cloud/filestore/libs/storage/model/block_buffer.cpp

@@ -1,6 +1,6 @@
 #include "block_buffer.h"
 
-#include <cloud/filestore/libs/storage/core/utils.h>
+#include <cloud/filestore/libs/storage/model/utils.h>
 
 namespace NCloud::NFileStore::NStorage {
 

cloud/filestore/libs/storage/model/public.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <util/system/defaults.h>
+
+#include <memory>
+
+namespace NCloud::NFileStore::NStorage {
+
+////////////////////////////////////////////////////////////////////////////////
+
+struct IBlockBuffer;
+using IBlockBufferPtr = std::shared_ptr<IBlockBuffer>;
+
+}   // namespace NCloud::NFileStore::NStorage

cloud/filestore/libs/storage/tablet/model/range.h → cloud/filestore/libs/storage/model/range.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#include "public.h"
-
 #include <util/generic/size_literals.h>
 #include <util/string/builder.h>
 #include <util/system/align.h>
@@ -146,6 +144,12 @@ struct TByteRange
     {
         return {blockIndex * blockSize, blockSize, blockSize};
     }
+
+    bool operator==(const TByteRange& other) const
+    {
+        return std::tie(Offset, Length, BlockSize) ==
+               std::tie(other.Offset, other.Length, other.BlockSize);
+    }
 };
 
 }   // namespace NCloud::NFileStore::NStorage

cloud/filestore/libs/storage/model/ut/CMakeLists.linux-x86_64.txt

@@ -0,0 +1,80 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(cloud-filestore-libs-storage-model-ut)
+target_include_directories(cloud-filestore-libs-storage-model-ut PRIVATE
+  ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/model
+)
+target_link_libraries(cloud-filestore-libs-storage-model-ut PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  library-cpp-cpuid_check
+  cpp-testing-unittest_main
+  libs-storage-model
+)
+target_link_options(cloud-filestore-libs-storage-model-ut PRIVATE
+  -ldl
+  -lrt
+  -Wl,--no-as-needed
+  -fPIC
+  -fPIC
+  -lpthread
+  -lrt
+  -ldl
+)
+target_sources(cloud-filestore-libs-storage-model-ut PRIVATE
+  ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/model/block_buffer_ut.cpp
+  ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/model/range_ut.cpp
+)
+set_property(
+  TARGET
+  cloud-filestore-libs-storage-model-ut
+  PROPERTY
+  SPLIT_FACTOR
+  15
+)
+add_yunittest(
+  NAME
+  cloud-filestore-libs-storage-model-ut
+  TEST_TARGET
+  cloud-filestore-libs-storage-model-ut
+  TEST_ARG
+  --print-before-suite
+  --print-before-test
+  --fork-tests
+  --print-times
+  --show-fails
+)
+set_yunittest_property(
+  TEST
+  cloud-filestore-libs-storage-model-ut
+  PROPERTY
+  LABELS
+  SMALL
+)
+set_yunittest_property(
+  TEST
+  cloud-filestore-libs-storage-model-ut
+  PROPERTY
+  PROCESSORS
+  2
+)
+set_yunittest_property(
+  TEST
+  cloud-filestore-libs-storage-model-ut
+  PROPERTY
+  TIMEOUT
+  60
+)
+target_allocator(cloud-filestore-libs-storage-model-ut
+  cpp-malloc-tcmalloc
+  libs-tcmalloc-no_percpu_cache
+)
+vcs_info(cloud-filestore-libs-storage-model-ut)

cloud/filestore/libs/storage/model/ut/CMakeLists.txt

@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+  include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+  include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-x86_64.txt)
+endif()

cloud/filestore/libs/storage/model/ut/ya.make

@@ -0,0 +1,12 @@
+UNITTEST_FOR(cloud/filestore/libs/storage/model)
+
+INCLUDE(${ARCADIA_ROOT}/cloud/filestore/tests/recipes/small.inc)
+
+SRCS(
+    block_buffer_ut.cpp
+    range_ut.cpp
+)
+
+END()
+
+RECURSE_FOR_TESTS(ut)

cloud/filestore/libs/storage/core/utils.h → cloud/filestore/libs/storage/model/utils.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#include "public.h"
-
 #include <util/generic/bitops.h>
 
 namespace NCloud::NFileStore::NStorage {

cloud/filestore/libs/storage/model/ya.make

@@ -7,7 +7,9 @@ GENERATE_ENUM_SERIALIZATION(
 )
 
 SRCS(
+    block_buffer.cpp
     channel_data_kind.cpp
+    utils.cpp
 )
 
 END()

cloud/filestore/libs/storage/service/CMakeLists.linux-x86_64.txt

@@ -51,6 +51,7 @@ target_sources(filestore-libs-storage-service PRIVATE
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_monitoring_search.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_ping.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_pingsession.cpp
+  ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_readdata.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_statfs.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_actor_update_stats.cpp
   ${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/service/service_state.cpp

cloud/filestore/libs/storage/service/service_actor.h

@@ -11,7 +11,7 @@
 #include <cloud/filestore/libs/storage/api/service.h>
 #include <cloud/filestore/libs/storage/core/config.h>
 #include <cloud/filestore/libs/storage/core/request_info.h>
-#include <cloud/filestore/libs/storage/core/utils.h>
+#include <cloud/filestore/libs/storage/model/utils.h>
 
 #include <cloud/storage/core/libs/diagnostics/public.h>
 

cloud/filestore/libs/storage/service/service_actor_createsession.cpp

@@ -792,21 +792,26 @@ void TStorageServiceActor::HandleSessionCreated(
                 msg->FileStore.GetFileSystemId(),
                 msg->ClientId);
 
+            const auto mediaKind = static_cast<NProto::EStorageMediaKind>(
+                msg->FileStore.GetStorageMediaKind());
+
             session = State->CreateSession(
                 msg->ClientId,
-                msg->FileStore.GetFileSystemId(),
+                msg->FileStore,
                 msg->SessionId,
                 msg->SessionState,
                 msg->SessionSeqNo,
                 msg->ReadOnly,
-                static_cast<NProto::EStorageMediaKind>(msg->FileStore.GetStorageMediaKind()),
+                mediaKind,
                 std::move(stats),
                 actorId,
                 msg->TabletId);
 
             Y_ABORT_UNLESS(session);
         } else {
-            session->UpdateSessionState(msg->SessionState);
+            session->UpdateSessionState(
+                msg->SessionState,
+                msg->FileStore);
             session->AddSubSession(msg->SessionSeqNo, msg->ReadOnly);
             session->SessionActor = actorId;
         }

cloud/filestore/libs/storage/service/service_actor_destroysession.cpp

@@ -226,7 +226,9 @@ void TStorageServiceActor::HandleDestroySession(
             "Another create or destroy request is in progress"));
     }
 
-    if (session->ClientId != clientId || session->FileSystemId != fsId) {
+    if (session->ClientId != clientId
+            || session->FileStore.GetFileSystemId() != fsId)
+    {
         return reply(ErrorInvalidSession(clientId, sessionId, seqNo));
     }
 
@@ -265,7 +267,7 @@ void TStorageServiceActor::HandleDestroySession(
         StorageConfig,
         std::move(requestInfo),
         session->ClientId,
-        session->FileSystemId,
+        session->FileStore.GetFileSystemId(),
         session->SessionId,
         seqNo);
 
@@ -281,7 +283,7 @@ void TStorageServiceActor::HandleSessionDestroyed(
     auto* session = State->FindSession(msg->SessionId, msg->SeqNo);
     if (session) {
         session->CreateDestroyState = ESessionCreateDestroyState::STATE_NONE;
-        const auto fsId = session->FileSystemId;
+        const auto fsId = session->FileStore.GetFileSystemId();
         const auto clientId = session->ClientId;
         if (!State->RemoveSession(msg->SessionId, msg->SeqNo)) {
             StatsRegistry->Unregister(fsId, clientId);

cloud/filestore/libs/storage/service/service_actor_forward.cpp

@@ -72,8 +72,16 @@ void TStorageServiceActor::ForwardRequest(
         ForwardRequest<ns::T##name##Method>(ctx, ev);                          \
     }                                                                          \
 
-    FILESTORE_SERVICE_REQUESTS_FWD(FILESTORE_FORWARD_REQUEST, TEvService)
+    FILESTORE_SERVICE_REQUESTS_HANDLE(FILESTORE_FORWARD_REQUEST, TEvService)
 
 #undef FILESTORE_FORWARD_REQUEST
 
+#define FILESTORE_DEFINE_NO_HANDLE_FORWARD(name, ns)                           \
+template void TStorageServiceActor::ForwardRequest<ns::T##name##Method>(       \
+    const TActorContext&, const ns::TEv##name##Request::TPtr&);                \
+
+    FILESTORE_SERVICE_REQUESTS_NO_HANDLE(FILESTORE_DEFINE_NO_HANDLE_FORWARD, TEvService)
+
+#undef FILESTORE_DEFINE_NO_HANDLE_FORWARD
+
 }   // namespace NCloud::NFileStore::NStorage

cloud/filestore/libs/storage/service/service_actor_monitoring.cpp

@@ -98,7 +98,7 @@ void TStorageServiceActor::RenderSessions(IOutputStream& out)
                         DumpFsLink(
                             out,
                             session.TabletId,
-                            session.FileSystemId
+                            session.FileStore.GetFileSystemId()
                         );
                     }
                     TABLED() { out << session.SessionId; }