From 153a6609841ac3e7b506a408298299c2ac7573af Mon Sep 17 00:00:00 2001 From: Matheus Marchini Date: Mon, 25 Dec 2017 22:17:25 -0200 Subject: [PATCH 1/2] src, test: node internals' postmortem metadata Before these changes, only V8 added postmortem metadata to Node's binary, limiting the possibilities for debugger's developers to add some features that rely on investigating Node's internal structures. These changes are first steps towards empowering debug tools to navigate Node's internal structures. One example of what can be achieved with this is shown at nodejs/llnode#122 (a command which prints information about handles and requests on the queue for a core dump file). Node postmortem metadata are prefixed with nodedbg_. This also adds tests to validate if all postmortem metadata are calculated correctly, plus some documentation on what is postmortem metadata and a few care to be taken to avoid breaking it. Ref: https://github.com/nodejs/llnode/pull/122 Ref: https://github.com/nodejs/post-mortem/issues/46 --- doc/guides/node-postmortem-support.md | 72 ++++++++++ node.gyp | 3 + src/base_object.h | 5 + src/env.h | 6 + src/handle_wrap.h | 5 + src/node_postmortem_metadata.cc | 118 +++++++++++++++ src/req_wrap.h | 10 +- src/util-inl.h | 12 +- test/cctest/node_test_fixture.h | 55 +++++++ test/cctest/test_environment.cc | 46 +----- test/cctest/test_node_postmortem_metadata.cc | 143 +++++++++++++++++++ 11 files changed, 427 insertions(+), 48 deletions(-) create mode 100644 doc/guides/node-postmortem-support.md create mode 100644 src/node_postmortem_metadata.cc create mode 100644 test/cctest/test_node_postmortem_metadata.cc diff --git a/doc/guides/node-postmortem-support.md b/doc/guides/node-postmortem-support.md new file mode 100644 index 00000000000000..e29d9ca3a1fdf0 --- /dev/null +++ b/doc/guides/node-postmortem-support.md @@ -0,0 +1,72 @@ +# Postmortem Support + +Postmortem metadata are constants present in the final build which can be used +by debuggers and other tools to navigate through internal structures of software +when analyzing its memory (either on a running process or a core dump). Node +provides this metadata in its builds for V8 and Node internal structures. + + +### V8 Postmortem metadata + +V8 prefixes all postmortem constants with `v8dbg_`, and they allow inspection of +objects on the heap as well as object properties and references. V8 generates +those symbols with a script (`deps/v8/tools/gen-postmortem-metadata.py`), and +Node always includes these constants in the final build. + +### Node Debug Symbols + +Node prefixes all postmortem constants with `nodedbg_`, and they complement V8 +constants by providing ways to inspect Node-specific structures, like +`node::Environment`, `node::BaseObject` and its descendants, classes from +`src/utils.h` and others. Those constants are declared in +`src/node_postmortem_metadata.cc`, and most of them are calculated at compile +time. + +#### Calculating offset of class members + +Node constants referring to the offset of class members in memory are calculated +at compile time. Because of that, those class members must be at a fixed offset +from the start of the class. That's not a problem in most cases, but it also +means that those members should always come after any templated member on the +class definition. + +For example, if we want to add a constant with the offset for +`ReqWrap::req_wrap_queue_`, it should be defined after `ReqWrap::req_`, because +`sizeof(req_)` depends on the type of T, which means the class definition should +be like this: + +```c++ +template +class ReqWrap : public AsyncWrap { + private: + // req_wrap_queue_ comes before any templated member, which places it in a + // fixed offset from the start of the class + ListNode req_wrap_queue_; + + T req_; +}; +``` + +instead of: + +```c++ +template +class ReqWrap : public AsyncWrap { + private: + T req_; + + // req_wrap_queue_ comes after a templated member, which means it won't be in + // a fixed offset from the start of the class + ListNode req_wrap_queue_; +}; +``` + +There are also tests on `test/cctest/test_node_postmortem_metadata.cc` to make +sure all Node postmortem metadata are calculated correctly. + +## Tools and References + +* [llnode](https://github.com/nodejs/llnode): LLDB plugin +* [`mdb_v8`](https://github.com/joyent/mdb_v8): mdb plugin +* [nodejs/post-mortem](https://github.com/nodejs/post-mortem): Node.js +post-mortem working group diff --git a/node.gyp b/node.gyp index d0c8f05e66a9ea..4e42b44d7f06e7 100644 --- a/node.gyp +++ b/node.gyp @@ -305,6 +305,7 @@ 'src/node_os.cc', 'src/node_platform.cc', 'src/node_perf.cc', + 'src/node_postmortem_metadata.cc', 'src/node_serdes.cc', 'src/node_trace_events.cc', 'src/node_url.cc', @@ -962,6 +963,7 @@ 'test/cctest/node_test_fixture.cc', 'test/cctest/test_aliased_buffer.cc', 'test/cctest/test_base64.cc', + 'test/cctest/test_node_postmortem_metadata.cc', 'test/cctest/test_environment.cc', 'test/cctest/test_util.cc', 'test/cctest/test_url.cc' @@ -969,6 +971,7 @@ 'libraries': [ '<(OBJ_PATH)<(OBJ_SEPARATOR)async_wrap.<(OBJ_SUFFIX)', + '<(OBJ_PATH)<(OBJ_SEPARATOR)handle_wrap.<(OBJ_SUFFIX)', '<(OBJ_PATH)<(OBJ_SEPARATOR)env.<(OBJ_SUFFIX)', '<(OBJ_PATH)<(OBJ_SEPARATOR)node.<(OBJ_SUFFIX)', '<(OBJ_PATH)<(OBJ_SEPARATOR)node_buffer.<(OBJ_SUFFIX)', diff --git a/src/base_object.h b/src/base_object.h index 0998920f49dd15..5852f764066fbc 100644 --- a/src/base_object.h +++ b/src/base_object.h @@ -65,6 +65,11 @@ class BaseObject { static inline void WeakCallback( const v8::WeakCallbackInfo& data); + // persistent_handle_ needs to be at a fixed offset from the start of the + // class because it is used by src/node_postmortem_metadata.cc to calculate + // offsets and generate debug symbols for BaseObject, which assumes that the + // position of members in memory are predictable. For more information please + // refer to `doc/guides/node-postmortem-support.md` v8::Persistent persistent_handle_; Environment* env_; }; diff --git a/src/env.h b/src/env.h index a1f505c4fe1da1..8e88c7dc5c20bf 100644 --- a/src/env.h +++ b/src/env.h @@ -765,6 +765,12 @@ class Environment { std::unique_ptr inspector_agent_; #endif + // handle_wrap_queue_ and req_wrap_queue_ needs to be at a fixed offset from + // the start of the class because it is used by + // src/node_postmortem_metadata.cc to calculate offsets and generate debug + // symbols for Environment, which assumes that the position of members in + // memory are predictable. For more information please refer to + // `doc/guides/node-postmortem-support.md` HandleWrapQueue handle_wrap_queue_; ReqWrapQueue req_wrap_queue_; ListHead&); static void OnClose(uv_handle_t* handle); + // handle_wrap_queue_ needs to be at a fixed offset from the start of the + // class because it is used by src/node_postmortem_metadata.cc to calculate + // offsets and generate debug symbols for HandleWrap, which assumes that the + // position of members in memory are predictable. For more information please + // refer to `doc/guides/node-postmortem-support.md` ListNode handle_wrap_queue_; enum { kInitialized, kClosing, kClosingWithCallback, kClosed } state_; uv_handle_t* const handle_; diff --git a/src/node_postmortem_metadata.cc b/src/node_postmortem_metadata.cc new file mode 100644 index 00000000000000..4a463958f54509 --- /dev/null +++ b/src/node_postmortem_metadata.cc @@ -0,0 +1,118 @@ +// Need to import standard headers before redefining private, otherwise it +// won't compile. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace node { +// Forward declaration needed before redefining private. +int GenDebugSymbols(); +} // namespace node + + +#define private friend int GenDebugSymbols(); private + +#include "env.h" +#include "base_object-inl.h" +#include "handle_wrap.h" +#include "util-inl.h" +#include "req_wrap.h" +#include "v8abbr.h" + +#define NODEDBG_SYMBOL(Name) nodedbg_ ## Name + +// nodedbg_offset_CLASS__MEMBER__TYPE: Describes the offset to a class member. +#define NODEDBG_OFFSET(Class, Member, Type) \ + NODEDBG_SYMBOL(offset_ ## Class ## __ ## Member ## __ ## Type) + +// These are the constants describing Node internal structures. Every constant +// should use the format described above. These constants are declared as +// global integers so that they'll be present in the generated node binary. They +// also need to be declared outside any namespace to avoid C++ name-mangling. +#define NODE_OFFSET_POSTMORTEM_METADATA(V) \ + V(BaseObject, persistent_handle_, v8_Persistent_v8_Object, \ + BaseObject::persistent_handle_) \ + V(Environment, handle_wrap_queue_, Environment_HandleWrapQueue, \ + Environment::handle_wrap_queue_) \ + V(Environment, req_wrap_queue_, Environment_ReqWrapQueue, \ + Environment::req_wrap_queue_) \ + V(HandleWrap, handle_wrap_queue_, ListNode_HandleWrap, \ + HandleWrap::handle_wrap_queue_) \ + V(Environment_HandleWrapQueue, head_, ListNode_HandleWrap, \ + Environment::HandleWrapQueue::head_) \ + V(ListNode_HandleWrap, next_, uintptr_t, ListNode::next_) \ + V(ReqWrap, req_wrap_queue_, ListNode_ReqWrapQueue, \ + ReqWrap::req_wrap_queue_) \ + V(Environment_ReqWrapQueue, head_, ListNode_ReqWrapQueue, \ + Environment::ReqWrapQueue::head_) \ + V(ListNode_ReqWrap, next_, uintptr_t, ListNode>::next_) + +extern "C" { +int nodedbg_const_Environment__kContextEmbedderDataIndex__int; +uintptr_t nodedbg_offset_ExternalString__data__uintptr_t; + +#define V(Class, Member, Type, Accessor) \ + NODE_EXTERN uintptr_t NODEDBG_OFFSET(Class, Member, Type); + NODE_OFFSET_POSTMORTEM_METADATA(V) +#undef V +} + +namespace node { + +int GenDebugSymbols() { + nodedbg_const_Environment__kContextEmbedderDataIndex__int = + Environment::kContextEmbedderDataIndex; + + nodedbg_offset_ExternalString__data__uintptr_t = NODE_OFF_EXTSTR_DATA; + + #define V(Class, Member, Type, Accessor) \ + NODEDBG_OFFSET(Class, Member, Type) = OffsetOf(&Accessor); + NODE_OFFSET_POSTMORTEM_METADATA(V) + #undef V + + return 1; +} + +int debug_symbols_generated = GenDebugSymbols(); + +} // namespace node diff --git a/src/req_wrap.h b/src/req_wrap.h index 83baf9d2a35285..05bc558570abf1 100644 --- a/src/req_wrap.h +++ b/src/req_wrap.h @@ -27,9 +27,13 @@ class ReqWrap : public AsyncWrap { protected: // req_wrap_queue_ needs to be at a fixed offset from the start of the class // because it is used by ContainerOf to calculate the address of the embedding - // ReqWrap. ContainerOf compiles down to simple, fixed pointer arithmetic. - // sizeof(req_) depends on the type of T, so req_wrap_queue_ would - // no longer be at a fixed offset if it came after req_. + // ReqWrap. ContainerOf compiles down to simple, fixed pointer arithmetic. It + // is also used by src/node_postmortem_metadata.cc to calculate offsets and + // generate debug symbols for ReqWrap, which assumes that the position of + // members in memory are predictable. sizeof(req_) depends on the type of T, + // so req_wrap_queue_ would no longer be at a fixed offset if it came after + // req_. For more information please refer to + // `doc/guides/node-postmortem-support.md` T req_; }; diff --git a/src/util-inl.h b/src/util-inl.h index 558a0ab2b42611..c5a25c91ffb088 100644 --- a/src/util-inl.h +++ b/src/util-inl.h @@ -141,13 +141,17 @@ typename ListHead::Iterator ListHead::end() const { return Iterator(const_cast*>(&head_)); } +template +constexpr uintptr_t OffsetOf(Inner Outer::*field) { + return reinterpret_cast(&(static_cast(0)->*field)); +} + template ContainerOfHelper::ContainerOfHelper(Inner Outer::*field, Inner* pointer) - : pointer_(reinterpret_cast( - reinterpret_cast(pointer) - - reinterpret_cast(&(static_cast(0)->*field)))) { -} + : pointer_( + reinterpret_cast( + reinterpret_cast(pointer) - OffsetOf(field))) {} template template diff --git a/test/cctest/node_test_fixture.h b/test/cctest/node_test_fixture.h index d5be613c0bd105..ffecd34d71ba40 100644 --- a/test/cctest/node_test_fixture.h +++ b/test/cctest/node_test_fixture.h @@ -5,6 +5,7 @@ #include "gtest/gtest.h" #include "node.h" #include "node_platform.h" +#include "node_internals.h" #include "env.h" #include "v8.h" #include "libplatform/libplatform.h" @@ -75,6 +76,13 @@ class NodeTestFixture : public ::testing::Test { v8::Isolate::CreateParams params_; params_.array_buffer_allocator = allocator_.get(); isolate_ = v8::Isolate::New(params_); + + // As the TracingController is stored globally, we only need to create it + // one time for all tests. + if (node::tracing::TraceEventHelper::GetTracingController() == nullptr) { + node::tracing::TraceEventHelper::SetTracingController( + new v8::TracingController()); + } } virtual void TearDown() { @@ -95,4 +103,51 @@ class NodeTestFixture : public ::testing::Test { v8::ArrayBuffer::Allocator::NewDefaultAllocator()}; }; + +class EnvironmentTestFixture : public NodeTestFixture { + public: + class Env { + public: + Env(const v8::HandleScope& handle_scope, + const Argv& argv, + NodeTestFixture* test_fixture) { + auto isolate = handle_scope.GetIsolate(); + context_ = node::NewContext(isolate); + CHECK(!context_.IsEmpty()); + context_->Enter(); + + isolate_data_ = node::CreateIsolateData(isolate, + NodeTestFixture::CurrentLoop(), + test_fixture->Platform()); + CHECK_NE(nullptr, isolate_data_); + environment_ = node::CreateEnvironment(isolate_data_, + context_, + 1, *argv, + argv.nr_args(), *argv); + CHECK_NE(nullptr, environment_); + } + + ~Env() { + environment_->CleanupHandles(); + node::FreeEnvironment(environment_); + node::FreeIsolateData(isolate_data_); + context_->Exit(); + } + + node::Environment* operator*() const { + return environment_; + } + + v8::Local context() const { + return context_; + } + + private: + v8::Local context_; + node::IsolateData* isolate_data_; + node::Environment* environment_; + DISALLOW_COPY_AND_ASSIGN(Env); + }; +}; + #endif // TEST_CCTEST_NODE_TEST_FIXTURE_H_ diff --git a/test/cctest/test_environment.cc b/test/cctest/test_environment.cc index 704efd7a88358f..352fed1fb62ed9 100644 --- a/test/cctest/test_environment.cc +++ b/test/cctest/test_environment.cc @@ -20,43 +20,7 @@ static void at_exit_callback1(void* arg); static void at_exit_callback2(void* arg); static std::string cb_1_arg; // NOLINT(runtime/string) -class EnvironmentTest : public NodeTestFixture { - public: - class Env { - public: - Env(const v8::HandleScope& handle_scope, - v8::Isolate* isolate, - const Argv& argv, - NodeTestFixture* test_fixture) { - context_ = v8::Context::New(isolate); - CHECK(!context_.IsEmpty()); - isolate_data_ = CreateIsolateData(isolate, - NodeTestFixture::CurrentLoop(), - test_fixture->Platform()); - CHECK_NE(nullptr, isolate_data_); - environment_ = CreateEnvironment(isolate_data_, - context_, - 1, *argv, - argv.nr_args(), *argv); - CHECK_NE(nullptr, environment_); - } - - ~Env() { - environment_->CleanupHandles(); - FreeEnvironment(environment_); - FreeIsolateData(isolate_data_); - } - - Environment* operator*() const { - return environment_; - } - - private: - v8::Local context_; - IsolateData* isolate_data_; - Environment* environment_; - }; - +class EnvironmentTest : public EnvironmentTestFixture { private: virtual void TearDown() { NodeTestFixture::TearDown(); @@ -68,7 +32,7 @@ class EnvironmentTest : public NodeTestFixture { TEST_F(EnvironmentTest, AtExitWithEnvironment) { const v8::HandleScope handle_scope(isolate_); const Argv argv; - Env env {handle_scope, isolate_, argv, this}; + Env env {handle_scope, argv, this}; AtExit(*env, at_exit_callback1); RunAtExit(*env); @@ -78,7 +42,7 @@ TEST_F(EnvironmentTest, AtExitWithEnvironment) { TEST_F(EnvironmentTest, AtExitWithArgument) { const v8::HandleScope handle_scope(isolate_); const Argv argv; - Env env {handle_scope, isolate_, argv, this}; + Env env {handle_scope, argv, this}; std::string arg{"some args"}; AtExit(*env, at_exit_callback1, static_cast(&arg)); @@ -89,8 +53,8 @@ TEST_F(EnvironmentTest, AtExitWithArgument) { TEST_F(EnvironmentTest, MultipleEnvironmentsPerIsolate) { const v8::HandleScope handle_scope(isolate_); const Argv argv; - Env env1 {handle_scope, isolate_, argv, this}; - Env env2 {handle_scope, isolate_, argv, this}; + Env env1 {handle_scope, argv, this}; + Env env2 {handle_scope, argv, this}; AtExit(*env1, at_exit_callback1); AtExit(*env2, at_exit_callback2); diff --git a/test/cctest/test_node_postmortem_metadata.cc b/test/cctest/test_node_postmortem_metadata.cc new file mode 100644 index 00000000000000..10deeb668dd8e9 --- /dev/null +++ b/test/cctest/test_node_postmortem_metadata.cc @@ -0,0 +1,143 @@ +#include "node_postmortem_metadata.cc" + +#include "gtest/gtest.h" +#include "node.h" +#include "node_internals.h" +#include "node_test_fixture.h" +#include "req_wrap-inl.h" +#include "tracing/agent.h" +#include "v8.h" + + +class DebugSymbolsTest : public EnvironmentTestFixture {}; + + +class TestHandleWrap : public node::HandleWrap { + public: + size_t self_size() const override { return sizeof(*this); } + + TestHandleWrap(node::Environment* env, + v8::Local object, + uv_tcp_t* handle) + : node::HandleWrap(env, + object, + reinterpret_cast(handle), + node::AsyncWrap::PROVIDER_TIMERWRAP) {} +}; + + +class TestReqWrap : public node::ReqWrap { + public: + size_t self_size() const override { return sizeof(*this); } + + TestReqWrap(node::Environment* env, v8::Local object) + : node::ReqWrap(env, + object, + node::AsyncWrap::PROVIDER_TIMERWRAP) {} +}; + +TEST_F(DebugSymbolsTest, ContextEmbedderDataIndex) { + int kContextEmbedderDataIndex = node::Environment::kContextEmbedderDataIndex; + EXPECT_EQ(nodedbg_const_Environment__kContextEmbedderDataIndex__int, + kContextEmbedderDataIndex); +} + +TEST_F(DebugSymbolsTest, ExternalStringDataOffset) { + EXPECT_EQ(nodedbg_offset_ExternalString__data__uintptr_t, + NODE_OFF_EXTSTR_DATA); +} + +TEST_F(DebugSymbolsTest, BaseObjectPersistentHandle) { + const v8::HandleScope handle_scope(isolate_); + const Argv argv; + Env env{handle_scope, argv, this}; + + v8::Local object = v8::Object::New(isolate_); + node::BaseObject obj(*env, object); + + auto expected = reinterpret_cast(&obj.persistent()); + auto calculated = reinterpret_cast(&obj) + + nodedbg_offset_BaseObject__persistent_handle___v8_Persistent_v8_Object; + EXPECT_EQ(expected, calculated); + + obj.persistent().Reset(); // ~BaseObject() expects an empty handle. +} + + +TEST_F(DebugSymbolsTest, EnvironmentHandleWrapQueue) { + const v8::HandleScope handle_scope(isolate_); + const Argv argv; + Env env{handle_scope, argv, this}; + + auto expected = reinterpret_cast((*env)->handle_wrap_queue()); + auto calculated = reinterpret_cast(*env) + + nodedbg_offset_Environment__handle_wrap_queue___Environment_HandleWrapQueue; // NOLINT(whitespace/line_length) + EXPECT_EQ(expected, calculated); +} + +TEST_F(DebugSymbolsTest, EnvironmentReqWrapQueue) { + const v8::HandleScope handle_scope(isolate_); + const Argv argv; + Env env{handle_scope, argv, this}; + + auto expected = reinterpret_cast((*env)->req_wrap_queue()); + auto calculated = reinterpret_cast(*env) + + nodedbg_offset_Environment__req_wrap_queue___Environment_ReqWrapQueue; + EXPECT_EQ(expected, calculated); +} + +TEST_F(DebugSymbolsTest, HandleWrapList) { + const v8::HandleScope handle_scope(isolate_); + const Argv argv; + Env env{handle_scope, argv, this}; + + uv_tcp_t handle; + + auto obj_template = v8::FunctionTemplate::New(isolate_); + obj_template->InstanceTemplate()->SetInternalFieldCount(1); + + v8::Local object = + obj_template->GetFunction()->NewInstance(env.context()).ToLocalChecked(); + TestHandleWrap obj(*env, object, &handle); + + auto queue = reinterpret_cast((*env)->handle_wrap_queue()); + auto head = queue + + nodedbg_offset_Environment_HandleWrapQueue__head___ListNode_HandleWrap; + auto next = + head + nodedbg_offset_ListNode_HandleWrap__next___uintptr_t; + next = *reinterpret_cast(next); + + auto expected = reinterpret_cast(&obj); + auto calculated = next - + nodedbg_offset_HandleWrap__handle_wrap_queue___ListNode_HandleWrap; + EXPECT_EQ(expected, calculated); + + obj.persistent().Reset(); // ~HandleWrap() expects an empty handle. +} + +TEST_F(DebugSymbolsTest, ReqWrapList) { + const v8::HandleScope handle_scope(isolate_); + const Argv argv; + Env env{handle_scope, argv, this}; + + auto obj_template = v8::FunctionTemplate::New(isolate_); + obj_template->InstanceTemplate()->SetInternalFieldCount(1); + + v8::Local object = + obj_template->GetFunction()->NewInstance(env.context()).ToLocalChecked(); + TestReqWrap obj(*env, object); + + auto queue = reinterpret_cast((*env)->req_wrap_queue()); + auto head = queue + + nodedbg_offset_Environment_ReqWrapQueue__head___ListNode_ReqWrapQueue; + auto next = + head + nodedbg_offset_ListNode_ReqWrap__next___uintptr_t; + next = *reinterpret_cast(next); + + auto expected = reinterpret_cast(&obj); + auto calculated = + next - nodedbg_offset_ReqWrap__req_wrap_queue___ListNode_ReqWrapQueue; + EXPECT_EQ(expected, calculated); + + obj.Dispatched(); +} From f798beb546f3e7c9a62019f96c955c266af655d1 Mon Sep 17 00:00:00 2001 From: Matheus Marchini Date: Fri, 19 Jan 2018 19:51:03 +0000 Subject: [PATCH 2/2] [squash] workaround to fix arm64 failing test --- test/cctest/test_node_postmortem_metadata.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/cctest/test_node_postmortem_metadata.cc b/test/cctest/test_node_postmortem_metadata.cc index 10deeb668dd8e9..9ba6e15593a019 100644 --- a/test/cctest/test_node_postmortem_metadata.cc +++ b/test/cctest/test_node_postmortem_metadata.cc @@ -127,6 +127,11 @@ TEST_F(DebugSymbolsTest, ReqWrapList) { obj_template->GetFunction()->NewInstance(env.context()).ToLocalChecked(); TestReqWrap obj(*env, object); + // NOTE (mmarchini): Workaround to fix failing tests on ARM64 machines with + // older GCC. Should be removed once we upgrade the GCC version used on our + // ARM64 CI machinies. + for (auto it : *(*env)->req_wrap_queue()) {} + auto queue = reinterpret_cast((*env)->req_wrap_queue()); auto head = queue + nodedbg_offset_Environment_ReqWrapQueue__head___ListNode_ReqWrapQueue;