Added tensor utils to Eager Dygraph (PaddlePaddle#37478)

* Added GradTensorHolder to Eager Dygraph

* Added accumulation codes to Eager Dygraph

* Added tensor utils to Eager Dygraph

* Resolve compilation issues

* Fixed issues

paddle/fluid/eager/api/utils/CMakeLists.txt

@@ -1 +1,2 @@
+cc_library(tensor_utils SRCS tensor_utils.cc DEPS pten pten_api autograd_meta grad_node_info accumulation_node)
 cc_library(global_utils SRCS global_utils.cc DEPS place)

paddle/fluid/eager/api/utils/tensor_utils.cc

@@ -0,0 +1,61 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle/fluid/eager/api/utils/tensor_utils.h"
+#include "paddle/fluid/eager/accumulation/accumulation_node.h"
+#include "paddle/fluid/eager/api/utils/global_utils.h"
+#include "paddle/fluid/eager/autograd_meta.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/fluid/eager/utils.h"
+
+#include "paddle/pten/api/all.h"
+
+#include "paddle/fluid/framework/data_layout.h"
+#include "paddle/fluid/framework/pten_utils.h"
+#include "paddle/fluid/framework/variable.h"
+
+namespace egr {
+
+bool IsLeafTensor(const egr::EagerTensor& target) {
+  std::shared_ptr<GradNodeBase> grad_node = EagerUtils::grad_node(target);
+  if (std::dynamic_pointer_cast<GradNodeAccumulation>(grad_node)) {
+    return true;
+  }
+
+  return false;
+}
+
+egr::EagerTensor CreateTensorWithValue(const pten::DDim& ddim,
+                                       const paddle::platform::Place& place,
+                                       const pten::DataType& dtype,
+                                       const pten::DataLayout& layout,
+                                       float value, bool is_leaf) {
+  paddle::experimental::Tensor tensor = paddle::experimental::full(
+      paddle::framework::vectorize(ddim), paddle::experimental::Scalar(value),
+      dtype, pten::TransToPtenBackend(place), layout);
+
+  egr::EagerTensor out = egr::EagerTensor();
+  out.set_tensor(std::make_shared<paddle::experimental::Tensor>(tensor));
+  auto meta = EagerUtils::autograd_meta(&out);
+
+  if (is_leaf) {
+    auto accumulation_node = std::make_shared<GradNodeAccumulation>();
+    meta->SetGradNode(accumulation_node);
+    meta->SetStopGradient(false);
+  }
+
+  return out;
+}
+
+}  // namespace egr

paddle/fluid/eager/api/utils/tensor_utils.h

@@ -0,0 +1,32 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/pten/api/all.h"
+
+namespace egr {
+
+// If and only if the tensor holds an AccumulationNode
+// Then it's treated as a leaf tensor
+bool IsLeafTensor(const egr::EagerTensor& target);
+
+egr::EagerTensor CreateTensorWithValue(const pten::DDim& ddim,
+                                       const paddle::platform::Place& place,
+                                       const pten::DataType& dtype,
+                                       const pten::DataLayout& layout,
+                                       float value, bool is_leaf = true);
+
+}  // namespace egr

paddle/fluid/eager/tests/CMakeLists.txt

@@ -1,3 +1,3 @@
-set(eager_deps pten pten_api utils global_utils pten_tensor autograd_meta grad_node_info grad_tensor_holder gradient_accumulation accumulation_node)
+set(eager_deps pten pten_api utils tensor_utils global_utils pten_tensor autograd_meta grad_node_info grad_tensor_holder gradient_accumulation accumulation_node)
 add_subdirectory(data_structure_tests)
 add_subdirectory(task_tests)

paddle/fluid/eager/tests/data_structure_tests/CMakeLists.txt

@@ -1,6 +1,6 @@
-cc_test(test_egr_ds_eager_tensor SRCS eager_tensor_test.cc DEPS ${eager_deps} )
+cc_test(test_egr_ds_eager_tensor SRCS eager_tensor_test.cc DEPS ${eager_deps})
 cc_test(test_egr_ds_auotgrad_meta SRCS autograd_meta_test.cc DEPS ${eager_deps})
 cc_test(test_egr_ds_grad_node_info SRCS grad_node_info_test.cc DEPS ${eager_deps})
-cc_test(test_egr_ds_tensor_wrapper SRCS tensor_wrapper_test.cc DEPS ${eager_deps})
 cc_test(test_egr_ds_grad_tensor_holder SRCS grad_tensor_holder_test.cc DEPS ${eager_deps})
 cc_test(test_egr_ds_accumulation_node SRCS accumulation_node_test.cc DEPS ${eager_deps})
+cc_test(test_egr_ds_tensor_wrapper SRCS tensor_wrapper_test.cc DEPS ${eager_deps})

paddle/fluid/eager/tests/data_structure_tests/accumulation_node_test.cc

@@ -42,6 +42,7 @@ TEST(AccumulationNode, EagerTensor) {
       std::make_shared<paddle::experimental::DefaultAllocator>(
           paddle::platform::CPUPlace()),
       meta);
+
   dt1->mutable_data<paddle::platform::float16>()[0] = 20.0;
   EagerTensor et1 = EagerTensor(dt1);
 

paddle/fluid/eager/tests/task_tests/CMakeLists.txt

@@ -1 +1,2 @@
+cc_test(test_egr_task_tensor_utils SRCS tensor_utils_test.cc DEPS ${eager_deps})
 cc_test(test_egr_task_eager_utils SRCS eager_utils_test.cc DEPS ${eager_deps})

paddle/fluid/eager/tests/task_tests/tensor_utils_test.cc

@@ -0,0 +1,61 @@
+// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sstream>
+
+#include "gtest/gtest.h"
+
+#include "paddle/fluid/eager/api/utils/tensor_utils.h"
+#include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/fluid/eager/grad_node_info.h"
+#include "paddle/fluid/eager/grad_tensor_holder.h"
+#include "paddle/fluid/eager/tests/test_utils.h"
+#include "paddle/pten/api/lib/utils/allocator.h"
+
+#include "paddle/pten/core/kernel_registry.h"
+
+// TODO(jiabin): remove nolint here!!!
+using namespace egr;  // NOLINT
+
+namespace eager_test {
+
+TEST(TensorUtils, Test) {
+  // Prepare Device Contexts
+  InitEnv(paddle::platform::CPUPlace());
+
+  // Prepare Inputs
+  std::vector<egr::EagerTensor> target_tensors;
+  paddle::framework::DDim ddim = paddle::framework::make_ddim({4, 16, 16, 32});
+
+  // Create Target Tensor
+  egr::EagerTensor t = CreateTensorWithValue(
+      ddim, paddle::platform::CPUPlace(), pten::DataType::FLOAT32,
+      pten::DataLayout::NCHW, 5.0 /*value*/, true /*is_leaf*/);
+
+  egr::EagerTensor t_grad = CreateTensorWithValue(
+      ddim, paddle::platform::CPUPlace(), pten::DataType::FLOAT32,
+      pten::DataLayout::NCHW, 1.0 /*value*/, false /*is_leaf*/);
+
+  CHECK_EQ(IsLeafTensor(t), true);
+
+  // Test Utils
+  CompareTensorWithValue<float>(t, 5.0);
+
+  egr::AutogradMeta* meta = egr::EagerUtils::autograd_meta(&t);
+  *meta->MutableGrad() = t_grad;
+
+  CompareGradTensorWithValue<float>(t, 1.0);
+}
+
+}  // namespace eager_test

paddle/fluid/eager/tests/test_utils.h

@@ -172,4 +172,5 @@ inline void InitEnv(paddle::platform::Place place) {
   // Init Tracer Place
   egr::Controller::Instance().SetExpectedPlace(place);
 }
+
 }  // namespace eager_test