Merge branch 'develop' into remove_deprecated_unittest_alias

paddle/fluid/eager/auto_code_generator/generate_file_structures.py

@@ -143,7 +143,9 @@ def GenerateFileStructureForIntermediateDygraph(eager_dir, split_count):
         for i in range(split_count):
             f.write("nodes" + str(i + 1) + ".cc ")
         f.write("${fluid_manual_nodes} DEPS ${eager_deps} ${fluid_deps})\n")
-        f.write("add_dependencies(dygraph_node copy_dygraph_node)\n")
+        f.write(
+            "add_dependencies(dygraph_node copy_dygraph_node copy_dygraph_forward_functions)\n"
+        )
 
     with open(forwards_level_cmakelist_path, "w") as f:
         f.write("add_custom_target(\n")
@@ -181,7 +183,7 @@ def GenerateFileStructureForIntermediateDygraph(eager_dir, split_count):
             "${fluid_manual_functions} DEPS ${eager_deps} ${fluid_deps} ${GLOB_OP_LIB} ${GLOB_OPERATOR_DEPS})\n"
         )
         f.write(
-            "add_dependencies(dygraph_function copy_dygraph_forward_functions)\n"
+            "add_dependencies(dygraph_function copy_dygraph_forward_functions copy_dygraph_node)\n"
         )
 
     with open(generated_level_cmakelist_path, "w") as f:

paddle/fluid/eager/to_static/run_program_op_func.h

@@ -60,11 +60,6 @@ inline void run_program_ad_func(
     std::vector<paddle::framework::Scope*>& step_scope,  // NOLINT
     std::vector<paddle::Tensor*>& dout,                  // NOLINT
     const paddle::framework::AttributeMap& attrs) {
-  VLOG(2) << "start run run_program";
-  // Call forward function
-  RunProgramAPI(x, params, out, step_scope, dout, attrs);
-  VLOG(2) << "start run run_program grad";
-
   // Prepare Autograd Meta
   auto deref_out = details::DereferenceTensors(out);
   std::vector<egr::AutogradMeta*> p_autograd_x =
@@ -78,6 +73,13 @@ inline void run_program_ad_func(
   bool require_any_grad = egr::EagerUtils::ComputeRequireGrad(
       trace_backward, &p_autograd_x, &p_autograd_params);
 
+  VLOG(2) << "start run run_program with require_any_grad = "
+          << require_any_grad;
+  // Call forward function
+  // if require_any_grad is False, don't save any middle vars.
+  RunProgramAPI(x, params, out, step_scope, dout, require_any_grad, attrs);
+  VLOG(2) << "start run run_program grad";
+
   if (require_any_grad) {
     egr::EagerUtils::PassStopGradient(false, &p_autograd_outs);
     // Create GradOpNode (1 means [out_grad], 2 means [x_grad, paramx_grad])

paddle/fluid/eager/to_static/run_program_op_node.h

@@ -283,6 +283,7 @@ inline void RunProgramAPI(
     std::vector<paddle::Tensor *> &out,                   // NOLINT
     std::vector<paddle::framework::Scope *> &step_scope,  // NOLINT
     std::vector<paddle::Tensor *> &dout,                  // NOLINT
+    bool require_any_grad,
     const paddle::framework::AttributeMap &attrs) {
   VLOG(2) << "RunProgramOpKernel Compute";
   // In the original run_program OP, the default value of the is_test
@@ -436,8 +437,10 @@ inline void RunProgramAPI(
 
     VLOG(3) << paddle::framework::GenScopeTreeDebugInfo(out_scope_vec->front());
 
-    if (is_test || !egr::Controller::Instance().HasGrad()) {
-      VLOG(4) << "is test, set this scope can reused";
+    if (is_test || !require_any_grad) {
+      VLOG(4) << "don't require any grad, set this scope can reused";
+      VLOG(4) << "is_test: " << is_test
+              << ", require_any_grad: " << require_any_grad;
       global_inner_scope->SetCanReuesd(true);
       details::GcScope(global_inner_scope);
     } else {
@@ -576,7 +579,8 @@ inline void RunProgramGradAPI(
                                                    *backward_global_block,
                                                    global_inner_scope);
     VLOG(4) << "after backward gc all vars";
-    global_inner_scope->SetCanReuesd(true);
+    global_inner_scope->SetCanReuesd(
+        false);  // can't reuse util call `~GradNodeRunProgram`
     details::GcScope(global_inner_scope);
   }
 }
@@ -586,7 +590,15 @@ class GradNodeRunProgram : public egr::GradNodeBase {
   GradNodeRunProgram(size_t bwd_in_slot_num, size_t bwd_out_slot_num)
       : egr::GradNodeBase(bwd_in_slot_num, bwd_out_slot_num) {}
 
-  ~GradNodeRunProgram() override = default;
+  ~GradNodeRunProgram() {
+    auto *out_scope_vec = &step_scope_;
+    // Normally out_scope_vec.size() == 1. for safty, we add for-loop here.
+    for (size_t i = 0; i < out_scope_vec->size(); ++i) {
+      paddle::framework::Scope *global_inner_scope = out_scope_vec->at(i);
+      global_inner_scope->SetCanReuesd(true);  // set this to reuse scope.
+      details::GcScope(global_inner_scope);
+    }
+  }
   // Functor: perform backward computations
   virtual paddle::small_vector<std::vector<paddle::Tensor>,
                                egr::kSlotSmallVectorSize>

paddle/fluid/prim/api/api.yaml

@@ -2,6 +2,12 @@
 - subtract
 - multiply
 - divide
+- less_equal
+- less_than
+- equal
+- not_equal
+- greater_equal
+- greater_than
 - bitwise_and
 - bitwise_not
 - bitwise_or
@@ -33,3 +39,4 @@
 - put_along_axis
 - greater_than
 - less_equal
+- where

paddle/fluid/prim/api/composite_backward/composite_backward_api.h

@@ -30,6 +30,18 @@ using Tensor = paddle::Tensor;
 using IntArray = paddle::experimental::IntArrayBase<paddle::Tensor>;
 //  This function should have as same signature as phi, which defined in
 //  paddle/phi/api/backward/backward_api.h
+template <typename T>
+void relu_grad(const Tensor& out, const Tensor& out_grad, Tensor* x_grad) {
+  if (x_grad) {
+    auto condition = greater_than<T>(
+        out, full<T>(phi::vectorize(out.dims()), 0.0, out.dtype()));
+    auto res = where<T>(condition,
+                        out_grad,
+                        full<T>(phi::vectorize(out.dims()), 0.0, out.dtype()));
+    set_output<T>(res, x_grad);
+  }
+}
+
 template <typename T>
 void softmax_grad(const Tensor& out,
                   const Tensor& out_grad,

paddle/fluid/prim/tests/CMakeLists.txt

@@ -38,7 +38,8 @@ cc_test_old(
   static_global_utils
   static_tensor_operants
   tensor_api
-  operants_manager)
+  operants_manager
+  generated_static_op)
 
 if(NOT (NOT WITH_PYTHON AND ON_INFER))
   cc_library(

paddle/fluid/prim/tests/test_eager_prim.cc

@@ -35,6 +35,12 @@ PD_DECLARE_KERNEL(tanh_grad, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(pow, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(scale, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(multiply, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(less_equal, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(less_than, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(equal, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(not_equal, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(greater_equal, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(greater_than, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(bitwise_and, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(bitwise_or, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(bitwise_xor, CPU, ALL_LAYOUT);
@@ -46,6 +52,12 @@ PD_DECLARE_KERNEL(tanh_grad, GPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(pow, GPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(scale, GPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(multiply, KPS, ALL_LAYOUT);
+PD_DECLARE_KERNEL(less_equal, KPS, ALL_LAYOUT);
+PD_DECLARE_KERNEL(less_than, KPS, ALL_LAYOUT);
+PD_DECLARE_KERNEL(equal, KPS, ALL_LAYOUT);
+PD_DECLARE_KERNEL(not_equal, KPS, ALL_LAYOUT);
+PD_DECLARE_KERNEL(greater_equal, KPS, ALL_LAYOUT);
+PD_DECLARE_KERNEL(greater_than, KPS, ALL_LAYOUT);
 PD_DECLARE_KERNEL(bitwise_and, KPS, ALL_LAYOUT);
 PD_DECLARE_KERNEL(bitwise_or, KPS, ALL_LAYOUT);
 PD_DECLARE_KERNEL(bitwise_xor, KPS, ALL_LAYOUT);
@@ -151,6 +163,50 @@ TEST(EagerPrim, LogicalOperantsTest) {
   EXPECT_EQ(out0.data<int>()[0], out1.data<int>()[0]);
 }
 
+TEST(EagerPrim, CompareOperantsTest) {
+  // 1. Initialized
+  eager_test::InitEnv(paddle::platform::CPUPlace());
+  FLAGS_tensor_operants_mode = "eager";
+  paddle::prim::InitTensorOperants();
+  // 2. pre
+  paddle::framework::DDim ddim = phi::make_ddim({4, 16, 16, 32});
+  paddle::Tensor tensor0 =
+      ::egr::egr_utils_api::CreateTensorWithValue(ddim,
+                                                  paddle::platform::CPUPlace(),
+                                                  phi::DataType::INT32,
+                                                  phi::DataLayout::NCHW,
+                                                  1 /*value*/,
+                                                  true /*is_leaf*/);
+  ::egr::egr_utils_api::RetainGradForTensor(tensor0);
+  paddle::Tensor tensor1 =
+      ::egr::egr_utils_api::CreateTensorWithValue(ddim,
+                                                  paddle::platform::CPUPlace(),
+                                                  phi::DataType::INT32,
+                                                  phi::DataLayout::NCHW,
+                                                  0 /*value*/,
+                                                  true /*is_leaf*/);
+  ::egr::egr_utils_api::RetainGradForTensor(tensor1);
+  // 3. Run Forward once
+  paddle::Tensor out0 = (tensor0 < tensor1);
+  paddle::Tensor out1 = less_than_ad_func(tensor0, tensor1);
+  EXPECT_EQ(out0.data<bool>()[0], out1.data<bool>()[0]);
+  out0 = (tensor0 <= tensor1);
+  out1 = less_equal_ad_func(tensor0, tensor1);
+  EXPECT_EQ(out0.data<bool>()[0], out1.data<bool>()[0]);
+  out0 = (tensor0 == tensor1);
+  out1 = equal_ad_func(tensor0, tensor1);
+  EXPECT_EQ(out0.data<bool>()[0], out1.data<bool>()[0]);
+  out0 = (tensor0 != tensor1);
+  out1 = not_equal_ad_func(tensor0, tensor1);
+  EXPECT_EQ(out0.data<bool>()[0], out1.data<bool>()[0]);
+  out0 = (tensor0 > tensor1);
+  out1 = greater_than_ad_func(tensor0, tensor1);
+  EXPECT_EQ(out0.data<bool>()[0], out1.data<bool>()[0]);
+  out0 = (tensor0 >= tensor1);
+  out1 = greater_equal_ad_func(tensor0, tensor1);
+  EXPECT_EQ(out0.data<bool>()[0], out1.data<bool>()[0]);
+}
+
 TEST(EagerPrim, TestFlags) {
   PrimCommonUtils::SetBwdPrimEnabled(true);
   ASSERT_TRUE(PrimCommonUtils::IsBwdPrimEnabled());

paddle/fluid/prim/tests/test_static_prim.cc

@@ -38,6 +38,12 @@ PD_DECLARE_KERNEL(scale, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(subtract, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(multiply, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(concat, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(less_equal, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(less_than, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(equal, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(not_equal, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(greater_equal, CPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(greater_than, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(bitwise_and, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(bitwise_or, CPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(bitwise_xor, CPU, ALL_LAYOUT);
@@ -51,6 +57,12 @@ PD_DECLARE_KERNEL(scale, GPU, ALL_LAYOUT);
 PD_DECLARE_KERNEL(subtract, KPS, ALL_LAYOUT);
 PD_DECLARE_KERNEL(multiply, KPS, ALL_LAYOUT);
 PD_DECLARE_KERNEL(concat, GPU, ALL_LAYOUT);
+PD_DECLARE_KERNEL(less_equal, KPS, ALL_LAYOUT);
+PD_DECLARE_KERNEL(less_than, KPS, ALL_LAYOUT);
+PD_DECLARE_KERNEL(equal, KPS, ALL_LAYOUT);
+PD_DECLARE_KERNEL(not_equal, KPS, ALL_LAYOUT);
+PD_DECLARE_KERNEL(greater_equal, KPS, ALL_LAYOUT);
+PD_DECLARE_KERNEL(greater_than, KPS, ALL_LAYOUT);
 PD_DECLARE_KERNEL(bitwise_and, KPS, ALL_LAYOUT);
 PD_DECLARE_KERNEL(bitwise_or, KPS, ALL_LAYOUT);
 PD_DECLARE_KERNEL(bitwise_xor, KPS, ALL_LAYOUT);
@@ -429,6 +441,99 @@ TEST(StaticCompositeGradMaker, LogicalOperantsTest) {
             std::size_t(1));
 }
 
+TEST(StaticCompositeGradMaker, CompareOperantsTest) {
+  // Initialized environment
+  FLAGS_tensor_operants_mode = "static";
+  paddle::OperantsManager::Instance().static_operants.reset(
+      new paddle::prim::StaticTensorOperants());
+
+  TestBaseProgram base_program = TestBaseProgram();
+  auto* target_block = base_program.GetBlock(0);
+  std::vector<int64_t> shape = {2, 2};
+  StaticCompositeContext::Instance().SetBlock(target_block);
+  Tensor x0 = prim::empty<prim::DescTensor>(
+      shape, phi::DataType::INT32, phi::CPUPlace());
+  std::string x0_name =
+      std::static_pointer_cast<prim::DescTensor>(x0.impl())->Name();
+  Tensor x1 = prim::empty<prim::DescTensor>(
+      shape, phi::DataType::INT32, phi::CPUPlace());
+  std::string x1_name =
+      std::static_pointer_cast<prim::DescTensor>(x1.impl())->Name();
+  Tensor x2 = prim::empty<prim::DescTensor>(
+      shape, phi::DataType::INT32, phi::CPUPlace());
+  std::string x2_name =
+      std::static_pointer_cast<prim::DescTensor>(x2.impl())->Name();
+  Tensor x3 = prim::empty<prim::DescTensor>(
+      shape, phi::DataType::INT32, phi::CPUPlace());
+  std::string x3_name =
+      std::static_pointer_cast<prim::DescTensor>(x3.impl())->Name();
+  Tensor x4 = prim::empty<prim::DescTensor>(
+      shape, phi::DataType::INT32, phi::CPUPlace());
+  std::string x4_name =
+      std::static_pointer_cast<prim::DescTensor>(x4.impl())->Name();
+  Tensor x5 = prim::empty<prim::DescTensor>(
+      shape, phi::DataType::INT32, phi::CPUPlace());
+  std::string x5_name =
+      std::static_pointer_cast<prim::DescTensor>(x5.impl())->Name();
+  Tensor x6 = prim::empty<prim::DescTensor>(
+      shape, phi::DataType::INT32, phi::CPUPlace());
+  std::string x6_name =
+      std::static_pointer_cast<prim::DescTensor>(x6.impl())->Name();
+
+  Tensor out_less = (x0 < x1);
+  Tensor out_less_equal = (out_less <= x2);
+  Tensor out_equal = (out_less_equal == x3);
+  Tensor out_not_equal = (out_equal != x4);
+  Tensor out_greater = (out_not_equal > x5);
+  Tensor out_greater_equal = (out_greater >= x6);
+
+  ASSERT_EQ(target_block->AllOps().size(), static_cast<std::size_t>(6));
+  ASSERT_EQ(target_block->AllOps()[0]->Type(), "less_than");
+  ASSERT_EQ(target_block->AllOps()[0]->Inputs().at("X").size(),
+            static_cast<std::size_t>(1));
+  ASSERT_EQ(target_block->AllOps()[0]->Inputs().at("X")[0], x0_name);
+  ASSERT_EQ(target_block->AllOps()[0]->Inputs().at("Y").size(),
+            static_cast<std::size_t>(1));
+  ASSERT_EQ(target_block->AllOps()[0]->Inputs().at("Y")[0], x1_name);
+  ASSERT_EQ(target_block->AllOps()[0]->Outputs().at("Out").size(),
+            std::size_t(1));
+
+  ASSERT_EQ(target_block->AllOps()[1]->Type(), "less_equal");
+  ASSERT_EQ(target_block->AllOps()[1]->Inputs().at("Y").size(),
+            static_cast<std::size_t>(1));
+  ASSERT_EQ(target_block->AllOps()[1]->Inputs().at("Y")[0], x2_name);
+  ASSERT_EQ(target_block->AllOps()[1]->Outputs().at("Out").size(),
+            std::size_t(1));
+
+  ASSERT_EQ(target_block->AllOps()[2]->Type(), "equal");
+  ASSERT_EQ(target_block->AllOps()[2]->Inputs().at("Y").size(),
+            static_cast<std::size_t>(1));
+  ASSERT_EQ(target_block->AllOps()[2]->Inputs().at("Y")[0], x3_name);
+  ASSERT_EQ(target_block->AllOps()[2]->Outputs().at("Out").size(),
+            std::size_t(1));
+
+  ASSERT_EQ(target_block->AllOps()[3]->Type(), "not_equal");
+  ASSERT_EQ(target_block->AllOps()[3]->Inputs().at("Y").size(),
+            static_cast<std::size_t>(1));
+  ASSERT_EQ(target_block->AllOps()[3]->Inputs().at("Y")[0], x4_name);
+  ASSERT_EQ(target_block->AllOps()[3]->Outputs().at("Out").size(),
+            std::size_t(1));
+
+  ASSERT_EQ(target_block->AllOps()[4]->Type(), "greater_than");
+  ASSERT_EQ(target_block->AllOps()[4]->Inputs().at("Y").size(),
+            static_cast<std::size_t>(1));
+  ASSERT_EQ(target_block->AllOps()[4]->Inputs().at("Y")[0], x5_name);
+  ASSERT_EQ(target_block->AllOps()[4]->Outputs().at("Out").size(),
+            std::size_t(1));
+
+  ASSERT_EQ(target_block->AllOps()[5]->Type(), "greater_equal");
+  ASSERT_EQ(target_block->AllOps()[5]->Inputs().at("Y").size(),
+            static_cast<std::size_t>(1));
+  ASSERT_EQ(target_block->AllOps()[5]->Inputs().at("Y")[0], x6_name);
+  ASSERT_EQ(target_block->AllOps()[5]->Outputs().at("Out").size(),
+            std::size_t(1));
+}
+
 TEST(StaticPrim, TestFlags) {
   PrimCommonUtils::SetBwdPrimEnabled(true);
   ASSERT_TRUE(PrimCommonUtils::IsBwdPrimEnabled());
@@ -445,6 +550,12 @@ USE_OP_ITSELF(elementwise_mul);
 USE_OP_ITSELF(elementwise_sub);
 USE_OP_ITSELF(elementwise_pow);
 USE_OP_ITSELF(scale);
+USE_OP_ITSELF(less_equal);
+USE_OP_ITSELF(less_than);
+USE_OP_ITSELF(equal);
+USE_OP_ITSELF(not_equal);
+USE_OP_ITSELF(greater_equal);
+USE_OP_ITSELF(greater_than);
 USE_OP_ITSELF(bitwise_xor);
 USE_OP_ITSELF(bitwise_and);
 USE_OP_ITSELF(bitwise_not);