Merge branch 'anuragd/flag_based_test_selection' into 'release/1.0'

(//tests): CI based masking on test cases

See merge request adlsa/TRTorch!13

.bazelrc

@@ -35,8 +35,8 @@ build:pre_cxx11_abi --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"
 build:pre_cxx11_abi --linkopt="-D_GLIBCXX_USE_CXX11_ABI=0"
 build:pre_cxx11_abi --define=abi=pre_cxx11_abi
 
-build:ci_testing --define=torchtrt_src=pre_built
+build:ci_testing --define=torchtrt_src=pre_built --cxxopt="-DDISABLE_TEST_IN_CI" --action_env "NVIDIA_TF32_OVERRIDE=0"
 build:use_precompiled_torchtrt --define=torchtrt_src=pre_built
 
-test:ci_testing --define=torchtrt_src=pre_built
-test:use_precompiled_torchtrt --define=torchtrt_src=pre_built
+test:ci_testing --define=torchtrt_src=pre_built --cxxopt="-DDISABLE_TEST_IN_CI" --action_env "NVIDIA_TF32_OVERRIDE=0"
+test:use_precompiled_torchtrt --define=torchtrt_src=pre_built

tests/core/conversion/converters/test_activation.cpp

@@ -200,6 +200,8 @@ TEST(Converters, ATenEluConvertsCorrectly) {
   ASSERT_TRUE(torch_tensorrt::tests::util::almostEqual(jit_results[0], trt_results[0], 2e-6));
 }
 
+#ifndef DISABLE_TEST_IN_CI
+
 TEST(Converters, ATenGELUConvertsCorrectly) {
   const auto graph = R"IR(
       graph(%0 : Tensor):
@@ -226,3 +228,4 @@ TEST(Converters, ATenGELUConvertsCorrectly) {
 
   ASSERT_TRUE(torch_tensorrt::tests::util::almostEqual(jit_results[0], trt_results[0], 5e-3));
 }
+#endif

tests/core/conversion/converters/test_lstm_cell.cpp

@@ -4,6 +4,8 @@
 #include "tests/util/util.h"
 #include "torch/csrc/jit/ir/irparser.h"
 
+#ifndef DISABLE_TEST_IN_CI
+
 TEST(Converters, ATenGRUCellConvertsCorrectlyWithBiasCheckHidden) {
   const auto graph = R"IR(
       graph(%0 : Tensor,
@@ -283,3 +285,5 @@ TEST(Converters, ATenLSTMCellConvertsCorrectlyWithoutBiasCheckCell) {
   ASSERT_TRUE(
       torch_tensorrt::tests::util::almostEqual(jit_results[0], trt_results[0].reshape_as(jit_results[0]), 2e-5));
 }
+
+#endif

tests/core/conversion/converters/test_softmax.cpp

@@ -4,6 +4,8 @@
 #include "tests/util/util.h"
 #include "torch/csrc/jit/ir/irparser.h"
 
+#ifndef DISABLE_TEST_IN_CI
+
 TEST(Converters, ATenSoftmax1DConvertsCorrectly) {
   const auto graph = R"IR(
       graph(%0 : Tensor):
@@ -127,4 +129,5 @@ TEST(Converters, ATenSoftmaxNDConvertsCorrectlyNegtiveIndex) {
   auto trt = trt_results[0].reshape_as(jit_results[0]);
 
   ASSERT_TRUE(torch_tensorrt::tests::util::almostEqual(jit_results[0], trt, 2e-6));
-}
+}
+#endif

tests/core/partitioning/test_fallback_graph_output.cpp

@@ -1,102 +1,105 @@
-#include <string>
-#include <unordered_set>
-#include "core/compiler.h"
-#include "gtest/gtest.h"
-#include "tests/util/util.h"
-#include "torch/script.h"
-
-TEST(Partitioning, ComputeResNet50FallbackGraphCorrectly) {
-  torch::jit::script::Module mod;
-  try {
-    mod = torch::jit::load("tests/modules/resnet50_traced.jit.pt");
-  } catch (const c10::Error& e) {
-    std::cerr << "error loading the model\n";
-    return;
-  }
-
-  const std::vector<std::vector<int64_t>> input_shapes = {{1, 3, 224, 224}};
-  std::vector<torch::jit::IValue> jit_inputs_ivalues;
-  std::vector<torch::jit::IValue> trt_inputs_ivalues;
-  for (auto in_shape : input_shapes) {
-    auto in = at::randint(5, in_shape, {at::kCUDA});
-    jit_inputs_ivalues.push_back(in.clone());
-    trt_inputs_ivalues.push_back(in.clone());
-  }
-
-  std::vector<torch_tensorrt::core::ir::Input> input_ranges{torch_tensorrt::core::ir::Input({1, 3, 224, 224})};
-
-  torch_tensorrt::core::CompileSpec cfg(input_ranges);
-  cfg.partition_info.enabled = true;
-  cfg.partition_info.forced_fallback_operators.push_back("aten::add");
-
-  auto jit_results = mod.forward(jit_inputs_ivalues).toTensor();
-  auto trt_mod = torch_tensorrt::core::CompileGraph(mod, cfg);
-  auto trt_results = trt_mod.forward(trt_inputs_ivalues).toTensor();
-  ASSERT_TRUE(torch_tensorrt::tests::util::almostEqual(jit_results, trt_results, 2e-6));
-}
-
-TEST(Partitioning, ComputeMobileNetFallbackGraphCorrectly) {
-  torch::jit::script::Module mod;
-  try {
-    mod = torch::jit::load("tests/modules/mobilenet_v2_traced.jit.pt");
-  } catch (const c10::Error& e) {
-    std::cerr << "error loading the model\n";
-    return;
-  }
-
-  const std::vector<std::vector<int64_t>> input_shapes = {{1, 3, 224, 224}};
-  std::vector<torch::jit::IValue> jit_inputs_ivalues;
-  std::vector<torch::jit::IValue> trt_inputs_ivalues;
-  for (auto in_shape : input_shapes) {
-    auto in = at::randint(5, in_shape, {at::kCUDA});
-    jit_inputs_ivalues.push_back(in.clone());
-    trt_inputs_ivalues.push_back(in.clone());
-  }
-
-  std::vector<torch_tensorrt::core::ir::Input> input_ranges{torch_tensorrt::core::ir::Input({1, 3, 224, 224})};
-  auto g = mod.get_method("forward").graph();
-  torch_tensorrt::core::CompileSpec cfg(input_ranges);
-  cfg.partition_info.enabled = true;
-  cfg.partition_info.forced_fallback_operators.push_back("aten::hardtanh");
-
-  auto jit_results = mod.forward(jit_inputs_ivalues).toTensor();
-  auto trt_mod = torch_tensorrt::core::CompileGraph(mod, cfg);
-  auto trt_results = trt_mod.forward(trt_inputs_ivalues).toTensor();
-  ASSERT_TRUE(torch_tensorrt::tests::util::almostEqual(jit_results, trt_results, 2e-6));
-}
-
-TEST(Partitioning, ComputeResNet50HalfFallbackGraphCorrectly) {
-  torch::jit::script::Module mod;
-  try {
-    mod = torch::jit::load("tests/modules/resnet50_traced.jit.pt");
-  } catch (const c10::Error& e) {
-    std::cerr << "error loading the model\n";
-    return;
-  }
-
-  mod.to(torch::kHalf);
-
-  const std::vector<std::vector<int64_t>> input_shapes = {{1, 3, 224, 224}};
-  std::vector<torch::jit::IValue> jit_inputs_ivalues;
-  std::vector<torch::jit::IValue> trt_inputs_ivalues;
-  for (auto in_shape : input_shapes) {
-    auto in = at::randint(5, in_shape, {at::kCUDA}).to(torch::kHalf);
-    jit_inputs_ivalues.push_back(in.clone());
-    trt_inputs_ivalues.push_back(in.clone());
-  }
-
-  auto in_shape = torch_tensorrt::core::ir::Input({1, 3, 224, 224});
-  in_shape.dtype = nvinfer1::DataType::kHALF;
-
-  std::vector<torch_tensorrt::core::ir::Input> input_ranges({in_shape});
-  auto g = mod.get_method("forward").graph();
-  torch_tensorrt::core::CompileSpec cfg(input_ranges);
-  cfg.partition_info.enabled = true;
-  cfg.partition_info.forced_fallback_operators.push_back("aten::add");
-
-  auto jit_results = mod.forward(jit_inputs_ivalues).toTensor();
-  auto trt_mod = torch_tensorrt::core::CompileGraph(mod, cfg);
-  auto trt_results = trt_mod.forward(trt_inputs_ivalues).toTensor();
-  // Lower threshold because FP16
-  ASSERT_TRUE(torch_tensorrt::tests::util::almostEqual(jit_results, trt_results, 2e-1));
-}
+#include <string>
+#include <unordered_set>
+#include "core/compiler.h"
+#include "gtest/gtest.h"
+#include "tests/util/util.h"
+#include "torch/script.h"
+
+#ifndef DISABLE_TEST_IN_CI
+
+TEST(Partitioning, ComputeResNet50FallbackGraphCorrectly) {
+  torch::jit::script::Module mod;
+  try {
+    mod = torch::jit::load("tests/modules/resnet50_traced.jit.pt");
+  } catch (const c10::Error& e) {
+    std::cerr << "error loading the model\n";
+    return;
+  }
+
+  const std::vector<std::vector<int64_t>> input_shapes = {{1, 3, 224, 224}};
+  std::vector<torch::jit::IValue> jit_inputs_ivalues;
+  std::vector<torch::jit::IValue> trt_inputs_ivalues;
+  for (auto in_shape : input_shapes) {
+    auto in = at::randint(5, in_shape, {at::kCUDA});
+    jit_inputs_ivalues.push_back(in.clone());
+    trt_inputs_ivalues.push_back(in.clone());
+  }
+
+  std::vector<torch_tensorrt::core::ir::Input> input_ranges{torch_tensorrt::core::ir::Input({1, 3, 224, 224})};
+
+  torch_tensorrt::core::CompileSpec cfg(input_ranges);
+  cfg.partition_info.enabled = true;
+  cfg.partition_info.forced_fallback_operators.push_back("aten::add");
+
+  auto jit_results = mod.forward(jit_inputs_ivalues).toTensor();
+  auto trt_mod = torch_tensorrt::core::CompileGraph(mod, cfg);
+  auto trt_results = trt_mod.forward(trt_inputs_ivalues).toTensor();
+  ASSERT_TRUE(torch_tensorrt::tests::util::almostEqual(jit_results, trt_results, 2e-6));
+}
+
+TEST(Partitioning, ComputeMobileNetFallbackGraphCorrectly) {
+  torch::jit::script::Module mod;
+  try {
+    mod = torch::jit::load("tests/modules/mobilenet_v2_traced.jit.pt");
+  } catch (const c10::Error& e) {
+    std::cerr << "error loading the model\n";
+    return;
+  }
+
+  const std::vector<std::vector<int64_t>> input_shapes = {{1, 3, 224, 224}};
+  std::vector<torch::jit::IValue> jit_inputs_ivalues;
+  std::vector<torch::jit::IValue> trt_inputs_ivalues;
+  for (auto in_shape : input_shapes) {
+    auto in = at::randint(5, in_shape, {at::kCUDA});
+    jit_inputs_ivalues.push_back(in.clone());
+    trt_inputs_ivalues.push_back(in.clone());
+  }
+
+  std::vector<torch_tensorrt::core::ir::Input> input_ranges{torch_tensorrt::core::ir::Input({1, 3, 224, 224})};
+  auto g = mod.get_method("forward").graph();
+  torch_tensorrt::core::CompileSpec cfg(input_ranges);
+  cfg.partition_info.enabled = true;
+  cfg.partition_info.forced_fallback_operators.push_back("aten::hardtanh");
+
+  auto jit_results = mod.forward(jit_inputs_ivalues).toTensor();
+  auto trt_mod = torch_tensorrt::core::CompileGraph(mod, cfg);
+  auto trt_results = trt_mod.forward(trt_inputs_ivalues).toTensor();
+  ASSERT_TRUE(torch_tensorrt::tests::util::almostEqual(jit_results, trt_results, 2e-6));
+}
+
+TEST(Partitioning, ComputeResNet50HalfFallbackGraphCorrectly) {
+  torch::jit::script::Module mod;
+  try {
+    mod = torch::jit::load("tests/modules/resnet50_traced.jit.pt");
+  } catch (const c10::Error& e) {
+    std::cerr << "error loading the model\n";
+    return;
+  }
+
+  mod.to(torch::kHalf);
+
+  const std::vector<std::vector<int64_t>> input_shapes = {{1, 3, 224, 224}};
+  std::vector<torch::jit::IValue> jit_inputs_ivalues;
+  std::vector<torch::jit::IValue> trt_inputs_ivalues;
+  for (auto in_shape : input_shapes) {
+    auto in = at::randint(5, in_shape, {at::kCUDA}).to(torch::kHalf);
+    jit_inputs_ivalues.push_back(in.clone());
+    trt_inputs_ivalues.push_back(in.clone());
+  }
+
+  auto in_shape = torch_tensorrt::core::ir::Input({1, 3, 224, 224});
+  in_shape.dtype = nvinfer1::DataType::kHALF;
+
+  std::vector<torch_tensorrt::core::ir::Input> input_ranges({in_shape});
+  auto g = mod.get_method("forward").graph();
+  torch_tensorrt::core::CompileSpec cfg(input_ranges);
+  cfg.partition_info.enabled = true;
+  cfg.partition_info.forced_fallback_operators.push_back("aten::add");
+
+  auto jit_results = mod.forward(jit_inputs_ivalues).toTensor();
+  auto trt_mod = torch_tensorrt::core::CompileGraph(mod, cfg);
+  auto trt_results = trt_mod.forward(trt_inputs_ivalues).toTensor();
+  // Lower threshold because FP16
+  ASSERT_TRUE(torch_tensorrt::tests::util::almostEqual(jit_results, trt_results, 2e-1));
+}
+#endif

tests/cpp/test_compiled_modules.cpp

@@ -24,6 +24,8 @@ TEST_P(CppAPITests, CompiledModuleIsClose) {
   }
 }
 
+#ifndef DISABLE_TEST_IN_CI
+
 INSTANTIATE_TEST_SUITE_P(
     CompiledModuleForwardIsCloseSuite,
     CppAPITests,
@@ -36,3 +38,5 @@ INSTANTIATE_TEST_SUITE_P(
         PathAndInSize({"tests/modules/mobilenet_v2_scripted.jit.pt", {{1, 3, 224, 224}}, 2e-5}),
         PathAndInSize({"tests/modules/efficientnet_b0_scripted.jit.pt", {{1, 3, 224, 224}}, 8e-3}),
         PathAndInSize({"tests/modules/vit_scripted.jit.pt", {{1, 3, 224, 224}}, 8e-2})));
+
+#endif

tests/cpp/test_module_fallback.cpp

@@ -4,6 +4,8 @@
 #include "torch/script.h"
 #include "torch_tensorrt/torch_tensorrt.h"
 
+#ifndef DISABLE_TEST_IN_CI
+
 TEST(CppAPITest, ResNetModuleFallbacksCorrectly) {
   torch::jit::script::Module mod;
   try {
@@ -69,3 +71,4 @@ TEST(CppAPITest, MobileNetModuleFallbacksCorrectlyWithOneEngine) {
   auto trt_results = trt_mod.forward(trt_inputs_ivalues).toTensor();
   ASSERT_TRUE(torch_tensorrt::tests::util::almostEqual(jit_results, trt_results, 2e-6));
 }
+#endif

tests/cpp/test_modules_as_engines.cpp

@@ -51,6 +51,8 @@ TEST_P(CppAPITests, ModuleToEngineToModuleIsClose) {
       torch_tensorrt::tests::util::almostEqual(jit_results[0], trt_results[0].reshape_as(jit_results[0]), threshold));
 }
 
+#ifndef DISABLE_TEST_IN_CI
+
 INSTANTIATE_TEST_SUITE_P(
     ModuleAsEngineForwardIsCloseSuite,
     CppAPITests,
@@ -63,3 +65,5 @@ INSTANTIATE_TEST_SUITE_P(
         PathAndInSize({"tests/modules/mobilenet_v2_scripted.jit.pt", {{1, 3, 224, 224}}, 2e-5}),
         PathAndInSize({"tests/modules/efficientnet_b0_scripted.jit.pt", {{1, 3, 224, 224}}, 2e-5}),
         PathAndInSize({"tests/modules/vit_scripted.jit.pt", {{1, 3, 224, 224}}, 8e-2})));
+
+#endif

tests/cpp/test_multiple_registered_engines.cpp

@@ -4,6 +4,8 @@
 #include "torch/script.h"
 #include "torch_tensorrt/torch_tensorrt.h"
 
+#ifndef DISABLE_TEST_IN_CI
+
 TEST(CppAPITest, CanRunMultipleEngines) {
   torch::jit::script::Module mod1;
   torch::jit::script::Module mod2;
@@ -63,3 +65,4 @@ TEST(CppAPITest, CanRunMultipleEngines) {
         torch_tensorrt::tests::util::almostEqual(jit2_results[i], trt2_results[i].reshape_as(jit2_results[i]), 2e-5));
   }
 }
+#endif

tests/cpp/test_runtime_thread_safety.cpp

@@ -5,6 +5,8 @@
 #include "torch/script.h"
 #include "torch_tensorrt/torch_tensorrt.h"
 
+#ifndef DISABLE_TEST_IN_CI
+
 void run_infer(
     int thread_id,
     torch::jit::Module& mod,
@@ -80,4 +82,5 @@ TEST(CppAPITests, RuntimeThreadSafety) {
     flag = flag && f;
   }
   ASSERT_TRUE(flag);
-}
+}
+#endif