Improvements to OSS Test CMake generation and add tests (pytorch#3649)

Summary:
* Fix the bash script looping through exectubales
* For the generator, add a way to add additional libs in generated CMakeLists.txt
* Build extension/data_loader in main target as an example for the fix

Pull Request resolved: pytorch#3649

Test Plan: `sh test/run_oss_cpp_tests.sh`

Reviewed By: larryliu0820

Differential Revision: D57467885

Pulled By: kirklandsign

fbshipit-source-id: 49f9b8dac9f091aca05a122aaf0cb9020d192b68

extension/data_loader/test/CMakeLists.txt

@@ -0,0 +1,45 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# @generated by test/utils/generate_gtest_cmakelists.py
+#
+# This file should be formatted with
+# ~~~
+# cmake-format -i CMakeLists.txt
+# ~~~
+# It should also be cmake-lint clean.
+#
+
+cmake_minimum_required(VERSION 3.19)
+project(extension_data_loader_test)
+
+# Use C++14 for test.
+set(CMAKE_CXX_STANDARD 14)
+
+set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
+
+include(${EXECUTORCH_ROOT}/build/Utils.cmake)
+
+# Find prebuilt executorch library
+find_package(executorch CONFIG REQUIRED)
+
+enable_testing()
+find_package(GTest CONFIG REQUIRED)
+
+# Let files say "include <executorch/path/to/header.h>".
+set(_common_include_directories ${EXECUTORCH_ROOT}/..)
+target_include_directories(executorch INTERFACE ${_common_include_directories})
+
+set(_test_srcs buffer_data_loader_test.cpp shared_ptr_data_loader_test.cpp
+               file_data_loader_test.cpp mmap_data_loader_test.cpp
+)
+
+add_executable(extension_data_loader_test ${_test_srcs})
+target_link_libraries(
+  extension_data_loader_test GTest::gtest GTest::gtest_main GTest::gmock
+  executorch extension_data_loader
+)
+add_test(ExecuTorchTest extension_data_loader_test)

test/run_oss_cpp_tests.sh

@@ -8,18 +8,23 @@
 set -ex
 
 build_executorch() {
-  cmake . -DCMAKE_INSTALL_PREFIX=cmake-out -DEXECUTORCH_BUILD_GTESTS=ON -Bcmake-out
+  cmake . \
+    -DCMAKE_INSTALL_PREFIX=cmake-out \
+    -DEXECUTORCH_BUILD_GTESTS=ON \
+    -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
+    -Bcmake-out
   cmake --build cmake-out -j9 --target install
 }
 
 build_and_run_test() {
   local test_dir=$1
   cmake "${test_dir}" -Bcmake-out/"${test_dir}" -DCMAKE_INSTALL_PREFIX=cmake-out
   cmake --build cmake-out/"${test_dir}" -j9
-  for t in $(cmake-out/"${test_dir}"/*test); do ./"$t"; done
+  for t in cmake-out/"${test_dir}"/*test; do ./"$t"; done
 }
 
 build_executorch
+build_and_run_test extension/data_loader/test/
 build_and_run_test runtime/core/portable_type/test/
 build_and_run_test runtime/core/test/
 build_and_run_test runtime/core/exec_aten/util/test/

test/utils/OSSTest.cmake.in

@@ -40,5 +40,6 @@ set(_test_srcs {test_srcs})
 add_executable({project_name} ${{_test_srcs}})
 target_link_libraries(
   {project_name} GTest::gtest GTest::gtest_main GTest::gmock executorch
+  {additional_libs}
 )
 add_test(ExecuTorchTest {project_name})

test/utils/OSSTestConfig.json

@@ -1,4 +1,16 @@
 { "tests": [
+    {
+        "directory": "extension/data_loader/test",
+        "sources": [
+            "buffer_data_loader_test.cpp",
+            "shared_ptr_data_loader_test.cpp",
+            "file_data_loader_test.cpp",
+            "mmap_data_loader_test.cpp"
+        ],
+        "additional_libs": [
+            "extension_data_loader"
+        ]
+    },
     {
         "directory": "runtime/core/portable_type/test",
         "sources": [

test/utils/generate_gtest_cmakelists.py

@@ -29,7 +29,7 @@ def calculate_relative_path(path_to_root):
     return os.path.relpath("/", "/" + path_to_root)
 
 
-def format_template(path_to_root, test_srcs):
+def format_template(path_to_root, test_srcs, additional_libs):
     """
     Format the template with the given path_to_root and test_srcs.
     """
@@ -39,30 +39,34 @@ def format_template(path_to_root, test_srcs):
         project_name=calculate_project_name(path_to_root),
         path_to_root=calculate_relative_path(path_to_root),
         test_srcs=" ".join(test_srcs),
+        additional_libs=" ".join(additional_libs),
     )
 
 
-def write_template(path_to_root, test_srcs):
+def write_template(path_to_root, test_srcs, additional_libs):
     """
     Write the template to the given path_to_root.
     """
     with open(os.path.join(path_to_root, "CMakeLists.txt"), "w") as f:
-        f.write(format_template(path_to_root, test_srcs))
+        f.write(format_template(path_to_root, test_srcs, additional_libs))
 
 
 def read_config_json(json_path):
     """
-    Read the config.json file and return the list of (path_to_root, test_srcs)
+    Read the config.json file
     """
     with open(json_path) as f:
         config = json.load(f)
-    return [(d["directory"], d["sources"]) for d in config["tests"]]
+    return config["tests"]
 
 
 if __name__ == "__main__":
     json_path = os.path.dirname(os.path.abspath(__file__)) + "/OSSTestConfig.json"
-    for path_to_root, test_srcs in read_config_json(json_path):
-        write_template(path_to_root, test_srcs)
+    for d in read_config_json(json_path):
+        path_to_root = d["directory"]
+        test_srcs = d["sources"]
+        additional_libs = d.get("additional_libs", [])
+        write_template(path_to_root, test_srcs, additional_libs)
         if shutil.which("cmake-format") is not None:
             subprocess.run(
                 ["cmake-format", "-i", path_to_root + "/CMakeLists.txt"], check=True