Implement global index store cache

swift/internal/compiling.bzl

@@ -58,6 +58,7 @@ load(
     "SWIFT_FEATURE_SUPPORTS_SYSTEM_MODULE_FLAG",
     "SWIFT_FEATURE_SYSTEM_MODULE",
     "SWIFT_FEATURE_USE_C_MODULES",
+    "SWIFT_FEATURE_USE_GLOBAL_INDEX_STORE",
     "SWIFT_FEATURE_USE_GLOBAL_MODULE_CACHE",
     "SWIFT_FEATURE_VFSOVERLAY",
     "SWIFT_FEATURE__NUM_THREADS_0_IN_SWIFTCOPTS",
@@ -805,6 +806,14 @@ def compile_action_configs(
             ],
             features = [SWIFT_FEATURE_ENABLE_SKIP_FUNCTION_BODIES],
         ),
+        swift_toolchain_config.action_config(
+            actions = [swift_action_names.COMPILE],
+            configurators = [_global_index_store_configurator],
+            features = [
+                SWIFT_FEATURE_INDEX_WHILE_BUILDING,
+                SWIFT_FEATURE_USE_GLOBAL_INDEX_STORE,
+            ],
+        ),
 
         # Configure index-while-building.
         swift_toolchain_config.action_config(
@@ -1419,6 +1428,12 @@ def _index_while_building_configurator(prerequisites, args):
     if not _index_store_path_overridden(prerequisites.user_compile_flags):
         args.add("-index-store-path", prerequisites.indexstore_directory.path)
 
+def _global_index_store_configurator(prerequisites, args):
+    """Adds flags for index-store generation to the command line."""
+    out_dir = prerequisites.indexstore_directory.dirname.split("/")[0]
+    path = out_dir + "/_global_index_store"
+    args.add("-Xwrapped-swift=-global-index-store-import-path=" + path)
+
 def _conditional_compilation_flag_configurator(prerequisites, args):
     """Adds (non-Clang) conditional compilation flags to the command line."""
     all_defines = depset(

swift/internal/feature_names.bzl

@@ -99,11 +99,15 @@ SWIFT_FEATURE_ENABLE_TESTING = "swift.enable_testing"
 SWIFT_FEATURE_FULL_DEBUG_INFO = "swift.full_debug_info"
 
 # If enabled, the compilation action for a target will produce an index store.
+# https://docs.google.com/document/d/1cH2sTpgSnJZCkZtJl1aY-rzy4uGPcrI-6RrUpdATO2Q/
 SWIFT_FEATURE_INDEX_WHILE_BUILDING = "swift.index_while_building"
 
 # If enabled the compilation action will not produce indexes for system modules.
 SWIFT_FEATURE_DISABLE_SYSTEM_INDEX = "swift.disable_system_index"
 
+# Index while building - using a global index store cache
+SWIFT_FEATURE_USE_GLOBAL_INDEX_STORE = "swift.use_global_index_store"
+
 # If enabled, compilation actions and module map generation will assume that the
 # header paths in module maps are relative to the current working directory
 # (i.e., the workspace root); if disabled, header paths in module maps are

swift/repositories.bzl

@@ -108,6 +108,27 @@ def swift_rules_dependencies():
         type = "zip",
     )
 
+    # It relies on `index-import` to import indexes into Bazel's remote
+    # cache and allow using a global index internally in workers.
+    # Note: this is only loaded if swift.index_while_building_v2 is enabled
+    _maybe(
+        http_archive,
+        name = "build_bazel_rules_swift_index_import",
+        build_file_content = """\
+load("@bazel_skylib//rules:native_binary.bzl", "native_binary")
+
+native_binary(
+    name = "index_import",
+    src = "index-import",
+    out = "index-import",
+    visibility = ["//visibility:public"],
+)
+""",
+        canonical_id = "index-import-5.3.2.6",
+        urls = ["https://github.com/MobileNativeFoundation/index-import/releases/download/5.3.2.6/index-import.zip"],
+        sha256 = "61a58363f56c5fd84d4ebebe0d9b5dd90c74ae170405a7b9018e8cf698e679de",
+    )
+
     _maybe(
         swift_autoconfiguration,
         name = "build_bazel_rules_swift_local_config",

tools/worker/BUILD

@@ -18,12 +18,18 @@ config_setting(
     },
 )
 
+# Internal hinge for index while building V2 feature
+config_setting(
+    name = "use_global_index_store",
+    values = {
+        "features": "swift.use_global_index_store",
+    },
+)
+
 cc_library(
     name = "compile_with_worker",
     srcs = [
         "compile_with_worker.cc",
-        "output_file_map.cc",
-        "output_file_map.h",
         "work_processor.cc",
         "work_processor.h",
     ],
@@ -34,7 +40,6 @@ cc_library(
         "//tools/common:file_system",
         "//tools/common:path_utils",
         "//tools/common:temp_file",
-        "@com_github_nlohmann_json//:json",
         "@com_google_protobuf//:protobuf",
     ],
 )
@@ -50,13 +55,30 @@ cc_library(
 
 cc_library(
     name = "swift_runner",
-    srcs = ["swift_runner.cc"],
+    srcs = [
+        "output_file_map.cc",
+        "output_file_map.h",
+        "swift_runner.cc",
+    ],
     hdrs = ["swift_runner.h"],
+    copts = select({
+        ":use_global_index_store": [
+            "-DINDEX_IMPORT_PATH=\\\"$(rootpath @build_bazel_rules_swift_index_import//:index_import)\\\"",
+        ],
+        "//conditions:default": [],
+    }),
+    data = select({
+        ":use_global_index_store": [
+            "@build_bazel_rules_swift_index_import//:index_import",
+        ],
+        "//conditions:default": [],
+    }),
     deps = [
         "//tools/common:bazel_substitutions",
         "//tools/common:file_system",
         "//tools/common:process",
         "//tools/common:temp_file",
+        "@com_github_nlohmann_json//:json",
     ],
 )
 

tools/worker/swift_runner.cc

@@ -20,6 +20,7 @@
 #include "tools/common/file_system.h"
 #include "tools/common/process.h"
 #include "tools/common/temp_file.h"
+#include "tools/worker/output_file_map.h"
 
 namespace {
 
@@ -129,29 +130,105 @@ int SwiftRunner::Run(std::ostream *stderr_stream, bool stdout_to_stderr) {
     exit_code = RunSubProcess(rewriter_args, stderr_stream, stdout_to_stderr);
   }
 
+  auto enable_global_index_store = global_index_store_import_path_ != "";
+  if (enable_global_index_store) {
+    OutputFileMap output_file_map;
+    output_file_map.ReadFromPath(output_file_map_path_);
+
+    auto outputs = output_file_map.incremental_outputs();
+    std::map<std::string, std::string>::iterator it;
+
+    std::vector<std::string> ii_args;
+// The index-import runfile path is passed as a define
+#if defined(INDEX_IMPORT_PATH)
+    ii_args.push_back(INDEX_IMPORT_PATH);
+#else
+    // Logical error
+    std::cerr << "Incorrectly compiled work_processor.cc";
+    exit_code = EXIT_FAILURE;
+    return exit_code;
+#endif
+
+    for (it = outputs.begin(); it != outputs.end(); it++) {
+      // Need the actual output paths of the compiler - not bazel
+      auto output_path = it->first;
+      auto file_type = output_path.substr(output_path.find_last_of(".") + 1);
+      if (file_type == "o") {
+        ii_args.push_back("-import-output-file");
+        ii_args.push_back(output_path);
+      }
+    }
+
+    auto exec_root = GetCurrentDirectory();
+    // Copy back from the global index store to bazel's index store
+    ii_args.push_back(exec_root + "/" + global_index_store_import_path_);
+    ii_args.push_back(exec_root + "/" + index_store_path_);
+    exit_code =
+        RunSubProcess(ii_args, stderr_stream, /*stdout_to_stderr=*/true);
+  }
   return exit_code;
 }
 
+// Marker for end of iteration
+class StreamIteratorEnd {};
+
+// Basic iterator over an ifstream
+class StreamIterator {
+ public:
+  StreamIterator(std::ifstream &file) : file_{file} { next(); }
+
+  const std::string &operator*() const { return str_; }
+
+  StreamIterator &operator++() {
+    next();
+    return *this;
+  }
+
+  bool operator!=(StreamIteratorEnd) const { return !!file_; }
+
+ private:
+  void next() { std::getline(file_, str_); }
+
+  std::ifstream &file_;
+  std::string str_;
+};
+
+class ArgsFile {
+ public:
+  ArgsFile(std::ifstream &file) : file_(file) {}
+
+  StreamIterator begin() { return StreamIterator{file_}; }
+
+  StreamIteratorEnd end() { return StreamIteratorEnd{}; }
+
+ private:
+  std::ifstream &file_;
+};
+
 bool SwiftRunner::ProcessPossibleResponseFile(
     const std::string &arg, std::function<void(const std::string &)> consumer) {
   auto path = arg.substr(1);
   std::ifstream original_file(path);
+  ArgsFile args_file(original_file);
+
   // If we couldn't open it, maybe it's not a file; maybe it's just some other
   // argument that starts with "@" such as "@loader_path/..."
   if (!original_file.good()) {
     consumer(arg);
     return false;
   }
 
+  // Read the file to a vector to prevent double I/O
+  auto args = ParseArguments(args_file);
+
   // If we're forcing response files, process and send the arguments from this
   // file directly to the consumer; they'll all get written to the same response
   // file at the end of processing all the arguments.
   if (force_response_file_) {
-    std::string arg_from_file;
-    while (std::getline(original_file, arg_from_file)) {
+    for (auto it = args.begin(); it != args.end(); ++it) {
       // Arguments in response files might be quoted/escaped, so we need to
       // unescape them ourselves.
-      ProcessArgument(Unescape(arg_from_file), consumer);
+      ProcessArgument(it, Unescape(*it), consumer);
     }
     return true;
   }
@@ -161,12 +238,10 @@ bool SwiftRunner::ProcessPossibleResponseFile(
   bool changed = false;
   std::string arg_from_file;
   std::vector<std::string> new_args;
-
-  while (std::getline(original_file, arg_from_file)) {
-    changed |=
-        ProcessArgument(arg_from_file, [&](const std::string &processed_arg) {
-          new_args.push_back(processed_arg);
-        });
+  for (auto it = args.begin(); it != args.end(); ++it) {
+    changed |= ProcessArgument(it, *it, [&](const std::string &processed_arg) {
+      new_args.push_back(processed_arg);
+    });
   }
 
   if (changed) {
@@ -182,10 +257,11 @@ bool SwiftRunner::ProcessPossibleResponseFile(
   return changed;
 }
 
+template <typename Iterator>
 bool SwiftRunner::ProcessArgument(
-    const std::string &arg, std::function<void(const std::string &)> consumer) {
+    Iterator &itr, const std::string &arg,
+    std::function<void(const std::string &)> consumer) {
   bool changed = false;
-
   if (arg[0] == '@') {
     changed = ProcessPossibleResponseFile(arg, consumer);
   } else {
@@ -198,8 +274,8 @@ bool SwiftRunner::ProcessArgument(
         consumer(GetCurrentDirectory() + "=.");
         changed = true;
       } else if (new_arg == "-coverage-prefix-pwd-is-dot") {
-        // Get the actual current working directory (the workspace root), which we
-        // didn't know at analysis time.
+        // Get the actual current working directory (the workspace root), which
+        // we didn't know at analysis time.
         consumer("-coverage-prefix-map");
         consumer(GetCurrentDirectory() + "=.");
         changed = true;
@@ -213,14 +289,37 @@ bool SwiftRunner::ProcessArgument(
         temp_directories_.push_back(std::move(module_cache_dir));
         changed = true;
       } else if (StripPrefix("-generated-header-rewriter=", new_arg)) {
-        generated_header_rewriter_path_ = new_arg;
+        changed = true;
+      } else if (StripPrefix("-global-index-store-import-path=", new_arg)) {
         changed = true;
       } else {
         // TODO(allevato): Report that an unknown wrapper arg was found and give
         // the caller a way to exit gracefully.
         changed = true;
       }
     } else {
+      // Process default arguments
+      if (arg == "-index-store-path") {
+        consumer("-index-store-path");
+        ++itr;
+
+        // If there was a global index store set, pass that to swiftc.
+        // Otherwise, pass the users. We later copy index data onto the users.
+        if (global_index_store_import_path_ != "") {
+          new_arg = global_index_store_import_path_;
+        } else {
+          new_arg = index_store_path_;
+        }
+        changed = true;
+      } else if (arg == "-output-file-map") {
+        // Save the output file map to the value proceeding
+        // `-output-file-map`
+        consumer("-output-file-map");
+        ++itr;
+        new_arg = output_file_map_path_;
+        changed = true;
+      }
+
       // Apply any other text substitutions needed in the argument (i.e., for
       // Apple toolchains).
       //
@@ -236,6 +335,36 @@ bool SwiftRunner::ProcessArgument(
   return changed;
 }
 
+template <typename Iterator>
+std::vector<std::string> SwiftRunner::ParseArguments(Iterator itr) {
+  std::vector<std::string> out_args;
+  for (auto it = itr.begin(); it != itr.end(); ++it) {
+    auto arg = *it;
+    out_args.push_back(arg);
+
+    if (StripPrefix("-Xwrapped-swift=", arg)) {
+      if (StripPrefix("-global-index-store-import-path=", arg)) {
+        global_index_store_import_path_ = arg;
+      } else if (StripPrefix("-generated-header-rewriter=", arg)) {
+        generated_header_rewriter_path_ = arg;
+      }
+    } else {
+      if (arg == "-output-file-map") {
+        ++it;
+        arg = *it;
+        output_file_map_path_ = arg;
+        out_args.push_back(arg);
+      } else if (arg == "-index-store-path") {
+        ++it;
+        arg = *it;
+        index_store_path_ = arg;
+        out_args.push_back(arg);
+      }
+    }
+  }
+  return out_args;
+}
+
 std::vector<std::string> SwiftRunner::ProcessArguments(
     const std::vector<std::string> &args) {
   std::vector<std::string> new_args;
@@ -248,16 +377,19 @@ std::vector<std::string> SwiftRunner::ProcessArguments(
 #endif
 
   // The tool is assumed to be the first argument. Push it directly.
-  auto it = args.begin();
+  auto parsed_args = ParseArguments(args);
+
+  auto it = parsed_args.begin();
   new_args.push_back(*it++);
 
   // If we're forcing response files, push the remaining processed args onto a
   // different vector that we write out below. If not, push them directly onto
   // the vector being returned.
   auto &args_destination = force_response_file_ ? response_file_args : new_args;
-  while (it != args.end()) {
-    ProcessArgument(
-        *it, [&](const std::string &arg) { args_destination.push_back(arg); });
+  while (it != parsed_args.end()) {
+    ProcessArgument(it, *it, [&](const std::string &arg) {
+      args_destination.push_back(arg);
+    });
     ++it;
   }