diff --git a/swift/internal/compiling.bzl b/swift/internal/compiling.bzl index 332a7ded3..f05de598c 100644 --- a/swift/internal/compiling.bzl +++ b/swift/internal/compiling.bzl @@ -46,6 +46,7 @@ load( "SWIFT_FEATURE_FASTBUILD", "SWIFT_FEATURE_FULL_DEBUG_INFO", "SWIFT_FEATURE_INDEX_WHILE_BUILDING", + "SWIFT_FEATURE_INDEX_WHILE_BUILDING_V2", "SWIFT_FEATURE_MINIMAL_DEPS", "SWIFT_FEATURE_MODULE_MAP_HOME_IS_CWD", "SWIFT_FEATURE_NO_EMBED_DEBUG_MODULE", @@ -731,6 +732,13 @@ def compile_action_configs(): ], ), + swift_toolchain_config.action_config( + actions = [swift_action_names.COMPILE], + + configurators = [_index_while_building_configurator], + features = [SWIFT_FEATURE_INDEX_WHILE_BUILDING_V2], + ), + # User-defined conditional compilation flags (defined for Swift; those # passed directly to ClangImporter are handled above). swift_toolchain_config.action_config( @@ -1487,6 +1495,15 @@ def compile( else: vfsoverlay_file = None + # For index while building v2 it uses a global index store which is + # configured by this flag + additional_copts = [] + if is_feature_enabled( + feature_configuration = feature_configuration, + feature_name = SWIFT_FEATURE_INDEX_WHILE_BUILDING_V2, + ): + additional_copts.append("-Xwrapped-swift=-enable-global-index-store") + prerequisites = struct( additional_inputs = additional_inputs, bin_dir = bin_dir, @@ -1502,7 +1519,7 @@ def compile( source_files = srcs, transitive_modules = transitive_modules, transitive_swiftmodules = transitive_swiftmodules, - user_compile_flags = copts + swift_toolchain.command_line_copts, + user_compile_flags = copts + swift_toolchain.command_line_copts + additional_copts, vfsoverlay_file = vfsoverlay_file, vfsoverlay_search_path = _SWIFTMODULES_VFS_ROOT, # Merge the compile outputs into the prerequisites. @@ -1834,10 +1851,19 @@ def _declare_compile_outputs( # Configure index-while-building if requested. IDEs and other indexing tools # can enable this feature on the command line during a build and then access # the index store artifacts that are produced. - index_while_building = is_feature_enabled( + index_while_building_v1 = is_feature_enabled( feature_configuration = feature_configuration, feature_name = SWIFT_FEATURE_INDEX_WHILE_BUILDING, ) + index_while_building_v2 = is_feature_enabled( + feature_configuration = feature_configuration, + feature_name = SWIFT_FEATURE_INDEX_WHILE_BUILDING_V2, + ) + + if index_while_building_v1 and index_while_building_v2: + fail("can't use both swift.index_while_building and swift.index_while_building_v2") + + index_while_building = (index_while_building_v1 or index_while_building_v2) if ( index_while_building and not _index_store_path_overridden(user_compile_flags) diff --git a/swift/internal/feature_names.bzl b/swift/internal/feature_names.bzl index 583e33178..fadefa348 100644 --- a/swift/internal/feature_names.bzl +++ b/swift/internal/feature_names.bzl @@ -99,6 +99,10 @@ SWIFT_FEATURE_ENABLE_TESTING = "swift.enable_testing" # warnings otherwise. SWIFT_FEATURE_FULL_DEBUG_INFO = "swift.full_debug_info" +# Index while building - using a global index store cache +# https://docs.google.com/document/d/1cH2sTpgSnJZCkZtJl1aY-rzy4uGPcrI-6RrUpdATO2Q/ +SWIFT_FEATURE_INDEX_WHILE_BUILDING_V2 = "swift.index_while_building_v2" + # If enabled, the compilation action for a target will produce an index store. SWIFT_FEATURE_INDEX_WHILE_BUILDING = "swift.index_while_building" diff --git a/swift/repositories.bzl b/swift/repositories.bzl index 4bef54bae..a3461ffd4 100644 --- a/swift/repositories.bzl +++ b/swift/repositories.bzl @@ -108,6 +108,40 @@ def swift_rules_dependencies(): type = "zip", ) + # It relies on `index-import` to import indexes into Bazel's remote + # cache and allow using a global index internally in workers + _maybe( + http_archive, + name = "build_bazel_rules_swift_index_import", + build_file_content = """\ +load("@bazel_skylib//rules:native_binary.bzl", "native_binary") + +native_binary( + name = "index_import", + src = "index-import", + out = "index-import", + visibility = ["//visibility:public"], +) + +native_binary( + name = "validate_index", + src = "validate-index", + out = "validate-index", + visibility = ["//visibility:public"], +) + +native_binary( + name = "absolute_unit", + src = "absolute-unit", + out = "absolute-unit", + visibility = ["//visibility:public"], +) +""", + canonical_id = "index-import-5.3.2.5", + urls = ["https://github.com/bazel-ios/index-import/releases/download/5.3.2.5/index-import.zip"], + sha256 = "79e9b2cd3e988155b86668c56d95705e1a4a7c7b6d702ff5ded3a18d1291a39a", + ) + _maybe( swift_autoconfiguration, name = "build_bazel_rules_swift_local_config", diff --git a/tools/worker/BUILD b/tools/worker/BUILD index 23bb274ae..2a3ee4d1c 100644 --- a/tools/worker/BUILD +++ b/tools/worker/BUILD @@ -18,6 +18,14 @@ config_setting( }, ) +# Internal hinge for index while building V2 feature +config_setting( + name = "index_while_building_v2", + values = { + "features": "swift.index_while_building_v2" + }, +) + cc_library( name = "compile_with_worker", srcs = [ @@ -38,6 +46,18 @@ cc_library( "@com_github_nlohmann_json//:json", "@com_google_protobuf//:protobuf", ], + copts = select({ + ":index_while_building_v2": [ + "-DINDEX_IMPORT_PATH=\\\"$(rootpath @build_bazel_rules_swift_index_import//:index_import)\\\"" + ], + "//conditions:default": [], + }), + data = select({ + ":index_while_building_v2": [ + "@build_bazel_rules_swift_index_import//:index_import" + ], + "//conditions:default": [], + }) ) cc_library( diff --git a/tools/worker/work_processor.cc b/tools/worker/work_processor.cc index a738041c1..8dce0fe3f 100644 --- a/tools/worker/work_processor.cc +++ b/tools/worker/work_processor.cc @@ -22,9 +22,11 @@ #include #include #include +#include #include "tools/common/file_system.h" #include "tools/common/path_utils.h" +#include "tools/common/process.h" #include "tools/common/string_utils.h" #include "tools/common/temp_file.h" #include "tools/worker/output_file_map.h" @@ -64,6 +66,11 @@ void WorkProcessor::ProcessWorkRequest( bool is_wmo = false; std::string prev_arg; + + std::string bazel_index_store_path; + bool enable_global_index_store = false; + + for (auto arg : request.arguments()) { auto original_arg = arg; // Peel off the `-output-file-map` argument, so we can rewrite it if @@ -77,6 +84,19 @@ void WorkProcessor::ProcessWorkRequest( is_wmo = true; } + // Peel off index-store-path, we will conditionally pop on a new one if + // it passes -Xwrapped-swift-enable-global-index-store + if (arg == "-index-store-path") { + arg.clear(); + } else if (prev_arg == "-index-store-path") { + bazel_index_store_path = arg; + arg.clear(); + } + if (arg == "-Xwrapped-swift=-enable-global-index-store") { + arg.clear(); + enable_global_index_store = true; + } + if (!arg.empty()) { params_file_stream << arg << '\n'; } @@ -84,9 +104,21 @@ void WorkProcessor::ProcessWorkRequest( prev_arg = original_arg; } + // Material within this index store is namespaced to arcitecture, os, etc + auto exec_root = GetCurrentDirectory(); + auto global_index_store = exec_root + "/bazel-out/global_index_store"; + if (enable_global_index_store) { + processed_args.push_back("-index-store-path"); + processed_args.push_back(global_index_store); + } else if (!bazel_index_store_path.empty()) { + processed_args.push_back("-index-store-path"); + processed_args.push_back(bazel_index_store_path); + } + if (!output_file_map_path.empty()) { + output_file_map.ReadFromPath(output_file_map_path); + if (!is_wmo) { - output_file_map.ReadFromPath(output_file_map_path); // Rewrite the output file map to use the incremental storage area and // pass the compiler the path to the rewritten file. @@ -130,7 +162,6 @@ void WorkProcessor::ProcessWorkRequest( SwiftRunner swift_runner(processed_args, /*force_response_file=*/true); int exit_code = swift_runner.Run(&stderr_stream, /*stdout_to_stderr=*/true); - if (!is_wmo) { // Copy the output files from the incremental storage area back to the // locations where Bazel declared the files. @@ -144,6 +175,43 @@ void WorkProcessor::ProcessWorkRequest( } } + // Import global indexes if compilation succeeds + if (exit_code != EXIT_FAILURE && enable_global_index_store) { + std::vector ii_args; + + // The index-import runfile path is pased as a define + #if defined(INDEX_IMPORT_PATH) + ii_args.push_back(INDEX_IMPORT_PATH); + #else + // Logical error + std::cerr << "Incorrectly compiled work_processor.cc"; + exit_code = EXIT_FAILURE; + #endif + + // Use the output path map to deterine what compilation ouputs to import + auto outputs = output_file_map.incremental_outputs(); + std::map::iterator it; + for (it = outputs.begin(); it != outputs.end(); it++) { + // Need the actual output paths of the compiler - not bazel + auto output_path = is_wmo ? it->first : it->second; + + auto file_type = output_path.substr(output_path.find_last_of(".") + 1); + if (file_type == "o") { + ii_args.push_back("-import-output-file"); + ii_args.push_back(output_path); + } + } + + // Copy back from the global index store to bazel's index store + ii_args.push_back(global_index_store); + + ii_args.push_back(exec_root + "/" + bazel_index_store_path); + + const std::vector& ii_args_ = ii_args; + // Dupes the output into the action for failure cases + exit_code = RunSubProcess(ii_args_, &stderr_stream, /*stdout_to_stderr=*/true); + } + response->set_exit_code(exit_code); response->set_output(stderr_stream.str()); }