Have BES uploads fail gracefully if the BES is not reachable due to b…

…ad network connection. PiperOrigin-RevId: 249268815
keith · May 21, 2019 · acf55d8 · acf55d8
1 parent 7f49531
commit acf55d8
Show file tree

Hide file tree

Showing 7 changed files with 62 additions and 0 deletions.
diff --git a/src/BUILD b/src/BUILD
@@ -435,6 +435,7 @@ filegroup(
         "//src/main/cpp:srcs",
         "//src/main/java/com/google/devtools/build/docgen:srcs",
         "//src/main/java/com/google/devtools/build/lib:srcs",
+        "//src/main/java/com/google/devtools/build/lib/network:srcs",
         "//src/main/java/com/google/devtools/build/skydoc:srcs",
         "//src/main/java/com/google/devtools/build/skyframe:srcs",
         "//src/main/java/com/google/devtools/common/options:srcs",

diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD
@@ -841,6 +841,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/buildeventservice",
         "//src/main/java/com/google/devtools/build/lib/dynamic",
         "//src/main/java/com/google/devtools/build/lib/metrics:metrics_module",
+        "//src/main/java/com/google/devtools/build/lib/network:connectivity",
         "//src/main/java/com/google/devtools/build/lib/profiler/callcounts:callcounts_module",
         "//src/main/java/com/google/devtools/build/lib/profiler/memory:allocationtracker_module",
         "//src/main/java/com/google/devtools/build/lib/remote",

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java
@@ -60,6 +60,7 @@ public final class Bazel {
           com.google.devtools.build.lib.dynamic.DynamicExecutionModule.class,
           com.google.devtools.build.lib.bazel.rules.BazelRulesModule.class,
           com.google.devtools.build.lib.bazel.rules.BazelStrategyModule.class,
+          com.google.devtools.build.lib.network.ConnectivityModule.class,
           com.google.devtools.build.lib.buildeventservice.BazelBuildEventServiceModule.class,
           com.google.devtools.build.lib.profiler.callcounts.CallcountsModule.class,
           com.google.devtools.build.lib.profiler.memory.AllocationTrackerModule.class,

diff --git a/src/main/java/com/google/devtools/build/lib/buildeventservice/BUILD b/src/main/java/com/google/devtools/build/lib/buildeventservice/BUILD
@@ -36,6 +36,8 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/buildeventstream",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto",
         "//src/main/java/com/google/devtools/build/lib/buildeventstream/transports",
+        "//src/main/java/com/google/devtools/build/lib/network:connectivity",
+        "//src/main/java/com/google/devtools/build/lib/network:connectivity_status",
         "//src/main/java/com/google/devtools/build/lib/profiler",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/common/options",

diff --git a/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java b/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java
@@ -48,6 +48,9 @@
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.EventHandler;
 import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.network.ConnectivityStatus;
+import com.google.devtools.build.lib.network.ConnectivityStatus.Status;
+import com.google.devtools.build.lib.network.ConnectivityStatusProvider;
 import com.google.devtools.build.lib.profiler.AutoProfiler;
 import com.google.devtools.build.lib.runtime.BlazeCommandEventHandler;
 import com.google.devtools.build.lib.runtime.BlazeModule;
@@ -128,6 +131,8 @@ public abstract class BuildEventServiceModule<BESOptionsT extends BuildEventServ
   @Nullable private String invocationId;
   @Nullable private Reporter cmdLineReporter;
   @Nullable private BuildEventStreamer streamer;
+  @Nullable private ConnectivityStatusProvider connectivityProvider;
+  private static final String CONNECTIVITY_CACHE_KEY = "BES";
 
   protected BESOptionsT besOptions;
 
@@ -227,6 +232,15 @@ public void beforeCommand(CommandEnvironment cmdEnv) {
     this.buildRequestId = cmdEnv.getBuildRequestId();
     this.cmdLineReporter = cmdEnv.getReporter();
 
+    for (BlazeModule module : cmdEnv.getRuntime().getBlazeModules()) {
+      if (module instanceof ConnectivityStatusProvider) {
+        this.connectivityProvider = (ConnectivityStatusProvider) module;
+        break;
+      }
+    }
+    Preconditions.checkNotNull(
+        this.connectivityProvider, "No ConnectivityStatusProvider found in modules list");
+
     OptionsParsingResult parsingResult = cmdEnv.getOptions();
     this.besOptions = Preconditions.checkNotNull(parsingResult.getOptions(optionsClass()));
     this.bepOptions =
@@ -576,6 +590,18 @@ private BuildEventServiceTransport createBesTransport(
 
     constructAndReportIds();
 
+    ConnectivityStatus status = connectivityProvider.getStatus(CONNECTIVITY_CACHE_KEY);
+    if (status.status != Status.OK) {
+      clearBesClient();
+      String message =
+          String.format(
+              "Build Event Service uploads disabled due to a connectivity problem: %s",
+              status.toString());
+      cmdLineReporter.handle(Event.warn(message));
+      googleLogger.atWarning().log(message);
+      return null;
+    }
+
     final BuildEventServiceClient besClient;
     try {
       besClient = getBesClient(besOptions, authTlsOptions);

diff --git a/src/main/java/com/google/devtools/build/lib/network/BUILD b/src/main/java/com/google/devtools/build/lib/network/BUILD
@@ -0,0 +1,29 @@
+package(
+    default_visibility = ["//src:__subpackages__"],
+)
+
+filegroup(
+    name = "srcs",
+    srcs = glob(["*"]),
+)
+
+java_library(
+    name = "connectivity",
+    srcs = ["ConnectivityModule.java"],
+    deps = [
+        ":connectivity_status",
+        "//src/main/java/com/google/devtools/build/lib:runtime",
+        "//src/main/java/com/google/devtools/build/lib:util",
+        "//src/main/java/com/google/devtools/common/options",
+        "//third_party:guava",
+    ],
+)
+
+java_library(
+    name = "connectivity_status",
+    srcs = [
+        "ConnectivityStatus.java",
+        "ConnectivityStatusProvider.java",
+    ],
+    deps = ["//third_party:guava"],
+)
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
@@ -466,6 +466,8 @@ public BuildOptions getDefaultBuildOptions() {
 
   @SuppressWarnings("unchecked")
   public <T extends BlazeModule> T getBlazeModule(Class<T> moduleClass) {
+    // TODO(steinman): Change this to include subclasses, and clean up places where we're iterating
+    // through all the modules to find an instance of the superclass.
     for (BlazeModule module : blazeModules) {
       if (module.getClass() == moduleClass) {
         return (T) module;