Implement RemoteExecutionBuildTargetsQueue to be used by LOCAL_BUILD_…

…WITH_REMOTE_EXECUTION.

Summary: This remote strategy will be used by remote execution.

Reviewed By: shivanker

fbshipit-source-id: 5a8a7db

src/com/facebook/buck/distributed/build_client/AbstractRemoteExecutionInfo.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2018-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.distributed.build_client;
+
+import com.facebook.buck.core.util.immutables.BuckStyleImmutable;
+import java.util.Optional;
+import org.immutables.value.Value;
+
+/** In flight information about RemoteExecution. */
+@Value.Immutable
+@BuckStyleImmutable
+interface AbstractRemoteExecutionInfo {
+
+  /** The state of execution. */
+  RemoteExecutionState getState();
+
+  /** The BuildTarget this refers to. */
+  String getBuildTarget();
+
+  /** Elapsed time since this target started being processed. */
+  Optional<Long> getDurationMillis();
+}

src/com/facebook/buck/distributed/build_client/RemoteExecutionBuildTargetsQueue.java

@@ -0,0 +1,173 @@
+/*
+ * Copyright 2018-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.distributed.build_client;
+
+import com.facebook.buck.distributed.build_slave.BuildTargetsQueue;
+import com.facebook.buck.distributed.build_slave.DistributableBuildGraph;
+import com.facebook.buck.distributed.thrift.CoordinatorBuildProgress;
+import com.facebook.buck.distributed.thrift.WorkUnit;
+import com.facebook.buck.event.BuckEventBus;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Queues;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import javax.annotation.concurrent.GuardedBy;
+import sun.reflect.generics.reflectiveObjects.NotImplementedException;
+
+/** BuildTargetsQueue implementation used to run in the Remote Execution model. */
+public class RemoteExecutionBuildTargetsQueue implements BuildTargetsQueue {
+  @GuardedBy("this")
+  private final Queue<TargetToBuild> targetsWaitingToBeBuilt;
+
+  @GuardedBy("this")
+  private final Map<String, TargetToBuild> targetsBuilding;
+
+  private final BuckEventBus eventBus;
+
+  private volatile boolean haveRemoteMachinesConnected;
+  private volatile int totalTargetsEnqueued;
+  private volatile int totalTargetsBuilt;
+
+  private static class TargetToBuild {
+    private final String targetName;
+    private final SettableFuture<Void> completionFuture;
+
+    private TargetToBuild(String targetName) {
+      this.targetName = targetName;
+      this.completionFuture = SettableFuture.create();
+    }
+
+    public String getTargetName() {
+      return targetName;
+    }
+
+    public SettableFuture<Void> getCompletionFuture() {
+      return completionFuture;
+    }
+
+    @Override
+    public int hashCode() {
+      return targetName.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (obj instanceof TargetToBuild) {
+        return targetName.equals(((TargetToBuild) obj).targetName);
+      }
+
+      return false;
+    }
+  }
+
+  public RemoteExecutionBuildTargetsQueue(BuckEventBus eventBus) {
+    this.eventBus = eventBus;
+    this.targetsWaitingToBeBuilt = Queues.newArrayDeque();
+    this.targetsBuilding = Maps.newConcurrentMap();
+    this.haveRemoteMachinesConnected = false;
+    this.totalTargetsEnqueued = 0;
+    this.totalTargetsBuilt = 0;
+  }
+
+  /** Async enqueues a build target to be executed remotely asap. */
+  public ListenableFuture<?> enqueueForRemoteBuild(String buildTarget) {
+    eventBus.post(
+        new RemoteExecutionEvent(
+            RemoteExecutionInfo.builder()
+                .setState(RemoteExecutionState.ENQUEUED)
+                .setBuildTarget(buildTarget)
+                .build()));
+    TargetToBuild target = new TargetToBuild(buildTarget);
+    synchronized (this) {
+      targetsWaitingToBeBuilt.add(target);
+    }
+
+    return target.getCompletionFuture();
+  }
+
+  public boolean haveRemoteMachinesConnected() {
+    return this.haveRemoteMachinesConnected;
+  }
+
+  @Override
+  public boolean hasReadyZeroDependencyNodes() {
+    synchronized (this) {
+      return !targetsWaitingToBeBuilt.isEmpty();
+    }
+  }
+
+  @Override
+  public List<WorkUnit> dequeueZeroDependencyNodes(List<String> finishedNodes, int maxUnitsOfWork) {
+    this.haveRemoteMachinesConnected = true;
+    List<WorkUnit> newWorkload = Lists.newArrayList();
+
+    synchronized (this) {
+      for (String finishedTarget : finishedNodes) {
+        TargetToBuild target = Preconditions.checkNotNull(targetsBuilding.remove(finishedTarget));
+        target.getCompletionFuture().set(null);
+        eventBus.post(
+            new RemoteExecutionEvent(
+                RemoteExecutionInfo.builder()
+                    .setState(RemoteExecutionState.REMOTE_BUILD_FINISHED)
+                    .setBuildTarget(target.getTargetName())
+                    .build()));
+      }
+
+      int newWorkCount = Math.min(targetsWaitingToBeBuilt.size(), maxUnitsOfWork);
+      while (newWorkCount-- > 0) {
+        TargetToBuild target = targetsWaitingToBeBuilt.remove();
+        targetsBuilding.put(target.getTargetName(), target);
+        WorkUnit unit = new WorkUnit().setBuildTargets(Lists.newArrayList(target.getTargetName()));
+        newWorkload.add(unit);
+
+        eventBus.post(
+            new RemoteExecutionEvent(
+                RemoteExecutionInfo.builder()
+                    .setState(RemoteExecutionState.REMOTE_BUILD_STARTED)
+                    .setBuildTarget(target.getTargetName())
+                    .build()));
+      }
+    }
+
+    return newWorkload;
+  }
+
+  @Override
+  public boolean haveMostBuildRulesFinished() {
+    return false;
+  }
+
+  @Override
+  public CoordinatorBuildProgress getBuildProgress() {
+    CoordinatorBuildProgress progress =
+        new CoordinatorBuildProgress()
+            .setBuiltRulesCount(totalTargetsBuilt)
+            .setTotalRulesCount(totalTargetsEnqueued);
+    return progress;
+  }
+
+  @Override
+  public DistributableBuildGraph getDistributableBuildGraph() {
+    // This class has no knowledge of the build graph.
+    throw new NotImplementedException();
+  }
+}

src/com/facebook/buck/distributed/build_client/RemoteExecutionEvent.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2018-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.distributed.build_client;
+
+import com.facebook.buck.event.AbstractBuckEvent;
+import com.facebook.buck.event.EventKey;
+import com.facebook.buck.event.WorkAdvanceEvent;
+
+/** Remote execution events sent to the event bus. */
+public class RemoteExecutionEvent extends AbstractBuckEvent implements WorkAdvanceEvent {
+
+  private final RemoteExecutionInfo info;
+
+  public RemoteExecutionEvent(RemoteExecutionInfo info) {
+    super(EventKey.unique());
+    this.info = info;
+  }
+
+  public RemoteExecutionInfo getInfo() {
+    return info;
+  }
+
+  @Override
+  protected String getValueString() {
+    return String.format("%s:%s", info.getBuildTarget(), info.getState());
+  }
+
+  @Override
+  public String getEventName() {
+    return info.getState().toString();
+  }
+}

src/com/facebook/buck/distributed/build_client/RemoteExecutionState.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2018-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.distributed.build_client;
+
+/** State of the current remote execution unit. */
+public enum RemoteExecutionState {
+  ENQUEUED,
+  REMOTE_BUILD_STARTED,
+  REMOTE_BUILD_FINISHED,
+  DOWNLOADING_RESULT,
+  BUILD_FINISHED,
+  BUILT_LOCALLY,
+  BUILT_LOCALLY_BECAUSE_OF_CACHING_ISSUE,
+}

test/com/facebook/buck/distributed/build_client/BUCK

@@ -1,4 +1,7 @@
-load("//tools/build_rules:java_rules.bzl", "standard_java_test")
+load(
+    "//tools/build_rules:java_rules.bzl",
+    "standard_java_test",
+)
 
 standard_java_test(
     name = "build_client",
@@ -17,6 +20,7 @@ standard_java_test(
         "//src/com/facebook/buck/core/sourcepath/resolver/impl:impl",
         "//src/com/facebook/buck/distributed:common",
         "//src/com/facebook/buck/distributed/build_client:build_client",
+        "//src/com/facebook/buck/distributed/build_slave:build_slave",
         "//src/com/facebook/buck/event:event",
         "//src/com/facebook/buck/event:interfaces",
         "//src/com/facebook/buck/graph:graph",
@@ -49,6 +53,7 @@ standard_java_test(
         "//test/com/facebook/buck/util/concurrent:testutil",
         "//test/com/facebook/buck/util/config:testutil",
         "//third-party/java/easymock:easymock",
+        "//third-party/java/jackson:jackson-annotations",
         "//third-party/java/junit:junit",
     ],
 )

test/com/facebook/buck/distributed/build_client/RemoteExecutionBuildTargetsQueueTest.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2018-present Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.facebook.buck.distributed.build_client;
+
+import com.facebook.buck.distributed.thrift.WorkUnit;
+import com.facebook.buck.event.BuckEventBus;
+import com.google.common.collect.Lists;
+import java.util.List;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class RemoteExecutionBuildTargetsQueueTest {
+
+  private static final int MAX_WORK_UNITS = Integer.MAX_VALUE;
+
+  private BuckEventBus buckEventBus;
+  private RemoteExecutionBuildTargetsQueue queue;
+
+  @Before
+  public void setUp() {
+    buckEventBus = EasyMock.createNiceMock(BuckEventBus.class);
+    queue = new RemoteExecutionBuildTargetsQueue(buckEventBus);
+  }
+
+  @Test
+  public void testEnqueuingOneTarget() {
+    String target = "super target";
+    queue.enqueueForRemoteBuild(target);
+
+    List<String> finishedNodes = Lists.newArrayList();
+    List<WorkUnit> workUnits = queue.dequeueZeroDependencyNodes(finishedNodes, MAX_WORK_UNITS);
+    Assert.assertEquals(1, workUnits.size());
+    Assert.assertEquals(1, workUnits.get(0).getBuildTargetsSize());
+    Assert.assertEquals(target, workUnits.get(0).getBuildTargets().get(0));
+  }
+}