diff --git a/src/com/facebook/buck/distributed/build_client/AbstractRemoteExecutionInfo.java b/src/com/facebook/buck/distributed/build_client/AbstractRemoteExecutionInfo.java new file mode 100644 index 00000000000..353f57f3329 --- /dev/null +++ b/src/com/facebook/buck/distributed/build_client/AbstractRemoteExecutionInfo.java @@ -0,0 +1,36 @@ +/* + * Copyright 2018-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.distributed.build_client; + +import com.facebook.buck.core.util.immutables.BuckStyleImmutable; +import java.util.Optional; +import org.immutables.value.Value; + +/** In flight information about RemoteExecution. */ +@Value.Immutable +@BuckStyleImmutable +interface AbstractRemoteExecutionInfo { + + /** The state of execution. */ + RemoteExecutionState getState(); + + /** The BuildTarget this refers to. */ + String getBuildTarget(); + + /** Elapsed time since this target started being processed. */ + Optional getDurationMillis(); +} diff --git a/src/com/facebook/buck/distributed/build_client/RemoteExecutionBuildTargetsQueue.java b/src/com/facebook/buck/distributed/build_client/RemoteExecutionBuildTargetsQueue.java new file mode 100644 index 00000000000..0f3538b3996 --- /dev/null +++ b/src/com/facebook/buck/distributed/build_client/RemoteExecutionBuildTargetsQueue.java @@ -0,0 +1,173 @@ +/* + * Copyright 2018-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.distributed.build_client; + +import com.facebook.buck.distributed.build_slave.BuildTargetsQueue; +import com.facebook.buck.distributed.build_slave.DistributableBuildGraph; +import com.facebook.buck.distributed.thrift.CoordinatorBuildProgress; +import com.facebook.buck.distributed.thrift.WorkUnit; +import com.facebook.buck.event.BuckEventBus; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Queues; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import javax.annotation.concurrent.GuardedBy; +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +/** BuildTargetsQueue implementation used to run in the Remote Execution model. */ +public class RemoteExecutionBuildTargetsQueue implements BuildTargetsQueue { + @GuardedBy("this") + private final Queue targetsWaitingToBeBuilt; + + @GuardedBy("this") + private final Map targetsBuilding; + + private final BuckEventBus eventBus; + + private volatile boolean haveRemoteMachinesConnected; + private volatile int totalTargetsEnqueued; + private volatile int totalTargetsBuilt; + + private static class TargetToBuild { + private final String targetName; + private final SettableFuture completionFuture; + + private TargetToBuild(String targetName) { + this.targetName = targetName; + this.completionFuture = SettableFuture.create(); + } + + public String getTargetName() { + return targetName; + } + + public SettableFuture getCompletionFuture() { + return completionFuture; + } + + @Override + public int hashCode() { + return targetName.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof TargetToBuild) { + return targetName.equals(((TargetToBuild) obj).targetName); + } + + return false; + } + } + + public RemoteExecutionBuildTargetsQueue(BuckEventBus eventBus) { + this.eventBus = eventBus; + this.targetsWaitingToBeBuilt = Queues.newArrayDeque(); + this.targetsBuilding = Maps.newConcurrentMap(); + this.haveRemoteMachinesConnected = false; + this.totalTargetsEnqueued = 0; + this.totalTargetsBuilt = 0; + } + + /** Async enqueues a build target to be executed remotely asap. */ + public ListenableFuture enqueueForRemoteBuild(String buildTarget) { + eventBus.post( + new RemoteExecutionEvent( + RemoteExecutionInfo.builder() + .setState(RemoteExecutionState.ENQUEUED) + .setBuildTarget(buildTarget) + .build())); + TargetToBuild target = new TargetToBuild(buildTarget); + synchronized (this) { + targetsWaitingToBeBuilt.add(target); + } + + return target.getCompletionFuture(); + } + + public boolean haveRemoteMachinesConnected() { + return this.haveRemoteMachinesConnected; + } + + @Override + public boolean hasReadyZeroDependencyNodes() { + synchronized (this) { + return !targetsWaitingToBeBuilt.isEmpty(); + } + } + + @Override + public List dequeueZeroDependencyNodes(List finishedNodes, int maxUnitsOfWork) { + this.haveRemoteMachinesConnected = true; + List newWorkload = Lists.newArrayList(); + + synchronized (this) { + for (String finishedTarget : finishedNodes) { + TargetToBuild target = Preconditions.checkNotNull(targetsBuilding.remove(finishedTarget)); + target.getCompletionFuture().set(null); + eventBus.post( + new RemoteExecutionEvent( + RemoteExecutionInfo.builder() + .setState(RemoteExecutionState.REMOTE_BUILD_FINISHED) + .setBuildTarget(target.getTargetName()) + .build())); + } + + int newWorkCount = Math.min(targetsWaitingToBeBuilt.size(), maxUnitsOfWork); + while (newWorkCount-- > 0) { + TargetToBuild target = targetsWaitingToBeBuilt.remove(); + targetsBuilding.put(target.getTargetName(), target); + WorkUnit unit = new WorkUnit().setBuildTargets(Lists.newArrayList(target.getTargetName())); + newWorkload.add(unit); + + eventBus.post( + new RemoteExecutionEvent( + RemoteExecutionInfo.builder() + .setState(RemoteExecutionState.REMOTE_BUILD_STARTED) + .setBuildTarget(target.getTargetName()) + .build())); + } + } + + return newWorkload; + } + + @Override + public boolean haveMostBuildRulesFinished() { + return false; + } + + @Override + public CoordinatorBuildProgress getBuildProgress() { + CoordinatorBuildProgress progress = + new CoordinatorBuildProgress() + .setBuiltRulesCount(totalTargetsBuilt) + .setTotalRulesCount(totalTargetsEnqueued); + return progress; + } + + @Override + public DistributableBuildGraph getDistributableBuildGraph() { + // This class has no knowledge of the build graph. + throw new NotImplementedException(); + } +} diff --git a/src/com/facebook/buck/distributed/build_client/RemoteExecutionEvent.java b/src/com/facebook/buck/distributed/build_client/RemoteExecutionEvent.java new file mode 100644 index 00000000000..dc17da60b17 --- /dev/null +++ b/src/com/facebook/buck/distributed/build_client/RemoteExecutionEvent.java @@ -0,0 +1,46 @@ +/* + * Copyright 2018-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.distributed.build_client; + +import com.facebook.buck.event.AbstractBuckEvent; +import com.facebook.buck.event.EventKey; +import com.facebook.buck.event.WorkAdvanceEvent; + +/** Remote execution events sent to the event bus. */ +public class RemoteExecutionEvent extends AbstractBuckEvent implements WorkAdvanceEvent { + + private final RemoteExecutionInfo info; + + public RemoteExecutionEvent(RemoteExecutionInfo info) { + super(EventKey.unique()); + this.info = info; + } + + public RemoteExecutionInfo getInfo() { + return info; + } + + @Override + protected String getValueString() { + return String.format("%s:%s", info.getBuildTarget(), info.getState()); + } + + @Override + public String getEventName() { + return info.getState().toString(); + } +} diff --git a/src/com/facebook/buck/distributed/build_client/RemoteExecutionState.java b/src/com/facebook/buck/distributed/build_client/RemoteExecutionState.java new file mode 100644 index 00000000000..9fadf4d2b0c --- /dev/null +++ b/src/com/facebook/buck/distributed/build_client/RemoteExecutionState.java @@ -0,0 +1,28 @@ +/* + * Copyright 2018-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.distributed.build_client; + +/** State of the current remote execution unit. */ +public enum RemoteExecutionState { + ENQUEUED, + REMOTE_BUILD_STARTED, + REMOTE_BUILD_FINISHED, + DOWNLOADING_RESULT, + BUILD_FINISHED, + BUILT_LOCALLY, + BUILT_LOCALLY_BECAUSE_OF_CACHING_ISSUE, +} diff --git a/test/com/facebook/buck/distributed/build_client/BUCK b/test/com/facebook/buck/distributed/build_client/BUCK index 6818f8d8ba4..433fff286ac 100644 --- a/test/com/facebook/buck/distributed/build_client/BUCK +++ b/test/com/facebook/buck/distributed/build_client/BUCK @@ -1,4 +1,7 @@ -load("//tools/build_rules:java_rules.bzl", "standard_java_test") +load( + "//tools/build_rules:java_rules.bzl", + "standard_java_test", +) standard_java_test( name = "build_client", @@ -17,6 +20,7 @@ standard_java_test( "//src/com/facebook/buck/core/sourcepath/resolver/impl:impl", "//src/com/facebook/buck/distributed:common", "//src/com/facebook/buck/distributed/build_client:build_client", + "//src/com/facebook/buck/distributed/build_slave:build_slave", "//src/com/facebook/buck/event:event", "//src/com/facebook/buck/event:interfaces", "//src/com/facebook/buck/graph:graph", @@ -49,6 +53,7 @@ standard_java_test( "//test/com/facebook/buck/util/concurrent:testutil", "//test/com/facebook/buck/util/config:testutil", "//third-party/java/easymock:easymock", + "//third-party/java/jackson:jackson-annotations", "//third-party/java/junit:junit", ], ) diff --git a/test/com/facebook/buck/distributed/build_client/RemoteExecutionBuildTargetsQueueTest.java b/test/com/facebook/buck/distributed/build_client/RemoteExecutionBuildTargetsQueueTest.java new file mode 100644 index 00000000000..8a5b1137251 --- /dev/null +++ b/test/com/facebook/buck/distributed/build_client/RemoteExecutionBuildTargetsQueueTest.java @@ -0,0 +1,52 @@ +/* + * Copyright 2018-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.facebook.buck.distributed.build_client; + +import com.facebook.buck.distributed.thrift.WorkUnit; +import com.facebook.buck.event.BuckEventBus; +import com.google.common.collect.Lists; +import java.util.List; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class RemoteExecutionBuildTargetsQueueTest { + + private static final int MAX_WORK_UNITS = Integer.MAX_VALUE; + + private BuckEventBus buckEventBus; + private RemoteExecutionBuildTargetsQueue queue; + + @Before + public void setUp() { + buckEventBus = EasyMock.createNiceMock(BuckEventBus.class); + queue = new RemoteExecutionBuildTargetsQueue(buckEventBus); + } + + @Test + public void testEnqueuingOneTarget() { + String target = "super target"; + queue.enqueueForRemoteBuild(target); + + List finishedNodes = Lists.newArrayList(); + List workUnits = queue.dequeueZeroDependencyNodes(finishedNodes, MAX_WORK_UNITS); + Assert.assertEquals(1, workUnits.size()); + Assert.assertEquals(1, workUnits.get(0).getBuildTargetsSize()); + Assert.assertEquals(target, workUnits.get(0).getBuildTargets().get(0)); + } +}