Skip to content

Commit

Permalink
Automated rollback of commit 526ec17.
Browse files Browse the repository at this point in the history 
*** Reason for rollback ***

b/119190539

*** Original change description ***

Add a new BEP event for the location of output from the `query`, `cquery`, and `aquery` commands. Add a new common query flag --upload_query_output_using_bep that controls whether these commands print their output to the console or whether they upload them to remote storage using BEP.

RELNOTES: The new --upload_query_output_using_bep query/cquery/aquery flag causes query outputs to be uploaded via BEP.
PiperOrigin-RevId: 220530561
  • Loading branch information
@haxorz
haxorz authored and Copybara-Service committed Nov 7, 2018
1 parent 2b98ef7 commit f4023b9
Show file tree
Hide file tree
Showing 24 changed files with 181 additions and 790 deletions.
73 changes: 9 additions & 64 deletions src/main/java/com/google/devtools/build/lib/analysis/NoBuildEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import com.google.devtools.build.lib.buildeventstream.GenericBuildEvent;
import com.google.devtools.build.lib.buildeventstream.ProgressEvent;
import com.google.devtools.build.lib.util.ProcessUtils;
import java.util.ArrayList;
import java.util.Collection;

/** This event raised to indicate that no build will be happening for the given command. */
Expand All @@ -32,89 +31,35 @@ public final class NoBuildEvent implements BuildEvent {
private final Long startTimeMillis;
private final boolean separateFinishedEvent;
private final boolean showProgress;
private final ImmutableList<BuildEventId> additionalChildrenEvents;

private NoBuildEvent(
public NoBuildEvent(
String command,
Long startTimeMillis,
boolean separateFinishedEvent,
boolean showProgress,
String id,
ImmutableList<BuildEventId> additionalChildrenEvents) {
String id) {
this.command = command;
this.startTimeMillis = startTimeMillis;
this.separateFinishedEvent = separateFinishedEvent;
this.showProgress = showProgress;
this.id = id;
this.additionalChildrenEvents = additionalChildrenEvents;
}

public static Builder newBuilder() {
return new Builder();
public NoBuildEvent(String command, Long startTimeMillis, boolean separateFinishedEvent) {
this(command, startTimeMillis, separateFinishedEvent, false, null);
}

/** Builder for {@link NoBuildEvent}. */
public static class Builder {
private String command = null;
private String id = null;
private boolean showProgress = false;
private Long startTimeMillis = null;
private boolean separateFinishedEvent = false;
private ArrayList<BuildEventId> additionalChildrenEvents = new ArrayList<>();

private Builder() {
}

public Builder setCommand(String command) {
this.command = command;
return this;
}

public Builder setId(String id) {
this.id = id;
return this;
}

public Builder setShowProgress(boolean showProgress) {
this.showProgress = showProgress;
return this;
}

public Builder setStartTimeMillis(long startTimeMillis) {
this.startTimeMillis = startTimeMillis;
return this;
}

public Builder setSeparateFinishedEvent(boolean separateFinishedEvent) {
this.separateFinishedEvent = separateFinishedEvent;
return this;
}

public Builder addAdditionalChildrenEvents(Iterable<BuildEventId> additionalChildrenEvents) {
additionalChildrenEvents.forEach(this.additionalChildrenEvents::add);
return this;
}

public NoBuildEvent build() {
return new NoBuildEvent(
command,
startTimeMillis,
separateFinishedEvent,
showProgress,
id,
ImmutableList.copyOf(additionalChildrenEvents));
}
public NoBuildEvent() {
this(null, null, false);
}

@Override
public Collection<BuildEventId> getChildrenEvents() {
ImmutableList.Builder allChildrenEventsBuilder = ImmutableList.builder();
allChildrenEventsBuilder.add(ProgressEvent.INITIAL_PROGRESS_UPDATE);
allChildrenEventsBuilder.addAll(additionalChildrenEvents);
if (separateFinishedEvent) {
allChildrenEventsBuilder.add(BuildEventId.buildFinished());
return ImmutableList.of(ProgressEvent.INITIAL_PROGRESS_UPDATE, BuildEventId.buildFinished());
} else {
return ImmutableList.of(ProgressEvent.INITIAL_PROGRESS_UPDATE);
}
return allChildrenEventsBuilder.build();
}

@Override
Expand Down
5 changes: 1 addition & 4 deletions src/main/java/com/google/devtools/build/lib/analysis/NoBuildRequestFinishedEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,7 @@
import com.google.devtools.build.lib.buildeventstream.BuildCompletingEvent;
import com.google.devtools.build.lib.util.ExitCode;

/**
* A {@link BuildEvent} indicating that a request (that does not involve building) should be treated
* as finished.
*/
/** {@link BuildEvent} indicating that a request that does not involve building as finished. */
public final class NoBuildRequestFinishedEvent extends BuildCompletingEvent {
public NoBuildRequestFinishedEvent(ExitCode exitCode, long finishTimeMillis) {
super(exitCode, finishTimeMillis);
Expand Down
13 changes: 6 additions & 7 deletions src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,13 +116,12 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti

env.getReporter()
.post(
NoBuildEvent.newBuilder()
.setCommand(env.getCommandName())
.setStartTimeMillis(env.getCommandStartTime())
.setSeparateFinishedEvent(true)
.setShowProgress(true)
.setId(env.getCommandId().toString())
.build());
new NoBuildEvent(
env.getCommandName(),
env.getCommandStartTime(),
true,
true,
env.getCommandId().toString()));

// 2. Evaluate expression:
QueryEvalResult queryEvalResult = null;
Expand Down
13 changes: 6 additions & 7 deletions src/main/java/com/google/devtools/build/lib/bazel/commands/SyncCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,12 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti
try {
env.getReporter()
.post(
NoBuildEvent.newBuilder()
.setCommand(env.getCommandName())
.setStartTimeMillis(env.getCommandStartTime())
.setSeparateFinishedEvent(true)
.setShowProgress(true)
.setId(env.getCommandId().toString())
.build());
new NoBuildEvent(
env.getCommandName(),
env.getCommandStartTime(),
true,
true,
env.getCommandId().toString()));
env.setupPackageCache(options, env.getRuntime().getDefaultsPackageContent());
SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor();
skyframeExecutor.injectExtraPrecomputedValues(
Expand Down
8 changes: 0 additions & 8 deletions src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventId.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,12 +313,4 @@ public static BuildEventId buildMetrics() {
BuildEventStreamProtos.BuildEventId.BuildMetricsId.getDefaultInstance())
.build());
}

public static BuildEventId queryOutput() {
return new BuildEventId(
BuildEventStreamProtos.BuildEventId.newBuilder()
.setQueryOutput(
BuildEventStreamProtos.BuildEventId.QueryOutputId.getDefaultInstance())
.build());
}
}
31 changes: 0 additions & 31 deletions src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,11 +193,6 @@ message BuildEventId {
message BuildMetricsId {
}

// Identifier of an event providing information about the location of
// query/cquery/acquery command output.
message QueryOutputId {
}

oneof id {
UnknownBuildEventId unknown = 1;
ProgressId progress = 2;
Expand All @@ -221,7 +216,6 @@ message BuildEventId {
BuildFinishedId build_finished = 9;
BuildToolLogsId build_tool_logs = 20;
BuildMetricsId build_metrics = 22;
QueryOutputId query_output = 23;
}
}

Expand Down Expand Up @@ -671,30 +665,6 @@ message BuildToolLogs {
repeated File log = 1;
}

// For 'query', 'cquery', and 'aquery' commands, an Event that describes where
// the output was written to.
message QueryOutput {
// Marker message type to indicate that Bazel printed the query output to
// stdout on the console. Note that this also implies that the output is in a
// Progress event.
message OutputPrintedToConsole {
}

// Marker message type to indicate that the query output upload didn't occur
// for some reason. Either there was an error, or the appropriate flags to
// enable remote file uploading weren't set,
message UploadFailed {
}

oneof payload {
OutputPrintedToConsole output_printed_to_console = 1;
// If this field is set, 'query_output_file.uri' will be the URI of where
// the query output was uploaded.
File query_output_file = 2;
UploadFailed upload_failed = 3;
};
}

// Message describing a build event. Events will have an identifier that
// is unique within a given build invocation; they also announce follow-up
// events as children. More details, which are specific to the kind of event
Expand Down Expand Up @@ -725,6 +695,5 @@ message BuildEvent {
BuildFinished finished = 14;
BuildToolLogs build_tool_logs = 23;
BuildMetrics build_metrics = 24;
QueryOutput query_output = 25;
};
}
9 changes: 1 addition & 8 deletions src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,10 +117,7 @@ public void buildTargets(BuildRequest request, BuildResult result, TargetValidat
boolean catastrophe = false;
try {
try (SilentCloseable c = Profiler.instance().profile("BuildStartingEvent")) {
env.getEventBus().post(new BuildStartingEvent(
env,
request,
getAdditionalChildrenEventsForBuildStartingEvent()));
env.getEventBus().post(new BuildStartingEvent(env, request));
}
logger.info("Build identifier: " + request.getId());

Expand Down Expand Up @@ -221,10 +218,6 @@ protected void postProcessAnalysisResult(
PostAnalysisQueryCommandLineException {
}

protected ImmutableList<BuildEventId> getAdditionalChildrenEventsForBuildStartingEvent() {
return ImmutableList.of();
}

private void reportExceptionError(Exception e) {
if (e.getMessage() != null) {
getReporter().handle(Event.error(e.getMessage()));
Expand Down
66 changes: 28 additions & 38 deletions src/main/java/com/google/devtools/build/lib/buildtool/PostAnalysisQueryBuildTool.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,9 @@
// limitations under the License.
package com.google.devtools.build.lib.buildtool;

import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.analysis.AnalysisResult;
import com.google.devtools.build.lib.analysis.ViewCreationFailedException;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.buildeventstream.BuildEventId;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.query2.NamedThreadSafeOutputFormatterCallback;
import com.google.devtools.build.lib.query2.PostAnalysisQueryEnvironment;
Expand All @@ -26,7 +24,6 @@
import com.google.devtools.build.lib.query2.engine.QueryException;
import com.google.devtools.build.lib.query2.engine.QueryExpression;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
import com.google.devtools.build.lib.runtime.QueryBEPHelper;
import com.google.devtools.build.lib.skyframe.SkyframeExecutorWrappingWalkableGraph;
import com.google.devtools.build.skyframe.WalkableGraph;
import java.io.IOException;
Expand Down Expand Up @@ -82,11 +79,6 @@ protected abstract PostAnalysisQueryEnvironment<T> getQueryEnvironment(
TopLevelConfigurations topLevelConfigurations,
WalkableGraph walkableGraph);

@Override
protected ImmutableList<BuildEventId> getAdditionalChildrenEventsForBuildStartingEvent() {
return ImmutableList.of(QueryBEPHelper.getBuildEventIdOfQueryOutputEvent());
}

private void doPostAnalysisQuery(
BuildRequest request,
BuildConfiguration hostConfiguration,
Expand All @@ -95,38 +87,36 @@ private void doPostAnalysisQuery(
throws InterruptedException, QueryException, IOException {
WalkableGraph walkableGraph =
SkyframeExecutorWrappingWalkableGraph.of(env.getSkyframeExecutor());

PostAnalysisQueryEnvironment<T> postAnalysisQueryEnvironment =
getQueryEnvironment(request, hostConfiguration, topLevelConfigurations, walkableGraph);
try (QueryBEPHelper queryBepHelper =
QueryBEPHelper.create(env, postAnalysisQueryEnvironment.getCommonQueryOptions())) {
Iterable<NamedThreadSafeOutputFormatterCallback<T>> callbacks =
postAnalysisQueryEnvironment.getDefaultOutputFormatters(
postAnalysisQueryEnvironment.getAccessor(),
env.getReporter(),
queryBepHelper.getOutputStreamForQueryOutput(),
env.getSkyframeExecutor(),
hostConfiguration,
runtime.getRuleClassProvider().getTrimmingTransitionFactory(),
env.getPackageManager());
String outputFormat = postAnalysisQueryEnvironment.getOutputFormat();
NamedThreadSafeOutputFormatterCallback<T> callback =
NamedThreadSafeOutputFormatterCallback.selectCallback(outputFormat, callbacks);
if (callback == null) {
env.getReporter()
.handle(
Event.error(
String.format(
"Invalid output format '%s'. Valid values are: %s",
outputFormat,
NamedThreadSafeOutputFormatterCallback.callbackNames(callbacks))));
return;
}
QueryEvalResult result =
postAnalysisQueryEnvironment.evaluateQuery(queryExpression, callback);
if (result.isEmpty()) {
env.getReporter().handle(Event.info("Empty query results"));
}
queryBepHelper.afterQueryOutputIsWritten();

Iterable<NamedThreadSafeOutputFormatterCallback<T>> callbacks =
postAnalysisQueryEnvironment.getDefaultOutputFormatters(
postAnalysisQueryEnvironment.getAccessor(),
env.getReporter(),
env.getReporter().getOutErr().getOutputStream(),
env.getSkyframeExecutor(),
hostConfiguration,
runtime.getRuleClassProvider().getTrimmingTransitionFactory(),
env.getPackageManager());
String outputFormat = postAnalysisQueryEnvironment.getOutputFormat();
NamedThreadSafeOutputFormatterCallback<T> callback =
NamedThreadSafeOutputFormatterCallback.selectCallback(outputFormat, callbacks);
if (callback == null) {
env.getReporter()
.handle(
Event.error(
String.format(
"Invalid output format '%s'. Valid values are: %s",
outputFormat,
NamedThreadSafeOutputFormatterCallback.callbackNames(callbacks))));
return;
}
QueryEvalResult result =
postAnalysisQueryEnvironment.evaluateQuery(queryExpression, callback);
if (result.isEmpty()) {
env.getReporter().handle(Event.info("Empty query results"));
}
}

Expand Down
Loading

0 comments on commit f4023b9

Please sign in to comment.