Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make TargetPatternParsing repository renaming aware so that platforms and toolchains get remapped #7902

Closed
wants to merge 5 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ public String getOffset() {
* {@code Type.TARGETS_BELOW_DIRECTORY}.
*/
public abstract <T, E extends Exception> void eval(
RepositoryName repositoryName,
TargetPatternResolver<T> resolver,
ImmutableSet<PathFragment> blacklistedSubdirectories,
ImmutableSet<PathFragment> excludedSubdirectories,
Expand All @@ -168,13 +169,14 @@ public abstract <T, E extends Exception> void eval(
* {@link TargetParsingException} or the given {@code exceptionClass}.
*/
public final <T, E extends Exception> ListenableFuture<Void> evalAdaptedForAsync(
RepositoryName repositoryName,
TargetPatternResolver<T> resolver,
ImmutableSet<PathFragment> blacklistedSubdirectories,
ImmutableSet<PathFragment> excludedSubdirectories,
ThreadSafeBatchCallback<T, E> callback,
Class<E> exceptionClass) {
try {
eval(resolver, blacklistedSubdirectories, excludedSubdirectories, callback, exceptionClass);
eval(repositoryName, resolver, blacklistedSubdirectories, excludedSubdirectories, callback, exceptionClass);
return Futures.immediateFuture(null);
} catch (TargetParsingException e) {
return Futures.immediateFailedFuture(e);
Expand All @@ -197,14 +199,15 @@ public final <T, E extends Exception> ListenableFuture<Void> evalAdaptedForAsync
* {@link TargetParsingException} or the given {@code exceptionClass}.
*/
public <T, E extends Exception> ListenableFuture<Void> evalAsync(
RepositoryName repositoryName,
TargetPatternResolver<T> resolver,
ImmutableSet<PathFragment> blacklistedSubdirectories,
ImmutableSet<PathFragment> excludedSubdirectories,
ThreadSafeBatchCallback<T, E> callback,
Class<E> exceptionClass,
ListeningExecutorService executor) {
return evalAdaptedForAsync(
resolver, blacklistedSubdirectories, excludedSubdirectories, callback, exceptionClass);
repositoryName, resolver, blacklistedSubdirectories, excludedSubdirectories, callback, exceptionClass);
}

/**
Expand Down Expand Up @@ -277,7 +280,7 @@ public PackageIdentifier getDirectoryForTargetsUnderDirectory() {
* For patterns of type {@link Type#PATH_AS_TARGET}, returns the path in question.
*
* <p>The interpretation of this path, of course, depends on the existence of packages.
* See {@link InterpretPathAsTarget#eval}.
* See {@link TargetPattern#eval}.
*/
public String getPathForPathAsTarget() {
throw new IllegalStateException();
Expand Down Expand Up @@ -318,6 +321,7 @@ private SingleTarget(

@Override
public <T, E extends Exception> void eval(
RepositoryName repositoryName,
TargetPatternResolver<T> resolver,
ImmutableSet<PathFragment> blacklistedSubdirectories,
ImmutableSet<PathFragment> excludedSubdirectories,
Expand Down Expand Up @@ -375,6 +379,7 @@ private InterpretPathAsTarget(String path, String originalPattern, String offset

@Override
public <T, E extends Exception> void eval(
RepositoryName repositoryName,
TargetPatternResolver<T> resolver,
ImmutableSet<PathFragment> blacklistedSubdirectories,
ImmutableSet<PathFragment> excludedSubdirectories,
Expand Down Expand Up @@ -460,6 +465,7 @@ private TargetsInPackage(String originalPattern, String offset,

@Override
public <T, E extends Exception> void eval(
RepositoryName repositoryName,
TargetPatternResolver<T> resolver,
ImmutableSet<PathFragment> blacklistedSubdirectories,
ImmutableSet<PathFragment> excludedSubdirectories,
Expand Down Expand Up @@ -569,6 +575,7 @@ private TargetsBelowDirectory(

@Override
public <T, E extends Exception> void eval(
RepositoryName repositoryName,
TargetPatternResolver<T> resolver,
ImmutableSet<PathFragment> blacklistedSubdirectories,
ImmutableSet<PathFragment> excludedSubdirectories,
Expand All @@ -588,6 +595,7 @@ public <T, E extends Exception> void eval(

@Override
public <T, E extends Exception> ListenableFuture<Void> evalAsync(
RepositoryName repositoryName,
TargetPatternResolver<T> resolver,
ImmutableSet<PathFragment> blacklistedSubdirectories,
ImmutableSet<PathFragment> excludedSubdirectories,
Expand Down
63 changes: 63 additions & 0 deletions src/main/java/com/google/devtools/build/lib/packages/Package.java
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import com.google.devtools.build.lib.skyframe.serialization.SerializationContext;
import com.google.devtools.build.lib.skyframe.serialization.SerializationException;
import com.google.devtools.build.lib.syntax.StarlarkSemantics;
import com.google.devtools.build.lib.util.Pair;
import com.google.devtools.build.lib.util.SpellChecker;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
Expand Down Expand Up @@ -217,6 +218,8 @@ private NameConflictException(String message) {

private ImmutableList<String> registeredExecutionPlatforms;
private ImmutableList<String> registeredToolchains;
private ImmutableMap<RepositoryName, ImmutableList<String>> registeredToolchainsAndRepoName;
private ImmutableMap<RepositoryName, ImmutableList<String>> registeredExecutionPlatformsAndRepoName;

/**
* Package initialization, part 1 of 3: instantiates a new package with the
Expand Down Expand Up @@ -406,6 +409,20 @@ private void finishInit(Builder builder) {
this.posts = ImmutableList.copyOf(builder.posts);
this.registeredExecutionPlatforms = ImmutableList.copyOf(builder.registeredExecutionPlatforms);
this.registeredToolchains = ImmutableList.copyOf(builder.registeredToolchains);

ImmutableMap.Builder<RepositoryName, ImmutableList<String>>
registeredToolchainsAndRepoNameBuilder = ImmutableMap.builder();
builder.registeredToolchainsAndRepoName.forEach((k, v) ->
registeredToolchainsAndRepoNameBuilder.put(k, ImmutableList.copyOf(v)));
this.registeredToolchainsAndRepoName = registeredToolchainsAndRepoNameBuilder.build();

ImmutableMap.Builder<RepositoryName, ImmutableList<String>>
registeredExecutionPlatformsAndRepoNameBuilder = ImmutableMap.builder();
builder.registeredExecutionPlatformsAndRepoName.forEach((k, v) ->
registeredExecutionPlatformsAndRepoNameBuilder.put(k, ImmutableList.copyOf(v)));
this.registeredExecutionPlatformsAndRepoName = registeredExecutionPlatformsAndRepoNameBuilder.build();


this.repositoryMapping = Preconditions.checkNotNull(builder.repositoryMapping);
ImmutableMap.Builder<RepositoryName, ImmutableMap<RepositoryName, RepositoryName>>
repositoryMappingsBuilder = ImmutableMap.builder();
Expand Down Expand Up @@ -705,6 +722,14 @@ public ImmutableList<String> getRegisteredToolchains() {
return registeredToolchains;
}

public ImmutableMap<RepositoryName, ImmutableList<String>> getRegisteredToolchainsAndRepoName() {
return registeredToolchainsAndRepoName;
}

public ImmutableMap<RepositoryName, ImmutableList<String>> getRegisteredExecutionPlatformsAndRepoName() {
return registeredExecutionPlatformsAndRepoName;
}

@Override
public String toString() {
return "Package(" + name + ")="
Expand Down Expand Up @@ -837,6 +862,8 @@ public void onLoadingComplete(

private final List<String> registeredExecutionPlatforms = new ArrayList<>();
private final List<String> registeredToolchains = new ArrayList<>();
private final Map<RepositoryName, List<String>> registeredToolchainsAndRepoName = new HashMap<>();
private final Map<RepositoryName, List<String>> registeredExecutionPlatformsAndRepoName = new HashMap<>();

private ThirdPartyLicenseExistencePolicy thirdPartyLicenceExistencePolicy =
ThirdPartyLicenseExistencePolicy.USER_CONTROLLABLE;
Expand Down Expand Up @@ -941,6 +968,22 @@ ImmutableMap<RepositoryName, RepositoryName> getRepositoryMapping() {
return this.repositoryMapping;
}

// /**
// * Returns the repository mapping for the requested external repository.
// *
// * @throws UnsupportedOperationException if called from a package other than
// * the //external package
// */
// public ImmutableMap<RepositoryName, RepositoryName> getRepositoryMapping(
// RepositoryName repository) {
// if (!isWorkspace()) {
// throw new UnsupportedOperationException("Can only access the external package repository"
// + "mappings from the //external package");
// }
// return ImmutableMap.copyOf(externalPackageRepositoryMappings.getOrDefault(repository, new HashMap<>()));
// }


Interner<ImmutableList<?>> getListInterner() {
return listInterner;
}
Expand Down Expand Up @@ -1409,6 +1452,26 @@ void addRegisteredToolchains(List<String> toolchains) {
this.registeredToolchains.addAll(toolchains);
}

void addRegisteredToolchainsAndRepositoryName(List<String> toolchains, RepositoryName repositoryName) {
// The add needs to be additive. If a list already exists for a particular repository name,
// we must add to the list
if (this.registeredToolchainsAndRepoName.containsKey(repositoryName)) {
List<String> existingToolchains = this.registeredToolchainsAndRepoName.get(repositoryName);
toolchains.addAll(existingToolchains);
}
this.registeredToolchainsAndRepoName.put(repositoryName, toolchains);
}

void addRegisteredExecutionPlatformsAndRepositoryName(List<String> platforms, RepositoryName repositoryName) {
// The add needs to be additive. If a list already exists for a particular repository name,
// we must add to the list
if (this.registeredExecutionPlatformsAndRepoName.containsKey(repositoryName)) {
List<String> existingToolchains = this.registeredExecutionPlatformsAndRepoName.get(repositoryName);
platforms.addAll(existingToolchains);
}
this.registeredExecutionPlatformsAndRepoName.put(repositoryName, platforms);
}

private Builder beforeBuild(boolean discoverAssumedInputFiles) throws NoSuchPackageException {
Preconditions.checkNotNull(pkg);
Preconditions.checkNotNull(filename);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ private void execute(
new BazelStarlarkContext(
/* toolsRepository= */ null,
/* fragmentNameToClass= */ null,
ImmutableMap.of(),
ImmutableMap.of(RepositoryName.createFromValidStrippedName("ALKSDF"), RepositoryName.createFromValidStrippedName("ALKA")),
new SymbolGenerator<>(workspaceFileKey)))
.build();
SkylarkUtils.setPhase(workspaceEnv, Phase.WORKSPACE);
Expand Down Expand Up @@ -423,8 +423,8 @@ public Object invoke(

// Add to the package definition for later.
Package.Builder builder = PackageFactory.getContext(env, location).pkgBuilder;
builder.addRegisteredExecutionPlatforms(
platformLabels.getContents(String.class, "platform_labels"));
RepositoryName repositoryName = env.getGlobals().getLabel().getPackageIdentifier().getRepository();
builder.addRegisteredToolchainsAndRepositoryName(platformLabels.getContents(String.class, "platform_labels"), repositoryName);

return NONE;
}
Expand Down Expand Up @@ -460,8 +460,8 @@ public Object invoke(

// Add to the package definition for later.
Package.Builder builder = PackageFactory.getContext(env, location).pkgBuilder;
builder.addRegisteredToolchains(
toolchainLabels.getContents(String.class, "toolchain_labels"));
RepositoryName repositoryName = env.getGlobals().getLabel().getPackageIdentifier().getRepository();
builder.addRegisteredToolchainsAndRepositoryName(toolchainLabels.getContents(String.class, "toolchain_labels"), repositoryName);

return NONE;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey;
import com.google.devtools.build.lib.skyframe.ConfiguredTargetValue;
import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
import com.google.devtools.build.lib.skyframe.TargetPatternValue.TargetPatternKey;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.WalkableGraph;
import java.io.OutputStream;
Expand Down Expand Up @@ -269,9 +270,11 @@ protected ConfiguredTargetKey getSkyKey(ConfiguredTargetValue configuredTargetVa
@Override
public QueryTaskFuture<Void> getTargetsMatchingPattern(
QueryExpression owner, String pattern, Callback<ConfiguredTargetValue> callback) {
TargetPatternKey patternKey;
TargetPattern patternToEval;
try {
patternToEval = getPattern(pattern);
patternKey = getPatternKey(pattern);
patternToEval = patternKey.getParsedPattern();
} catch (TargetParsingException tpe) {
try {
reportBuildFileError(owner, tpe.getMessage());
Expand All @@ -290,6 +293,7 @@ public QueryTaskFuture<Void> getTargetsMatchingPattern(
return QueryTaskFutureImpl.ofDelegate(
Futures.catchingAsync(
patternToEval.evalAdaptedForAsync(
patternKey.getRepositoryName(),
resolver,
getBlacklistedPackagePrefixesPathFragments(),
/* excludedSubdirectories= */ ImmutableSet.of(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey;
import com.google.devtools.build.lib.skyframe.ConfiguredTargetValue;
import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
import com.google.devtools.build.lib.skyframe.TargetPatternValue.TargetPatternKey;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.WalkableGraph;
import java.io.OutputStream;
Expand Down Expand Up @@ -210,8 +211,10 @@ public ConfiguredTargetAccessor getAccessor() {
public QueryTaskFuture<Void> getTargetsMatchingPattern(
QueryExpression owner, String pattern, Callback<ConfiguredTarget> callback) {
TargetPattern patternToEval;
TargetPatternKey patternKey;
try {
patternToEval = getPattern(pattern);
patternKey = getPatternKey(pattern);
patternToEval = patternKey.getParsedPattern();
} catch (TargetParsingException tpe) {
try {
reportBuildFileError(owner, tpe.getMessage());
Expand All @@ -230,6 +233,7 @@ public QueryTaskFuture<Void> getTargetsMatchingPattern(
return QueryTaskFutureImpl.ofDelegate(
Futures.catchingAsync(
patternToEval.evalAdaptedForAsync(
patternKey.getRepositoryName(),
resolver,
getBlacklistedPackagePrefixesPathFragments(),
/* excludedSubdirectories= */ ImmutableSet.of(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.cmdline.TargetParsingException;
import com.google.devtools.build.lib.cmdline.TargetPattern;
import com.google.devtools.build.lib.collect.compacthashset.CompactHashSet;
Expand Down Expand Up @@ -113,7 +114,8 @@ public abstract class PostAnalysisQueryEnvironment<T> extends AbstractBlazeQuery
ALL_PATTERNS =
ImmutableList.of(
new TargetPatternKey(
targetPattern, FilteringPolicies.NO_FILTER, false, "", ImmutableSet.of()));
// TODO: FIGURE OUT WHAT THE REPO SHOULD BE!!!!
targetPattern, FilteringPolicies.NO_FILTER, false, "", ImmutableSet.of(), RepositoryName.MAIN));
}

protected RecursivePackageProviderBackedTargetPatternResolver resolver;
Expand Down Expand Up @@ -238,13 +240,13 @@ ImmutableSet<PathFragment> getBlacklistedPackagePrefixesPathFragments()
@Nullable
protected abstract T getValueFromKey(SkyKey key) throws InterruptedException;

protected TargetPattern getPattern(String pattern) throws TargetParsingException {
protected TargetPatternKey getPatternKey(String pattern) throws TargetParsingException {
TargetPatternKey targetPatternKey =
((TargetPatternKey)
TargetPatternValue.key(
pattern, TargetPatternEvaluator.DEFAULT_FILTERING_POLICY, parserPrefix)
.argument());
return targetPatternKey.getParsedPattern();
return targetPatternKey;
}

ThreadSafeMutableSet<T> getFwdDeps(Iterable<T> targets) throws InterruptedException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -738,6 +738,7 @@ public QueryTaskFuture<Void> evalTargetPatternKey(
}
ListenableFuture<Void> evalFuture =
patternToEval.evalAsync(
targetPatternKey.getRepositoryName(),
resolver,
blacklistedSubdirectoriesToExclude,
additionalSubdirectoriesToExclude,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,11 +101,11 @@ public SkyValue compute(SkyKey key, Environment env)

try {
parsedPattern.eval(
patternKey.getRepositoryName(),
preparer,
blacklistedSubdirectories,
excludedSubdirectories,
NullCallback.<Void>instance(),
RuntimeException.class);
NullCallback.<Void>instance(), RuntimeException.class);
} catch (TargetParsingException e) {
throw new PrepareDepsOfPatternFunctionException(e);
} catch (MissingDepException e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@
package com.google.devtools.build.lib.skyframe;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Interner;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.cmdline.TargetParsingException;
import com.google.devtools.build.lib.cmdline.TargetPattern;
import com.google.devtools.build.lib.cmdline.TargetPattern.Type;
Expand Down Expand Up @@ -91,8 +93,10 @@ public static PrepareDepsOfPatternSkyKeysAndExceptions keys(
ImmutableList.builder();
ImmutableList.Builder<PrepareDepsOfPatternSkyKeyException> resultExceptionsBuilder =
ImmutableList.builder();
// TODO aaahhhhh???
ImmutableMap<RepositoryName, ImmutableList<String>> patternsMap = ImmutableMap.of(RepositoryName.MAIN, ImmutableList.copyOf(patterns));
Iterable<TargetPatternSkyKeyOrException> keysMaybe =
TargetPatternValue.keys(patterns, FilteringPolicies.NO_FILTER, offset);
TargetPatternValue.keys(patternsMap, FilteringPolicies.NO_FILTER, offset);
ImmutableList.Builder<TargetPatternKey> targetPatternKeysBuilder = ImmutableList.builder();
for (TargetPatternSkyKeyOrException keyMaybe : keysMaybe) {
try {
Expand Down
Loading