From 9860051afe7ccf0902f1ce3dbbc9066088d7c849 Mon Sep 17 00:00:00 2001 From: salma-samy Date: Mon, 2 Jan 2023 06:48:47 -0800 Subject: [PATCH] Separate selection from bazel dependency graph * Created new K/F/V for Bazel dependency graph named "BazelDepGraph" * Refactored "BazelModuleResolution" to only run resolution (discovery, selection and checks) to create pruned and unpruned graph and not create any dependency value * BazelDepGraphResolution calls BazelModuleResolution and extracts the dependency graph from it * Updated tests PiperOrigin-RevId: 499026445 Change-Id: Id1237f9d09015ffe8987d933431fccfcfa0c0963 --- .../lib/bazel/BazelRepositoryModule.java | 4 +- .../devtools/build/lib/bazel/bzlmod/BUILD | 2 + .../bazel/bzlmod/BazelDepGraphFunction.java | 120 ++++++ .../lib/bazel/bzlmod/BazelDepGraphValue.java | 100 +++++ .../bzlmod/BazelModuleInspectorFunction.java | 2 +- .../bzlmod/BazelModuleResolutionFunction.java | 181 +++------ .../bzlmod/BazelModuleResolutionValue.java | 95 +---- .../build/lib/bazel/bzlmod/Module.java | 2 +- .../build/lib/bazel/bzlmod/Selection.java | 36 +- .../bzlmod/SingleExtensionUsagesFunction.java | 14 +- .../bzlmod/SingleExtensionUsagesValue.java | 3 +- .../RepositoryDelegatorFunction.java | 10 +- .../lib/skyframe/BzlmodRepoRuleFunction.java | 23 +- .../RegisteredExecutionPlatformsFunction.java | 12 +- .../RegisteredToolchainsFunction.java | 12 +- .../skyframe/RepositoryMappingFunction.java | 36 +- .../build/lib/skyframe/SkyFunctions.java | 2 + .../build/lib/analysis/util/AnalysisMock.java | 3 + .../devtools/build/lib/bazel/bzlmod/BUILD | 1 + .../bzlmod/BazelDepGraphFunctionTest.java | 348 ++++++++++++++++++ .../BazelModuleResolutionFunctionTest.java | 188 +--------- .../bzlmod/BzlmodRepoRuleFunctionTest.java | 1 + .../bzlmod/ModuleExtensionResolutionTest.java | 17 +- .../build/lib/bazel/bzlmod/SelectionTest.java | 23 +- .../repository/RepositoryDelegatorTest.java | 8 +- 25 files changed, 741 insertions(+), 502 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphValue.java create mode 100644 src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 60a1becf68679a..219cb430b92706 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -25,6 +25,7 @@ import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; +import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; @@ -248,8 +249,9 @@ public ResolutionReason getResolutionReason() { .addSkyFunction( SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, directories.getWorkspace(), builtinModules)) - .addSkyFunction(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) + .addSkyFunction(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .addSkyFunction(SkyFunctions.BAZEL_MODULE_INSPECTION, new BazelModuleInspectorFunction()) + .addSkyFunction(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction) .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_USAGES, new SingleExtensionUsagesFunction()); filesystem = runtime.getFileSystem(); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index a20cc3d8d7609c..3c37edd9667c22 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -86,6 +86,7 @@ java_library( srcs = [ "AbridgedModule.java", "ArchiveOverride.java", + "BazelDepGraphValue.java", "BazelModuleResolutionValue.java", "GitOverride.java", "LocalPathOverride.java", @@ -126,6 +127,7 @@ java_library( java_library( name = "resolution_impl", srcs = [ + "BazelDepGraphFunction.java", "BazelModuleResolutionFunction.java", "Discovery.java", "ModuleExtensionContext.java", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java new file mode 100644 index 00000000000000..3091a04226fb96 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java @@ -0,0 +1,120 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; + +import com.google.common.collect.BiMap; +import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableTable; +import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction.BazelModuleResolutionFunctionException; +import com.google.devtools.build.lib.cmdline.LabelSyntaxException; +import com.google.devtools.build.lib.cmdline.PackageIdentifier; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.packages.LabelConverter; +import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import javax.annotation.Nullable; + +/** + * This function runs Bazel module resolution, extracts the dependency graph from it and creates a + * value containing all Bazel modules, along with a few lookup maps that help with further usage. By + * this stage, module extensions are not evaluated yet. + */ +public class BazelDepGraphFunction implements SkyFunction { + + @Override + @Nullable + public SkyValue compute(SkyKey skyKey, Environment env) + throws SkyFunctionException, InterruptedException { + + BazelModuleResolutionValue selectionResult = + (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY); + if (env.valuesMissing()) { + return null; + } + + ImmutableMap depGraph = selectionResult.getResolvedDepGraph(); + ImmutableMap canonicalRepoNameLookup = + depGraph.keySet().stream() + .collect(toImmutableMap(ModuleKey::getCanonicalRepoName, key -> key)); + + // For each extension usage, we resolve (i.e. canonicalize) its bzl file label. Then we can + // group all usages by the label + name (the ModuleExtensionId). + ImmutableTable.Builder + extensionUsagesTableBuilder = ImmutableTable.builder(); + for (Module module : depGraph.values()) { + LabelConverter labelConverter = + new LabelConverter( + PackageIdentifier.create(module.getCanonicalRepoName(), PathFragment.EMPTY_FRAGMENT), + module.getRepoMappingWithBazelDepsOnly()); + for (ModuleExtensionUsage usage : module.getExtensionUsages()) { + try { + ModuleExtensionId moduleExtensionId = + ModuleExtensionId.create( + labelConverter.convert(usage.getExtensionBzlFile()), usage.getExtensionName()); + extensionUsagesTableBuilder.put(moduleExtensionId, module.getKey(), usage); + } catch (LabelSyntaxException e) { + throw new BazelModuleResolutionFunctionException( + ExternalDepsException.withCauseAndMessage( + Code.BAD_MODULE, + e, + "invalid label for module extension found at %s", + usage.getLocation()), + Transience.PERSISTENT); + } + } + } + ImmutableTable extensionUsagesById = + extensionUsagesTableBuilder.buildOrThrow(); + + // Calculate a unique name for each used extension id. + BiMap extensionUniqueNames = HashBiMap.create(); + for (ModuleExtensionId id : extensionUsagesById.rowKeySet()) { + // Ensure that the resulting extension name (and thus the repository names derived from it) do + // not start with a tilde. + RepositoryName repository = id.getBzlFileLabel().getRepository(); + String nonEmptyRepoPart; + if (repository.isMain()) { + nonEmptyRepoPart = "_main"; + } else { + nonEmptyRepoPart = repository.getName(); + } + String bestName = nonEmptyRepoPart + "~" + id.getExtensionName(); + if (extensionUniqueNames.putIfAbsent(bestName, id) == null) { + continue; + } + int suffix = 2; + while (extensionUniqueNames.putIfAbsent(bestName + suffix, id) != null) { + suffix++; + } + } + + return BazelDepGraphValue.create( + depGraph, + canonicalRepoNameLookup, + depGraph.values().stream().map(AbridgedModule::from).collect(toImmutableList()), + extensionUsagesById, + ImmutableMap.copyOf(extensionUniqueNames.inverse())); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphValue.java new file mode 100644 index 00000000000000..fab2435d52d307 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphValue.java @@ -0,0 +1,100 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableTable; +import com.google.devtools.build.lib.cmdline.RepositoryMapping; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.skyframe.SkyFunctions; +import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import java.util.Map; + +/** + * The result of running Bazel module resolution, containing the Bazel module dependency graph + * post-version-resolution. + */ +@AutoValue +public abstract class BazelDepGraphValue implements SkyValue { + @SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.BAZEL_DEP_GRAPH; + + public static BazelDepGraphValue create( + ImmutableMap depGraph, + ImmutableMap canonicalRepoNameLookup, + ImmutableList abridgedModules, + ImmutableTable extensionUsagesTable, + ImmutableMap extensionUniqueNames) { + return new AutoValue_BazelDepGraphValue( + depGraph, + canonicalRepoNameLookup, + abridgedModules, + extensionUsagesTable, + extensionUniqueNames); + } + + /** + * The post-selection dep graph. Must have BFS iteration order, starting from the root module. For + * any KEY in the returned map, it's guaranteed that {@code depGraph[KEY].getKey() == KEY}. + */ + public abstract ImmutableMap getDepGraph(); + + /** A mapping from a canonical repo name to the key of the module backing it. */ + public abstract ImmutableMap getCanonicalRepoNameLookup(); + + /** All modules in the same order as {@link #getDepGraph}, but with limited information. */ + public abstract ImmutableList getAbridgedModules(); + + /** + * All module extension usages grouped by the extension's ID and the key of the module where this + * usage occurs. For each extension identifier ID, extensionUsagesTable[ID][moduleKey] is the + * ModuleExtensionUsage of ID in the module keyed by moduleKey. + */ + public abstract ImmutableTable + getExtensionUsagesTable(); + + /** + * A mapping from the ID of a module extension to a unique string that serves as its "name". This + * is not the same as the extension's declared name, as the declared name is only unique within + * the .bzl file, whereas this unique name is guaranteed to be unique across the workspace. + */ + public abstract ImmutableMap getExtensionUniqueNames(); + + /** + * Returns the full {@link RepositoryMapping} for the given module, including repos from Bazel + * module deps and module extensions. + */ + public final RepositoryMapping getFullRepoMapping(ModuleKey key) { + ImmutableMap.Builder mapping = ImmutableMap.builder(); + for (Map.Entry e : + getExtensionUsagesTable().column(key).entrySet()) { + ModuleExtensionId extensionId = e.getKey(); + ModuleExtensionUsage usage = e.getValue(); + for (Map.Entry entry : usage.getImports().entrySet()) { + String canonicalRepoName = + getExtensionUniqueNames().get(extensionId) + "~" + entry.getValue(); + mapping.put(entry.getKey(), RepositoryName.createUnvalidated(canonicalRepoName)); + } + } + return getDepGraph() + .get(key) + .getRepoMappingWithBazelDepsOnly() + .withAdditionalMappings(mapping.buildOrThrow()); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorFunction.java index 67f8cfdfb495f1..0394de65277e22 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleInspectorFunction.java @@ -56,7 +56,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) } ImmutableMap overrides = root.getOverrides(); ImmutableMap unprunedDepGraph = resolutionValue.getUnprunedDepGraph(); - ImmutableMap resolvedDepGraph = resolutionValue.getDepGraph(); + ImmutableMap resolvedDepGraph = resolutionValue.getResolvedDepGraph(); ImmutableMap depGraph = computeAugmentedGraph(unprunedDepGraph, resolvedDepGraph.keySet(), overrides); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java index 033f02894dd6d1..c44ca83ea57bc7 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java @@ -1,4 +1,4 @@ -// Copyright 2021 The Bazel Authors. All rights reserved. +// Copyright 2022 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,38 +15,26 @@ package com.google.devtools.build.lib.bazel.bzlmod; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.ImmutableMap.toImmutableMap; - -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Splitter; import com.google.common.base.Strings; -import com.google.common.collect.BiMap; -import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.ImmutableTable; import com.google.devtools.build.lib.analysis.BlazeVersionInfo; import com.google.devtools.build.lib.bazel.BazelVersion; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; -import com.google.devtools.build.lib.bazel.bzlmod.Selection.SelectionResult; import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; -import com.google.devtools.build.lib.cmdline.LabelSyntaxException; -import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.events.ExtendedEventHandler; -import com.google.devtools.build.lib.packages.LabelConverter; import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; -import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; @@ -60,9 +48,8 @@ import javax.annotation.Nullable; /** - * Runs Bazel module resolution. This function produces the dependency graph containing all Bazel - * modules, along with a few lookup maps that help with further usage. By this stage, module - * extensions are not evaluated yet. + * Discovers the whole dependency graph and runs selection algorithm on it to produce the pruned + * dependency graph and runs checks on it. */ public class BazelModuleResolutionFunction implements SkyFunction { @@ -70,15 +57,16 @@ public class BazelModuleResolutionFunction implements SkyFunction { new Precomputed<>("check_direct_dependency"); public static final Precomputed BAZEL_COMPATIBILITY_MODE = new Precomputed<>("bazel_compatibility_mode"); - public static final Precomputed> ALLOWED_YANKED_VERSIONS = new Precomputed<>("allowed_yanked_versions"); + private static final String BZLMOD_ALLOWED_YANKED_VERSIONS_ENV = "BZLMOD_ALLOW_YANKED_VERSIONS"; @Override @Nullable public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, InterruptedException { + ClientEnvironmentValue allowedYankedVersionsFromEnv = (ClientEnvironmentValue) env.getValue(ClientEnvironmentFunction.key(BZLMOD_ALLOWED_YANKED_VERSIONS_ENV)); @@ -94,28 +82,70 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (initialDepGraph == null) { return null; } - ImmutableMap overrides = root.getOverrides(); - SelectionResult selectionResult; + + BazelModuleResolutionValue selectionResultValue; try { - selectionResult = Selection.run(initialDepGraph, overrides); + selectionResultValue = Selection.run(initialDepGraph, root.getOverrides()); } catch (ExternalDepsException e) { throw new BazelModuleResolutionFunctionException(e, Transience.PERSISTENT); } - ImmutableMap resolvedDepGraph = selectionResult.getResolvedDepGraph(); + ImmutableMap resolvedDepGraph = selectionResultValue.getResolvedDepGraph(); + + verifyRootModuleDirectDepsAreAccurate( + initialDepGraph.get(ModuleKey.ROOT), + resolvedDepGraph.get(ModuleKey.ROOT), + Objects.requireNonNull(CHECK_DIRECT_DEPENDENCIES.get(env)), + env.getListener()); checkBazelCompatibility( resolvedDepGraph.values(), Objects.requireNonNull(BAZEL_COMPATIBILITY_MODE.get(env)), env.getListener()); + verifyYankedVersions( resolvedDepGraph, parseYankedVersions( allowedYankedVersionsFromEnv.getValue(), Objects.requireNonNull(ALLOWED_YANKED_VERSIONS.get(env))), env.getListener()); - verifyRootModuleDirectDepsAreAccurate( - env, initialDepGraph.get(ModuleKey.ROOT), resolvedDepGraph.get(ModuleKey.ROOT)); - return createValue(resolvedDepGraph, selectionResult.getUnprunedDepGraph(), overrides); + + return selectionResultValue; + } + + private static void verifyRootModuleDirectDepsAreAccurate( + Module discoveredRootModule, + Module resolvedRootModule, + CheckDirectDepsMode mode, + EventHandler eventHandler) + throws BazelModuleResolutionFunctionException { + if (mode == CheckDirectDepsMode.OFF) { + return; + } + + boolean failure = false; + for (Map.Entry dep : discoveredRootModule.getDeps().entrySet()) { + ModuleKey resolved = resolvedRootModule.getDeps().get(dep.getKey()); + if (!dep.getValue().equals(resolved)) { + String message = + String.format( + "For repository '%s', the root module requires module version %s, but got %s in the" + + " resolved dependency graph.", + dep.getKey(), dep.getValue(), resolved); + if (mode == CheckDirectDepsMode.WARNING) { + eventHandler.handle(Event.warn(message)); + } else { + eventHandler.handle(Event.error(message)); + failure = true; + } + } + } + + if (failure) { + throw new BazelModuleResolutionFunctionException( + ExternalDepsException.withMessage( + Code.VERSION_RESOLUTION_ERROR, "Direct dependency check failed."), + Transience.PERSISTENT); + } } public static void checkBazelCompatibility( @@ -291,109 +321,6 @@ private void verifyYankedVersions( } } - private static void verifyRootModuleDirectDepsAreAccurate( - Environment env, Module discoveredRootModule, Module resolvedRootModule) - throws InterruptedException, BazelModuleResolutionFunctionException { - CheckDirectDepsMode mode = Objects.requireNonNull(CHECK_DIRECT_DEPENDENCIES.get(env)); - if (mode == CheckDirectDepsMode.OFF) { - return; - } - boolean failure = false; - for (Map.Entry dep : discoveredRootModule.getDeps().entrySet()) { - ModuleKey resolved = resolvedRootModule.getDeps().get(dep.getKey()); - if (!dep.getValue().equals(resolved)) { - String message = - String.format( - "For repository '%s', the root module requires module version %s, but got %s in the" - + " resolved dependency graph.", - dep.getKey(), dep.getValue(), resolved); - if (mode == CheckDirectDepsMode.WARNING) { - env.getListener().handle(Event.warn(message)); - } else { - env.getListener().handle(Event.error(message)); - failure = true; - } - } - } - if (failure) { - throw new BazelModuleResolutionFunctionException( - ExternalDepsException.withMessage( - Code.VERSION_RESOLUTION_ERROR, "Direct dependency check failed."), - Transience.PERSISTENT); - } - } - - @VisibleForTesting - static BazelModuleResolutionValue createValue( - ImmutableMap depGraph, - ImmutableMap unprunedDepGraph, - ImmutableMap overrides) - throws BazelModuleResolutionFunctionException { - // Build some reverse lookups for later use. - ImmutableMap canonicalRepoNameLookup = - depGraph.keySet().stream() - .collect(toImmutableMap(ModuleKey::getCanonicalRepoName, key -> key)); - - // For each extension usage, we resolve (i.e. canonicalize) its bzl file label. Then we can - // group all usages by the label + name (the ModuleExtensionId). - ImmutableTable.Builder - extensionUsagesTableBuilder = ImmutableTable.builder(); - for (Module module : depGraph.values()) { - LabelConverter labelConverter = - new LabelConverter( - PackageIdentifier.create(module.getCanonicalRepoName(), PathFragment.EMPTY_FRAGMENT), - module.getRepoMappingWithBazelDepsOnly()); - for (ModuleExtensionUsage usage : module.getExtensionUsages()) { - try { - ModuleExtensionId moduleExtensionId = - ModuleExtensionId.create( - labelConverter.convert(usage.getExtensionBzlFile()), usage.getExtensionName()); - extensionUsagesTableBuilder.put(moduleExtensionId, module.getKey(), usage); - } catch (LabelSyntaxException e) { - throw new BazelModuleResolutionFunctionException( - ExternalDepsException.withCauseAndMessage( - Code.BAD_MODULE, - e, - "invalid label for module extension found at %s", - usage.getLocation()), - Transience.PERSISTENT); - } - } - } - ImmutableTable extensionUsagesById = - extensionUsagesTableBuilder.buildOrThrow(); - - // Calculate a unique name for each used extension id. - BiMap extensionUniqueNames = HashBiMap.create(); - for (ModuleExtensionId id : extensionUsagesById.rowKeySet()) { - // Ensure that the resulting extension name (and thus the repository names derived from it) do - // not start with a tilde. - RepositoryName repository = id.getBzlFileLabel().getRepository(); - String nonEmptyRepoPart; - if (repository.isMain()) { - nonEmptyRepoPart = "_main"; - } else { - nonEmptyRepoPart = repository.getName(); - } - String bestName = nonEmptyRepoPart + "~" + id.getExtensionName(); - if (extensionUniqueNames.putIfAbsent(bestName, id) == null) { - continue; - } - int suffix = 2; - while (extensionUniqueNames.putIfAbsent(bestName + suffix, id) != null) { - suffix++; - } - } - - return BazelModuleResolutionValue.create( - depGraph, - unprunedDepGraph, - canonicalRepoNameLookup, - depGraph.values().stream().map(AbridgedModule::from).collect(toImmutableList()), - extensionUsagesById, - ImmutableMap.copyOf(extensionUniqueNames.inverse())); - } - static class BazelModuleResolutionFunctionException extends SkyFunctionException { BazelModuleResolutionFunctionException(ExternalDepsException e, Transience transience) { super(e, transience); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java index f48eee06904a3f..f2c734647478ed 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java @@ -1,4 +1,4 @@ -// Copyright 2021 The Bazel Authors. All rights reserved. +// Copyright 2022 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,96 +16,39 @@ package com.google.devtools.build.lib.bazel.bzlmod; import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableTable; -import com.google.devtools.build.lib.cmdline.RepositoryMapping; -import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; -import java.util.Map; /** - * The result of running Bazel module resolution, containing the Bazel module dependency graph - * post-version-resolution. + * The result of the selection process, containing both the pruned and the un-pruned dependency + * graphs. */ @AutoValue -public abstract class BazelModuleResolutionValue implements SkyValue { +abstract class BazelModuleResolutionValue implements SkyValue { + /* TODO(andreisolo): Also load the modules overridden by {@code single_version_override} or + NonRegistryOverride if we need to detect changes in the dependency graph caused by them. + */ + @SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.BAZEL_MODULE_RESOLUTION; - public static BazelModuleResolutionValue create( - ImmutableMap depGraph, - ImmutableMap unprunedDepGraph, - ImmutableMap canonicalRepoNameLookup, - ImmutableList abridgedModules, - ImmutableTable extensionUsagesTable, - ImmutableMap extensionUniqueNames) { - return new AutoValue_BazelModuleResolutionValue( - depGraph, - unprunedDepGraph, - canonicalRepoNameLookup, - abridgedModules, - extensionUsagesTable, - extensionUniqueNames); - } - - /** - * The post-selection dep graph. Must have BFS iteration order, starting from the root module. For - * any KEY in the returned map, it's guaranteed that {@code depGraph[KEY].getKey() == KEY}. - */ - public abstract ImmutableMap getDepGraph(); + /** Final dep graph sorted in BFS iteration order, with unused modules removed. */ + abstract ImmutableMap getResolvedDepGraph(); /** - * The post-selection un-pruned dep graph, used for in-depth inspection. TODO(andreisolo): decide - * whether to store the un-pruned graph or just the removed modules? Random order (depends on - * SkyFrame execution order). For any KEY in the returned map, it's guaranteed that {@code - * depGraph[KEY].getKey() == KEY}. + * Un-pruned dep graph, with updated dep keys, and additionally containing the unused modules + * which were initially discovered (and their MODULE.bazel files loaded). Does not contain modules + * overridden by {@code single_version_override} or {@link NonRegistryOverride}, only by {@code + * multiple_version_override}. */ - public abstract ImmutableMap getUnprunedDepGraph(); - - /** A mapping from a canonical repo name to the key of the module backing it. */ - public abstract ImmutableMap getCanonicalRepoNameLookup(); + abstract ImmutableMap getUnprunedDepGraph(); - /** All modules in the same order as {@link #getDepGraph}, but with limited information. */ - public abstract ImmutableList getAbridgedModules(); - - /** - * All module extension usages grouped by the extension's ID and the key of the module where this - * usage occurs. For each extension identifier ID, extensionUsagesTable[ID][moduleKey] is the - * ModuleExtensionUsage of ID in the module keyed by moduleKey. - */ - public abstract ImmutableTable - getExtensionUsagesTable(); - - /** - * A mapping from the ID of a module extension to a unique string that serves as its "name". This - * is not the same as the extension's declared name, as the declared name is only unique within - * the .bzl file, whereas this unique name is guaranteed to be unique across the workspace. - */ - public abstract ImmutableMap getExtensionUniqueNames(); - - /** - * Returns the full {@link RepositoryMapping} for the given module, including repos from Bazel - * module deps and module extensions. - */ - public final RepositoryMapping getFullRepoMapping(ModuleKey key) { - ImmutableMap.Builder mapping = ImmutableMap.builder(); - for (Map.Entry e : - getExtensionUsagesTable().column(key).entrySet()) { - ModuleExtensionId extensionId = e.getKey(); - ModuleExtensionUsage usage = e.getValue(); - for (Map.Entry entry : usage.getImports().entrySet()) { - String canonicalRepoName = - getExtensionUniqueNames().get(extensionId) + "~" + entry.getValue(); - mapping.put(entry.getKey(), RepositoryName.createUnvalidated(canonicalRepoName)); - } - } - return getDepGraph() - .get(key) - .getRepoMappingWithBazelDepsOnly() - .withAdditionalMappings(mapping.buildOrThrow()); + static BazelModuleResolutionValue create( + ImmutableMap resolvedDepGraph, + ImmutableMap unprunedDepGraph) { + return new AutoValue_BazelModuleResolutionValue(resolvedDepGraph, unprunedDepGraph); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java index 807c7118c4a3a8..2952548e851bd5 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java @@ -115,7 +115,7 @@ public final RepositoryName getCanonicalRepoName() { /** * Returns a {@link RepositoryMapping} with only Bazel module repos and no repos from module - * extensions. For the full mapping, see {@link BazelModuleResolutionValue#getFullRepoMapping}. + * extensions. For the full mapping, see {@link BazelDepGraphValue#getFullRepoMapping}. */ public final RepositoryMapping getRepoMappingWithBazelDepsOnly() { ImmutableMap.Builder mapping = ImmutableMap.builder(); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Selection.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Selection.java index 615feac57a776c..bcacb7b1928ba9 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Selection.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Selection.java @@ -67,34 +67,6 @@ final class Selection { private Selection() {} - /** - * The result of the selection process, containing both the pruned and the un-pruned dependency - * graphs. - */ - @AutoValue - abstract static class SelectionResult { - /* TODO(andreisolo): Also load the modules overridden by {@code single_version_override} or - NonRegistryOverride if we need to detect changes in the dependency graph caused by them. - */ - - /** Final dep graph sorted in BFS iteration order, with unused modules removed. */ - abstract ImmutableMap getResolvedDepGraph(); - - /** - * Un-pruned dep graph, with updated dep keys, and additionally containing the unused modules - * which were initially discovered (and their MODULE.bazel files loaded). Does not contain - * modules overridden by {@code single_version_override} or {@link NonRegistryOverride}, only by - * {@code multiple_version_override}. - */ - abstract ImmutableMap getUnprunedDepGraph(); - - static SelectionResult create( - ImmutableMap resolvedDepGraph, - ImmutableMap unprunedDepGraph) { - return new AutoValue_Selection_SelectionResult(resolvedDepGraph, unprunedDepGraph); - } - } - /** During selection, a version is selected for each distinct "selection group". */ @AutoValue abstract static class SelectionGroup { @@ -193,8 +165,10 @@ private static SelectionGroup computeSelectionGroup( allowedVersionSet.ceiling(module.getVersion())); } - /** Runs module selection (aka version resolution). Returns a {@link SelectionResult}. */ - public static SelectionResult run( + /** + * Runs module selection (aka version resolution). Returns a {@link BazelModuleResolutionValue}. + */ + public static BazelModuleResolutionValue run( ImmutableMap depGraph, ImmutableMap overrides) throws ExternalDepsException { // For any multiple-version overrides, build a mapping from (moduleName, compatibilityLevel) to @@ -254,7 +228,7 @@ public static SelectionResult run( new DepGraphWalker(newDepGraph, overrides, selectionGroups).walk(); // Return the result containing both the pruned and un-pruned dep graphs - return SelectionResult.create(prunedDepGraph, unprunedDepGraph); + return BazelModuleResolutionValue.create(prunedDepGraph, unprunedDepGraph); } /** diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionUsagesFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionUsagesFunction.java index e5f001679bca5a..821c759d258d56 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionUsagesFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionUsagesFunction.java @@ -44,24 +44,24 @@ public class SingleExtensionUsagesFunction implements SkyFunction { @Nullable public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, InterruptedException { - BazelModuleResolutionValue bazelModuleResolutionValue = - (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY); - if (bazelModuleResolutionValue == null) { + BazelDepGraphValue bazelDepGraphValue = + (BazelDepGraphValue) env.getValue(BazelDepGraphValue.KEY); + if (bazelDepGraphValue == null) { return null; } ModuleExtensionId id = (ModuleExtensionId) skyKey.argument(); ImmutableTable usagesTable = - bazelModuleResolutionValue.getExtensionUsagesTable(); + bazelDepGraphValue.getExtensionUsagesTable(); return SingleExtensionUsagesValue.create( usagesTable.row(id), - bazelModuleResolutionValue.getExtensionUniqueNames().get(id), + bazelDepGraphValue.getExtensionUniqueNames().get(id), // Filter abridged modules down to only those that actually used this extension. - bazelModuleResolutionValue.getAbridgedModules().stream() + bazelDepGraphValue.getAbridgedModules().stream() .filter(module -> usagesTable.contains(id, module.getKey())) .collect(toImmutableList()), // TODO(wyv): Maybe cache these mappings? usagesTable.row(id).keySet().stream() - .collect(toImmutableMap(key -> key, bazelModuleResolutionValue::getFullRepoMapping))); + .collect(toImmutableMap(key -> key, bazelDepGraphValue::getFullRepoMapping))); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionUsagesValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionUsagesValue.java index 037054bb02d4b9..f00e1bbd4852b3 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionUsagesValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionUsagesValue.java @@ -34,8 +34,7 @@ public abstract class SingleExtensionUsagesValue implements SkyValue { public abstract ImmutableMap getExtensionUsages(); /** - * The "unique name" (see {@link BazelModuleResolutionValue#getExtensionUniqueNames} of this - * extension. + * The "unique name" (see {@link BazelDepGraphValue#getExtensionUniqueNames} of this extension. */ public abstract String getExtensionUniqueName(); diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java index e0bded5e01f126..b52373016cd737 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java @@ -250,10 +250,6 @@ public SkyValue compute(SkyKey skyKey, Environment env) } Map overrides = REPOSITORY_OVERRIDES.get(env); - boolean doNotFetchUnconditionally = - DONT_FETCH_UNCONDITIONALLY.equals(DEPENDENCY_FOR_UNCONDITIONAL_FETCHING.get(env)); - boolean needsConfiguring = false; - Path repoRoot = RepositoryFunction.getExternalRepositoryDirectory(directories) .getRelative(repositoryName.getName()); @@ -264,7 +260,6 @@ public SkyValue compute(SkyKey skyKey, Environment env) } Rule rule = null; - if (starlarkSemantics.getBool(BuildLanguageOptions.ENABLE_BZLMOD)) { // Tries to get a repository rule instance from Bzlmod generated repos. SkyKey key = BzlmodRepoRuleValue.key(repositoryName); @@ -299,15 +294,18 @@ public SkyValue compute(SkyKey skyKey, Environment env) String.format("'%s' is not a repository rule", repositoryName.getCanonicalForm())); } + boolean needsConfiguring = false; if (handler.isConfigure(rule)) { needsConfiguring = !DONT_FETCH_UNCONDITIONALLY.equals(DEPENDENCY_FOR_UNCONDITIONAL_CONFIGURING.get(env)); } - if (env.valuesMissing()) { return null; } + DigestWriter digestWriter = new DigestWriter(directories, repositoryName, rule); + boolean doNotFetchUnconditionally = + DONT_FETCH_UNCONDITIONALLY.equals(DEPENDENCY_FOR_UNCONDITIONAL_FETCHING.get(env)); // Local repositories are fetched regardless of the marker file because the operation is // generally fast and they do not depend on non-local data, so it does not make much sense to diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java index 6970db5caf11b0..fe4977fe1e001b 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.analysis.BlazeDirectories; -import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionValue; +import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphValue; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleCreator; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; @@ -104,7 +104,10 @@ public SkyValue compute(SkyKey skyKey, Environment env) } RepositoryName repositoryName = ((BzlmodRepoRuleValue.Key) skyKey).argument(); - BazelModuleResolutionValue moduleResolution; + BazelDepGraphValue bazelDepGraphValue; + + // Try to find a module from this repository name. If not found, try to find + // an extension. If none found, it is an invalid name (return not found) // Look for the repo from Bazel module generated repos. try { @@ -114,9 +117,9 @@ public SkyValue compute(SkyKey skyKey, Environment env) return createRuleFromSpec(repoSpec.get(), starlarkSemantics, env); } - // BazelModuleResolutionValue is affected by repos found in Step 1, therefore it should NOT + // BazelDepGraphValue is affected by repos found in Step 1, therefore it should NOT // be requested in Step 1 to avoid cycle dependency. - moduleResolution = (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY); + bazelDepGraphValue = (BazelDepGraphValue) env.getValue(BazelDepGraphValue.KEY); if (env.valuesMissing()) { return null; } @@ -124,7 +127,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) // Step 2: Look for repositories derived from Bazel Modules. repoSpec = checkRepoFromBazelModules( - moduleResolution, root.getOverrides(), env.getListener(), repositoryName); + bazelDepGraphValue, root.getOverrides(), env.getListener(), repositoryName); if (repoSpec.isPresent()) { return createRuleFromSpec(repoSpec.get(), starlarkSemantics, env); } @@ -134,7 +137,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) // Otherwise, look for the repo from module extension evaluation results. Optional extensionId = - moduleResolution.getExtensionUniqueNames().entrySet().stream() + bazelDepGraphValue.getExtensionUniqueNames().entrySet().stream() .filter(e -> repositoryName.getName().startsWith(e.getValue() + "~")) .map(Entry::getKey) .findFirst(); @@ -170,18 +173,17 @@ private static Optional checkRepoFromNonRegistryOverrides( } private Optional checkRepoFromBazelModules( - BazelModuleResolutionValue bazelModuleResolutionValue, + BazelDepGraphValue bazelDepGraphValue, ImmutableMap overrides, ExtendedEventHandler eventListener, RepositoryName repositoryName) throws InterruptedException, IOException { - ModuleKey moduleKey = - bazelModuleResolutionValue.getCanonicalRepoNameLookup().get(repositoryName); + ModuleKey moduleKey = bazelDepGraphValue.getCanonicalRepoNameLookup().get(repositoryName); if (moduleKey == null) { return Optional.empty(); } com.google.devtools.build.lib.bazel.bzlmod.Module module = - bazelModuleResolutionValue.getDepGraph().get(moduleKey); + bazelDepGraphValue.getDepGraph().get(moduleKey); Registry registry = checkNotNull(module.getRegistry()); RepoSpec repoSpec = registry.getRepoSpec(moduleKey, repositoryName, eventListener); repoSpec = maybeAppendAdditionalPatches(repoSpec, overrides.get(moduleKey.getName())); @@ -229,6 +231,7 @@ private BzlmodRepoRuleValue createRuleFromSpec( } ruleClass = getStarlarkRuleClass(repoSpec, loadedModules); } + try { Rule rule = BzlmodRepoRuleCreator.createRule( diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunction.java index bb464a6328539a..0f1ace19597acd 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunction.java @@ -24,7 +24,7 @@ import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.analysis.platform.PlatformInfo; import com.google.devtools.build.lib.analysis.platform.PlatformProviderUtils; -import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionValue; +import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphValue; import com.google.devtools.build.lib.bazel.bzlmod.Module; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelConstants; @@ -162,18 +162,18 @@ private static ImmutableList getBzlmodExecutionPlatforms( if (!semantics.getBool(BuildLanguageOptions.ENABLE_BZLMOD)) { return ImmutableList.of(); } - BazelModuleResolutionValue bazelModuleResolutionValue = - (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY); - if (bazelModuleResolutionValue == null) { + BazelDepGraphValue bazelDepGraphValue = + (BazelDepGraphValue) env.getValue(BazelDepGraphValue.KEY); + if (bazelDepGraphValue == null) { return null; } ImmutableList.Builder executionPlatforms = ImmutableList.builder(); - for (Module module : bazelModuleResolutionValue.getDepGraph().values()) { + for (Module module : bazelDepGraphValue.getDepGraph().values()) { TargetPattern.Parser parser = new TargetPattern.Parser( PathFragment.EMPTY_FRAGMENT, module.getCanonicalRepoName(), - bazelModuleResolutionValue.getFullRepoMapping(module.getKey())); + bazelDepGraphValue.getFullRepoMapping(module.getKey())); for (String pattern : module.getExecutionPlatformsToRegister()) { try { executionPlatforms.add(parser.parse(pattern)); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunction.java index 90f88c0b673af1..4c52e42d1b36ed 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunction.java @@ -24,7 +24,7 @@ import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; import com.google.devtools.build.lib.analysis.platform.DeclaredToolchainInfo; import com.google.devtools.build.lib.analysis.platform.PlatformProviderUtils; -import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionValue; +import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphValue; import com.google.devtools.build.lib.bazel.bzlmod.ExternalDepsException; import com.google.devtools.build.lib.bazel.bzlmod.Module; import com.google.devtools.build.lib.cmdline.Label; @@ -151,18 +151,18 @@ private static ImmutableList getBzlmodToolchains( if (!semantics.getBool(BuildLanguageOptions.ENABLE_BZLMOD)) { return ImmutableList.of(); } - BazelModuleResolutionValue bazelModuleResolutionValue = - (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY); - if (bazelModuleResolutionValue == null) { + BazelDepGraphValue bazelDepGraphValue = + (BazelDepGraphValue) env.getValue(BazelDepGraphValue.KEY); + if (bazelDepGraphValue == null) { return null; } ImmutableList.Builder toolchains = ImmutableList.builder(); - for (Module module : bazelModuleResolutionValue.getDepGraph().values()) { + for (Module module : bazelDepGraphValue.getDepGraph().values()) { TargetPattern.Parser parser = new TargetPattern.Parser( PathFragment.EMPTY_FRAGMENT, module.getCanonicalRepoName(), - bazelModuleResolutionValue.getFullRepoMapping(module.getKey())); + bazelDepGraphValue.getFullRepoMapping(module.getKey())); for (String pattern : module.getToolchainsToRegister()) { try { toolchains.add(parser.parse(pattern)); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunction.java index 74cf9101263ad1..acfbb11dd8de1b 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunction.java @@ -16,7 +16,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionValue; +import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphValue; import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionEvalValue; @@ -77,9 +77,9 @@ public SkyValue compute(SkyKey skyKey, Environment env) .withAdditionalMappings(bazelToolsMapping.getRepositoryMapping())); } - BazelModuleResolutionValue bazelModuleResolutionValue = - (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY); - if (bazelModuleResolutionValue == null) { + BazelDepGraphValue bazelDepGraphValue = + (BazelDepGraphValue) env.getValue(BazelDepGraphValue.KEY); + if (bazelDepGraphValue == null) { return null; } @@ -103,7 +103,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) Collectors.toMap( Entry::getKey, entry -> RepositoryName.createUnvalidated(entry.getKey()))); return RepositoryMappingValue.withMapping( - computeForBazelModuleRepo(repositoryName, bazelModuleResolutionValue) + computeForBazelModuleRepo(repositoryName, bazelDepGraphValue) .get() // For the transitional period, we need to map the workspace name to the main repo. .withAdditionalMappings( @@ -114,14 +114,14 @@ public SkyValue compute(SkyKey skyKey, Environment env) // Try and see if this is a repo generated from a Bazel module. Optional mapping = - computeForBazelModuleRepo(repositoryName, bazelModuleResolutionValue); + computeForBazelModuleRepo(repositoryName, bazelDepGraphValue); if (mapping.isPresent()) { return RepositoryMappingValue.withMapping(mapping.get()); } // Now try and see if this is a repo generated from a module extension. Optional moduleExtensionId = - maybeGetModuleExtensionForRepo(repositoryName, bazelModuleResolutionValue); + maybeGetModuleExtensionForRepo(repositoryName, bazelDepGraphValue); if (moduleExtensionId.isPresent()) { SingleExtensionEvalValue extensionEvalValue = @@ -132,10 +132,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) } return RepositoryMappingValue.withMapping( computeForModuleExtensionRepo( - repositoryName, - moduleExtensionId.get(), - extensionEvalValue, - bazelModuleResolutionValue)); + repositoryName, moduleExtensionId.get(), extensionEvalValue, bazelDepGraphValue)); } } @@ -166,13 +163,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) * Optional.empty(). */ private Optional computeForBazelModuleRepo( - RepositoryName repositoryName, BazelModuleResolutionValue bazelModuleResolutionValue) { - ModuleKey moduleKey = - bazelModuleResolutionValue.getCanonicalRepoNameLookup().get(repositoryName); + RepositoryName repositoryName, BazelDepGraphValue bazelDepGraphValue) { + ModuleKey moduleKey = bazelDepGraphValue.getCanonicalRepoNameLookup().get(repositoryName); if (moduleKey == null) { return Optional.empty(); } - return Optional.of(bazelModuleResolutionValue.getFullRepoMapping(moduleKey)); + return Optional.of(bazelDepGraphValue.getFullRepoMapping(moduleKey)); } /** @@ -187,12 +183,12 @@ private RepositoryMapping computeForModuleExtensionRepo( RepositoryName repositoryName, ModuleExtensionId extensionId, SingleExtensionEvalValue extensionEvalValue, - BazelModuleResolutionValue bazelModuleResolutionValue) { + BazelDepGraphValue bazelDepGraphValue) { // Find the key of the module containing this extension. This will be used to compute additional // mappings -- any repo generated by an extension contained in the module "foo" can additionally // see all repos that "foo" can see. ModuleKey moduleKey = - bazelModuleResolutionValue + bazelDepGraphValue .getCanonicalRepoNameLookup() .get(extensionId.getBzlFileLabel().getRepository()); // NOTE(wyv): This means that if "foo" has a bazel_dep with the repo name "bar", and the @@ -201,7 +197,7 @@ private RepositoryMapping computeForModuleExtensionRepo( // the two to avoid any surprises. return RepositoryMapping.create( extensionEvalValue.getCanonicalRepoNameToInternalNames().inverse(), repositoryName) - .withAdditionalMappings(bazelModuleResolutionValue.getFullRepoMapping(moduleKey)); + .withAdditionalMappings(bazelDepGraphValue.getFullRepoMapping(moduleKey)); } private SkyValue computeFromWorkspace( @@ -236,8 +232,8 @@ private SkyValue computeFromWorkspace( } private static Optional maybeGetModuleExtensionForRepo( - RepositoryName repositoryName, BazelModuleResolutionValue bazelModuleResolutionValue) { - return bazelModuleResolutionValue.getExtensionUniqueNames().entrySet().stream() + RepositoryName repositoryName, BazelDepGraphValue bazelDepGraphValue) { + return bazelDepGraphValue.getExtensionUniqueNames().entrySet().stream() .filter(e -> repositoryName.getName().startsWith(e.getValue() + "~")) .map(Entry::getKey) .findFirst(); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java index aadb01b42ac508..863a9d472976c3 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java @@ -151,6 +151,8 @@ public final class SkyFunctions { SkyFunctionName.createHermetic("SINGLE_EXTENSION_USAGES"); public static final SkyFunctionName SINGLE_EXTENSION_EVAL = SkyFunctionName.createNonHermetic("SINGLE_EXTENSION_EVAL"); + public static final SkyFunctionName BAZEL_DEP_GRAPH = + SkyFunctionName.createHermetic("BAZEL_DEP_GRAPH"); public static Predicate isSkyFunction(SkyFunctionName functionName) { return key -> key.functionName().equals(functionName); diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java index 55fca945220288..2501f17dc55937 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java @@ -17,6 +17,7 @@ import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; @@ -140,6 +141,8 @@ public ImmutableMap getSkyFunctions(BlazeDirectori FakeRegistry.DEFAULT_FACTORY, directories.getWorkspace(), getBuiltinModules(directories)), + SkyFunctions.BAZEL_DEP_GRAPH, + new BazelDepGraphFunction(), SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction(), SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 504211647adedd..34491a53c96890 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -79,6 +79,7 @@ java_library( "//third_party:auto_value", "//third_party:caffeine", "//third_party:guava", + "//third_party:jsr305", "//third_party:junit4", "//third_party:mockito", "//third_party:truth", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java new file mode 100644 index 00000000000000..bdba9bca8400e3 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java @@ -0,0 +1,348 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil.createModuleKey; +import static com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil.createRepositoryMapping; +import static org.junit.Assert.fail; + +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.FileValue; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.ServerDirectories; +import com.google.devtools.build.lib.analysis.util.AnalysisMock; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.clock.BlazeClock; +import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; +import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; +import com.google.devtools.build.lib.skyframe.ExternalFilesHelper; +import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction; +import com.google.devtools.build.lib.skyframe.FileFunction; +import com.google.devtools.build.lib.skyframe.FileStateFunction; +import com.google.devtools.build.lib.skyframe.PrecomputedFunction; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.skyframe.SkyFunctions; +import com.google.devtools.build.lib.testutil.FoundationTestCase; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.FileStateKey; +import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.SyscallCache; +import com.google.devtools.build.skyframe.EvaluationContext; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator; +import com.google.devtools.build.skyframe.MemoizingEvaluator; +import com.google.devtools.build.skyframe.RecordingDifferencer; +import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import java.util.concurrent.atomic.AtomicReference; +import javax.annotation.Nullable; +import net.starlark.java.eval.StarlarkSemantics; +import net.starlark.java.syntax.Location; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link BazelDepGraphFunction}. */ +@RunWith(JUnit4.class) +public class BazelDepGraphFunctionTest extends FoundationTestCase { + + private MemoizingEvaluator evaluator; + private RecordingDifferencer differencer; + private EvaluationContext evaluationContext; + private FakeRegistry.Factory registryFactory; + private BazelModuleResolutionFunctionMock resolutionFunctionMock; + + @Before + public void setup() throws Exception { + differencer = new SequencedRecordingDifferencer(); + registryFactory = new FakeRegistry.Factory(); + evaluationContext = + EvaluationContext.newBuilder().setNumThreads(8).setEventHandler(reporter).build(); + + AtomicReference packageLocator = + new AtomicReference<>( + new PathPackageLocator( + outputBase, + ImmutableList.of(Root.fromPath(rootDirectory)), + BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY)); + BlazeDirectories directories = + new BlazeDirectories( + new ServerDirectories(rootDirectory, outputBase, rootDirectory), + rootDirectory, + /* defaultSystemJavabase= */ null, + AnalysisMock.get().getProductName()); + ExternalFilesHelper externalFilesHelper = + ExternalFilesHelper.createForTesting( + packageLocator, + ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, + directories); + + resolutionFunctionMock = new BazelModuleResolutionFunctionMock(); + + evaluator = + new InMemoryMemoizingEvaluator( + ImmutableMap.builder() + .put(FileValue.FILE, new FileFunction(packageLocator, directories)) + .put( + FileStateKey.FILE_STATE, + new FileStateFunction( + Suppliers.ofInstance( + new TimestampGranularityMonitor(BlazeClock.instance())), + SyscallCache.NO_CACHE, + externalFilesHelper)) + .put( + SkyFunctions.MODULE_FILE, + new ModuleFileFunction(registryFactory, rootDirectory, ImmutableMap.of())) + .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, resolutionFunctionMock) + .put( + SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, + new ClientEnvironmentFunction(new AtomicReference<>(ImmutableMap.of()))) + .buildOrThrow(), + differencer); + + PrecomputedValue.STARLARK_SEMANTICS.set( + differencer, + StarlarkSemantics.builder().setBool(BuildLanguageOptions.ENABLE_BZLMOD, true).build()); + ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); + ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); + BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( + differencer, CheckDirectDepsMode.OFF); + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( + differencer, BazelCompatibilityMode.ERROR); + BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); + } + + @Test + public void createValue_basic() throws Exception { + // Root depends on dep@1.0 and dep@2.0 at the same time with a multiple-version override. + // Root also depends on rules_cc as a normal dep. + // dep@1.0 depends on rules_java, which is overridden by a non-registry override (see below). + ImmutableMap depGraph = + ImmutableMap.builder() + .put( + ModuleKey.ROOT, + Module.builder() + .setName("my_root") + .setVersion(Version.parse("1.0")) + .setKey(ModuleKey.ROOT) + .addDep("my_dep_1", createModuleKey("dep", "1.0")) + .addDep("my_dep_2", createModuleKey("dep", "2.0")) + .addDep("rules_cc", createModuleKey("rules_cc", "1.0")) + .build()) + .put( + createModuleKey("dep", "1.0"), + Module.builder() + .setName("dep") + .setVersion(Version.parse("1.0")) + .setKey(createModuleKey("dep", "1.0")) + .addDep("rules_java", createModuleKey("rules_java", "")) + .build()) + .put( + createModuleKey("dep", "2.0"), + Module.builder() + .setName("dep") + .setVersion(Version.parse("2.0")) + .setKey(createModuleKey("dep", "2.0")) + .build()) + .put( + createModuleKey("rules_cc", "1.0"), + Module.builder() + .setName("rules_cc") + .setVersion(Version.parse("1.0")) + .setKey(createModuleKey("rules_cc", "1.0")) + .build()) + .put( + createModuleKey("rules_java", ""), + Module.builder() + .setName("rules_java") + .setVersion(Version.parse("1.0")) + .setKey(createModuleKey("rules_java", "")) + .build()) + .buildOrThrow(); + + // TODO we need to mock bazelModuleResolution function to return depGraph + resolutionFunctionMock.setDepGraph(depGraph); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(BazelDepGraphValue.KEY), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + BazelDepGraphValue value = result.get(BazelDepGraphValue.KEY); + assertThat(value.getCanonicalRepoNameLookup()) + .containsExactly( + RepositoryName.MAIN, + ModuleKey.ROOT, + RepositoryName.create("dep~1.0"), + createModuleKey("dep", "1.0"), + RepositoryName.create("dep~2.0"), + createModuleKey("dep", "2.0"), + RepositoryName.create("rules_cc~1.0"), + createModuleKey("rules_cc", "1.0"), + RepositoryName.create("rules_java~override"), + createModuleKey("rules_java", "")); + assertThat(value.getAbridgedModules()) + .containsExactlyElementsIn( + depGraph.values().stream().map(AbridgedModule::from).collect(toImmutableList())); + } + + private static ModuleExtensionUsage createModuleExtensionUsage( + String bzlFile, String name, String... imports) { + ImmutableBiMap.Builder importsBuilder = ImmutableBiMap.builder(); + for (int i = 0; i < imports.length; i += 2) { + importsBuilder.put(imports[i], imports[i + 1]); + } + return ModuleExtensionUsage.builder() + .setExtensionBzlFile(bzlFile) + .setExtensionName(name) + .setImports(importsBuilder.buildOrThrow()) + .setLocation(Location.BUILTIN) + .build(); + } + + @Test + public void createValue_moduleExtensions() throws Exception { + Module root = + Module.builder() + .setName("root") + .setVersion(Version.parse("1.0")) + .setKey(ModuleKey.ROOT) + .addDep("rje", createModuleKey("rules_jvm_external", "1.0")) + .addDep("rpy", createModuleKey("rules_python", "2.0")) + .addExtensionUsage( + createModuleExtensionUsage("@rje//:defs.bzl", "maven", "av", "autovalue")) + .addExtensionUsage( + createModuleExtensionUsage("@rpy//:defs.bzl", "pip", "numpy", "numpy")) + .build(); + ModuleKey depKey = createModuleKey("dep", "2.0"); + Module dep = + Module.builder() + .setName("dep") + .setVersion(Version.parse("2.0")) + .setKey(depKey) + .addDep("rules_python", createModuleKey("rules_python", "2.0")) + .addExtensionUsage( + createModuleExtensionUsage("@rules_python//:defs.bzl", "pip", "np", "numpy")) + .addExtensionUsage( + createModuleExtensionUsage("//:defs.bzl", "myext", "oneext", "myext")) + .addExtensionUsage( + createModuleExtensionUsage("//incredible:conflict.bzl", "myext", "twoext", "myext")) + .build(); + ImmutableMap depGraph = ImmutableMap.of(ModuleKey.ROOT, root, depKey, dep); + + ModuleExtensionId maven = + ModuleExtensionId.create( + Label.parseCanonical("@@rules_jvm_external~1.0//:defs.bzl"), "maven"); + ModuleExtensionId pip = + ModuleExtensionId.create(Label.parseCanonical("@@rules_python~2.0//:defs.bzl"), "pip"); + ModuleExtensionId myext = + ModuleExtensionId.create(Label.parseCanonical("@@dep~2.0//:defs.bzl"), "myext"); + ModuleExtensionId myext2 = + ModuleExtensionId.create( + Label.parseCanonical("@@dep~2.0//incredible:conflict.bzl"), "myext"); + + resolutionFunctionMock.setDepGraph(depGraph); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(BazelDepGraphValue.KEY), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + BazelDepGraphValue value = result.get(BazelDepGraphValue.KEY); + + assertThat(value.getExtensionUsagesTable()).hasSize(5); + assertThat(value.getExtensionUsagesTable()) + .containsCell(maven, ModuleKey.ROOT, root.getExtensionUsages().get(0)); + assertThat(value.getExtensionUsagesTable()) + .containsCell(pip, ModuleKey.ROOT, root.getExtensionUsages().get(1)); + assertThat(value.getExtensionUsagesTable()) + .containsCell(pip, depKey, dep.getExtensionUsages().get(0)); + assertThat(value.getExtensionUsagesTable()) + .containsCell(myext, depKey, dep.getExtensionUsages().get(1)); + assertThat(value.getExtensionUsagesTable()) + .containsCell(myext2, depKey, dep.getExtensionUsages().get(2)); + + assertThat(value.getExtensionUniqueNames()) + .containsExactly( + maven, "rules_jvm_external~1.0~maven", + pip, "rules_python~2.0~pip", + myext, "dep~2.0~myext", + myext2, "dep~2.0~myext2"); + + assertThat(value.getFullRepoMapping(ModuleKey.ROOT)) + .isEqualTo( + createRepositoryMapping( + ModuleKey.ROOT, + "", + "", + "root", + "", + "rje", + "rules_jvm_external~1.0", + "rpy", + "rules_python~2.0", + "av", + "rules_jvm_external~1.0~maven~autovalue", + "numpy", + "rules_python~2.0~pip~numpy")); + assertThat(value.getFullRepoMapping(depKey)) + .isEqualTo( + createRepositoryMapping( + depKey, + "dep", + "dep~2.0", + "rules_python", + "rules_python~2.0", + "np", + "rules_python~2.0~pip~numpy", + "oneext", + "dep~2.0~myext~myext", + "twoext", + "dep~2.0~myext2~myext")); + } + + private static class BazelModuleResolutionFunctionMock implements SkyFunction { + + private ImmutableMap depGraph = ImmutableMap.of(); + + public void setDepGraph(ImmutableMap depGraph) { + this.depGraph = depGraph; + } + + @Override + @Nullable + public SkyValue compute(SkyKey skyKey, Environment env) + throws SkyFunctionException, InterruptedException { + + return BazelModuleResolutionValue.create(depGraph, ImmutableMap.of()); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java index 8ad8b3d995ea34..8d77b6622be04d 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java @@ -15,13 +15,10 @@ package com.google.devtools.build.lib.bazel.bzlmod; -import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.truth.Truth.assertThat; import static com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil.createModuleKey; -import static com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil.createRepositoryMapping; import com.google.common.base.Suppliers; -import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.FileValue; @@ -32,8 +29,6 @@ import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; import com.google.devtools.build.lib.clock.BlazeClock; -import com.google.devtools.build.lib.cmdline.Label; -import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; @@ -62,7 +57,6 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import net.starlark.java.eval.StarlarkSemantics; -import net.starlark.java.syntax.Location; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -117,6 +111,7 @@ public void setup() throws Exception { SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootDirectory, ImmutableMap.of())) .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, @@ -136,187 +131,6 @@ public void setup() throws Exception { BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); } - @Test - public void createValue_basic() throws Exception { - // Root depends on dep@1.0 and dep@2.0 at the same time with a multiple-version override. - // Root also depends on rules_cc as a normal dep. - // dep@1.0 depends on rules_java, which is overridden by a non-registry override (see below). - ImmutableMap depGraph = - ImmutableMap.builder() - .put( - ModuleKey.ROOT, - Module.builder() - .setName("my_root") - .setVersion(Version.parse("1.0")) - .setKey(ModuleKey.ROOT) - .addDep("my_dep_1", createModuleKey("dep", "1.0")) - .addDep("my_dep_2", createModuleKey("dep", "2.0")) - .addDep("rules_cc", createModuleKey("rules_cc", "1.0")) - .build()) - .put( - createModuleKey("dep", "1.0"), - Module.builder() - .setName("dep") - .setVersion(Version.parse("1.0")) - .setKey(createModuleKey("dep", "1.0")) - .addDep("rules_java", createModuleKey("rules_java", "")) - .build()) - .put( - createModuleKey("dep", "2.0"), - Module.builder() - .setName("dep") - .setVersion(Version.parse("2.0")) - .setKey(createModuleKey("dep", "2.0")) - .build()) - .put( - createModuleKey("rules_cc", "1.0"), - Module.builder() - .setName("rules_cc") - .setVersion(Version.parse("1.0")) - .setKey(createModuleKey("rules_cc", "1.0")) - .build()) - .put( - createModuleKey("rules_java", ""), - Module.builder() - .setName("rules_java") - .setVersion(Version.parse("1.0")) - .setKey(createModuleKey("rules_java", "")) - .build()) - .buildOrThrow(); - ImmutableMap overrides = - ImmutableMap.of( - "dep", - MultipleVersionOverride.create( - ImmutableList.of(Version.parse("1.0"), Version.parse("2.0")), ""), - "rules_java", LocalPathOverride.create("bleh")); - - BazelModuleResolutionValue value = - BazelModuleResolutionFunction.createValue(depGraph, depGraph, overrides); - assertThat(value.getCanonicalRepoNameLookup()) - .containsExactly( - RepositoryName.MAIN, - ModuleKey.ROOT, - RepositoryName.create("dep~1.0"), - createModuleKey("dep", "1.0"), - RepositoryName.create("dep~2.0"), - createModuleKey("dep", "2.0"), - RepositoryName.create("rules_cc~1.0"), - createModuleKey("rules_cc", "1.0"), - RepositoryName.create("rules_java~override"), - createModuleKey("rules_java", "")); - assertThat(value.getAbridgedModules()) - .containsExactlyElementsIn( - depGraph.values().stream().map(AbridgedModule::from).collect(toImmutableList())); - } - - private static ModuleExtensionUsage createModuleExtensionUsage( - String bzlFile, String name, String... imports) { - ImmutableBiMap.Builder importsBuilder = ImmutableBiMap.builder(); - for (int i = 0; i < imports.length; i += 2) { - importsBuilder.put(imports[i], imports[i + 1]); - } - return ModuleExtensionUsage.builder() - .setExtensionBzlFile(bzlFile) - .setExtensionName(name) - .setImports(importsBuilder.buildOrThrow()) - .setLocation(Location.BUILTIN) - .build(); - } - - @Test - public void createValue_moduleExtensions() throws Exception { - Module root = - Module.builder() - .setName("root") - .setVersion(Version.parse("1.0")) - .setKey(ModuleKey.ROOT) - .addDep("rje", createModuleKey("rules_jvm_external", "1.0")) - .addDep("rpy", createModuleKey("rules_python", "2.0")) - .addExtensionUsage( - createModuleExtensionUsage("@rje//:defs.bzl", "maven", "av", "autovalue")) - .addExtensionUsage( - createModuleExtensionUsage("@rpy//:defs.bzl", "pip", "numpy", "numpy")) - .build(); - ModuleKey depKey = createModuleKey("dep", "2.0"); - Module dep = - Module.builder() - .setName("dep") - .setVersion(Version.parse("2.0")) - .setKey(depKey) - .addDep("rules_python", createModuleKey("rules_python", "2.0")) - .addExtensionUsage( - createModuleExtensionUsage("@rules_python//:defs.bzl", "pip", "np", "numpy")) - .addExtensionUsage( - createModuleExtensionUsage("//:defs.bzl", "myext", "oneext", "myext")) - .addExtensionUsage( - createModuleExtensionUsage("//incredible:conflict.bzl", "myext", "twoext", "myext")) - .build(); - ImmutableMap depGraph = ImmutableMap.of(ModuleKey.ROOT, root, depKey, dep); - - ModuleExtensionId maven = - ModuleExtensionId.create( - Label.parseCanonical("@@rules_jvm_external~1.0//:defs.bzl"), "maven"); - ModuleExtensionId pip = - ModuleExtensionId.create(Label.parseCanonical("@@rules_python~2.0//:defs.bzl"), "pip"); - ModuleExtensionId myext = - ModuleExtensionId.create(Label.parseCanonical("@@dep~2.0//:defs.bzl"), "myext"); - ModuleExtensionId myext2 = - ModuleExtensionId.create( - Label.parseCanonical("@@dep~2.0//incredible:conflict.bzl"), "myext"); - - BazelModuleResolutionValue value = - BazelModuleResolutionFunction.createValue(depGraph, depGraph, ImmutableMap.of()); - assertThat(value.getExtensionUsagesTable()).hasSize(5); - assertThat(value.getExtensionUsagesTable()) - .containsCell(maven, ModuleKey.ROOT, root.getExtensionUsages().get(0)); - assertThat(value.getExtensionUsagesTable()) - .containsCell(pip, ModuleKey.ROOT, root.getExtensionUsages().get(1)); - assertThat(value.getExtensionUsagesTable()) - .containsCell(pip, depKey, dep.getExtensionUsages().get(0)); - assertThat(value.getExtensionUsagesTable()) - .containsCell(myext, depKey, dep.getExtensionUsages().get(1)); - assertThat(value.getExtensionUsagesTable()) - .containsCell(myext2, depKey, dep.getExtensionUsages().get(2)); - - assertThat(value.getExtensionUniqueNames()) - .containsExactly( - maven, "rules_jvm_external~1.0~maven", - pip, "rules_python~2.0~pip", - myext, "dep~2.0~myext", - myext2, "dep~2.0~myext2"); - - assertThat(value.getFullRepoMapping(ModuleKey.ROOT)) - .isEqualTo( - createRepositoryMapping( - ModuleKey.ROOT, - "", - "", - "root", - "", - "rje", - "rules_jvm_external~1.0", - "rpy", - "rules_python~2.0", - "av", - "rules_jvm_external~1.0~maven~autovalue", - "numpy", - "rules_python~2.0~pip~numpy")); - assertThat(value.getFullRepoMapping(depKey)) - .isEqualTo( - createRepositoryMapping( - depKey, - "dep", - "dep~2.0", - "rules_python", - "rules_python~2.0", - "np", - "rules_python~2.0~pip~numpy", - "oneext", - "dep~2.0~myext~myext", - "twoext", - "dep~2.0~myext2~myext")); - } - @Test public void testBazelInvalidCompatibility() throws Exception { scratch.file( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index 6ac576ea5a2f92..37b92c3609dfcc 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -123,6 +123,7 @@ public void setup() throws Exception { .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( SkyFunctions.MODULE_FILE, diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index 634f7318543cc7..f8ad86388377d7 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -196,12 +196,12 @@ public void setup() throws Exception { .put( SkyFunctions.PACKAGE, new PackageFunction( - /*packageFactory=*/ null, - /*pkgLocator=*/ null, - /*showLoadingProgress=*/ null, - /*numPackagesSuccessfullyLoaded=*/ new AtomicInteger(), - /*bzlLoadFunctionForInlining=*/ null, - /*packageProgress=*/ null, + /* packageFactory= */ null, + /* pkgLocator= */ null, + /* showLoadingProgress= */ null, + /* numPackagesSuccessfullyLoaded= */ new AtomicInteger(), + /* bzlLoadFunctionForInlining= */ null, + /* packageProgress= */ null, PackageFunction.ActionOnIOExceptionReadingBuildFile.UseOriginalIOException .INSTANCE, GlobbingStrategy.SKYFRAME_HYBRID, @@ -221,7 +221,7 @@ public void setup() throws Exception { .put( SkyFunctions.IGNORED_PACKAGE_PREFIXES, new IgnoredPackagePrefixesFunction( - /*ignoredPackagePrefixesFile=*/ PathFragment.EMPTY_FRAGMENT)) + /* ignoredPackagePrefixesFile= */ PathFragment.EMPTY_FRAGMENT)) .put(SkyFunctions.RESOLVED_HASH_VALUES, new ResolvedHashesFunction()) .put(SkyFunctions.REPOSITORY_MAPPING, new RepositoryMappingFunction()) .put( @@ -234,7 +234,7 @@ public void setup() throws Exception { ruleClassProvider, packageFactory, directories, - /*bzlLoadFunctionForInlining=*/ null)) + /* bzlLoadFunctionForInlining= */ null)) .put( SkyFunctions.REPOSITORY_DIRECTORY, new RepositoryDelegatorFunction( @@ -247,6 +247,7 @@ public void setup() throws Exception { .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put(SkyFunctions.SINGLE_EXTENSION_USAGES, new SingleExtensionUsagesFunction()) .put(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/SelectionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/SelectionTest.java index 5316c2bc5e1614..2aed3463828d41 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/SelectionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/SelectionTest.java @@ -22,7 +22,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil.ModuleBuilder; -import com.google.devtools.build.lib.bazel.bzlmod.Selection.SelectionResult; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -53,7 +52,8 @@ public void diamond_simple() throws Exception { .put(ModuleBuilder.create("ddd", "2.0", 1).buildEntry()) .buildOrThrow(); - SelectionResult selectionResult = Selection.run(depGraph, /*overrides=*/ ImmutableMap.of()); + BazelModuleResolutionValue selectionResult = + Selection.run(depGraph, /* overrides= */ ImmutableMap.of()); assertThat(selectionResult.getResolvedDepGraph().entrySet()) .containsExactly( ModuleBuilder.create("aaa", Version.EMPTY) @@ -117,7 +117,8 @@ public void diamond_withFurtherRemoval() throws Exception { .put(ModuleBuilder.create("eee", "1.0").buildEntry()) .build(); - SelectionResult selectionResult = Selection.run(depGraph, /*overrides=*/ ImmutableMap.of()); + BazelModuleResolutionValue selectionResult = + Selection.run(depGraph, /* overrides= */ ImmutableMap.of()); assertThat(selectionResult.getResolvedDepGraph().entrySet()) .containsExactly( ModuleBuilder.create("aaa", Version.EMPTY) @@ -180,7 +181,8 @@ public void circularDependencyDueToSelection() throws Exception { .put(ModuleBuilder.create("ddd", "1.0").buildEntry()) .buildOrThrow(); - SelectionResult selectionResult = Selection.run(depGraph, /*overrides=*/ ImmutableMap.of()); + BazelModuleResolutionValue selectionResult = + Selection.run(depGraph, /* overrides= */ ImmutableMap.of()); assertThat(selectionResult.getResolvedDepGraph().entrySet()) .containsExactly( ModuleBuilder.create("aaa", Version.EMPTY) @@ -287,7 +289,8 @@ public void differentCompatibilityLevelIsOkIfUnreferenced() throws Exception { // \-> ccc 1.1 // \-> ddd 1.0 -> bbb 1.1 // \-> eee 1.0 -> ccc 1.1 - SelectionResult selectionResult = Selection.run(depGraph, /*overrides=*/ ImmutableMap.of()); + BazelModuleResolutionValue selectionResult = + Selection.run(depGraph, /* overrides= */ ImmutableMap.of()); assertThat(selectionResult.getResolvedDepGraph().entrySet()) .containsExactly( ModuleBuilder.create("aaa", "1.0") @@ -384,7 +387,7 @@ public void multipleVersionOverride_fork_goodCase() throws Exception { MultipleVersionOverride.create( ImmutableList.of(Version.parse("1.0"), Version.parse("2.0")), "")); - SelectionResult selectionResult = Selection.run(depGraph, overrides); + BazelModuleResolutionValue selectionResult = Selection.run(depGraph, overrides); assertThat(selectionResult.getResolvedDepGraph().entrySet()) .containsExactly( ModuleBuilder.create("aaa", Version.EMPTY) @@ -457,7 +460,7 @@ public void multipleVersionOverride_diamond_differentCompatibilityLevels() throw MultipleVersionOverride.create( ImmutableList.of(Version.parse("1.0"), Version.parse("2.0")), "")); - SelectionResult selectionResult = Selection.run(depGraph, overrides); + BazelModuleResolutionValue selectionResult = Selection.run(depGraph, overrides); assertThat(selectionResult.getResolvedDepGraph().entrySet()) .containsExactly( ModuleBuilder.create("aaa", Version.EMPTY) @@ -506,7 +509,7 @@ public void multipleVersionOverride_diamond_sameCompatibilityLevel() throws Exce MultipleVersionOverride.create( ImmutableList.of(Version.parse("1.0"), Version.parse("2.0")), "")); - SelectionResult selectionResult = Selection.run(depGraph, overrides); + BazelModuleResolutionValue selectionResult = Selection.run(depGraph, overrides); assertThat(selectionResult.getResolvedDepGraph().entrySet()) .containsExactly( ModuleBuilder.create("aaa", Version.EMPTY) @@ -584,7 +587,7 @@ public void multipleVersionOverride_diamond_snappingToNextHighestVersion() throw // \-> bbb3@1.0 -> ccc@1.7 [originally ccc@1.5] // \-> bbb4@1.0 -> ccc@1.7 [allowed] // \-> bbb5@1.0 -> ccc@2.0 [allowed] - SelectionResult selectionResult = Selection.run(depGraph, overrides); + BazelModuleResolutionValue selectionResult = Selection.run(depGraph, overrides); assertThat(selectionResult.getResolvedDepGraph().entrySet()) .containsExactly( ModuleBuilder.create("aaa", Version.EMPTY) @@ -797,7 +800,7 @@ public void multipleVersionOverride_diamond_badVersionsAreOkayIfUnreferenced() t // \ \-> bbb4@1.1 // \-> bbb4@1.1 // ccc@1.5 and ccc@3.0, the versions violating the allowlist, are gone. - SelectionResult selectionResult = Selection.run(depGraph, overrides); + BazelModuleResolutionValue selectionResult = Selection.run(depGraph, overrides); assertThat(selectionResult.getResolvedDepGraph().entrySet()) .containsExactly( ModuleBuilder.create("aaa", Version.EMPTY) diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java index e36870fb9b8525..decb4e5e2544f9 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java @@ -30,6 +30,7 @@ import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ServerDirectories; import com.google.devtools.build.lib.analysis.util.AnalysisMock; +import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; @@ -180,7 +181,7 @@ public void setupDelegator() throws Exception { null, null, null, - /*packageProgress=*/ null, + /* packageProgress= */ null, PackageFunction.ActionOnIOExceptionReadingBuildFile.UseOriginalIOException .INSTANCE, GlobbingStrategy.SKYFRAME_HYBRID, @@ -198,7 +199,7 @@ public void setupDelegator() throws Exception { ruleClassProvider, pkgFactory, directories, - /*bzlLoadFunctionForInlining=*/ null)) + /* bzlLoadFunctionForInlining= */ null)) .put( SkyFunctions.LOCAL_REPOSITORY_LOOKUP, new LocalRepositoryLookupFunction( @@ -217,11 +218,12 @@ public void setupDelegator() throws Exception { .put( SkyFunctions.IGNORED_PACKAGE_PREFIXES, new IgnoredPackagePrefixesFunction( - /*ignoredPackagePrefixesFile=*/ PathFragment.EMPTY_FRAGMENT)) + /* ignoredPackagePrefixesFile= */ PathFragment.EMPTY_FRAGMENT)) .put(SkyFunctions.RESOLVED_HASH_VALUES, new ResolvedHashesFunction()) .put( SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootPath, ImmutableMap.of())) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE,