From 9c230f122a99bb378426e20f814029ad8dea3eeb Mon Sep 17 00:00:00 2001 From: schmitt Date: Wed, 13 Mar 2019 13:42:51 -0700 Subject: [PATCH] Transformation for build configurations based on a platform/flags mapping. Introduces a new SkyValue which stores the information obtained from a mapping file (parser yet to be written) and provides logic to transform a build configuration (key) based on that. Step 3/N towards the platforms mapping functionality for https://github.com/bazelbuild/bazel/issues/6426 RELNOTES: None. PiperOrigin-RevId: 238298127 --- .../analysis/PlatformConfigurationLoader.java | 32 +-- .../build/lib/analysis/PlatformOptions.java | 69 ++++-- .../lib/analysis/config/BuildOptions.java | 5 + .../lib/skyframe/PlatformMappingValue.java | 196 +++++++++++++++ .../build/lib/skyframe/SkyFunctions.java | 6 +- .../skyframe/PlatformMappingValueTest.java | 224 ++++++++++++++++++ 6 files changed, 482 insertions(+), 50 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/skyframe/PlatformMappingValue.java create mode 100644 src/test/java/com/google/devtools/build/lib/skyframe/PlatformMappingValueTest.java diff --git a/src/main/java/com/google/devtools/build/lib/analysis/PlatformConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/analysis/PlatformConfigurationLoader.java index ce340050bad976..adad85ac8883e4 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/PlatformConfigurationLoader.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/PlatformConfigurationLoader.java @@ -16,13 +16,11 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; import com.google.devtools.build.lib.analysis.config.BuildConfiguration; import com.google.devtools.build.lib.analysis.config.BuildOptions; import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory; import com.google.devtools.build.lib.analysis.config.FragmentOptions; import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; -import com.google.devtools.build.lib.cmdline.Label; /** A loader that creates {@link PlatformConfiguration} instances based on command-line options. */ public class PlatformConfigurationLoader implements ConfigurationFragmentFactory { @@ -35,36 +33,10 @@ public ImmutableSet> requiredOptions() { public PlatformConfiguration create(BuildOptions buildOptions) throws InvalidConfigurationException { PlatformOptions platformOptions = buildOptions.get(PlatformOptions.class); - - // Handle default values for the host and target platform. - // TODO(https://github.com/bazelbuild/bazel/issues/6849): After migration, set the defaults - // directly. - Label hostPlatform; - if (platformOptions.hostPlatform != null) { - hostPlatform = platformOptions.hostPlatform; - } else if (platformOptions.autoConfigureHostPlatform) { - // Use the auto-configured host platform. - hostPlatform = PlatformOptions.DEFAULT_HOST_PLATFORM; - } else { - // Use the legacy host platform. - hostPlatform = PlatformOptions.LEGACY_DEFAULT_HOST_PLATFORM; - } - - Label targetPlatform; - if (!platformOptions.platforms.isEmpty()) { - targetPlatform = Iterables.getFirst(platformOptions.platforms, null); - } else if (platformOptions.autoConfigureHostPlatform) { - // Default to the host platform, whatever it is. - targetPlatform = hostPlatform; - } else { - // Use the legacy target platform - targetPlatform = PlatformOptions.LEGACY_DEFAULT_TARGET_PLATFORM; - } - return new PlatformConfiguration( - hostPlatform, + platformOptions.computeHostPlatform(), ImmutableList.copyOf(platformOptions.extraExecutionPlatforms), - targetPlatform, + platformOptions.computeTargetPlatform(), ImmutableList.copyOf(platformOptions.extraToolchains), ImmutableList.copyOf(platformOptions.enabledToolchainTypes)); } diff --git a/src/main/java/com/google/devtools/build/lib/analysis/PlatformOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/PlatformOptions.java index 7681871fee2bc0..788dc3a85956e0 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/PlatformOptions.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/PlatformOptions.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.analysis; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; import com.google.devtools.build.lib.analysis.config.BuildConfiguration; import com.google.devtools.build.lib.analysis.config.BuildConfiguration.LabelListConverter; import com.google.devtools.build.lib.analysis.config.FragmentOptions; @@ -101,23 +102,22 @@ public class PlatformOptions extends FragmentOptions { public List extraToolchains; @Option( - name = "toolchain_resolution_override", - allowMultiple = true, - defaultValue = "", - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = { - OptionEffectTag.AFFECTS_OUTPUTS, - OptionEffectTag.CHANGES_INPUTS, - OptionEffectTag.LOADING_AND_ANALYSIS - }, - deprecationWarning = - "toolchain_resolution_override is now a no-op and will be removed in" - + " an upcoming release", - help = - "Override toolchain resolution for a toolchain type with a specific toolchain. " - + "Example: --toolchain_resolution_override=@io_bazel_rules_go//:toolchain=" - + "@io_bazel_rules_go//:linux-arm64-toolchain" - ) + name = "toolchain_resolution_override", + allowMultiple = true, + defaultValue = "", + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + effectTags = { + OptionEffectTag.AFFECTS_OUTPUTS, + OptionEffectTag.CHANGES_INPUTS, + OptionEffectTag.LOADING_AND_ANALYSIS + }, + deprecationWarning = + "toolchain_resolution_override is now a no-op and will be removed in" + + " an upcoming release", + help = + "Override toolchain resolution for a toolchain type with a specific toolchain. " + + "Example: --toolchain_resolution_override=@io_bazel_rules_go//:toolchain=" + + "@io_bazel_rules_go//:linux-arm64-toolchain") public List toolchainResolutionOverrides; @Option( @@ -184,4 +184,39 @@ public PlatformOptions getHost() { host.useToolchainResolutionForJavaRules = this.useToolchainResolutionForJavaRules; return host; } + + /** Returns the intended target platform value based on options defined in this fragment. */ + public Label computeTargetPlatform() { + // Handle default values for the host and target platform. + // TODO(https://github.com/bazelbuild/bazel/issues/6849): After migration, set the defaults + // directly. + + if (!platforms.isEmpty()) { + return Iterables.getFirst(platforms, null); + } else if (autoConfigureHostPlatform) { + // Default to the host platform, whatever it is. + return computeHostPlatform(); + } else { + // Use the legacy target platform + return LEGACY_DEFAULT_TARGET_PLATFORM; + } + } + + /** Returns the intended host platform value based on options defined in this fragment. */ + public Label computeHostPlatform() { + // Handle default values for the host and target platform. + // TODO(https://github.com/bazelbuild/bazel/issues/6849): After migration, set the defaults + // directly. + + Label hostPlatform; + if (this.hostPlatform != null) { + return this.hostPlatform; + } else if (autoConfigureHostPlatform) { + // Use the auto-configured host platform. + return DEFAULT_HOST_PLATFORM; + } else { + // Use the legacy host platform. + return LEGACY_DEFAULT_HOST_PLATFORM; + } + } } diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildOptions.java index 7775d33c08969f..a47b2d6bb0b6a2 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildOptions.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildOptions.java @@ -192,6 +192,11 @@ public Collection getNativeOptions() { return fragmentOptionsMap.values(); } + /** Returns the set of fragment classes contained in these options. */ + public Set> getFragmentClasses() { + return fragmentOptionsMap.keySet(); + } + public ImmutableMap getStarlarkOptions() { return skylarkOptionsMap; } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PlatformMappingValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PlatformMappingValue.java new file mode 100644 index 00000000000000..4ad16cb904a48c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/skyframe/PlatformMappingValue.java @@ -0,0 +1,196 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Interner; +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.analysis.PlatformOptions; +import com.google.devtools.build.lib.analysis.config.BuildOptions; +import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.concurrent.BlazeInterners; +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.common.options.OptionsParser; +import com.google.devtools.common.options.OptionsParsingException; +import com.google.devtools.common.options.OptionsParsingResult; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Stores contents of a platforms/flags mapping file for transforming one {@link + * BuildConfigurationValue.Key} into another. + * + *

See + * the design for more details on how the mapping can be defined and the desired logic on how it + * is applied to configuration keys. + */ +public final class PlatformMappingValue implements SkyValue { + + /** Key for {@link PlatformMappingValue} based on the location of the mapping file. */ + @ThreadSafety.Immutable + @AutoCodec + public static final class Key implements SkyKey { + private static final Interner interner = BlazeInterners.newWeakInterner(); + + private final RootedPath path; + + private Key(RootedPath path) { + this.path = path; + } + + @AutoCodec.VisibleForSerialization + @AutoCodec.Instantiator + static Key create(RootedPath path) { + return interner.intern(new Key(path)); + } + + @Override + public SkyFunctionName functionName() { + return SkyFunctions.PLATFORM_MAPPING; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Key key = (Key) o; + return Objects.equals(path, key.path); + } + + @Override + public int hashCode() { + return Objects.hash(path); + } + + @Override + public String toString() { + return "PlatformMappingValue.Key{" + "path=" + path + '}'; + } + } + + private final Map> platformsToFlags; + private final Map, Label> flagsToPlatforms; + + /** + * Creates a new mapping value which will match on the given platforms (if a target platform is + * set on the key to be mapped), otherwise on the set of flags. + * + * @param platformsToFlags mapping from target platform label to the command line style flags that + * should be parsed & modified if that platform is set + * @param flagsToPlatforms mapping from a collection of command line style flags to a target + * platform that should be set if the flags match the mapped options + */ + PlatformMappingValue( + Map> platformsToFlags, + Map, Label> flagsToPlatforms) { + this.platformsToFlags = platformsToFlags; + this.flagsToPlatforms = flagsToPlatforms; + } + + /** + * Maps one {@link BuildConfigurationValue.Key} to another by way of mappings provided in a file. + * + *

The + * full design contains the details for the mapping logic but in short: + * + *

    + *
  1. If a target platform is set on the original then mappings from platform to flags will be + * applied. + *
  2. If no target platform is set then mappings from flags to platforms will be applied. + *
  3. If no matching flags to platforms mapping was found, the default target platform will be + * used. + *
+ * + * @param original the key representing the configuration to be mapped + * @param defaultBuildOptions build options as set by default in this server + * @return the mapped key if any mapping matched the original or else the original + * @throws OptionsParsingException if any of the user configured flags cannot be parsed + * @throws IllegalArgumentException if the original does not contain a {@link PlatformOptions} + * fragment + */ + public BuildConfigurationValue.Key map( + BuildConfigurationValue.Key original, BuildOptions defaultBuildOptions) + throws OptionsParsingException { + BuildOptions.OptionsDiffForReconstruction originalDiff = original.getOptionsDiff(); + BuildOptions originalOptions = defaultBuildOptions.applyDiff(originalDiff); + + Preconditions.checkArgument( + originalOptions.contains(PlatformOptions.class), + "When using platform mappings, all configurations must contain platform options"); + + BuildOptions modifiedOptions = null; + + if (!originalOptions.get(PlatformOptions.class).platforms.isEmpty()) { + List