Skip to content

Commit

Permalink
Lockfile cherry picks (bazelbuild#18143)
Browse files Browse the repository at this point in the history
* Module Lockfile

- Created lockfile K/F/V holding module information and its related flags
- Updated BazelDepGraph function to use the lockfile
- Created type adapters for module, related classes & immutable collections
- Updated tests with Module.bazel to work after enabling the lockfile
- Added new java test class for the lockfile
- Added a flag to enable/disable lockfile (while it's still in progress)
- Added integration tests

PiperOrigin-RevId: 520925174
Change-Id: Ib8da99aa86a1797da44dc9a899d932f58c4a84c9

# Conflicts:
#	src/test/py/bazel/bzlmod/bazel_module_test.py

* Make lockfile human readable

PiperOrigin-RevId: 521009432
Change-Id: I2b2e5e0c845f05c428ff8772aba2abce72a3830f

* Split bazel_module_tests into smaller ones

PiperOrigin-RevId: 521766028
Change-Id: I44f5f68e4357bcc1219985e983814c5e4afec65d

# Conflicts:
#	src/test/py/bazel/BUILD
#	src/test/py/bazel/bzlmod/bazel_module_test.py

* Update lockfile name

PiperOrigin-RevId: 521871482
Change-Id: Ib4357c4d35fa8ebd2e636c7f15740247ee8dfab2

* Add repospec to the module

- With this we can get the repospec directly from the module not the registry (no need for internet access)
- Since Repospec & Tag classes have an "attributes" field that can hold starlark values, an adapter was needed to serialize/deserialize starlark values

PiperOrigin-RevId: 523364539
Change-Id: Ifcdf0c6b4b6fbbcdae9c14a4b0cd7f53ae91c161

* Update lockfile flag from experimental to different modes flag:
- Update: run resolution and update the lock file when it mismatches the module
- Error: throw an error if the module doesn't match the lockfile
- Off: don't read/update the lockfile

PiperOrigin-RevId: 524813416
Change-Id: I5cc3577fdbed8339ada50001081b75b4932c017c

* Hash local overrides Module files into the lockfile

PiperOrigin-RevId: 525405790
Change-Id: I7c9b26469c889435ed4adc4a49197c10e0284af0

# Conflicts:
#	src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java
#	src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java

* update functions names

* Add auto_value_gson and fixes

* update auto value import

* increase install base size

RELNOTES: Added preliminary support for a lockfile for Bzlmod. It's disabled by default; use `--lockfile_mode=update` to enable it. This lockfile contains only Bazel module information; it does not involve module extensions.
  • Loading branch information
SalmaSamy authored Apr 20, 2023
1 parent 76ad4a9 commit 5afb8b6
Show file tree
Hide file tree
Showing 68 changed files with 3,169 additions and 840 deletions.
4 changes: 4 additions & 0 deletions .bazelci/postsubmit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,10 @@ tasks:
- "-//src/java_tools/import_deps_checker/..."
# We hit connection timeout error when downloading multiple URLs on RBE, see b/217865760
- "-//src/test/py/bazel:bazel_module_test"
- "-//src/test/py/bazel:bazel_lockfile_test"
- "-//src/test/py/bazel:bazel_overrides_test"
- "-//src/test/py/bazel:bazel_repo_mapping_test"
- "-//src/test/py/bazel:bazel_yanked_versions_test"
- "-//src/test/shell/bazel:verify_workspace"
include_json_profile:
- build
Expand Down
4 changes: 4 additions & 0 deletions .bazelci/presubmit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,10 @@ tasks:
- "-//src/java_tools/import_deps_checker/..."
# We hit connection timeout error when downloading multiple URLs on RBE, see b/217865760
- "-//src/test/py/bazel:bazel_module_test"
- "-//src/test/py/bazel:bazel_lockfile_test"
- "-//src/test/py/bazel:bazel_overrides_test"
- "-//src/test/py/bazel:bazel_repo_mapping_test"
- "-//src/test/py/bazel:bazel_yanked_versions_test"
- "-//src/test/shell/bazel:verify_workspace"
kythe_ubuntu2004:
shell_commands:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@
import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
import com.google.devtools.build.lib.analysis.RuleDefinition;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
import com.google.devtools.build.lib.bazel.bzlmod.AttributeValues;
import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction;
import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction;
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorFunction;
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason;
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction;
Expand All @@ -46,6 +48,7 @@
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions;
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode;
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode;
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode;
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.RepositoryOverride;
import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache;
import com.google.devtools.build.lib.bazel.repository.downloader.DelegatingDownloader;
Expand Down Expand Up @@ -139,6 +142,7 @@ public class BazelRepositoryModule extends BlazeModule {
private final AtomicBoolean ignoreDevDeps = new AtomicBoolean(false);
private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING;
private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR;
private LockfileMode bazelLockfileMode = LockfileMode.OFF;
private List<String> allowedYankedVersions = ImmutableList.of();
private SingleExtensionEvalFunction singleExtensionEvalFunction;

Expand Down Expand Up @@ -232,7 +236,8 @@ public void workspaceInit(
public RepoSpec getRepoSpec(RepositoryName repoName) {
return RepoSpec.builder()
.setRuleClassName("local_config_platform")
.setAttributes(ImmutableMap.of("name", repoName.getName()))
.setAttributes(
AttributeValues.create(ImmutableMap.of("name", repoName.getName())))
.build();
}

Expand All @@ -249,7 +254,10 @@ public ResolutionReason getResolutionReason() {
.addSkyFunction(
SkyFunctions.MODULE_FILE,
new ModuleFileFunction(registryFactory, directories.getWorkspace(), builtinModules))
.addSkyFunction(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction())
.addSkyFunction(
SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(directories.getWorkspace()))
.addSkyFunction(
SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace()))
.addSkyFunction(SkyFunctions.BAZEL_MODULE_INSPECTION, new BazelModuleInspectorFunction())
.addSkyFunction(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction())
.addSkyFunction(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction)
Expand Down Expand Up @@ -421,6 +429,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
ignoreDevDeps.set(repoOptions.ignoreDevDependency);
checkDirectDepsMode = repoOptions.checkDirectDependencies;
bazelCompatibilityMode = repoOptions.bazelCompatibilityMode;
bazelLockfileMode = repoOptions.lockfileMode;
allowedYankedVersions = repoOptions.allowedYankedVersions;

if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) {
Expand Down Expand Up @@ -512,6 +521,7 @@ public ImmutableList<Injected> getPrecomputedValues() {
BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, checkDirectDepsMode),
PrecomputedValue.injected(
BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, bazelCompatibilityMode),
PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, bazelLockfileMode),
PrecomputedValue.injected(
BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, allowedYankedVersions));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ public RepoSpec build() {
return RepoSpec.builder()
.setBzlFile("@bazel_tools//tools/build_defs/repo:http.bzl")
.setRuleClassName("http_archive")
.setAttributes(attrBuilder.buildOrThrow())
.setAttributes(AttributeValues.create(attrBuilder.buildOrThrow()))
.build();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// Copyright 2023 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package com.google.devtools.build.lib.bazel.bzlmod;

import com.google.auto.value.AutoValue;
import com.google.common.collect.Maps;
import com.ryanharter.auto.value.gson.GenerateTypeAdapter;
import java.util.List;
import java.util.Map;
import net.starlark.java.eval.Dict;
import net.starlark.java.eval.Starlark;

/** Wraps a dictionary of attribute names and values. Always uses a dict to represent them */
@AutoValue
@GenerateTypeAdapter
public abstract class AttributeValues {

public static AttributeValues create(Dict<String, Object> attribs) {
return new AutoValue_AttributeValues(attribs);
}

public static AttributeValues create(Map<String, Object> attribs) {
return new AutoValue_AttributeValues(
Dict.immutableCopyOf(Maps.transformValues(attribs, AttributeValues::valueToStarlark)));
}

public abstract Dict<String, Object> attributes();

// TODO(salmasamy) this is a copy of Attribute::valueToStarlark, Maybe think of a better place?
private static Object valueToStarlark(Object x) {
// Is x a non-empty string_list_dict?
if (x instanceof Map) {
Map<?, ?> map = (Map<?, ?>) x;
if (!map.isEmpty() && map.values().iterator().next() instanceof List) {
Dict.Builder<Object, Object> dict = Dict.builder();
for (Map.Entry<?, ?> e : map.entrySet()) {
dict.put(e.getKey(), Starlark.fromJava(e.getValue(), null));
}
return dict.buildImmutable();
}
}
// For all other attribute values, shallow conversion is safe.
return Starlark.fromJava(x, null);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
// Copyright 2023 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package com.google.devtools.build.lib.bazel.bzlmod;

import com.google.devtools.build.lib.cmdline.Label;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import com.google.gson.TypeAdapter;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import net.starlark.java.eval.Dict;
import net.starlark.java.eval.EvalException;
import net.starlark.java.eval.Mutability;
import net.starlark.java.eval.Starlark;
import net.starlark.java.eval.StarlarkInt;
import net.starlark.java.eval.StarlarkList;

/** Helps serialize/deserialize {@link AttributeValues}, which contains Starlark values. */
public class AttributeValuesAdapter extends TypeAdapter<AttributeValues> {

@Override
public void write(JsonWriter out, AttributeValues attributeValues) throws IOException {
JsonObject jsonObject = new JsonObject();
for (Map.Entry<String, Object> entry : attributeValues.attributes().entrySet()) {
jsonObject.add(entry.getKey(), serializeObject(entry.getValue()));
}
out.jsonValue(jsonObject.toString());
}

@Override
public AttributeValues read(JsonReader in) throws IOException {
JsonObject jsonObject = JsonParser.parseReader(in).getAsJsonObject();
Dict.Builder<String, Object> dict = Dict.builder();
for (Map.Entry<String, JsonElement> entry : jsonObject.entrySet()) {
dict.put(entry.getKey(), deserializeObject(entry.getValue()));
}
return AttributeValues.create(dict.buildImmutable());
}

/**
* Starlark Object Types Bool Integer String Label List (Int, label, string) Dict (String,list) &
* (Label, String)
*/
private JsonElement serializeObject(Object obj) {
if (obj.equals(Starlark.NONE)) {
return JsonNull.INSTANCE;
} else if (obj instanceof Boolean) {
return new JsonPrimitive((Boolean) obj);
} else if (obj instanceof StarlarkInt) {
try {
return new JsonPrimitive(((StarlarkInt) obj).toInt("serialization into the lockfile"));
} catch (EvalException e) {
throw new IllegalArgumentException("Unable to parse StarlarkInt to Integer: " + e);
}
} else if (obj instanceof String || obj instanceof Label) {
return new JsonPrimitive(serializeObjToString(obj));
} else if (obj instanceof Dict) {
JsonObject jsonObject = new JsonObject();
for (Map.Entry<?, ?> entry : ((Dict<?, ?>) obj).entrySet()) {
jsonObject.add(serializeObjToString(entry.getKey()), serializeObject(entry.getValue()));
}
return jsonObject;
} else if (obj instanceof StarlarkList) {
JsonArray jsonArray = new JsonArray();
for (Object item : (StarlarkList<?>) obj) {
jsonArray.add(serializeObject(item));
}
return jsonArray;
} else {
throw new IllegalArgumentException("Unsupported type: " + obj.getClass());
}
}

private Object deserializeObject(JsonElement json) {
if (json == null || json.isJsonNull()) {
return Starlark.NONE;
} else if (json.isJsonPrimitive()) {
JsonPrimitive jsonPrimitive = json.getAsJsonPrimitive();
if (jsonPrimitive.isBoolean()) {
return jsonPrimitive.getAsBoolean();
} else if (jsonPrimitive.isNumber()) {
return StarlarkInt.of(jsonPrimitive.getAsInt());
} else if (jsonPrimitive.isString()) {
return deserializeStringToObject(jsonPrimitive.getAsString());
} else {
throw new IllegalArgumentException("Unsupported JSON primitive: " + jsonPrimitive);
}
} else if (json.isJsonObject()) {
JsonObject jsonObject = json.getAsJsonObject();
Dict.Builder<Object, Object> dict = Dict.builder();
for (Map.Entry<String, JsonElement> entry : jsonObject.entrySet()) {
dict.put(deserializeStringToObject(entry.getKey()), deserializeObject(entry.getValue()));
}
return dict.buildImmutable();
} else if (json.isJsonArray()) {
JsonArray jsonArray = json.getAsJsonArray();
List<Object> list = new ArrayList<>();
for (JsonElement item : jsonArray) {
list.add(deserializeObject(item));
}
return StarlarkList.copyOf(Mutability.IMMUTABLE, list);
} else {
throw new IllegalArgumentException("Unsupported JSON element: " + json);
}
}

/**
* Serializes an object (Label or String) to String A label is converted to a String as it is. A
* String is being modified with a delimiter to be easily differentiated from the label when
* deserializing.
*
* @param obj String or Label
* @return serialized object
*/
private String serializeObjToString(Object obj) {
if (obj instanceof Label) {
return ((Label) obj).getUnambiguousCanonicalForm();
}
return "--" + obj;
}

/**
* Deserializes a string to either a label or a String depending on the prefix. A string will have
* a delimiter at the start, else it is converted to a label.
*
* @param value String to be deserialized
* @return Object of type String of Label
*/
private Object deserializeStringToObject(String value) {
if (value.startsWith("--")) {
return value.substring(2);
}
return Label.parseCanonicalUnchecked(value);
}
}
Loading

0 comments on commit 5afb8b6

Please sign in to comment.