Skip to content

Commit

Permalink
Add a codec for Module.
Browse files Browse the repository at this point in the history
Adds a BzlLoadValue.Key to BazelModuleContext to achieve this. However,
since BazelModuleContext is inseparable from Label, which BzlLoadValue.Key
depends on, that would create a circular dependency. Adds the BazelModuleKey
interface to break the cycle.

PiperOrigin-RevId: 702700386
Change-Id: I59c93167b97b86a524f6a62ae4b5ffa991a9b729
  • Loading branch information
aoeui authored and copybara-github committed Dec 4, 2024
1 parent eae9d4d commit 54237be
Show file tree
Hide file tree
Showing 14 changed files with 266 additions and 19 deletions.
1 change: 1 addition & 0 deletions src/main/java/com/google/devtools/build/lib/cmdline/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ java_library(
srcs = [
"BazelCompileContext.java",
"BazelModuleContext.java",
"BazelModuleKey.java",
"IgnoredSubdirectories.java",
"Label.java",
"LabelConstants.java",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,12 @@
@AutoValue
public abstract class BazelModuleContext {
/** Label associated with the Starlark {@link net.starlark.java.eval.Module}. */
public abstract Label label();
public final Label label() {
return key().getLabel();
}

/** {@link com.google.devtools.build.lib.skyframe.BzlLoadValue.Key} used to create the module. */
public abstract BazelModuleKey key();

/** The repository mapping applicable to the repo where the .bzl file is located in. */
public abstract RepositoryMapping repoMapping();
Expand Down Expand Up @@ -158,13 +163,12 @@ public static BazelModuleContext ofInnermostBzlOrFail(StarlarkThread thread, Str
}

public static BazelModuleContext create(
Label label,
BazelModuleKey key,
RepositoryMapping repoMapping,
String filename,
ImmutableList<Module> loads,
byte[] bzlTransitiveDigest) {
return new AutoValue_BazelModuleContext(
label, repoMapping, filename, loads, bzlTransitiveDigest);
return new AutoValue_BazelModuleContext(key, repoMapping, filename, loads, bzlTransitiveDigest);
}

public final Label.PackageContext packageContext() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// Copyright 2020 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.cmdline;

import com.google.devtools.build.lib.skyframe.SkyFunctions;
import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;

/**
* Interface for {@link com.google.devtools.build.lib.skyframe.BzlLoadValue.Key}.
*
* <p>This exists to break what would otherwise be a circular dependency between {@link Label},
* {@link BazelModuleContext} and {@link com.google.devtools.build.lib.skyframe.BzlLoadValue.Key}.
*/
public interface BazelModuleKey extends SkyKey {
/** Absolute label of the .bzl file to be loaded. */
Label getLabel();

@Override
default SkyFunctionName functionName() {
return SkyFunctions.BZL_LOAD;
}

/** Creates a fake instance for testing. */
static BazelModuleKey createFakeModuleKeyForTesting(Label label) {
return new FakeModuleKey(label);
}

/** Key for {@link BazelModuleContext}s created outside of Skyframe for testing */
static final class FakeModuleKey implements BazelModuleKey {
private final Label label;

private FakeModuleKey(Label label) {
this.label = label;
}

@Override
public Label getLabel() {
return label;
}

@Override
public SkyFunctionName functionName() {
throw new UnsupportedOperationException();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -882,7 +882,7 @@ private BzlLoadValue computeInternalWithCompiledBzl(
// including the label and a reified copy of the load DAG.
BazelModuleContext bazelModuleContext =
BazelModuleContext.create(
label,
key,
repoMapping,
prog.getFilename(),
ImmutableList.copyOf(loadMap.values()),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableTable;
import com.google.devtools.build.lib.cmdline.BazelModuleKey;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
Expand All @@ -32,7 +33,6 @@
import com.google.devtools.build.lib.util.HashCodes;
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyKey.SkyKeyInterner;
import com.google.devtools.build.skyframe.SkyValue;
Expand Down Expand Up @@ -102,7 +102,7 @@ public ImmutableTable<RepositoryName, String, RepositoryName> getRecordedRepoMap
private static final SkyKeyInterner<Key> keyInterner = SkyKey.newInterner();

/** SkyKey for a Starlark load. */
public abstract static sealed class Key implements SkyKey
public abstract static sealed class Key implements BazelModuleKey
permits KeyForBuild, KeyForWorkspace, KeyForBuiltins, KeyForBzlmod {
// Closed, for class-based equals()/hashCode().
private Key() {}
Expand All @@ -114,6 +114,7 @@ private Key() {}
* other keys to use {@code @_builtins}, but since no real repo by that name may be defined,
* they won't evaluate to a successful result.)
*/
@Override
public abstract Label getLabel();

/** Returns true if this is a request for the special BUILD prelude file. */
Expand Down Expand Up @@ -158,11 +159,6 @@ final boolean isSclDialect() {
*/
abstract BzlCompileValue.Key getCompileKey(Root root);

@Override
public SkyFunctionName functionName() {
return SkyFunctions.BZL_LOAD;
}

@Override
public final boolean valueIsShareable() {
// We don't guarantee that all constructs implement equality, meaning we can't correctly
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ java_library(
name = "serialization_registry_setup_helpers",
srcs = ["SerializationRegistrySetupHelpers.java"],
deps = [
":module_codec",
":serialization",
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib/actions:artifacts",
Expand Down Expand Up @@ -65,6 +66,7 @@ java_library(
["**/*.java"],
exclude = [
"AbstractExportedStarlarkSymbolCodec.java",
"ModuleCodec.java",
"CodecScanningConstants.java",
"SerializationRegistrySetupHelpers.java",
"SerializationConstants.java",
Expand Down Expand Up @@ -124,3 +126,16 @@ java_library(
"@com_google_protobuf//:protobuf_java",
],
)

java_library(
name = "module_codec",
srcs = ["ModuleCodec.java"],
deps = [
":serialization",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/skyframe:bzl_load_value",
"//src/main/java/net/starlark/java/eval",
"//third_party:guava",
"@com_google_protobuf//:protobuf_java",
],
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// Copyright 2024 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe.serialization;

import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.devtools.build.lib.skyframe.BzlLoadValue.bzlLoadKeyCodec;

import com.google.devtools.build.lib.cmdline.BazelModuleContext;
import com.google.devtools.build.lib.skyframe.BzlLoadValue;
import com.google.protobuf.CodedInputStream;
import com.google.protobuf.CodedOutputStream;
import java.io.IOException;
import net.starlark.java.eval.Module;

/**
* Codec for {@link Module}.
*
* <p>Serializes using the module's associated {@link BzlLoadValue.Key}.
*/
public final class ModuleCodec extends DeferredObjectCodec<Module> {
private static final ModuleCodec INSTANCE = new ModuleCodec();

public static ModuleCodec moduleCodec() {
return INSTANCE;
}

@Override
public Class<Module> getEncodedClass() {
return Module.class;
}

@Override
public boolean autoRegister() {
// Unit tests that bypass Skyframe for Module loading cannot use this codec.
return false;
}

@Override
public void serialize(SerializationContext context, Module obj, CodedOutputStream codedOut)
throws IOException, SerializationException {
var moduleContext = checkNotNull(BazelModuleContext.of(obj), "module %s missing context", obj);
context.serializeLeaf((BzlLoadValue.Key) moduleContext.key(), bzlLoadKeyCodec(), codedOut);
}

@Override
public DeferredValue<Module> deserializeDeferred(
AsyncDeserializationContext context, CodedInputStream codedIn)
throws IOException, SerializationException {
BzlLoadValue.Key bzlLoadKey = context.deserializeLeaf(codedIn, bzlLoadKeyCodec());
var builder = new DeserializationBuilder();
context.getSkyValue(bzlLoadKey, builder, DeserializationBuilder::setBzlLoadValue);
return builder;
}

private static final class DeserializationBuilder implements DeferredValue<Module> {
private BzlLoadValue loadValue;

@Override
public Module call() {
return checkNotNull(loadValue, "Skyframe lookup value not set").getModule();
}

private static void setBzlLoadValue(DeserializationBuilder builder, Object value) {
builder.loadValue = (BzlLoadValue) value;
}
}

private ModuleCodec() {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ public static ObjectCodecRegistry.Builder addStarlarkFunctionality(
.add(ArrayCodec.forComponentType(Artifact.class))
.add(new DeferredNestedSetCodec())
.add(Label.valueSharingCodec())
.add(ModuleCodec.moduleCodec())
.add(PackageIdentifier.valueSharingCodec())
.add(ConfiguredTargetKey.valueSharingCodec())
.add(TransitiveInfoProviderMapImpl.valueSharingCodec())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import com.google.devtools.build.lib.analysis.util.ActionTester;
import com.google.devtools.build.lib.analysis.util.BuildViewTestCase;
import com.google.devtools.build.lib.cmdline.BazelModuleContext;
import com.google.devtools.build.lib.cmdline.BazelModuleKey;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.RepositoryMapping;
import com.google.devtools.build.lib.events.StoredEventHandler;
Expand Down Expand Up @@ -76,7 +77,8 @@ private static Object exec(String... lines) throws Exception {
StarlarkSemantics.DEFAULT,
ImmutableMap.of(),
BazelModuleContext.create(
Label.parseCanonicalUnchecked("//test:label"),
BazelModuleKey.createFakeModuleKeyForTesting(
Label.parseCanonicalUnchecked("//test:label")),
RepositoryMapping.ALWAYS_FALLBACK,
"test/label.bzl",
/* loads= */ ImmutableList.of(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import com.google.devtools.build.lib.analysis.config.CoreOptions;
import com.google.devtools.build.lib.analysis.starlark.StarlarkCustomCommandLine.VectorArg;
import com.google.devtools.build.lib.cmdline.BazelModuleContext;
import com.google.devtools.build.lib.cmdline.BazelModuleKey;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.RepositoryMapping;
import com.google.devtools.build.lib.testutil.Scratch;
Expand Down Expand Up @@ -451,7 +452,8 @@ private static Object execStarlark(String code) throws Exception {
StarlarkSemantics.DEFAULT,
ImmutableMap.of(),
BazelModuleContext.create(
Label.parseCanonicalUnchecked("//test:label"),
BazelModuleKey.createFakeModuleKeyForTesting(
Label.parseCanonicalUnchecked("//test:label")),
RepositoryMapping.ALWAYS_FALLBACK,
"test/label.bzl",
/* loads= */ ImmutableList.of(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -313,10 +313,20 @@ java_test(
srcs = ["ModuleCodecTest.java"],
deps = [
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/packages",
"//src/main/java/com/google/devtools/build/lib/skyframe:bzl_load_value",
"//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_cluster",
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization",
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization:module_codec",
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils",
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils:round-tripping",
"//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
"//src/main/java/net/starlark/java/eval",
"//src/test/java/com/google/devtools/build/lib/analysis/util",
"//third_party:guava",
"//third_party:jsr305",
"//third_party:junit4",
"//third_party:truth",
],
)

Expand Down
Loading

0 comments on commit 54237be

Please sign in to comment.