diff --git a/src/test/java/com/google/devtools/build/lib/buildtool/util/BUILD b/src/test/java/com/google/devtools/build/lib/buildtool/util/BUILD index cbc1c1dd33d526..3a70cfb641a69b 100644 --- a/src/test/java/com/google/devtools/build/lib/buildtool/util/BUILD +++ b/src/test/java/com/google/devtools/build/lib/buildtool/util/BUILD @@ -98,6 +98,7 @@ java_library( "//src/test/java/com/google/devtools/build/lib/testutil:TestUtils", "//src/test/java/com/google/devtools/build/lib/vfs/util", "//third_party:guava", + "//third_party:guava-testlib", "//third_party:jsr305", "//third_party:junit4", "//third_party:truth", diff --git a/src/test/java/com/google/devtools/build/lib/buildtool/util/SkyframeIntegrationTestBase.java b/src/test/java/com/google/devtools/build/lib/buildtool/util/SkyframeIntegrationTestBase.java new file mode 100644 index 00000000000000..6976b497c32623 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/buildtool/util/SkyframeIntegrationTestBase.java @@ -0,0 +1,88 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.buildtool.util; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.testing.GcFinalization; +import com.google.devtools.build.lib.skyframe.SkyframeExecutor; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** Infrastructure to support Skyframe integration tests. */ +public abstract class SkyframeIntegrationTestBase extends BuildIntegrationTestCase { + + protected SkyframeExecutor skyframeExecutor() { + return runtimeWrapper.getSkyframeExecutor(); + } + + protected static List> weakRefs(Object... strongRefs) throws Exception { + List> result = new ArrayList<>(); + for (Object ref : strongRefs) { + result.add(new WeakReference<>(ref)); + } + return result; + } + + protected static void assertAllReleased(Iterable> refs) { + for (WeakReference ref : refs) { + GcFinalization.awaitClear(ref); + } + } + + private String makeGenruleContents(String value) { + return String.format( + "genrule(name='target', outs=['out'], cmd='/bin/echo %s > $(location out)')", value); + } + + protected void writeGenrule(String filename, String value) throws Exception { + write(filename, makeGenruleContents(value)); + } + + protected void writeGenruleAbsolute(Path file, String value) throws Exception { + writeAbsolute(file, makeGenruleContents(value)); + } + + protected void assertCharContentsIgnoringOrderAndWhitespace( + String expectedCharContents, String target) throws Exception { + Path path = Iterables.getOnlyElement(getArtifacts(target)).getPath(); + char[] actualChars = FileSystemUtils.readContentAsLatin1(path); + char[] expectedChars = expectedCharContents.toCharArray(); + Arrays.sort(actualChars); + Arrays.sort(expectedChars); + assertThat(new String(actualChars).trim()).isEqualTo(new String(expectedChars).trim()); + } + + protected void assertContents(String expectedContents, String target) throws Exception { + assertContents(expectedContents, Iterables.getOnlyElement(getArtifacts(target)).getPath()); + } + + protected void assertContents(String expectedContents, Path path) throws Exception { + String actualContents = new String(FileSystemUtils.readContentAsLatin1(path)); + assertThat(actualContents.trim()).isEqualTo(expectedContents); + } + + protected ImmutableList getOnlyOutputContentAsLines(String target) throws Exception { + return FileSystemUtils.readLines( + Iterables.getOnlyElement(getArtifacts(target)).getPath(), UTF_8); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/BUILD b/src/test/java/com/google/devtools/build/lib/skyframe/BUILD index 68cd782bd8f573..271862ad793356 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/BUILD +++ b/src/test/java/com/google/devtools/build/lib/skyframe/BUILD @@ -20,6 +20,7 @@ CROSS_PLATFORM_WINDOWS_TESTS = [ # Tests that are broken out from the SkyframeTests target into separate targets. EXCLUDED_FROM_SKYFRAME_TESTS = [ + "LocalDiffAwarenessIntegrationTest.java", "PrepareDepsOfTargetsUnderDirectoryFunctionTest.java", # b/179148968 ] + CROSS_PLATFORM_WINDOWS_TESTS @@ -378,3 +379,28 @@ java_test( "//third_party:truth", ], ) + +java_test( + name = "LocalDiffAwarenessIntegrationTest", + srcs = ["LocalDiffAwarenessIntegrationTest.java"], + # TODO(pcloudy): Even with --experimental_windows_watchfs, there's an extra + # getValues() on the second build in + # externalSymlink_doesNotTriggerFullGraphTraversal with Windows, and + # non-deterministic failure to detect changes (watchfs bug?). + tags = ["no_windows"], + deps = [ + "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories", + "//src/main/java/com/google/devtools/build/lib/skyframe:local_diff_awareness", + "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception", + "//src/main/java/com/google/devtools/build/lib/util:os", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//src/main/java/com/google/devtools/common/options", + "//src/test/java/com/google/devtools/build/lib/buildtool/util", + "//src/test/java/com/google/devtools/build/skyframe:testutil", + "//third_party:guava", + "//third_party:junit4", + "//third_party:truth", + ], +) diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/LocalDiffAwarenessIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/LocalDiffAwarenessIntegrationTest.java new file mode 100644 index 00000000000000..2129bc2cd05fa0 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/skyframe/LocalDiffAwarenessIntegrationTest.java @@ -0,0 +1,180 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.skyframe; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static org.junit.Assert.assertThrows; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.buildtool.util.SkyframeIntegrationTestBase; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; +import com.google.devtools.build.lib.runtime.WorkspaceBuilder; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.vfs.DelegateFileSystem; +import com.google.devtools.build.lib.vfs.FileStatus; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.NotifyingHelper; +import com.google.devtools.common.options.OptionsBase; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.After; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for local diff awareness. A good place for general tests of Bazel's interactions with + * "smart" filesystems, so that open-source changes don't break Google-internal features around + * smart filesystems. + */ +@RunWith(JUnit4.class) +public class LocalDiffAwarenessIntegrationTest extends SkyframeIntegrationTestBase { + private final Map throwOnNextStatIfFound = new HashMap<>(); + + @Override + protected BlazeRuntime.Builder getRuntimeBuilder() throws Exception { + return super.getRuntimeBuilder() + .addBlazeModule( + new BlazeModule() { + @Override + public void workspaceInit( + BlazeRuntime runtime, BlazeDirectories directories, WorkspaceBuilder builder) { + builder.addDiffAwarenessFactory(new LocalDiffAwareness.Factory(ImmutableList.of())); + } + + @Override + public Iterable> getCommandOptions(Command command) { + return ImmutableList.of(LocalDiffAwareness.Options.class); + } + }); + } + + @Override + public FileSystem createFileSystem() throws Exception { + return new DelegateFileSystem(super.createFileSystem()) { + @Override + protected FileStatus statIfFound(PathFragment path, boolean followSymlinks) + throws IOException { + IOException e = throwOnNextStatIfFound.remove(path); + if (e != null) { + throw e; + } + return super.statIfFound(path, followSymlinks); + } + }; + } + + @Before + public void addOptions() { + addOptions("--watchfs", "--experimental_windows_watchfs"); + } + + @After + public void checkExceptionsThrown() { + assertWithMessage("Injected exception(s) not thrown").that(throwOnNextStatIfFound).isEmpty(); + } + + @Test + public void changedFile_detectsChange() throws Exception { + // TODO(bazel-team): Understand why these tests are flaky on Mac. Probably real watchfs bug? + Assume.assumeFalse(OS.DARWIN.equals(OS.getCurrent())); + write("foo/BUILD", "genrule(name='foo', outs=['out'], cmd='echo hello > $@')"); + buildTarget("//foo"); + assertContents("hello", "//foo"); + write("foo/BUILD", "genrule(name='foo', outs=['out'], cmd='echo there > $@')"); + + buildTarget("//foo"); + + assertContents("there", "//foo"); + } + + @Test + public void changedFile_statFails_throwsError() throws Exception { + Assume.assumeFalse(OS.DARWIN.equals(OS.getCurrent())); + write("foo/BUILD", "genrule(name='foo', outs=['out'], cmd='echo hello > $@')"); + buildTarget("//foo"); + assertContents("hello", "//foo"); + Path buildFile = write("foo/BUILD", "genrule(name='foo', outs=['out'], cmd='echo there > $@')"); + IOException injectedException = new IOException("oh no!"); + throwOnNextStatIfFound.put(buildFile.asFragment(), injectedException); + + AbruptExitException e = assertThrows(AbruptExitException.class, () -> buildTarget("//foo")); + + assertThat(e.getCause()).hasCauseThat().hasCauseThat().isSameInstanceAs(injectedException); + } + + @Test + public void externalSymlink_doesNotTriggerFullGraphTraversal() throws Exception { + addOptions("--symlink_prefix=/"); + AtomicInteger calledGetValues = new AtomicInteger(0); + skyframeExecutor() + .getEvaluator() + .injectGraphTransformerForTesting( + NotifyingHelper.makeNotifyingTransformer( + (key, type, order, context) -> { + if (type == NotifyingHelper.EventType.GET_VALUES) { + calledGetValues.incrementAndGet(); + } + })); + write( + "hello/BUILD", + "genrule(name='target', srcs = ['external'], outs=['out'], cmd='/bin/cat $(SRCS) > $@')"); + String externalLink = System.getenv("TEST_TMPDIR") + "/target"; + write(externalLink, "one"); + createSymlink(externalLink, "hello/external"); + + // Trivial build: external symlink is not seen, so normal diff awareness is in play. + buildTarget("//hello:BUILD"); + // New package path on first build triggers full-graph work. + calledGetValues.set(0); + // getValues() called during output file checking (although if an output service is able to + // report modified files in practice there is no iteration). + // If external repositories are being used, getValues called because of that too. + // TODO(bazel-team): get rid of this when we can disable checks for external repositories. + int numGetValuesInFullDiffAwarenessBuild = + 1 + ("bazel".equals(this.getRuntime().getProductName()) ? 1 : 0); + + buildTarget("//hello:BUILD"); + assertThat(calledGetValues.getAndSet(0)).isEqualTo(numGetValuesInFullDiffAwarenessBuild); + + // Now bring the external symlink into Bazel's awareness. + buildTarget("//hello:target"); + assertContents("one", "//hello:target"); + assertThat(calledGetValues.getAndSet(0)).isEqualTo(numGetValuesInFullDiffAwarenessBuild); + + // Builds that follow a build containing an external file don't trigger a traversal. + buildTarget("//hello:target"); + assertContents("one", "//hello:target"); + assertThat(calledGetValues.getAndSet(0)).isEqualTo(numGetValuesInFullDiffAwarenessBuild); + + write(externalLink, "two"); + + buildTarget("//hello:target"); + // External file changes are tracked. + assertContents("two", "//hello:target"); + assertThat(calledGetValues.getAndSet(0)).isEqualTo(numGetValuesInFullDiffAwarenessBuild); + } +}