diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index f24f4d8e01..6465378a37 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -42,12 +42,12 @@ jobs: ORG_GRADLE_PROJECT_epApiVersion: ${{ matrix.epVersion }} uses: eskatos/gradle-command-action@v1 with: - arguments: verGJF build + arguments: verGJF build -x :jmh:test if: matrix.java == '8' - name: Build and test using Gradle and Java 11 uses: eskatos/gradle-command-action@v1 with: - arguments: :nullaway:test + arguments: :nullaway:test :jmh:test if: matrix.java == '11' - name: Report jacoco coverage uses: eskatos/gradle-command-action@v1 diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 3764e5a474..b311a64d78 100755 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -58,6 +58,7 @@ def build = [ asmTree : "org.ow2.asm:asm-tree:${versions.asm}", errorProneCheckApi : "com.google.errorprone:error_prone_check_api:${versions.errorProneApi}", errorProneCore : "com.google.errorprone:error_prone_core:${versions.errorProne}", + errorProneCoreForApi : "com.google.errorprone:error_prone_core:${versions.errorProneApi}", errorProneJavac : "com.google.errorprone:javac:9+181-r4173-1", errorProneTestHelpers : "com.google.errorprone:error_prone_test_helpers:${versions.errorProneApi}", checkerDataflow : "org.checkerframework:dataflow-nullaway:${versions.checkerFramework}", diff --git a/jmh/build.gradle b/jmh/build.gradle new file mode 100644 index 0000000000..c034b46fae --- /dev/null +++ b/jmh/build.gradle @@ -0,0 +1,60 @@ +plugins { + id 'java-library' + id "me.champeau.jmh" version "0.6.5" +} + +configurations { + // create a configuration for the sources and dependencies of each benchmark + caffeineSources + caffeineDeps +} +dependencies { + + // Add NullAway and Error Prone Core as dependencies. This ensures that the classes get included + // in the jmh-generated jar, and hence get JIT-compiled during benchmarking. Without this dependence, NullAway + // can still be loaded via the processor path, but it gets reloaded on each run of compilation, skewing + // performance measurements + implementation project(':nullaway') + // use the same version of Error Prone Core that we are compiling NullAway against, so we can + // benchmark against different versions of Error Prone + implementation deps.build.errorProneCoreForApi + + + // Source jars for our desired benchmarks + caffeineSources('com.github.ben-manes.caffeine:caffeine:3.0.2:sources') { + transitive = false + } + + caffeineDeps 'com.github.ben-manes.caffeine:caffeine:3.0.2' + + testImplementation deps.test.junit4 +} + +def caffeineSourceDir = project.layout.buildDirectory.dir('caffeineSources') + +task extractCaffeineSources(type: Copy) { + from zipTree(configurations.caffeineSources.singleFile) + into caffeineSourceDir +} + +compileJava.dependsOn(extractCaffeineSources) + +jmh { + // seems we need more iterations to fully warm up the JIT + warmupIterations = 10 + + // a trick: to get the classpath for a benchmark, create a configuration that depends on the benchmark, and + // then filter out the benchmark itself + def caffeineClasspath = configurations.caffeineDeps.filter({f -> !f.toString().contains("caffeine-3.0.2")}).asPath + // pass source directories and classpaths for benchmarks as JVM arguments + // though jvmArgsAppend is a list, it just gets converted to a single string (probably a plugin bug) + jvmArgsAppend = [ + "-Dnullaway.caffeine.sources=${caffeineSourceDir.get()} " + + "-Dnullaway.caffeine.classpath=$caffeineClasspath" + ] + + // commented-out examples of how to tweak other jmh parameters; they show the default values + // for more examples see https://github.com/melix/jmh-gradle-plugin/blob/master/README.adoc#configuration-options + // iterations = 5 + // fork = 5 +} diff --git a/jmh/src/jmh/java/com/uber/nullaway/jmh/CaffeineBenchmark.java b/jmh/src/jmh/java/com/uber/nullaway/jmh/CaffeineBenchmark.java new file mode 100644 index 0000000000..ffcc6abf71 --- /dev/null +++ b/jmh/src/jmh/java/com/uber/nullaway/jmh/CaffeineBenchmark.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2021 Uber Technologies, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.uber.nullaway.jmh; + +import com.google.common.collect.ImmutableList; +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.infra.Blackhole; + +@State(Scope.Benchmark) +public class CaffeineBenchmark { + + /** + * we use a subset of the source files since many are auto-generated and annotated with + * {@code @SuppressWarnings("NullAway")} + */ + private static final ImmutableList SOURCE_FILE_NAMES = + ImmutableList.of( + "com/github/benmanes/caffeine/cache/AbstractLinkedDeque.java", + "com/github/benmanes/caffeine/cache/AccessOrderDeque.java", + "com/github/benmanes/caffeine/cache/Async.java", + "com/github/benmanes/caffeine/cache/AsyncCache.java", + "com/github/benmanes/caffeine/cache/AsyncCacheLoader.java", + "com/github/benmanes/caffeine/cache/AsyncLoadingCache.java", + "com/github/benmanes/caffeine/cache/BoundedBuffer.java", + "com/github/benmanes/caffeine/cache/BoundedLocalCache.java", + "com/github/benmanes/caffeine/cache/Buffer.java", + "com/github/benmanes/caffeine/cache/Cache.java", + "com/github/benmanes/caffeine/cache/CacheLoader.java", + "com/github/benmanes/caffeine/cache/Caffeine.java", + "com/github/benmanes/caffeine/cache/CaffeineSpec.java", + "com/github/benmanes/caffeine/cache/Expiry.java", + "com/github/benmanes/caffeine/cache/FrequencySketch.java", + "com/github/benmanes/caffeine/cache/LinkedDeque.java", + "com/github/benmanes/caffeine/cache/LoadingCache.java", + "com/github/benmanes/caffeine/cache/LocalAsyncCache.java", + "com/github/benmanes/caffeine/cache/LocalAsyncLoadingCache.java", + "com/github/benmanes/caffeine/cache/LocalCache.java", + "com/github/benmanes/caffeine/cache/LocalCacheFactory.java", + "com/github/benmanes/caffeine/cache/LocalLoadingCache.java", + "com/github/benmanes/caffeine/cache/LocalManualCache.java", + "com/github/benmanes/caffeine/cache/MpscGrowableArrayQueue.java", + "com/github/benmanes/caffeine/cache/Node.java", + "com/github/benmanes/caffeine/cache/NodeFactory.java", + "com/github/benmanes/caffeine/cache/Pacer.java", + "com/github/benmanes/caffeine/cache/Policy.java", + "com/github/benmanes/caffeine/cache/References.java", + "com/github/benmanes/caffeine/cache/RemovalCause.java", + "com/github/benmanes/caffeine/cache/RemovalListener.java", + "com/github/benmanes/caffeine/cache/Scheduler.java", + "com/github/benmanes/caffeine/cache/SerializationProxy.java", + "com/github/benmanes/caffeine/cache/StripedBuffer.java", + "com/github/benmanes/caffeine/cache/Ticker.java", + "com/github/benmanes/caffeine/cache/TimerWheel.java", + "com/github/benmanes/caffeine/cache/UnboundedLocalCache.java", + "com/github/benmanes/caffeine/cache/Weigher.java", + "com/github/benmanes/caffeine/cache/WriteOrderDeque.java", + "com/github/benmanes/caffeine/cache/WriteThroughEntry.java", + "com/github/benmanes/caffeine/cache/stats/CacheStats.java", + "com/github/benmanes/caffeine/cache/stats/ConcurrentStatsCounter.java", + "com/github/benmanes/caffeine/cache/stats/DisabledStatsCounter.java", + "com/github/benmanes/caffeine/cache/stats/GuardedStatsCounter.java", + "com/github/benmanes/caffeine/cache/stats/StatsCounter.java"); + + private NullawayJavac nullawayJavac; + + @Setup + public void setup() throws IOException { + String caffeineSourceDir = System.getProperty("nullaway.caffeine.sources"); + String caffeineClasspath = System.getProperty("nullaway.caffeine.classpath"); + List realSourceFileNames = + SOURCE_FILE_NAMES + .stream() + .map(s -> caffeineSourceDir + File.separator + s.replaceAll("/", File.separator)) + .collect(Collectors.toList()); + nullawayJavac = + NullawayJavac.create( + realSourceFileNames, "com.github.benmanes.caffeine", caffeineClasspath); + } + + @Benchmark + public void compile(Blackhole bh) throws Exception { + bh.consume(nullawayJavac.compile()); + } +} diff --git a/jmh/src/main/java/com/uber/nullaway/jmh/NullawayJavac.java b/jmh/src/main/java/com/uber/nullaway/jmh/NullawayJavac.java new file mode 100644 index 0000000000..e7da10a1cc --- /dev/null +++ b/jmh/src/main/java/com/uber/nullaway/jmh/NullawayJavac.java @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2021 Uber Technologies, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.uber.nullaway.jmh; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import javax.annotation.Nullable; +import javax.tools.DiagnosticListener; +import javax.tools.JavaCompiler; +import javax.tools.JavaFileObject; +import javax.tools.SimpleJavaFileObject; +import javax.tools.StandardJavaFileManager; +import javax.tools.ToolProvider; + +/** + * Code to run Javac with NullAway enabled, designed to aid benchmarking. Construction of {@code + * NullawayJavac} objects performs one-time operations whose cost we do not care to benchmark, so + * that {@link #compile()} can be run repeatedly to measure performance in the steady state. + */ +public class NullawayJavac { + + ////////////////////// + // state required to run javac via the standard APIs + ////////////////////// + private List compilationUnits; + private JavaCompiler compiler; + @Nullable private DiagnosticListener diagnosticListener; + private StandardJavaFileManager fileManager; + private List options; + + /** + * Sets up compilation for a simple single source file, for testing / sanity checking purposes. + * Running {@link #compile()} on the resulting object will return {@code false}, as the sample + * input source has NullAway errors. + * + * @throws IOException if output temporary directory cannot be created + */ + public static NullawayJavac createSimpleTest() throws IOException { + String testClass = + "package com.uber;\n" + + "import java.util.*;\n" + + "class Test { \n" + + " public static void main(String args[]) {\n" + + " Set s = null;\n" + + " for (short i = 0; i < 100; i++) {\n" + + " s.add(i);\n" + + " s.remove(i - 1);\n" + + " }\n" + + " System.out.println(s.size());" + + " }\n" + + "}\n"; + return new NullawayJavac( + Collections.singletonList(new JavaSourceFromString("Test", testClass)), "com.uber", null); + } + + /** + * Creates a NullawayJavac object to compile a set of source files. + * + * @param sourceFileNames absolute paths to the source files to be compiled + * @param annotatedPackages argument to pass for "-XepOpt:NullAway:AnnotatedPackages" option + * @param classpath classpath for the benchmark + * @throws IOException if a temporary output directory cannot be created + */ + public static NullawayJavac create( + List sourceFileNames, String annotatedPackages, String classpath) throws IOException { + List compilationUnits = new ArrayList<>(); + for (String sourceFileName : sourceFileNames) { + // we read every source file into memory in the prepare phase, to avoid some I/O during + // compilations + String content = readFile(sourceFileName); + String classname = + sourceFileName.substring( + sourceFileName.lastIndexOf(File.separatorChar) + 1, sourceFileName.indexOf(".java")); + compilationUnits.add(new JavaSourceFromString(classname, content)); + } + + return new NullawayJavac(compilationUnits, annotatedPackages, classpath); + } + + /** + * Configures compilation with javac and NullAway. + * + *

To pass NullAway in the {@code -processorpath} argument to the spawned javac and ensure it + * gets JIT-compiled during benchmarking, we make this project depend on NullAway and Error Prone + * Core, and then pass our own classpath as the processorpath. Note that this makes (dependencies + * of) NullAway and Error Prone visible on the classpath for the spawned javac + * instance as well. Note that this could lead to problems for benchmarks that depend on a + * conflicting version of a library that NullAway depends on. + * + * @param compilationUnits input sources to be compiled + * @param annotatedPackages argument to pass for "-XepOpt:NullAway:AnnotatedPackages" option + * @param classpath classpath for the program to be compiled + * @throws IOException if a temporary output directory cannot be created + */ + private NullawayJavac( + List compilationUnits, String annotatedPackages, @Nullable String classpath) + throws IOException { + this.compilationUnits = compilationUnits; + this.compiler = ToolProvider.getSystemJavaCompiler(); + this.diagnosticListener = + diagnostic -> { + // do nothing + }; + // uncomment this if you want to see compile errors get printed out + // this.diagnosticListener = null; + this.fileManager = compiler.getStandardFileManager(diagnosticListener, null, null); + Path outputDir = Files.createTempDirectory("classes"); + outputDir.toFile().deleteOnExit(); + this.options = new ArrayList<>(); + if (classpath != null) { + options.addAll(Arrays.asList("-classpath", classpath)); + } + options.addAll( + Arrays.asList( + "-processorpath", + System.getProperty("java.class.path"), + "-d", + outputDir.toAbsolutePath().toString(), + "-XDcompilePolicy=simple", + "-Xplugin:ErrorProne -XepDisableAllChecks -Xep:NullAway:ERROR -XepOpt:NullAway:AnnotatedPackages=" + + annotatedPackages)); + } + + /** + * Runs the compilation. + * + * @return true if the input files compile without error; false otherwise + */ + public boolean compile() { + JavaCompiler.CompilationTask task = + compiler.getTask(null, fileManager, diagnosticListener, options, null, compilationUnits); + return task.call(); + } + + private static String readFile(String path) throws IOException { + byte[] encoded = Files.readAllBytes(Paths.get(path)); + return new String(encoded, StandardCharsets.UTF_8); + } + + /** + * This class allows code to be generated directly from a String, instead of having to be on disk. + * + *

Based on code in Apache Pig; see here. + */ + private static class JavaSourceFromString extends SimpleJavaFileObject { + final String code; + + JavaSourceFromString(String name, String code) { + super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension), Kind.SOURCE); + this.code = code; + } + + @Override + public CharSequence getCharContent(boolean ignoreEncodingErrors) { + return code; + } + } +} diff --git a/jmh/src/test/java/com/uber/nullaway/jmh/NullawayJavacTest.java b/jmh/src/test/java/com/uber/nullaway/jmh/NullawayJavacTest.java new file mode 100644 index 0000000000..88221fa945 --- /dev/null +++ b/jmh/src/test/java/com/uber/nullaway/jmh/NullawayJavacTest.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 Uber Technologies, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package com.uber.nullaway.jmh; + +import java.io.IOException; +import org.junit.Test; + +public class NullawayJavacTest { + + @Test + public void simpleTest() throws IOException { + NullawayJavac n = NullawayJavac.createSimpleTest(); + // run compile in a loop to make sure that works + for (int i = 0; i < 5; i++) { + // we do not expect the simple test to compile + org.junit.Assert.assertTrue(!n.compile()); + } + } +} diff --git a/settings.gradle b/settings.gradle index 19ecec180f..b995b6bf2c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -29,3 +29,4 @@ include ':jar-infer:jar-infer-cli' include ':jar-infer:test-java-lib-jarinfer' include ':jar-infer:nullaway-integration-test' include ':jar-infer:test-android-lib-jarinfer' +include ':jmh'