Add a jmh module for benchmarking (#487)

This is a first crack at creating a jmh module for benchmarking NullAway, so we can more easily detect performance regressions.  We have a single benchmark for now, Caffeine 3.0.2.  `./gradlew jmh` runs the benchmark.  This is an MVP: in the future, we should add more benchmarks, make the benchmark runner more versatile, etc.

For now, running benchmarks only works on JDK 11.  We can add support for JDK 8 in the future if we want (by adding EP javac to bootclasspaths, etc.).

.github/workflows/continuous-integration.yml

@@ -42,12 +42,12 @@ jobs:
           ORG_GRADLE_PROJECT_epApiVersion: ${{ matrix.epVersion }}
         uses: eskatos/gradle-command-action@v1
         with:
-          arguments: verGJF build
+          arguments: verGJF build -x :jmh:test
         if: matrix.java == '8'
       - name: Build and test using Gradle and Java 11
         uses: eskatos/gradle-command-action@v1
         with:
-          arguments: :nullaway:test
+          arguments: :nullaway:test :jmh:test
         if: matrix.java == '11'
       - name: Report jacoco coverage
         uses: eskatos/gradle-command-action@v1

gradle/dependencies.gradle

@@ -58,6 +58,7 @@ def build = [
     asmTree                 : "org.ow2.asm:asm-tree:${versions.asm}",
     errorProneCheckApi      : "com.google.errorprone:error_prone_check_api:${versions.errorProneApi}",
     errorProneCore          : "com.google.errorprone:error_prone_core:${versions.errorProne}",
+    errorProneCoreForApi    : "com.google.errorprone:error_prone_core:${versions.errorProneApi}",
     errorProneJavac         : "com.google.errorprone:javac:9+181-r4173-1",
     errorProneTestHelpers   : "com.google.errorprone:error_prone_test_helpers:${versions.errorProneApi}",
     checkerDataflow         : "org.checkerframework:dataflow-nullaway:${versions.checkerFramework}",

jmh/build.gradle

@@ -0,0 +1,60 @@
+plugins {
+    id 'java-library'
+    id "me.champeau.jmh" version "0.6.5"
+}
+
+configurations {
+    // create a configuration for the sources and dependencies of each benchmark
+    caffeineSources
+    caffeineDeps
+}
+dependencies {
+
+    // Add NullAway and Error Prone Core as dependencies.  This ensures that the classes get included
+    // in the jmh-generated jar, and hence get JIT-compiled during benchmarking.  Without this dependence, NullAway
+    // can still be loaded via the processor path, but it gets reloaded on each run of compilation, skewing
+    // performance measurements
+    implementation project(':nullaway')
+    // use the same version of Error Prone Core that we are compiling NullAway against, so we can
+    // benchmark against different versions of Error Prone
+    implementation deps.build.errorProneCoreForApi
+
+
+    // Source jars for our desired benchmarks
+    caffeineSources('com.github.ben-manes.caffeine:caffeine:3.0.2:sources') {
+        transitive = false
+    }
+
+    caffeineDeps 'com.github.ben-manes.caffeine:caffeine:3.0.2'
+
+    testImplementation deps.test.junit4
+}
+
+def caffeineSourceDir = project.layout.buildDirectory.dir('caffeineSources')
+
+task extractCaffeineSources(type: Copy) {
+    from zipTree(configurations.caffeineSources.singleFile)
+    into caffeineSourceDir
+}
+
+compileJava.dependsOn(extractCaffeineSources)
+
+jmh {
+    // seems we need more iterations to fully warm up the JIT
+    warmupIterations = 10
+
+    // a trick: to get the classpath for a benchmark, create a configuration that depends on the benchmark, and
+    // then filter out the benchmark itself
+    def caffeineClasspath = configurations.caffeineDeps.filter({f -> !f.toString().contains("caffeine-3.0.2")}).asPath
+    // pass source directories and classpaths for benchmarks as JVM arguments
+    // though jvmArgsAppend is a list, it just gets converted to a single string (probably a plugin bug)
+    jvmArgsAppend = [
+            "-Dnullaway.caffeine.sources=${caffeineSourceDir.get()} " +
+            "-Dnullaway.caffeine.classpath=$caffeineClasspath"
+    ]
+
+    // commented-out examples of how to tweak other jmh parameters; they show the default values
+    // for more examples see https://github.com/melix/jmh-gradle-plugin/blob/master/README.adoc#configuration-options
+    // iterations = 5
+    // fork = 5
+}

jmh/src/jmh/java/com/uber/nullaway/jmh/CaffeineBenchmark.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2021 Uber Technologies, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.uber.nullaway.jmh;
+
+import com.google.common.collect.ImmutableList;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.infra.Blackhole;
+
+@State(Scope.Benchmark)
+public class CaffeineBenchmark {
+
+  /**
+   * we use a subset of the source files since many are auto-generated and annotated with
+   * {@code @SuppressWarnings("NullAway")}
+   */
+  private static final ImmutableList<String> SOURCE_FILE_NAMES =
+      ImmutableList.of(
+          "com/github/benmanes/caffeine/cache/AbstractLinkedDeque.java",
+          "com/github/benmanes/caffeine/cache/AccessOrderDeque.java",
+          "com/github/benmanes/caffeine/cache/Async.java",
+          "com/github/benmanes/caffeine/cache/AsyncCache.java",
+          "com/github/benmanes/caffeine/cache/AsyncCacheLoader.java",
+          "com/github/benmanes/caffeine/cache/AsyncLoadingCache.java",
+          "com/github/benmanes/caffeine/cache/BoundedBuffer.java",
+          "com/github/benmanes/caffeine/cache/BoundedLocalCache.java",
+          "com/github/benmanes/caffeine/cache/Buffer.java",
+          "com/github/benmanes/caffeine/cache/Cache.java",
+          "com/github/benmanes/caffeine/cache/CacheLoader.java",
+          "com/github/benmanes/caffeine/cache/Caffeine.java",
+          "com/github/benmanes/caffeine/cache/CaffeineSpec.java",
+          "com/github/benmanes/caffeine/cache/Expiry.java",
+          "com/github/benmanes/caffeine/cache/FrequencySketch.java",
+          "com/github/benmanes/caffeine/cache/LinkedDeque.java",
+          "com/github/benmanes/caffeine/cache/LoadingCache.java",
+          "com/github/benmanes/caffeine/cache/LocalAsyncCache.java",
+          "com/github/benmanes/caffeine/cache/LocalAsyncLoadingCache.java",
+          "com/github/benmanes/caffeine/cache/LocalCache.java",
+          "com/github/benmanes/caffeine/cache/LocalCacheFactory.java",
+          "com/github/benmanes/caffeine/cache/LocalLoadingCache.java",
+          "com/github/benmanes/caffeine/cache/LocalManualCache.java",
+          "com/github/benmanes/caffeine/cache/MpscGrowableArrayQueue.java",
+          "com/github/benmanes/caffeine/cache/Node.java",
+          "com/github/benmanes/caffeine/cache/NodeFactory.java",
+          "com/github/benmanes/caffeine/cache/Pacer.java",
+          "com/github/benmanes/caffeine/cache/Policy.java",
+          "com/github/benmanes/caffeine/cache/References.java",
+          "com/github/benmanes/caffeine/cache/RemovalCause.java",
+          "com/github/benmanes/caffeine/cache/RemovalListener.java",
+          "com/github/benmanes/caffeine/cache/Scheduler.java",
+          "com/github/benmanes/caffeine/cache/SerializationProxy.java",
+          "com/github/benmanes/caffeine/cache/StripedBuffer.java",
+          "com/github/benmanes/caffeine/cache/Ticker.java",
+          "com/github/benmanes/caffeine/cache/TimerWheel.java",
+          "com/github/benmanes/caffeine/cache/UnboundedLocalCache.java",
+          "com/github/benmanes/caffeine/cache/Weigher.java",
+          "com/github/benmanes/caffeine/cache/WriteOrderDeque.java",
+          "com/github/benmanes/caffeine/cache/WriteThroughEntry.java",
+          "com/github/benmanes/caffeine/cache/stats/CacheStats.java",
+          "com/github/benmanes/caffeine/cache/stats/ConcurrentStatsCounter.java",
+          "com/github/benmanes/caffeine/cache/stats/DisabledStatsCounter.java",
+          "com/github/benmanes/caffeine/cache/stats/GuardedStatsCounter.java",
+          "com/github/benmanes/caffeine/cache/stats/StatsCounter.java");
+
+  private NullawayJavac nullawayJavac;
+
+  @Setup
+  public void setup() throws IOException {
+    String caffeineSourceDir = System.getProperty("nullaway.caffeine.sources");
+    String caffeineClasspath = System.getProperty("nullaway.caffeine.classpath");
+    List<String> realSourceFileNames =
+        SOURCE_FILE_NAMES
+            .stream()
+            .map(s -> caffeineSourceDir + File.separator + s.replaceAll("/", File.separator))
+            .collect(Collectors.toList());
+    nullawayJavac =
+        NullawayJavac.create(
+            realSourceFileNames, "com.github.benmanes.caffeine", caffeineClasspath);
+  }
+
+  @Benchmark
+  public void compile(Blackhole bh) throws Exception {
+    bh.consume(nullawayJavac.compile());
+  }
+}

jmh/src/main/java/com/uber/nullaway/jmh/NullawayJavac.java

@@ -0,0 +1,188 @@
+/*
+ * Copyright (c) 2021 Uber Technologies, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package com.uber.nullaway.jmh;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import javax.annotation.Nullable;
+import javax.tools.DiagnosticListener;
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileObject;
+import javax.tools.SimpleJavaFileObject;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.ToolProvider;
+
+/**
+ * Code to run Javac with NullAway enabled, designed to aid benchmarking. Construction of {@code
+ * NullawayJavac} objects performs one-time operations whose cost we do not care to benchmark, so
+ * that {@link #compile()} can be run repeatedly to measure performance in the steady state.
+ */
+public class NullawayJavac {
+
+  //////////////////////
+  // state required to run javac via the standard APIs
+  //////////////////////
+  private List<JavaFileObject> compilationUnits;
+  private JavaCompiler compiler;
+  @Nullable private DiagnosticListener<JavaFileObject> diagnosticListener;
+  private StandardJavaFileManager fileManager;
+  private List<String> options;
+
+  /**
+   * Sets up compilation for a simple single source file, for testing / sanity checking purposes.
+   * Running {@link #compile()} on the resulting object will return {@code false}, as the sample
+   * input source has NullAway errors.
+   *
+   * @throws IOException if output temporary directory cannot be created
+   */
+  public static NullawayJavac createSimpleTest() throws IOException {
+    String testClass =
+        "package com.uber;\n"
+            + "import java.util.*;\n"
+            + "class Test {   \n"
+            + "  public static void main(String args[]) {\n"
+            + "    Set<Short> s = null;\n"
+            + "    for (short i = 0; i < 100; i++) {\n"
+            + "      s.add(i);\n"
+            + "      s.remove(i - 1);\n"
+            + "    }\n"
+            + "    System.out.println(s.size());"
+            + "  }\n"
+            + "}\n";
+    return new NullawayJavac(
+        Collections.singletonList(new JavaSourceFromString("Test", testClass)), "com.uber", null);
+  }
+
+  /**
+   * Creates a NullawayJavac object to compile a set of source files.
+   *
+   * @param sourceFileNames absolute paths to the source files to be compiled
+   * @param annotatedPackages argument to pass for "-XepOpt:NullAway:AnnotatedPackages" option
+   * @param classpath classpath for the benchmark
+   * @throws IOException if a temporary output directory cannot be created
+   */
+  public static NullawayJavac create(
+      List<String> sourceFileNames, String annotatedPackages, String classpath) throws IOException {
+    List<JavaFileObject> compilationUnits = new ArrayList<>();
+    for (String sourceFileName : sourceFileNames) {
+      // we read every source file into memory in the prepare phase, to avoid some I/O during
+      // compilations
+      String content = readFile(sourceFileName);
+      String classname =
+          sourceFileName.substring(
+              sourceFileName.lastIndexOf(File.separatorChar) + 1, sourceFileName.indexOf(".java"));
+      compilationUnits.add(new JavaSourceFromString(classname, content));
+    }
+
+    return new NullawayJavac(compilationUnits, annotatedPackages, classpath);
+  }
+
+  /**
+   * Configures compilation with javac and NullAway.
+   *
+   * <p>To pass NullAway in the {@code -processorpath} argument to the spawned javac and ensure it
+   * gets JIT-compiled during benchmarking, we make this project depend on NullAway and Error Prone
+   * Core, and then pass our own classpath as the processorpath. Note that this makes (dependencies
+   * of) NullAway and Error Prone visible on the <emph>classpath</emph> for the spawned javac
+   * instance as well. Note that this could lead to problems for benchmarks that depend on a
+   * conflicting version of a library that NullAway depends on.
+   *
+   * @param compilationUnits input sources to be compiled
+   * @param annotatedPackages argument to pass for "-XepOpt:NullAway:AnnotatedPackages" option
+   * @param classpath classpath for the program to be compiled
+   * @throws IOException if a temporary output directory cannot be created
+   */
+  private NullawayJavac(
+      List<JavaFileObject> compilationUnits, String annotatedPackages, @Nullable String classpath)
+      throws IOException {
+    this.compilationUnits = compilationUnits;
+    this.compiler = ToolProvider.getSystemJavaCompiler();
+    this.diagnosticListener =
+        diagnostic -> {
+          // do nothing
+        };
+    // uncomment this if you want to see compile errors get printed out
+    // this.diagnosticListener = null;
+    this.fileManager = compiler.getStandardFileManager(diagnosticListener, null, null);
+    Path outputDir = Files.createTempDirectory("classes");
+    outputDir.toFile().deleteOnExit();
+    this.options = new ArrayList<>();
+    if (classpath != null) {
+      options.addAll(Arrays.asList("-classpath", classpath));
+    }
+    options.addAll(
+        Arrays.asList(
+            "-processorpath",
+            System.getProperty("java.class.path"),
+            "-d",
+            outputDir.toAbsolutePath().toString(),
+            "-XDcompilePolicy=simple",
+            "-Xplugin:ErrorProne -XepDisableAllChecks -Xep:NullAway:ERROR -XepOpt:NullAway:AnnotatedPackages="
+                + annotatedPackages));
+  }
+
+  /**
+   * Runs the compilation.
+   *
+   * @return true if the input files compile without error; false otherwise
+   */
+  public boolean compile() {
+    JavaCompiler.CompilationTask task =
+        compiler.getTask(null, fileManager, diagnosticListener, options, null, compilationUnits);
+    return task.call();
+  }
+
+  private static String readFile(String path) throws IOException {
+    byte[] encoded = Files.readAllBytes(Paths.get(path));
+    return new String(encoded, StandardCharsets.UTF_8);
+  }
+
+  /**
+   * This class allows code to be generated directly from a String, instead of having to be on disk.
+   *
+   * <p>Based on code in Apache Pig; see <a
+   * href="https://github.com/apache/pig/blob/59ec4a326079c9f937a052194405415b1e3a2b06/src/org/apache/pig/impl/util/JavaCompilerHelper.java#L42-L58">here</a>.
+   */
+  private static class JavaSourceFromString extends SimpleJavaFileObject {
+    final String code;
+
+    JavaSourceFromString(String name, String code) {
+      super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension), Kind.SOURCE);
+      this.code = code;
+    }
+
+    @Override
+    public CharSequence getCharContent(boolean ignoreEncodingErrors) {
+      return code;
+    }
+  }
+}