Merge branch 'microsoft:main' into main

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '32 17 * * 5'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'java' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.gitignore

@@ -5,6 +5,7 @@ gclogs/streaming
 gclogs/unified
 gclogs/zgc
 
+unifiedrules.txt
 gctoolkit-testdata-*.zip
 
 # Maven

IT/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.microsoft.gctoolkit</groupId>
         <artifactId>gctoolkit</artifactId>
-        <version>2.0.6-SNAPSHOT</version>
+        <version>2.0.8-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -44,7 +44,7 @@
         <dependency>
             <groupId>com.microsoft.gctoolkit</groupId>
             <artifactId>gctoolkit-sample</artifactId>
-            <version>2.0.6-SNAPSHOT</version>
+            <version>2.0.8-SNAPSHOT</version>
             <scope>test</scope>
         </dependency>
     </dependencies>

IT/src/test/java/module-info.java → IT/src/main/java/module-info.java

@@ -10,9 +10,6 @@
     requires com.microsoft.gctoolkit.parser;
     requires com.microsoft.gctoolkit.vertx;
     requires java.logging;
-    requires org.junit.jupiter.api;
-
-    exports com.microsoft.gctoolkit.integration;
 
     exports com.microsoft.gctoolkit.integration.aggregation to
             com.microsoft.gctoolkit.vertx;

IT/src/test/java/com/microsoft/gctoolkit/integration/EndToEndIntegrationTest.java

@@ -10,9 +10,12 @@
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
+import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Optional;
 
+import static org.junit.jupiter.api.Assertions.fail;
+
 public class EndToEndIntegrationTest {
 
     @Test
@@ -43,7 +46,12 @@ public void analyze(String gcLogFile) {
          * The JavaVirtualMachine contains the aggregations as filled out by the Aggregators.
          * It also contains configuration information about how the JVM was configured for the runtime.
          */
-        JavaVirtualMachine machine = gcToolKit.analyze(logFile);
+        JavaVirtualMachine machine = null;
+        try {
+            machine = gcToolKit.analyze(logFile);
+        } catch (IOException e) {
+            fail(e.getMessage());
+        }
 
         // Retrieves the Aggregation for HeapOccupancyAfterCollectionSummary. This is a time-series aggregation.
         String message = "The XYDataSet for %s contains %s items.\n";

IT/src/test/java/com/microsoft/gctoolkit/integration/aggregation/RuntimeAggregationTest.java → IT/src/test/java/com/microsoft/gctoolkit/integration/RuntimeAggregationTest.java

@@ -1,5 +1,6 @@
-package com.microsoft.gctoolkit.integration.aggregation;
+package com.microsoft.gctoolkit.integration;
 
+import com.microsoft.gctoolkit.integration.aggregation.RuntimeAggregation;
 import com.microsoft.gctoolkit.time.DateTimeStamp;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;

IT/src/test/java/com/microsoft/gctoolkit/integration/collections/XYDataSetTest.java → IT/src/test/java/com/microsoft/gctoolkit/integration/XYDataSetTest.java

@@ -1,7 +1,8 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
-package com.microsoft.gctoolkit.integration.collections;
+package com.microsoft.gctoolkit.integration;
 
+import com.microsoft.gctoolkit.integration.collections.XYDataSet;
 import com.microsoft.gctoolkit.integration.collections.XYDataSet.Point;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;

IT/src/test/java/com/microsoft/gctoolkit/integration/core/JavaVirtualMachineConfigurationTest.java

@@ -0,0 +1,47 @@
+package com.microsoft.gctoolkit.integration.core;
+
+import com.microsoft.gctoolkit.GCToolKit;
+import com.microsoft.gctoolkit.integration.io.TestLogFile;
+import com.microsoft.gctoolkit.io.GCLogFile;
+import com.microsoft.gctoolkit.io.RotatingGCLogFile;
+import com.microsoft.gctoolkit.io.SingleGCLogFile;
+import com.microsoft.gctoolkit.jvm.JavaVirtualMachine;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+
+import static org.junit.jupiter.api.Assertions.fail;
+
+public class JavaVirtualMachineConfigurationTest {
+
+    private String logFile = "rolling/jdk14/rollinglogs/long_restart.log";
+    private int[][] times = { { 0, 13, 262172, 262172}, { 259077, 259077, 262172, 3095}};
+
+    @Test
+    public void testRotating() {
+        TestLogFile log = new TestLogFile(logFile);
+        test(new RotatingGCLogFile(log.getFile().toPath()), times[0]);
+    }
+
+    @Test
+    public void testSingle() {
+        TestLogFile log = new TestLogFile(logFile);
+        test(new SingleGCLogFile(log.getFile().toPath()), times[1]);
+    }
+
+    private void test(GCLogFile log, int[] endStartTimes ) {
+        GCToolKit gcToolKit = new GCToolKit();
+        gcToolKit.loadAggregationsFromServiceLoader();
+        JavaVirtualMachine machine = null;
+        try {
+            machine = gcToolKit.analyze(log);
+        } catch (IOException e) {
+            fail(e.getMessage());
+        }
+        Assertions.assertEquals( endStartTimes[0], (int)(machine.getEstimatedJVMStartTime().getTimeStamp() * 1000.0d));
+        Assertions.assertEquals( endStartTimes[1], (int)(machine.getTimeOfFirstEvent().getTimeStamp() * 1000.0d));
+        Assertions.assertEquals( endStartTimes[2], (int)(machine.getJVMTerminationTime().getTimeStamp() * 1000.0d));
+        Assertions.assertEquals( endStartTimes[3], (int)(machine.getRuntimeDuration() * 1000.0d));
+    }
+}

api/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>com.microsoft.gctoolkit</groupId>
         <artifactId>gctoolkit</artifactId>
-        <version>2.0.6-SNAPSHOT</version>
+        <version>2.0.8-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

api/src/main/java/com/microsoft/gctoolkit/GCToolKit.java

@@ -7,8 +7,13 @@
 import com.microsoft.gctoolkit.io.GCLogFile;
 import com.microsoft.gctoolkit.io.RotatingGCLogFile;
 import com.microsoft.gctoolkit.io.SingleGCLogFile;
+import com.microsoft.gctoolkit.jvm.Diary;
 import com.microsoft.gctoolkit.jvm.JavaVirtualMachine;
+import com.microsoft.gctoolkit.message.DataSourceBus;
+import com.microsoft.gctoolkit.message.JVMEventBus;
+import com.microsoft.gctoolkit.message.DataSourceParser;
 
+import java.io.IOException;
 import java.lang.reflect.Method;
 import java.util.HashSet;
 import java.util.ServiceConfigurationError;
@@ -30,13 +35,40 @@ public class GCToolKit {
      * @param logFile GCLogFile DataSource
      * @return JavaVirtualMachine implementation.
      */
-    private static JavaVirtualMachine loadJavaVirtualMachine(GCLogFile logFile) {
+    private JavaVirtualMachine loadJavaVirtualMachine(GCLogFile logFile) {
         return ServiceLoader.load(JavaVirtualMachine.class)
                 .stream()
                 .map(ServiceLoader.Provider::get)
                 .filter(jvm -> jvm.accepts(logFile))
                 .findFirst()
-                .orElseThrow(() -> new ServiceConfigurationError("No suitable service provider found"));
+                .orElseThrow(() -> new ServiceConfigurationError("No suitable JavaVirtualMachine implementation found"));
+    }
+
+    private DataSourceBus setupDataSourceBus() {
+        return ServiceLoader.load(DataSourceBus.class)
+                .stream()
+                .map(ServiceLoader.Provider::get)
+                .findFirst()
+                .orElseThrow(() -> new ServiceConfigurationError("No suitable DataSourceBus implementation found"));
+
+    }
+
+    private JVMEventBus setupJVMEventBus() {
+        return ServiceLoader.load(JVMEventBus.class)
+                .stream()
+                .map(ServiceLoader.Provider::get)
+                .findFirst()
+                .orElseThrow(() -> new ServiceConfigurationError("No suitable JVMEventBus implementation found"));
+
+    }
+
+    private void registerParsers(DataSourceBus bus, Diary diary) {
+        ServiceLoader.load(DataSourceParser.class)
+                .stream()
+                .map(ServiceLoader.Provider::get)
+                .filter(p->p.accepts(diary))
+                .forEach(parser->bus.register(parser));
+
     }
 
     private final Set<Class<? extends Aggregation>> registeredAggregations;
@@ -107,14 +139,16 @@ public void registerAggregation(Class<? extends Aggregation> aggregationClass) {
      * @return a representation of the state of the Java Virtual Machine resulting
      * from the analysis of the GC log file.
      */
-    public JavaVirtualMachine analyze(GCLogFile logFile) {
+    public JavaVirtualMachine analyze(GCLogFile logFile) throws IOException  {
         //todo: revert this to DataSource to account for non-GC log data sources once we have a use case (maybe JFR but JFR timers drift badly ATM)
-        // Potential NPE, but would have logged if there was trouble creating the instance.
+        //setup message bus
+        //DataSourceBus dataSourceBus = setupDataSourceBus();
+        //JVMEventBus eventBus = setupJVMEventBus();
+        //registerParsers(dataSourceBus,logFile.diary());
         JavaVirtualMachine javaVirtualMachine = loadJavaVirtualMachine(logFile);
         //todo: do we need reflection????
         try {
-            Method analyze = javaVirtualMachine.getClass()
-                    .getMethod("analyze", Set.class, DataSource.class);
+            Method analyze = javaVirtualMachine.getClass().getMethod("analyze", Set.class, DataSource.class);
             analyze.invoke(javaVirtualMachine, this.registeredAggregations, logFile);
         } catch (ReflectiveOperationException e) {
             LOGGER.log(Level.SEVERE, "Cannot invoke analyze method", e);

api/src/main/java/com/microsoft/gctoolkit/event/MalformedEvent.java

@@ -0,0 +1,7 @@
+package com.microsoft.gctoolkit.event;
+
+public class MalformedEvent extends Exception {
+    public MalformedEvent(String message) {
+        super(message);
+    }
+}

api/src/main/java/com/microsoft/gctoolkit/event/g1gc/G1GCPauseEvent.java

@@ -64,15 +64,12 @@ public MemoryPoolSummary getTenured() {
         if ((getEden() == null) || (getHeap() == null)) {
             return NULL_POOL;
         } else if (getSurvivor() == null) {
-            return new MemoryPoolSummary(getHeap().getOccupancyBeforeCollection() - this.getEden().getOccupancyBeforeCollection(),
-                    getHeap().getSizeBeforeCollection() - getEden().getSizeBeforeCollection(),
-                    getHeap().getOccupancyAfterCollection() - getEden().getOccupancyAfterCollection(),
-                    getHeap().getSizeAfterCollection() - getEden().getSizeAfterCollection());
+            return getHeap().minus(getEden());
         } else {
-            return new MemoryPoolSummary(getHeap().getOccupancyBeforeCollection() - this.getEden().getOccupancyBeforeCollection(),
+            return new MemoryPoolSummary(getHeap().getOccupancyBeforeCollection() - this.getEden().getOccupancyBeforeCollection() - getSurvivor().getOccupancyBeforeCollection(),
                     getHeap().getSizeBeforeCollection() - getEden().getSizeBeforeCollection() - getSurvivor().getOccupancyBeforeCollection(),
                     getHeap().getOccupancyAfterCollection() - getEden().getOccupancyAfterCollection() - getSurvivor().getOccupancyAfterCollection(),
-                    getHeap().getSizeAfterCollection() - getEden().getSizeAfterCollection());
+                    getHeap().getSizeAfterCollection() - getEden().getSizeAfterCollection() - getSurvivor().getOccupancyAfterCollection());
         }
     }
 

api/src/main/java/com/microsoft/gctoolkit/jvm/Diary.java

@@ -3,6 +3,7 @@
 package com.microsoft.gctoolkit.jvm;
 
 import com.microsoft.gctoolkit.parser.datatype.TripleState;
+import com.microsoft.gctoolkit.time.DateTimeStamp;
 
 import java.util.Arrays;
 
@@ -45,6 +46,7 @@
 public class Diary {
 
     private final TripleState[] states;
+    private DateTimeStamp timeOfFirstEvent;
 
     public Diary() {
         states = new TripleState[SupportedFlags.values().length];
@@ -366,4 +368,12 @@ public boolean isMaxTenuringThresholdViolationKnown() {
     public boolean isJVMEventsKnown() {
         return isApplicationStoppedTimeKnown() && isApplicationRunningTime();
     }
+
+    public void setTimeOfFirstEvent(DateTimeStamp startTime) {
+        if ( this.timeOfFirstEvent == null)
+            this.timeOfFirstEvent = startTime;
+    }
+    public DateTimeStamp getTimeOfFirstEvent() {
+        return this.timeOfFirstEvent;
+    }
 }

api/src/main/java/com/microsoft/gctoolkit/jvm/JavaVirtualMachine.java

@@ -5,7 +5,6 @@
 
 import com.microsoft.gctoolkit.GCToolKit;
 import com.microsoft.gctoolkit.aggregator.Aggregation;
-import com.microsoft.gctoolkit.io.DataSource;
 import com.microsoft.gctoolkit.io.GCLogFile;
 import com.microsoft.gctoolkit.time.DateTimeStamp;
 
@@ -69,15 +68,22 @@ public interface JavaVirtualMachine {
 
     /**
      * Return the time of the first event in the GC log file.
-     * @return The time of the first event.
+     * @return The time of the last event.
      */
     DateTimeStamp getTimeOfFirstEvent();
 
+    /**
+     * Estimates the initial start time of the log in the case that the log
+     * is determined to be a fragment. Otherwise, return a start time of 0.000 seconds
+     * @return The time of the first event.
+     */
+    DateTimeStamp getEstimatedJVMStartTime();
+
     /**
      * Return the time of the last event in the GC log file.
      * @return The time of the last event.
      */
-    DateTimeStamp getTimeOfLastEvent();
+    DateTimeStamp getJVMTerminationTime();
 
     /**
      * Return the runtime duration. This is not necessarily the difference
@@ -87,13 +93,6 @@ public interface JavaVirtualMachine {
      */
     double getRuntimeDuration();
 
-    /**
-     * Get the configuration metadata. This configuration data is either
-     * known from JVM flags, or inferred from parsing the GC log file.
-     * @return The {@code JvmConfiguration}
-     */
-    JvmConfiguration getJvmConfiguration();
-
     /**
      * Return the {@code Aggregation} that was used in the analysis of the GC log file
      * that is the same class as {@code aggregationClass}. In other words, {@code aggregationClass}
@@ -106,4 +105,5 @@ public interface JavaVirtualMachine {
      * if given aggregationClass is not available.
      */
     <T extends Aggregation> Optional<T> getAggregation(Class<T> aggregationClass);
+
 }