Skip to content

Commit

Permalink
Ensure backwards compatibility to cpg version < 4.4.0 (#769)
Browse files Browse the repository at this point in the history
Co-authored-by: Christian Banse <oxisto@aybaze.com>
  • Loading branch information
KuechA and oxisto committed Apr 26, 2022
1 parent df04e7d commit 7a767b6
Show file tree
Hide file tree
Showing 11 changed files with 96 additions and 40 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,15 @@ public Map<String, String> getSymbols() {
return this.symbols;
}

/** Returns a list of all analyzed files. */
public List<File> getSourceLocations() {
List<File> sourceLocations = new ArrayList<>();
for (var entry : softwareComponents.entrySet()) {
sourceLocations.addAll(entry.getValue());
}
return sourceLocations;
}

public Map<String, List<File>> getSoftwareComponents() {
return this.softwareComponents;
}
Expand Down Expand Up @@ -283,25 +292,25 @@ public Builder symbols(Map<String, String> symbols) {

/**
* Files or directories containing the source code to analyze. Generates a dummy software
* component called "SWC".
* component called "application".
*
* @param sourceLocations The files with the source code
* @return this
*/
public Builder sourceLocations(File... sourceLocations) {
this.softwareComponents.put("SWC", Arrays.asList(sourceLocations));
this.softwareComponents.put("application", Arrays.asList(sourceLocations));
return this;
}

/**
* Files or directories containing the source code to analyze. Generates a dummy software
* component called "SWC".
* component called "application".
*
* @param sourceLocations The files with the source code
* @return this
*/
public Builder sourceLocations(List<File> sourceLocations) {
this.softwareComponents.put("SWC", sourceLocations);
this.softwareComponents.put("application", sourceLocations);
return this;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ import de.fraunhofer.aisec.cpg.frontends.cpp.CXXLanguageFrontend
import de.fraunhofer.aisec.cpg.graph.Component
import de.fraunhofer.aisec.cpg.graph.TypeManager
import de.fraunhofer.aisec.cpg.helpers.Benchmark
import de.fraunhofer.aisec.cpg.helpers.MeasurementHolder
import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker
import de.fraunhofer.aisec.cpg.helpers.TimeBenchmark
import de.fraunhofer.aisec.cpg.helpers.Util
import de.fraunhofer.aisec.cpg.passes.Pass
import de.fraunhofer.aisec.cpg.passes.scopes.ScopeManager
Expand Down Expand Up @@ -77,7 +77,7 @@ private constructor(
return CompletableFuture.supplyAsync {
val scopesBuildForAnalysis = ScopeManager()
val outerBench =
TimeBenchmark(
Benchmark(
TranslationManager::class.java,
"Translation into full graph",
false,
Expand All @@ -88,8 +88,7 @@ private constructor(

try {
// Parse Java/C/CPP files
var bench =
TimeBenchmark(this.javaClass, "Executing Language Frontend", false, result)
var bench = Benchmark(this.javaClass, "Executing Language Frontend", false, result)
frontendsNeedCleanup = runFrontends(result, config, scopesBuildForAnalysis)
bench.addMeasurement()

Expand All @@ -99,7 +98,7 @@ private constructor(
// Apply passes
for (pass in config.registeredPasses) {
passesNeedCleanup.add(pass)
bench = TimeBenchmark(pass.javaClass, "Executing Pass", false, result)
bench = Benchmark(pass.javaClass, "Executing Pass", false, result)
pass.accept(result)
bench.addMeasurement()
if (result.isCancelled) {
Expand Down Expand Up @@ -227,14 +226,17 @@ private constructor(
result.components.forEach { s ->
s.translationUnits.forEach {
val bench =
Benchmark(this.javaClass, "Activating types for ${it.name}", true)
MeasurementHolder(
this.javaClass,
"Activating types for ${it.name}",
true
)
SubgraphWalker.activateTypes(it, scopeManager)
bench.addMeasurement()
}
}
result.translationUnits.forEach {
val bench =
TimeBenchmark(this.javaClass, "Activating types for ${it.name}", true)
val bench = Benchmark(this.javaClass, "Activating types for ${it.name}", true)
SubgraphWalker.activateTypes(it, scopeManager)
bench.addMeasurement()
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@
import de.fraunhofer.aisec.cpg.graph.Node;
import de.fraunhofer.aisec.cpg.graph.SubGraph;
import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration;
import de.fraunhofer.aisec.cpg.helpers.Benchmark;
import de.fraunhofer.aisec.cpg.helpers.BenchmarkResults;
import de.fraunhofer.aisec.cpg.helpers.MeasurementHolder;
import de.fraunhofer.aisec.cpg.helpers.StatisticsHolder;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
Expand Down Expand Up @@ -61,7 +61,7 @@ public class TranslationResult extends Node implements StatisticsHolder {
*/
private final Set<Node> additionalNodes = new HashSet<>();

private final List<Benchmark> benchmarks = new ArrayList<>();
private final List<MeasurementHolder> benchmarks = new ArrayList<>();

public TranslationResult(TranslationManager translationManager) {
this.translationManager = translationManager;
Expand All @@ -88,6 +88,45 @@ public List<TranslationUnitDeclaration> getTranslationUnits() {
return result;
}

/**
* If no component exists, it generates a [Component] called "application" and adds [tu]. If a
* component already exists, adds the tu to this component.
*
* @param tu The translation unit to add.
* @deprecated This should not be used anymore. Instead, the corresponding component should be
* selected and the translation unit should be added there.
*/
@Deprecated(since = "4.4.1")
public synchronized void addTranslationUnit(TranslationUnitDeclaration tu) {
Component swc = null;
if (components.size() == 1) {
// Only one component exists, so we take this one
swc = components.get(0);
} else if (components.isEmpty()) {
// No component exists, so we create the new dummy component.
swc = new Component();
swc.setName("application");
components.add(swc);
} else {
// Multiple components exist. As we don't know where to put the tu, we check if we have the
// component we created and add it there or create a new one.
for (var component : components) {
if (component.getName().equals("application")) {
swc = component;
break;
}
}

if (swc == null) {
swc = new Component();
swc.setName("application");
components.add(swc);
}
}

swc.getTranslationUnits().add(tu);
}

/**
* List of software components. Note that this list is immutable. Use {@link
* #addComponent(Component)} if you wish to modify it.
Expand Down Expand Up @@ -127,12 +166,12 @@ public TranslationManager getTranslationManager() {
}

@Override
public void addBenchmark(@NotNull Benchmark b) {
public void addBenchmark(@NotNull MeasurementHolder b) {
this.benchmarks.add(b);
}

@NotNull
public List<Benchmark> getBenchmarks() {
public List<MeasurementHolder> getBenchmarks() {
return benchmarks;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression
import de.fraunhofer.aisec.cpg.graph.types.TypeParser
import de.fraunhofer.aisec.cpg.graph.types.UnknownType
import de.fraunhofer.aisec.cpg.helpers.TimeBenchmark
import de.fraunhofer.aisec.cpg.helpers.Benchmark
import de.fraunhofer.aisec.cpg.passes.scopes.ScopeManager
import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation
import de.fraunhofer.aisec.cpg.sarif.Region
Expand Down Expand Up @@ -196,7 +196,7 @@ class CXXLanguageFrontend(config: TranslationConfiguration, scopeManager: ScopeM
val log = DefaultLogService()
val opts = ILanguage.OPTION_PARSE_INACTIVE_CODE // | ILanguage.OPTION_ADD_COMMENTS;
return try {
var bench = TimeBenchmark(this.javaClass, "Parsing sourcefile")
var bench = Benchmark(this.javaClass, "Parsing sourcefile")
val translationUnit =
GPPLanguage.getDefault()
.getASTTranslationUnit(
Expand All @@ -211,7 +211,7 @@ class CXXLanguageFrontend(config: TranslationConfiguration, scopeManager: ScopeM
val length = translationUnit.length
LOGGER.info("Parsed {} bytes corresponding roughly to {} LoC", length, length / 50)
bench.addMeasurement()
bench = TimeBenchmark(this.javaClass, "Transform to CPG")
bench = Benchmark(this.javaClass, "Transform to CPG")
if (config.debugParser) {
explore(translationUnit, 0)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression
import de.fraunhofer.aisec.cpg.graph.types.TypeParser
import de.fraunhofer.aisec.cpg.graph.types.UnknownType
import de.fraunhofer.aisec.cpg.helpers.Benchmark
import de.fraunhofer.aisec.cpg.helpers.CommonPath
import de.fraunhofer.aisec.cpg.helpers.TimeBenchmark
import de.fraunhofer.aisec.cpg.passes.scopes.Scope
import de.fraunhofer.aisec.cpg.passes.scopes.ScopeManager
import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation
Expand Down Expand Up @@ -105,11 +105,11 @@ open class JavaLanguageFrontend(config: TranslationConfiguration, scopeManager:
val parser = JavaParser(parserConfiguration)

// parse the file
var bench = TimeBenchmark(this.javaClass, "Parsing source file")
var bench = Benchmark(this.javaClass, "Parsing source file")

context = parse(file, parser)
bench.addMeasurement()
bench = TimeBenchmark(this.javaClass, "Transform to CPG")
bench = Benchmark(this.javaClass, "Transform to CPG")
context!!.setData(Node.SYMBOL_RESOLVER_KEY, javaSymbolResolver)

// starting point is always a translation declaration
Expand Down
4 changes: 2 additions & 2 deletions cpg-core/src/main/java/de/fraunhofer/aisec/cpg/graph/Graph.kt
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ package de.fraunhofer.aisec.cpg.graph
import de.fraunhofer.aisec.cpg.ExperimentalGraph
import de.fraunhofer.aisec.cpg.TranslationResult
import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge
import de.fraunhofer.aisec.cpg.helpers.Benchmark
import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker
import de.fraunhofer.aisec.cpg.helpers.TimeBenchmark
import java.io.Closeable
import java.util.*
import java.util.function.Predicate
Expand All @@ -45,7 +45,7 @@ import scala.Option

@ExperimentalGraph
class QueryBenchmark constructor(db: Graph, query: Query) :
TimeBenchmark(db.javaClass, "totalNodes: " + db.size() + " query: " + query.toString()),
Benchmark(db.javaClass, "totalNodes: " + db.size() + " query: " + query.toString()),
AutoCloseable,
Closeable {
override fun close() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,10 @@ class BenchmarkResults(val entries: List<List<Any>>) {
/** Interface definition to hold different statistics about the translation process. */
interface StatisticsHolder {
val translatedFiles: List<String>
val benchmarks: List<Benchmark>
val benchmarks: List<MeasurementHolder>
val config: TranslationConfiguration

fun addBenchmark(b: Benchmark)
fun addBenchmark(b: MeasurementHolder)

val benchmarkResults: BenchmarkResults
get() {
Expand Down Expand Up @@ -129,15 +129,19 @@ fun relativeOrAbsolute(path: Path, topLevel: File?): Path {
}

/** Measures the time between creating the object to calling its stop() method. */
open class TimeBenchmark(
open class Benchmark(
c: Class<*>,
message: String,
debug: Boolean = false,
holder: StatisticsHolder? = null
) : Benchmark(c, message, debug, holder) {
) : MeasurementHolder(c, message, debug, holder) {

private val start: Instant

fun stop() {
addMeasurement()
}

/** Stops the time and computes the difference between */
override fun addMeasurement(measurementKey: String?, measurementValue: String?): Any? {
val duration = Duration.between(start, Instant.now()).toMillis()
Expand All @@ -152,7 +156,7 @@ open class TimeBenchmark(
}

companion object {
val log: Logger = LoggerFactory.getLogger(Benchmark::class.java)
val log: Logger = LoggerFactory.getLogger(MeasurementHolder::class.java)
}

init {
Expand All @@ -162,7 +166,7 @@ open class TimeBenchmark(
}

/** Represents some kind of measurements, e.g., on the performance or problems. */
open class Benchmark
open class MeasurementHolder
@JvmOverloads
constructor(
/** The class which called this benchmark. */
Expand Down Expand Up @@ -198,6 +202,7 @@ constructor(
}

/** Adds a measurement for the respective benchmark and saves it to the map. */
@JvmOverloads
open fun addMeasurement(
measurementKey: String? = null,
measurementValue: String? = null
Expand All @@ -213,7 +218,7 @@ constructor(
}

companion object {
val log: Logger = LoggerFactory.getLogger(Benchmark::class.java)
val log: Logger = LoggerFactory.getLogger(MeasurementHolder::class.java)
}

init {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import de.fraunhofer.aisec.cpg.TranslationResult
import de.fraunhofer.aisec.cpg.graph.Node
import de.fraunhofer.aisec.cpg.graph.ProblemNode
import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration
import de.fraunhofer.aisec.cpg.helpers.Benchmark
import de.fraunhofer.aisec.cpg.helpers.MeasurementHolder
import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker.ScopedWalker

/**
Expand All @@ -54,7 +54,8 @@ class StatisticsCollectionPass : Pass() {
walker.iterate(tu)
}

val nodeMeasurement = Benchmark(this.javaClass, "Measuring Nodes", false, translationResult)
val nodeMeasurement =
MeasurementHolder(this.javaClass, "Measuring Nodes", false, translationResult)
nodeMeasurement.addMeasurement("Graph nodes", nodes.toString())
nodeMeasurement.addMeasurement("Problem nodes", problemNodes.toString())
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge
import de.fraunhofer.aisec.cpg.graph.statements.expressions.InitializerListExpression
import de.fraunhofer.aisec.cpg.graph.types.ObjectType
import de.fraunhofer.aisec.cpg.graph.types.Type
import de.fraunhofer.aisec.cpg.helpers.Benchmark
import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker
import de.fraunhofer.aisec.cpg.helpers.TimeBenchmark
import java.time.Duration
import java.time.temporal.ChronoUnit
import kotlin.io.path.writeText
Expand Down Expand Up @@ -103,7 +103,7 @@ class PerformanceRegressionTest {
// Even on a slow machine, this should not exceed 1 second (it should be more like
// 200-300ms)
assertTimeout(Duration.of(1, ChronoUnit.SECONDS)) {
val b = TimeBenchmark(PerformanceRegressionTest::class.java, "getAstChildren")
val b = Benchmark(PerformanceRegressionTest::class.java, "getAstChildren")
doNothing(tu)
b.addMeasurement()
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression
import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression
import de.fraunhofer.aisec.cpg.graph.types.*
import de.fraunhofer.aisec.cpg.helpers.TimeBenchmark
import de.fraunhofer.aisec.cpg.helpers.Benchmark
import de.fraunhofer.aisec.cpg.passes.VariableUsageResolver
import de.fraunhofer.aisec.cpg.passes.scopes.ScopeManager
import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation
Expand Down Expand Up @@ -71,7 +71,7 @@ class LLVMIRLanguageFrontend(config: TranslationConfiguration, scopeManager: Sco
}

override fun parse(file: File): TranslationUnitDeclaration {
var bench = TimeBenchmark(this.javaClass, "Parsing sourcefile")
var bench = Benchmark(this.javaClass, "Parsing sourcefile")
// clear the bindings cache, because it is just valid within one module
bindingsCache.clear()

Expand Down Expand Up @@ -109,7 +109,7 @@ class LLVMIRLanguageFrontend(config: TranslationConfiguration, scopeManager: Sco
throw TranslationException("Could not parse IR: $errorMsg")
}
bench.addMeasurement()
bench = TimeBenchmark(this.javaClass, "Transform to CPG")
bench = Benchmark(this.javaClass, "Transform to CPG")

val tu = TranslationUnitDeclaration()

Expand Down
Loading

0 comments on commit 7a767b6

Please sign in to comment.