From 3d2ec7e342ddfaf7d7acae744faeab3bd72d1a4e Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Thu, 27 May 2021 15:03:16 +0200 Subject: [PATCH 01/30] ++ --- cpg-console/.gitattributes | 6 ++ cpg-console/.gitignore | 5 ++ cpg-console/build.gradle.kts | 47 +++++++++++ .../de/fraunhofer/aisec/cpg/console/App.kt | 79 +++++++++++++++++++ settings.gradle.kts | 1 + 5 files changed, 138 insertions(+) create mode 100644 cpg-console/.gitattributes create mode 100644 cpg-console/.gitignore create mode 100644 cpg-console/build.gradle.kts create mode 100644 cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/App.kt diff --git a/cpg-console/.gitattributes b/cpg-console/.gitattributes new file mode 100644 index 0000000000..00a51aff5e --- /dev/null +++ b/cpg-console/.gitattributes @@ -0,0 +1,6 @@ +# +# https://help.github.com/articles/dealing-with-line-endings/ +# +# These are explicitly windows files and should use crlf +*.bat text eol=crlf + diff --git a/cpg-console/.gitignore b/cpg-console/.gitignore new file mode 100644 index 0000000000..1b6985c009 --- /dev/null +++ b/cpg-console/.gitignore @@ -0,0 +1,5 @@ +# Ignore Gradle project-specific cache directory +.gradle + +# Ignore Gradle build output directory +build diff --git a/cpg-console/build.gradle.kts b/cpg-console/build.gradle.kts new file mode 100644 index 0000000000..8c1d3db7d0 --- /dev/null +++ b/cpg-console/build.gradle.kts @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +plugins { + application +} + +application { + mainClass.set("de.fraunhofer.aisec.cpg_vis_neo4j.ApplicationKt") +} + +tasks.withType { + useJUnitPlatform { + if (!project.hasProperty("integration")) { + excludeTags("integration") + } + } +} + +dependencies { + // CPG + api(project(":cpg-library")) + + implementation("org.jetbrains.kotlinx:ki-shell:0.3.3") +} diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/App.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/App.kt new file mode 100644 index 0000000000..f0ded1536c --- /dev/null +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/App.kt @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package org.jetbrains.kotlinx.ki.shell + +import kotlin.script.experimental.api.ScriptCompilationConfiguration +import kotlin.script.experimental.api.ScriptEvaluationConfiguration +import kotlin.script.experimental.jvm.baseClassLoader +import kotlin.script.experimental.jvm.defaultJvmScriptingHostConfiguration +import kotlin.script.experimental.jvm.dependenciesFromClassloader +import kotlin.script.experimental.jvm.jvm +import org.jetbrains.kotlinx.ki.shell.configuration.CachedInstance +import org.jetbrains.kotlinx.ki.shell.configuration.ReplConfiguration +import org.jetbrains.kotlinx.ki.shell.configuration.ReplConfigurationBase + +object CpgConsole { + @JvmStatic + fun main(args: Array) { + val repl = + Shell( + configuration(), + defaultJvmScriptingHostConfiguration, + ScriptCompilationConfiguration { + jvm { + dependenciesFromClassloader( + classLoader = KotlinShell::class.java.classLoader, + wholeClasspath = true + ) + } + }, + ScriptEvaluationConfiguration { + jvm { baseClassLoader(Shell::class.java.classLoader) } + } + ) + + Runtime.getRuntime() + .addShutdownHook( + Thread { + println("\nBye!") + repl.cleanUp() + } + ) + + repl.doRun() + } + + fun configuration(): ReplConfiguration { + val instance = CachedInstance() + val klassName: String? = System.getProperty("config.class") + + return if (klassName != null) { + instance.load(klassName, ReplConfiguration::class) + } else { + instance.get { object : ReplConfigurationBase() {} } + } + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index e63d5caf63..dc4a11a360 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,3 +1,4 @@ include(":cpg-library") include(":cpg-neo4j") +include(":cpg-console") From 0a3d4e95f4c6fea20b225e2630ce4bcc7d808a17 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Tue, 1 Jun 2021 21:10:38 +0200 Subject: [PATCH 02/30] Added CPG console plugins --- cpg-console/build.gradle.kts | 3 +- .../aisec/cpg/console/AnalyzePlugin.kt | 78 +++++++++++++++++++ .../cpg/console/{App.kt => CpgConsole.kt} | 19 ++++- .../aisec/cpg/console/Neo4jPlugin.kt | 67 ++++++++++++++++ cpg-console/src/main/resources/log4j2.xml | 14 ++++ 5 files changed, 177 insertions(+), 4 deletions(-) create mode 100644 cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt rename cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/{App.kt => CpgConsole.kt} (80%) create mode 100644 cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Neo4jPlugin.kt create mode 100644 cpg-console/src/main/resources/log4j2.xml diff --git a/cpg-console/build.gradle.kts b/cpg-console/build.gradle.kts index 8c1d3db7d0..4e144eb2d0 100644 --- a/cpg-console/build.gradle.kts +++ b/cpg-console/build.gradle.kts @@ -28,7 +28,7 @@ plugins { } application { - mainClass.set("de.fraunhofer.aisec.cpg_vis_neo4j.ApplicationKt") + mainClass.set("de.fraunhofer.aisec.cpg.console.CpgConsole") } tasks.withType { @@ -42,6 +42,7 @@ tasks.withType { dependencies { // CPG api(project(":cpg-library")) + api(project(":cpg-neo4j")) implementation("org.jetbrains.kotlinx:ki-shell:0.3.3") } diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt new file mode 100644 index 0000000000..b38a989bda --- /dev/null +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.console + +import org.jetbrains.kotlinx.ki.shell.BaseCommand +import org.jetbrains.kotlinx.ki.shell.Command +import org.jetbrains.kotlinx.ki.shell.Plugin +import org.jetbrains.kotlinx.ki.shell.Shell +import org.jetbrains.kotlinx.ki.shell.configuration.ReplConfiguration + +class AnalyzePlugin : Plugin { + inner class Load(conf: ReplConfiguration) : BaseCommand() { + override val name: String by conf.get(default = "analyze") + override val short: String by conf.get(default = "a") + override val description: String = "analyzes the path" + + override val params = "" + + override fun execute(line: String): Command.Result { + val p = line.indexOf(' ') + val path = line.substring(p + 1).trim() + + return Command.Result.RunSnippets( + listOf( + "import de.fraunhofer.aisec.cpg.TranslationConfiguration", + "import de.fraunhofer.aisec.cpg.TranslationManager", + "import de.fraunhofer.aisec.cpg.graph.*", + "import java.io.File", + "val config =\n" + + " TranslationConfiguration.builder()\n" + + " .sourceLocations(File(\"" + + path + + "\"))\n" + + " .defaultLanguages()\n" + + " .defaultPasses()\n" + + " .build()", + "val analyzer = TranslationManager.builder().config(config).build()", + "val result = analyzer.analyze().get()" + ) + ) + + // return Command.Result.RunSnippets(listOf(content)) + } + } + + lateinit var repl: Shell + + override fun init(repl: Shell, config: ReplConfiguration) { + this.repl = repl + + repl.registerCommand(Load(config)) + } + + override fun cleanUp() {} +} diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/App.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt similarity index 80% rename from cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/App.kt rename to cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt index f0ded1536c..4f6f9738c6 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/App.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt @@ -23,7 +23,7 @@ * \______/ \__| \______/ * */ -package org.jetbrains.kotlinx.ki.shell +package de.fraunhofer.aisec.cpg.console import kotlin.script.experimental.api.ScriptCompilationConfiguration import kotlin.script.experimental.api.ScriptEvaluationConfiguration @@ -31,6 +31,9 @@ import kotlin.script.experimental.jvm.baseClassLoader import kotlin.script.experimental.jvm.defaultJvmScriptingHostConfiguration import kotlin.script.experimental.jvm.dependenciesFromClassloader import kotlin.script.experimental.jvm.jvm +import org.jetbrains.kotlinx.ki.shell.KotlinShell +import org.jetbrains.kotlinx.ki.shell.Plugin +import org.jetbrains.kotlinx.ki.shell.Shell import org.jetbrains.kotlinx.ki.shell.configuration.CachedInstance import org.jetbrains.kotlinx.ki.shell.configuration.ReplConfiguration import org.jetbrains.kotlinx.ki.shell.configuration.ReplConfigurationBase @@ -66,14 +69,24 @@ object CpgConsole { repl.doRun() } - fun configuration(): ReplConfiguration { + private fun configuration(): ReplConfiguration { val instance = CachedInstance() val klassName: String? = System.getProperty("config.class") return if (klassName != null) { instance.load(klassName, ReplConfiguration::class) } else { - instance.get { object : ReplConfigurationBase() {} } + instance.get { + object : ReplConfigurationBase() { + override fun plugins(): Iterator { + val list = super.plugins().asSequence().toList().toMutableList() + list += AnalyzePlugin() + list += Neo4jPlugin() + + return list.listIterator() + } + } + } } } } diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Neo4jPlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Neo4jPlugin.kt new file mode 100644 index 0000000000..9280315cdb --- /dev/null +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Neo4jPlugin.kt @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.console + +import org.jetbrains.kotlinx.ki.shell.BaseCommand +import org.jetbrains.kotlinx.ki.shell.Command +import org.jetbrains.kotlinx.ki.shell.Plugin +import org.jetbrains.kotlinx.ki.shell.Shell +import org.jetbrains.kotlinx.ki.shell.configuration.ReplConfiguration + +class Neo4jPlugin : Plugin { + inner class Load(conf: ReplConfiguration) : BaseCommand() { + override val name: String by conf.get(default = "neo4j") + override val short: String by conf.get(default = "n") + override val description: String = "persists the graph into neo4j" + + override val params = "" + + override fun execute(line: String): Command.Result { + val p = line.indexOf(' ') + val host = line.substring(p + 1).trim() + + return Command.Result.RunSnippets( + listOf( + "import de.fraunhofer.aisec.cpg_vis_neo4j.Application", + "val neo4j = Application()", + "neo4j.pushToNeo4j(result)" + ) + ) + + // return Command.Result.RunSnippets(listOf(content)) + } + } + + lateinit var repl: Shell + + override fun init(repl: Shell, config: ReplConfiguration) { + this.repl = repl + + repl.registerCommand(Load(config)) + } + + override fun cleanUp() {} +} diff --git a/cpg-console/src/main/resources/log4j2.xml b/cpg-console/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..d8be9b3a6e --- /dev/null +++ b/cpg-console/src/main/resources/log4j2.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file From 970da40f5d9643f6fb797ed65bb6cbd36687dcc0 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Tue, 1 Jun 2021 22:47:22 +0200 Subject: [PATCH 03/30] Added value resolver --- cpg-console/README.md | 17 ++ cpg-console/build.gradle.kts | 4 + .../aisec/cpg/analysis/Extensions.kt | 62 ++++++ .../aisec/cpg/analysis/ValueResolver.kt | 185 ++++++++++++++++++ .../aisec/cpg/console/AnalyzePlugin.kt | 11 ++ .../aisec/cpg/console/CpgConsole.kt | 1 + .../aisec/cpg/console/ShowCodePlugin.kt | 66 +++++++ cpg-console/src/test/resources/array.cpp | 3 + 8 files changed, 349 insertions(+) create mode 100644 cpg-console/README.md create mode 100644 cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt create mode 100644 cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueResolver.kt create mode 100644 cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt create mode 100644 cpg-console/src/test/resources/array.cpp diff --git a/cpg-console/README.md b/cpg-console/README.md new file mode 100644 index 0000000000..f4c049f6ad --- /dev/null +++ b/cpg-console/README.md @@ -0,0 +1,17 @@ +# CPG Console + +```bash +../gradlew installDist +build/install/cpg-console/bin/cpg-console +``` + +The following example snippet can be used: + +```kotlin +:a src/test/resources/array.cpp +var tu = result.translationUnits.first() +var main = tu.byName("main") +:code main? +var decl = main?.body(0) +var v = decl?.singleDeclaration as? VariableDeclaration +``` diff --git a/cpg-console/build.gradle.kts b/cpg-console/build.gradle.kts index 4e144eb2d0..00a35b97fc 100644 --- a/cpg-console/build.gradle.kts +++ b/cpg-console/build.gradle.kts @@ -39,6 +39,10 @@ tasks.withType { } } +tasks.withType { + kotlinOptions.jvmTarget = "1.8" // important, since ki is 1.8 and otherwise inlining wont work +} + dependencies { // CPG api(project(":cpg-library")) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt new file mode 100644 index 0000000000..da3c989144 --- /dev/null +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.analysis + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.declarations.Declaration +import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration +import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement +import de.fraunhofer.aisec.cpg.graph.statements.Statement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression + +fun Node?.printCode(): Unit { + println(this?.code) +} + +fun Expression.resolve(): Any? { + return ValueResolver().resolve(this) +} + +fun Declaration.resolve(): Any? { + return ValueResolver().resolveDeclaration(this) +} + +inline fun TranslationUnitDeclaration.byName(name: String): T? { + return this.declarations.filterIsInstance().firstOrNull { it.name == name } +} + +inline fun FunctionDeclaration.body(i: Int): T? { + return if (this.body is CompoundStatement) { + (this.body as? CompoundStatement)?.statements?.get(i) as? T + } else { + return if (i == 0) { + this.body as? T + } else { + null + } + } +} diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueResolver.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueResolver.kt new file mode 100644 index 0000000000..31fd6c7c8b --- /dev/null +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueResolver.kt @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.analysis + +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.declarations.Declaration +import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* + +/** + * The value resolver tries to resolve the value of an [Expression] basically by following edges + * until we reach a [Literal]. + * + * It contains some advanced mechanics such as resolution of values of arrays, if they contain + * literal values. Furthermore, its behaviour can be adjusted by implementing the [cannotResolve] + * function, which is called when the default behaviour would not be able to resolve the value. This + * way, language specific features such as string formatting can be modelled. + */ +class ValueResolver( + /** + * Contains a reference to a function that gets called if the value cannot be resolved by the + * standard behaviour. + */ + val cannotResolve: (Node?, ValueResolver) -> Any? = { node: Node?, _: ValueResolver -> + // end of the line, lets just keep the expression name + if (node != null) { + "{${node.name}}" + } else { + null + } + } +) { + + fun resolveDeclaration(decl: Declaration?): Any? { + when (decl) { + is VariableDeclaration -> return resolve(decl.initializer) + is FieldDeclaration -> return resolve(decl.initializer) + } + + return cannotResolve(decl, this) + } + + /** Tries to resolve this expression. Anything can happen. */ + fun resolve(expr: Expression?): Any? { + when (expr) { + is Literal<*> -> { + return expr.value + } + is DeclaredReferenceExpression -> return resolveDeclaration(expr.refersTo) + is BinaryOperator -> { + // resolve lhs + val lhsValue = resolve(expr.lhs) + + // resolve rhs + val rhsValue = resolve(expr.rhs) + + if (expr.operatorCode == "+") { + if (lhsValue is String) { + return lhsValue + rhsValue + } else if (lhsValue is Int && rhsValue is Number) { + return lhsValue + rhsValue.toInt() + } else if (lhsValue is Long && rhsValue is Number) { + return lhsValue + rhsValue.toLong() + } else if (lhsValue is Short && rhsValue is Number) { + return lhsValue + rhsValue.toShort() + } else if (lhsValue is Byte && rhsValue is Number) { + return lhsValue + rhsValue.toByte() + } else if (lhsValue is Double && rhsValue is Number) { + return lhsValue + rhsValue.toDouble() + } else if (lhsValue is Float && rhsValue is Number) { + return lhsValue + rhsValue.toDouble() + } + } else if (expr.operatorCode == "-") { + if (lhsValue is Int && rhsValue is Number) { + return lhsValue - rhsValue.toInt() + } else if (lhsValue is Long && rhsValue is Number) { + return lhsValue - rhsValue.toLong() + } else if (lhsValue is Short && rhsValue is Number) { + return lhsValue - rhsValue.toShort() + } else if (lhsValue is Byte && rhsValue is Number) { + return lhsValue - rhsValue.toByte() + } else if (lhsValue is Double && rhsValue is Number) { + return lhsValue - rhsValue.toDouble() + } else if (lhsValue is Float && rhsValue is Number) { + return lhsValue - rhsValue.toDouble() + } + } else if (expr.operatorCode == "/") { + if (lhsValue is Int && rhsValue is Number) { + return lhsValue / rhsValue.toInt() + } else if (lhsValue is Long && rhsValue is Number) { + return lhsValue / rhsValue.toLong() + } else if (lhsValue is Short && rhsValue is Number) { + return lhsValue / rhsValue.toShort() + } else if (lhsValue is Byte && rhsValue is Number) { + return lhsValue / rhsValue.toByte() + } else if (lhsValue is Double && rhsValue is Number) { + return lhsValue / rhsValue.toDouble() + } else if (lhsValue is Float && rhsValue is Number) { + return lhsValue / rhsValue.toDouble() + } + } else if (expr.operatorCode == "*") { + if (lhsValue is Int && rhsValue is Number) { + return lhsValue * rhsValue.toInt() + } else if (lhsValue is Long && rhsValue is Number) { + return lhsValue * rhsValue.toLong() + } else if (lhsValue is Short && rhsValue is Number) { + return lhsValue * rhsValue.toShort() + } else if (lhsValue is Byte && rhsValue is Number) { + return lhsValue * rhsValue.toByte() + } else if (lhsValue is Double && rhsValue is Number) { + return lhsValue * rhsValue.toDouble() + } else if (lhsValue is Float && rhsValue is Number) { + return lhsValue * rhsValue.toDouble() + } + } + + return cannotResolve(expr, this) + } + is CastExpression -> { + return this.resolve(expr.expression) + } + is ArraySubscriptionExpression -> { + val array = + (expr.arrayExpression as? DeclaredReferenceExpression)?.refersTo as? + VariableDeclaration + val ile = array?.initializer as? InitializerListExpression + + ile?.let { + return resolve( + it.initializers + .filterIsInstance(KeyValueExpression::class.java) + .firstOrNull { kve -> + (kve.key as? Literal<*>)?.value == + (expr.subscriptExpression as? Literal<*>)?.value + } + ?.value + ) + } + + return cannotResolve(expr, this) + } + is ConditionalExpression -> { + // assume that condition is a binary operator + if (expr.condition is BinaryOperator) { + val lhs = resolve((expr.condition as? BinaryOperator)?.lhs) + val rhs = resolve((expr.condition as? BinaryOperator)?.rhs) + + return if (lhs == rhs) { + resolve(expr.thenExpr) + } else { + resolve(expr.elseExpr) + } + } + + return cannotResolve(expr, this) + } + } + + return cannotResolve(expr, this) + } +} diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt index b38a989bda..b1b4286b5d 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt @@ -45,10 +45,21 @@ class AnalyzePlugin : Plugin { return Command.Result.RunSnippets( listOf( + // basics "import de.fraunhofer.aisec.cpg.TranslationConfiguration", "import de.fraunhofer.aisec.cpg.TranslationManager", + // all the graph nodes "import de.fraunhofer.aisec.cpg.graph.*", + "import de.fraunhofer.aisec.cpg.graph.declarations.*", + "import de.fraunhofer.aisec.cpg.graph.statements.*", + // helper builtins + "import de.fraunhofer.aisec.cpg.analysis.resolve", + "import de.fraunhofer.aisec.cpg.analysis.byName", + "import de.fraunhofer.aisec.cpg.analysis.body", + "import de.fraunhofer.aisec.cpg.analysis.printCode", + // some basic java stuff "import java.io.File", + // lets build and analyze "val config =\n" + " TranslationConfiguration.builder()\n" + " .sourceLocations(File(\"" + diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt index 4f6f9738c6..ef580ddf62 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt @@ -82,6 +82,7 @@ object CpgConsole { val list = super.plugins().asSequence().toList().toMutableList() list += AnalyzePlugin() list += Neo4jPlugin() + list += ShowCodePlugin() return list.listIterator() } diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt new file mode 100644 index 0000000000..435fb2fef8 --- /dev/null +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.console + +import de.fraunhofer.aisec.cpg.graph.Node +import org.jetbrains.kotlinx.ki.shell.BaseCommand +import org.jetbrains.kotlinx.ki.shell.Command +import org.jetbrains.kotlinx.ki.shell.Plugin +import org.jetbrains.kotlinx.ki.shell.Shell +import org.jetbrains.kotlinx.ki.shell.configuration.ReplConfiguration + +class ShowCodePlugin : Plugin { + inner class Load(conf: ReplConfiguration) : BaseCommand() { + override val name: String by conf.get(default = "code") + override val short: String by conf.get(default = "c") + override val description: String = "print code of node" + + override val params = "" + + override fun execute(line: String): Command.Result { + val p = line.indexOf(' ') + val node = line.substring(p + 1).trim() + + return Command.Result.RunSnippets(listOf("${node}.printCode()")) + + // return Command.Result.RunSnippets(listOf(content)) + } + } + + lateinit var repl: Shell + + override fun init(repl: Shell, config: ReplConfiguration) { + this.repl = repl + + repl.registerCommand(Load(config)) + } + + override fun cleanUp() {} +} + +fun Node.printNode() { + println(this.code) +} diff --git a/cpg-console/src/test/resources/array.cpp b/cpg-console/src/test/resources/array.cpp new file mode 100644 index 0000000000..b681f130e9 --- /dev/null +++ b/cpg-console/src/test/resources/array.cpp @@ -0,0 +1,3 @@ +int main() { + int a = 4 + 5; +} \ No newline at end of file From c5a1ffff9db4e63e5d7bbd355aec88f324104d97 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Tue, 1 Jun 2021 23:25:37 +0200 Subject: [PATCH 04/30] starting out of bounds check --- cpg-console/build.gradle.kts | 10 ++++ .../aisec/cpg/analysis/OutOfBoundsCheck.kt | 55 +++++++++++++++++++ .../aisec/cpg/analysis/AnalysisTest.kt | 48 ++++++++++++++++ cpg-console/src/test/resources/array.cpp | 7 ++- 4 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt create mode 100644 cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt diff --git a/cpg-console/build.gradle.kts b/cpg-console/build.gradle.kts index 00a35b97fc..35f8c22c43 100644 --- a/cpg-console/build.gradle.kts +++ b/cpg-console/build.gradle.kts @@ -43,10 +43,20 @@ tasks.withType { kotlinOptions.jvmTarget = "1.8" // important, since ki is 1.8 and otherwise inlining wont work } +val versions = mapOf( + "junit5" to "5.6.0" +) + dependencies { // CPG api(project(":cpg-library")) api(project(":cpg-neo4j")) + // JUnit + testImplementation("org.junit.jupiter", "junit-jupiter-api", versions["junit5"]) + testImplementation("org.junit.jupiter", "junit-jupiter-params", versions["junit5"]) + testRuntimeOnly("org.junit.jupiter", "junit-jupiter-engine", versions["junit5"]) + + implementation("org.jetbrains.kotlinx:ki-shell:0.3.3") } diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt new file mode 100644 index 0000000000..635cf6e4c9 --- /dev/null +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.analysis + +import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ArraySubscriptionExpression +import de.fraunhofer.aisec.cpg.processing.IVisitor +import de.fraunhofer.aisec.cpg.processing.strategy.Strategy + +class OutOfBoundsCheck { + + fun run(result: TranslationResult) { + for (tu in result.translationUnits) { + tu.accept( + Strategy::AST_FORWARD, + object : IVisitor() { + fun visit(v: ArraySubscriptionExpression) { + val resolvedIndex = v.subscriptExpression.resolve() + + if (resolvedIndex is Int) { + println("Index: $resolvedIndex") + println(v) + } else { + println("Could not resolved ${v.subscriptExpression}") + } + } + } + ) + } + } +} diff --git a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt new file mode 100644 index 0000000000..61681459ac --- /dev/null +++ b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.analysis + +import de.fraunhofer.aisec.cpg.TranslationConfiguration +import de.fraunhofer.aisec.cpg.TranslationManager +import java.io.File +import org.junit.jupiter.api.Test + +class AnalysisTest { + @Test + fun testOutOfBounds() { + val config = + TranslationConfiguration.builder() + .sourceLocations(File("src/test/resources/array.cpp")) + .defaultPasses() + .defaultLanguages() + .build() + + val analyzer = TranslationManager.builder().config(config).build() + val result = analyzer.analyze().get() + + OutOfBoundsCheck().run(result) + } +} diff --git a/cpg-console/src/test/resources/array.cpp b/cpg-console/src/test/resources/array.cpp index b681f130e9..b7d3ba5350 100644 --- a/cpg-console/src/test/resources/array.cpp +++ b/cpg-console/src/test/resources/array.cpp @@ -1,3 +1,6 @@ int main() { - int a = 4 + 5; -} \ No newline at end of file + char c[4]; + int a = 4 + 1; + + char b = c[a]; // out of bounds +} From 922cefe0fc9e2a61545f1486f1836a3e41af972a Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Wed, 2 Jun 2021 12:02:27 +0200 Subject: [PATCH 05/30] byName now supports a simple nested level --- .../aisec/cpg/analysis/Extensions.kt | 53 ++++++++++++--- .../aisec/cpg/analysis/OutOfBoundsCheck.kt | 29 +++++++- .../aisec/cpg/console/AnalyzePlugin.kt | 4 +- .../aisec/cpg/console/CpgConsole.kt | 1 + .../fraunhofer/aisec/cpg/console/RunPlugin.kt | 68 +++++++++++++++++++ cpg-console/src/test/resources/Array.java | 10 +++ cpg-console/src/test/resources/array.cpp | 7 +- .../cpg/frontends/cpp/ExpressionHandler.java | 51 ++++++++++---- .../aisec/cpg/graph/DeclarationHolder.java | 4 ++ .../declarations/DeclarationSequence.java | 6 ++ .../declarations/FunctionDeclaration.java | 10 +++ .../declarations/NamespaceDeclaration.java | 2 + .../graph/declarations/RecordDeclaration.java | 12 ++++ .../TranslationUnitDeclaration.java | 2 + .../statements/DeclarationStatement.java | 2 + .../aisec/cpg/graph/statements/Statement.java | 8 +++ .../expressions/ArrayCreationExpression.java | 6 +- 17 files changed, 247 insertions(+), 28 deletions(-) create mode 100644 cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/RunPlugin.kt create mode 100644 cpg-console/src/test/resources/Array.java diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt index da3c989144..b4c4ac2158 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt @@ -25,13 +25,14 @@ */ package de.fraunhofer.aisec.cpg.analysis +import de.fraunhofer.aisec.cpg.graph.DeclarationHolder import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import kotlin.jvm.Throws fun Node?.printCode(): Unit { println(this?.code) @@ -45,18 +46,54 @@ fun Declaration.resolve(): Any? { return ValueResolver().resolveDeclaration(this) } -inline fun TranslationUnitDeclaration.byName(name: String): T? { - return this.declarations.filterIsInstance().firstOrNull { it.name == name } +@Throws(DeclarationNotFound::class) +inline fun DeclarationHolder.byName(name: String): T { + var base = this + var lookup = name + + // lets do a _very_ simple FQN lookup (TODO(oxisto): we could do this with a for-loop for + // multiple nested levels) + if (name.contains(".")) { + // take the most left one + val baseName = name.split(".")[0] + + base = + this.declarations.filterIsInstance().firstOrNull { + (it as? Node)?.name == baseName + } + ?: throw DeclarationNotFound("base not found") + lookup = name.split(".")[1] + } + + val o = base.declarations.filterIsInstance().firstOrNull() { it.name == lookup } + + return o ?: throw DeclarationNotFound("declaration with name not found or incorrect type") } -inline fun FunctionDeclaration.body(i: Int): T? { +/** + * This inline function returns the n'th statement (in AST order) as specified in T. + * + * For convenience, n defaults to zero, so that the first statement is always easy to fetch. + */ +@Throws(StatementNotFound::class) +inline fun FunctionDeclaration.body(n: Int = 0): T { return if (this.body is CompoundStatement) { - (this.body as? CompoundStatement)?.statements?.get(i) as? T + val o = (this.body as? CompoundStatement)?.statements?.filterIsInstance()?.get(n) + + if (o == null) { + throw StatementNotFound() + } else { + return o + } } else { - return if (i == 0) { - this.body as? T + if (n == 0 && this.body is T) { + this.body as T } else { - null + throw StatementNotFound() } } } + +class StatementNotFound : Exception() + +class DeclarationNotFound(message: String) : Exception(message) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt index 635cf6e4c9..17d402221e 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt @@ -27,12 +27,21 @@ package de.fraunhofer.aisec.cpg.analysis import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ArrayCreationExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.ArraySubscriptionExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression +import de.fraunhofer.aisec.cpg.helpers.Util import de.fraunhofer.aisec.cpg.processing.IVisitor import de.fraunhofer.aisec.cpg.processing.strategy.Strategy +import org.slf4j.Logger +import org.slf4j.LoggerFactory class OutOfBoundsCheck { + private val log: Logger + get() = LoggerFactory.getLogger(OutOfBoundsCheck::class.java) + fun run(result: TranslationResult) { for (tu in result.translationUnits) { tu.accept( @@ -43,7 +52,25 @@ class OutOfBoundsCheck { if (resolvedIndex is Int) { println("Index: $resolvedIndex") - println(v) + + // check, if we know that the array was initialized with a fixed length + // TODO(oxisto): it would be nice to have a helper that follows the expr + val decl = + (v.arrayExpression as? DeclaredReferenceExpression)?.refersTo as? + VariableDeclaration + (decl?.initializer as? ArrayCreationExpression)?.let { + println("Found a ArrayCreationExpression") + + val dimension = it.dimensions.first().resolve() + + if (resolvedIndex >= dimension as Int) { + Util.errorWithFileLocation( + v, + log, + "Error: this expression will run out of bounds $resolvedIndex >= $dimension" + ) + } + } } else { println("Could not resolved ${v.subscriptExpression}") } diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt index b1b4286b5d..588a91d3f3 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt @@ -69,7 +69,9 @@ class AnalyzePlugin : Plugin { " .defaultPasses()\n" + " .build()", "val analyzer = TranslationManager.builder().config(config).build()", - "val result = analyzer.analyze().get()" + "val result = analyzer.analyze().get()", + // for convenience + "val tu = result.translationUnits.first()" ) ) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt index ef580ddf62..32a27807e5 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt @@ -83,6 +83,7 @@ object CpgConsole { list += AnalyzePlugin() list += Neo4jPlugin() list += ShowCodePlugin() + list += RunPlugin() return list.listIterator() } diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/RunPlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/RunPlugin.kt new file mode 100644 index 0000000000..b1be2fdde7 --- /dev/null +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/RunPlugin.kt @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.console + +import org.jetbrains.kotlinx.ki.shell.BaseCommand +import org.jetbrains.kotlinx.ki.shell.Command +import org.jetbrains.kotlinx.ki.shell.Plugin +import org.jetbrains.kotlinx.ki.shell.Shell +import org.jetbrains.kotlinx.ki.shell.configuration.ReplConfiguration + +class RunPlugin : Plugin { + inner class Load(conf: ReplConfiguration) : BaseCommand() { + override val name: String by conf.get(default = "run") + override val short: String by conf.get(default = "r") + override val description: String = "runs an analyzer" + + override val params = "" + + override fun execute(line: String): Command.Result { + val p = line.indexOf(' ') + val path = line.substring(p + 1).trim() + + return Command.Result.RunSnippets( + listOf( + // import + "import de.fraunhofer.aisec.cpg.analysis.OutOfBoundsCheck", + // run it + "OutOfBoundsCheck().run(result)" + ) + ) + + // return Command.Result.RunSnippets(listOf(content)) + } + } + + lateinit var repl: Shell + + override fun init(repl: Shell, config: ReplConfiguration) { + this.repl = repl + + repl.registerCommand(Load(config)) + } + + override fun cleanUp() {} +} diff --git a/cpg-console/src/test/resources/Array.java b/cpg-console/src/test/resources/Array.java new file mode 100644 index 0000000000..afe96516e0 --- /dev/null +++ b/cpg-console/src/test/resources/Array.java @@ -0,0 +1,10 @@ +class Array { + public static void main(String[] args) { + char[] c = new char[4]; + + int a = 4; + int b = a + 1; + + char d = c[b]; + } +} \ No newline at end of file diff --git a/cpg-console/src/test/resources/array.cpp b/cpg-console/src/test/resources/array.cpp index b7d3ba5350..2c0c5fb25b 100644 --- a/cpg-console/src/test/resources/array.cpp +++ b/cpg-console/src/test/resources/array.cpp @@ -1,6 +1,7 @@ int main() { - char c[4]; - int a = 4 + 1; + char* c = new char[4]; + int a = 4; + int b = a + 1; - char b = c[a]; // out of bounds + char d = c[b]; } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/frontends/cpp/ExpressionHandler.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/frontends/cpp/ExpressionHandler.java index a3c2536e71..f8c93d03bf 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/frontends/cpp/ExpressionHandler.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/frontends/cpp/ExpressionHandler.java @@ -172,15 +172,48 @@ private Expression handleArraySubscriptExpression(CPPASTArraySubscriptExpression return arraySubsExpression; } - private NewExpression handleNewExpression(CPPASTNewExpression ctx) { + private Expression handleNewExpression(CPPASTNewExpression ctx) { String name = ctx.getTypeId().getDeclSpecifier().toString(); String code = ctx.getRawSignature(); // TODO: obsolete? Type t = TypeParser.createFrom(expressionTypeProxy(ctx).toString(), true); - t.reference(PointerType.PointerOrigin.ARRAY); - NewExpression newExpression = NodeBuilder.newNewExpression(code, t); + Expression expr; + + IASTInitializer init = ctx.getInitializer(); + + // we need to check, whether this is an array initialization or a single new expression + if (ctx.isArrayAllocation()) { + t.reference(PointerType.PointerOrigin.ARRAY); + + var arrayMods = + ((IASTArrayDeclarator) ctx.getTypeId().getAbstractDeclarator()).getArrayModifiers(); + + var arrayCreate = NodeBuilder.newArrayCreationExpression(code); + + arrayCreate.setType(t); + + for (var arrayMod : arrayMods) { + arrayCreate.addDimension(this.handle(arrayMod.getConstantExpression())); + } + + if (init != null) { + arrayCreate.setInitializer(this.lang.getInitializerHandler().handle(init)); + } + + expr = arrayCreate; + } else { + t.reference(PointerType.PointerOrigin.POINTER); + + var newExpression = NodeBuilder.newNewExpression(code, t); + + if (init != null) { + newExpression.setInitializer(this.lang.getInitializerHandler().handle(init)); + } + + expr = newExpression; + } // try to actually resolve the type IASTDeclSpecifier declSpecifier = ctx.getTypeId().getDeclSpecifier(); @@ -190,22 +223,16 @@ private NewExpression handleNewExpression(CPPASTNewExpression ctx) { if (binding != null && !(binding instanceof CPPScope.CPPScopeProblem)) { // update the type - newExpression.setType(TypeParser.createFrom(binding.getName(), true)); + expr.setType(TypeParser.createFrom(binding.getName(), true)); } else { log.debug( "Could not resolve binding of type {} for {}, it is probably defined somewhere externally", name, - newExpression); + expr); } } - IASTInitializer init = ctx.getInitializer(); - - if (init != null) { - newExpression.setInitializer(this.lang.getInitializerHandler().handle(init)); - } - - return newExpression; + return expr; } private ConditionalExpression handleConditionalExpression(CPPASTConditionalExpression ctx) { diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/DeclarationHolder.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/DeclarationHolder.java index e10a9dd1a8..dabb51fb0f 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/DeclarationHolder.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/DeclarationHolder.java @@ -29,6 +29,7 @@ import de.fraunhofer.aisec.cpg.graph.edge.Properties; import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge; import java.util.Collection; +import java.util.List; import org.checkerframework.checker.nullness.qual.NonNull; public interface DeclarationHolder { @@ -83,4 +84,7 @@ default void addIfNotContains( collection.add(propertyEdge); } } + + @NonNull + List getDeclarations(); } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/DeclarationSequence.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/DeclarationSequence.java index e5cb3a4100..792e96e26c 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/DeclarationSequence.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/DeclarationSequence.java @@ -31,6 +31,7 @@ import java.util.Collections; import java.util.List; import org.checkerframework.checker.nullness.qual.NonNull; +import org.jetbrains.annotations.NotNull; import org.neo4j.ogm.annotation.Relationship; /** @@ -77,4 +78,9 @@ public boolean isSingle() { public Declaration first() { return children.get(0).getEnd(); } + + @NotNull + public List getDeclarations() { + return getChildren(); + } } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.java index 1d62d790b3..2659359ee6 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.java @@ -42,6 +42,7 @@ import org.apache.commons.lang3.builder.ToStringBuilder; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; +import org.jetbrains.annotations.NotNull; import org.neo4j.ogm.annotation.Relationship; /** Represents the declaration or definition of a function. */ @@ -397,4 +398,13 @@ public void addDeclaration(@NonNull Declaration declaration) { addIfNotContains(records, (RecordDeclaration) declaration); } } + + @NotNull + public List getDeclarations() { + var list = new ArrayList(); + list.addAll(this.getParameters()); + list.addAll(this.getRecords()); + + return list; + } } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.java index 40a6af1f29..a701b31914 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/NamespaceDeclaration.java @@ -34,6 +34,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.checkerframework.checker.nullness.qual.NonNull; +import org.jetbrains.annotations.NotNull; /** * Declares the scope of a namespace and appends its own name to the current namespace-prefix to @@ -69,6 +70,7 @@ public List getNamespaces() { return Util.filterCast(declarations, NamespaceDeclaration.class); } + @NotNull public List getDeclarations() { return declarations; } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.java index 7187d2e439..064f8fea2d 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.java @@ -38,6 +38,7 @@ import org.apache.commons.lang3.builder.ToStringBuilder; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; +import org.jetbrains.annotations.NotNull; import org.neo4j.ogm.annotation.Relationship; import org.neo4j.ogm.annotation.Transient; @@ -188,6 +189,17 @@ public void removeRecord(RecordDeclaration recordDeclaration) { this.records.removeIf(propertyEdge -> propertyEdge.getEnd().equals(recordDeclaration)); } + @NotNull + public List getDeclarations() { + var list = new ArrayList(); + list.addAll(this.getFields()); + list.addAll(this.getMethods()); + list.addAll(this.getConstructors()); + list.addAll(this.getRecords()); + + return list; + } + /** * Combines both implemented interfaces and extended classes. This is most commonly what you are * looking for when looking for method call targets etc. diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.java index 1ea5f55974..eab7acbc84 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/TranslationUnitDeclaration.java @@ -36,6 +36,7 @@ import org.apache.commons.lang3.builder.ToStringBuilder; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; +import org.jetbrains.annotations.NotNull; import org.neo4j.ogm.annotation.Relationship; /** The top most declaration, representing a translation unit, for example a file. */ @@ -108,6 +109,7 @@ public IncludeDeclaration getIncludeByName(@NonNull String name) { .orElse(null); } + @NotNull @NonNull public List getDeclarations() { return unwrap(this.declarations); diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.java index f45a5c5e74..59b2933df2 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.java @@ -37,6 +37,7 @@ import java.util.Objects; import org.apache.commons.lang3.builder.ToStringBuilder; import org.checkerframework.checker.nullness.qual.NonNull; +import org.jetbrains.annotations.NotNull; import org.neo4j.ogm.annotation.Relationship; /** @@ -74,6 +75,7 @@ public T getSingleDeclarationAs(Class clazz) { return clazz.cast(this.getSingleDeclaration()); } + @NotNull @NonNull public List getDeclarations() { return unwrap(this.declarations, true); diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/Statement.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/Statement.java index a1bdf76daa..48dd3045ed 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/Statement.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/Statement.java @@ -35,6 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge; import java.util.*; import org.checkerframework.checker.nullness.qual.NonNull; +import org.jetbrains.annotations.NotNull; import org.neo4j.ogm.annotation.Relationship; /** A statement. */ @@ -91,4 +92,11 @@ public void addDeclaration(@NonNull Declaration declaration) { addIfNotContains(this.locals, (VariableDeclaration) declaration); } } + + @NotNull + public List getDeclarations() { + var list = new ArrayList(this.getLocals()); + + return list; + } } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArrayCreationExpression.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArrayCreationExpression.java index 6cf63ba3b3..57a943a7c3 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArrayCreationExpression.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArrayCreationExpression.java @@ -48,7 +48,7 @@ public class ArrayCreationExpression extends Expression implements TypeListener * {@link #dimensions} or an initializer. */ @SubGraph("AST") - private InitializerListExpression initializer; + private Expression initializer; /** * Specifies the dimensions of the array that is to be created. Many languages, such as Java, @@ -59,11 +59,11 @@ public class ArrayCreationExpression extends Expression implements TypeListener @SubGraph("AST") private List> dimensions = new ArrayList<>(); - public InitializerListExpression getInitializer() { + public Expression getInitializer() { return initializer; } - public void setInitializer(InitializerListExpression initializer) { + public void setInitializer(Expression initializer) { if (this.initializer != null) { this.initializer.unregisterTypeListener(this); this.removePrevDFG(initializer); From 0d4f8babcf0905fe53db00ae4753af2567b81568 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Wed, 2 Jun 2021 12:25:57 +0200 Subject: [PATCH 06/30] Shorter, more concise toString() output for console --- .../aisec/cpg/console/CpgConsole.kt | 4 ++++ .../de/fraunhofer/aisec/cpg/graph/Node.kt | 4 +--- .../cpg/graph/declarations/Declaration.java | 9 +------- .../declarations/FunctionDeclaration.java | 8 ++++--- .../graph/declarations/ValueDeclaration.java | 4 +++- .../aisec/cpg/graph/types/IncompleteType.java | 15 ------------- .../aisec/cpg/graph/types/ObjectType.java | 21 ------------------- .../aisec/cpg/graph/types/PointerType.java | 17 --------------- .../aisec/cpg/graph/types/Type.java | 15 ++++--------- 9 files changed, 18 insertions(+), 79 deletions(-) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt index 32a27807e5..4de399545b 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt @@ -25,12 +25,14 @@ */ package de.fraunhofer.aisec.cpg.console +import de.fraunhofer.aisec.cpg.graph.Node import kotlin.script.experimental.api.ScriptCompilationConfiguration import kotlin.script.experimental.api.ScriptEvaluationConfiguration import kotlin.script.experimental.jvm.baseClassLoader import kotlin.script.experimental.jvm.defaultJvmScriptingHostConfiguration import kotlin.script.experimental.jvm.dependenciesFromClassloader import kotlin.script.experimental.jvm.jvm +import org.apache.commons.lang3.builder.ToStringStyle import org.jetbrains.kotlinx.ki.shell.KotlinShell import org.jetbrains.kotlinx.ki.shell.Plugin import org.jetbrains.kotlinx.ki.shell.Shell @@ -41,6 +43,8 @@ import org.jetbrains.kotlinx.ki.shell.configuration.ReplConfigurationBase object CpgConsole { @JvmStatic fun main(args: Array) { + Node.TO_STRING_STYLE = ToStringStyle.JSON_STYLE + val repl = Shell( configuration(), diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt index b1d891171f..9a0ca0c11e 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -222,10 +222,8 @@ open class Node : IVisitable, Persistable { override fun toString(): String { return ToStringBuilder(this, TO_STRING_STYLE) - .append("id", id) .append("name", name) .append("location", location) - .append("argumentIndex", argumentIndex) .toString() } @@ -253,7 +251,7 @@ open class Node : IVisitable, Persistable { } companion object { - @JvmField val TO_STRING_STYLE: ToStringStyle = ToStringStyle.SHORT_PREFIX_STYLE + @JvmField var TO_STRING_STYLE: ToStringStyle = ToStringStyle.SHORT_PREFIX_STYLE protected val log: Logger = LoggerFactory.getLogger(Node::class.java) diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.java index efdce9947f..b470913bf3 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/Declaration.java @@ -26,7 +26,6 @@ package de.fraunhofer.aisec.cpg.graph.declarations; import de.fraunhofer.aisec.cpg.graph.Node; -import org.apache.commons.lang3.builder.ToStringBuilder; /** * Represents a single declaration or definition, i.e. of a variable ({@link VariableDeclaration}) @@ -40,10 +39,4 @@ */ // TODO: expressionRefersToDeclaration definition and declaration nodes and introduce a field if its // declaration only -public class Declaration extends Node { - - @Override - public String toString() { - return new ToStringBuilder(this, Node.TO_STRING_STYLE).toString(); - } -} +public class Declaration extends Node {} diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.java index 2659359ee6..c4a78d16df 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.java @@ -320,17 +320,19 @@ public Optional getVariableDeclarationByName(String name) { return Optional.empty(); } + @NotNull @Override public String toString() { return new ToStringBuilder(this, Node.TO_STRING_STYLE) .appendSuper(super.toString()) - .append("type", type) + // .append("type", type) .append( "parameters", parameters.stream() .map(PropertyEdge::getEnd) - .map(ParamVariableDeclaration::getName) - .collect(Collectors.joining(", "))) + /*.map(ParamVariableDeclaration::toString) + .collect(Collectors.joining(", ")))*/ + .collect(Collectors.toList())) .toString(); } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.java index 5164e25a35..0bb02b4652 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.java @@ -38,6 +38,7 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang3.builder.ToStringBuilder; +import org.jetbrains.annotations.NotNull; import org.neo4j.ogm.annotation.Transient; /** A declaration who has a type. */ @@ -182,12 +183,13 @@ public void refreshType() { }); } + @NotNull @Override public String toString() { return new ToStringBuilder(this, Node.TO_STRING_STYLE) .appendSuper(super.toString()) .append("type", type) - .append("possibleSubTypes", possibleSubTypes) + // .append("possibleSubTypes", possibleSubTypes) .toString(); } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/IncompleteType.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/IncompleteType.java index 382e9c78f4..3b0ead459c 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/IncompleteType.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/IncompleteType.java @@ -72,19 +72,4 @@ public int hashCode() { return Objects.hash(super.hashCode()); } - - @Override - public String toString() { - return "IncompleteType{" - + "typeName='" - + getName() - + '\'' - + ", storage=" - + this.getStorage() - + ", qualifier=" - + this.getQualifier() - + ", origin=" - + this.getTypeOrigin() - + '}'; - } } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/ObjectType.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/ObjectType.java index a6f8aa5581..025a856615 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/ObjectType.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/ObjectType.java @@ -157,25 +157,4 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(super.hashCode(), generics, modifier, primitive); } - - @Override - public String toString() { - return "ObjectType{" - + "generics=" - + generics - + ", typeName='" - + getName() - + '\'' - + ", storage=" - + this.getStorage() - + ", qualifier=" - + this.getQualifier() - + ", modifier=" - + modifier - + ", primitive=" - + primitive - + ", origin=" - + this.getTypeOrigin() - + '}'; - } } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/PointerType.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/PointerType.java index 28f93fc1f3..bc950156ff 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/PointerType.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/PointerType.java @@ -158,21 +158,4 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(super.hashCode(), elementType); } - - @Override - public String toString() { - return "PointerType{" - + "elementType=" - + elementType - + ", typeName='" - + getName() - + '\'' - + ", storage=" - + this.getStorage() - + ", qualifier=" - + this.getQualifier() - + ", origin=" - + this.getTypeOrigin() - + '}'; - } } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/Type.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/Type.java index aee9fec8c0..7afeb9f739 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/Type.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/Type.java @@ -27,8 +27,10 @@ import de.fraunhofer.aisec.cpg.graph.Node; import java.util.*; +import org.apache.commons.lang3.builder.ToStringBuilder; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; +import org.jetbrains.annotations.NotNull; import org.neo4j.ogm.annotation.Relationship; import org.neo4j.ogm.annotation.typeconversion.Convert; @@ -347,18 +349,9 @@ public int hashCode() { return Objects.hash(getName(), storage, qualifier); } + @NotNull @Override public String toString() { - return "Type{" - + "typeName='" - + getName() - + '\'' - + ", storage=" - + storage - + ", qualifier=" - + qualifier - + ", origin=" - + origin - + '}'; + return new ToStringBuilder(this, TO_STRING_STYLE).append("name", getName()).toString(); } } From b3bb438fd8f04bec84561aa746f5214d26a786b0 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Wed, 2 Jun 2021 12:28:50 +0200 Subject: [PATCH 07/30] Fixed test --- .../aisec/cpg/frontends/java/JavaLanguageFrontendTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpg-library/src/test/java/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontendTest.java b/cpg-library/src/test/java/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontendTest.java index 641adb1f20..fb5ab4b75a 100644 --- a/cpg-library/src/test/java/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontendTest.java +++ b/cpg-library/src/test/java/de/fraunhofer/aisec/cpg/frontends/java/JavaLanguageFrontendTest.java @@ -419,7 +419,7 @@ void testArrays() throws Exception { assertNotNull(ace); // which has a initializer list (1 entry) - InitializerListExpression ile = ace.getInitializer(); + InitializerListExpression ile = (InitializerListExpression) ace.getInitializer(); assertNotNull(ile); assertEquals(1, ile.getInitializers().size()); From 45fc070355333c674b2ff464c215e9b4266892b8 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Wed, 2 Jun 2021 13:34:21 +0200 Subject: [PATCH 08/30] Simple Nullpointer detection --- .../aisec/cpg/analysis/Extensions.kt | 15 +++ .../aisec/cpg/analysis/NullPointerCheck.kt | 93 +++++++++++++++++++ .../aisec/cpg/console/AnalyzePlugin.kt | 1 + .../fraunhofer/aisec/cpg/console/RunPlugin.kt | 4 +- .../aisec/cpg/analysis/AnalysisTest.kt | 15 +++ cpg-console/src/test/resources/Array.java | 13 +++ .../cpg/frontends/java/ExpressionHandler.java | 3 +- .../de/fraunhofer/aisec/cpg/graph/HasBase.kt | 34 +++++++ .../de/fraunhofer/aisec/cpg/graph/Node.kt | 11 ++- .../aisec/cpg/graph/NodeBuilder.java | 2 +- .../declarations/FunctionDeclaration.java | 9 +- .../statements/DeclarationStatement.java | 3 +- .../ArraySubscriptionExpression.java | 10 +- .../expressions/CallExpression.java | 10 +- .../graph/statements/expressions/Literal.java | 5 +- .../expressions/MemberExpression.java | 3 +- .../aisec/cpg/passes/CallResolver.java | 7 +- 17 files changed, 213 insertions(+), 25 deletions(-) create mode 100644 cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt create mode 100644 cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/HasBase.kt diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt index b4c4ac2158..c3b12093e5 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt @@ -32,10 +32,25 @@ import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation.locationLink import kotlin.jvm.Throws fun Node?.printCode(): Unit { + val header = "--- ${locationLink(this?.location)} ---" + + println(header) println(this?.code) + println("-".repeat(header.length)) +} + +fun MutableSet.printCode(): Unit { + val it = this.iterator() + + while (it.hasNext()) { + val next = it.next() + next.printCode() + println("") + } } fun Expression.resolve(): Any? { diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt new file mode 100644 index 0000000000..0f72d120cd --- /dev/null +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.analysis + +import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.graph.HasBase +import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.graph.declarations.Declaration +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.helpers.Util +import de.fraunhofer.aisec.cpg.processing.IVisitor +import de.fraunhofer.aisec.cpg.processing.strategy.Strategy +import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation.locationLink +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class NullPointerCheck { + private val log: Logger + get() = LoggerFactory.getLogger(OutOfBoundsCheck::class.java) + + fun run(result: TranslationResult) { + for (tu in result.translationUnits) { + tu.accept( + Strategy::AST_FORWARD, + object : IVisitor() { + fun visit(v: MemberCallExpression) { + handleHasBase(v) + } + + fun visit(v: CallExpression) { + handleHasBase(v) + } + + fun visit(v: MemberExpression) { + handleHasBase(v) + } + + fun visit(v: ArraySubscriptionExpression) { + handleHasBase(v) + } + } + ) + } + } + + fun handleHasBase(node: HasBase) { + // check for all incoming DFG branches + node.base.prevDFG.forEach { + it + var resolved: Any? = + null // TODO(oxisto): make sure that we distinguish between null and not-resolved + if (it is Expression) { + // try to resolve them + resolved = it.resolve() + } else if (it is Declaration) { + // try to resolve them + resolved = it.resolve() + } + + if (resolved == null) { + // TODO(oxisto): would be nice to have the complete resolution path + Util.errorWithFileLocation( + node as Node, + log, + "Null pointer detected in branch. Relevant point that set it was here: ${locationLink(it.location)}" + ) + } + } + } +} diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt index 588a91d3f3..55042529c0 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt @@ -52,6 +52,7 @@ class AnalyzePlugin : Plugin { "import de.fraunhofer.aisec.cpg.graph.*", "import de.fraunhofer.aisec.cpg.graph.declarations.*", "import de.fraunhofer.aisec.cpg.graph.statements.*", + "import de.fraunhofer.aisec.cpg.graph.statements.expressions.*", // helper builtins "import de.fraunhofer.aisec.cpg.analysis.resolve", "import de.fraunhofer.aisec.cpg.analysis.byName", diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/RunPlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/RunPlugin.kt index b1be2fdde7..65c3c40778 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/RunPlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/RunPlugin.kt @@ -47,8 +47,10 @@ class RunPlugin : Plugin { listOf( // import "import de.fraunhofer.aisec.cpg.analysis.OutOfBoundsCheck", + "import de.fraunhofer.aisec.cpg.analysis.NullPointerCheck", // run it - "OutOfBoundsCheck().run(result)" + "OutOfBoundsCheck().run(result)", + "NullPointerCheck().run(result)" ) ) diff --git a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt index 61681459ac..f33fcb584e 100644 --- a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt +++ b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt @@ -45,4 +45,19 @@ class AnalysisTest { OutOfBoundsCheck().run(result) } + + @Test + fun testNullPointer() { + val config = + TranslationConfiguration.builder() + .sourceLocations(File("src/test/resources/Array.java")) + .defaultPasses() + .defaultLanguages() + .build() + + val analyzer = TranslationManager.builder().config(config).build() + val result = analyzer.analyze().get() + + NullPointerCheck().run(result) + } } diff --git a/cpg-console/src/test/resources/Array.java b/cpg-console/src/test/resources/Array.java index afe96516e0..cba249006f 100644 --- a/cpg-console/src/test/resources/Array.java +++ b/cpg-console/src/test/resources/Array.java @@ -6,5 +6,18 @@ public static void main(String[] args) { int b = a + 1; char d = c[b]; + + // obviously null + AnotherObject obj = null; + + // lets make it a little bit tricky at least + obj = AnotherObject(); + + if(something) { + // whoops, overriden with null again + obj = null; + } + + obj.doSomething(); } } \ No newline at end of file diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.java index dac805749b..76adbbe4df 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.java @@ -702,7 +702,8 @@ private CallExpression handleMethodCallExpression(Expression expr) { scopeName = scope.toString(); } - Statement base = handle(scope); + de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression base = + (de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression) handle(scope); // If the base directly refers to a record, then this is a static call if (base instanceof DeclaredReferenceExpression diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/HasBase.kt b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/HasBase.kt new file mode 100644 index 0000000000..d84adfbb3a --- /dev/null +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/HasBase.kt @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.graph + +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression + +/** Specifies that a certain node has a base on which it executes an operation. */ +interface HasBase { + + val base: Expression +} diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt index 9a0ca0c11e..0d011be94e 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -221,10 +221,13 @@ open class Node : IVisitable, Persistable { } override fun toString(): String { - return ToStringBuilder(this, TO_STRING_STYLE) - .append("name", name) - .append("location", location) - .toString() + val builder = ToStringBuilder(this, TO_STRING_STYLE) + + if (name != "") { + builder.append("name", name) + } + + return builder.append("location", location).toString() } override fun equals(other: Any?): Boolean { diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/NodeBuilder.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/NodeBuilder.java index 6403601777..9e6c8d49a3 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/NodeBuilder.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/NodeBuilder.java @@ -238,7 +238,7 @@ public static ExpressionList newExpressionList(String code) { } public static CallExpression newMemberCallExpression( - String name, String fqn, Node base, Node member, String operatorCode, String code) { + String name, String fqn, Expression base, Node member, String operatorCode, String code) { MemberCallExpression node = new MemberCallExpression(); node.setName(name); node.setBase(base); diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.java index c4a78d16df..28f8340449 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/FunctionDeclaration.java @@ -325,14 +325,7 @@ public Optional getVariableDeclarationByName(String name) { public String toString() { return new ToStringBuilder(this, Node.TO_STRING_STYLE) .appendSuper(super.toString()) - // .append("type", type) - .append( - "parameters", - parameters.stream() - .map(PropertyEdge::getEnd) - /*.map(ParamVariableDeclaration::toString) - .collect(Collectors.joining(", ")))*/ - .collect(Collectors.toList())) + .append("parameters", this.getParameters()) .toString(); } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.java index 59b2933df2..d1e1125d04 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.java @@ -95,11 +95,12 @@ public void addToPropertyEdgeDeclaration(@NonNull Declaration declaration) { this.declarations.add(propertyEdge); } + @NotNull @Override public String toString() { return new ToStringBuilder(this, Node.TO_STRING_STYLE) .appendSuper(super.toString()) - .append("declarations", declarations) + .append("declarations", this.getDeclarations()) .toString(); } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArraySubscriptionExpression.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArraySubscriptionExpression.java index 82ca80c846..558564964c 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArraySubscriptionExpression.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/ArraySubscriptionExpression.java @@ -25,6 +25,7 @@ */ package de.fraunhofer.aisec.cpg.graph.statements.expressions; +import de.fraunhofer.aisec.cpg.graph.HasBase; import de.fraunhofer.aisec.cpg.graph.HasType; import de.fraunhofer.aisec.cpg.graph.HasType.TypeListener; import de.fraunhofer.aisec.cpg.graph.SubGraph; @@ -33,13 +34,14 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; +import org.jetbrains.annotations.NotNull; /** * Represents the Subscription or access of an array of the form array[index], where * both array and index are of type {@link Expression}. CPP can overload * operators thus changing semantics of array access. */ -public class ArraySubscriptionExpression extends Expression implements TypeListener { +public class ArraySubscriptionExpression extends Expression implements TypeListener, HasBase { @SubGraph("AST") private Expression arrayExpression; @@ -107,4 +109,10 @@ public boolean equals(Object o) { public int hashCode() { return super.hashCode(); } + + @NotNull + @Override + public Expression getBase() { + return this.arrayExpression; + } } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.java index 5b9fbba277..1ea1ba7c4e 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.java @@ -43,7 +43,7 @@ * An expression, which calls another function. It has a list of arguments (list of {@link * Expression}s) and is connected via the INVOKES edge to its {@link FunctionDeclaration}. */ -public class CallExpression extends Expression implements TypeListener { +public class CallExpression extends Expression implements TypeListener, HasBase { /** * Connection to its {@link FunctionDeclaration}. This will be populated by the {@link @@ -63,15 +63,15 @@ public class CallExpression extends Expression implements TypeListener { * the original AST, but we treat it as such for better consistency */ @SubGraph("AST") - private Node base; + private Expression base; private String fqn; - public Node getBase() { - return base; + public Expression getBase() { + return (Expression) base; } - public void setBase(Node base) { + public void setBase(Expression base) { if (this.base instanceof HasType) { ((HasType) this.base).unregisterTypeListener(this); } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/Literal.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/Literal.java index dee9bc0587..ec7b9c6479 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/Literal.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/Literal.java @@ -49,7 +49,10 @@ public void setValue(T value) { @Override public String toString() { - return new ToStringBuilder(this, Node.TO_STRING_STYLE).append("value", value).toString(); + return new ToStringBuilder(this, Node.TO_STRING_STYLE) + .appendSuper(super.toString()) + .append("value", value) + .toString(); } @Override diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.java index f2c14d1bb8..6c78338831 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/MemberExpression.java @@ -25,6 +25,7 @@ */ package de.fraunhofer.aisec.cpg.graph.statements.expressions; +import de.fraunhofer.aisec.cpg.graph.HasBase; import de.fraunhofer.aisec.cpg.graph.Node; import de.fraunhofer.aisec.cpg.graph.SubGraph; import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration; @@ -35,7 +36,7 @@ /** * Represents access to a field of a {@link RecordDeclaration}, such as obj.property. */ -public class MemberExpression extends DeclaredReferenceExpression { +public class MemberExpression extends DeclaredReferenceExpression implements HasBase { @SubGraph("AST") @NonNull diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/passes/CallResolver.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/passes/CallResolver.java index 0dfa563327..231bb29703 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/passes/CallResolver.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/passes/CallResolver.java @@ -669,7 +669,12 @@ private void handleMethodCall(RecordDeclaration curClass, CallExpression call) { if (curClass != null && !(call instanceof MemberCallExpression || call instanceof StaticCallExpression)) { - call.setBase(curClass.getThis()); + // COMMENT(oxisto) if we run into this condition, parsing has gone wrong on some other parts. + // not sure if we + // can heal this here, so I deactivated this line. + + // TODO(oxisto): this should anyway be replaced by the new receiver field + // call.setBase(curClass.getThis()); } createMethodDummies(invocationCandidates, possibleContainingTypes, call); From cf0f943e5afe9b4b67b0b2a3740742dbaf10cf88 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Wed, 2 Jun 2021 13:55:09 +0200 Subject: [PATCH 09/30] Fixed go tests --- cpg-library/src/main/golang/expressions.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cpg-library/src/main/golang/expressions.go b/cpg-library/src/main/golang/expressions.go index 8684803457..a3a59e97b2 100644 --- a/cpg-library/src/main/golang/expressions.go +++ b/cpg-library/src/main/golang/expressions.go @@ -195,8 +195,8 @@ func (c *MemberCallExpression) SetFqn(s string) { (*CallExpression)(c).SetFqn(s) } -func (m *MemberCallExpression) SetBase(n *Node) { - (*jnigi.ObjectRef)(m).SetField(env, "base", (*jnigi.ObjectRef)(n).Cast("de/fraunhofer/aisec/cpg/graph/Node")) +func (m *MemberCallExpression) SetBase(e *Expression) { + (*jnigi.ObjectRef)(m).SetField(env, "base", (*jnigi.ObjectRef)(e).Cast("de/fraunhofer/aisec/cpg/graph/statements/expressions/Expression")) } func (m *MemberCallExpression) SetMember(n *Node) { @@ -211,14 +211,13 @@ func (m *MemberExpression) SetBase(e *Expression) { (*jnigi.ObjectRef)(m).SetField(env, "base", (*jnigi.ObjectRef)(e).Cast("de/fraunhofer/aisec/cpg/graph/statements/expressions/Expression")) } -func (m *MemberExpression) GetBase() *Node { +func (m *MemberExpression) GetBase() *Expression { i, err := (*jnigi.ObjectRef)(m).GetField(env, "base", jnigi.ObjectType("de/fraunhofer/aisec/cpg/graph/statements/expressions/Expression")) if err != nil { log.Fatal(err) - } - return (*Node)(i.(*jnigi.ObjectRef)) + return (*Expression)(i.(*jnigi.ObjectRef)) } func (r *DeclaredReferenceExpression) Expression() *Expression { From 1e9f25252e2bb53310b71f82484779b6e8e24813 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Wed, 2 Jun 2021 17:56:43 +0200 Subject: [PATCH 10/30] Code cleanup. Added CouldNotResolve class --- .../fraunhofer/aisec/cpg/analysis/Extensions.kt | 6 +++--- .../aisec/cpg/analysis/NullPointerCheck.kt | 4 +--- .../aisec/cpg/analysis/ValueResolver.kt | 8 ++++++-- cpg-console/src/test/resources/Array.java | 6 ++++-- .../graph/statements/DeclarationStatement.java | 1 + .../statements/expressions/CallExpression.java | 15 +++++++++------ .../ExplicitConstructorInvocation.java | 2 ++ 7 files changed, 26 insertions(+), 16 deletions(-) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt index c3b12093e5..ebed7314f1 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt @@ -66,8 +66,8 @@ inline fun DeclarationHolder.byName(name: String): T { var base = this var lookup = name - // lets do a _very_ simple FQN lookup (TODO(oxisto): we could do this with a for-loop for - // multiple nested levels) + // lets do a _very_ simple FQN lookup + // TODO(oxisto): we could do this with a for-loop for multiple nested levels if (name.contains(".")) { // take the most left one val baseName = name.split(".")[0] @@ -80,7 +80,7 @@ inline fun DeclarationHolder.byName(name: String): T { lookup = name.split(".")[1] } - val o = base.declarations.filterIsInstance().firstOrNull() { it.name == lookup } + val o = base.declarations.filterIsInstance().firstOrNull { it.name == lookup } return o ?: throw DeclarationNotFound("declaration with name not found or incorrect type") } diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt index 0f72d120cd..743d5c8c30 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt @@ -69,9 +69,7 @@ class NullPointerCheck { fun handleHasBase(node: HasBase) { // check for all incoming DFG branches node.base.prevDFG.forEach { - it - var resolved: Any? = - null // TODO(oxisto): make sure that we distinguish between null and not-resolved + var resolved: Any? = CouldNotResolve() if (it is Expression) { // try to resolve them resolved = it.resolve() diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueResolver.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueResolver.kt index 31fd6c7c8b..0f676046bf 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueResolver.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueResolver.kt @@ -31,6 +31,8 @@ import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +class CouldNotResolve + /** * The value resolver tries to resolve the value of an [Expression] basically by following edges * until we reach a [Literal]. @@ -50,7 +52,7 @@ class ValueResolver( if (node != null) { "{${node.name}}" } else { - null + CouldNotResolve() } } ) { @@ -58,7 +60,9 @@ class ValueResolver( fun resolveDeclaration(decl: Declaration?): Any? { when (decl) { is VariableDeclaration -> return resolve(decl.initializer) - is FieldDeclaration -> return resolve(decl.initializer) + is FieldDeclaration -> { + return resolve(decl.initializer) + } } return cannotResolve(decl, this) diff --git a/cpg-console/src/test/resources/Array.java b/cpg-console/src/test/resources/Array.java index cba249006f..61510b6e72 100644 --- a/cpg-console/src/test/resources/Array.java +++ b/cpg-console/src/test/resources/Array.java @@ -11,11 +11,13 @@ public static void main(String[] args) { AnotherObject obj = null; // lets make it a little bit tricky at least - obj = AnotherObject(); + obj = something; if(something) { + AnotherObject yetAnotherObject = null; + // whoops, overriden with null again - obj = null; + obj = yetAnotherObject; } obj.doSomething(); diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.java index d1e1125d04..4e282eea8a 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/DeclarationStatement.java @@ -96,6 +96,7 @@ public void addToPropertyEdgeDeclaration(@NonNull Declaration declaration) { } @NotNull + @NonNull @Override public String toString() { return new ToStringBuilder(this, Node.TO_STRING_STYLE) diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.java index 1ea1ba7c4e..41f0df1207 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/CallExpression.java @@ -37,6 +37,7 @@ import org.apache.commons.lang3.builder.ToStringBuilder; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; +import org.jetbrains.annotations.NotNull; import org.neo4j.ogm.annotation.Relationship; /** @@ -67,17 +68,18 @@ public class CallExpression extends Expression implements TypeListener, HasBase private String fqn; + @NotNull public Expression getBase() { - return (Expression) base; + return base; } public void setBase(Expression base) { - if (this.base instanceof HasType) { - ((HasType) this.base).unregisterTypeListener(this); + if (this.base != null) { + this.base.unregisterTypeListener(this); } this.base = base; - if (base instanceof HasType) { - ((HasType) base).registerTypeListener(this); + if (base != null) { + base.registerTypeListener(this); } } @@ -160,7 +162,7 @@ public void typeChanged(HasType src, HasType root, Type oldType) { Type previous = this.type; List types = invokes.stream() - .map(pe -> pe.getEnd()) + .map(PropertyEdge::getEnd) .map(FunctionDeclaration::getType) .filter(Objects::nonNull) .collect(Collectors.toList()); @@ -188,6 +190,7 @@ public void possibleSubTypesChanged(HasType src, HasType root, Set oldSubT } } + @NotNull @Override public String toString() { return new ToStringBuilder(this, Node.TO_STRING_STYLE) diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExplicitConstructorInvocation.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExplicitConstructorInvocation.java index 2965892a84..d18907ce2d 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExplicitConstructorInvocation.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/ExplicitConstructorInvocation.java @@ -28,6 +28,7 @@ import de.fraunhofer.aisec.cpg.graph.Node; import java.util.Objects; import org.apache.commons.lang3.builder.ToStringBuilder; +import org.jetbrains.annotations.NotNull; public class ExplicitConstructorInvocation extends CallExpression { @@ -41,6 +42,7 @@ public void setContainingClass(String containingClass) { this.containingClass = containingClass; } + @NotNull @Override public String toString() { return new ToStringBuilder(this, Node.TO_STRING_STYLE) From 07cefc1114303e9b3a4216f7045a99c9fe502910 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Wed, 2 Jun 2021 19:20:53 +0200 Subject: [PATCH 11/30] nice formatting of NPE --- cpg-console/build.gradle.kts | 1 + .../aisec/cpg/analysis/Extensions.kt | 24 +++++ .../aisec/cpg/analysis/NullPointerCheck.kt | 89 +++++++++++++++---- .../aisec/cpg/analysis/ValueResolver.kt | 4 + .../aisec/cpg/sarif/PhysicalLocation.java | 2 +- 5 files changed, 101 insertions(+), 19 deletions(-) diff --git a/cpg-console/build.gradle.kts b/cpg-console/build.gradle.kts index 35f8c22c43..4b7b983bb1 100644 --- a/cpg-console/build.gradle.kts +++ b/cpg-console/build.gradle.kts @@ -57,6 +57,7 @@ dependencies { testImplementation("org.junit.jupiter", "junit-jupiter-params", versions["junit5"]) testRuntimeOnly("org.junit.jupiter", "junit-jupiter-engine", versions["junit5"]) + implementation("org.jline:jline:3.20.0") implementation("org.jetbrains.kotlinx:ki-shell:0.3.3") } diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt index ebed7314f1..fb9391deb4 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt @@ -29,6 +29,7 @@ import de.fraunhofer.aisec.cpg.graph.DeclarationHolder import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression @@ -112,3 +113,26 @@ inline fun FunctionDeclaration.body(n: Int = 0): T { class StatementNotFound : Exception() class DeclarationNotFound(message: String) : Exception(message) + +fun Node.followPrevEOG(predicate: (PropertyEdge<*>) -> Boolean): List>? { + val path = mutableListOf>() + + for (edge in this.prevEOGEdges) { + val source = edge.start + + path.add(edge) + + if (predicate(edge)) { + return path + } + + val subPath = source.followPrevEOG(predicate) + if (subPath != null) { + path.addAll(subPath) + + return path + } + } + + return null +} diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt index 743d5c8c30..480399f8e9 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt @@ -23,17 +23,23 @@ * \______/ \__| \______/ * */ +@file:Suppress("JAVA_MODULE_DOES_NOT_EXPORT_PACKAGE") + package de.fraunhofer.aisec.cpg.analysis import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.HasBase import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.Declaration +import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import de.fraunhofer.aisec.cpg.graph.statements.IfStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.* -import de.fraunhofer.aisec.cpg.helpers.Util import de.fraunhofer.aisec.cpg.processing.IVisitor import de.fraunhofer.aisec.cpg.processing.strategy.Strategy import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation.locationLink +import org.jline.utils.AttributedString +import org.jline.utils.AttributedStringBuilder +import org.jline.utils.AttributedStyle.* import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -67,25 +73,72 @@ class NullPointerCheck { } fun handleHasBase(node: HasBase) { - // check for all incoming DFG branches - node.base.prevDFG.forEach { - var resolved: Any? = CouldNotResolve() - if (it is Expression) { - // try to resolve them - resolved = it.resolve() - } else if (it is Declaration) { - // try to resolve them - resolved = it.resolve() - } + try { + // check for all incoming DFG branches + node.base.prevDFG.forEach { + var resolved: Any? = CouldNotResolve() + val resolver = ValueResolver() + if (it is Expression) { + // try to resolve them + resolved = resolver.resolve(it) + } else if (it is Declaration) { + // try to resolve them + resolved = resolver.resolveDeclaration(it) + } + + if (resolved == null) { + println("") + val sb = AttributedStringBuilder() + sb.append("--- FINDING: Null pointer detected in ") + sb.append(node.javaClass.simpleName, DEFAULT.foreground(GREEN)) + sb.append(" when accessing base ") + sb.append(node.base.name, DEFAULT.foreground(CYAN)) + sb.append(" ---") - if (resolved == null) { - // TODO(oxisto): would be nice to have the complete resolution path - Util.errorWithFileLocation( - node as Node, - log, - "Null pointer detected in branch. Relevant point that set it was here: ${locationLink(it.location)}" - ) + val header = sb.toAnsi() + + println(header) + println( + "${AttributedString(locationLink((node as Node).location), DEFAULT.foreground(BLUE or BRIGHT)).toAnsi()}: ${(node as Node).code}" + ) + println("") + println( + "The following path was discovered that leads to ${AttributedString(node.base.name, DEFAULT.foreground(CYAN)).toAnsi()} being null:" + ) + for (p in resolver.path) { + + println( + "${AttributedString(locationLink(p.location), DEFAULT.foreground(BLUE or BRIGHT)).toAnsi()}: ${p.code}" + ) + } + + val path = + it.followPrevEOG { edge -> + return@followPrevEOG when (edge.start) { + is IfStatement -> { + true + } + is FunctionDeclaration -> { + true + } + else -> false + } + } + + val last = path?.last()?.start + + if (last is IfStatement) { + println() + println( + "Branch depends on ${AttributedString("IfStatement", DEFAULT.foreground(GREEN)).toAnsi()} with condition ${AttributedString(last.condition.code, DEFAULT.foreground(CYAN)).toAnsi()} in ${AttributedString(locationLink(last.location), DEFAULT.foreground(BLUE or BRIGHT)).toAnsi()}" + ) + } + + println("-".repeat(sb.toString().length)) + } } + } catch (ex: Throwable) { + log.error("Exception while running check: {}", ex) } } } diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueResolver.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueResolver.kt index 0f676046bf..f07fb7f02c 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueResolver.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueResolver.kt @@ -56,8 +56,10 @@ class ValueResolver( } } ) { + val path: MutableList = mutableListOf() fun resolveDeclaration(decl: Declaration?): Any? { + decl?.let { this.path += it } when (decl) { is VariableDeclaration -> return resolve(decl.initializer) is FieldDeclaration -> { @@ -70,6 +72,8 @@ class ValueResolver( /** Tries to resolve this expression. Anything can happen. */ fun resolve(expr: Expression?): Any? { + expr?.let { this.path += it } + when (expr) { is Literal<*> -> { return expr.value diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/sarif/PhysicalLocation.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/sarif/PhysicalLocation.java index 500ada4444..99e325539b 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/sarif/PhysicalLocation.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/sarif/PhysicalLocation.java @@ -36,7 +36,7 @@ public class PhysicalLocation { @NonNull public static String locationLink(@Nullable PhysicalLocation location) { if (location != null) { - return location.getArtifactLocation().getUri() + return location.getArtifactLocation().getUri().getPath() + ":" + location.getRegion().getStartLine() + ":" From 8ac3465af8e062f19bcd1d3784491050064bcb50 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Wed, 2 Jun 2021 21:06:04 +0200 Subject: [PATCH 12/30] First attempt at code coloring --- .../aisec/cpg/analysis/Extensions.kt | 107 ++++++++++++++++++ .../aisec/cpg/analysis/AnalysisTest.kt | 32 ++++++ .../cpg/frontends/java/ExpressionHandler.java | 3 +- 3 files changed, 141 insertions(+), 1 deletion(-) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt index fb9391deb4..74b638b5fa 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt @@ -29,12 +29,21 @@ import de.fraunhofer.aisec.cpg.graph.DeclarationHolder import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement +import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.Statement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression +import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation.locationLink +import de.fraunhofer.aisec.cpg.sarif.Region import kotlin.jvm.Throws +import org.jline.utils.AttributedString +import org.jline.utils.AttributedStyle.CYAN +import org.jline.utils.AttributedStyle.DEFAULT fun Node?.printCode(): Unit { val header = "--- ${locationLink(this?.location)} ---" @@ -136,3 +145,101 @@ fun Node.followPrevEOG(predicate: (PropertyEdge<*>) -> Boolean): List() + + val fancies = getFanciesFor(this, this) + + for (fancy in fancies) { + if (fancy.first) { + val region = fancy.third + + fancy.second.let { + val ansi = it.toAnsi() + + var extraCharsInLine = extraCharsInLines.getOrDefault(region.startLine, 0) + + // the amount of extra chars introduced by the ANSI control chars + val extraChars = ansi.length - it.length + + // the current line we want to tackle + val line = lines[region.startLine] + + // everything before the thing we want to replace. add the extra chars in line to + // correct for ANSI chars introduced before us + val before = line.substring(0, region.startColumn + extraCharsInLine) + + // everything after the thing we want to replace. add the extra chars in line to + // correct for ANSI chars introduced before us + val after = line.substring(region.endColumn + extraCharsInLine) + + // reconstruct the line + lines[region.startLine] = before + ansi + after + + // update extra chars in line + extraCharsInLine += extraChars + + // store it + extraCharsInLines.put(region.startLine, extraCharsInLine) + } + } + } + + return lines.joinToString("\n") +} + +fun getFanciesFor(parent: Node, node: Node): List> { + val list = mutableListOf>() + + when (node) { + is MemberCallExpression -> { + // only color the member + list.addAll(getFanciesFor(node, node.member)) + + return list + } + is DeclaredReferenceExpression -> { + val region = getRelativeLocation(parent, node) + + list += Triple(true, AttributedString(node.name, DEFAULT.foreground(CYAN)), region) + + return list + } + /*is DeclarationStatement -> { + // lets assume, that everything left of the variable name is some sort of type + var typeRegion = Region( + node.location?.region?.startLine, + node.location?.region?.startColumn, + node.location?.region?.endLine, + node.singleDeclaration?.location?.region?.startColumn + ) + + list += Triple(true, AttributedString(node.name, DEFAULT.foreground(CYAN)), region) + + return list + }*/ + } + + return list +} + +fun getRelativeLocation(parent: Node, node: Node): Region { + val columnOffset = (parent.location?.region?.startColumn ?: 0) + val lineOffset = (parent.location?.region?.startLine ?: 0) + return Region( + (node.location?.region?.startLine ?: 0) - lineOffset, + (node.location?.region?.startColumn ?: 0) - columnOffset, + (node.location?.region?.endLine ?: 0) - lineOffset, + (node.location?.region?.endColumn ?: 0) - columnOffset + ) +} diff --git a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt index f33fcb584e..bafd7d06fe 100644 --- a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt +++ b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt @@ -27,7 +27,11 @@ package de.fraunhofer.aisec.cpg.analysis import de.fraunhofer.aisec.cpg.TranslationConfiguration import de.fraunhofer.aisec.cpg.TranslationManager +import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import java.io.File +import junit.framework.Assert.assertEquals import org.junit.jupiter.api.Test class AnalysisTest { @@ -60,4 +64,32 @@ class AnalysisTest { NullPointerCheck().run(result) } + + @Test + fun testAttribute() { + val config = + TranslationConfiguration.builder() + .sourceLocations(File("src/test/resources/Array.java")) + .defaultPasses() + .defaultLanguages() + .build() + + val analyzer = TranslationManager.builder().config(config).build() + val result = analyzer.analyze().get() + val tu = result.translationUnits.first() + + val main = tu.byName("Array.main") + val call = main.body(0) + + var code = call.fancyCode() + + assertEquals("obj.\u001B[36mdoSomething\u001B[0m();", code) + println(code) + + val decl = main.body(0) + + code = decl.fancyCode() + + println(code) + } } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.java index 76adbbe4df..f1def6cc8d 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/frontends/java/ExpressionHandler.java @@ -723,7 +723,8 @@ private CallExpression handleMethodCallExpression(Expression expr) { lang.setCodeAndRegion( member, - methodCallExpr); // This will also overwrite the code set to the empty string set above + methodCallExpr + .getName()); // This will also overwrite the code set to the empty string set above callExpression = NodeBuilder.newMemberCallExpression( name, qualifiedName, base, member, ".", methodCallExpr.toString()); From ada296129e8ab2e2f01bdc069b293680d579d5e8 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Wed, 2 Jun 2021 22:04:12 +0200 Subject: [PATCH 13/30] First integration of syntax highlighting in console --- .../aisec/cpg/analysis/Extensions.kt | 143 +++++++++++++----- .../aisec/cpg/analysis/AnalysisTest.kt | 7 +- 2 files changed, 115 insertions(+), 35 deletions(-) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt index 74b638b5fa..10d3beeb10 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt @@ -34,22 +34,20 @@ import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.Statement -import de.fraunhofer.aisec.cpg.graph.statements.expressions.DeclaredReferenceExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation.locationLink import de.fraunhofer.aisec.cpg.sarif.Region import kotlin.jvm.Throws import org.jline.utils.AttributedString -import org.jline.utils.AttributedStyle.CYAN -import org.jline.utils.AttributedStyle.DEFAULT +import org.jline.utils.AttributedStyle +import org.jline.utils.AttributedStyle.* fun Node?.printCode(): Unit { val header = "--- ${locationLink(this?.location)} ---" println(header) - println(this?.code) + println(this?.fancyCode()) println("-".repeat(header.length)) } @@ -162,27 +160,36 @@ fun Node.fancyCode(): String { for (fancy in fancies) { if (fancy.first) { - val region = fancy.third + val region = getRelativeLocation(this, fancy.third) fancy.second.let { - val ansi = it.toAnsi() - - var extraCharsInLine = extraCharsInLines.getOrDefault(region.startLine, 0) - - // the amount of extra chars introduced by the ANSI control chars - val extraChars = ansi.length - it.length - // the current line we want to tackle val line = lines[region.startLine] + // the already accumulated extra chars on this line + var extraCharsInLine = extraCharsInLines.getOrDefault(region.startLine, 0) + // everything before the thing we want to replace. add the extra chars in line to // correct for ANSI chars introduced before us val before = line.substring(0, region.startColumn + extraCharsInLine) + // the actual content we want to fancy + val content = + line.substring( + region.startColumn + extraCharsInLine, + region.endColumn + extraCharsInLine + ) + // everything after the thing we want to replace. add the extra chars in line to // correct for ANSI chars introduced before us val after = line.substring(region.endColumn + extraCharsInLine) + // fancy it + val ansi = AttributedString(content, fancy.second).toAnsi() + + // the amount of extra chars introduced by the ANSI control chars + val extraChars = ansi.length - content.length + // reconstruct the line lines[region.startLine] = before + ansi + after @@ -198,8 +205,8 @@ fun Node.fancyCode(): String { return lines.joinToString("\n") } -fun getFanciesFor(parent: Node, node: Node): List> { - val list = mutableListOf>() +fun getFanciesFor(original: Node, node: Node): List> { + val list = mutableListOf>() when (node) { is MemberCallExpression -> { @@ -209,37 +216,105 @@ fun getFanciesFor(parent: Node, node: Node): List { - val region = getRelativeLocation(parent, node) + node.location?.let { list += Triple(true, DEFAULT.foreground(CYAN), it.region) } - list += Triple(true, AttributedString(node.name, DEFAULT.foreground(CYAN)), region) + return list + } + is DeclarationStatement -> { + node.location?.let { + // lets assume, that everything left of the variable name is some sort of type + val typeRegion = + Region( + it.region.startLine, + it.region.startColumn, + it.region.endLine, + node.singleDeclaration.location?.region?.startColumn ?: it.region.startLine + ) + + list += Triple(true, DEFAULT.foreground(RED or BRIGHT), typeRegion) + } + + for (declaration in node.declarations) { + list.addAll(getFanciesFor(original, declaration)) + } return list } - /*is DeclarationStatement -> { - // lets assume, that everything left of the variable name is some sort of type - var typeRegion = Region( - node.location?.region?.startLine, - node.location?.region?.startColumn, - node.location?.region?.endLine, - node.singleDeclaration?.location?.region?.startColumn - ) + is VariableDeclaration -> { + // only color initializer, if any + node.initializer?.let { list.addAll(getFanciesFor(original, it)) } - list += Triple(true, AttributedString(node.name, DEFAULT.foreground(CYAN)), region) + return list + } + is CompoundStatement -> { + // loop through statements + for (statement in node.statements) { + list.addAll(getFanciesFor(original, statement)) + } return list - }*/ + } + is Literal<*> -> { + if (node.value is Number) { + node.location?.let { + list += Triple(true, DEFAULT.foreground(BLUE or BRIGHT), it.region) + } + } + + return list + } + is ArrayCreationExpression -> { + // color the whole expression + node.location?.let { list += Triple(true, DEFAULT.foreground(YELLOW), it.region) } + + return list + } + is FunctionDeclaration -> { + // color the name + node.location?.let { + // look for the name in code; this assumes that it is on the first line for now + val offset = node.code?.indexOf(node.name) ?: -1 + + if (offset != -1) { + val region = + Region( + it.region.startLine, + it.region.startColumn + offset, + it.region.startLine, + it.region.startColumn + offset + node.name.length + ) + + list += Triple(true, DEFAULT.foreground(YELLOW or BRIGHT), region) + } + } + + // forward it to the body + list.addAll(getFanciesFor(original, node.body)) + + return list + } } return list } -fun getRelativeLocation(parent: Node, node: Node): Region { - val columnOffset = (parent.location?.region?.startColumn ?: 0) +fun getRelativeLocation(parent: Node, region: Region): Region { + var columnOffset = 0 + + // we only need a column offset, if the start line is the same + columnOffset = + if (region.startLine == (parent.location?.region?.startLine ?: 0)) { + (parent.location?.region?.startColumn ?: 0) + } else { + 1 // not sure why + } + val lineOffset = (parent.location?.region?.startLine ?: 0) + return Region( - (node.location?.region?.startLine ?: 0) - lineOffset, - (node.location?.region?.startColumn ?: 0) - columnOffset, - (node.location?.region?.endLine ?: 0) - lineOffset, - (node.location?.region?.endColumn ?: 0) - columnOffset + region.startLine - lineOffset, + region.startColumn - columnOffset, + region.endLine - lineOffset, + region.endColumn - columnOffset ) } diff --git a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt index bafd7d06fe..2cf3094954 100644 --- a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt +++ b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt @@ -86,10 +86,15 @@ class AnalysisTest { assertEquals("obj.\u001B[36mdoSomething\u001B[0m();", code) println(code) - val decl = main.body(0) + var decl = main.body(0) + code = decl.fancyCode() + println(code) + decl = main.body(1) code = decl.fancyCode() + println(code) + code = main.fancyCode() println(code) } } From d9c4bbc1a706ae69a9af4cc8a08b2c184f23835d Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Wed, 2 Jun 2021 23:23:52 +0200 Subject: [PATCH 14/30] Added more coloring --- .../aisec/cpg/analysis/Extensions.kt | 202 +++++++++++------- .../aisec/cpg/analysis/NullPointerCheck.kt | 2 +- .../aisec/cpg/analysis/AnalysisTest.kt | 3 +- cpg-console/src/test/resources/Array.java | 4 +- .../aisec/cpg/graph/types/TypeParser.java | 10 +- 5 files changed, 138 insertions(+), 83 deletions(-) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt index 10d3beeb10..b1e9121ed7 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg.analysis import de.fraunhofer.aisec.cpg.graph.DeclarationHolder +import de.fraunhofer.aisec.cpg.graph.HasType import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration @@ -33,18 +34,21 @@ import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.edge.PropertyEdge import de.fraunhofer.aisec.cpg.graph.statements.CompoundStatement import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement +import de.fraunhofer.aisec.cpg.graph.statements.IfStatement import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.statements.expressions.* -import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker +import de.fraunhofer.aisec.cpg.graph.types.TypeParser import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation.locationLink import de.fraunhofer.aisec.cpg.sarif.Region import kotlin.jvm.Throws +import org.jetbrains.kotlinx.ki.shell.configuration.ReplConfigurationBase +import org.jetbrains.kotlinx.ki.shell.plugins.SyntaxPlugin import org.jline.utils.AttributedString import org.jline.utils.AttributedStyle import org.jline.utils.AttributedStyle.* fun Node?.printCode(): Unit { - val header = "--- ${locationLink(this?.location)} ---" + val header = "--- ${this.fancyLocationLink()} ---" println(header) println(this?.fancyCode()) @@ -151,62 +155,59 @@ fun Node.fancyCode(): String { // split it into lines val lines = (code?.split("\n") ?: listOf()).toMutableList() - // next, we need to get all the AST children - val children = SubgraphWalker.getAstChildren(this) - val extraCharsInLines = mutableMapOf() val fancies = getFanciesFor(this, this) for (fancy in fancies) { - if (fancy.first) { - val region = getRelativeLocation(this, fancy.third) + val region = getRelativeLocation(this, fancy.second) - fancy.second.let { - // the current line we want to tackle - val line = lines[region.startLine] + fancy.first.let { + // the current line we want to tackle + val line = lines[region.startLine] - // the already accumulated extra chars on this line - var extraCharsInLine = extraCharsInLines.getOrDefault(region.startLine, 0) + // the already accumulated extra chars on this line + var extraCharsInLine = extraCharsInLines.getOrDefault(region.startLine, 0) - // everything before the thing we want to replace. add the extra chars in line to - // correct for ANSI chars introduced before us - val before = line.substring(0, region.startColumn + extraCharsInLine) + // everything before the thing we want to replace. add the extra chars in line to + // correct for ANSI chars introduced before us + val before = line.substring(0, region.startColumn + extraCharsInLine) - // the actual content we want to fancy - val content = - line.substring( - region.startColumn + extraCharsInLine, - region.endColumn + extraCharsInLine - ) + // the actual content we want to fancy + val content = + line.substring( + region.startColumn + extraCharsInLine, + region.endColumn + extraCharsInLine + ) - // everything after the thing we want to replace. add the extra chars in line to - // correct for ANSI chars introduced before us - val after = line.substring(region.endColumn + extraCharsInLine) + // everything after the thing we want to replace. add the extra chars in line to + // correct for ANSI chars introduced before us + val after = line.substring(region.endColumn + extraCharsInLine) - // fancy it - val ansi = AttributedString(content, fancy.second).toAnsi() + // fancy it + val ansi = AttributedString(content, fancy.first).toAnsi() - // the amount of extra chars introduced by the ANSI control chars - val extraChars = ansi.length - content.length + // the amount of extra chars introduced by the ANSI control chars + val extraChars = ansi.length - content.length - // reconstruct the line - lines[region.startLine] = before + ansi + after + // reconstruct the line + lines[region.startLine] = before + ansi + after - // update extra chars in line - extraCharsInLine += extraChars + // update extra chars in line + extraCharsInLine += extraChars - // store it - extraCharsInLines.put(region.startLine, extraCharsInLine) - } + // store it + extraCharsInLines.put(region.startLine, extraCharsInLine) } } return lines.joinToString("\n") } -fun getFanciesFor(original: Node, node: Node): List> { - val list = mutableListOf>() +val styles = SyntaxPlugin.HighlightStylesFromConfiguration(object : ReplConfigurationBase() {}) + +fun getFanciesFor(original: Node, node: Node): List> { + val list = mutableListOf>() when (node) { is MemberCallExpression -> { @@ -215,24 +216,13 @@ fun getFanciesFor(original: Node, node: Node): List { - node.location?.let { list += Triple(true, DEFAULT.foreground(CYAN), it.region) } + /*is DeclaredReferenceExpression -> { + node.location?.let { list += Pair(DEFAULT.foreground(CYAN), it.region) } return list - } + }*/ is DeclarationStatement -> { - node.location?.let { - // lets assume, that everything left of the variable name is some sort of type - val typeRegion = - Region( - it.region.startLine, - it.region.startColumn, - it.region.endLine, - node.singleDeclaration.location?.region?.startColumn ?: it.region.startLine - ) - - list += Triple(true, DEFAULT.foreground(RED or BRIGHT), typeRegion) - } + fancyType(node, (node.singleDeclaration as? HasType)!!, list) for (declaration in node.declarations) { list.addAll(getFanciesFor(original, declaration)) @@ -254,40 +244,61 @@ fun getFanciesFor(original: Node, node: Node): List { + // look for the if keyword + fancyWord("if", node, list, styles.keyword) + + list.addAll(getFanciesFor(original, node.thenStatement)) + + return list + } + is BinaryOperator -> { + list.addAll(getFanciesFor(original, node.lhs)) + list.addAll(getFanciesFor(original, node.rhs)) + } is Literal<*> -> { - if (node.value is Number) { - node.location?.let { - list += Triple(true, DEFAULT.foreground(BLUE or BRIGHT), it.region) - } + when (node.value) { + is Number -> + node.location?.let { + list += Pair(DEFAULT.foreground(BLUE or BRIGHT), it.region) + } + null -> node.location?.let { list += Pair(DEFAULT.foreground(CYAN), it.region) } + is Boolean -> + node.location?.let { list += Pair(DEFAULT.foreground(CYAN), it.region) } + is String -> + node.location?.let { list += Pair(DEFAULT.foreground(GREEN), it.region) } } return list } is ArrayCreationExpression -> { - // color the whole expression - node.location?.let { list += Triple(true, DEFAULT.foreground(YELLOW), it.region) } + fancyWord("new", node, list, styles.keyword) + + // check for primitive types + for (primitive in TypeParser.PRIMITIVES) { + fancyWord(primitive, node, list, styles.keyword) + } + + // color initializer, if any + node.initializer?.let { list.addAll(getFanciesFor(original, it)) } + + // color dimensions, if any + for (dimension in node.dimensions) { + list.addAll(getFanciesFor(original, dimension)) + } return list } is FunctionDeclaration -> { - // color the name - node.location?.let { - // look for the name in code; this assumes that it is on the first line for now - val offset = node.code?.indexOf(node.name) ?: -1 - - if (offset != -1) { - val region = - Region( - it.region.startLine, - it.region.startColumn + offset, - it.region.startLine, - it.region.startColumn + offset + node.name.length - ) - - list += Triple(true, DEFAULT.foreground(YELLOW or BRIGHT), region) - } + // color some keywords + val keywords = listOf("public", "private", "static") + for (keyword in keywords) { + fancyWord(node.name, node, list, styles.keyword) } + // color the name + fancyWord(node.name, node, list, styles.function) + // forward it to the body list.addAll(getFanciesFor(original, node.body)) @@ -298,6 +309,44 @@ fun getFanciesFor(original: Node, node: Node): List> +) { + val types = TypeParser.PRIMITIVES.toMutableSet() + types += node.type.name + + // check for primitive types + for (type in types) { + fancyWord(type, outer, list, styles.type) + } +} + +private fun fancyWord( + word: String, + node: Node, + list: MutableList>, + style: AttributedStyle +) { + node.location?.let { + // look for the name in code; this assumes that it is on the first line for now + val offset = node.code?.indexOf(word) ?: -1 + + if (offset != -1) { + val region = + Region( + it.region.startLine, + it.region.startColumn + offset, + it.region.startLine, + it.region.startColumn + offset + word.length + ) + + list += Pair(style, region) + } + } +} + fun getRelativeLocation(parent: Node, region: Region): Region { var columnOffset = 0 @@ -318,3 +367,8 @@ fun getRelativeLocation(parent: Node, region: Region): Region { region.endColumn - columnOffset ) } + +fun Node?.fancyLocationLink(): String { + return AttributedString(locationLink(this?.location), DEFAULT.foreground(BLUE or BRIGHT)) + .toAnsi() +} diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt index 480399f8e9..802bf125a9 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt @@ -130,7 +130,7 @@ class NullPointerCheck { if (last is IfStatement) { println() println( - "Branch depends on ${AttributedString("IfStatement", DEFAULT.foreground(GREEN)).toAnsi()} with condition ${AttributedString(last.condition.code, DEFAULT.foreground(CYAN)).toAnsi()} in ${AttributedString(locationLink(last.location), DEFAULT.foreground(BLUE or BRIGHT)).toAnsi()}" + "Branch depends on ${AttributedString("IfStatement", DEFAULT.foreground(GREEN)).toAnsi()} with condition ${AttributedString(last.condition.code, DEFAULT.foreground(CYAN)).toAnsi()} in ${last.fancyLocationLink()}" ) } diff --git a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt index 2cf3094954..5eb8856496 100644 --- a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt +++ b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt @@ -31,7 +31,6 @@ import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression import java.io.File -import junit.framework.Assert.assertEquals import org.junit.jupiter.api.Test class AnalysisTest { @@ -83,7 +82,7 @@ class AnalysisTest { var code = call.fancyCode() - assertEquals("obj.\u001B[36mdoSomething\u001B[0m();", code) + // assertEquals("obj.\u001B[36mdoSomething\u001B[0m();", code) println(code) var decl = main.body(0) diff --git a/cpg-console/src/test/resources/Array.java b/cpg-console/src/test/resources/Array.java index 61510b6e72..31c6d9376a 100644 --- a/cpg-console/src/test/resources/Array.java +++ b/cpg-console/src/test/resources/Array.java @@ -13,7 +13,7 @@ public static void main(String[] args) { // lets make it a little bit tricky at least obj = something; - if(something) { + if (something) { AnotherObject yetAnotherObject = null; // whoops, overriden with null again @@ -21,5 +21,7 @@ public static void main(String[] args) { } obj.doSomething(); + + String s = "some string"; } } \ No newline at end of file diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/TypeParser.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/TypeParser.java index 5a6c41a947..2b99b34d4e 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/TypeParser.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/TypeParser.java @@ -45,7 +45,7 @@ public class TypeParser { private static final Logger log = LoggerFactory.getLogger(TypeParser.class); public static final String UNKNOWN_TYPE_STRING = "UNKNOWN"; - private static final List primitives = + public static final List PRIMITIVES = List.of("byte", "short", "int", "long", "float", "double", "boolean", "char"); private static final Pattern functionPtrRegex = Pattern.compile( @@ -547,7 +547,7 @@ private static String replaceScopeResolutionOperator(@NonNull String type) { */ private static boolean isPrimitiveType(@NonNull List stringList) { for (String s : stringList) { - if (primitives.contains(s)) { + if (PRIMITIVES.contains(s)) { return true; } } @@ -568,7 +568,7 @@ private static List joinPrimitive(@NonNull List typeBlocks) { boolean foundPrimitive = false; for (String s : typeBlocks) { - if (primitives.contains(s)) { + if (PRIMITIVES.contains(s)) { if (primitiveType.length() > 0) { primitiveType.append(" "); } @@ -577,11 +577,11 @@ private static List joinPrimitive(@NonNull List typeBlocks) { } for (String s : typeBlocks) { - if (primitives.contains(s) && !foundPrimitive) { + if (PRIMITIVES.contains(s) && !foundPrimitive) { joinedTypeBlocks.add(primitiveType.toString()); foundPrimitive = true; } else { - if (!primitives.contains(s)) { + if (!PRIMITIVES.contains(s)) { joinedTypeBlocks.add(s); } } From d9081f9b3d7476f9a20dbe367e0e397b52b53735 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Thu, 3 Jun 2021 00:03:40 +0200 Subject: [PATCH 15/30] Even more syntax highlighting --- .../aisec/cpg/analysis/Extensions.kt | 28 +++++----- .../aisec/cpg/analysis/NullPointerCheck.kt | 8 +-- .../aisec/cpg/analysis/OutOfBoundsCheck.kt | 52 ++++++++++++++++--- 3 files changed, 63 insertions(+), 25 deletions(-) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt index b1e9121ed7..66cd84b4ff 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt @@ -216,11 +216,18 @@ fun getFanciesFor(original: Node, node: Node): List { - node.location?.let { list += Pair(DEFAULT.foreground(CYAN), it.region) } + is DeclaredReferenceExpression -> { + if ((original as? MemberCallExpression)?.member == node) { + node.location?.let { list += Pair(styles.identifier!!, it.region) } + } + + // also color it, if its on its own + if (original == node) { + node.location?.let { list += Pair(styles.identifier!!, it.region) } + } return list - }*/ + } is DeclarationStatement -> { fancyType(node, (node.singleDeclaration as? HasType)!!, list) @@ -258,15 +265,10 @@ fun getFanciesFor(original: Node, node: Node): List -> { when (node.value) { - is Number -> - node.location?.let { - list += Pair(DEFAULT.foreground(BLUE or BRIGHT), it.region) - } - null -> node.location?.let { list += Pair(DEFAULT.foreground(CYAN), it.region) } - is Boolean -> - node.location?.let { list += Pair(DEFAULT.foreground(CYAN), it.region) } - is String -> - node.location?.let { list += Pair(DEFAULT.foreground(GREEN), it.region) } + is Number -> node.location?.let { list += Pair(styles.number, it.region) } + null -> node.location?.let { list += Pair(styles.number, it.region) } + is Boolean -> node.location?.let { list += Pair(styles.number, it.region) } + is String -> node.location?.let { list += Pair(styles.string, it.region) } } return list @@ -293,7 +295,7 @@ fun getFanciesFor(original: Node, node: Node): List() { fun visit(v: ArraySubscriptionExpression) { - val resolvedIndex = v.subscriptExpression.resolve() + val resolver = ValueResolver() + val resolvedIndex = resolver.resolve(v.subscriptExpression) if (resolvedIndex is Int) { - println("Index: $resolvedIndex") - // check, if we know that the array was initialized with a fixed length // TODO(oxisto): it would be nice to have a helper that follows the expr val decl = @@ -64,11 +67,44 @@ class OutOfBoundsCheck { val dimension = it.dimensions.first().resolve() if (resolvedIndex >= dimension as Int) { - Util.errorWithFileLocation( - v, - log, - "Error: this expression will run out of bounds $resolvedIndex >= $dimension" + println("") + val sb = AttributedStringBuilder() + sb.append("--- FINDING: Out of bounds access in ") + sb.append( + it.javaClass.simpleName, + DEFAULT.foreground(AttributedStyle.GREEN) + ) + sb.append( + " when accessing index ${AttributedString(""+resolvedIndex, DEFAULT.foreground(AttributedStyle.CYAN)).toAnsi()} of " + ) + sb.append(decl.name, DEFAULT.foreground(AttributedStyle.YELLOW)) + sb.append( + ", an array of length ${AttributedString(""+dimension, DEFAULT.foreground(AttributedStyle.CYAN)).toAnsi()} ---" + ) + + val header = sb.toAnsi() + + println(header) + println( + "${ + AttributedString( + PhysicalLocation.locationLink(v.location), DEFAULT.foreground( + AttributedStyle.BLUE or AttributedStyle.BRIGHT + )).toAnsi()}: ${v.fancyCode()}" ) + println("") + println( + "The following path was discovered that leads to ${v.subscriptExpression.fancyCode()} being ${AttributedString(""+resolvedIndex, DEFAULT.foreground(AttributedStyle.CYAN)).toAnsi()}:" + ) + for (p in resolver.path) { + + println( + "${AttributedString( + PhysicalLocation.locationLink(p.location), DEFAULT.foreground( + AttributedStyle.BLUE or AttributedStyle.BRIGHT + )).toAnsi()}: ${p.fancyCode()}" + ) + } } } } else { From 6099892e37afb723b799e1e261fa98e64cd32cf4 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Thu, 3 Jun 2021 10:23:20 +0200 Subject: [PATCH 16/30] Tutorial --- .../aisec/cpg/analysis/Extensions.kt | 45 +++++++++++++++++-- .../aisec/cpg/console/CpgConsole.kt | 2 +- .../aisec/cpg/console/ShowCodePlugin.kt | 5 --- .../{AnalyzePlugin.kt => TranslatePlugin.kt} | 11 +++-- 4 files changed, 50 insertions(+), 13 deletions(-) rename cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/{AnalyzePlugin.kt => TranslatePlugin.kt} (89%) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt index 66cd84b4ff..220163bac2 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt @@ -25,6 +25,7 @@ */ package de.fraunhofer.aisec.cpg.analysis +import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.DeclarationHolder import de.fraunhofer.aisec.cpg.graph.HasType import de.fraunhofer.aisec.cpg.graph.Node @@ -38,6 +39,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.IfStatement import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.TypeParser +import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation.locationLink import de.fraunhofer.aisec.cpg.sarif.Region import kotlin.jvm.Throws @@ -47,15 +49,17 @@ import org.jline.utils.AttributedString import org.jline.utils.AttributedStyle import org.jline.utils.AttributedStyle.* -fun Node?.printCode(): Unit { +fun Node.printCode(): Node { val header = "--- ${this.fancyLocationLink()} ---" println(header) - println(this?.fancyCode()) + println(this.fancyCode()) println("-".repeat(header.length)) + + return this } -fun MutableSet.printCode(): Unit { +fun Collection.printCode(): Collection { val it = this.iterator() while (it.hasNext()) { @@ -63,6 +67,8 @@ fun MutableSet.printCode(): Unit { next.printCode() println("") } + + return this } fun Expression.resolve(): Any? { @@ -73,6 +79,39 @@ fun Declaration.resolve(): Any? { return ValueResolver().resolveDeclaration(this) } +@JvmName("allNodes") +fun TranslationResult.all(): List { + return this.all() +} + +inline fun TranslationResult.all(): List { + val children = SubgraphWalker.flattenAST(this) + + return children.filterIsInstance() +} + +@JvmName("allNodes") +fun Node.all(): List { + return this.all() +} + +inline fun Node.all(): List { + val children = SubgraphWalker.flattenAST(this) + + return children.filterIsInstance() +} + +@JvmName("astNodes") +fun Node.ast(): List { + return this.ast() +} + +inline fun Node.ast(): List { + val children = SubgraphWalker.getAstChildren(this) + + return children.filterIsInstance() +} + @Throws(DeclarationNotFound::class) inline fun DeclarationHolder.byName(name: String): T { var base = this diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt index 4de399545b..674f68d517 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt @@ -84,7 +84,7 @@ object CpgConsole { object : ReplConfigurationBase() { override fun plugins(): Iterator { val list = super.plugins().asSequence().toList().toMutableList() - list += AnalyzePlugin() + list += TranslatePlugin() list += Neo4jPlugin() list += ShowCodePlugin() list += RunPlugin() diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt index 435fb2fef8..56744fb0b6 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt @@ -25,7 +25,6 @@ */ package de.fraunhofer.aisec.cpg.console -import de.fraunhofer.aisec.cpg.graph.Node import org.jetbrains.kotlinx.ki.shell.BaseCommand import org.jetbrains.kotlinx.ki.shell.Command import org.jetbrains.kotlinx.ki.shell.Plugin @@ -60,7 +59,3 @@ class ShowCodePlugin : Plugin { override fun cleanUp() {} } - -fun Node.printNode() { - println(this.code) -} diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt similarity index 89% rename from cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt rename to cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt index 55042529c0..758a385d92 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/AnalyzePlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt @@ -31,11 +31,12 @@ import org.jetbrains.kotlinx.ki.shell.Plugin import org.jetbrains.kotlinx.ki.shell.Shell import org.jetbrains.kotlinx.ki.shell.configuration.ReplConfiguration -class AnalyzePlugin : Plugin { +class TranslatePlugin : Plugin { inner class Load(conf: ReplConfiguration) : BaseCommand() { - override val name: String by conf.get(default = "analyze") - override val short: String by conf.get(default = "a") - override val description: String = "analyzes the path" + override val name: String by conf.get(default = "translate") + override val short: String by conf.get(default = "tr") + override val description: String = + "translates the source code files at the path into the CPG" override val params = "" @@ -55,6 +56,8 @@ class AnalyzePlugin : Plugin { "import de.fraunhofer.aisec.cpg.graph.statements.expressions.*", // helper builtins "import de.fraunhofer.aisec.cpg.analysis.resolve", + "import de.fraunhofer.aisec.cpg.analysis.all", + "import de.fraunhofer.aisec.cpg.analysis.ast", "import de.fraunhofer.aisec.cpg.analysis.byName", "import de.fraunhofer.aisec.cpg.analysis.body", "import de.fraunhofer.aisec.cpg.analysis.printCode", From f7c56c91238aaa1660a44ca2219800ca49b8bdc3 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Thu, 3 Jun 2021 10:25:03 +0200 Subject: [PATCH 17/30] added tutorial --- tutorial.md | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 tutorial.md diff --git a/tutorial.md b/tutorial.md new file mode 100644 index 0000000000..f169ebab29 --- /dev/null +++ b/tutorial.md @@ -0,0 +1,91 @@ +# CPG Console + +While the CPG is mostly used as a libary in external tools, we decided to showcase its functionalities with a simple CLI based console that can be used to query the graph and run simple analysis steps. + +To launch the console, built it and then run `bin/cpg-console`. You will be greeted by the interactive prompt of our CPG console, which is implemented by the kotlin `ki` interactive shell. The commands on this shell follow the Kotlin language. For more information please see the [Kotlin documentation](https://kotlinlang.org/docs/home.html). + +In addition to that, commands prefixed by a `:` are plugin commands. To get a list of all available plugins use the `:h` or `:help` command. + +``` +ki-shell 0.3/1.4.32 +type :h for help +[0] :h +:quit or q quit the shell +:load or l load file and evaluate +:type or t display the type of an expression without evaluating it +:list or ls list defined symbols +:help or h [command] print this summary or command-specific help +``` + +## Launching the translation plugin + +One such a plugin is the `:translate` command, or `:tr` for short. It allows the translation of source code files into the code property graph. In this example, we will use all files in `src/test/resources` and translate them. + +```log +[1] :tr src/test/resources +10:15:27,086 INFO TranslationManager Parsing /Users/cpg/Repositories/cpg/cpg-console/src/test/resources/array.cpp +10:15:27,231 INFO CXXLanguageFrontend Parsed 96 bytes corresponding roughly to 1 LoC +10:15:27,231 INFO Benchmark CXXLanguageFrontend Parsing sourcefile done in 94 ms +10:15:27,273 INFO Benchmark CXXLanguageFrontend Transform to CPG done in 41 ms +10:15:27,273 INFO TranslationManager Parsing /Users/cpg/Repositories/cpg/cpg-console/src/test/resources/Array.java +10:15:27,312 INFO JavaLanguageFrontend Source file root used for type solver: /Users/cpg/Repositories/cpg/cpg-console/src/test/resources +10:15:27,343 ERROR TranslationManager Different frontends are used for multiple files. This will very likely break the following passes. +10:15:27,395 INFO Benchmark JavaLanguageFrontend Parsing source file done in 51 ms +10:15:27,452 INFO Benchmark JavaLanguageFrontend Transform to CPG done in 57 ms +10:15:27,452 INFO Benchmark TranslationManager Frontend done in 373 ms +10:15:27,465 INFO Benchmark TypeHierarchyResolver Executing Pass done in 12 ms +10:15:27,467 INFO Benchmark JavaExternalTypeHierarchyResolver Executing Pass done in 2 ms +10:15:27,470 INFO Benchmark ImportResolver Executing Pass done in 2 ms +10:15:27,482 INFO Benchmark VariableUsageResolver Executing Pass done in 11 ms +10:15:27,490 INFO Benchmark CallResolver Executing Pass done in 8 ms +10:15:27,493 INFO Benchmark EvaluationOrderGraphPass Executing Pass done in 2 ms +10:15:27,495 INFO Benchmark TypeResolver Executing Pass done in 1 ms +10:15:27,498 INFO Benchmark ControlFlowSensitiveDFGPass Executing Pass done in 2 ms +10:15:27,499 INFO Benchmark FilenameMapper Executing Pass done in 1 ms +10:15:27,499 INFO Benchmark TranslationManager Translation into full graph done in 420 ms +``` + +After the translation is done, several symbols are available on the console, to query and analyse the result. You can use the `:ls` command to get a quick overview. + +```kotlin +[2] :ls +val config: de.fraunhofer.aisec.cpg.TranslationConfiguration! +val analyzer: de.fraunhofer.aisec.cpg.TranslationManager! +val result: de.fraunhofer.aisec.cpg.TranslationResult! +val tu: de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration! +``` + +Most interesting for the user are the `result` object which holds the complete translation result, as well as the `tu` object, which is a shortcut to the first translation unit, i.e. the first file that was translated. It is of type `TranslationUnitDeclaration`, which is one of our node type in the graph; the most basic one being a `Node` itself. + +## Querying the translation result + +In the following, we will use the aforementioned objects to query the source code for interesting patterns. To do so, we will explore several built-in functions that can be used in exploring the graph. The first of these, is the `all` function, it returns a list of all nodes that are direct descendents of a particular node, basicically flattening the hierarchy. + +```kotlin +[3] result.all() +res3: List = ... +``` + +The output here can be quite huge, so additional filtering is needed. The `all` function takes additional type parameter, which can be used to further filter nodes of a particular type. In this case, we are interested in all `ArraySubscriptionExpression` nodes, i.e. those that represent access to an element of an array. These operations are often prone to out of bounds errors and we want to explore, whether our code is also affected by that. + +```kotlin +[4] result.all() +res4: List = [{"location":"array.cpp(6:12-6:18)","type":{"name":"char"},"possibleSubTypes":[{"name":"char"},"UNKNOWN"]}, {"location":"Array.java(8:18-8:22)","type":{"name":"char"},"possibleSubTypes":[{"name":"char"},"UNKNOWN"]}] +``` + +Much better. We have found two nodes that represent an array access. To see the corresponding source code of our result, we can prefix our previous command with `:code` or `:c`. This shows the raw source code as well as the location of the file where the code is located. + +```kotlin +[5] :code result.all() +--- src/test/resources/array.cpp:6:12 --- += c[b] +----------------------------------------------------------------------------------------------- + +--- src/test/resources/Array.java:8:18 --- +c[b] +------------------------------------------------------------------------------------------------ + +res5: Collection = [{"location":"array.cpp(6:12-6:18)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]}, {"location":"Array.java(8:18-8:22)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]}] +``` + +This also demonstrates quite nicely, that queries on the CPG work independently of the programming language. Our test folder contains both Java as well as C++ files and we can analyse them simultaneously. From 15f482a416a983f2bb2f097196290127c29a9571 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Thu, 3 Jun 2021 10:54:26 +0200 Subject: [PATCH 18/30] tutorial ++, but needs more shortcut functions --- tutorial.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/tutorial.md b/tutorial.md index f169ebab29..b3cd2ead27 100644 --- a/tutorial.md +++ b/tutorial.md @@ -66,11 +66,14 @@ In the following, we will use the aforementioned objects to query the source cod res3: List = ... ``` -The output here can be quite huge, so additional filtering is needed. The `all` function takes additional type parameter, which can be used to further filter nodes of a particular type. In this case, we are interested in all `ArraySubscriptionExpression` nodes, i.e. those that represent access to an element of an array. These operations are often prone to out of bounds errors and we want to explore, whether our code is also affected by that. +The output here can be quite huge, so additional filtering is needed. The `all` function takes an additional type parameter, which can be used to further filter nodes of a particular type. In this case, we are interested in all `ArraySubscriptionExpression` nodes, i.e. those that represent access to an element of an array. These operations are often prone to out of bounds errors and we want to explore, whether our code is also affected by that. ```kotlin [4] result.all() -res4: List = [{"location":"array.cpp(6:12-6:18)","type":{"name":"char"},"possibleSubTypes":[{"name":"char"},"UNKNOWN"]}, {"location":"Array.java(8:18-8:22)","type":{"name":"char"},"possibleSubTypes":[{"name":"char"},"UNKNOWN"]}] +res4: List = [ + {"location":"array.cpp(6:12-6:18)","type":{"name":"char"}}, + {"location":"Array.java(8:18-8:22)","type":{"name":"char"}} +] ``` Much better. We have found two nodes that represent an array access. To see the corresponding source code of our result, we can prefix our previous command with `:code` or `:c`. This shows the raw source code as well as the location of the file where the code is located. @@ -89,3 +92,45 @@ res5: Collection = [{"location":"array.cpp(6 ``` This also demonstrates quite nicely, that queries on the CPG work independently of the programming language. Our test folder contains both Java as well as C++ files and we can analyse them simultaneously. + +```kotlin +[24] result.all().map { it.subscriptExpression.resolve() } +res21: List = [5, 5] +``` + +TODO: make it simpler to query prev DFG with type +```kotlin +[30] result.all().map { + Triple( + it.subscriptExpression.resolve() as Int, + (it.arrayExpression.prevDFG.first() as? ArrayCreationExpression)?.dimensions?.first()?.resolve() as Int, + it + ) } +res29: List> = [(5, 4, {"location":"array.cpp(6:12-6:18)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]}), (5, 4, {"location":"Array.java(8:18-8:22)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]})] +``` + +We use that result to filter those where the resolved index is greater or equal to our dimension. + +```kotlin +[31] res39.filter { it.first >= it.second } +res41: List> = [(5, 4, {"location":"array.cpp(6:12-6:18)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]}), (5, 4, {"location":"Array.java(8:18-8:22)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]})] +``` + +```kotlin +[31] :code res39.filter { it.first >= it.second }.map { it.third } +--- /Users/chr55316/Repositories/cpg/cpg-console/src/test/resources/array.cpp:6:12 --- += c[b] +----------------------------------------------------------------------------------------------- + +--- /Users/chr55316/Repositories/cpg/cpg-console/src/test/resources/Array.java:8:18 --- +c[b] +------------------------------------------------------------------------------------------------ +``` +Of course the same can also be achieved in one, slightly larger query. + +```kotlin +[6] result.all().filter { + (it.subscriptExpression.resolve() as Int) >= + ((it.arrayExpression.prevDFG.first() as? ArrayCreationExpression)?.dimensions?.first()?.resolve() as Int) +} +``` From fe72ecb9bf8c309c222b316909d8a04e8ca70098 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Thu, 3 Jun 2021 20:43:06 +0200 Subject: [PATCH 19/30] Playing around with formatting --- .../aisec/cpg/analysis/Extensions.kt | 152 +++++++++++++----- .../aisec/cpg/analysis/NullPointerCheck.kt | 10 +- .../aisec/cpg/analysis/OutOfBoundsCheck.kt | 9 +- .../aisec/cpg/console/ShowCodePlugin.kt | 2 +- .../aisec/cpg/console/TranslatePlugin.kt | 1 + .../aisec/cpg/analysis/AnalysisTest.kt | 11 +- cpg-console/src/test/resources/array.cpp | 6 + tutorial.md | 13 +- 8 files changed, 145 insertions(+), 59 deletions(-) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt index 220163bac2..11f89aa661 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt @@ -42,6 +42,7 @@ import de.fraunhofer.aisec.cpg.graph.types.TypeParser import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation.locationLink import de.fraunhofer.aisec.cpg.sarif.Region +import java.io.File import kotlin.jvm.Throws import org.jetbrains.kotlinx.ki.shell.configuration.ReplConfigurationBase import org.jetbrains.kotlinx.ki.shell.plugins.SyntaxPlugin @@ -49,22 +50,25 @@ import org.jline.utils.AttributedString import org.jline.utils.AttributedStyle import org.jline.utils.AttributedStyle.* -fun Node.printCode(): Node { +fun Node.printCode(linesAhead: Int = 0, showNumbers: Boolean = false): Node { val header = "--- ${this.fancyLocationLink()} ---" println(header) - println(this.fancyCode()) + println(this.fancyCode(linesAhead, showNumbers)) println("-".repeat(header.length)) return this } -fun Collection.printCode(): Collection { +fun Collection.printCode( + linesAhead: Int = 0, + showNumbers: Boolean = false +): Collection { val it = this.iterator() while (it.hasNext()) { val next = it.next() - next.printCode() + next.printCode(linesAhead, showNumbers) println("") } @@ -112,6 +116,10 @@ inline fun Node.ast(): List { return children.filterIsInstance() } +inline fun Node.dfgFrom(): List { + return this.prevDFG.toList().filterIsInstance() +} + @Throws(DeclarationNotFound::class) inline fun DeclarationHolder.byName(name: String): T { var base = this @@ -187,60 +195,116 @@ fun Node.followPrevEOG(predicate: (PropertyEdge<*>) -> Boolean): List() + // update the start line + startLine = region.startLine - val fancies = getFanciesFor(this, this) + this.file?.let { file -> code = getCode(file, region) } + } - for (fancy in fancies) { - val region = getRelativeLocation(this, fancy.second) + // split it into lines + val lines = (code?.split("\n") ?: listOf()).toMutableList() - fancy.first.let { - // the current line we want to tackle - val line = lines[region.startLine] + val extraCharsInLines = mutableMapOf() - // the already accumulated extra chars on this line - var extraCharsInLine = extraCharsInLines.getOrDefault(region.startLine, 0) + val fancies = getFanciesFor(this, this) - // everything before the thing we want to replace. add the extra chars in line to - // correct for ANSI chars introduced before us - val before = line.substring(0, region.startColumn + extraCharsInLine) + for (fancy in fancies) { + val region = getRelativeLocation(it, fancy.second) - // the actual content we want to fancy - val content = - line.substring( - region.startColumn + extraCharsInLine, - region.endColumn + extraCharsInLine - ) + fancy.first.let { + // the current line we want to tackle + val line = lines[region.startLine] + + // the already accumulated extra chars on this line + var extraCharsInLine = extraCharsInLines.getOrDefault(region.startLine, 0) + + // everything before the thing we want to replace. add the extra chars in line to + // correct for ANSI chars introduced before us + val before = line.substring(0, region.startColumn + extraCharsInLine) + + // the actual content we want to fancy + val content = + line.substring( + region.startColumn + extraCharsInLine, + region.endColumn + extraCharsInLine + ) + + // everything after the thing we want to replace. add the extra chars in line to + // correct for ANSI chars introduced before us + val after = line.substring(region.endColumn + extraCharsInLine) + + // fancy it + val ansi = AttributedString(content, fancy.first).toAnsi() - // everything after the thing we want to replace. add the extra chars in line to - // correct for ANSI chars introduced before us - val after = line.substring(region.endColumn + extraCharsInLine) + // the amount of extra chars introduced by the ANSI control chars + val extraChars = ansi.length - content.length - // fancy it - val ansi = AttributedString(content, fancy.first).toAnsi() + // reconstruct the line + lines[region.startLine] = before + ansi + after - // the amount of extra chars introduced by the ANSI control chars - val extraChars = ansi.length - content.length + // update extra chars in line + extraCharsInLine += extraChars - // reconstruct the line - lines[region.startLine] = before + ansi + after + // store it + extraCharsInLines.put(region.startLine, extraCharsInLine) + } + } - // update extra chars in line - extraCharsInLine += extraChars + if (showNumbers) { + var output = "" + + // display line numbers + for (i in 0 until lines.size) { + var line = ((i + startLine).toString() + ": ").padStart(5) + lines[i] + if (i != lines.size - 1) { + line += "\n" + } + + /*if (i >= linesAhead) { + output += AttributedString(line, DEFAULT.background(128, 128, 128)).toAnsi() + } else {*/ + output += line + // } + } - // store it - extraCharsInLines.put(region.startLine, extraCharsInLine) + return output } + + return lines.joinToString("\n") + } + + // no location, no fancy + return this.code +} + +fun getCode(file: String, region: Region): String { + var code = "" + + val lines = File(file).readLines() + + for (i in region.startLine - 1 until region.endLine) { + code += + when (i) { + region.startLine - 1 -> lines[i].substring(region.startColumn - 1) + "\n" + region.endLine - 1 -> lines[i].substring(0, region.endColumn - 1) + else -> lines[i] + "\n" + } } - return lines.joinToString("\n") + return code } val styles = SyntaxPlugin.HighlightStylesFromConfiguration(object : ReplConfigurationBase() {}) @@ -388,18 +452,18 @@ private fun fancyWord( } } -fun getRelativeLocation(parent: Node, region: Region): Region { +fun getRelativeLocation(parentRegion: Region, region: Region): Region { var columnOffset = 0 // we only need a column offset, if the start line is the same columnOffset = - if (region.startLine == (parent.location?.region?.startLine ?: 0)) { - (parent.location?.region?.startColumn ?: 0) + if (region.startLine == (parentRegion.startLine ?: 0)) { + (parentRegion.startColumn ?: 0) } else { 1 // not sure why } - val lineOffset = (parent.location?.region?.startLine ?: 0) + val lineOffset = (parentRegion.startLine ?: 0) return Region( region.startLine - lineOffset, diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt index 1468ce4dd4..6a48976641 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/NullPointerCheck.kt @@ -99,7 +99,10 @@ class NullPointerCheck { println(header) println( - "${AttributedString(locationLink((node as Node).location), DEFAULT.foreground(BLUE or BRIGHT)).toAnsi()}: ${(node as Node).fancyCode()}" + "${AttributedString(locationLink((node as Node).location), DEFAULT.foreground(BLUE or BRIGHT)).toAnsi()}: ${(node as Node).fancyCode( + showNumbers = false + ) + }" ) println("") println( @@ -108,7 +111,10 @@ class NullPointerCheck { for (p in resolver.path) { println( - "${AttributedString(locationLink(p.location), DEFAULT.foreground(BLUE or BRIGHT)).toAnsi()}: ${p.fancyCode()}" + "${AttributedString(locationLink(p.location), DEFAULT.foreground(BLUE or BRIGHT)).toAnsi()}: ${p.fancyCode( + showNumbers = false + ) + }" ) } diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt index e2d36ce045..27cfca72df 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt @@ -90,11 +90,14 @@ class OutOfBoundsCheck { AttributedString( PhysicalLocation.locationLink(v.location), DEFAULT.foreground( AttributedStyle.BLUE or AttributedStyle.BRIGHT - )).toAnsi()}: ${v.fancyCode()}" + )).toAnsi()}: ${v.fancyCode(showNumbers = false)}" ) println("") println( - "The following path was discovered that leads to ${v.subscriptExpression.fancyCode()} being ${AttributedString(""+resolvedIndex, DEFAULT.foreground(AttributedStyle.CYAN)).toAnsi()}:" + "The following path was discovered that leads to ${v.subscriptExpression.fancyCode( + showNumbers = false + ) + } being ${AttributedString(""+resolvedIndex, DEFAULT.foreground(AttributedStyle.CYAN)).toAnsi()}:" ) for (p in resolver.path) { @@ -102,7 +105,7 @@ class OutOfBoundsCheck { "${AttributedString( PhysicalLocation.locationLink(p.location), DEFAULT.foreground( AttributedStyle.BLUE or AttributedStyle.BRIGHT - )).toAnsi()}: ${p.fancyCode()}" + )).toAnsi()}: ${p.fancyCode(showNumbers = false)}" ) } } diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt index 56744fb0b6..ca1ca17c77 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt @@ -43,7 +43,7 @@ class ShowCodePlugin : Plugin { val p = line.indexOf(' ') val node = line.substring(p + 1).trim() - return Command.Result.RunSnippets(listOf("${node}.printCode()")) + return Command.Result.RunSnippets(listOf("${node}.printCode(0, true)")) // return Command.Result.RunSnippets(listOf(content)) } diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt index 758a385d92..66cc91ab8b 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt @@ -58,6 +58,7 @@ class TranslatePlugin : Plugin { "import de.fraunhofer.aisec.cpg.analysis.resolve", "import de.fraunhofer.aisec.cpg.analysis.all", "import de.fraunhofer.aisec.cpg.analysis.ast", + "import de.fraunhofer.aisec.cpg.analysis.dfgFrom", "import de.fraunhofer.aisec.cpg.analysis.byName", "import de.fraunhofer.aisec.cpg.analysis.body", "import de.fraunhofer.aisec.cpg.analysis.printCode", diff --git a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt index 5eb8856496..7c892d99c2 100644 --- a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt +++ b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt @@ -80,20 +80,23 @@ class AnalysisTest { val main = tu.byName("Array.main") val call = main.body(0) - var code = call.fancyCode() + var code = call.fancyCode(showNumbers = false) // assertEquals("obj.\u001B[36mdoSomething\u001B[0m();", code) println(code) var decl = main.body(0) - code = decl.fancyCode() + code = decl.fancyCode(showNumbers = false) println(code) decl = main.body(1) - code = decl.fancyCode() + code = decl.fancyCode(showNumbers = false) println(code) - code = main.fancyCode() + code = main.fancyCode(showNumbers = false) + println(code) + + code = call.fancyCode(3, true) println(code) } } diff --git a/cpg-console/src/test/resources/array.cpp b/cpg-console/src/test/resources/array.cpp index 2c0c5fb25b..4429239d93 100644 --- a/cpg-console/src/test/resources/array.cpp +++ b/cpg-console/src/test/resources/array.cpp @@ -5,3 +5,9 @@ int main() { char d = c[b]; } + +void some_other_function() { + char* c = new char[100]; + + return c[0]; +} \ No newline at end of file diff --git a/tutorial.md b/tutorial.md index b3cd2ead27..ff550946f7 100644 --- a/tutorial.md +++ b/tutorial.md @@ -95,18 +95,21 @@ This also demonstrates quite nicely, that queries on the CPG work independently ```kotlin [24] result.all().map { it.subscriptExpression.resolve() } -res21: List = [5, 5] +res21: List = [5, 5, 0] ``` TODO: make it simpler to query prev DFG with type ```kotlin -[30] result.all().map { - Triple( +[30] result.all().map { Triple( it.subscriptExpression.resolve() as Int, - (it.arrayExpression.prevDFG.first() as? ArrayCreationExpression)?.dimensions?.first()?.resolve() as Int, + it.arrayExpression.dfgFrom().first().dimensions.first().resolve() as Int, it ) } -res29: List> = [(5, 4, {"location":"array.cpp(6:12-6:18)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]}), (5, 4, {"location":"Array.java(8:18-8:22)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]})] +res21: List> = [ + (5, 4, {"location":"array.cpp(6:12-6:18)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]}), + (5, 4, {"location":"Array.java(8:18-8:22)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]}), + (0, 100, {"location":"array.cpp(12:12-12:16)","type":{"name":"char"},"possibleSubTypes":[{"name":"char"}]}) +] ``` We use that result to filter those where the resolved index is greater or equal to our dimension. From d036bae7acbef4c18f16fd82e70f9ff50a87d48b Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Tue, 8 Jun 2021 19:32:21 +0200 Subject: [PATCH 20/30] Fixed stuff --- .../aisec/cpg/analysis/Extensions.kt | 5 + .../aisec/cpg/analysis/OutOfBoundsCheck.kt | 8 +- .../aisec/cpg/console/Neo4jPlugin.kt | 8 +- .../aisec/cpg/console/TranslatePlugin.kt | 6 +- .../aisec/cpg/analysis/AnalysisTest.kt | 24 +++ cpg-console/src/test/resources/array.go | 7 + cpg-console/src/test/resources/nullptr.cpp | 8 + cpg-library/src/main/golang/expressions.go | 22 ++ .../src/main/golang/frontend/handler.go | 11 + .../de/fraunhofer/aisec/cpg/graph/Node.kt | 3 +- .../ParamVariableDeclaration.java | 3 +- .../graph/declarations/ValueDeclaration.java | 6 +- .../DeclaredReferenceExpression.java | 4 +- .../statements/expressions/Expression.java | 7 +- .../aisec/cpg/graph/types/Type.java | 5 +- tutorial.md | 194 ++++++++++++------ 16 files changed, 239 insertions(+), 82 deletions(-) create mode 100644 cpg-console/src/test/resources/array.go create mode 100644 cpg-console/src/test/resources/nullptr.cpp diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt index 11f89aa661..e0b0f69132 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt @@ -83,6 +83,11 @@ fun Declaration.resolve(): Any? { return ValueResolver().resolveDeclaration(this) } +val ArrayCreationExpression.capacity: Int + get() { + return dimensions.first().resolve() as Int + } + @JvmName("allNodes") fun TranslationResult.all(): List { return this.all() diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt index 27cfca72df..f3dc9d7ae8 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/OutOfBoundsCheck.kt @@ -62,11 +62,9 @@ class OutOfBoundsCheck { (v.arrayExpression as? DeclaredReferenceExpression)?.refersTo as? VariableDeclaration (decl?.initializer as? ArrayCreationExpression)?.let { - println("Found a ArrayCreationExpression") + val capacity = it.capacity - val dimension = it.dimensions.first().resolve() - - if (resolvedIndex >= dimension as Int) { + if (resolvedIndex >= capacity) { println("") val sb = AttributedStringBuilder() sb.append("--- FINDING: Out of bounds access in ") @@ -79,7 +77,7 @@ class OutOfBoundsCheck { ) sb.append(decl.name, DEFAULT.foreground(AttributedStyle.YELLOW)) sb.append( - ", an array of length ${AttributedString(""+dimension, DEFAULT.foreground(AttributedStyle.CYAN)).toAnsi()} ---" + ", an array of length ${AttributedString(""+capacity, DEFAULT.foreground(AttributedStyle.CYAN)).toAnsi()} ---" ) val header = sb.toAnsi() diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Neo4jPlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Neo4jPlugin.kt index 9280315cdb..8a5bbd9dad 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Neo4jPlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Neo4jPlugin.kt @@ -33,11 +33,11 @@ import org.jetbrains.kotlinx.ki.shell.configuration.ReplConfiguration class Neo4jPlugin : Plugin { inner class Load(conf: ReplConfiguration) : BaseCommand() { - override val name: String by conf.get(default = "neo4j") - override val short: String by conf.get(default = "n") - override val description: String = "persists the graph into neo4j" + override val name: String by conf.get(default = "export") + override val short: String by conf.get(default = "e") + override val description: String = "exports the graph into neo4j" - override val params = "" + override val params = "neo4j" override fun execute(line: String): Command.Result { val p = line.indexOf(' ') diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt index 66cc91ab8b..8a9b289065 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt @@ -49,6 +49,8 @@ class TranslatePlugin : Plugin { // basics "import de.fraunhofer.aisec.cpg.TranslationConfiguration", "import de.fraunhofer.aisec.cpg.TranslationManager", + // go + "import de.fraunhofer.aisec.cpg.frontends.golang.GoLanguageFrontend", // all the graph nodes "import de.fraunhofer.aisec.cpg.graph.*", "import de.fraunhofer.aisec.cpg.graph.declarations.*", @@ -62,15 +64,17 @@ class TranslatePlugin : Plugin { "import de.fraunhofer.aisec.cpg.analysis.byName", "import de.fraunhofer.aisec.cpg.analysis.body", "import de.fraunhofer.aisec.cpg.analysis.printCode", + "import de.fraunhofer.aisec.cpg.analysis.capacity", // some basic java stuff "import java.io.File", // lets build and analyze - "val config =\n" + + "@OptIn(de.fraunhofer.aisec.cpg.ExperimentalGolang::class) val config =\n" + " TranslationConfiguration.builder()\n" + " .sourceLocations(File(\"" + path + "\"))\n" + " .defaultLanguages()\n" + + " .registerLanguage(GoLanguageFrontend::class.java, GoLanguageFrontend.GOLANG_EXTENSIONS)" + " .defaultPasses()\n" + " .build()", "val analyzer = TranslationManager.builder().config(config).build()", diff --git a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt index 7c892d99c2..65cf228b69 100644 --- a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt +++ b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt @@ -25,11 +25,14 @@ */ package de.fraunhofer.aisec.cpg.analysis +import de.fraunhofer.aisec.cpg.ExperimentalGolang import de.fraunhofer.aisec.cpg.TranslationConfiguration import de.fraunhofer.aisec.cpg.TranslationManager +import de.fraunhofer.aisec.cpg.frontends.golang.GoLanguageFrontend import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression import java.io.File import org.junit.jupiter.api.Test @@ -64,6 +67,27 @@ class AnalysisTest { NullPointerCheck().run(result) } + @OptIn(ExperimentalGolang::class) + @Test + fun testResolve() { + val config = + TranslationConfiguration.builder() + .sourceLocations(File("src/test/resources/struct.go")) + .defaultPasses() + .defaultLanguages() + .registerLanguage( + GoLanguageFrontend::class.java, + GoLanguageFrontend.GOLANG_EXTENSIONS + ) + .build() + + val analyzer = TranslationManager.builder().config(config).build() + val result = analyzer.analyze().get() + + var resolved = result.all().map { it.base.resolve() } + println(resolved) + } + @Test fun testAttribute() { val config = diff --git a/cpg-console/src/test/resources/array.go b/cpg-console/src/test/resources/array.go new file mode 100644 index 0000000000..ebb002dd3d --- /dev/null +++ b/cpg-console/src/test/resources/array.go @@ -0,0 +1,7 @@ +package main + +func main() { + var a []int = make([]int, 10) + + a[11] = 0 +} diff --git a/cpg-console/src/test/resources/nullptr.cpp b/cpg-console/src/test/resources/nullptr.cpp new file mode 100644 index 0000000000..96a6cc5dad --- /dev/null +++ b/cpg-console/src/test/resources/nullptr.cpp @@ -0,0 +1,8 @@ +int main() { + SomeClass* a = new SomeClass(); + a->doSomething(); + + a = nullptr; + + a->doSomethingElse(); +} \ No newline at end of file diff --git a/cpg-library/src/main/golang/expressions.go b/cpg-library/src/main/golang/expressions.go index a3a59e97b2..947f05769c 100644 --- a/cpg-library/src/main/golang/expressions.go +++ b/cpg-library/src/main/golang/expressions.go @@ -37,6 +37,7 @@ type Expression Statement type CallExpression Expression type NewExpression Expression type ArrayCreationExpression Expression +type ArraySubscriptionExpression Expression type ConstructExpression Expression type MemberCallExpression CallExpression type MemberExpression Expression @@ -110,6 +111,19 @@ func NewArrayCreationExpression(fset *token.FileSet, astNode ast.Node) *ArrayCre return (*ArrayCreationExpression)(c) } +func NewArraySubscriptionExpression(fset *token.FileSet, astNode ast.Node) *ArraySubscriptionExpression { + c, err := env.NewObject("de/fraunhofer/aisec/cpg/graph/statements/expressions/ArraySubscriptionExpression") + if err != nil { + log.Fatal(err) + + } + + updateCode(fset, (*Node)(c), astNode) + updateLocation(fset, (*Node)(c), astNode) + + return (*ArraySubscriptionExpression)(c) +} + func NewConstructExpression(fset *token.FileSet, astNode ast.Node) *ConstructExpression { c, err := env.NewObject("de/fraunhofer/aisec/cpg/graph/statements/expressions/ConstructExpression") if err != nil { @@ -281,6 +295,14 @@ func (r *ArrayCreationExpression) AddDimension(e *Expression) { (*jnigi.ObjectRef)(r).CallMethod(env, "addDimension", jnigi.Void, (*jnigi.ObjectRef)(e).Cast("de/fraunhofer/aisec/cpg/graph/statements/expressions/Expression")) } +func (r *ArraySubscriptionExpression) SetArrayExpression(e *Expression) { + (*jnigi.ObjectRef)(r).CallMethod(env, "setArrayExpression", jnigi.Void, (*jnigi.ObjectRef)(e).Cast("de/fraunhofer/aisec/cpg/graph/statements/expressions/Expression")) +} + +func (r *ArraySubscriptionExpression) SetSubscriptExpression(e *Expression) { + (*jnigi.ObjectRef)(r).CallMethod(env, "setSubscriptExpression", jnigi.Void, (*jnigi.ObjectRef)(e).Cast("de/fraunhofer/aisec/cpg/graph/statements/expressions/Expression")) +} + func (c *ConstructExpression) AddArgument(e *Expression) { (*jnigi.ObjectRef)(c).CallMethod(env, "addArgument", jnigi.Void, (*jnigi.ObjectRef)(e).Cast("de/fraunhofer/aisec/cpg/graph/statements/expressions/Expression")) } diff --git a/cpg-library/src/main/golang/frontend/handler.go b/cpg-library/src/main/golang/frontend/handler.go index a96e4ded2d..6b20a846b5 100644 --- a/cpg-library/src/main/golang/frontend/handler.go +++ b/cpg-library/src/main/golang/frontend/handler.go @@ -495,6 +495,8 @@ func (this *GoLanguageFrontend) handleExpr(fset *token.FileSet, expr ast.Expr) * switch v := expr.(type) { case *ast.CallExpr: return (*cpg.Expression)(this.handleCallExpr(fset, v)) + case *ast.IndexExpr: + return (*cpg.Expression)(this.handleIndexExpr(fset, v)) case *ast.BinaryExpr: return (*cpg.Expression)(this.handleBinaryExpr(fset, v)) case *ast.UnaryExpr: @@ -722,6 +724,15 @@ func (this *GoLanguageFrontend) handleCallExpr(fset *token.FileSet, callExpr *as return (*cpg.Expression)(c) } +func (this *GoLanguageFrontend) handleIndexExpr(fset *token.FileSet, indexExpr *ast.IndexExpr) *cpg.Expression { + a := cpg.NewArraySubscriptionExpression(fset, indexExpr) + + a.SetArrayExpression(this.handleExpr(fset, indexExpr.X)) + a.SetSubscriptExpression(this.handleExpr(fset, indexExpr.Index)) + + return (*cpg.Expression)(a) +} + func (this *GoLanguageFrontend) handleNewExpr(fset *token.FileSet, callExpr *ast.CallExpr) *cpg.Expression { n := cpg.NewNewExpression(fset, callExpr) diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt index 0d011be94e..311346cb56 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -221,7 +221,8 @@ open class Node : IVisitable, Persistable { } override fun toString(): String { - val builder = ToStringBuilder(this, TO_STRING_STYLE) + val builder = + ToStringBuilder(this, TO_STRING_STYLE).append("@type", this.javaClass.simpleName) if (name != "") { builder.append("name", name) diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/ParamVariableDeclaration.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/ParamVariableDeclaration.java index 5007d66215..3ca7a83290 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/ParamVariableDeclaration.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/ParamVariableDeclaration.java @@ -27,12 +27,13 @@ import de.fraunhofer.aisec.cpg.graph.SubGraph; import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression; +import org.jetbrains.annotations.NotNull; import org.neo4j.ogm.annotation.Relationship; /** A declaration of a function parameter. */ public class ParamVariableDeclaration extends ValueDeclaration { - private boolean variadic = false; + @NotNull private boolean variadic = false; @Relationship(value = "DEFAULT", direction = "OUTGOING") @SubGraph("AST") diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.java index 0bb02b4652..f9fee9e8fa 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/declarations/ValueDeclaration.java @@ -186,11 +186,7 @@ public void refreshType() { @NotNull @Override public String toString() { - return new ToStringBuilder(this, Node.TO_STRING_STYLE) - .appendSuper(super.toString()) - .append("type", type) - // .append("possibleSubTypes", possibleSubTypes) - .toString(); + return new ToStringBuilder(this, Node.TO_STRING_STYLE).appendSuper(super.toString()).toString(); } @Override diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeclaredReferenceExpression.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeclaredReferenceExpression.java index f4b33f180f..3f325bda42 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeclaredReferenceExpression.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/DeclaredReferenceExpression.java @@ -155,9 +155,7 @@ public void possibleSubTypesChanged(HasType src, HasType root, Set oldSubT @Override public String toString() { return new ToStringBuilder(this, Node.TO_STRING_STYLE) - .append("name", getName()) - .append("type", type) - .append("location", getLocation()) + .append(super.toString()) .append("refersTo", refersTo) .toString(); } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/Expression.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/Expression.java index 0f45ac43d8..d4bc92a207 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/Expression.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/statements/expressions/Expression.java @@ -64,7 +64,8 @@ public class Expression extends Statement implements HasType { @Override public Type getType() { - // just to make sure that we REALLY always return a valid type in case this somehow gets set to + // just to make sure that we REALLY always return a valid type in case this + // somehow gets set to // null return type != null ? type : UnknownType.getUnknownType(); } @@ -89,7 +90,8 @@ public void updatePossibleSubtypes(Set types) { @Override public void setType(Type type, HasType root) { - // TODO Document this method. It is called very often (potentially for each AST node) and + // TODO Document this method. It is called very often (potentially for each AST + // node) and // performs less than optimal. if (type == null || root == this) { return; @@ -215,7 +217,6 @@ public String toString() { return new ToStringBuilder(this, Node.TO_STRING_STYLE) .appendSuper(super.toString()) .append("type", type) - .append("possibleSubTypes", possibleSubTypes) .toString(); } diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/Type.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/Type.java index 7afeb9f739..3d7012fb53 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/Type.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/Type.java @@ -352,6 +352,9 @@ public int hashCode() { @NotNull @Override public String toString() { - return new ToStringBuilder(this, TO_STRING_STYLE).append("name", getName()).toString(); + return new ToStringBuilder(this, TO_STRING_STYLE) + .append("@type", this.getClass().getSimpleName()) + .append("name", getName()) + .toString(); } } diff --git a/tutorial.md b/tutorial.md index ff550946f7..d798115069 100644 --- a/tutorial.md +++ b/tutorial.md @@ -1,8 +1,18 @@ -# CPG Console +# Code Propery Graph -While the CPG is mostly used as a libary in external tools, we decided to showcase its functionalities with a simple CLI based console that can be used to query the graph and run simple analysis steps. +Ensuring the correct behavior of software is crucial to avoid security issues stemming from incorrect implementations. In this video, we present our CPG tool, a language-independent analysis platform for source code based on an adaption of a code property graph. Our platform has support for multiple passes that can extend the analysis after the graph is constructed and it currently supports C/C++, Java and has experimental support for Golang and Python. -To launch the console, built it and then run `bin/cpg-console`. You will be greeted by the interactive prompt of our CPG console, which is implemented by the kotlin `ki` interactive shell. The commands on this shell follow the Kotlin language. For more information please see the [Kotlin documentation](https://kotlinlang.org/docs/home.html). +(TODO: Insert slides here) + +## What is a Code Property Graph? + +A code property graph (CPG) is a representation of source code in form of a labelled directed multigraph. Think of it as a graph where each node and edge is assigned a set of key-value pairs, called properties. This representation is supported by a range of graph databases and can be used to store source code of a program in a searchable data structure. Thus, the code property graph allows to use existing graph query languages in order to either manually navigate through interesting parts of the source code or to automatically find "interesting" patterns. + +## CPG Console + +While the CPG tool is mostly used as a libary in external tools, such as [Codyze](http://github.com/Fraunhofer-AISEC/codyze), we decided to showcase its functionalities with a simple CLI based console that can be used to query the graph and run simple analysis steps. + +To launch the console, built it according to the instructions in our `README.md` and then run `bin/cpg-console`. You will be greeted by the interactive prompt of our CPG console, which is implemented by the kotlin `ki` interactive shell. The commands on this shell follow the Kotlin language. For more information please see the [Kotlin documentation](https://kotlinlang.org/docs/home.html). In addition to that, commands prefixed by a `:` are plugin commands. To get a list of all available plugins use the `:h` or `:help` command. @@ -17,32 +27,43 @@ type :h for help :help or h [command] print this summary or command-specific help ``` -## Launching the translation plugin +### Launching the translation plugin One such a plugin is the `:translate` command, or `:tr` for short. It allows the translation of source code files into the code property graph. In this example, we will use all files in `src/test/resources` and translate them. -```log +```kotlin [1] :tr src/test/resources -10:15:27,086 INFO TranslationManager Parsing /Users/cpg/Repositories/cpg/cpg-console/src/test/resources/array.cpp -10:15:27,231 INFO CXXLanguageFrontend Parsed 96 bytes corresponding roughly to 1 LoC -10:15:27,231 INFO Benchmark CXXLanguageFrontend Parsing sourcefile done in 94 ms -10:15:27,273 INFO Benchmark CXXLanguageFrontend Transform to CPG done in 41 ms -10:15:27,273 INFO TranslationManager Parsing /Users/cpg/Repositories/cpg/cpg-console/src/test/resources/Array.java -10:15:27,312 INFO JavaLanguageFrontend Source file root used for type solver: /Users/cpg/Repositories/cpg/cpg-console/src/test/resources -10:15:27,343 ERROR TranslationManager Different frontends are used for multiple files. This will very likely break the following passes. -10:15:27,395 INFO Benchmark JavaLanguageFrontend Parsing source file done in 51 ms -10:15:27,452 INFO Benchmark JavaLanguageFrontend Transform to CPG done in 57 ms -10:15:27,452 INFO Benchmark TranslationManager Frontend done in 373 ms -10:15:27,465 INFO Benchmark TypeHierarchyResolver Executing Pass done in 12 ms -10:15:27,467 INFO Benchmark JavaExternalTypeHierarchyResolver Executing Pass done in 2 ms -10:15:27,470 INFO Benchmark ImportResolver Executing Pass done in 2 ms -10:15:27,482 INFO Benchmark VariableUsageResolver Executing Pass done in 11 ms -10:15:27,490 INFO Benchmark CallResolver Executing Pass done in 8 ms -10:15:27,493 INFO Benchmark EvaluationOrderGraphPass Executing Pass done in 2 ms -10:15:27,495 INFO Benchmark TypeResolver Executing Pass done in 1 ms -10:15:27,498 INFO Benchmark ControlFlowSensitiveDFGPass Executing Pass done in 2 ms -10:15:27,499 INFO Benchmark FilenameMapper Executing Pass done in 1 ms -10:15:27,499 INFO Benchmark TranslationManager Translation into full graph done in 420 ms +18:29:01,138 INFO TranslationManager Parsing src/test/resources/array.go +18:29:01,151 INFO TranslationManager Parsing src/test/resources/nullptr.cpp +18:29:01,183 ERROR TranslationManager Different frontends are used for multiple files. This will very likely break the following passes. +18:29:01,284 INFO CXXLanguageFrontend Parsed 114 bytes corresponding roughly to 2 LoC +18:29:01,284 INFO Benchmark CXXLanguageFrontend Parsing sourcefile done in 98 ms +18:29:01,307 INFO Benchmark CXXLanguageFrontend Transform to CPG done in 22 ms +18:29:01,307 INFO TranslationManager Parsing src/test/resources/array.cpp +18:29:01,308 ERROR TranslationManager Different frontends are used for multiple files. This will very likely break the following passes. +18:29:01,310 INFO CXXLanguageFrontend Parsed 174 bytes corresponding roughly to 3 LoC +18:29:01,310 INFO Benchmark CXXLanguageFrontend Parsing sourcefile done in 2 ms +18:29:01,326 INFO Benchmark CXXLanguageFrontend Transform to CPG done in 16 ms +18:29:01,327 INFO TranslationManager Parsing src/test/resources/Array.java +18:29:01,347 INFO JavaLanguageFrontend Source file root used for type solver: src/test/resources +18:29:01,373 ERROR TranslationManager Different frontends are used for multiple files. This will very likely break the following passes. +18:29:01,373 ERROR TranslationManager Different frontends are used for multiple files. This will very likely break the following passes. +18:29:01,373 ERROR TranslationManager Different frontends are used for multiple files. This will very likely break the following passes. +18:29:01,421 INFO Benchmark JavaLanguageFrontend Parsing source file done in 47 ms +18:29:01,472 INFO Benchmark JavaLanguageFrontend Transform to CPG done in 50 ms +18:29:01,472 INFO Benchmark TranslationManager Frontend done in 338 ms +18:29:01,486 INFO Benchmark TypeHierarchyResolver Executing Pass done in 13 ms +18:29:01,488 INFO Benchmark JavaExternalTypeHierarchyResolver Executing Pass done in 1 ms +18:29:01,490 INFO Benchmark ImportResolver Executing Pass done in 1 ms +18:29:01,508 WARN Util src/test/resources/struct.go:6:6: Did not find a declaration for nil +18:29:01,509 WARN Util src/test/resources/nullptr.cpp:5:9: Did not find a declaration for null +18:29:01,513 INFO Benchmark VariableUsageResolver Executing Pass done in 22 ms +18:29:01,521 INFO Benchmark CallResolver Executing Pass done in 7 ms +18:29:01,525 INFO Benchmark EvaluationOrderGraphPass Executing Pass done in 3 ms +18:29:01,527 INFO Benchmark TypeResolver Executing Pass done in 2 ms +18:29:01,531 INFO Benchmark ControlFlowSensitiveDFGPass Executing Pass done in 4 ms +18:29:01,533 INFO Benchmark FilenameMapper Executing Pass done in 1 ms +18:29:01,533 INFO Benchmark TranslationManager Translation into full graph done in 399 ms ``` After the translation is done, several symbols are available on the console, to query and analyse the result. You can use the `:ls` command to get a quick overview. @@ -57,7 +78,7 @@ val tu: de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration! Most interesting for the user are the `result` object which holds the complete translation result, as well as the `tu` object, which is a shortcut to the first translation unit, i.e. the first file that was translated. It is of type `TranslationUnitDeclaration`, which is one of our node type in the graph; the most basic one being a `Node` itself. -## Querying the translation result +### Querying the translation result In the following, we will use the aforementioned objects to query the source code for interesting patterns. To do so, we will explore several built-in functions that can be used in exploring the graph. The first of these, is the `all` function, it returns a list of all nodes that are direct descendents of a particular node, basicically flattening the hierarchy. @@ -66,74 +87,131 @@ In the following, we will use the aforementioned objects to query the source cod res3: List = ... ``` -The output here can be quite huge, so additional filtering is needed. The `all` function takes an additional type parameter, which can be used to further filter nodes of a particular type. In this case, we are interested in all `ArraySubscriptionExpression` nodes, i.e. those that represent access to an element of an array. These operations are often prone to out of bounds errors and we want to explore, whether our code is also affected by that. +The output here can be quite verbose, so additional filtering is needed. The `all` function takes an additional type parameter, which can be used to further filter nodes of a particular type. In this case, we are interested in all `ArraySubscriptionExpression` nodes, i.e. those that represent access to an element of an array. These operations are often prone to out of bounds errors and we want to explore, whether our code is also affected by that. ```kotlin [4] result.all() res4: List = [ - {"location":"array.cpp(6:12-6:18)","type":{"name":"char"}}, - {"location":"Array.java(8:18-8:22)","type":{"name":"char"}} + {"@type":"ArraySubscriptionExpression","location":"array.go(6:2-6:7)","type":{"@type":"ObjectType","name":"int"}}, + {"@type":"ArraySubscriptionExpression","location":"array.cpp(6:12-6:18)","type":{"@type":"ObjectType","name":"char"}}, + {"@type":"ArraySubscriptionExpression","location":"Array.java(8:18-8:22)","type":{"@type":"ObjectType","name":"char"}}, + {"@type":"ArraySubscriptionExpression","location":"array.cpp(12:12-12:16)","type":{"@type":"ObjectType","name":"char"}} ] ``` -Much better. We have found two nodes that represent an array access. To see the corresponding source code of our result, we can prefix our previous command with `:code` or `:c`. This shows the raw source code as well as the location of the file where the code is located. +Much better. We have found four nodes that represent an array access. To see the corresponding source code of our result, we can prefix our previous command with `:code` or `:c`. This shows the raw source code as well as the location of the file where the code is located. ```kotlin [5] :code result.all() ---- src/test/resources/array.cpp:6:12 --- -= c[b] +--- src/test/resources/array.go:6:2 --- + 6: a[11] +------------------------------------------------ + +--- array.cpp:6:12 --- + 6: = c[b] ----------------------------------------------------------------------------------------------- ---- src/test/resources/Array.java:8:18 --- -c[b] +--- Array.java:8:18 --- + 8: c[b] ------------------------------------------------------------------------------------------------ -res5: Collection = [{"location":"array.cpp(6:12-6:18)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]}, {"location":"Array.java(8:18-8:22)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]}] +--- array.cpp:12:12 --- + 12: c[0] +------------------------------------------------------------------------------------------------ ``` -This also demonstrates quite nicely, that queries on the CPG work independently of the programming language. Our test folder contains both Java as well as C++ files and we can analyse them simultaneously. +This also demonstrates quite nicely, that queries on the CPG work independently of the programming language. Our test folder contains Java, Go and C++ files and we can analyse all of them simultaneously. + +### Looking for software errors + +In a next step, we want to identify, which of those expression are accessing an array index that is greater than its capacity, thus leading to an error. From the code output before we can already see that two expressions are using a static value of `0` and `11`, but the other two are using a variable `b`. Using the `resolve` function, we can try to resolve the `b` variable, to check if it has a constant value. In this case we are in luck and we see that, next to the `0` and `11` we already know, the other two expression were resolved to `5`. ```kotlin -[24] result.all().map { it.subscriptExpression.resolve() } -res21: List = [5, 5, 0] +[6] result.all().map { it.subscriptExpression.resolve() } +res6: List = [11, 5, 5, 0] ``` -TODO: make it simpler to query prev DFG with type +In a next step, we want to check to capacity of the array the expression is referring to. We can make use of two helper functions `dfgFrom` and `capacity` to quickly check this. + ```kotlin -[30] result.all().map { Triple( +[7] var expr = result.all().map { Triple( it.subscriptExpression.resolve() as Int, - it.arrayExpression.dfgFrom().first().dimensions.first().resolve() as Int, + it.arrayExpression.dfgFrom().first().capacity, it ) } -res21: List> = [ - (5, 4, {"location":"array.cpp(6:12-6:18)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]}), - (5, 4, {"location":"Array.java(8:18-8:22)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]}), - (0, 100, {"location":"array.cpp(12:12-12:16)","type":{"name":"char"},"possibleSubTypes":[{"name":"char"}]}) +res7: List> = [ + (11, 10, {"@type":"ArraySubscriptionExpression","location":"array.go(6:2-6:7)","type":{"@type":"ObjectType","name":"int"}}), + (5, 4, {"@type":"ArraySubscriptionExpression","location":"array.cpp(6:12-6:18)","type":{"@type":"ObjectType","name":"char"}}), + (5, 4, {"@type":"ArraySubscriptionExpression","location":"Array.java(8:18-8:22)","type":{"@type":"ObjectType","name":"char"}}), + (0, 100, {"@type":"ArraySubscriptionExpression","location":"array.cpp(12:12-12:16)","type":{"@type":"ObjectType","name":"char"}}) ] ``` -We use that result to filter those where the resolved index is greater or equal to our dimension. +Lastly, we can make use of the `filter` function to return only those expressions where the resolved index is greater or equal to the capacity, leading to an out of bounds error and a possible program crash. Using the alredy known `:code` command, we can also show the relevant code locations. ```kotlin -[31] res39.filter { it.first >= it.second } -res41: List> = [(5, 4, {"location":"array.cpp(6:12-6:18)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]}), (5, 4, {"location":"Array.java(8:18-8:22)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]})] -``` +[8] expr.filter { it.first >= it.second } +res8: List> = [(5, 4, {"location":"array.cpp(6:12-6:18)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]}), (5, 4, {"location":"Array.java(8:18-8:22)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]})] -```kotlin -[31] :code res39.filter { it.first >= it.second }.map { it.third } ---- /Users/chr55316/Repositories/cpg/cpg-console/src/test/resources/array.cpp:6:12 --- +[9] :code expr.filter { it.first >= it.second }.map { it.third } +--- src/test/resources/array.cpp:6:12 --- = c[b] ----------------------------------------------------------------------------------------------- ---- /Users/chr55316/Repositories/cpg/cpg-console/src/test/resources/Array.java:8:18 --- +--- src/test/resources/Array.java:8:18 --- c[b] ------------------------------------------------------------------------------------------------ ``` -Of course the same can also be achieved in one, slightly larger query. + +### Futher analysis + +Because the way we have shown can be quite tedious, we already included several example analyis steps that can be perfomed on the currently loaded graph simply by executing the `:run` command. This includes the aforementioned check for out of bounds as well as check for null pointers. + +```kotlin +[10] :run + +--- FINDING: Out of bounds access in ArrayCreationExpression when accessing index 11 of a, an array of length 10 --- +src/test/resources/array.go:6:2: a[11] + +The following path was discovered that leads to 11 being 11: +src/test/resources/array.go:6:4: 11 + +--- FINDING: Out of bounds access in ArrayCreationExpression when accessing index 5 of c, an array of length 4 --- +src/test/resources/array.cpp:6:12: = c[b] + +The following path was discovered that leads to b being 5: +src/test/resources/array.cpp:6:16: b +src/test/resources/array.cpp:4:5: int b = a + 1; +src/test/resources/array.cpp:4:11: = a + 1 +src/test/resources/array.cpp:4:13: a +src/test/resources/array.cpp:3:5: int a = 4; +src/test/resources/array.cpp:3:11: = 4 +src/test/resources/array.cpp:4:17: 1 +``` + +Lastly, it is also possible to export the complete graph structure to a graph database, such as Neo4J with a simple `:export` command. ```kotlin -[6] result.all().filter { - (it.subscriptExpression.resolve() as Int) >= - ((it.arrayExpression.prevDFG.first() as? ArrayCreationExpression)?.dimensions?.first()?.resolve() as Int) -} +[11] :export neo4j +19:26:41,642 INFO Application Using import depth: -1 +19:26:41,643 INFO Application Count base nodes to save: 4 +Jun 08, 2021 7:26:41 PM org.neo4j.driver.internal.logging.JULogger info +INFO: Direct driver instance 1156771703 created for server address localhost:7687 +19:26:42,006 INFO DomainInfo Starting Post-processing phase +19:26:42,006 INFO DomainInfo Building byLabel lookup maps +19:26:42,006 INFO DomainInfo Building interface class map for 106 classes +19:26:42,027 INFO DomainInfo Post-processing complete +19:26:44,471 INFO BoltDriver Shutting down Bolt driver org.neo4j.driver.internal.InternalDriver@44f2ef77 +Jun 08, 2021 7:26:44 PM org.neo4j.driver.internal.logging.JULogger info +INFO: Closing driver instance 1156771703 +Jun 08, 2021 7:26:44 PM org.neo4j.driver.internal.logging.JULogger info +INFO: Closing connection pool towards localhost:7687 ``` + +Then, additional tools, such as the Neo4j browser can be used to further explore the graph. + +(TODO: screenshot of neo4j browser) + +## Conclusion + +In conclusion, the CPG tool can be used to \ No newline at end of file From 8d80537c3670666b0cec653ff8779e2c5a4d2dcd Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Tue, 8 Jun 2021 19:36:46 +0200 Subject: [PATCH 21/30] ++ --- tutorial.md | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tutorial.md b/tutorial.md index d798115069..b38e8aac18 100644 --- a/tutorial.md +++ b/tutorial.md @@ -139,7 +139,8 @@ In a next step, we want to check to capacity of the array the expression is refe it.arrayExpression.dfgFrom().first().capacity, it ) } -res7: List> = [ +[8]: expr +res8: List> = [ (11, 10, {"@type":"ArraySubscriptionExpression","location":"array.go(6:2-6:7)","type":{"@type":"ObjectType","name":"int"}}), (5, 4, {"@type":"ArraySubscriptionExpression","location":"array.cpp(6:12-6:18)","type":{"@type":"ObjectType","name":"char"}}), (5, 4, {"@type":"ArraySubscriptionExpression","location":"Array.java(8:18-8:22)","type":{"@type":"ObjectType","name":"char"}}), @@ -150,25 +151,33 @@ res7: List= it.second } -res8: List> = [(5, 4, {"location":"array.cpp(6:12-6:18)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]}), (5, 4, {"location":"Array.java(8:18-8:22)","type":{"name":"char"},"possibleSubTypes":["UNKNOWN",{"name":"char"}]})] +[9] expr.filter { it.first >= it.second } +res8: List> = [ + (11, 10, {"@type":"ArraySubscriptionExpression","location":"array.go(6:2-6:7)","type":{"@type":"ObjectType","name":"int"}}), + (5, 4, {"@type":"ArraySubscriptionExpression","location":"array.cpp(6:12-6:18)","type":{"@type":"ObjectType","name":"char"}}), + (5, 4, {"@type":"ArraySubscriptionExpression","location":"Array.java(8:18-8:22)","type":{"@type":"ObjectType","name":"char"}}) +] + +[10] :code expr.filter { it.first >= it.second }.map { it.third } +--- src/test/resources/array.go:6:2 --- + 6: a[11] +------------------------------------------------ -[9] :code expr.filter { it.first >= it.second }.map { it.third } --- src/test/resources/array.cpp:6:12 --- -= c[b] + 6: = c[b] ----------------------------------------------------------------------------------------------- --- src/test/resources/Array.java:8:18 --- -c[b] + 8: c[b] ------------------------------------------------------------------------------------------------ ``` ### Futher analysis -Because the way we have shown can be quite tedious, we already included several example analyis steps that can be perfomed on the currently loaded graph simply by executing the `:run` command. This includes the aforementioned check for out of bounds as well as check for null pointers. +Because the way we have shown can be quite tedious, we already included several example analyis steps that can be perfomed on the currently loaded graph simply by executing the `:run` command. This includes the aforementioned check for out of bounds as well as check for null pointers and will be extended in the future. ```kotlin -[10] :run +[11] :run --- FINDING: Out of bounds access in ArrayCreationExpression when accessing index 11 of a, an array of length 10 --- src/test/resources/array.go:6:2: a[11] @@ -192,7 +201,7 @@ src/test/resources/array.cpp:4:17: 1 Lastly, it is also possible to export the complete graph structure to a graph database, such as Neo4J with a simple `:export` command. ```kotlin -[11] :export neo4j +[12] :export neo4j 19:26:41,642 INFO Application Using import depth: -1 19:26:41,643 INFO Application Count base nodes to save: 4 Jun 08, 2021 7:26:41 PM org.neo4j.driver.internal.logging.JULogger info From 2dc1d845308255cd7b5a8dd155aa2c7f6d569400 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Tue, 8 Jun 2021 19:40:10 +0200 Subject: [PATCH 22/30] ++ --- tutorial.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tutorial.md b/tutorial.md index b38e8aac18..9d0ae04fd3 100644 --- a/tutorial.md +++ b/tutorial.md @@ -1,8 +1,8 @@ -# Code Propery Graph +# Code Property Graph Ensuring the correct behavior of software is crucial to avoid security issues stemming from incorrect implementations. In this video, we present our CPG tool, a language-independent analysis platform for source code based on an adaption of a code property graph. Our platform has support for multiple passes that can extend the analysis after the graph is constructed and it currently supports C/C++, Java and has experimental support for Golang and Python. -(TODO: Insert slides here) +(TODO: Insert finished video here) ## What is a Code Property Graph? From 3c9149ee1333c7208d66a98d7f2d0d82e9fd017d Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Tue, 8 Jun 2021 19:54:24 +0200 Subject: [PATCH 23/30] ++ --- .../aisec/cpg/analysis/AnalysisTest.kt | 21 ---------------- tutorial.md | 24 ++++++++++++------- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt index 65cf228b69..0c43fcbe9a 100644 --- a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt +++ b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt @@ -67,27 +67,6 @@ class AnalysisTest { NullPointerCheck().run(result) } - @OptIn(ExperimentalGolang::class) - @Test - fun testResolve() { - val config = - TranslationConfiguration.builder() - .sourceLocations(File("src/test/resources/struct.go")) - .defaultPasses() - .defaultLanguages() - .registerLanguage( - GoLanguageFrontend::class.java, - GoLanguageFrontend.GOLANG_EXTENSIONS - ) - .build() - - val analyzer = TranslationManager.builder().config(config).build() - val result = analyzer.analyze().get() - - var resolved = result.all().map { it.base.resolve() } - println(resolved) - } - @Test fun testAttribute() { val config = diff --git a/tutorial.md b/tutorial.md index 9d0ae04fd3..42aad42223 100644 --- a/tutorial.md +++ b/tutorial.md @@ -12,7 +12,7 @@ A code property graph (CPG) is a representation of source code in form of a labe While the CPG tool is mostly used as a libary in external tools, such as [Codyze](http://github.com/Fraunhofer-AISEC/codyze), we decided to showcase its functionalities with a simple CLI based console that can be used to query the graph and run simple analysis steps. -To launch the console, built it according to the instructions in our `README.md` and then run `bin/cpg-console`. You will be greeted by the interactive prompt of our CPG console, which is implemented by the kotlin `ki` interactive shell. The commands on this shell follow the Kotlin language. For more information please see the [Kotlin documentation](https://kotlinlang.org/docs/home.html). +To launch the console, first build it according to the instructions in our `README.md` and then run `bin/cpg-console`. You will be greeted by the interactive prompt of our console, which is implemented by the kotlin `ki` interactive shell. The commands on this shell follow the syntax of the Kotlin language. For more information please see the [Kotlin documentation](https://kotlinlang.org/docs/home.html). In addition to that, commands prefixed by a `:` are plugin commands. To get a list of all available plugins use the `:h` or `:help` command. @@ -66,7 +66,7 @@ One such a plugin is the `:translate` command, or `:tr` for short. It allows the 18:29:01,533 INFO Benchmark TranslationManager Translation into full graph done in 399 ms ``` -After the translation is done, several symbols are available on the console, to query and analyse the result. You can use the `:ls` command to get a quick overview. +After the translation is done, several symbols are available on the console, to query and analyse the translation result. You can use the `:ls` command to get a quick overview. ```kotlin [2] :ls @@ -76,7 +76,7 @@ val result: de.fraunhofer.aisec.cpg.TranslationResult! val tu: de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration! ``` -Most interesting for the user are the `result` object which holds the complete translation result, as well as the `tu` object, which is a shortcut to the first translation unit, i.e. the first file that was translated. It is of type `TranslationUnitDeclaration`, which is one of our node type in the graph; the most basic one being a `Node` itself. +Most interesting for the user is the `result` object which holds the complete translation result, as well as the `tu` object, which is a shortcut to the first translation unit, i.e. the first file that was translated. It is of type `TranslationUnitDeclaration`, which is one of our node types in the graph; the most basic one being a `Node` itself. ### Querying the translation result @@ -124,14 +124,16 @@ This also demonstrates quite nicely, that queries on the CPG work independently ### Looking for software errors -In a next step, we want to identify, which of those expression are accessing an array index that is greater than its capacity, thus leading to an error. From the code output before we can already see that two expressions are using a static value of `0` and `11`, but the other two are using a variable `b`. Using the `resolve` function, we can try to resolve the `b` variable, to check if it has a constant value. In this case we are in luck and we see that, next to the `0` and `11` we already know, the other two expression were resolved to `5`. +In a next step, we want to identify, which of those expression are accessing an array index that is greater than its capacity, thus leading to an error. From the code output we have seen before we can already identify two array indicies: `0` and `11`. But the other two are using a variable `b` as the index. Using the `resolve` function, we can try to resolve the variable `b`, to check if it has a constant value. ```kotlin [6] result.all().map { it.subscriptExpression.resolve() } res6: List = [11, 5, 5, 0] ``` -In a next step, we want to check to capacity of the array the expression is referring to. We can make use of two helper functions `dfgFrom` and `capacity` to quickly check this. +In this case we are in luck and we see that, next to the `0` and `11` we already know, the other two expression were resolved to `5`. + +In a next step, we want to check to capacity of the array the access is referring to. We can make use of two helper functions `dfgFrom` and `capacity` to quickly check this, using the built-in data flow analysis. ```kotlin [7] var expr = result.all().map { Triple( @@ -148,7 +150,9 @@ res8: List= it.second } @@ -157,7 +161,11 @@ res8: List= it.second }.map { it.third } --- src/test/resources/array.go:6:2 --- 6: a[11] @@ -174,7 +182,7 @@ res8: List Date: Tue, 8 Jun 2021 19:54:39 +0200 Subject: [PATCH 24/30] ++ --- tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorial.md b/tutorial.md index 42aad42223..2a1762b7ed 100644 --- a/tutorial.md +++ b/tutorial.md @@ -231,4 +231,4 @@ Then, additional tools, such as the Neo4j browser can be used to further explore ## Conclusion -In conclusion, the CPG tool can be used to translate source code of different programming languages to a uniform, language-independed represetation in the form of a code property graph. It can either be used as a library, in which it forms the underlying basis of the [Codyze](http://github.com/Fraunhofer-AISEC/codyze) analyizer or it it's console can be used to quickly explore source code and find weaknesses. +In conclusion, the CPG tool can be used to translate source code of different programming languages to a uniform, language-independed represetation in the form of a code property graph. It can either be used as a library, in which it forms the underlying basis of the [Codyze](http://github.com/Fraunhofer-AISEC/codyze) analyizer or it's console can be used to quickly explore source code and find weaknesses. From 3bd434f4035cf4115b1967a76edb44fb2f116273 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Wed, 9 Jun 2021 19:08:43 +0200 Subject: [PATCH 25/30] better tostring --- .../aisec/cpg/analysis/Extensions.kt | 29 ++++++++++++------- .../aisec/cpg/analysis/AnalysisTest.kt | 3 -- tutorial.md | 2 ++ 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt index e0b0f69132..5ad285b00b 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt @@ -60,6 +60,13 @@ fun Node.printCode(linesAhead: Int = 0, showNumbers: Boolean = false): Node { return this } +class NodeList(list: Collection) : ArrayList(list) { + + override fun toString(): String { + return this.joinToString(", \n ", "[\n ", "\n]") + } +} + fun Collection.printCode( linesAhead: Int = 0, showNumbers: Boolean = false @@ -89,40 +96,40 @@ val ArrayCreationExpression.capacity: Int } @JvmName("allNodes") -fun TranslationResult.all(): List { +fun TranslationResult.all(): NodeList { return this.all() } -inline fun TranslationResult.all(): List { +inline fun TranslationResult.all(): NodeList { val children = SubgraphWalker.flattenAST(this) - return children.filterIsInstance() + return NodeList(children.filterIsInstance()) } @JvmName("allNodes") -fun Node.all(): List { +fun Node.all(): NodeList { return this.all() } -inline fun Node.all(): List { +inline fun Node.all(): NodeList { val children = SubgraphWalker.flattenAST(this) - return children.filterIsInstance() + return NodeList(children.filterIsInstance()) } @JvmName("astNodes") -fun Node.ast(): List { +fun Node.ast(): NodeList { return this.ast() } -inline fun Node.ast(): List { +inline fun Node.ast(): NodeList { val children = SubgraphWalker.getAstChildren(this) - return children.filterIsInstance() + return NodeList(children.filterIsInstance()) } -inline fun Node.dfgFrom(): List { - return this.prevDFG.toList().filterIsInstance() +inline fun Node.dfgFrom(): NodeList { + return NodeList(this.prevDFG.toList().filterIsInstance()) } @Throws(DeclarationNotFound::class) diff --git a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt index 0c43fcbe9a..7c892d99c2 100644 --- a/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt +++ b/cpg-console/src/test/kotlin/de/fraunhofer/aisec/cpg/analysis/AnalysisTest.kt @@ -25,14 +25,11 @@ */ package de.fraunhofer.aisec.cpg.analysis -import de.fraunhofer.aisec.cpg.ExperimentalGolang import de.fraunhofer.aisec.cpg.TranslationConfiguration import de.fraunhofer.aisec.cpg.TranslationManager -import de.fraunhofer.aisec.cpg.frontends.golang.GoLanguageFrontend import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberCallExpression import java.io.File import org.junit.jupiter.api.Test diff --git a/tutorial.md b/tutorial.md index 2a1762b7ed..f190666caa 100644 --- a/tutorial.md +++ b/tutorial.md @@ -232,3 +232,5 @@ Then, additional tools, such as the Neo4j browser can be used to further explore ## Conclusion In conclusion, the CPG tool can be used to translate source code of different programming languages to a uniform, language-independed represetation in the form of a code property graph. It can either be used as a library, in which it forms the underlying basis of the [Codyze](http://github.com/Fraunhofer-AISEC/codyze) analyizer or it's console can be used to quickly explore source code and find weaknesses. + +It is available as open source on GitHub: https://github.com/Fraunhofer-AISEC/cpg From f53ebf47dcd910427e1479089fbdff50646adbb2 Mon Sep 17 00:00:00 2001 From: "Banse, Christian" Date: Wed, 9 Jun 2021 20:30:21 +0200 Subject: [PATCH 26/30] ++ --- .../aisec/cpg/analysis/Extensions.kt | 29 ++++----- .../aisec/cpg/analysis/MyToJsonStyle.kt | 60 +++++++++++++++++++ .../aisec/cpg/console/CpgConsole.kt | 4 +- .../aisec/cpg/console/TranslatePlugin.kt | 1 + .../de/fraunhofer/aisec/cpg/graph/Node.kt | 3 +- .../aisec/cpg/graph/types/Type.java | 5 +- 6 files changed, 76 insertions(+), 26 deletions(-) create mode 100644 cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MyToJsonStyle.kt diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt index 5ad285b00b..e0b0f69132 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/Extensions.kt @@ -60,13 +60,6 @@ fun Node.printCode(linesAhead: Int = 0, showNumbers: Boolean = false): Node { return this } -class NodeList(list: Collection) : ArrayList(list) { - - override fun toString(): String { - return this.joinToString(", \n ", "[\n ", "\n]") - } -} - fun Collection.printCode( linesAhead: Int = 0, showNumbers: Boolean = false @@ -96,40 +89,40 @@ val ArrayCreationExpression.capacity: Int } @JvmName("allNodes") -fun TranslationResult.all(): NodeList { +fun TranslationResult.all(): List { return this.all() } -inline fun TranslationResult.all(): NodeList { +inline fun TranslationResult.all(): List { val children = SubgraphWalker.flattenAST(this) - return NodeList(children.filterIsInstance()) + return children.filterIsInstance() } @JvmName("allNodes") -fun Node.all(): NodeList { +fun Node.all(): List { return this.all() } -inline fun Node.all(): NodeList { +inline fun Node.all(): List { val children = SubgraphWalker.flattenAST(this) - return NodeList(children.filterIsInstance()) + return children.filterIsInstance() } @JvmName("astNodes") -fun Node.ast(): NodeList { +fun Node.ast(): List { return this.ast() } -inline fun Node.ast(): NodeList { +inline fun Node.ast(): List { val children = SubgraphWalker.getAstChildren(this) - return NodeList(children.filterIsInstance()) + return children.filterIsInstance() } -inline fun Node.dfgFrom(): NodeList { - return NodeList(this.prevDFG.toList().filterIsInstance()) +inline fun Node.dfgFrom(): List { + return this.prevDFG.toList().filterIsInstance() } @Throws(DeclarationNotFound::class) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MyToJsonStyle.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MyToJsonStyle.kt new file mode 100644 index 0000000000..9d91dbb789 --- /dev/null +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MyToJsonStyle.kt @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2021, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.analysis + +import org.apache.commons.lang3.builder.ToStringStyle + +// ---------------------------------------------------------------------------- +class MultiLineToStringStyle internal constructor() : ToStringStyle() { + /** + * + * Ensure `Singleton` after serialization. + * + * @return the singleton + */ + private fun readResolve(): Any { + return MULTI_LINE_STYLE + } + + companion object { + private const val serialVersionUID = 1L + } + + /** + * + * Constructor. + * + * Use the static constant rather than instantiating. + */ + init { + this.isUseIdentityHashCode = false + this.isUseShortClassName = true + contentStart = "{" + fieldSeparator = System.lineSeparator() + " " + this.isFieldSeparatorAtStart = true + contentEnd = System.lineSeparator() + "}" + } +} diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt index 674f68d517..62f3c792cf 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/CpgConsole.kt @@ -25,6 +25,7 @@ */ package de.fraunhofer.aisec.cpg.console +import de.fraunhofer.aisec.cpg.analysis.MultiLineToStringStyle import de.fraunhofer.aisec.cpg.graph.Node import kotlin.script.experimental.api.ScriptCompilationConfiguration import kotlin.script.experimental.api.ScriptEvaluationConfiguration @@ -32,7 +33,6 @@ import kotlin.script.experimental.jvm.baseClassLoader import kotlin.script.experimental.jvm.defaultJvmScriptingHostConfiguration import kotlin.script.experimental.jvm.dependenciesFromClassloader import kotlin.script.experimental.jvm.jvm -import org.apache.commons.lang3.builder.ToStringStyle import org.jetbrains.kotlinx.ki.shell.KotlinShell import org.jetbrains.kotlinx.ki.shell.Plugin import org.jetbrains.kotlinx.ki.shell.Shell @@ -43,7 +43,7 @@ import org.jetbrains.kotlinx.ki.shell.configuration.ReplConfigurationBase object CpgConsole { @JvmStatic fun main(args: Array) { - Node.TO_STRING_STYLE = ToStringStyle.JSON_STYLE + Node.TO_STRING_STYLE = MultiLineToStringStyle() val repl = Shell( diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt index 8a9b289065..bfc89cd772 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt @@ -57,6 +57,7 @@ class TranslatePlugin : Plugin { "import de.fraunhofer.aisec.cpg.graph.statements.*", "import de.fraunhofer.aisec.cpg.graph.statements.expressions.*", // helper builtins + "import de.fraunhofer.aisec.cpg.analysis.NodeList", "import de.fraunhofer.aisec.cpg.analysis.resolve", "import de.fraunhofer.aisec.cpg.analysis.all", "import de.fraunhofer.aisec.cpg.analysis.ast", diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt index 311346cb56..0d011be94e 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -221,8 +221,7 @@ open class Node : IVisitable, Persistable { } override fun toString(): String { - val builder = - ToStringBuilder(this, TO_STRING_STYLE).append("@type", this.javaClass.simpleName) + val builder = ToStringBuilder(this, TO_STRING_STYLE) if (name != "") { builder.append("name", name) diff --git a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/Type.java b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/Type.java index 3d7012fb53..7afeb9f739 100644 --- a/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/Type.java +++ b/cpg-library/src/main/java/de/fraunhofer/aisec/cpg/graph/types/Type.java @@ -352,9 +352,6 @@ public int hashCode() { @NotNull @Override public String toString() { - return new ToStringBuilder(this, TO_STRING_STYLE) - .append("@type", this.getClass().getSimpleName()) - .append("name", getName()) - .toString(); + return new ToStringBuilder(this, TO_STRING_STYLE).append("name", getName()).toString(); } } From a0be96cbaaedf734fca093dca7a9ecc12fcc43bf Mon Sep 17 00:00:00 2001 From: Konrad Weiss Date: Mon, 21 Jun 2021 18:31:44 +0200 Subject: [PATCH 27/30] Deactivate go as default language --- .../kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt index bfc89cd772..aaa2303acf 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt @@ -75,7 +75,9 @@ class TranslatePlugin : Plugin { path + "\"))\n" + " .defaultLanguages()\n" + - " .registerLanguage(GoLanguageFrontend::class.java, GoLanguageFrontend.GOLANG_EXTENSIONS)" + + // " + // .registerLanguage(GoLanguageFrontend::class.java, + // GoLanguageFrontend.GOLANG_EXTENSIONS)" + " .defaultPasses()\n" + " .build()", "val analyzer = TranslationManager.builder().config(config).build()", From 85bf357ff0ba09d6c38f47839e7b6aba9d845eb0 Mon Sep 17 00:00:00 2001 From: Konrad Weiss Date: Mon, 21 Jun 2021 18:42:32 +0200 Subject: [PATCH 28/30] Revert "Deactivate go as default language" This reverts commit a0be96cbaaedf734fca093dca7a9ecc12fcc43bf. --- .../kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt index aaa2303acf..bfc89cd772 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt @@ -75,9 +75,7 @@ class TranslatePlugin : Plugin { path + "\"))\n" + " .defaultLanguages()\n" + - // " - // .registerLanguage(GoLanguageFrontend::class.java, - // GoLanguageFrontend.GOLANG_EXTENSIONS)" + + " .registerLanguage(GoLanguageFrontend::class.java, GoLanguageFrontend.GOLANG_EXTENSIONS)" + " .defaultPasses()\n" + " .build()", "val analyzer = TranslationManager.builder().config(config).build()", From d027ae0450093463624d5e34d81f8e76e5d4da83 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Mon, 21 Jun 2021 20:01:13 +0200 Subject: [PATCH 29/30] Some cleanup --- .../fraunhofer/aisec/cpg/analysis/MyToJsonStyle.kt | 14 +------------- .../de/fraunhofer/aisec/cpg/console/Neo4jPlugin.kt | 7 +++---- .../de/fraunhofer/aisec/cpg/console/RunPlugin.kt | 4 +++- .../fraunhofer/aisec/cpg/console/ShowCodePlugin.kt | 6 +++--- .../aisec/cpg/console/TranslatePlugin.kt | 4 +++- 5 files changed, 13 insertions(+), 22 deletions(-) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MyToJsonStyle.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MyToJsonStyle.kt index 9d91dbb789..a8da1f39cb 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MyToJsonStyle.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MyToJsonStyle.kt @@ -29,26 +29,14 @@ import org.apache.commons.lang3.builder.ToStringStyle // ---------------------------------------------------------------------------- class MultiLineToStringStyle internal constructor() : ToStringStyle() { - /** - * - * Ensure `Singleton` after serialization. - * - * @return the singleton - */ private fun readResolve(): Any { - return MULTI_LINE_STYLE + return this } companion object { private const val serialVersionUID = 1L } - /** - * - * Constructor. - * - * Use the static constant rather than instantiating. - */ init { this.isUseIdentityHashCode = false this.isUseShortClassName = true diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Neo4jPlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Neo4jPlugin.kt index 8a5bbd9dad..be343fef21 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Neo4jPlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/Neo4jPlugin.kt @@ -41,7 +41,6 @@ class Neo4jPlugin : Plugin { override fun execute(line: String): Command.Result { val p = line.indexOf(' ') - val host = line.substring(p + 1).trim() return Command.Result.RunSnippets( listOf( @@ -50,8 +49,6 @@ class Neo4jPlugin : Plugin { "neo4j.pushToNeo4j(result)" ) ) - - // return Command.Result.RunSnippets(listOf(content)) } } @@ -63,5 +60,7 @@ class Neo4jPlugin : Plugin { repl.registerCommand(Load(config)) } - override fun cleanUp() {} + override fun cleanUp() { + // nothing to do + } } diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/RunPlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/RunPlugin.kt index 65c3c40778..42c4598a80 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/RunPlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/RunPlugin.kt @@ -66,5 +66,7 @@ class RunPlugin : Plugin { repl.registerCommand(Load(config)) } - override fun cleanUp() {} + override fun cleanUp() { + // nothing to do + } } diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt index ca1ca17c77..1df06bc86d 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/ShowCodePlugin.kt @@ -44,8 +44,6 @@ class ShowCodePlugin : Plugin { val node = line.substring(p + 1).trim() return Command.Result.RunSnippets(listOf("${node}.printCode(0, true)")) - - // return Command.Result.RunSnippets(listOf(content)) } } @@ -57,5 +55,7 @@ class ShowCodePlugin : Plugin { repl.registerCommand(Load(config)) } - override fun cleanUp() {} + override fun cleanUp() { + // nothing to do + } } diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt index bfc89cd772..b12073cec8 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt @@ -97,5 +97,7 @@ class TranslatePlugin : Plugin { repl.registerCommand(Load(config)) } - override fun cleanUp() {} + override fun cleanUp() { + // nothing to do + } } From c8c699921e2fb2072dfd9ec2f6502267cb8c7f36 Mon Sep 17 00:00:00 2001 From: Konrad Weiss Date: Tue, 22 Jun 2021 00:04:55 +0200 Subject: [PATCH 30/30] Removing addition of experimental language --- .../de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt index b12073cec8..0304d44b22 100644 --- a/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt +++ b/cpg-console/src/main/kotlin/de/fraunhofer/aisec/cpg/console/TranslatePlugin.kt @@ -75,7 +75,7 @@ class TranslatePlugin : Plugin { path + "\"))\n" + " .defaultLanguages()\n" + - " .registerLanguage(GoLanguageFrontend::class.java, GoLanguageFrontend.GOLANG_EXTENSIONS)" + +// " .registerLanguage(GoLanguageFrontend::class.java, GoLanguageFrontend.GOLANG_EXTENSIONS)" + " .defaultPasses()\n" + " .build()", "val analyzer = TranslationManager.builder().config(config).build()", @@ -97,7 +97,5 @@ class TranslatePlugin : Plugin { repl.registerCommand(Load(config)) } - override fun cleanUp() { - // nothing to do - } + override fun cleanUp() {} }