From 2686e5732ea1ddbd5da2504dae78fcd5e1555592 Mon Sep 17 00:00:00 2001 From: Fabian Yamaguchi Date: Thu, 4 Feb 2021 16:37:18 +0100 Subject: [PATCH] :white_check_mark: Tests and minor changes to pass them (#53) * Pass a few more tests * Change `language` to `Plume` * NamespaceBlockTests pass now * FileTests pass * Pass Method tests * Added passing MethodParameterTests * Set correct CPG version * Add fields to mapper * Remove exclusions * Adapt schemas * :bug: Fixed TigerGraph tests * :bug: Fixed Graph update bug * :bug: Fixed failing extractor tests Co-authored-by: David Baker Effendi --- gradle.properties | 2 +- .../kotlin/io/github/plume/oss/Extractor.kt | 6 +- .../plume/oss/domain/mappers/VertexMapper.kt | 8 ++- .../io/github/plume/oss/graph/ASTBuilder.kt | 2 +- .../github/plume/oss/util/SootToPlumeUtil.kt | 44 ++++++++++--- .../github/plume/oss/TestDomainResources.kt | 12 ++-- .../plume/oss/extractor/BasicExtractorTest.kt | 20 +++--- .../plume/oss/extractor/UpdateGraphTest.kt | 1 - .../BasicIntraproceduralTest.kt | 34 +++------- .../test/resources/schema/jg_schema.groovy | 3 + .../src/test/resources/schema/tg_schema.gsql | 3 + .../plume/oss/querying/FileNodeTests.scala | 37 ----------- .../github/plume/oss/querying/FileTests.scala | 39 ++++++++++++ .../oss/querying/MetaDataNodeTests.scala | 22 ------- .../plume/oss/querying/MetaDataTests.scala | 25 ++++++++ .../oss/querying/MethodParameterTests.scala | 39 ++++++++++++ .../plume/oss/querying/MethodTests.scala | 63 +++++++------------ .../oss/querying/NamespaceBlockTests.scala | 37 +++++------ 18 files changed, 219 insertions(+), 178 deletions(-) delete mode 100644 testing/src/test/scala/io/github/plume/oss/querying/FileNodeTests.scala create mode 100644 testing/src/test/scala/io/github/plume/oss/querying/FileTests.scala delete mode 100644 testing/src/test/scala/io/github/plume/oss/querying/MetaDataNodeTests.scala create mode 100644 testing/src/test/scala/io/github/plume/oss/querying/MetaDataTests.scala create mode 100644 testing/src/test/scala/io/github/plume/oss/querying/MethodParameterTests.scala diff --git a/gradle.properties b/gradle.properties index 6e363122..c08d4fe1 100644 --- a/gradle.properties +++ b/gradle.properties @@ -6,7 +6,7 @@ jgVersion=0.5.3 neo4jDriverVersion=4.2.0 khttpVersion=1.0.0 jacksonVersion=2.12.0 -shiftleftVersion=1.3.48 +shiftleftVersion=1.3.57 sootVersion=4.2.1 lz4Version=1.7.1 diff --git a/plume/src/main/kotlin/io/github/plume/oss/Extractor.kt b/plume/src/main/kotlin/io/github/plume/oss/Extractor.kt index 664cac7d..4416bda0 100644 --- a/plume/src/main/kotlin/io/github/plume/oss/Extractor.kt +++ b/plume/src/main/kotlin/io/github/plume/oss/Extractor.kt @@ -210,7 +210,7 @@ class Extractor(val driver: IDriver) { } } if (splitFiles.keys.contains(SupportedFile.JAVA) || splitFiles.keys.contains(SupportedFile.JVM_CLASS)) { - driver.addVertex(NewMetaDataBuilder().language("Java").version(System.getProperty("java.runtime.version"))) + driver.addVertex(NewMetaDataBuilder().language("Plume").version("0.1")) } return splitFiles.keys.map { val filesToCompile = (splitFiles[it] ?: emptyList()).toList() @@ -222,7 +222,7 @@ class Extractor(val driver: IDriver) { } /** - * Projects all loaded classes currently loaded. + * Projects all loaded classes. */ fun project() { configureSoot() @@ -313,7 +313,7 @@ class Extractor(val driver: IDriver) { val currentFileHash = getFileHashPair(cls) val files = programStructure.nodes { it == ODBFile.Label() }.asSequence() logger.debug("Looking for existing file vertex for ${cls.name} from given file hash $currentFileHash") - files.firstOrNull { it.property("NAME") == cls.name }?.let { fileV -> + files.firstOrNull { it.property("NAME") == SootToPlumeUtil.sootClassToFileName(cls)}?.let { fileV -> if (fileV.property("HASH") != currentFileHash) { logger.debug("Existing class was found and file hashes do not match, marking for rebuild.") // Rebuild diff --git a/plume/src/main/kotlin/io/github/plume/oss/domain/mappers/VertexMapper.kt b/plume/src/main/kotlin/io/github/plume/oss/domain/mappers/VertexMapper.kt index c49d8d04..d5823b6d 100644 --- a/plume/src/main/kotlin/io/github/plume/oss/domain/mappers/VertexMapper.kt +++ b/plume/src/main/kotlin/io/github/plume/oss/domain/mappers/VertexMapper.kt @@ -59,6 +59,8 @@ object VertexMapper { .order(map["ORDER"] as Int) .id(map["id"] as Long) Method.Label() -> NewMethodBuilder() + .astparentfullname(map["AST_PARENT_FULL_NAME"] as String) + .astparenttype(map["AST_PARENT_TYPE"] as String) .name(map["NAME"] as String) .code(map["CODE"] as String) .fullname(map["FULL_NAME"] as String) @@ -94,6 +96,8 @@ object VertexMapper { .typedeclfullname(map["TYPE_DECL_FULL_NAME"] as String) .id(map["id"] as Long) TypeDecl.Label() -> NewTypeDeclBuilder() + .astparentfullname(map["AST_PARENT_FULL_NAME"] as String) + .astparenttype(map["AST_PARENT_TYPE"] as String) .name(map["NAME"] as String) .fullname(map["FULL_NAME"] as String) .order(map["ORDER"] as Int) @@ -113,6 +117,7 @@ object VertexMapper { .id(map["id"] as Long) NamespaceBlock.Label() -> NewNamespaceBlockBuilder() .fullname(map["FULL_NAME"] as String) + .filename(map["FILENAME"] as String) .name(map["NAME"] as String) .order(map["ORDER"] as Int) .id(map["id"] as Long) @@ -324,12 +329,9 @@ object VertexMapper { propertyMap.forEach { val key: Optional = when (it.key) { "PARSER_TYPE_NAME" -> Optional.empty() - "AST_PARENT_TYPE" -> Optional.empty() - "AST_PARENT_FULL_NAME" -> Optional.empty() "POLICY_DIRECTORIES" -> Optional.empty() "INHERITS_FROM_TYPE_FULL_NAME" -> Optional.empty() "OVERLAYS" -> Optional.empty() - "FILENAME" -> Optional.empty() "IS_EXTERNAL" -> Optional.empty() else -> Optional.of(it.key) } diff --git a/plume/src/main/kotlin/io/github/plume/oss/graph/ASTBuilder.kt b/plume/src/main/kotlin/io/github/plume/oss/graph/ASTBuilder.kt index 7d589fc7..85cfc092 100644 --- a/plume/src/main/kotlin/io/github/plume/oss/graph/ASTBuilder.kt +++ b/plume/src/main/kotlin/io/github/plume/oss/graph/ASTBuilder.kt @@ -82,7 +82,7 @@ class ASTBuilder(private val driver: IDriver) : IGraphBuilder { val localVertices = mutableListOf() graph.body.parameterLocals .mapIndexed { i, local -> - SootToPlumeUtil.projectMethodParameterIn(local, currentLine, currentCol, i) + SootToPlumeUtil.projectMethodParameterIn(local, currentLine, currentCol, i + 1) .apply { addSootToPlumeAssociation(local, this) } } .forEach { diff --git a/plume/src/main/kotlin/io/github/plume/oss/util/SootToPlumeUtil.kt b/plume/src/main/kotlin/io/github/plume/oss/util/SootToPlumeUtil.kt index 5ec2786e..3bd28a67 100644 --- a/plume/src/main/kotlin/io/github/plume/oss/util/SootToPlumeUtil.kt +++ b/plume/src/main/kotlin/io/github/plume/oss/util/SootToPlumeUtil.kt @@ -98,16 +98,19 @@ object SootToPlumeUtil { fun buildMethodHead(mtd: SootMethod, driver: IDriver): NewMethodBuilder { val currentLine = mtd.javaSourceStartLineNumber val currentCol = mtd.javaSourceStartColumnNumber - var childIdx = 0 + var childIdx = 1 // Method vertex val mtdVertex = NewMethodBuilder() .name(mtd.name) .fullname("${mtd.declaringClass}.${mtd.name}") + .filename(sootClassToFileName(mtd.declaringClass)) .signature(mtd.subSignature) .code(mtd.declaration) .linenumber(Option.apply(currentLine)) .columnnumber(Option.apply(currentCol)) .order(childIdx++) + .astparentfullname("${mtd.declaringClass}") + .astparenttype("TYPE_DECL") addSootToPlumeAssociation(mtd, mtdVertex) // Store method vertex NewBlockBuilder() @@ -162,7 +165,7 @@ object SootToPlumeUtil { mtdVertex: NewMethodBuilder, currentLine: Int, currentCol: Int, - initialChildIdx: Int = 0 + initialChildIdx: Int = 1 ) { var childIdx = initialChildIdx mtd.parameterTypes.forEachIndexed { i, type -> @@ -210,19 +213,21 @@ object SootToPlumeUtil { fun buildClassStructure(cls: SootClass, driver: IDriver): NewFileBuilder { val classChildrenVertices = mutableListOf() val fileHash = Extractor.getFileHashPair(cls) + val filename = sootClassToFileName(cls) var nbv: NewNamespaceBlockBuilder? = null if (cls.packageName.isNotEmpty()) { // Populate namespace block chain - val namespaceList = cls.packageName.split(".").toTypedArray() - if (namespaceList.isNotEmpty()) nbv = populateNamespaceChain(namespaceList, driver) + val namespaceList = arrayOf(cls.packageName) + // TODO : the CPG spec doesn't know these chains, simplify + if (namespaceList.isNotEmpty()) nbv = populateNamespaceChain(namespaceList, filename, driver) } val order = if (nbv != null) { driver.getNeighbours(nbv).use { ns -> - ns.node(nbv.id())?.outE()?.asSequence()?.toList()?.size ?: 0 + ns.node(nbv.id())?.outE()?.asSequence()?.toList()?.size ?: 1 } - } else 0 + } else 1 return NewFileBuilder() - .name(cls.name) + .name(sootClassToFileName(cls)) .hash(Option.apply(fileHash.toString())) .order(order) .apply { @@ -235,13 +240,27 @@ object SootToPlumeUtil { } } + /** + * Derive a file name from an object of type `SootClass` + * @param cls the soot class + * @return the filename in string form + * */ + fun sootClassToFileName(cls : SootClass) : String { + val packageName = cls.packageName + return if (packageName != null) { + "/" + cls.name.replace(".", "/") + ".class" + } else { + io.shiftleft.semanticcpg.language.types.structure.File.UNKNOWN() + } + } + /** * Creates a change of [NewNamespaceBlockBuilder]s and returns the final one in the chain. * * @param namespaceList A list of package names. * @return The final [NewNamespaceBlockBuilder] in the chain (the one associated with the file). */ - private fun populateNamespaceChain(namespaceList: Array, driver: IDriver): NewNamespaceBlockBuilder { + private fun populateNamespaceChain(namespaceList: Array, filename : String, driver: IDriver): NewNamespaceBlockBuilder { var prevNamespaceBlock: NewNamespaceBlockBuilder var currNamespaceBlock: NewNamespaceBlockBuilder? = null driver.getProgramStructure().use { programStructure -> @@ -254,7 +273,8 @@ object SootToPlumeUtil { ?: NewNamespaceBlockBuilder() .name(namespaceList[0]) .fullname(namespaceList[0]) - .order(0) + .filename(filename) + .order(1) if (namespaceList.size == 1) return prevNamespaceBlock val namespaceBuilder = StringBuilder(namespaceList[0]) @@ -262,7 +282,7 @@ object SootToPlumeUtil { namespaceBuilder.append("." + namespaceList[i]) val order: Int driver.getNeighbours(prevNamespaceBlock).use { ns -> - order = ns.node(prevNamespaceBlock.id())?.outE()?.asSequence()?.toList()?.size ?: 0 + order = 1 + (ns.node(prevNamespaceBlock.id())?.outE()?.asSequence()?.toList()?.size ?: 0) } val maybeCurrNamespaceBlock = programStructure.nodes { it == NamespaceBlock.Label() }.asSequence() .firstOrNull { @@ -272,6 +292,7 @@ object SootToPlumeUtil { currNamespaceBlock = maybeCurrNamespaceBlock ?: NewNamespaceBlockBuilder() .name(namespaceList[i]) .fullname(namespaceBuilder.toString()) + .filename(filename) .order(order) if (currNamespaceBlock != null) { driver.addEdge(currNamespaceBlock!!, prevNamespaceBlock, AST) @@ -293,6 +314,9 @@ object SootToPlumeUtil { NewTypeDeclBuilder() .name(cls.shortName) .fullname(cls.name) + .filename(sootClassToFileName(cls)) + .astparentfullname(cls.getPackageName()) + .astparenttype("NAMESPACE_BLOCK") .order(0) .apply { // Attach fields to the TypeDecl diff --git a/plume/src/test/kotlin/io/github/plume/oss/TestDomainResources.kt b/plume/src/test/kotlin/io/github/plume/oss/TestDomainResources.kt index cde6b304..d76bfabf 100644 --- a/plume/src/test/kotlin/io/github/plume/oss/TestDomainResources.kt +++ b/plume/src/test/kotlin/io/github/plume/oss/TestDomainResources.kt @@ -59,7 +59,7 @@ class TestDomainResources { NewMethodBuilder().name(STRING_1).fullname(STRING_1).signature(STRING_1).code(STRING_1).order(INT_1) .linenumber(Option.apply(INT_1)).columnnumber(Option.apply(INT_1)), NewModifierBuilder().modifiertype(MOD_1).order(INT_1), - NewNamespaceBlockBuilder().name(STRING_1).fullname(STRING_1).order(INT_1), + NewNamespaceBlockBuilder().name(STRING_1).fullname(STRING_1).order(INT_1).filename(STRING_1), NewReturnBuilder().order(INT_1).argumentindex(INT_1).code(STRING_1).linenumber(Option.apply(INT_1)) .columnnumber(Option.apply(INT_1)), NewTypeArgumentBuilder().order(INT_1), @@ -76,7 +76,8 @@ class TestDomainResources { val methodVertex: NewMethodBuilder = NewMethodBuilder().code(STRING_1).name(STRING_1).fullname(STRING_1).order(INT_1) - .linenumber(Option.apply(INT_1)).columnnumber(Option.apply(INT_1)).signature(STRING_2) + .linenumber(Option.apply(INT_1)).columnnumber(Option.apply(INT_1)).signature(STRING_2).filename(STRING_1) + .astparentfullname(STRING_1).astparenttype(STRING_2) val mtdParamInVertex: NewMethodParameterInBuilder = NewMethodParameterInBuilder().code(STRING_1).evaluationstrategy(EVAL_1).typefullname(STRING_1) .name(STRING_1).order(INT_1).linenumber(Option.apply(INT_1)).columnnumber(Option.apply(INT_1)) @@ -96,6 +97,7 @@ class TestDomainResources { NewIdentifierBuilder().name(STRING_1).typefullname(STRING_1).code(STRING_1).order(INT_1) .argumentindex(INT_1).linenumber(Option.apply(INT_1)).columnnumber(Option.apply(INT_1)) val typeDeclVertex: NewTypeDeclBuilder = NewTypeDeclBuilder().name(STRING_1).fullname(STRING_1).order(INT_1) + .astparentfullname(STRING_1).astparenttype(STRING_2) val literalVertex: NewLiteralBuilder = NewLiteralBuilder().typefullname(STRING_1).code(STRING_1).order(INT_1).argumentindex(INT_1) .linenumber(Option.apply(INT_1)).columnnumber(Option.apply(INT_1)) @@ -107,9 +109,9 @@ class TestDomainResources { .order(INT_1).linenumber(Option.apply(INT_1)).columnnumber(Option.apply(INT_1)) val fileVertex: NewFileBuilder = NewFileBuilder().name(STRING_1).hash(Option.apply(STRING_2)).order(INT_1) val namespaceBlockVertex1: NewNamespaceBlockBuilder = - NewNamespaceBlockBuilder().name(STRING_1).fullname(STRING_1).order(INT_1) + NewNamespaceBlockBuilder().name(STRING_1).fullname(STRING_1).order(INT_1).filename(STRING_1) val namespaceBlockVertex2: NewNamespaceBlockBuilder = - NewNamespaceBlockBuilder().name(STRING_2).fullname(STRING_2).order(INT_1) + NewNamespaceBlockBuilder().name(STRING_2).fullname(STRING_2).order(INT_1).filename(STRING_2) val metaDataVertex: NewMetaDataBuilder = NewMetaDataBuilder().language(STRING_1).version(STRING_2) val controlStructureVertex: NewControlStructureBuilder = NewControlStructureBuilder().code(STRING_1).linenumber(Option.apply(INT_1)) @@ -131,7 +133,7 @@ class TestDomainResources { .code(STRING_1).argumentindex(INT_1).order(INT_1).linenumber(Option.apply(INT_1)) .columnnumber(Option.apply(INT_1)) val unknownVertex: NewUnknownBuilder = - NewUnknownBuilder().typefullname("brug").code(STRING_1).order(INT_1).argumentindex(INT_1) + NewUnknownBuilder().typefullname(STRING_1).code(STRING_1).order(INT_1).argumentindex(INT_1) .linenumber(Option.apply(INT_1)).columnnumber(Option.apply(INT_1)) val modifierVertex: NewModifierBuilder = NewModifierBuilder().modifiertype(MOD_1).order(INT_1) diff --git a/plume/src/test/kotlin/io/github/plume/oss/extractor/BasicExtractorTest.kt b/plume/src/test/kotlin/io/github/plume/oss/extractor/BasicExtractorTest.kt index b319a482..61e2a5f4 100644 --- a/plume/src/test/kotlin/io/github/plume/oss/extractor/BasicExtractorTest.kt +++ b/plume/src/test/kotlin/io/github/plume/oss/extractor/BasicExtractorTest.kt @@ -9,9 +9,7 @@ import io.shiftleft.codepropertygraph.generated.nodes.Local import io.shiftleft.codepropertygraph.generated.nodes.Method import io.shiftleft.codepropertygraph.generated.nodes.NamespaceBlock import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Assertions -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import overflowdb.Graph @@ -59,7 +57,7 @@ class BasicExtractorTest { g = driver.getWholeGraph() val ns = g.nodes().asSequence().toList() assertNotNull(ns.filterIsInstance().find { it.name() == "extractor_tests" }) - ns.filterIsInstance().find { it.name() == "extractor_tests.Test1" } + ns.filterIsInstance().find { it.name() == "/extractor_tests/Test1.class" } .let { assertNotNull(it) } ns.filterIsInstance().find { it.name() == "main" }.let { assertNotNull(it) } ns.filterIsInstance().find { it.name() == "a" } @@ -79,7 +77,7 @@ class BasicExtractorTest { val ns = g.nodes().asSequence().toList() assertNotNull( ns.filterIsInstance().find { it.name() == "extractor_tests" }) - ns.filterIsInstance().find { it.name() == "extractor_tests.Test2" } + ns.filterIsInstance().find { it.name() == "/extractor_tests/Test2.class" } .let { assertNotNull(it) } ns.filterIsInstance().find { it.name() == "main" }.let { assertNotNull(it) } ns.filterIsInstance().find { it.name() == "l1" } @@ -98,18 +96,14 @@ class BasicExtractorTest { g = driver.getProgramStructure() val ns = g.nodes().asSequence().toList() ns.filterIsInstance().let { fileList -> - assertNotNull(fileList.firstOrNull { it.name() == "extractor_tests.dir_test.Dir1" }) - assertNotNull(fileList.firstOrNull { it.name() == "extractor_tests.dir_test.pack.Dir2" }) - } - ns.filterIsInstance().let { fileList -> - assertNotNull(fileList.firstOrNull { it.name() == "dir_test" }) - assertNotNull(fileList.firstOrNull { it.name() == "extractor_tests" }) - assertNotNull(fileList.firstOrNull { it.name() == "pack" }) + assertNotNull(fileList.firstOrNull { it.name() == "/extractor_tests/dir_test/Dir1.class" }) + assertNotNull(fileList.firstOrNull { it.name() == "/extractor_tests/dir_test/pack/Dir2.class" }) } + assertNotNull(ns.filterIsInstance().firstOrNull { it.name() == "extractor_tests.dir_test.pack" }) } @Test fun loadFileThatDoesNotExistTest() { - Assertions.assertThrows(NullPointerException::class.java) { extractor.load(File("dne.class")) } + assertThrows(NullPointerException::class.java) { extractor.load(File("dne.class")) } } } \ No newline at end of file diff --git a/plume/src/test/kotlin/io/github/plume/oss/extractor/UpdateGraphTest.kt b/plume/src/test/kotlin/io/github/plume/oss/extractor/UpdateGraphTest.kt index cfbdf0ca..2a439af1 100644 --- a/plume/src/test/kotlin/io/github/plume/oss/extractor/UpdateGraphTest.kt +++ b/plume/src/test/kotlin/io/github/plume/oss/extractor/UpdateGraphTest.kt @@ -10,7 +10,6 @@ import io.github.plume.oss.drivers.DriverFactory import io.github.plume.oss.drivers.GraphDatabase import io.github.plume.oss.drivers.TinkerGraphDriver import io.shiftleft.codepropertygraph.generated.nodes.Literal -import overflowdb.Graph import java.io.File import java.io.FileInputStream import java.io.FileOutputStream diff --git a/plume/src/test/kotlin/io/github/plume/oss/extractor/intraprocedural/BasicIntraproceduralTest.kt b/plume/src/test/kotlin/io/github/plume/oss/extractor/intraprocedural/BasicIntraproceduralTest.kt index 4e401b7c..cd9c4ebe 100644 --- a/plume/src/test/kotlin/io/github/plume/oss/extractor/intraprocedural/BasicIntraproceduralTest.kt +++ b/plume/src/test/kotlin/io/github/plume/oss/extractor/intraprocedural/BasicIntraproceduralTest.kt @@ -57,12 +57,9 @@ class BasicIntraproceduralTest { @Test fun basic1Test() { val ns = g.nodes().asSequence().toList() - ns.filterIsInstance().let { nbv -> - assertNotNull(nbv.find { it.name() == "basic" }) - assertNotNull(nbv.find { it.name() == "intraprocedural" }) - } + assertNotNull(ns.filterIsInstance().find { it.name() == "intraprocedural.basic" }) ns.filterIsInstance() - .find { it.name() == "intraprocedural.basic.Basic$currentTestNumber" } + .find { it.name() == "/intraprocedural/basic/Basic$currentTestNumber.class" } .let { assertNotNull(it) } ns.filterIsInstance().find { it.name() == "main" }.let { assertNotNull(it) } ns.filterIsInstance().find { it.name() == "a" } @@ -77,12 +74,9 @@ class BasicIntraproceduralTest { @Test fun basic2Test() { val ns = g.nodes().asSequence().toList() - ns.filterIsInstance().let { nbv -> - assertNotNull(nbv.find { it.name() == "basic" }) - assertNotNull(nbv.find { it.name() == "intraprocedural" }) - } + assertNotNull(ns.filterIsInstance().find { it.name() == "intraprocedural.basic" }) ns.filterIsInstance() - .find { it.name() == "intraprocedural.basic.Basic$currentTestNumber" } + .find { it.name() == "/intraprocedural/basic/Basic$currentTestNumber.class" } .let { assertNotNull(it) } ns.filterIsInstance().find { it.name() == "main" }.let { assertNotNull(it) } ns.filterIsInstance().find { it.name() == "a" } @@ -97,12 +91,9 @@ class BasicIntraproceduralTest { @Test fun basic3Test() { val ns = g.nodes().asSequence().toList() - ns.filterIsInstance().let { nbv -> - assertNotNull(nbv.find { it.name() == "basic" }) - assertNotNull(nbv.find { it.name() == "intraprocedural" }) - } + assertNotNull(ns.filterIsInstance().find { it.name() == "intraprocedural.basic" }) ns.filterIsInstance() - .find { it.name() == "intraprocedural.basic.Basic$currentTestNumber" } + .find { it.name() == "/intraprocedural/basic/Basic$currentTestNumber.class" } .let { assertNotNull(it) } ns.filterIsInstance().find { it.name() == "main" }.let { assertNotNull(it) } ns.filterIsInstance().find { it.name() == "a" } @@ -117,12 +108,9 @@ class BasicIntraproceduralTest { @Test fun basic4Test() { val ns = g.nodes().asSequence().toList() - ns.filterIsInstance().let { nbv -> - assertNotNull(nbv.find { it.name() == "basic" }) - assertNotNull(nbv.find { it.name() == "intraprocedural" }) - } + assertNotNull(ns.filterIsInstance().find { it.name() == "intraprocedural.basic" }) ns.filterIsInstance() - .find { it.name() == "intraprocedural.basic.Basic$currentTestNumber" } + .find { it.name() == "/intraprocedural/basic/Basic$currentTestNumber.class" } .let { assertNotNull(it) } ns.filterIsInstance().find { it.name() == "main" }.let { assertNotNull(it) } ns.filterIsInstance().find { it.name() == "Sally" }.let { assertNotNull(it) } @@ -148,10 +136,8 @@ class BasicIntraproceduralTest { g = driver.getWholeGraph() val ns = g.nodes().asSequence().toList() ns.filterIsInstance().let { nbv -> - assertNotNull(nbv.find { it.name() == "basic" }) - assertNotNull(nbv.find { it.name() == "intraprocedural" }) - assertNotNull(nbv.find { it.name() == "basic5" }) - assertEquals(3, nbv.toList().size) + assertNotNull(nbv.find { it.name() == "intraprocedural.basic.basic5" }) + assertEquals(2, nbv.toList().size) } ns.filterIsInstance().let { mrv -> assertNotNull(mrv.find { it.fullName() == "intraprocedural.basic.Basic5" }) diff --git a/plume/src/test/resources/schema/jg_schema.groovy b/plume/src/test/resources/schema/jg_schema.groovy index 9c34e64e..8e48e59e 100644 --- a/plume/src/test/resources/schema/jg_schema.groovy +++ b/plume/src/test/resources/schema/jg_schema.groovy @@ -37,6 +37,9 @@ mgmt.containsEdgeLabel('CAPTURED_BY') ?: mgmt.makeEdgeLabel('CAPTURED_BY').make( mgmt.containsEdgeLabel('REF') ?: mgmt.makeEdgeLabel('REF').make() mgmt.containsEdgeLabel('BINDS_TO') ?: mgmt.makeEdgeLabel('BINDS_TO').make() // Properties +mgmt.containsPropertyKey('AST_PARENT_TYPE') ?: mgmt.makePropertyKey('AST_PARENT_TYPE').dataType(String.class).cardinality(Cardinality.SINGLE).make() +mgmt.containsPropertyKey('AST_PARENT_FULL_NAME') ?: mgmt.makePropertyKey('AST_PARENT_FULL_NAME').dataType(String.class).cardinality(Cardinality.SINGLE).make() +mgmt.containsPropertyKey('FILENAME') ?: mgmt.makePropertyKey('FILENAME').dataType(String.class).cardinality(Cardinality.SINGLE).make() mgmt.containsPropertyKey('SIGNATURE') ?: mgmt.makePropertyKey('SIGNATURE').dataType(String.class).cardinality(Cardinality.SINGLE).make() mgmt.containsPropertyKey('EVALUATION_STRATEGY') ?: mgmt.makePropertyKey('EVALUATION_STRATEGY').dataType(String.class).cardinality(Cardinality.SINGLE).make() mgmt.containsPropertyKey('NAME') ?: mgmt.makePropertyKey('NAME').dataType(String.class).cardinality(Cardinality.SINGLE).make() diff --git a/plume/src/test/resources/schema/tg_schema.gsql b/plume/src/test/resources/schema/tg_schema.gsql index 9c8e6a21..cb1f3e0f 100644 --- a/plume/src/test/resources/schema/tg_schema.gsql +++ b/plume/src/test/resources/schema/tg_schema.gsql @@ -7,6 +7,8 @@ CREATE VERTEX CPG_VERT ( PRIMARY_ID id UINT, label STRING DEFAULT "UNKNOWN", ARGUMENT_INDEX INT DEFAULT -1, + AST_PARENT_FULL_NAME STRING DEFAULT "null", + AST_PARENT_TYPE STRING DEFAULT "null", DISPATCH_TYPE STRING DEFAULT "STATIC_DISPATCH", EVALUATION_STRATEGY STRING DEFAULT "BY_REFERENCE", METHOD_FULL_NAME STRING DEFAULT "null", @@ -20,6 +22,7 @@ CREATE VERTEX CPG_VERT ( COLUMN_NUMBER INT DEFAULT -1, LINE_NUMBER INT DEFAULT -1, NAME STRING DEFAULT "null", + FILENAME STRING DEFAULT "null", FULL_NAME STRING DEFAULT "null", CANONICAL_NAME STRING DEFAULT "null", AST_ORDER INT DEFAULT -1, diff --git a/testing/src/test/scala/io/github/plume/oss/querying/FileNodeTests.scala b/testing/src/test/scala/io/github/plume/oss/querying/FileNodeTests.scala deleted file mode 100644 index 9722c703..00000000 --- a/testing/src/test/scala/io/github/plume/oss/querying/FileNodeTests.scala +++ /dev/null @@ -1,37 +0,0 @@ -package io.github.plume.oss.querying - -import io.github.plume.oss.PlumeCodeToCpgSuite -import io.shiftleft.semanticcpg.language._ -import io.shiftleft.semanticcpg.language.types.structure.File - -class FileNodeTests extends PlumeCodeToCpgSuite { - - override val code: String = - """ - | class Foo { } - |""".stripMargin - - "should contain two file nodes in total, both with order=0" in { -// cpg.file.order.l shouldBe List(0, 0) -// cpg.file.name(File.UNKNOWN).size shouldBe 1 -// cpg.file.nameNot().size shouldBe 1 - } - - "should contain exactly one placeholder file node with `name=\"\"/order=0`" in { -// cpg.file(File.UNKNOWN).l match { -// case List(x) => -// x.order shouldBe 0 -// case _ => fail() -// } - } - - "should contain exactly one non-placeholder file with absolute path in `name`" in { -// cpg.file.nameNot(File.UNKNOWN).l match { -// case List(x) => -// x.name should startWith("/") -// case _ => fail() -// } - } - -} - diff --git a/testing/src/test/scala/io/github/plume/oss/querying/FileTests.scala b/testing/src/test/scala/io/github/plume/oss/querying/FileTests.scala new file mode 100644 index 00000000..7dd21a43 --- /dev/null +++ b/testing/src/test/scala/io/github/plume/oss/querying/FileTests.scala @@ -0,0 +1,39 @@ +package io.github.plume.oss.querying + +import io.github.plume.oss.PlumeCodeToCpgSuite +import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.types.structure.File + +class FileTests extends PlumeCodeToCpgSuite { + + override val code: String = + """ + | package a.b; + | class Foo { int bar() { return 1; } } + |""".stripMargin + + "should contain one file node in total with order=1" in { + cpg.file.order.l shouldBe List(1) + cpg.file.nameNot(File.UNKNOWN).size shouldBe 1 + } + + "should contain exactly one non-placeholder file with absolute path in `name`" in { + val List(x) = cpg.file.nameNot(File.UNKNOWN).l + x.name should startWith("/") + x.hash.isDefined shouldBe true + } + + "should allow traversing from file to its namespace blocks" in { + cpg.file.nameNot(File.UNKNOWN).namespaceBlock.name.toSet shouldBe Set("a.b") + } + + "should allow traversing from file to its methods via namespace block" in { + cpg.file.nameNot(File.UNKNOWN).method.name.toSet shouldBe Set("", "bar") + } + + "should allow traversing from file to its type declarations via namespace block" in { + cpg.file.nameNot(File.UNKNOWN).typeDecl.name.toSet shouldBe Set("Foo") + } + +} + diff --git a/testing/src/test/scala/io/github/plume/oss/querying/MetaDataNodeTests.scala b/testing/src/test/scala/io/github/plume/oss/querying/MetaDataNodeTests.scala deleted file mode 100644 index bcb98e64..00000000 --- a/testing/src/test/scala/io/github/plume/oss/querying/MetaDataNodeTests.scala +++ /dev/null @@ -1,22 +0,0 @@ -package io.github.plume.oss.querying - -import io.github.plume.oss.PlumeCodeToCpgSuite -import io.shiftleft.semanticcpg.language._ - -class MetaDataNodeTests extends PlumeCodeToCpgSuite { - override val code = - """ - |class Foo {} - |""".stripMargin - - "should contain exactly one node with all mandatory fields set" in { - cpg.metaData.l match { - case List(x) => - x.language shouldBe "Java" - // TODO - // x.version shouldBe "0.1" - x.overlays shouldBe List("semanticcpg") - case _ => fail() - } - } -} diff --git a/testing/src/test/scala/io/github/plume/oss/querying/MetaDataTests.scala b/testing/src/test/scala/io/github/plume/oss/querying/MetaDataTests.scala new file mode 100644 index 00000000..36262b73 --- /dev/null +++ b/testing/src/test/scala/io/github/plume/oss/querying/MetaDataTests.scala @@ -0,0 +1,25 @@ +package io.github.plume.oss.querying + +import io.github.plume.oss.PlumeCodeToCpgSuite +import io.shiftleft.semanticcpg.language._ + +class MetaDataTests extends PlumeCodeToCpgSuite { + override val code = + """ + |class Foo {} + |""".stripMargin + + "should contain exactly one node with all mandatory fields set" in { + val List(x) = cpg.metaData.l + x.language shouldBe "Plume" + x.version shouldBe "0.1" + x.overlays shouldBe List("semanticcpg") + } + + "should not have any incoming or outgoing edges" in { + cpg.metaData.size shouldBe 1 + cpg.metaData.in.l shouldBe List() + cpg.metaData.out.l shouldBe List() + } + +} diff --git a/testing/src/test/scala/io/github/plume/oss/querying/MethodParameterTests.scala b/testing/src/test/scala/io/github/plume/oss/querying/MethodParameterTests.scala new file mode 100644 index 00000000..02bc23c9 --- /dev/null +++ b/testing/src/test/scala/io/github/plume/oss/querying/MethodParameterTests.scala @@ -0,0 +1,39 @@ +package io.github.plume.oss.querying + +import io.github.plume.oss.PlumeCodeToCpgSuite +import io.shiftleft.semanticcpg.language._ + +class MethodParameterTests extends PlumeCodeToCpgSuite { + + override val code = + """package a; + |class Foo { + | int foo(int param1, int param2) { + | return 0; + | } + |} + """.stripMargin + + "should return exactly two parameters with correct fields" in { + cpg.parameter.name.toSet shouldBe Set("param1", "param2") + + val List(x) = cpg.parameter.name("param1").l + x.code shouldBe "int param1" + x.typeFullName shouldBe "int" + x.lineNumber shouldBe Some(2) + // x.columnNumber shouldBe Some(11) + x.order shouldBe 1 + + val List(y) = cpg.parameter.name("param2").l + y.code shouldBe "int param2" + y.typeFullName shouldBe "int" + y.lineNumber shouldBe Some(2) + // y.columnNumber shouldBe Some(21) + y.order shouldBe 2 + } + + "should allow traversing from parameter to method" in { + cpg.parameter.name("param1").method.name.l shouldBe List("foo") + } + +} diff --git a/testing/src/test/scala/io/github/plume/oss/querying/MethodTests.scala b/testing/src/test/scala/io/github/plume/oss/querying/MethodTests.scala index 978a988e..95efd8c5 100644 --- a/testing/src/test/scala/io/github/plume/oss/querying/MethodTests.scala +++ b/testing/src/test/scala/io/github/plume/oss/querying/MethodTests.scala @@ -6,59 +6,42 @@ class MethodTests extends PlumeCodeToCpgSuite { override val code = """ class Foo { - | int foo(int param1, int param2) { + | int foo(int param1, int param2) { | return 1; | } |} |""".stripMargin - "should return correct function/method name" in { - cpg.method.name.toSet shouldBe Set("foo", "") + "should contain exactly one method node with correct fields" in { + val List(x) = cpg.method.nameNot("").l + x.name shouldBe "foo" + x.fullName shouldBe "Foo.foo" + // x.code shouldBe "int foo (int param1,int param2)" + x.signature shouldBe "int foo(int,int)" + x.isExternal shouldBe false + x.order shouldBe 1 + x.filename.startsWith("/") shouldBe true + x.filename.endsWith(".class") shouldBe true + x.lineNumber shouldBe Some(2) + // x.lineNumberEnd shouldBe Some(3) + // x.columnNumber shouldBe Some(2) + // x.columnNumberEnd shouldBe Some(1) } - "should return correct line number" in { - cpg.method.lineNumber.toSet shouldBe Set(0,2) - } - - // TODO If there is a way to obtain a `lineNumber` end, that - // should be implemented and the following two tests should - // be commented in - - // "should return correct end line number" in { - // cpg.method.name("foo").lineNumberEnd.l shouldBe List(4) - // } - // - // "should return correct number of lines" in { - // cpg.method.name("foo").numberOfLines.l shouldBe List(2) - // } - - "should have correct method signature" in { - cpg.method.name("foo").signature.toSet shouldBe Set("int foo(int,int)") - } +// "should return correct number of lines" in { +// cpg.method.name("foo").numberOfLines.l shouldBe List(2) +// } - "should return correct number of parameters" in { + "should allow traversing to parameters" in { cpg.method.name("foo").parameter.name.toSet shouldBe Set("param1", "param2") } - // TODO `EVAL_TYPE/REF` edges seem to not be correct - // "should return correct parameter types" in { - // cpg.parameter.name("param1").evalType.l shouldBe List("int") - // cpg.parameter.name("param2").evalType.l shouldBe List("int") - // } - -// "should return correct return type" in { -// cpg.methodReturn.evalType.l shouldBe List("int") -// cpg.method.name("foo").methodReturn.evalType.l shouldBe List("int") -// cpg.parameter.name("argc").method.methodReturn.evalType.l shouldBe List("int") -// } - - "should return a filename for method 'foo'" in { - cpg.method.name("foo").file.name.l should not be empty + "should allow traversing to methodReturn" in { + cpg.method.name("foo").methodReturn.typeFullName.l shouldBe List("int") } - "should allow filtering by number of parameters" in { - cpg.method.filter(_.parameter.size == 2).name.l shouldBe List("foo") - cpg.method.filter(_.parameter.size == 1).name.l shouldBe List() + "should allow traversing to file" in { + cpg.method.name("foo").file.name.l should not be empty } } diff --git a/testing/src/test/scala/io/github/plume/oss/querying/NamespaceBlockTests.scala b/testing/src/test/scala/io/github/plume/oss/querying/NamespaceBlockTests.scala index 04d418f5..39b0e852 100644 --- a/testing/src/test/scala/io/github/plume/oss/querying/NamespaceBlockTests.scala +++ b/testing/src/test/scala/io/github/plume/oss/querying/NamespaceBlockTests.scala @@ -14,28 +14,29 @@ class NamespaceBlockTests extends PlumeCodeToCpgSuite { |} |""".stripMargin - "should contain two namespace blocks in total" in { -// cpg.namespaceBlock.size shouldBe 2 + "should contain one namespace blocks in total" in { + cpg.namespaceBlock.size shouldBe 1 + // There is no global namespace block in Java } - "should contain a correct global namespace block for the `` file" in { -// cpg.namespaceBlock.filename(File.UNKNOWN).l match { -// case List(x) => -// x.name shouldBe Namespace.globalNamespaceName -// x.fullName shouldBe Namespace.globalNamespaceName -// x.order shouldBe 0 -// case _ => fail() -// } + "should contain correct namespace block for known file" in { + val List(x) = cpg.namespaceBlock.filenameNot(File.UNKNOWN).l + x.name shouldBe "foo.bar" + x.filename should not be "" + x.fullName shouldBe "foo.bar" + x.order shouldBe 1 } - "should contain correct namespace block for known file" in { -// cpg.namespaceBlock.filenameNot(File.UNKNOWN).l match { -// case List(x) => -// x.name shouldBe "bar" -// x.fullName shouldBe "foo.bar" -// x.order shouldBe 0 -// case _ => fail() -// } + "should allow traversing from namespace block to method" in { + cpg.namespaceBlock.filenameNot(File.UNKNOWN).typeDecl.method.name.toSet shouldBe Set("foo", "") + } + + "should allow traversing from namespace block to type declaration" in { + cpg.namespaceBlock.filenameNot(File.UNKNOWN).typeDecl.name.l shouldBe List("A") + } + + "should allow traversing from namespace block to namespace" in { + cpg.namespaceBlock.filenameNot(File.UNKNOWN).namespace.name.l shouldBe List("foo.bar") } }