From ffae4fc32e20f6daca07672c3e51857fe973ab10 Mon Sep 17 00:00:00 2001 From: Fabian Yamaguchi Date: Mon, 8 Feb 2021 12:12:17 +0100 Subject: [PATCH] :white_check_mark: Type Handling Tests (#54) * TYPE and TYPE_DECL handling * :sparkles: Added TYPE_DECL to getProgramStructure * :zap: Improved TG queries with v2 syntax * :sparkles: TypeDecl is in properly * :sparkles: TypeDecl tests are green Co-authored-by: David Baker Effendi --- .../io/github/plume/oss/Traversals.scala | 5 +- gradle.properties | 2 +- .../kotlin/io/github/plume/oss/Extractor.kt | 36 ++++++++- .../plume/oss/domain/mappers/VertexMapper.kt | 2 +- .../github/plume/oss/drivers/GremlinDriver.kt | 15 +++- .../io/github/plume/oss/drivers/IDriver.kt | 2 +- .../github/plume/oss/drivers/Neo4jDriver.kt | 14 +++- .../plume/oss/drivers/OverflowDbDriver.kt | 7 +- .../github/plume/oss/graphio/GraphMLWriter.kt | 1 + .../plume/oss/graphio/GraphSONWriter.kt | 1 + .../github/plume/oss/util/SootToPlumeUtil.kt | 71 +++++++++++------- .../github/plume/oss/TestDomainResources.kt | 3 +- .../oss/drivers/JanusGraphDriverIntTest.kt | 7 +- .../plume/oss/drivers/Neo4jDriverIntTest.kt | 5 +- .../plume/oss/drivers/NeptuneDriverIntTest.kt | 5 +- .../oss/drivers/OverflowDbDriverIntTest.kt | 5 +- .../oss/drivers/TigerGraphDriverIntTest.kt | 5 +- .../oss/drivers/TinkerGraphDriverIntTest.kt | 9 ++- .../BasicIntraproceduralTest.kt | 5 +- .../test/resources/schema/jg_schema.groovy | 1 + .../src/test/resources/schema/tg_schema.gsql | 73 ++++++++----------- .../plume/oss/querying/TypeDeclTests.scala | 45 ++++++++++++ .../github/plume/oss/querying/TypeTests.scala | 69 ++++++++++++++++++ 23 files changed, 290 insertions(+), 98 deletions(-) create mode 100644 testing/src/test/scala/io/github/plume/oss/querying/TypeDeclTests.scala create mode 100644 testing/src/test/scala/io/github/plume/oss/querying/TypeTests.scala diff --git a/cpgconv/src/main/scala/io/github/plume/oss/Traversals.scala b/cpgconv/src/main/scala/io/github/plume/oss/Traversals.scala index b3e72f87..596721bc 100644 --- a/cpgconv/src/main/scala/io/github/plume/oss/Traversals.scala +++ b/cpgconv/src/main/scala/io/github/plume/oss/Traversals.scala @@ -1,10 +1,9 @@ package io.github.plume.oss import java.util - import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.EdgeTypes -import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, HasOrder, StoredNode} +import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, HasOrder, StoredNode, TypeDecl} import io.shiftleft.semanticcpg.language._ import overflowdb.{Edge, Graph} import io.shiftleft.codepropertygraph.generated.nodes @@ -55,6 +54,8 @@ object Traversals { (edgesFromFile ++ edgesFromNamespaceBlock).asJava } + def getTypeDecls(graph: Graph):util.List[TypeDecl] = Cpg(graph).typeDecl.l.asJava + def getNeighbours(graph: Graph, nodeId: Long): util.List[Edge] = { Cpg(graph) .id[StoredNode](nodeId) diff --git a/gradle.properties b/gradle.properties index 8019dd2b..01a1a40f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -6,7 +6,7 @@ jgVersion=0.5.3 neo4jDriverVersion=4.2.0 khttpVersion=1.0.0 jacksonVersion=2.12.0 -shiftleftVersion=1.3.61 +shiftleftVersion=1.3.62 sootVersion=4.2.1 lz4Version=1.7.1 diff --git a/plume/src/main/kotlin/io/github/plume/oss/Extractor.kt b/plume/src/main/kotlin/io/github/plume/oss/Extractor.kt index 4416bda0..c1fdca02 100644 --- a/plume/src/main/kotlin/io/github/plume/oss/Extractor.kt +++ b/plume/src/main/kotlin/io/github/plume/oss/Extractor.kt @@ -32,12 +32,16 @@ import io.github.plume.oss.util.ResourceCompilationUtil.compileJavaFiles import io.github.plume.oss.util.ResourceCompilationUtil.deleteClassFiles import io.github.plume.oss.util.ResourceCompilationUtil.moveClassFiles import io.github.plume.oss.util.SootToPlumeUtil +import io.shiftleft.codepropertygraph.generated.EdgeTypes +import io.shiftleft.codepropertygraph.generated.NodeKeyNames.FULL_NAME +import io.shiftleft.codepropertygraph.generated.NodeTypes.TYPE_DECL import io.shiftleft.codepropertygraph.generated.nodes.* import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger import overflowdb.Graph import overflowdb.Node import soot.* +import soot.jimple.* import soot.jimple.spark.SparkTransformer import soot.jimple.toolkits.callgraph.CHATransformer import soot.jimple.toolkits.callgraph.Edge @@ -248,6 +252,27 @@ class Extractor(val driver: IDriver) { } .distinct().toList().let { if (it.size >= 100000) it.parallelStream() else it.stream() } .filter { !it.isPhantom }.map { BriefUnitGraph(it.retrieveActiveBody()) }.toList() + // Build types from fields + classStream.asSequence().map { it.fields }.flatten().map { it.type }.distinct() + .filter { t -> + !classStream.any { it.name == t.toQuotedString() } + && + !programStructure.nodes { n -> n == TYPE_DECL }.asSequence() + .any { n -> n.property(FULL_NAME) == t.toQuotedString() } + } + .map(SootToPlumeUtil::buildTypeDeclaration) + .forEach(driver::addVertex) + // Build types from locals + graphs.asSequence().map { it.body.locals + it.body.parameterLocals }.flatten().map { it.type } + .distinct() + .filter { t -> + !classStream.any { it.name == t.toQuotedString() } + && + !programStructure.nodes { n -> n == TYPE_DECL }.asSequence() + .any { n -> n.property(FULL_NAME) == t.toQuotedString() } + } + .map(SootToPlumeUtil::buildTypeDeclaration) + .forEach(driver::addVertex) // Construct the CPGs for methods graphs.map(this::constructCPG) .toList().asSequence() @@ -281,7 +306,14 @@ class Extractor(val driver: IDriver) { if (programStructure.nodes { it == ODBFile.Label() }.asSequence().none { it.property("NAME") == cls.name }) { logger.debug("Building file, namespace, and type declaration for ${cls.name}") SootToPlumeUtil.buildClassStructure(cls, driver) - SootToPlumeUtil.buildTypeDeclaration(cls, driver) + val typeDecl = SootToPlumeUtil.buildTypeDeclaration(cls.type, false) + addSootToPlumeAssociation(cls, typeDecl) + cls.fields.forEachIndexed { i, field -> + SootToPlumeUtil.projectMember(field, i + 1).let { memberVertex -> + driver.addEdge(typeDecl, memberVertex, EdgeTypes.AST) + addSootToPlumeAssociation(field, memberVertex) + } + } } } @@ -313,7 +345,7 @@ class Extractor(val driver: IDriver) { val currentFileHash = getFileHashPair(cls) val files = programStructure.nodes { it == ODBFile.Label() }.asSequence() logger.debug("Looking for existing file vertex for ${cls.name} from given file hash $currentFileHash") - files.firstOrNull { it.property("NAME") == SootToPlumeUtil.sootClassToFileName(cls)}?.let { fileV -> + files.firstOrNull { it.property("NAME") == SootToPlumeUtil.sootClassToFileName(cls) }?.let { fileV -> if (fileV.property("HASH") != currentFileHash) { logger.debug("Existing class was found and file hashes do not match, marking for rebuild.") // Rebuild diff --git a/plume/src/main/kotlin/io/github/plume/oss/domain/mappers/VertexMapper.kt b/plume/src/main/kotlin/io/github/plume/oss/domain/mappers/VertexMapper.kt index d5823b6d..cce225fb 100644 --- a/plume/src/main/kotlin/io/github/plume/oss/domain/mappers/VertexMapper.kt +++ b/plume/src/main/kotlin/io/github/plume/oss/domain/mappers/VertexMapper.kt @@ -101,6 +101,7 @@ object VertexMapper { .name(map["NAME"] as String) .fullname(map["FULL_NAME"] as String) .order(map["ORDER"] as Int) + .isexternal(map["IS_EXTERNAL"] as Boolean) .id(map["id"] as Long) TypeParameter.Label() -> NewTypeParameterBuilder() .name(map["NAME"] as String) @@ -332,7 +333,6 @@ object VertexMapper { "POLICY_DIRECTORIES" -> Optional.empty() "INHERITS_FROM_TYPE_FULL_NAME" -> Optional.empty() "OVERLAYS" -> Optional.empty() - "IS_EXTERNAL" -> Optional.empty() else -> Optional.of(it.key) } if (key.isPresent) attributes[key.get()] = it.value diff --git a/plume/src/main/kotlin/io/github/plume/oss/drivers/GremlinDriver.kt b/plume/src/main/kotlin/io/github/plume/oss/drivers/GremlinDriver.kt index 56677588..9bc8cff3 100644 --- a/plume/src/main/kotlin/io/github/plume/oss/drivers/GremlinDriver.kt +++ b/plume/src/main/kotlin/io/github/plume/oss/drivers/GremlinDriver.kt @@ -166,13 +166,24 @@ abstract class GremlinDriver : IDriver { } override fun getProgramStructure(): overflowdb.Graph { - val programStructureSubGraph = g.V().hasLabel(File.Label()) + val sg = g.V().hasLabel(File.Label()) .repeat(un.outE(AST).inV()).emit() .inE() .subgraph("sg") .cap("sg") .next() - return gremlinToPlume(programStructureSubGraph.traversal()) + // Transfer type decl vertices to the result, this needs to be done with the tokens step to get all properties + // from the remote server + g.V().hasLabel(TypeDecl.Label()).valueMap() + .with(WithOptions.tokens) + .by(un.unfold()) + .toStream() + .map { Pair(sg.addVertex(T.label, it[T.label], T.id, it[T.id]), it as Map<*, *>) } + .forEach { (v, map) -> + map.filter { it.key != T.id && it.key != T.label } + .forEach { (key, value) -> v.property(key.toString(), value) } + } + return gremlinToPlume(sg.traversal()) } override fun getNeighbours(v: NewNodeBuilder): overflowdb.Graph { diff --git a/plume/src/main/kotlin/io/github/plume/oss/drivers/IDriver.kt b/plume/src/main/kotlin/io/github/plume/oss/drivers/IDriver.kt index 2318daed..1b8f551d 100644 --- a/plume/src/main/kotlin/io/github/plume/oss/drivers/IDriver.kt +++ b/plume/src/main/kotlin/io/github/plume/oss/drivers/IDriver.kt @@ -74,7 +74,7 @@ interface IDriver : AutoCloseable { fun getMethod(fullName: String, signature: String, includeBody: Boolean = false): Graph /** - * Obtains all program structure related vertices. + * Obtains all program structure related vertices. These are NAMESPACE_BLOCK, FILE, and TYPE_DECL vertices. * * @return The [Graph] containing the program structure related sub-graphs. */ diff --git a/plume/src/main/kotlin/io/github/plume/oss/drivers/Neo4jDriver.kt b/plume/src/main/kotlin/io/github/plume/oss/drivers/Neo4jDriver.kt index 460606bb..69abac72 100644 --- a/plume/src/main/kotlin/io/github/plume/oss/drivers/Neo4jDriver.kt +++ b/plume/src/main/kotlin/io/github/plume/oss/drivers/Neo4jDriver.kt @@ -5,8 +5,7 @@ import io.github.plume.oss.domain.mappers.VertexMapper import io.github.plume.oss.domain.mappers.VertexMapper.checkSchemaConstraints import io.github.plume.oss.domain.mappers.VertexMapper.mapToVertex import io.shiftleft.codepropertygraph.generated.EdgeTypes.AST -import io.shiftleft.codepropertygraph.generated.NodeTypes.FILE -import io.shiftleft.codepropertygraph.generated.NodeTypes.METHOD +import io.shiftleft.codepropertygraph.generated.NodeTypes.* import io.shiftleft.codepropertygraph.generated.nodes.NewNodeBuilder import org.apache.logging.log4j.LogManager import org.neo4j.driver.AuthTokens @@ -320,6 +319,17 @@ class Neo4jDriver : IDriver { ).list().map { it["x"] } } neo4jToOverflowGraph(result, graph) + val typeDecl = session.writeTransaction { tx -> + tx.run( + """ + MATCH (n:$TYPE_DECL) + RETURN n + """.trimIndent() + ).list() + } + typeDecl.map { it["n"].asNode() } + .map { mapToVertex(it.asMap() + mapOf("id" to it.id())) } + .forEach { addNodeToGraph(graph, it) } } return graph } diff --git a/plume/src/main/kotlin/io/github/plume/oss/drivers/OverflowDbDriver.kt b/plume/src/main/kotlin/io/github/plume/oss/drivers/OverflowDbDriver.kt index 325feae8..1eddfb8b 100644 --- a/plume/src/main/kotlin/io/github/plume/oss/drivers/OverflowDbDriver.kt +++ b/plume/src/main/kotlin/io/github/plume/oss/drivers/OverflowDbDriver.kt @@ -134,7 +134,12 @@ class OverflowDbDriver : IDriver { } override fun getProgramStructure(): Graph { - return deepCopyGraph(Traversals.getProgramStructure(graph)) + val g = deepCopyGraph(Traversals.getProgramStructure(graph)) + Traversals.getTypeDecls(graph).forEach { t -> + val node = g.addNode(t.id(), t.label()) + t.propertyMap().forEach { (key, value) -> node.setProperty(key, value) } + } + return g } override fun getNeighbours(v: NewNodeBuilder): Graph { diff --git a/plume/src/main/kotlin/io/github/plume/oss/graphio/GraphMLWriter.kt b/plume/src/main/kotlin/io/github/plume/oss/graphio/GraphMLWriter.kt index 380abb7e..5ad97f8f 100644 --- a/plume/src/main/kotlin/io/github/plume/oss/graphio/GraphMLWriter.kt +++ b/plume/src/main/kotlin/io/github/plume/oss/graphio/GraphMLWriter.kt @@ -54,6 +54,7 @@ object GraphMLWriter { when (u) { is String -> keySet[t] = "string" is Int -> keySet[t] = "int" + is Boolean -> keySet[t] = "boolean" is `$colon$colon`<*> -> keySet[t] = "string" is `Nil$` -> keySet[t] = "string" } diff --git a/plume/src/main/kotlin/io/github/plume/oss/graphio/GraphSONWriter.kt b/plume/src/main/kotlin/io/github/plume/oss/graphio/GraphSONWriter.kt index d922d4ef..136a0237 100644 --- a/plume/src/main/kotlin/io/github/plume/oss/graphio/GraphSONWriter.kt +++ b/plume/src/main/kotlin/io/github/plume/oss/graphio/GraphSONWriter.kt @@ -99,6 +99,7 @@ object GraphSONWriter { is String -> sb.append(",\"value\":\"$v\"") is Int -> sb.append(",\"value\":{\"@type\":\"g:Int32\",\"@value\":$v}") is Long -> sb.append(",\"value\":{\"@type\":\"g:Int64\",\"@value\":$v}") + is Boolean -> sb.append(",\"value\":$v") else -> println("Unsupported type $v ${v.javaClass}") } sb.append("}]") diff --git a/plume/src/main/kotlin/io/github/plume/oss/util/SootToPlumeUtil.kt b/plume/src/main/kotlin/io/github/plume/oss/util/SootToPlumeUtil.kt index f7b21720..c179aa8e 100644 --- a/plume/src/main/kotlin/io/github/plume/oss/util/SootToPlumeUtil.kt +++ b/plume/src/main/kotlin/io/github/plume/oss/util/SootToPlumeUtil.kt @@ -21,6 +21,7 @@ import io.github.plume.oss.domain.mappers.VertexMapper.mapToVertex import io.github.plume.oss.drivers.IDriver import io.github.plume.oss.util.SootParserUtil.determineEvaluationStrategy import io.shiftleft.codepropertygraph.generated.EdgeTypes.* +import io.shiftleft.codepropertygraph.generated.NodeTypes.UNKNOWN import io.shiftleft.codepropertygraph.generated.nodes.* import scala.Option import scala.jdk.CollectionConverters @@ -44,7 +45,7 @@ object SootToPlumeUtil { * * @param field The [SootField] from which the class member information is constructed from. */ - private fun projectMember(field: SootField, childIdx: Int): NewMemberBuilder = + fun projectMember(field: SootField, childIdx: Int): NewMemberBuilder = NewMemberBuilder() .name(field.name) .code(field.declaration) @@ -153,8 +154,16 @@ object SootToPlumeUtil { (Extractor.getSootAssociation(mtd)?.size ?: 0) + 1 ) // Create program structure - buildClassStructure(mtd.declaringClass, driver) - buildTypeDeclaration(mtd.declaringClass, driver) + val cls = mtd.declaringClass + buildClassStructure(cls, driver) + val typeDecl = buildTypeDeclaration(cls.type) + addSootToPlumeAssociation(cls, typeDecl) + cls.fields.forEachIndexed { i, field -> + projectMember(field, i + 1).let { memberVertex -> + driver.addEdge(typeDecl, memberVertex, AST) + addSootToPlumeAssociation(field, memberVertex) + } + } connectMethodToTypeDecls(mtd, driver) return mtdVertex } @@ -245,10 +254,10 @@ object SootToPlumeUtil { * @param cls the soot class * @return the filename in string form * */ - fun sootClassToFileName(cls : SootClass) : String { + fun sootClassToFileName(cls: SootClass): String { val packageName = cls.packageName return if (packageName != null) { - "/" + cls.name.replace(".", "/") + ".class" + "/" + cls.name.replace(".", "/") + ".class" } else { io.shiftleft.semanticcpg.language.types.structure.File.UNKNOWN() } @@ -260,7 +269,11 @@ object SootToPlumeUtil { * @param namespaceList A list of package names. * @return The final [NewNamespaceBlockBuilder] in the chain (the one associated with the file). */ - private fun populateNamespaceChain(namespaceList: Array, filename : String, driver: IDriver): NewNamespaceBlockBuilder { + private fun populateNamespaceChain( + namespaceList: Array, + filename: String, + driver: IDriver + ): NewNamespaceBlockBuilder { var prevNamespaceBlock: NewNamespaceBlockBuilder var currNamespaceBlock: NewNamespaceBlockBuilder? = null driver.getProgramStructure().use { programStructure -> @@ -304,30 +317,36 @@ object SootToPlumeUtil { } /** - * Given a class will construct a type declaration with members. + * Given a type will construct a type declaration with members. * - * @param cls The [SootClass] to create the declaration from. - * @param driver The driver to construct the type declaration to. + * @param type The [soot.Type] to create the declaration from. + * @param isExternal Whether the type is part of the application or it is external. * @return The [NewTypeDecl] representing this newly created vertex. */ - fun buildTypeDeclaration(cls: SootClass, driver: IDriver): NewTypeDeclBuilder = - NewTypeDeclBuilder() - .name(cls.shortName) - .fullname(cls.name) - .filename(sootClassToFileName(cls)) - .astparentfullname(cls.getPackageName()) + fun buildTypeDeclaration(type: soot.Type, isExternal: Boolean = true): NewTypeDeclBuilder { + val filename = if (isExternal) { + "<${UNKNOWN.toLowerCase()}>" + } else { + if (type.toQuotedString().contains('.')) "/${ + type.toQuotedString().replace(".", "/").removeSuffix("[]") + }.class" + else type.toQuotedString() + } + val parentType = if (type.toQuotedString().contains('.')) type.toQuotedString().substringBeforeLast(".") + else type.toQuotedString() + val shortName = if (type.toQuotedString().contains('.')) type.toQuotedString().substringAfterLast('.') + else type.toQuotedString() + + return NewTypeDeclBuilder() + .name(shortName) + .fullname(type.toQuotedString()) + .filename(filename) + .astparentfullname(parentType) .astparenttype("NAMESPACE_BLOCK") - .order(0) - .apply { - // Attach fields to the TypeDecl - cls.fields.forEachIndexed { i, field -> - projectMember(field, i + 1).let { memberVertex -> - driver.addEdge(this, memberVertex, AST) - addSootToPlumeAssociation(field, memberVertex) - } - } - addSootToPlumeAssociation(cls, this) - } + .order(if (isExternal) -1 else 1) + .isexternal(isExternal) + .apply { addSootToPlumeAssociation(type, this) } + } /** * Connects the given method's [BriefUnitGraph] to its type declaration and source file (if present). diff --git a/plume/src/test/kotlin/io/github/plume/oss/TestDomainResources.kt b/plume/src/test/kotlin/io/github/plume/oss/TestDomainResources.kt index d76bfabf..c2e24e84 100644 --- a/plume/src/test/kotlin/io/github/plume/oss/TestDomainResources.kt +++ b/plume/src/test/kotlin/io/github/plume/oss/TestDomainResources.kt @@ -24,6 +24,7 @@ class TestDomainResources { val EVAL_2 = BY_SHARING val MOD_1 = ABSTRACT val MOD_2 = CONSTRUCTOR + val BOOL_1 = false val vertices = listOf( NewArrayInitializerBuilder().order(INT_1), @@ -63,7 +64,7 @@ class TestDomainResources { NewReturnBuilder().order(INT_1).argumentindex(INT_1).code(STRING_1).linenumber(Option.apply(INT_1)) .columnnumber(Option.apply(INT_1)), NewTypeArgumentBuilder().order(INT_1), - NewTypeDeclBuilder().name(STRING_1).fullname(STRING_1).order(INT_1), + NewTypeDeclBuilder().name(STRING_1).fullname(STRING_1).order(INT_1).isexternal(BOOL_1), NewTypeParameterBuilder().name(STRING_1).order(INT_1), NewTypeRefBuilder().typefullname(STRING_1) .dynamictypehintfullname(createScalaList(STRING_1)) diff --git a/plume/src/test/kotlin/io/github/plume/oss/drivers/JanusGraphDriverIntTest.kt b/plume/src/test/kotlin/io/github/plume/oss/drivers/JanusGraphDriverIntTest.kt index 55f2bb0f..e41c92d3 100644 --- a/plume/src/test/kotlin/io/github/plume/oss/drivers/JanusGraphDriverIntTest.kt +++ b/plume/src/test/kotlin/io/github/plume/oss/drivers/JanusGraphDriverIntTest.kt @@ -31,6 +31,7 @@ import io.github.plume.oss.TestDomainResources.Companion.typeParameterVertex import io.github.plume.oss.TestDomainResources.Companion.typeRefVertex import io.github.plume.oss.TestDomainResources.Companion.unknownVertex import io.github.plume.oss.domain.exceptions.PlumeSchemaViolationException +import io.github.plume.oss.graphio.GraphMLWriter import io.github.plume.oss.util.SootToPlumeUtil import io.shiftleft.codepropertygraph.generated.EdgeTypes.* import io.shiftleft.codepropertygraph.generated.nodes.* @@ -39,6 +40,7 @@ import org.junit.jupiter.api.* import org.junit.jupiter.api.Assertions.* import overflowdb.Graph import scala.Option +import java.io.FileWriter import kotlin.properties.Delegates class JanusGraphDriverIntTest { @@ -504,15 +506,16 @@ class JanusGraphDriverIntTest { g = driver.getProgramStructure() val ns = g.nodes().asSequence().toList() val es = g.edges().asSequence().toList() - assertEquals(3, ns.size) + assertEquals(4, ns.size) assertEquals(2, es.size) val file = g.V(fileVertex.id()).next() val ns1 = g.V(namespaceBlockVertex1.id()).next() - // Assert no program structure vertices part of the method body + // Assert program structure vertices are present assertTrue(ns.any { it.id() == namespaceBlockVertex2.id() }) assertTrue(ns.any { it.id() == namespaceBlockVertex1.id() }) assertTrue(ns.any { it.id() == fileVertex.id() }) + assertTrue(ns.any { it.id() == typeDeclVertex.id() }) // Check that vertices are connected by AST edges assertTrue(file.out(AST).asSequence().any { it.id() == namespaceBlockVertex1.id() }) assertTrue(ns1.out(AST).asSequence().any { it.id() == namespaceBlockVertex2.id() }) diff --git a/plume/src/test/kotlin/io/github/plume/oss/drivers/Neo4jDriverIntTest.kt b/plume/src/test/kotlin/io/github/plume/oss/drivers/Neo4jDriverIntTest.kt index 892aceac..4e401d10 100644 --- a/plume/src/test/kotlin/io/github/plume/oss/drivers/Neo4jDriverIntTest.kt +++ b/plume/src/test/kotlin/io/github/plume/oss/drivers/Neo4jDriverIntTest.kt @@ -508,15 +508,16 @@ class Neo4jDriverIntTest { g = driver.getProgramStructure() val ns = g.nodes().asSequence().toList() val es = g.edges().asSequence().toList() - assertEquals(3, ns.size) + assertEquals(4, ns.size) assertEquals(2, es.size) val file = g.V(fileVertex.id()).next() val ns1 = g.V(namespaceBlockVertex1.id()).next() - // Assert no program structure vertices part of the method body + // Assert program structure vertices are present assertTrue(ns.any { it.id() == namespaceBlockVertex2.id() }) assertTrue(ns.any { it.id() == namespaceBlockVertex1.id() }) assertTrue(ns.any { it.id() == fileVertex.id() }) + assertTrue(ns.any { it.id() == typeDeclVertex.id() }) // Check that vertices are connected by AST edges assertTrue(file.out(AST).asSequence().any { it.id() == namespaceBlockVertex1.id() }) assertTrue(ns1.out(AST).asSequence().any { it.id() == namespaceBlockVertex2.id() }) diff --git a/plume/src/test/kotlin/io/github/plume/oss/drivers/NeptuneDriverIntTest.kt b/plume/src/test/kotlin/io/github/plume/oss/drivers/NeptuneDriverIntTest.kt index 1794bdb4..fe01caae 100644 --- a/plume/src/test/kotlin/io/github/plume/oss/drivers/NeptuneDriverIntTest.kt +++ b/plume/src/test/kotlin/io/github/plume/oss/drivers/NeptuneDriverIntTest.kt @@ -503,15 +503,16 @@ class NeptuneDriverIntTest { g = driver.getProgramStructure() val ns = g.nodes().asSequence().toList() val es = g.edges().asSequence().toList() - assertEquals(3, ns.size) + assertEquals(4, ns.size) assertEquals(2, es.size) val file = g.V(fileVertex.id()).next() val ns1 = g.V(namespaceBlockVertex1.id()).next() - // Assert no program structure vertices part of the method body + // Assert program structure vertices are present assertTrue(ns.any { it.id() == namespaceBlockVertex2.id() }) assertTrue(ns.any { it.id() == namespaceBlockVertex1.id() }) assertTrue(ns.any { it.id() == fileVertex.id() }) + assertTrue(ns.any { it.id() == typeDeclVertex.id() }) // Check that vertices are connected by AST edges assertTrue(file.out(AST).asSequence().any { it.id() == namespaceBlockVertex1.id() }) assertTrue(ns1.out(AST).asSequence().any { it.id() == namespaceBlockVertex2.id() }) diff --git a/plume/src/test/kotlin/io/github/plume/oss/drivers/OverflowDbDriverIntTest.kt b/plume/src/test/kotlin/io/github/plume/oss/drivers/OverflowDbDriverIntTest.kt index a54aa934..9cee933d 100644 --- a/plume/src/test/kotlin/io/github/plume/oss/drivers/OverflowDbDriverIntTest.kt +++ b/plume/src/test/kotlin/io/github/plume/oss/drivers/OverflowDbDriverIntTest.kt @@ -526,15 +526,16 @@ class OverflowDbDriverIntTest { g = driver.getProgramStructure() val ns = g.nodes().asSequence().toList() val es = g.edges().asSequence().toList() - assertEquals(3, ns.size) + assertEquals(4, ns.size) assertEquals(2, es.size) val file = g.V(fileVertex.id()).next() val ns1 = g.V(namespaceBlockVertex1.id()).next() - // Assert no program structure vertices part of the method body + // Assert program structure vertices are present assertTrue(ns.any { it.id() == namespaceBlockVertex2.id() }) assertTrue(ns.any { it.id() == namespaceBlockVertex1.id() }) assertTrue(ns.any { it.id() == fileVertex.id() }) + assertTrue(ns.any { it.id() == typeDeclVertex.id() }) // Check that vertices are connected by AST edges assertTrue(file.out(AST).asSequence().any { it.id() == namespaceBlockVertex1.id() }) assertTrue(ns1.out(AST).asSequence().any { it.id() == namespaceBlockVertex2.id() }) diff --git a/plume/src/test/kotlin/io/github/plume/oss/drivers/TigerGraphDriverIntTest.kt b/plume/src/test/kotlin/io/github/plume/oss/drivers/TigerGraphDriverIntTest.kt index 44f255c0..ab1309d8 100644 --- a/plume/src/test/kotlin/io/github/plume/oss/drivers/TigerGraphDriverIntTest.kt +++ b/plume/src/test/kotlin/io/github/plume/oss/drivers/TigerGraphDriverIntTest.kt @@ -515,15 +515,16 @@ class TigerGraphDriverIntTest { g = driver.getProgramStructure() val ns = g.nodes().asSequence().toList() val es = g.edges().asSequence().toList() - assertEquals(3, ns.size) + assertEquals(4, ns.size) assertEquals(2, es.size) val file = g.V(fileVertex.id()).next() val ns1 = g.V(namespaceBlockVertex1.id()).next() - // Assert no program structure vertices part of the method body + // Assert program structure vertices are present assertTrue(ns.any { it.id() == namespaceBlockVertex2.id() }) assertTrue(ns.any { it.id() == namespaceBlockVertex1.id() }) assertTrue(ns.any { it.id() == fileVertex.id() }) + assertTrue(ns.any { it.id() == typeDeclVertex.id() }) // Check that vertices are connected by AST edges assertTrue(file.out(AST).asSequence().any { it.id() == namespaceBlockVertex1.id() }) assertTrue(ns1.out(AST).asSequence().any { it.id() == namespaceBlockVertex2.id() }) diff --git a/plume/src/test/kotlin/io/github/plume/oss/drivers/TinkerGraphDriverIntTest.kt b/plume/src/test/kotlin/io/github/plume/oss/drivers/TinkerGraphDriverIntTest.kt index 7eb2bf96..d5c23d7f 100644 --- a/plume/src/test/kotlin/io/github/plume/oss/drivers/TinkerGraphDriverIntTest.kt +++ b/plume/src/test/kotlin/io/github/plume/oss/drivers/TinkerGraphDriverIntTest.kt @@ -567,15 +567,16 @@ class TinkerGraphDriverIntTest { g = driver.getProgramStructure() val ns = g.nodes().asSequence().toList() val es = g.edges().asSequence().toList() - assertEquals(3, ns.size) + assertEquals(4, ns.size) assertEquals(2, es.size) val file = g.V(fileVertex.id()).next() val ns1 = g.V(namespaceBlockVertex1.id()).next() - // Assert no program structure vertices part of the method body + // Assert program structure vertices are present assertTrue(ns.any { it.id() == namespaceBlockVertex2.id() }) assertTrue(ns.any { it.id() == namespaceBlockVertex1.id() }) assertTrue(ns.any { it.id() == fileVertex.id() }) + assertTrue(ns.any { it.id() == typeDeclVertex.id() }) // Check that vertices are connected by AST edges assertTrue(file.out(AST).asSequence().any { it.id() == namespaceBlockVertex1.id() }) assertTrue(ns1.out(AST).asSequence().any { it.id() == namespaceBlockVertex2.id() }) @@ -592,8 +593,8 @@ class TinkerGraphDriverIntTest { val file = g.V(fileVertex.id()).next() val mtd = g.V(methodVertex.id()).next() // Check that vertices are connected by AST edges - assertTrue(file.out(AST).asSequence().any { it.id() == namespaceBlockVertex1.id()}) - assertTrue(mtd.out(SOURCE_FILE).asSequence().any { it.id() == fileVertex.id()}) + assertTrue(file.out(AST).asSequence().any { it.id() == namespaceBlockVertex1.id() }) + assertTrue(mtd.out(SOURCE_FILE).asSequence().any { it.id() == fileVertex.id() }) } } diff --git a/plume/src/test/kotlin/io/github/plume/oss/extractor/intraprocedural/BasicIntraproceduralTest.kt b/plume/src/test/kotlin/io/github/plume/oss/extractor/intraprocedural/BasicIntraproceduralTest.kt index cd9c4ebe..d3695746 100644 --- a/plume/src/test/kotlin/io/github/plume/oss/extractor/intraprocedural/BasicIntraproceduralTest.kt +++ b/plume/src/test/kotlin/io/github/plume/oss/extractor/intraprocedural/BasicIntraproceduralTest.kt @@ -142,7 +142,10 @@ class BasicIntraproceduralTest { ns.filterIsInstance().let { mrv -> assertNotNull(mrv.find { it.fullName() == "intraprocedural.basic.Basic5" }) assertNotNull(mrv.find { it.fullName() == "intraprocedural.basic.basic5.Basic5" }) - assertEquals(2, mrv.toList().size) + assertNotNull(mrv.find { it.fullName() == "int" }) + assertNotNull(mrv.find { it.fullName() == "byte" }) + assertNotNull(mrv.find { it.fullName() == "java.lang.String[]" }) + assertEquals(5, mrv.toList().size) } ns.filterIsInstance().let { mv -> assertNotNull(mv.find { it.fullName() == "intraprocedural.basic.Basic5.main" }) diff --git a/plume/src/test/resources/schema/jg_schema.groovy b/plume/src/test/resources/schema/jg_schema.groovy index 8e48e59e..5eac10c2 100644 --- a/plume/src/test/resources/schema/jg_schema.groovy +++ b/plume/src/test/resources/schema/jg_schema.groovy @@ -39,6 +39,7 @@ mgmt.containsEdgeLabel('BINDS_TO') ?: mgmt.makeEdgeLabel('BINDS_TO').make() // Properties mgmt.containsPropertyKey('AST_PARENT_TYPE') ?: mgmt.makePropertyKey('AST_PARENT_TYPE').dataType(String.class).cardinality(Cardinality.SINGLE).make() mgmt.containsPropertyKey('AST_PARENT_FULL_NAME') ?: mgmt.makePropertyKey('AST_PARENT_FULL_NAME').dataType(String.class).cardinality(Cardinality.SINGLE).make() +mgmt.containsPropertyKey('IS_EXTERNAL') ?: mgmt.makePropertyKey('IS_EXTERNAL').dataType(Boolean.class).cardinality(Cardinality.SINGLE).make() mgmt.containsPropertyKey('FILENAME') ?: mgmt.makePropertyKey('FILENAME').dataType(String.class).cardinality(Cardinality.SINGLE).make() mgmt.containsPropertyKey('SIGNATURE') ?: mgmt.makePropertyKey('SIGNATURE').dataType(String.class).cardinality(Cardinality.SINGLE).make() mgmt.containsPropertyKey('EVALUATION_STRATEGY') ?: mgmt.makePropertyKey('EVALUATION_STRATEGY').dataType(String.class).cardinality(Cardinality.SINGLE).make() diff --git a/plume/src/test/resources/schema/tg_schema.gsql b/plume/src/test/resources/schema/tg_schema.gsql index cb1f3e0f..150677a8 100644 --- a/plume/src/test/resources/schema/tg_schema.gsql +++ b/plume/src/test/resources/schema/tg_schema.gsql @@ -27,7 +27,8 @@ CREATE VERTEX CPG_VERT ( CANONICAL_NAME STRING DEFAULT "null", AST_ORDER INT DEFAULT -1, SIGNATURE STRING DEFAULT "()", - HASH STRING DEFAULT "null" + HASH STRING DEFAULT "null", + IS_EXTERNAL BOOL DEFAULT "TRUE" ) WITH primary_id_as_attribute="true" ## Language Dependent CREATE VERTEX META_DATA_VERT ( @@ -91,57 +92,51 @@ CREATE QUERY getMethodHead(STRING FULL_NAME, STRING SIGNATURE) FOR GRAPH cpg { PRINT @@edges; } -CREATE QUERY getMethod(STRING FULL_NAME, STRING SIGNATURE) FOR GRAPH cpg { +CREATE QUERY getMethod(STRING FULL_NAME, STRING SIGNATURE) FOR GRAPH cpg SYNTAX v2 { SetAccum @@edges; - OrAccum @visited; allV = {ANY}; + # Get method start = SELECT src FROM allV:src WHERE src.FULL_NAME == FULL_NAME AND src.SIGNATURE == SIGNATURE; allVert = start; - - WHILE start.size() > 0 DO - start = SELECT t - FROM start:s -((AST|REF|CFG|ARGUMENT|CAPTURED_BY|BINDS_TO|RECEIVER|CONDITION|BINDS):e) ->:t - WHERE NOT t.@visited - ACCUM t.@visited = TRUE, @@edges += e; - allVert = allVert UNION start; - END; - + # Get method's body vertices + start = SELECT t + FROM start:s -((AST>|REF>|CFG>|ARGUMENT>|CAPTURED_BY>|BINDS_TO>|RECEIVER>|CONDITION>|BINDS>)*) - :t; + allVert = allVert UNION start; + # Get edges between body methods finalEdges = SELECT t - FROM allVert -((AST|REF|CFG|ARGUMENT|CAPTURED_BY|BINDS_TO|RECEIVER|CONDITION|BINDS):e)->:t + FROM allVert -((AST>|REF>|CFG>|ARGUMENT>|CAPTURED_BY>|BINDS_TO>|RECEIVER>|CONDITION>|BINDS>):e)-:t ACCUM @@edges += e; - PRINT allVert; PRINT @@edges; } -CREATE QUERY getProgramStructure() FOR GRAPH cpg { +CREATE QUERY getProgramStructure() FOR GRAPH cpg SYNTAX v2 { SetAccum @@edges; - OrAccum @visited; + start = {CPG_VERT.*}; start = SELECT s - FROM start:s -(:e)-> :t - WHERE s.label == "FILE"; + FROM start:s + WHERE s.label == "FILE" OR s.label == "TYPE_DECL"; allVert = start; - WHILE start.size() > 0 DO - start = SELECT t - FROM start:s -(AST:e) ->:t - WHERE NOT t.@visited - ACCUM t.@visited = TRUE, @@edges += e; - allVert = allVert UNION start; - END; + start = SELECT t + FROM start:s -(AST>*)- :t + WHERE t.label == "NAMESPACE_BLOCK"; + allVert = allVert UNION start; finalEdges = SELECT t - FROM allVert -(AST:e)->:t + FROM allVert -(AST>:e)- :t + WHERE t.label == "NAMESPACE_BLOCK" ACCUM @@edges += e; + start = {CPG_VERT.*}; PRINT allVert; PRINT @@edges; } -CREATE QUERY getNeighbours(VERTEX SOURCE) FOR GRAPH cpg { +CREATE QUERY getNeighbours(VERTEX SOURCE) FOR GRAPH cpg SYNTAX v2 { SetAccum @@edges; seed = {CPG_VERT.*}; sourceSet = {SOURCE}; @@ -149,33 +144,23 @@ CREATE QUERY getNeighbours(VERTEX SOURCE) FOR GRAPH cpg { FROM seed:src -(:e)- CPG_VERT:tgt WHERE src == SOURCE ACCUM @@edges += e; - inVert = SELECT src - FROM seed:src -(:e)- CPG_VERT:tgt - WHERE tgt == SOURCE - ACCUM @@edges += e; - allVert = inVert UNION outVert; - allVert = allVert UNION sourceSet; + allVert = outVert UNION sourceSet; PRINT allVert; PRINT @@edges; } -CREATE QUERY deleteMethod(STRING FULL_NAME, STRING SIGNATURE) FOR GRAPH cpg { - SetAccum @@edges; - OrAccum @visited; +CREATE QUERY deleteMethod(STRING FULL_NAME, STRING SIGNATURE) FOR GRAPH cpg SYNTAX v2 { allV = {ANY}; + # Get method start = SELECT src FROM allV:src WHERE src.FULL_NAME == FULL_NAME AND src.SIGNATURE == SIGNATURE; allVert = start; - - WHILE start.size() > 0 DO - start = SELECT t - FROM start:s -((AST|REF|CFG|ARGUMENT|CAPTURED_BY|BINDS_TO|RECEIVER|CONDITION|BINDS):e) ->:t - WHERE NOT t.@visited - ACCUM t.@visited = TRUE, @@edges += e; - allVert = allVert UNION start; - END; + # Get method's body vertices + start = SELECT t + FROM start:s -((AST>|REF>|CFG>|ARGUMENT>|CAPTURED_BY>|BINDS_TO>|RECEIVER>|CONDITION>|BINDS>)*) - :t; + allVert = allVert UNION start; DELETE s FROM allVert:s; } diff --git a/testing/src/test/scala/io/github/plume/oss/querying/TypeDeclTests.scala b/testing/src/test/scala/io/github/plume/oss/querying/TypeDeclTests.scala new file mode 100644 index 00000000..2fd90f1f --- /dev/null +++ b/testing/src/test/scala/io/github/plume/oss/querying/TypeDeclTests.scala @@ -0,0 +1,45 @@ +package io.github.plume.oss.querying + +import io.github.plume.oss.PlumeCodeToCpgSuite +import io.shiftleft.semanticcpg.language._ +import io.shiftleft.semanticcpg.language.types.structure.File + +class TypeDeclTests extends PlumeCodeToCpgSuite { + + override val code = + """ + | package Foo; + | class Bar extends Woo { + | int x; + | int method () { return 1; } + | }; + | class Woo {} + """.stripMargin + + "should contain a type decl for `foo` with correct fields" in { + val List(x) = cpg.typeDecl.name("Bar").l + x.name shouldBe "Bar" + x.fullName shouldBe "Foo.Bar" + x.isExternal shouldBe false + // TODO: Inheritance still needs to be added + // x.inheritsFromTypeFullName shouldBe List("Woo") + x.aliasTypeFullName shouldBe None + x.order shouldBe 1 + x.filename.startsWith("/") shouldBe true + x.filename.endsWith(".class") shouldBe true + } + + "should contain type decl for external type `int`" in { + cpg.typeDecl.l.foreach(t => println(t.fullName)) + val List(x) = cpg.typeDecl("int").l + x.name shouldBe "int" + x.fullName shouldBe "int" + x.isExternal shouldBe true +// x.inheritsFromTypeFullName shouldBe List() + x.aliasTypeFullName shouldBe None + x.order shouldBe -1 + x.filename shouldBe File.UNKNOWN + } + + +} \ No newline at end of file diff --git a/testing/src/test/scala/io/github/plume/oss/querying/TypeTests.scala b/testing/src/test/scala/io/github/plume/oss/querying/TypeTests.scala new file mode 100644 index 00000000..310ffa96 --- /dev/null +++ b/testing/src/test/scala/io/github/plume/oss/querying/TypeTests.scala @@ -0,0 +1,69 @@ +package io.github.plume.oss.querying + +import io.github.plume.oss.PlumeCodeToCpgSuite +import io.shiftleft.semanticcpg.language._ + +class TypeTests extends PlumeCodeToCpgSuite { + + override val code = + """ + | package foo; + | class Foo { + | Long x; + | + | Integer myFunc(Object param) { + | Double y; + | return 1; + | } + | }; + |""".stripMargin + + "should create TYPE node with correct fields for class member" in { +// val List(x) = cpg.typ.name("Long").l +// x.name shouldBe "Long" +// x.fullName shouldBe "java.lang.Long" +// x.typeDeclFullName shouldBe "java.long.Long" + } + + "should create TYPE node with correct fields for return type" in { +// val List(x) = cpg.typ.name("Integer").l +// x.name shouldBe "Integer" +// x.fullName shouldBe "java.lang.Integer" +// x.typeDeclFullName shouldBe "java.lang.Integer" + } + + "should create TYPE node with correct fields for parameter type" in { +// val List(x) = cpg.typ.name("Object").l +// x.name shouldBe "Object" +// x.fullName shouldBe "java.lang.Object" +// x.typeDeclFullName shouldBe "java.lang.Object" + } + + "should create TYPE node with correct fields for local type" in { +// val List(x) = cpg.typ.name("Double").l +// x.name shouldBe "Double" +// x.fullName shouldBe "java.lang.Double" +// x.typeDeclFullName shouldBe "java.lang.Double" + } + + "should allow traversing from member's TYPE to member" in { +// val List(x) = cpg.typ("java.lang.Long").memberOfType.l +// x.name shouldBe "x" + } + + "should allow traversing from return params TYPE to return param" in { +// val List(x) = cpg.typ("java.lang.Integer").methodReturnOfType.l +// x.typeFullName shouldBe "java.lang.Integer" + } + + "should allow traversing from params TYPE to param" in { +// val List(x) = cpg.typ("java.lang.Object").parameterOfType.l +// x.name shouldBe "param" + } + + "should allow traversing from local's TYPE to local" in { +// val List(x) = cpg.typ("java.lang.Double").localOfType.l +// x.name shouldBe "y" + } + +}