Handing functionality off to jimple2cpg that includes many bug fixes (#…

…241) * Included jimple2cpg functionality and updated tests * Styling * Fixed driver fixture import * Fixed diff detection on operators
plume-oss · Mar 9, 2022 · 7c15c6c · 7c15c6c
1 parent 51af989
commit 7c15c6c
Show file tree

Hide file tree

Showing 32 changed files with 224 additions and 1,363 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+### Added
+
+- Object data flow tests that explores fields and inter-procedural calls (both static + virtual).
+- Data flow tests followed from `JavaSrc2Cpg`, specifically `OperatorTests`
+
+### Changed
+
+- Importing `jimple2cpg` instead of duplicating work here.
+- Removed parsing Jimple `IdentityStmt` since they end up duplicating parameters as locals.
+- Swapped out the deprecated `ParallelCpgPass` for `ConcurrentWriterCpgPass`
+
+### Fixed
+
+- Virtual calls now get passed the correct `this` object as the object the method is actually invoked with.
+- Virtual call `code` properties now include the object invoking the method.
+- Instance where `Local` nodes were duplicated for each time they were referenced
+- Issue where `NewCall(Operators.assignment)` did not have `methodFullName = Operators.assignment` and thus messing up data flow paths
+
 ## [1.0.17] - 2022-03-04
 
 ### Added

diff --git a/build.sbt b/build.sbt
@@ -15,18 +15,18 @@ inThisBuild(
 )
 
 val cpgVersion         = "1.3.509"
-val joernVersion       = "1.1.590"
+val joernVersion       = "1.1.606"
 val sootVersion        = "4.2.1"
 val tinkerGraphVersion = "3.4.8"
 val neo4jVersion       = "4.4.3"
 val tigerGraphVersion  = "3.1.0"
-val sttpVersion        = "3.4.1"
-val jacksonVersion     = "2.13.1"
+val sttpVersion        = "3.4.2"
+val jacksonVersion     = "2.13.2"
 val scalajHttpVersion  = "2.4.2"
 val lz4Version         = "1.8.0"
 val slf4jVersion       = "1.7.36"
 val logbackVersion     = "1.2.10"
-val scalatestVersion   = "3.2.9"
+val scalatestVersion   = "3.2.11"
 val circeVersion       = "0.14.1"
 
 lazy val scalatest         = "org.scalatest" %% "scalatest" % scalatestVersion
@@ -43,7 +43,8 @@ libraryDependencies ++= Seq(
   "io.shiftleft"                  %% "semanticcpg"          % cpgVersion,
   "io.joern"                      %% "dataflowengineoss"    % joernVersion,
   "io.joern"                      %% "x2cpg"                % joernVersion,
-  "io.shiftleft"                  %% "semanticcpg"          % cpgVersion       % Test classifier "tests",
+  "io.joern"                      %% "jimple2cpg"           % joernVersion,
+  "io.joern"                      %% "x2cpg"                % joernVersion     % Test classifier "tests",
   "org.soot-oss"                   % "soot"                 % sootVersion,
   "org.apache.tinkerpop"           % "tinkergraph-gremlin"  % tinkerGraphVersion,
   "org.apache.tinkerpop"           % "gremlin-driver"       % tinkerGraphVersion,
@@ -56,7 +57,7 @@ libraryDependencies ++= Seq(
   "org.scalaj"                     % "scalaj-http_2.13"     % scalajHttpVersion,
   "org.lz4"                        % "lz4-java"             % lz4Version,
   "org.slf4j"                      % "slf4j-api"            % slf4jVersion,
-  "org.slf4j"                      % "slf4j-simple"         % slf4jVersion % Runtime,
+  "org.slf4j"                      % "slf4j-simple"         % slf4jVersion     % Runtime,
   "org.scala-lang"                 % "scala-reflect"        % scalaVersion.value,
   "ch.qos.logback"                 % "logback-classic"      % logbackVersion   % Test,
   "org.scalatest"                 %% "scalatest"            % scalatestVersion % Test

diff --git a/src/main/scala/com/github/plume/oss/Jimple2Cpg.scala b/src/main/scala/com/github/plume/oss/Jimple2Cpg.scala
@@ -2,13 +2,23 @@ package com.github.plume.oss
 
 import com.github.plume.oss.drivers.{IDriver, OverflowDbDriver}
 import com.github.plume.oss.passes._
-import com.github.plume.oss.passes.concurrent.{
-  PlumeCfgCreationPass,
+import com.github.plume.oss.passes.base.{
+  AstCreationPass,
   PlumeContainsEdgePass,
-  PlumeHashPass
+  PlumeCpgPassBase,
+  PlumeFileCreationPass,
+  PlumeMetaDataPass,
+  PlumeMethodDecoratorPass,
+  PlumeMethodStubCreator,
+  PlumeNamespaceCreator,
+  PlumeTypeDeclStubCreator,
+  PlumeTypeNodePass
 }
-import com.github.plume.oss.passes.forkjoin.{PlumeCdgPass, PlumeCfgDominatorPass}
-import com.github.plume.oss.passes.parallel._
+import com.github.plume.oss.passes.incremental.{PlumeDiffPass, PlumeHashPass}
+import com.github.plume.oss.passes.controlflow.PlumeCfgCreationPass
+import com.github.plume.oss.passes.controlflow.cfgdominator.PlumeCfgDominatorPass
+import com.github.plume.oss.passes.controlflow.codepencegraph.PlumeCdgPass
+import com.github.plume.oss.passes.reachingdef._
 import com.github.plume.oss.util.ProgramHandlingUtil
 import com.github.plume.oss.util.ProgramHandlingUtil.{extractSourceFilesFromArchive, moveClassFiles}
 import io.joern.x2cpg.SourceFiles
@@ -120,7 +130,6 @@ class Jimple2Cpg {
           return cpg
         } else {
           logger.info(s"Processing ${codeToProcess.size} new or changed program files")
-          logger.debug(s"Files to process are: $sourceFileNames")
         }
 
         // After the diff pass any changed types are removed. Remaining types should be black listed to avoid duplicates

diff --git a/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala b/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala
@@ -8,7 +8,7 @@ import com.github.plume.oss.domain.{
   deserializeResultTable
 }
 import com.github.plume.oss.drivers.OverflowDbDriver.newOverflowGraph
-import com.github.plume.oss.passes.PlumeDynamicCallLinker
+import com.github.plume.oss.passes.callgraph.PlumeDynamicCallLinker
 import com.github.plume.oss.util.BatchedUpdateUtil._
 import io.joern.dataflowengineoss.language.toExtendedCfgNode
 import io.joern.dataflowengineoss.queryengine._
@@ -191,11 +191,12 @@ final case class OverflowDbDriver(
   }
 
   override def bulkTx(dg: AppliedDiff): Unit = {
-    dg.getDiffGraph.iterator.forEachRemaining {
-      case node: DetachedNodeData =>
-        val id      = idFromNodeData(node)
-        val newNode = cpg.graph.addNode(id, node.label)
-        propertiesFromNodeData(node).foreach { case (k, v) => newNode.setProperty(k, v) }
+    dg.getDiffGraph.iterator.collect { case x: DetachedNodeData => x }.foreach { node =>
+      val id      = idFromNodeData(node)
+      val newNode = cpg.graph.addNode(id, node.label)
+      propertiesFromNodeData(node).foreach { case (k, v) => newNode.setProperty(k, v) }
+    }
+    dg.getDiffGraph.iterator.filterNot(_.isInstanceOf[DetachedNodeData]).foreach {
       case c: BatchedUpdate.CreateEdge =>
         val srcId = idFromNodeData(c.src)
         val dstId = idFromNodeData(c.dst)
@@ -211,16 +212,16 @@ final case class OverflowDbDriver(
     }
   }
 
-  private def dfsDelete(
+  private def accumNodesToDelete(
       n: Node,
       visitedNodes: mutable.Set[Node],
       edgeToFollow: String*
   ): Unit = {
     if (!visitedNodes.contains(n)) {
       visitedNodes.add(n)
-      n.out(edgeToFollow: _*).forEachRemaining(dfsDelete(_, visitedNodes, edgeToFollow: _*))
+      n.out(edgeToFollow: _*)
+        .forEachRemaining(accumNodesToDelete(_, visitedNodes, edgeToFollow: _*))
     }
-    n.remove()
   }
 
   override def removeSourceFiles(filenames: String*): Unit = {
@@ -237,8 +238,11 @@ final case class OverflowDbDriver(
         // Remove TYPE nodes
         typeDecls.flatMap(_.in(EdgeTypes.REF)).foreach(_.remove())
         // Remove NAMESPACE_BLOCKs and their AST children (TYPE_DECL, METHOD, etc.)
-        val visitedNodes = mutable.Set.empty[Node]
-        namespaceBlocks.foreach(dfsDelete(_, visitedNodes, EdgeTypes.AST, EdgeTypes.CONDITION))
+        val nodesToDelete = mutable.Set.empty[Node]
+        namespaceBlocks.foreach(
+          accumNodesToDelete(_, nodesToDelete, EdgeTypes.AST, EdgeTypes.CONDITION)
+        )
+        nodesToDelete.foreach(_.remove)
         // Finally remove FILE node
         f.remove()
       }

diff --git a/...s/concurrent/PlumeConcurrentCpgPass.scala → ...e/oss/passes/PlumeConcurrentCpgPass.scala b/...s/concurrent/PlumeConcurrentCpgPass.scala → ...e/oss/passes/PlumeConcurrentCpgPass.scala
@@ -1,22 +1,22 @@
-package com.github.plume.oss.passes.concurrent
+package com.github.plume.oss.passes
 
 import com.github.plume.oss.drivers.IDriver
-import com.github.plume.oss.passes.concurrent.PlumeConcurrentCpgPass.concurrentCreateApply
+import com.github.plume.oss.passes.PlumeConcurrentCpgPass.concurrentCreateApply
 import io.shiftleft.codepropertygraph.Cpg
-import io.shiftleft.codepropertygraph.generated.nodes.AstNode
-import io.shiftleft.passes.{ConcurrentWriterCpgPass, KeyPool}
+import io.shiftleft.passes.{ConcurrentWriterCpgPass, CpgPass, KeyPool}
 import io.shiftleft.utils.ExecutionContextProvider
-import org.slf4j.{Logger, MDC}
+import org.slf4j.{Logger, LoggerFactory, MDC}
 import overflowdb.BatchedUpdate.{DiffGraph, DiffGraphBuilder}
 
+import java.util.concurrent.LinkedBlockingQueue
 import scala.collection.mutable
 import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, ExecutionContext, Future}
 
-abstract class PlumeConcurrentCpgPass[T <: AstNode](cpg: Cpg, keyPool: Option[KeyPool])
+abstract class PlumeConcurrentCpgPass[T <: AnyRef](cpg: Cpg, keyPool: Option[KeyPool])
     extends ConcurrentWriterCpgPass[T](cpg) {
 
-  override def generateParts(): Array[_ <: AstNode] = Array[AstNode](null)
+  override def generateParts(): Array[_ <: AnyRef] = Array(null)
 
   def createAndApply(driver: IDriver): Unit = {
     createApplySerializeAndStore(driver) // Apply to driver
@@ -51,12 +51,14 @@ abstract class PlumeConcurrentCpgPass[T <: AstNode](cpg: Cpg, keyPool: Option[Ke
 object PlumeConcurrentCpgPass {
   private val producerQueueCapacity = 2 + 4 * Runtime.getRuntime.availableProcessors()
 
+  MDC.setContextMap(new java.util.HashMap())
+
   def concurrentCreateApply[T](
       producerQueueCapacity: Long,
       driver: IDriver,
       name: String,
       baseLogger: Logger,
-      parts: Array[_ <: AstNode],
+      parts: Array[_ <: AnyRef],
       cpg: Cpg,
       runOnPart: (DiffGraphBuilder, T) => Unit,
       keyPool: Option[KeyPool],
@@ -113,3 +115,51 @@ object PlumeConcurrentCpgPass {
     }
   }
 }
+
+object PlumeConcurrentWriter {
+  private val writerQueueCapacity = 4
+}
+class PlumeConcurrentWriter(
+    driver: IDriver,
+    cpg: Cpg,
+    baseLogger: Logger = LoggerFactory.getLogger(classOf[CpgPass]),
+    keyPool: Option[KeyPool] = None,
+    mdc: java.util.Map[String, String],
+    setDiffT: Int => Int
+) extends Runnable {
+
+  val queue: LinkedBlockingQueue[Option[DiffGraph]] =
+    new LinkedBlockingQueue[Option[DiffGraph]](PlumeConcurrentWriter.writerQueueCapacity)
+
+  @volatile var raisedException: Exception = null
+
+  override def run(): Unit = {
+    try {
+      var nDiffT = setDiffT(0)
+      MDC.setContextMap(mdc)
+      var terminate = false
+      while (!terminate) {
+        queue.take() match {
+          case None =>
+            baseLogger.debug("Shutting down WriterThread")
+            terminate = true
+          case Some(diffGraph) =>
+            val appliedDiffGraph = overflowdb.BatchedUpdate
+              .applyDiff(cpg.graph, diffGraph, keyPool.orNull, null)
+
+            nDiffT = setDiffT(
+              nDiffT + appliedDiffGraph
+                .transitiveModifications()
+            )
+            driver.bulkTx(appliedDiffGraph)
+        }
+      }
+    } catch {
+      case exception: InterruptedException => baseLogger.warn("Interrupted WriterThread", exception)
+      case exc: Exception =>
+        raisedException = exc
+        queue.clear()
+        throw exc
+    }
+  }
+}
diff --git a/...rkjoin/PlumeForkJoinParallelCpgPass.scala → ...passes/PlumeForkJoinParallelCpgPass.scala b/...rkjoin/PlumeForkJoinParallelCpgPass.scala → ...passes/PlumeForkJoinParallelCpgPass.scala
@@ -1,8 +1,8 @@
-package com.github.plume.oss.passes.forkjoin
+package com.github.plume.oss.passes
 
 import com.github.plume.oss.drivers.IDriver
-import com.github.plume.oss.passes.PlumeCpgPass
-import com.github.plume.oss.passes.forkjoin.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore
+import com.github.plume.oss.passes.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore
+import com.github.plume.oss.passes.base.PlumeCpgPass
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.passes.{ForkJoinParallelCpgPass, KeyPool}
 import org.slf4j.Logger

diff --git a/src/main/scala/com/github/plume/oss/passes/base/AstCreationPass.scala b/src/main/scala/com/github/plume/oss/passes/base/AstCreationPass.scala
@@ -0,0 +1,36 @@
+package com.github.plume.oss.passes.base
+
+import com.github.plume.oss.Jimple2Cpg
+import com.github.plume.oss.passes.{IncrementalKeyPool, PlumeConcurrentCpgPass}
+import io.joern.jimple2cpg.passes.Global
+import io.shiftleft.codepropertygraph.Cpg
+import org.slf4j.LoggerFactory
+import soot.Scene
+
+/** Creates the AST layer from the given class file and stores all types in the given global parameter.
+  */
+class AstCreationPass(
+    filenames: List[String],
+    cpg: Cpg,
+    keyPool: IncrementalKeyPool
+) extends PlumeConcurrentCpgPass[String](cpg, keyPool = Some(keyPool)) {
+
+  val global: Global = Global()
+  private val logger = LoggerFactory.getLogger(classOf[AstCreationPass])
+
+  override def generateParts(): Array[_ <: AnyRef] = filenames.toArray
+
+  override def runOnPart(builder: DiffGraphBuilder, part: String): Unit = {
+    val qualifiedClassName = Jimple2Cpg.getQualifiedClassPath(part)
+    try {
+      val sootClass = Scene.v().loadClassAndSupport(qualifiedClassName)
+      sootClass.setApplicationClass()
+      new io.joern.jimple2cpg.passes.AstCreator(part, builder, global).createAst(sootClass)
+    } catch {
+      case e: Exception =>
+        logger.warn(s"Cannot parse: $part ($qualifiedClassName)", e)
+        Iterator()
+    }
+  }
+
+}
diff --git a/...es/concurrent/PlumeContainsEdgePass.scala → ...s/passes/base/PlumeContainsEdgePass.scala b/...es/concurrent/PlumeContainsEdgePass.scala → ...s/passes/base/PlumeContainsEdgePass.scala
@@ -1,11 +1,10 @@
-package com.github.plume.oss.passes.concurrent
+package com.github.plume.oss.passes.base
 
 import com.github.plume.oss.drivers.IDriver
-import com.github.plume.oss.passes.PlumeCpgPassBase
-import com.github.plume.oss.passes.concurrent.PlumeConcurrentCpgPass.concurrentCreateApply
+import com.github.plume.oss.passes.PlumeConcurrentCpgPass.concurrentCreateApply
+import io.joern.x2cpg.passes.base.ContainsEdgePass
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.AstNode
-import io.shiftleft.semanticcpg.passes.base.ContainsEdgePass
 
 object PlumeContainsEdgePass {
   private val producerQueueCapacity = 2 + 4 * Runtime.getRuntime.availableProcessors()

diff --git a/...ithub/plume/oss/passes/PlumeCpgPass.scala → .../plume/oss/passes/base/PlumeCpgPass.scala b/...ithub/plume/oss/passes/PlumeCpgPass.scala → .../plume/oss/passes/base/PlumeCpgPass.scala
@@ -1,23 +1,16 @@
-package com.github.plume.oss.passes
+package com.github.plume.oss.passes.base
 
 import com.github.plume.oss.drivers.IDriver
-import com.github.plume.oss.passes.forkjoin.PlumeForkJoinParallelCpgPass.{
-  DiffGraphBuilder,
-  forkJoinSerializeAndStore
-}
+import com.github.plume.oss.passes.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore
 import com.github.plume.oss.util.BatchedUpdateUtil
+import io.joern.x2cpg.passes.base._
+import io.joern.x2cpg.passes.frontend._
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.PropertyNames
 import io.shiftleft.codepropertygraph.generated.nodes.{AbstractNode, Method, NewNode, StoredNode}
 import io.shiftleft.passes.DiffGraph.Change
 import io.shiftleft.passes.{DiffGraph, ForkJoinParallelCpgPass, KeyPool}
-import io.shiftleft.semanticcpg.passes.base.{
-  FileCreationPass,
-  MethodDecoratorPass,
-  NamespaceCreator,
-  TypeDeclStubCreator
-}
-import io.shiftleft.semanticcpg.passes.frontend.{MetaDataPass, TypeNodePass}
+import overflowdb.BatchedUpdate.DiffGraphBuilder
 import overflowdb.traversal.jIteratortoTraversal
 import overflowdb.{BatchedUpdate, DetachedNodeData, DetachedNodeGeneric, Node, NodeOrDetachedNode}
 

diff --git a/...b/plume/oss/passes/PlumeCpgPassBase.scala → ...me/oss/passes/base/PlumeCpgPassBase.scala b/...b/plume/oss/passes/PlumeCpgPassBase.scala → ...me/oss/passes/base/PlumeCpgPassBase.scala
@@ -1,4 +1,4 @@
-package com.github.plume.oss.passes
+package com.github.plume.oss.passes.base
 
 import com.github.plume.oss.drivers.IDriver
 

diff --git a/...ses/parallel/PlumeMethodStubCreator.scala → .../passes/base/PlumeMethodStubCreator.scala b/...ses/parallel/PlumeMethodStubCreator.scala → .../passes/base/PlumeMethodStubCreator.scala
@@ -1,13 +1,13 @@
-package com.github.plume.oss.passes.parallel
+package com.github.plume.oss.passes.base
 
 import com.github.plume.oss.drivers.IDriver
-import com.github.plume.oss.passes.forkjoin.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore
-import com.github.plume.oss.passes.{IncrementalKeyPool, PlumeCpgPassBase, PlumeSimpleCpgPass}
+import com.github.plume.oss.passes.IncrementalKeyPool
+import com.github.plume.oss.passes.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore
+import io.joern.x2cpg.passes.base.NameAndSignature
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes._
 import io.shiftleft.codepropertygraph.generated.{EdgeTypes, EvaluationStrategies, NodeTypes}
 import io.shiftleft.semanticcpg.language._
-import io.shiftleft.semanticcpg.passes.base.NameAndSignature
 import overflowdb.BatchedUpdate
 
 import scala.collection.mutable
@@ -27,7 +27,8 @@ class PlumeMethodStubCreator(
 
   private def filter(name: NameAndSignature): Boolean = {
     val methodTypeName = name.fullName.replace(s".${name.name}:${name.signature}", "")
-    !(blacklist.contains(methodTypeName) || (blacklist.nonEmpty && methodTypeName == "<empty>"))
+    !(blacklist
+      .contains(methodTypeName) || (blacklist.nonEmpty && methodTypeName.contains("<operator>")))
   }
 
   override def run(dstGraph: BatchedUpdate.DiffGraphBuilder): Unit = {

diff --git a/...e/oss/passes/PlumeDynamicCallLinker.scala → ...es/callgraph/PlumeDynamicCallLinker.scala b/...e/oss/passes/PlumeDynamicCallLinker.scala → ...es/callgraph/PlumeDynamicCallLinker.scala
@@ -1,4 +1,4 @@
-package com.github.plume.oss.passes
+package com.github.plume.oss.passes.callgraph
 
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method, TypeDecl}