From 7c15c6ca8c44815422b4d2a793a0bf2389486d3f Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Wed, 9 Mar 2022 21:45:43 +0200 Subject: [PATCH] Handing functionality off to jimple2cpg that includes many bug fixes (#241) * Included jimple2cpg functionality and updated tests * Styling * Fixed driver fixture import * Fixed diff detection on operators --- CHANGELOG.md | 18 + build.sbt | 13 +- .../com/github/plume/oss/Jimple2Cpg.scala | 21 +- .../plume/oss/drivers/OverflowDbDriver.scala | 26 +- .../PlumeConcurrentCpgPass.scala | 66 +- .../PlumeForkJoinParallelCpgPass.scala | 6 +- .../oss/passes/base/AstCreationPass.scala | 36 + .../PlumeContainsEdgePass.scala | 7 +- .../oss/passes/{ => base}/PlumeCpgPass.scala | 17 +- .../passes/{ => base}/PlumeCpgPassBase.scala | 2 +- .../PlumeMethodStubCreator.scala | 11 +- .../PlumeDynamicCallLinker.scala | 2 +- .../concurrent/PlumeConcurrentWriter.scala | 57 -- .../PlumeCfgCreationPass.scala | 8 +- .../cfgdominator}/PlumeCfgDominatorPass.scala | 13 +- .../codepencegraph}/PlumeCdgPass.scala | 13 +- .../{ => incremental}/PlumeDiffPass.scala | 2 +- .../PlumeHashPass.scala | 3 +- .../parallel/PlumeAstCreationPass.scala | 43 - .../oss/passes/parallel/PlumeAstCreator.scala | 939 ------------------ .../parallel/PlumeParallelCpgPass.scala | 120 --- .../passes/parallel/PlumeParallelWriter.scala | 44 - .../PlumeReachingDefPass.scala | 6 +- .../com/github/plume/oss/DiffTests.scala | 4 +- .../github/plume/oss/querying/CfgTests.scala | 10 +- .../querying/ConstructorInvocationTests.scala | 59 +- .../plume/oss/querying/LocalTests.scala | 4 +- .../plume/oss/querying/MetaDataTests.scala | 4 +- .../oss/querying/MethodParameterTests.scala | 2 +- .../oss/querying/StaticCallGraphTests.scala | 19 +- .../oss/testfixtures/Jimple2CpgFixture.scala | 6 +- .../oss/testfixtures/PlumeDriverFixture.scala | 6 +- 32 files changed, 224 insertions(+), 1363 deletions(-) rename src/main/scala/com/github/plume/oss/passes/{concurrent => }/PlumeConcurrentCpgPass.scala (64%) rename src/main/scala/com/github/plume/oss/passes/{forkjoin => }/PlumeForkJoinParallelCpgPass.scala (94%) create mode 100644 src/main/scala/com/github/plume/oss/passes/base/AstCreationPass.scala rename src/main/scala/com/github/plume/oss/passes/{concurrent => base}/PlumeContainsEdgePass.scala (79%) rename src/main/scala/com/github/plume/oss/passes/{ => base}/PlumeCpgPass.scala (95%) rename src/main/scala/com/github/plume/oss/passes/{ => base}/PlumeCpgPassBase.scala (74%) rename src/main/scala/com/github/plume/oss/passes/{parallel => base}/PlumeMethodStubCreator.scala (91%) rename src/main/scala/com/github/plume/oss/passes/{ => callgraph}/PlumeDynamicCallLinker.scala (99%) delete mode 100644 src/main/scala/com/github/plume/oss/passes/concurrent/PlumeConcurrentWriter.scala rename src/main/scala/com/github/plume/oss/passes/{concurrent => controlflow}/PlumeCfgCreationPass.scala (79%) rename src/main/scala/com/github/plume/oss/passes/{forkjoin => controlflow/cfgdominator}/PlumeCfgDominatorPass.scala (63%) rename src/main/scala/com/github/plume/oss/passes/{forkjoin => controlflow/codepencegraph}/PlumeCdgPass.scala (63%) rename src/main/scala/com/github/plume/oss/passes/{ => incremental}/PlumeDiffPass.scala (97%) rename src/main/scala/com/github/plume/oss/passes/{concurrent => incremental}/PlumeHashPass.scala (93%) delete mode 100644 src/main/scala/com/github/plume/oss/passes/parallel/PlumeAstCreationPass.scala delete mode 100644 src/main/scala/com/github/plume/oss/passes/parallel/PlumeAstCreator.scala delete mode 100644 src/main/scala/com/github/plume/oss/passes/parallel/PlumeParallelCpgPass.scala delete mode 100644 src/main/scala/com/github/plume/oss/passes/parallel/PlumeParallelWriter.scala rename src/main/scala/com/github/plume/oss/passes/{parallel => reachingdef}/PlumeReachingDefPass.scala (85%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ad4f7c6..5a599838 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +### Added + +- Object data flow tests that explores fields and inter-procedural calls (both static + virtual). +- Data flow tests followed from `JavaSrc2Cpg`, specifically `OperatorTests` + +### Changed + +- Importing `jimple2cpg` instead of duplicating work here. +- Removed parsing Jimple `IdentityStmt` since they end up duplicating parameters as locals. +- Swapped out the deprecated `ParallelCpgPass` for `ConcurrentWriterCpgPass` + +### Fixed + +- Virtual calls now get passed the correct `this` object as the object the method is actually invoked with. +- Virtual call `code` properties now include the object invoking the method. +- Instance where `Local` nodes were duplicated for each time they were referenced +- Issue where `NewCall(Operators.assignment)` did not have `methodFullName = Operators.assignment` and thus messing up data flow paths + ## [1.0.17] - 2022-03-04 ### Added diff --git a/build.sbt b/build.sbt index 6c24fc16..35663646 100644 --- a/build.sbt +++ b/build.sbt @@ -15,18 +15,18 @@ inThisBuild( ) val cpgVersion = "1.3.509" -val joernVersion = "1.1.590" +val joernVersion = "1.1.606" val sootVersion = "4.2.1" val tinkerGraphVersion = "3.4.8" val neo4jVersion = "4.4.3" val tigerGraphVersion = "3.1.0" -val sttpVersion = "3.4.1" -val jacksonVersion = "2.13.1" +val sttpVersion = "3.4.2" +val jacksonVersion = "2.13.2" val scalajHttpVersion = "2.4.2" val lz4Version = "1.8.0" val slf4jVersion = "1.7.36" val logbackVersion = "1.2.10" -val scalatestVersion = "3.2.9" +val scalatestVersion = "3.2.11" val circeVersion = "0.14.1" lazy val scalatest = "org.scalatest" %% "scalatest" % scalatestVersion @@ -43,7 +43,8 @@ libraryDependencies ++= Seq( "io.shiftleft" %% "semanticcpg" % cpgVersion, "io.joern" %% "dataflowengineoss" % joernVersion, "io.joern" %% "x2cpg" % joernVersion, - "io.shiftleft" %% "semanticcpg" % cpgVersion % Test classifier "tests", + "io.joern" %% "jimple2cpg" % joernVersion, + "io.joern" %% "x2cpg" % joernVersion % Test classifier "tests", "org.soot-oss" % "soot" % sootVersion, "org.apache.tinkerpop" % "tinkergraph-gremlin" % tinkerGraphVersion, "org.apache.tinkerpop" % "gremlin-driver" % tinkerGraphVersion, @@ -56,7 +57,7 @@ libraryDependencies ++= Seq( "org.scalaj" % "scalaj-http_2.13" % scalajHttpVersion, "org.lz4" % "lz4-java" % lz4Version, "org.slf4j" % "slf4j-api" % slf4jVersion, - "org.slf4j" % "slf4j-simple" % slf4jVersion % Runtime, + "org.slf4j" % "slf4j-simple" % slf4jVersion % Runtime, "org.scala-lang" % "scala-reflect" % scalaVersion.value, "ch.qos.logback" % "logback-classic" % logbackVersion % Test, "org.scalatest" %% "scalatest" % scalatestVersion % Test diff --git a/src/main/scala/com/github/plume/oss/Jimple2Cpg.scala b/src/main/scala/com/github/plume/oss/Jimple2Cpg.scala index eb0bf0b3..850748cc 100644 --- a/src/main/scala/com/github/plume/oss/Jimple2Cpg.scala +++ b/src/main/scala/com/github/plume/oss/Jimple2Cpg.scala @@ -2,13 +2,23 @@ package com.github.plume.oss import com.github.plume.oss.drivers.{IDriver, OverflowDbDriver} import com.github.plume.oss.passes._ -import com.github.plume.oss.passes.concurrent.{ - PlumeCfgCreationPass, +import com.github.plume.oss.passes.base.{ + AstCreationPass, PlumeContainsEdgePass, - PlumeHashPass + PlumeCpgPassBase, + PlumeFileCreationPass, + PlumeMetaDataPass, + PlumeMethodDecoratorPass, + PlumeMethodStubCreator, + PlumeNamespaceCreator, + PlumeTypeDeclStubCreator, + PlumeTypeNodePass } -import com.github.plume.oss.passes.forkjoin.{PlumeCdgPass, PlumeCfgDominatorPass} -import com.github.plume.oss.passes.parallel._ +import com.github.plume.oss.passes.incremental.{PlumeDiffPass, PlumeHashPass} +import com.github.plume.oss.passes.controlflow.PlumeCfgCreationPass +import com.github.plume.oss.passes.controlflow.cfgdominator.PlumeCfgDominatorPass +import com.github.plume.oss.passes.controlflow.codepencegraph.PlumeCdgPass +import com.github.plume.oss.passes.reachingdef._ import com.github.plume.oss.util.ProgramHandlingUtil import com.github.plume.oss.util.ProgramHandlingUtil.{extractSourceFilesFromArchive, moveClassFiles} import io.joern.x2cpg.SourceFiles @@ -120,7 +130,6 @@ class Jimple2Cpg { return cpg } else { logger.info(s"Processing ${codeToProcess.size} new or changed program files") - logger.debug(s"Files to process are: $sourceFileNames") } // After the diff pass any changed types are removed. Remaining types should be black listed to avoid duplicates diff --git a/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala b/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala index 1ef77263..0c2007d5 100644 --- a/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala +++ b/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala @@ -8,7 +8,7 @@ import com.github.plume.oss.domain.{ deserializeResultTable } import com.github.plume.oss.drivers.OverflowDbDriver.newOverflowGraph -import com.github.plume.oss.passes.PlumeDynamicCallLinker +import com.github.plume.oss.passes.callgraph.PlumeDynamicCallLinker import com.github.plume.oss.util.BatchedUpdateUtil._ import io.joern.dataflowengineoss.language.toExtendedCfgNode import io.joern.dataflowengineoss.queryengine._ @@ -191,11 +191,12 @@ final case class OverflowDbDriver( } override def bulkTx(dg: AppliedDiff): Unit = { - dg.getDiffGraph.iterator.forEachRemaining { - case node: DetachedNodeData => - val id = idFromNodeData(node) - val newNode = cpg.graph.addNode(id, node.label) - propertiesFromNodeData(node).foreach { case (k, v) => newNode.setProperty(k, v) } + dg.getDiffGraph.iterator.collect { case x: DetachedNodeData => x }.foreach { node => + val id = idFromNodeData(node) + val newNode = cpg.graph.addNode(id, node.label) + propertiesFromNodeData(node).foreach { case (k, v) => newNode.setProperty(k, v) } + } + dg.getDiffGraph.iterator.filterNot(_.isInstanceOf[DetachedNodeData]).foreach { case c: BatchedUpdate.CreateEdge => val srcId = idFromNodeData(c.src) val dstId = idFromNodeData(c.dst) @@ -211,16 +212,16 @@ final case class OverflowDbDriver( } } - private def dfsDelete( + private def accumNodesToDelete( n: Node, visitedNodes: mutable.Set[Node], edgeToFollow: String* ): Unit = { if (!visitedNodes.contains(n)) { visitedNodes.add(n) - n.out(edgeToFollow: _*).forEachRemaining(dfsDelete(_, visitedNodes, edgeToFollow: _*)) + n.out(edgeToFollow: _*) + .forEachRemaining(accumNodesToDelete(_, visitedNodes, edgeToFollow: _*)) } - n.remove() } override def removeSourceFiles(filenames: String*): Unit = { @@ -237,8 +238,11 @@ final case class OverflowDbDriver( // Remove TYPE nodes typeDecls.flatMap(_.in(EdgeTypes.REF)).foreach(_.remove()) // Remove NAMESPACE_BLOCKs and their AST children (TYPE_DECL, METHOD, etc.) - val visitedNodes = mutable.Set.empty[Node] - namespaceBlocks.foreach(dfsDelete(_, visitedNodes, EdgeTypes.AST, EdgeTypes.CONDITION)) + val nodesToDelete = mutable.Set.empty[Node] + namespaceBlocks.foreach( + accumNodesToDelete(_, nodesToDelete, EdgeTypes.AST, EdgeTypes.CONDITION) + ) + nodesToDelete.foreach(_.remove) // Finally remove FILE node f.remove() } diff --git a/src/main/scala/com/github/plume/oss/passes/concurrent/PlumeConcurrentCpgPass.scala b/src/main/scala/com/github/plume/oss/passes/PlumeConcurrentCpgPass.scala similarity index 64% rename from src/main/scala/com/github/plume/oss/passes/concurrent/PlumeConcurrentCpgPass.scala rename to src/main/scala/com/github/plume/oss/passes/PlumeConcurrentCpgPass.scala index 5743cde9..43e17575 100644 --- a/src/main/scala/com/github/plume/oss/passes/concurrent/PlumeConcurrentCpgPass.scala +++ b/src/main/scala/com/github/plume/oss/passes/PlumeConcurrentCpgPass.scala @@ -1,22 +1,22 @@ -package com.github.plume.oss.passes.concurrent +package com.github.plume.oss.passes import com.github.plume.oss.drivers.IDriver -import com.github.plume.oss.passes.concurrent.PlumeConcurrentCpgPass.concurrentCreateApply +import com.github.plume.oss.passes.PlumeConcurrentCpgPass.concurrentCreateApply import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.codepropertygraph.generated.nodes.AstNode -import io.shiftleft.passes.{ConcurrentWriterCpgPass, KeyPool} +import io.shiftleft.passes.{ConcurrentWriterCpgPass, CpgPass, KeyPool} import io.shiftleft.utils.ExecutionContextProvider -import org.slf4j.{Logger, MDC} +import org.slf4j.{Logger, LoggerFactory, MDC} import overflowdb.BatchedUpdate.{DiffGraph, DiffGraphBuilder} +import java.util.concurrent.LinkedBlockingQueue import scala.collection.mutable import scala.concurrent.duration.Duration import scala.concurrent.{Await, ExecutionContext, Future} -abstract class PlumeConcurrentCpgPass[T <: AstNode](cpg: Cpg, keyPool: Option[KeyPool]) +abstract class PlumeConcurrentCpgPass[T <: AnyRef](cpg: Cpg, keyPool: Option[KeyPool]) extends ConcurrentWriterCpgPass[T](cpg) { - override def generateParts(): Array[_ <: AstNode] = Array[AstNode](null) + override def generateParts(): Array[_ <: AnyRef] = Array(null) def createAndApply(driver: IDriver): Unit = { createApplySerializeAndStore(driver) // Apply to driver @@ -51,12 +51,14 @@ abstract class PlumeConcurrentCpgPass[T <: AstNode](cpg: Cpg, keyPool: Option[Ke object PlumeConcurrentCpgPass { private val producerQueueCapacity = 2 + 4 * Runtime.getRuntime.availableProcessors() + MDC.setContextMap(new java.util.HashMap()) + def concurrentCreateApply[T]( producerQueueCapacity: Long, driver: IDriver, name: String, baseLogger: Logger, - parts: Array[_ <: AstNode], + parts: Array[_ <: AnyRef], cpg: Cpg, runOnPart: (DiffGraphBuilder, T) => Unit, keyPool: Option[KeyPool], @@ -113,3 +115,51 @@ object PlumeConcurrentCpgPass { } } } + +object PlumeConcurrentWriter { + private val writerQueueCapacity = 4 +} +class PlumeConcurrentWriter( + driver: IDriver, + cpg: Cpg, + baseLogger: Logger = LoggerFactory.getLogger(classOf[CpgPass]), + keyPool: Option[KeyPool] = None, + mdc: java.util.Map[String, String], + setDiffT: Int => Int +) extends Runnable { + + val queue: LinkedBlockingQueue[Option[DiffGraph]] = + new LinkedBlockingQueue[Option[DiffGraph]](PlumeConcurrentWriter.writerQueueCapacity) + + @volatile var raisedException: Exception = null + + override def run(): Unit = { + try { + var nDiffT = setDiffT(0) + MDC.setContextMap(mdc) + var terminate = false + while (!terminate) { + queue.take() match { + case None => + baseLogger.debug("Shutting down WriterThread") + terminate = true + case Some(diffGraph) => + val appliedDiffGraph = overflowdb.BatchedUpdate + .applyDiff(cpg.graph, diffGraph, keyPool.orNull, null) + + nDiffT = setDiffT( + nDiffT + appliedDiffGraph + .transitiveModifications() + ) + driver.bulkTx(appliedDiffGraph) + } + } + } catch { + case exception: InterruptedException => baseLogger.warn("Interrupted WriterThread", exception) + case exc: Exception => + raisedException = exc + queue.clear() + throw exc + } + } +} diff --git a/src/main/scala/com/github/plume/oss/passes/forkjoin/PlumeForkJoinParallelCpgPass.scala b/src/main/scala/com/github/plume/oss/passes/PlumeForkJoinParallelCpgPass.scala similarity index 94% rename from src/main/scala/com/github/plume/oss/passes/forkjoin/PlumeForkJoinParallelCpgPass.scala rename to src/main/scala/com/github/plume/oss/passes/PlumeForkJoinParallelCpgPass.scala index bdefc8ed..1a64eb0f 100644 --- a/src/main/scala/com/github/plume/oss/passes/forkjoin/PlumeForkJoinParallelCpgPass.scala +++ b/src/main/scala/com/github/plume/oss/passes/PlumeForkJoinParallelCpgPass.scala @@ -1,8 +1,8 @@ -package com.github.plume.oss.passes.forkjoin +package com.github.plume.oss.passes import com.github.plume.oss.drivers.IDriver -import com.github.plume.oss.passes.PlumeCpgPass -import com.github.plume.oss.passes.forkjoin.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore +import com.github.plume.oss.passes.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore +import com.github.plume.oss.passes.base.PlumeCpgPass import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.passes.{ForkJoinParallelCpgPass, KeyPool} import org.slf4j.Logger diff --git a/src/main/scala/com/github/plume/oss/passes/base/AstCreationPass.scala b/src/main/scala/com/github/plume/oss/passes/base/AstCreationPass.scala new file mode 100644 index 00000000..7c3c4919 --- /dev/null +++ b/src/main/scala/com/github/plume/oss/passes/base/AstCreationPass.scala @@ -0,0 +1,36 @@ +package com.github.plume.oss.passes.base + +import com.github.plume.oss.Jimple2Cpg +import com.github.plume.oss.passes.{IncrementalKeyPool, PlumeConcurrentCpgPass} +import io.joern.jimple2cpg.passes.Global +import io.shiftleft.codepropertygraph.Cpg +import org.slf4j.LoggerFactory +import soot.Scene + +/** Creates the AST layer from the given class file and stores all types in the given global parameter. + */ +class AstCreationPass( + filenames: List[String], + cpg: Cpg, + keyPool: IncrementalKeyPool +) extends PlumeConcurrentCpgPass[String](cpg, keyPool = Some(keyPool)) { + + val global: Global = Global() + private val logger = LoggerFactory.getLogger(classOf[AstCreationPass]) + + override def generateParts(): Array[_ <: AnyRef] = filenames.toArray + + override def runOnPart(builder: DiffGraphBuilder, part: String): Unit = { + val qualifiedClassName = Jimple2Cpg.getQualifiedClassPath(part) + try { + val sootClass = Scene.v().loadClassAndSupport(qualifiedClassName) + sootClass.setApplicationClass() + new io.joern.jimple2cpg.passes.AstCreator(part, builder, global).createAst(sootClass) + } catch { + case e: Exception => + logger.warn(s"Cannot parse: $part ($qualifiedClassName)", e) + Iterator() + } + } + +} diff --git a/src/main/scala/com/github/plume/oss/passes/concurrent/PlumeContainsEdgePass.scala b/src/main/scala/com/github/plume/oss/passes/base/PlumeContainsEdgePass.scala similarity index 79% rename from src/main/scala/com/github/plume/oss/passes/concurrent/PlumeContainsEdgePass.scala rename to src/main/scala/com/github/plume/oss/passes/base/PlumeContainsEdgePass.scala index 15217296..f5da53d7 100644 --- a/src/main/scala/com/github/plume/oss/passes/concurrent/PlumeContainsEdgePass.scala +++ b/src/main/scala/com/github/plume/oss/passes/base/PlumeContainsEdgePass.scala @@ -1,11 +1,10 @@ -package com.github.plume.oss.passes.concurrent +package com.github.plume.oss.passes.base import com.github.plume.oss.drivers.IDriver -import com.github.plume.oss.passes.PlumeCpgPassBase -import com.github.plume.oss.passes.concurrent.PlumeConcurrentCpgPass.concurrentCreateApply +import com.github.plume.oss.passes.PlumeConcurrentCpgPass.concurrentCreateApply +import io.joern.x2cpg.passes.base.ContainsEdgePass import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.AstNode -import io.shiftleft.semanticcpg.passes.base.ContainsEdgePass object PlumeContainsEdgePass { private val producerQueueCapacity = 2 + 4 * Runtime.getRuntime.availableProcessors() diff --git a/src/main/scala/com/github/plume/oss/passes/PlumeCpgPass.scala b/src/main/scala/com/github/plume/oss/passes/base/PlumeCpgPass.scala similarity index 95% rename from src/main/scala/com/github/plume/oss/passes/PlumeCpgPass.scala rename to src/main/scala/com/github/plume/oss/passes/base/PlumeCpgPass.scala index b2e52955..7353c438 100644 --- a/src/main/scala/com/github/plume/oss/passes/PlumeCpgPass.scala +++ b/src/main/scala/com/github/plume/oss/passes/base/PlumeCpgPass.scala @@ -1,23 +1,16 @@ -package com.github.plume.oss.passes +package com.github.plume.oss.passes.base import com.github.plume.oss.drivers.IDriver -import com.github.plume.oss.passes.forkjoin.PlumeForkJoinParallelCpgPass.{ - DiffGraphBuilder, - forkJoinSerializeAndStore -} +import com.github.plume.oss.passes.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore import com.github.plume.oss.util.BatchedUpdateUtil +import io.joern.x2cpg.passes.base._ +import io.joern.x2cpg.passes.frontend._ import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.PropertyNames import io.shiftleft.codepropertygraph.generated.nodes.{AbstractNode, Method, NewNode, StoredNode} import io.shiftleft.passes.DiffGraph.Change import io.shiftleft.passes.{DiffGraph, ForkJoinParallelCpgPass, KeyPool} -import io.shiftleft.semanticcpg.passes.base.{ - FileCreationPass, - MethodDecoratorPass, - NamespaceCreator, - TypeDeclStubCreator -} -import io.shiftleft.semanticcpg.passes.frontend.{MetaDataPass, TypeNodePass} +import overflowdb.BatchedUpdate.DiffGraphBuilder import overflowdb.traversal.jIteratortoTraversal import overflowdb.{BatchedUpdate, DetachedNodeData, DetachedNodeGeneric, Node, NodeOrDetachedNode} diff --git a/src/main/scala/com/github/plume/oss/passes/PlumeCpgPassBase.scala b/src/main/scala/com/github/plume/oss/passes/base/PlumeCpgPassBase.scala similarity index 74% rename from src/main/scala/com/github/plume/oss/passes/PlumeCpgPassBase.scala rename to src/main/scala/com/github/plume/oss/passes/base/PlumeCpgPassBase.scala index f340fe67..bd877c43 100644 --- a/src/main/scala/com/github/plume/oss/passes/PlumeCpgPassBase.scala +++ b/src/main/scala/com/github/plume/oss/passes/base/PlumeCpgPassBase.scala @@ -1,4 +1,4 @@ -package com.github.plume.oss.passes +package com.github.plume.oss.passes.base import com.github.plume.oss.drivers.IDriver diff --git a/src/main/scala/com/github/plume/oss/passes/parallel/PlumeMethodStubCreator.scala b/src/main/scala/com/github/plume/oss/passes/base/PlumeMethodStubCreator.scala similarity index 91% rename from src/main/scala/com/github/plume/oss/passes/parallel/PlumeMethodStubCreator.scala rename to src/main/scala/com/github/plume/oss/passes/base/PlumeMethodStubCreator.scala index d6ea4593..5ac3c51b 100644 --- a/src/main/scala/com/github/plume/oss/passes/parallel/PlumeMethodStubCreator.scala +++ b/src/main/scala/com/github/plume/oss/passes/base/PlumeMethodStubCreator.scala @@ -1,13 +1,13 @@ -package com.github.plume.oss.passes.parallel +package com.github.plume.oss.passes.base import com.github.plume.oss.drivers.IDriver -import com.github.plume.oss.passes.forkjoin.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore -import com.github.plume.oss.passes.{IncrementalKeyPool, PlumeCpgPassBase, PlumeSimpleCpgPass} +import com.github.plume.oss.passes.IncrementalKeyPool +import com.github.plume.oss.passes.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore +import io.joern.x2cpg.passes.base.NameAndSignature import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes._ import io.shiftleft.codepropertygraph.generated.{EdgeTypes, EvaluationStrategies, NodeTypes} import io.shiftleft.semanticcpg.language._ -import io.shiftleft.semanticcpg.passes.base.NameAndSignature import overflowdb.BatchedUpdate import scala.collection.mutable @@ -27,7 +27,8 @@ class PlumeMethodStubCreator( private def filter(name: NameAndSignature): Boolean = { val methodTypeName = name.fullName.replace(s".${name.name}:${name.signature}", "") - !(blacklist.contains(methodTypeName) || (blacklist.nonEmpty && methodTypeName == "")) + !(blacklist + .contains(methodTypeName) || (blacklist.nonEmpty && methodTypeName.contains(""))) } override def run(dstGraph: BatchedUpdate.DiffGraphBuilder): Unit = { diff --git a/src/main/scala/com/github/plume/oss/passes/PlumeDynamicCallLinker.scala b/src/main/scala/com/github/plume/oss/passes/callgraph/PlumeDynamicCallLinker.scala similarity index 99% rename from src/main/scala/com/github/plume/oss/passes/PlumeDynamicCallLinker.scala rename to src/main/scala/com/github/plume/oss/passes/callgraph/PlumeDynamicCallLinker.scala index 9f431824..dc3fcc7b 100644 --- a/src/main/scala/com/github/plume/oss/passes/PlumeDynamicCallLinker.scala +++ b/src/main/scala/com/github/plume/oss/passes/callgraph/PlumeDynamicCallLinker.scala @@ -1,4 +1,4 @@ -package com.github.plume.oss.passes +package com.github.plume.oss.passes.callgraph import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method, TypeDecl} diff --git a/src/main/scala/com/github/plume/oss/passes/concurrent/PlumeConcurrentWriter.scala b/src/main/scala/com/github/plume/oss/passes/concurrent/PlumeConcurrentWriter.scala deleted file mode 100644 index afbc5a5e..00000000 --- a/src/main/scala/com/github/plume/oss/passes/concurrent/PlumeConcurrentWriter.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.github.plume.oss.passes.concurrent - -import com.github.plume.oss.drivers.IDriver -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.passes.{CpgPass, KeyPool} -import org.slf4j.{Logger, LoggerFactory, MDC} -import overflowdb.BatchedUpdate.DiffGraph - -import java.util.concurrent.LinkedBlockingQueue - -object PlumeConcurrentWriter { - private val writerQueueCapacity = 4 -} -class PlumeConcurrentWriter( - driver: IDriver, - cpg: Cpg, - baseLogger: Logger = LoggerFactory.getLogger(classOf[CpgPass]), - keyPool: Option[KeyPool] = None, - mdc: java.util.Map[String, String], - setDiffT: Int => Int -) extends Runnable { - - val queue: LinkedBlockingQueue[Option[DiffGraph]] = - new LinkedBlockingQueue[Option[DiffGraph]](PlumeConcurrentWriter.writerQueueCapacity) - - @volatile var raisedException: Exception = null - - override def run(): Unit = { - try { - var nDiffT = setDiffT(0) - MDC.setContextMap(mdc) - var terminate = false - while (!terminate) { - queue.take() match { - case None => - baseLogger.debug("Shutting down WriterThread") - terminate = true - case Some(diffGraph) => - val appliedDiffGraph = overflowdb.BatchedUpdate - .applyDiff(cpg.graph, diffGraph, keyPool.orNull, null) - - nDiffT = setDiffT( - nDiffT + appliedDiffGraph - .transitiveModifications() - ) - driver.bulkTx(appliedDiffGraph) - } - } - } catch { - case exception: InterruptedException => baseLogger.warn("Interrupted WriterThread", exception) - case exc: Exception => - raisedException = exc - queue.clear() - throw exc - } - } -} diff --git a/src/main/scala/com/github/plume/oss/passes/concurrent/PlumeCfgCreationPass.scala b/src/main/scala/com/github/plume/oss/passes/controlflow/PlumeCfgCreationPass.scala similarity index 79% rename from src/main/scala/com/github/plume/oss/passes/concurrent/PlumeCfgCreationPass.scala rename to src/main/scala/com/github/plume/oss/passes/controlflow/PlumeCfgCreationPass.scala index e02f3750..0dd360a3 100644 --- a/src/main/scala/com/github/plume/oss/passes/concurrent/PlumeCfgCreationPass.scala +++ b/src/main/scala/com/github/plume/oss/passes/controlflow/PlumeCfgCreationPass.scala @@ -1,11 +1,11 @@ -package com.github.plume.oss.passes.concurrent +package com.github.plume.oss.passes.controlflow import com.github.plume.oss.drivers.IDriver -import com.github.plume.oss.passes.PlumeCpgPassBase -import com.github.plume.oss.passes.concurrent.PlumeConcurrentCpgPass.concurrentCreateApply +import com.github.plume.oss.passes.PlumeConcurrentCpgPass.concurrentCreateApply +import com.github.plume.oss.passes.base.PlumeCpgPassBase +import io.joern.x2cpg.passes.controlflow.CfgCreationPass import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Method -import io.shiftleft.semanticcpg.passes.controlflow.CfgCreationPass object PlumeCfgCreationPass { private val producerQueueCapacity = 2 + 4 * Runtime.getRuntime.availableProcessors() diff --git a/src/main/scala/com/github/plume/oss/passes/forkjoin/PlumeCfgDominatorPass.scala b/src/main/scala/com/github/plume/oss/passes/controlflow/cfgdominator/PlumeCfgDominatorPass.scala similarity index 63% rename from src/main/scala/com/github/plume/oss/passes/forkjoin/PlumeCfgDominatorPass.scala rename to src/main/scala/com/github/plume/oss/passes/controlflow/cfgdominator/PlumeCfgDominatorPass.scala index 4dc52652..57f0edc0 100644 --- a/src/main/scala/com/github/plume/oss/passes/forkjoin/PlumeCfgDominatorPass.scala +++ b/src/main/scala/com/github/plume/oss/passes/controlflow/cfgdominator/PlumeCfgDominatorPass.scala @@ -1,17 +1,12 @@ -package com.github.plume.oss.passes.forkjoin +package com.github.plume.oss.passes.controlflow.cfgdominator import com.github.plume.oss.drivers.IDriver -import com.github.plume.oss.passes.PlumeCpgPassBase -import com.github.plume.oss.passes.forkjoin.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore -import com.github.plume.oss.passes.parallel.PlumeParallelCpgPass.{ - parallelEnqueue, - parallelWithWriter -} -import com.github.plume.oss.passes.parallel.PlumeParallelWriter +import com.github.plume.oss.passes.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore +import com.github.plume.oss.passes.base.PlumeCpgPassBase +import io.joern.x2cpg.passes.controlflow.cfgdominator.CfgDominatorPass import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Method import io.shiftleft.passes.KeyPool -import io.shiftleft.semanticcpg.passes.controlflow.cfgdominator.CfgDominatorPass class PlumeCfgDominatorPass(cpg: Cpg, keyPool: Option[KeyPool] = None) extends CfgDominatorPass(cpg) diff --git a/src/main/scala/com/github/plume/oss/passes/forkjoin/PlumeCdgPass.scala b/src/main/scala/com/github/plume/oss/passes/controlflow/codepencegraph/PlumeCdgPass.scala similarity index 63% rename from src/main/scala/com/github/plume/oss/passes/forkjoin/PlumeCdgPass.scala rename to src/main/scala/com/github/plume/oss/passes/controlflow/codepencegraph/PlumeCdgPass.scala index 5eb6cd8d..6c7a67cd 100644 --- a/src/main/scala/com/github/plume/oss/passes/forkjoin/PlumeCdgPass.scala +++ b/src/main/scala/com/github/plume/oss/passes/controlflow/codepencegraph/PlumeCdgPass.scala @@ -1,17 +1,12 @@ -package com.github.plume.oss.passes.forkjoin +package com.github.plume.oss.passes.controlflow.codepencegraph import com.github.plume.oss.drivers.IDriver -import com.github.plume.oss.passes.PlumeCpgPassBase -import com.github.plume.oss.passes.forkjoin.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore -import com.github.plume.oss.passes.parallel.PlumeParallelCpgPass.{ - parallelEnqueue, - parallelWithWriter -} -import com.github.plume.oss.passes.parallel.PlumeParallelWriter +import com.github.plume.oss.passes.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore +import com.github.plume.oss.passes.base.PlumeCpgPassBase +import io.joern.x2cpg.passes.controlflow.codepencegraph.CdgPass import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Method import io.shiftleft.passes.KeyPool -import io.shiftleft.semanticcpg.passes.controlflow.codepencegraph.CdgPass class PlumeCdgPass(cpg: Cpg, keyPool: Option[KeyPool] = None) extends CdgPass(cpg) diff --git a/src/main/scala/com/github/plume/oss/passes/PlumeDiffPass.scala b/src/main/scala/com/github/plume/oss/passes/incremental/PlumeDiffPass.scala similarity index 97% rename from src/main/scala/com/github/plume/oss/passes/PlumeDiffPass.scala rename to src/main/scala/com/github/plume/oss/passes/incremental/PlumeDiffPass.scala index 37ebb0bf..ca2614af 100644 --- a/src/main/scala/com/github/plume/oss/passes/PlumeDiffPass.scala +++ b/src/main/scala/com/github/plume/oss/passes/incremental/PlumeDiffPass.scala @@ -1,4 +1,4 @@ -package com.github.plume.oss.passes +package com.github.plume.oss.passes.incremental import com.github.plume.oss.drivers.IDriver import com.github.plume.oss.util.HashUtil diff --git a/src/main/scala/com/github/plume/oss/passes/concurrent/PlumeHashPass.scala b/src/main/scala/com/github/plume/oss/passes/incremental/PlumeHashPass.scala similarity index 93% rename from src/main/scala/com/github/plume/oss/passes/concurrent/PlumeHashPass.scala rename to src/main/scala/com/github/plume/oss/passes/incremental/PlumeHashPass.scala index 76209026..b8d9078b 100644 --- a/src/main/scala/com/github/plume/oss/passes/concurrent/PlumeHashPass.scala +++ b/src/main/scala/com/github/plume/oss/passes/incremental/PlumeHashPass.scala @@ -1,5 +1,6 @@ -package com.github.plume.oss.passes.concurrent +package com.github.plume.oss.passes.incremental +import com.github.plume.oss.passes.PlumeConcurrentCpgPass import com.github.plume.oss.util.HashUtil import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.PropertyNames diff --git a/src/main/scala/com/github/plume/oss/passes/parallel/PlumeAstCreationPass.scala b/src/main/scala/com/github/plume/oss/passes/parallel/PlumeAstCreationPass.scala deleted file mode 100644 index 5cb5df6e..00000000 --- a/src/main/scala/com/github/plume/oss/passes/parallel/PlumeAstCreationPass.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.github.plume.oss.passes.parallel - -import com.github.plume.oss.Jimple2Cpg -import com.github.plume.oss.passes.IncrementalKeyPool -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.passes.DiffGraph -import org.slf4j.LoggerFactory -import soot.Scene - -import java.nio.file.Paths -import java.util.concurrent.ConcurrentSkipListSet - -final case class Global( - usedTypes: ConcurrentSkipListSet[String] = new ConcurrentSkipListSet[String]() -) - -/** Creates the AST layer from the given class file and stores all types in the given global parameter. - */ -class AstCreationPass( - filenames: List[String], - cpg: Cpg, - keyPool: IncrementalKeyPool -) extends PlumeParallelCpgPass[String](cpg, keyPools = Some(keyPool.split(filenames.size))) { - - val global: Global = Global() - private val logger = LoggerFactory.getLogger(classOf[AstCreationPass]) - - override def partIterator: Iterator[String] = filenames.iterator - - override def runOnPart(filename: String): Iterator[DiffGraph] = { - val qualifiedClassName = Jimple2Cpg.getQualifiedClassPath(filename) - try { - val sootClass = Scene.v().loadClassAndSupport(qualifiedClassName) - sootClass.setApplicationClass() - new PlumeAstCreator(filename, global).createAst(sootClass) - } catch { - case e: Exception => - logger.warn(s"Cannot parse: $filename ($qualifiedClassName)", e) - Iterator() - } - } - -} diff --git a/src/main/scala/com/github/plume/oss/passes/parallel/PlumeAstCreator.scala b/src/main/scala/com/github/plume/oss/passes/parallel/PlumeAstCreator.scala deleted file mode 100644 index a5782760..00000000 --- a/src/main/scala/com/github/plume/oss/passes/parallel/PlumeAstCreator.scala +++ /dev/null @@ -1,939 +0,0 @@ -package com.github.plume.oss.passes.parallel - -import io.shiftleft.codepropertygraph.generated.nodes._ -import io.shiftleft.codepropertygraph.generated._ -import io.shiftleft.passes.DiffGraph -import io.joern.x2cpg.Ast -import org.slf4j.LoggerFactory -import soot.jimple._ -import soot.tagkit.Host -import soot.{Local => _, _} - -import scala.collection.mutable -import scala.jdk.CollectionConverters.CollectionHasAsScala -import scala.util.{Failure, Success, Try} - -/** Manages the creation of the AST layer from the given class file and stores all types in the given global parameter. - */ -class PlumeAstCreator(filename: String, global: Global) { - - import PlumeAstCreator._ - - private val logger = LoggerFactory.getLogger(classOf[AstCreationPass]) - private val unitToAsts = mutable.HashMap[soot.Unit, Seq[Ast]]() - private val controlTargets = mutable.HashMap[Seq[Ast], soot.Unit]() - - /** The changes generated by this pass. - */ - val diffGraph: DiffGraph.Builder = DiffGraph.newBuilder - - /** Add `typeName` to a global map and return it. The - * map is later passed to a pass that creates TYPE - * nodes for each key in the map. - */ - private def registerType(typeName: String): String = { - global.usedTypes.add(typeName) - typeName - } - - /** Entry point of AST creation. Translates a compilation - * unit created by JavaParser into a DiffGraph containing - * the corresponding CPG AST. - */ - def createAst(cls: SootClass): Iterator[DiffGraph] = { - val astRoot = astForCompilationUnit(cls) - storeInDiffGraph(astRoot) - Iterator(diffGraph.build()) - } - - /** Copy nodes/edges of given `AST` into the diff graph - */ - private def storeInDiffGraph(ast: Ast): scala.Unit = { - ast.nodes.foreach { node => - diffGraph.addNode(node) - } - ast.edges.foreach { edge => - diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.AST) - } - ast.conditionEdges.foreach { edge => - diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.CONDITION) - } - ast.argEdges.foreach { edge => - diffGraph.addEdge(edge.src, edge.dst, EdgeTypes.ARGUMENT) - } - } - - /** Translate compilation unit into AST - */ - private def astForCompilationUnit(cls: SootClass): Ast = { - val ast = astForPackageDeclaration(cls.getPackageName) - val namespaceBlockFullName = - ast.root.collect { case x: NewNamespaceBlock => x.fullName }.getOrElse("none") - ast.withChild(astForTypeDecl(cls.getType, namespaceBlockFullName)) - } - - /** Translate package declaration into AST consisting of - * a corresponding namespace block. - */ - private def astForPackageDeclaration(packageDecl: String): Ast = { - val absolutePath = new java.io.File(filename).toPath.toAbsolutePath.normalize().toString - val name = packageDecl.split("\\.").lastOption.getOrElse("") - val namespaceBlock = NewNamespaceBlock() - .name(name) - .fullName(packageDecl) - Ast(namespaceBlock.filename(absolutePath).order(1)) - } - - /** Creates the AST root for type declarations and acts as the entry point for method generation. - */ - private def astForTypeDecl(typ: RefType, namespaceBlockFullName: String): Ast = { - val fullName = typ.toQuotedString - val shortName = - if (fullName.contains('.')) fullName.substring(fullName.lastIndexOf('.') + 1) - else fullName - - val relatedClass = typ.getSootClass - val inheritsFromTypeFullName = - if (relatedClass.hasSuperclass) { - if (!relatedClass.getSuperclass.isApplicationClass) - registerType(relatedClass.getSuperclass.getType.toQuotedString) - List(relatedClass.getSuperclass.toString) - } else List(registerType("java.lang.Object")) - val implementsTypeFullName = relatedClass.getInterfaces.asScala.map { i: SootClass => - if (!i.isApplicationClass) - registerType(i.getType.toQuotedString) - i.getType.toQuotedString - }.toList - - val typeDecl = NewTypeDecl() - .name(shortName) - .fullName(registerType(fullName)) - .order(1) // Jimple always has 1 class per file - .filename(filename) - .code(shortName) - .inheritsFromTypeFullName(inheritsFromTypeFullName ++ implementsTypeFullName) - .astParentType(NodeTypes.NAMESPACE_BLOCK) - .astParentFullName(namespaceBlockFullName) - val methodAsts = withOrder( - typ.getSootClass.getMethods.asScala.toList.sortWith((x, y) => x.getName > y.getName) - ) { (m, order) => - astForMethod(m, typ, order) - } - - val memberAsts = typ.getSootClass.getFields.asScala - .filter(_.isDeclared) - .zipWithIndex - .map { case (v, i) => - astForField(v, i + methodAsts.size + 1) - } - .toList - - Ast(typeDecl) - .withChildren(memberAsts) - .withChildren(methodAsts) - } - - private def astForField(v: SootField, order: Int): Ast = { - val typeFullName = registerType(v.getType.toQuotedString) - val name = v.getName - Ast( - NewMember() - .name(name) - .typeFullName(typeFullName) - .order(order) - .code(s"$typeFullName $name") - ) - } - - private def astForMethod( - methodDeclaration: SootMethod, - typeDecl: RefType, - childNum: Int - ): Ast = { - val methodNode = createMethodNode(methodDeclaration, typeDecl, childNum) - val lastOrder = 2 + methodDeclaration.getParameterCount - try { - if (!methodDeclaration.isConcrete) { - Ast(methodNode) - .withChildren(astsForModifiers(methodDeclaration)) - .withChild(astForMethodReturn(methodDeclaration)) - } else { - val methodBody = Try(methodDeclaration.getActiveBody) match { - case Failure(_) => methodDeclaration.retrieveActiveBody() - case Success(body) => body - } - val parameterAsts = - Seq(createThisNode(methodDeclaration, NewMethodParameterIn())) ++ withOrder( - methodBody.getParameterLocals - ) { (p, order) => - astForParameter(p, order, methodDeclaration) - } - - Ast(methodNode) - .withChildren(astsForModifiers(methodDeclaration)) - .withChildren(parameterAsts) - .withChild(astForMethodBody(methodBody, lastOrder)) - .withChild(astForMethodReturn(methodDeclaration)) - } - } catch { - case e: RuntimeException => - logger.warn( - s"Unexpected exception while parsing method body! Will stub the method ${methodNode.fullName}", - e - ) - Ast(methodNode) - .withChildren(astsForModifiers(methodDeclaration)) - .withChild(astForMethodReturn(methodDeclaration)) - } finally { - // Join all targets with CFG edges - this seems to work from what is seen on DotFiles - controlTargets.foreach({ case (asts, units) => - asts.headOption match { - case Some(value) => - diffGraph.addEdge(value.root.get, unitToAsts(units).last.root.get, EdgeTypes.CFG) - case None => - } - }) - // Clear these maps - controlTargets.clear() - unitToAsts.clear() - } - } - - private def getEvaluationStrategy(typ: soot.Type): String = - typ match { - case _: PrimType => EvaluationStrategies.BY_VALUE - case _: VoidType => EvaluationStrategies.BY_VALUE - case _: NullType => EvaluationStrategies.BY_VALUE - case _: RefLikeType => EvaluationStrategies.BY_REFERENCE - case _ => EvaluationStrategies.BY_SHARING - } - - private def astForParameter( - parameter: soot.Local, - childNum: Int, - methodDeclaration: SootMethod - ): Ast = { - val typeFullName = registerType(parameter.getType.toQuotedString) - val parameterNode = NewMethodParameterIn() - .name(parameter.getName) - .code(s"${parameter.getType.toQuotedString} ${parameter.getName}") - .typeFullName(typeFullName) - .order(childNum) - .lineNumber(line(methodDeclaration)) - .columnNumber(column(methodDeclaration)) - .evaluationStrategy(getEvaluationStrategy(parameter.getType)) - Ast(parameterNode) - } - - private def astForMethodBody(body: Body, order: Int): Ast = { - val block = NewBlock().order(order).lineNumber(line(body)).columnNumber(column(body)) - Ast(block).withChildren( - withOrder(body.getUnits.asScala) { (x, order) => - astsForStatement(x, order) - }.flatten - ) - } - - private def astsForStatement(statement: soot.Unit, order: Int): Seq[Ast] = { - val stmt = statement match { - case x: AssignStmt => astsForDefinition(x, order) - case x: IdentityStmt => astsForDefinition(x, order) - case x: InvokeStmt => astsForExpression(x.getInvokeExpr, order, statement) - case x: ReturnStmt => astsForReturnNode(x, order) - case x: ReturnVoidStmt => astsForReturnVoidNode(x, order) - case x: IfStmt => astsForIfStmt(x, order) - case x: GotoStmt => astsForGotoStmt(x, order) - case x: LookupSwitchStmt => astsForLookupSwitchStmt(x, order) - case x: TableSwitchStmt => astsForTableSwitchStmt(x, order) - case x: ThrowStmt => Seq(astForUnknownStmt(x, x.getOp, order)) - case x: MonitorStmt => Seq(astForUnknownStmt(x, x.getOp, order)) - case x => - logger.warn(s"Unhandled soot.Unit type ${x.getClass}") - Seq() - } - unitToAsts.put(statement, stmt) - stmt - } - - private def astForBinOpExpr(binOp: BinopExpr, order: Int, parentUnit: soot.Unit): Ast = { - val operatorName = binOp match { - case _: AddExpr => Operators.addition - case _: SubExpr => Operators.subtraction - case _: MulExpr => Operators.multiplication - case _: DivExpr => Operators.division - case _: RemExpr => Operators.modulo - case _: GeExpr => Operators.greaterEqualsThan - case _: GtExpr => Operators.greaterThan - case _: LeExpr => Operators.lessEqualsThan - case _: LtExpr => Operators.lessThan - case _: ShlExpr => Operators.shiftLeft - case _: ShrExpr => Operators.logicalShiftRight - case _: UshrExpr => Operators.arithmeticShiftRight - case _: CmpExpr => Operators.compare - case _: CmpgExpr => Operators.compare - case _: CmplExpr => Operators.compare - case _: AndExpr => Operators.and - case _: OrExpr => Operators.or - case _: XorExpr => Operators.xor - case _: EqExpr => Operators.equals - case _ => "" - } - - val callNode = NewCall() - .name(operatorName) - .methodFullName(operatorName) - .dispatchType(DispatchTypes.STATIC_DISPATCH) - .code(binOp.toString) - .argumentIndex(order) - .order(order) - - val args = - astsForValue(binOp.getOp1, 1, parentUnit) ++ astsForValue(binOp.getOp2, 2, parentUnit) - callAst(callNode, args) - } - - private def astsForExpression(expr: Expr, order: Int, parentUnit: soot.Unit): Seq[Ast] = { - expr match { - case x: BinopExpr => Seq(astForBinOpExpr(x, order, parentUnit)) - case x: InvokeExpr => Seq(astForInvokeExpr(x, order, parentUnit)) - case x: AnyNewExpr => Seq(astForNewExpr(x, order, parentUnit)) - case x: CastExpr => Seq(astForUnaryExpr(Operators.cast, x, x.getOp, order, parentUnit)) - case x: InstanceOfExpr => - Seq(astForUnaryExpr(Operators.instanceOf, x, x.getOp, order, parentUnit)) - case x: LengthExpr => - Seq(astForUnaryExpr(Operators.lengthOf, x, x.getOp, order, parentUnit)) - case x: NegExpr => Seq(astForUnaryExpr(Operators.minus, x, x.getOp, order, parentUnit)) - case x => - logger.warn(s"Unhandled soot.Expr type ${x.getClass}") - Seq() - } - } - - private def astsForValue(value: soot.Value, order: Int, parentUnit: soot.Unit): Seq[Ast] = { - value match { - case x: Expr => astsForExpression(x, order, parentUnit) - case x: soot.Local => Seq(astForLocal(x, order, parentUnit)) - case x: CaughtExceptionRef => Seq(astForCaughtExceptionRef(x, order, parentUnit)) - case x: Constant => Seq(astForConstantExpr(x, order)) - case x: FieldRef => Seq(astForFieldRef(x, order, parentUnit)) - case x: ThisRef => Seq(createThisNode(x)) - case x: ParameterRef => Seq(createParameterNode(x, order)) - case x: IdentityRef => Seq(astForIdentityRef(x, order, parentUnit)) - case x: ArrayRef => Seq(astForArrayRef(x, order, parentUnit)) - case x => - logger.warn(s"Unhandled soot.Value type ${x.getClass}") - Seq() - } - } - - private def astForArrayRef( - arrRef: ArrayRef, - order: Int, - parentUnit: soot.Unit - ): Ast = { - val indexAccess = NewCall() - .name(Operators.indexAccess) - .methodFullName(Operators.indexAccess) - .dispatchType(DispatchTypes.STATIC_DISPATCH) - .code(arrRef.toString()) - .order(order) - .argumentIndex(order) - .lineNumber(line(parentUnit)) - .columnNumber(column(parentUnit)) - .typeFullName(registerType(arrRef.getType.toQuotedString)) - - val astChildren = - astsForValue(arrRef.getBase, 1, parentUnit) ++ astsForValue(arrRef.getIndex, 2, parentUnit) - Ast(indexAccess) - .withChildren(astChildren) - .withArgEdges(indexAccess, astChildren.flatMap(_.root)) - } - - private def astForLocal(local: soot.Local, order: Int, parentUnit: soot.Unit): Ast = { - val name = local.getName - val typeFullName = registerType(local.getType.toQuotedString) - Ast( - NewIdentifier() - .name(name) - .lineNumber(line(parentUnit)) - .columnNumber(column(parentUnit)) - .order(order) - .argumentIndex(order) - .code(name) - .typeFullName(typeFullName) - ) - } - - private def astForIdentityRef(x: IdentityRef, order: Int, parentUnit: soot.Unit): Ast = { - Ast( - NewIdentifier() - .code(x.toString()) - .name(x.toString()) - .order(order) - .argumentIndex(order) - .typeFullName(registerType(x.getType.toQuotedString)) - .lineNumber(line(parentUnit)) - .columnNumber(column(parentUnit)) - ) - } - - private def astForInvokeExpr(invokeExpr: InvokeExpr, order: Int, parentUnit: soot.Unit): Ast = { - val method = invokeExpr.getMethodRef - val dispatchType = invokeExpr match { - case _ if method.isConstructor => DispatchTypes.STATIC_DISPATCH - case _: DynamicInvokeExpr => DispatchTypes.DYNAMIC_DISPATCH - case _: InstanceInvokeExpr => DispatchTypes.DYNAMIC_DISPATCH - case _ => DispatchTypes.STATIC_DISPATCH - } - val signature = - s"${method.getReturnType.toQuotedString}(${(for (i <- 0 until method.getParameterTypes().size()) - yield method.getParameterType(i).toQuotedString).mkString(",")})" - val thisAsts = Seq(createThisNode(method, NewIdentifier())) - - val methodName = - if (method.isConstructor) - registerType(method.getDeclaringClass.getType.getClassName) - else - method.getName - - val callType = - if (method.isConstructor) "void" - else registerType(method.getDeclaringClass.getType.toQuotedString) - - val callNode = NewCall() - .name(method.getName) - .code(s"$methodName(${invokeExpr.getArgs.asScala.mkString(", ")})") - .dispatchType(dispatchType) - .order(order) - .argumentIndex(order) - .methodFullName(s"${method.getDeclaringClass.toString}.${method.getName}:$signature") - .signature(signature) - .typeFullName(callType) - .lineNumber(line(parentUnit)) - .columnNumber(column(parentUnit)) - - val argAsts = withOrder(invokeExpr match { - case x: DynamicInvokeExpr => x.getArgs.asScala ++ x.getBootstrapArgs.asScala - case x => x.getArgs.asScala - }) { case (arg, order) => - astsForValue(arg, order, parentUnit) - }.flatten - - Ast(callNode) - .withChildren(thisAsts) - .withChildren(argAsts) - .withArgEdges(callNode, thisAsts.flatMap(_.root)) - .withArgEdges(callNode, argAsts.flatMap(_.root)) - } - - private def astForNewExpr(x: AnyNewExpr, order: Int, parentUnit: soot.Unit): Ast = { - x match { - case u: NewArrayExpr => - astForArrayInitializeExpr(x, List(u.getSize), order, parentUnit) - case u: NewMultiArrayExpr => - astForArrayInitializeExpr(x, u.getSizes.asScala, order, parentUnit) - case _ => - val parentType = registerType(x.getType.toQuotedString) - Ast( - NewCall() - .name(".alloc") - .methodFullName(".alloc") - .typeFullName(parentType) - .code(s"new ${x.getType}") - .dispatchType(DispatchTypes.STATIC_DISPATCH) - .order(order) - .argumentIndex(order) - .lineNumber(line(parentUnit)) - .columnNumber(column(parentUnit)) - ) - } - } - - private def astForArrayInitializeExpr( - arrayInitExpr: Expr, - sizes: Iterable[Value], - order: Int, - parentUnit: soot.Unit - ): Ast = { - val callBlock = NewCall() - .name(Operators.arrayInitializer) - .methodFullName(Operators.arrayInitializer) - .code(arrayInitExpr.toString()) - .dispatchType(DispatchTypes.STATIC_DISPATCH) - .order(order) - .typeFullName(registerType(arrayInitExpr.getType.toQuotedString)) - .argumentIndex(order) - .lineNumber(line(parentUnit)) - .columnNumber(column(parentUnit)) - val valueAsts = withOrder(sizes) { (s, o) => astsForValue(s, o, parentUnit) }.flatten - Ast(callBlock) - .withChildren(valueAsts) - .withArgEdges(callBlock, valueAsts.flatMap(_.root)) - } - - private def astForUnaryExpr( - methodName: String, - unaryExpr: Expr, - op: Value, - order: Int, - parentUnit: soot.Unit - ): Ast = { - val callBlock = NewCall() - .name(methodName) - .methodFullName(methodName) - .code(unaryExpr.toString()) - .dispatchType(DispatchTypes.STATIC_DISPATCH) - .order(order) - .typeFullName(registerType(unaryExpr.getType.toQuotedString)) - .argumentIndex(order) - .lineNumber(line(parentUnit)) - .columnNumber(column(parentUnit)) - val valueAsts = astsForValue(op, 1, parentUnit) - Ast(callBlock) - .withChildren(valueAsts) - .withArgEdges(callBlock, valueAsts.flatMap(_.root)) - } - - private def createThisNode(method: ThisRef): Ast = { - Ast( - NewIdentifier() - .name("this") - .code("this") - .typeFullName(registerType(method.getType.toQuotedString)) - .dynamicTypeHintFullName(Seq(method.getType.toQuotedString)) - .order(0) - .argumentIndex(0) - ) - } - - private def createThisNode(method: SootMethod, builder: NewNode): Ast = - createThisNode(method.makeRef(), builder) - - private def createThisNode(method: SootMethodRef, builder: NewNode): Ast = { - if (!method.isStatic || method.isConstructor) { - val parentType = registerType(method.getDeclaringClass.getType.toQuotedString) - Ast( - builder match { - case x: NewIdentifier => - x.name("this") - .code("this") - .typeFullName(parentType) - .order(0) - .argumentIndex(0) - .dynamicTypeHintFullName(Seq(parentType)) - case x: NewMethodParameterIn => - x.name("this") - .code("this") - .lineNumber(line(method.tryResolve())) - .typeFullName(parentType) - .order(0) - .evaluationStrategy(EvaluationStrategies.BY_SHARING) - .dynamicTypeHintFullName(Seq(parentType)) - case x => x - } - ) - } else { - Ast() - } - } - - private def createParameterNode(parameterRef: ParameterRef, order: Int): Ast = { - val name = s"@parameter${parameterRef.getIndex}" - Ast( - NewIdentifier() - .name(name) - .code(name) - .typeFullName(registerType(parameterRef.getType.toQuotedString)) - .order(order) - .argumentIndex(order) - ) - } - - /** Creates the AST for assignment statements keeping in mind Jimple is a 3-address code language. - */ - private def astsForDefinition(assignStmt: DefinitionStmt, order: Int): Seq[Ast] = { - val initializer = assignStmt.getRightOp - val leftOp = assignStmt.getLeftOp - val name = assignStmt.getLeftOp match { - case x: soot.Local => x.getName - case x: FieldRef => x.getFieldRef.name - case x: ArrayRef => x.toString() - case x => logger.warn(s"Unhandled LHS type in definition ${x.getClass}"); x.toString() - } - val typeFullName = registerType(leftOp.getType.toQuotedString) - val code = s"$typeFullName $name" - val identifier = leftOp match { - case x: soot.Local => Seq(astForLocal(x, 1, assignStmt)) - case x: FieldRef => Seq(astForFieldRef(x, 1, assignStmt)) - case x => astsForValue(x, 1, assignStmt) - } - val initAsts = astsForValue(initializer, 2, assignStmt) - val assignmentRhsCode = initAsts - .flatMap(_.root) - .map(_.properties.getOrElse(PropertyNames.CODE, "")) - .mkString(", ") - val assignment = NewCall() - .name(Operators.assignment) - .code(s"$name = $assignmentRhsCode") - .dispatchType(DispatchTypes.STATIC_DISPATCH) - .order(order) - .argumentIndex(order) - .typeFullName(registerType(assignStmt.getLeftOp.getType.toQuotedString)) - val initializerAst = Seq(callAst(assignment, identifier ++ initAsts)) - Seq( - Ast( - NewLocal().name(name).code(code).typeFullName(typeFullName).order(order) - ) - ) ++ initializerAst.toList - } - - private def astsForIfStmt(ifStmt: IfStmt, order: Int): Seq[Ast] = { - // bytecode/jimple ASTs are flat so there will not be nested bodies - val condition = astsForValue(ifStmt.getCondition, order, ifStmt) - controlTargets.put(condition, ifStmt.getTarget) - condition - } - - private def astsForGotoStmt(gotoStmt: GotoStmt, order: Int): Seq[Ast] = { - // bytecode/jimple ASTs are flat so there will not be nested bodies - val gotoAst = Seq( - Ast( - NewUnknown() - .code(gotoStmt.toString) - .order(order) - .argumentIndex(order) - .lineNumber(line(gotoStmt)) - .columnNumber(column(gotoStmt)) - ) - ) - controlTargets.put(gotoAst, gotoStmt.getTarget) - gotoAst - } - - private def astForSwitchWithDefaultAndCondition(switchStmt: SwitchStmt, order: Int): Ast = { - val jimple = switchStmt.toString() - val totalTgts = switchStmt.getTargets.size() - val switch = NewControlStructure() - .controlStructureType(ControlStructureTypes.SWITCH) - .code(jimple.dropRight(jimple.length - jimple.indexOf("{") + 1)) - .lineNumber(line(switchStmt)) - .columnNumber(column(switchStmt)) - .order(order) - .argumentIndex(order) - - val conditionalAst = astsForValue(switchStmt.getKey, totalTgts + 1, switchStmt) - val defaultAst = Seq( - Ast( - NewJumpTarget() - .name("default") - .code("default:") - .order(totalTgts + 2) - .argumentIndex(totalTgts + 2) - .lineNumber(line(switchStmt.getDefaultTarget)) - .columnNumber(column(switchStmt.getDefaultTarget)) - ) - ) - Ast(switch) - .withConditionEdge(switch, conditionalAst.flatMap(_.root).head) - .withChildren(conditionalAst ++ defaultAst) - } - - private def astsForLookupSwitchStmt(lookupSwitchStmt: LookupSwitchStmt, order: Int): Seq[Ast] = { - val totalTgts = lookupSwitchStmt.getTargets.size() - val switchAst = astForSwitchWithDefaultAndCondition(lookupSwitchStmt, order) - - val tgts = for { - i <- 0 until totalTgts - if lookupSwitchStmt.getTarget(i) != lookupSwitchStmt.getDefaultTarget - } yield (lookupSwitchStmt.getLookupValue(i), lookupSwitchStmt.getTarget(i)) - val tgtAsts = tgts.map { case (lookup, target) => - Ast( - NewJumpTarget() - .name(s"case $lookup") - .code(s"case $lookup:") - .argumentIndex(lookup) - .order(lookup) - .lineNumber(line(target)) - .columnNumber(column(target)) - ) - } - - Seq( - switchAst - .withChildren(tgtAsts) - ) - } - - private def astsForTableSwitchStmt(tableSwitchStmt: SwitchStmt, order: Int): Seq[Ast] = { - val switchAst = astForSwitchWithDefaultAndCondition(tableSwitchStmt, order) - val tgtAsts = tableSwitchStmt.getTargets.asScala - .filter(x => tableSwitchStmt.getDefaultTarget != x) - .zipWithIndex - .map({ case (tgt, i) => - Ast( - NewJumpTarget() - .name(s"case $i") - .code(s"case $i:") - .argumentIndex(i) - .order(i) - .lineNumber(line(tgt)) - .columnNumber(column(tgt)) - ) - }) - .toSeq - - Seq( - switchAst - .withChildren(tgtAsts) - ) - } - - private def astForUnknownStmt(stmt: Stmt, op: Value, order: Int): Ast = { - val opAst = astsForValue(op, 1, stmt) - val unknown = NewUnknown() - .order(order) - .code(stmt.toString()) - .lineNumber(line(stmt)) - .columnNumber(column(stmt)) - .typeFullName(registerType("void")) - Ast(unknown) - .withChildren(opAst) - } - - private def astsForReturnNode(returnStmt: ReturnStmt, order: Int): Seq[Ast] = { - Seq( - Ast(NewReturn().order(order).lineNumber(line(returnStmt)).columnNumber(column(returnStmt))) - .withChildren(astsForValue(returnStmt.getOp, order + 1, returnStmt)) - ) - } - - private def astsForReturnVoidNode(returnVoidStmt: ReturnVoidStmt, order: Int): Seq[Ast] = { - Seq( - Ast( - NewReturn() - .order(order) - .lineNumber(line(returnVoidStmt)) - .columnNumber(column(returnVoidStmt)) - ) - ) - } - - private def astForFieldRef(fieldRef: FieldRef, order: Int, parentUnit: soot.Unit): Ast = { - val leftOpString = fieldRef match { - case x: StaticFieldRef => x.getFieldRef.declaringClass().toString - case x: InstanceFieldRef => x.getBase.toString() - case _ => fieldRef.getFieldRef.declaringClass().toString - } - val leftOpType = fieldRef match { - case x: StaticFieldRef => x.getFieldRef.declaringClass().getType - case x: InstanceFieldRef => x.getBase.getType - case _ => fieldRef.getFieldRef.declaringClass().getType - } - val fieldAccessBlock = NewCall() - .name(Operators.fieldAccess) - .code(s"${leftOpType.toQuotedString}.${fieldRef.getFieldRef.name()}") - .typeFullName(registerType(fieldRef.getType.toQuotedString)) - .methodFullName(Operators.fieldAccess) - .dispatchType(DispatchTypes.STATIC_DISPATCH) - .order(order) - .argumentIndex(order) - .lineNumber(line(parentUnit)) - .columnNumber(column(parentUnit)) - - val argAsts = Seq( - NewIdentifier() - .order(1) - .argumentIndex(1) - .lineNumber(line(parentUnit)) - .columnNumber(column(parentUnit)) - .name(leftOpString) - .code(leftOpString) - .typeFullName(registerType(leftOpType.toQuotedString)), - NewFieldIdentifier() - .order(2) - .argumentIndex(2) - .lineNumber(line(parentUnit)) - .columnNumber(column(parentUnit)) - .canonicalName(fieldRef.getFieldRef.getSignature) - .code(fieldRef.getFieldRef.name()) - ).map(Ast(_)) - - Ast(fieldAccessBlock) - .withChildren(argAsts) - .withArgEdges(fieldAccessBlock, argAsts.flatMap(_.root)) - } - - private def astForCaughtExceptionRef( - caughtException: CaughtExceptionRef, - order: Int, - parentUnit: soot.Unit - ): Ast = { - Ast( - NewIdentifier() - .order(order) - .argumentIndex(order) - .lineNumber(line(parentUnit)) - .columnNumber(column(parentUnit)) - .name(caughtException.toString()) - .code(caughtException.toString()) - .typeFullName(registerType(caughtException.getType.toQuotedString)) - ) - } - - private def astForConstantExpr(constant: Constant, order: Int): Ast = { - constant match { - case _: ClassConstant => Ast() - case _: NullConstant => Ast() - case _: IntConstant => - Ast( - NewLiteral() - .order(order) - .argumentIndex(order) - .code(constant.toString) - .typeFullName(registerType("int")) - ) - case _: LongConstant => - Ast( - NewLiteral() - .order(order) - .argumentIndex(order) - .code(constant.toString) - .typeFullName(registerType("long")) - ) - case _: DoubleConstant => - Ast( - NewLiteral() - .order(order) - .argumentIndex(order) - .code(constant.toString) - .typeFullName(registerType("double")) - ) - case _: FloatConstant => - Ast( - NewLiteral() - .order(order) - .argumentIndex(order) - .code(constant.toString) - .typeFullName(registerType("float")) - ) - case _: StringConstant => - Ast( - NewLiteral() - .order(order) - .argumentIndex(order) - .code(constant.toString) - .typeFullName(registerType("java.lang.String")) - ) - } - } - - private def callAst(rootNode: NewNode, args: Seq[Ast]): Ast = { - Ast(rootNode) - .withChildren(args) - .withArgEdges(rootNode, args.flatMap(_.root)) - } - - private def astsForModifiers(methodDeclaration: SootMethod): Seq[Ast] = { - Seq( - if (methodDeclaration.isStatic) Some(ModifierTypes.STATIC) else None, - if (methodDeclaration.isPublic) Some(ModifierTypes.PUBLIC) else None, - if (methodDeclaration.isProtected) Some(ModifierTypes.PROTECTED) else None, - if (methodDeclaration.isPrivate) Some(ModifierTypes.PRIVATE) else None, - if (methodDeclaration.isAbstract) Some(ModifierTypes.ABSTRACT) else None, - if (methodDeclaration.isConstructor) Some(ModifierTypes.CONSTRUCTOR) else None, - if (!methodDeclaration.isFinal && !methodDeclaration.isStatic && methodDeclaration.isPublic) - Some(ModifierTypes.VIRTUAL) - else None, - if (methodDeclaration.isSynchronized) Some("SYNCHRONIZED") else None - ).flatten.map { modifier => - Ast(NewModifier().modifierType(modifier).code(modifier.toLowerCase)) - } - } - - private def astForMethodReturn(methodDeclaration: SootMethod): Ast = { - val typeFullName = registerType(methodDeclaration.getReturnType.toQuotedString) - val methodReturnNode = - NewMethodReturn() - .order(methodDeclaration.getParameterCount + 2) - .typeFullName(typeFullName) - .code(methodDeclaration.getReturnType.toQuotedString) - .lineNumber(line(methodDeclaration)) - Ast(methodReturnNode) - } - - private def createMethodNode( - methodDeclaration: SootMethod, - typeDecl: RefType, - childNum: Int - ) = { - val fullName = methodFullName(typeDecl, methodDeclaration) - val code = if (!methodDeclaration.isConstructor) { - s"${methodDeclaration.getReturnType.toQuotedString} ${methodDeclaration.getName}${paramListSignature(methodDeclaration, withParams = true)}" - } else { - s"${typeDecl.getClassName}${paramListSignature(methodDeclaration, withParams = true)}" - } - NewMethod() - .name(methodDeclaration.getName) - .fullName(fullName) - .code(code) - .signature( - methodDeclaration.getReturnType.toQuotedString + paramListSignature(methodDeclaration) - ) - .isExternal(false) - .order(childNum) - .filename(filename) - .lineNumber(line(methodDeclaration)) - .columnNumber(column(methodDeclaration)) - } - - private def methodFullName( - typeDecl: RefType, - methodDeclaration: SootMethod - ): String = { - val typeName = typeDecl.toQuotedString - val returnType = methodDeclaration.getReturnType.toQuotedString - val methodName = methodDeclaration.getName - s"$typeName.$methodName:$returnType${paramListSignature(methodDeclaration)}" - } - - private def paramListSignature(methodDeclaration: SootMethod, withParams: Boolean = false) = { - val paramTypes = methodDeclaration.getParameterTypes.asScala.map(_.toQuotedString) - - val paramNames = - if (!methodDeclaration.isPhantom && Try(methodDeclaration.retrieveActiveBody()).isSuccess) - methodDeclaration.retrieveActiveBody().getParameterLocals.asScala.map(_.getName) - else - paramTypes.zipWithIndex.map(x => { s"param${x._2 + 1}" }) - if (!withParams) { - "(" + paramTypes.mkString(",") + ")" - } else { - "(" + paramTypes.zip(paramNames).map(x => s"${x._1} ${x._2}").mkString(", ") + ")" - } - } -} - -object PlumeAstCreator { - def line(node: Host): Option[Integer] = { - if (node == null) None - else if (node.getJavaSourceStartLineNumber == -1) None - else Option(node.getJavaSourceStartLineNumber) - } - - def column(node: Host): Option[Integer] = { - if (node == null) None - else if (node.getJavaSourceStartColumnNumber == -1) None - else Option(node.getJavaSourceStartColumnNumber) - } - - def withOrder[T, X](nodeList: java.util.List[T])(f: (T, Int) => X): Seq[X] = { - nodeList.asScala.zipWithIndex.map { case (x, i) => - f(x, i + 1) - }.toSeq - } - - def withOrder[T, X](nodeList: Iterable[T])(f: (T, Int) => X): Seq[X] = { - nodeList.zipWithIndex.map { case (x, i) => - f(x, i + 1) - }.toSeq - } -} diff --git a/src/main/scala/com/github/plume/oss/passes/parallel/PlumeParallelCpgPass.scala b/src/main/scala/com/github/plume/oss/passes/parallel/PlumeParallelCpgPass.scala deleted file mode 100644 index bc858b8f..00000000 --- a/src/main/scala/com/github/plume/oss/passes/parallel/PlumeParallelCpgPass.scala +++ /dev/null @@ -1,120 +0,0 @@ -package com.github.plume.oss.passes.parallel - -import com.github.plume.oss.drivers.IDriver -import com.github.plume.oss.passes.PlumeCpgPassBase -import com.github.plume.oss.passes.parallel.PlumeParallelCpgPass.{ - parallelEnqueue, - parallelWithWriter -} -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.passes.{DiffGraph, KeyPool, ParallelCpgPass, ParallelIteratorExecutor} -import org.slf4j.Logger - -import scala.concurrent.ExecutionContext.Implicits.global - -abstract class PlumeParallelCpgPass[T]( - cpg: Cpg, - keyPools: Option[Iterator[KeyPool]] = None -) extends ParallelCpgPass[T](cpg, keyPools = keyPools) - with PlumeCpgPassBase { - - override def createAndApply(driver: IDriver): Unit = { - withWriter(driver) { writer => - enqueueInParallel(writer) - } - } - - private def withWriter[X](driver: IDriver)(f: PlumeParallelWriter => Unit): Unit = - parallelWithWriter[X](driver, f, cpg, baseLogger) - - private def enqueueInParallel(writer: PlumeParallelWriter): Unit = - withStartEndTimesLogged { - init() - parallelEnqueue[T]( - baseLogger, - name, - writer, - (part: T) => runOnPart(part), - keyPools, - partIterator - ) - } - -} - -object PlumeParallelCpgPass { - def parallelWithWriter[X]( - driver: IDriver, - f: PlumeParallelWriter => Unit, - cpg: Cpg, - baseLogger: Logger - ): Unit = { - val writer = new PlumeParallelWriter(driver, cpg) - val writerThread = new Thread(writer) - writerThread.setName("Writer") - writerThread.start() - try { - f(writer) - } catch { - case exception: Exception => - baseLogger.warn("pass failed", exception) - } finally { - writer.enqueue(None, None) - writerThread.join() - } - } - - def parallelEnqueue[T]( - baseLogger: Logger, - name: String, - writer: PlumeParallelWriter, - runOnPart: T => Iterator[DiffGraph], - keyPools: Option[Iterator[KeyPool]], - partIterator: Iterator[T] - ): Unit = { - try { - val it = new ParallelIteratorExecutor( - parallelItWithKeyPools[T]( - baseLogger, - keyPools, - partIterator - ) - ).map { case (part, keyPool) => - runOnPart(part).foreach(diffGraph => writer.enqueue(Some(diffGraph), keyPool)) - } - consume(it) - } catch { - case exception: Exception => - baseLogger.warn(s"Exception in parallel CPG pass $name:", exception) - } - } - - def parallelItWithKeyPools[T]( - baseLogger: Logger, - keyPools: Option[Iterator[KeyPool]], - partIterator: Iterator[T] - ): Iterator[(T, Option[KeyPool])] = - if (keyPools.isEmpty) { - partIterator.map(p => (p, None)) - } else { - val pools = keyPools.get - partIterator.map { p => - ( - p, - pools.nextOption() match { - case Some(pool) => Some(pool) - case None => - baseLogger.warn("Not enough key pools provided. Ids may not be constant across runs") - None - } - ) - } - } - - private def consume(it: Iterator[_]): Unit = { - while (it.hasNext) { - it.next() - } - } - -} diff --git a/src/main/scala/com/github/plume/oss/passes/parallel/PlumeParallelWriter.scala b/src/main/scala/com/github/plume/oss/passes/parallel/PlumeParallelWriter.scala deleted file mode 100644 index c701a869..00000000 --- a/src/main/scala/com/github/plume/oss/passes/parallel/PlumeParallelWriter.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.github.plume.oss.passes.parallel - -import com.github.plume.oss.drivers.IDriver -import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.passes.{CpgPass, DiffGraph, KeyPool} -import org.slf4j.{Logger, LoggerFactory} - -import java.util.concurrent.LinkedBlockingQueue - -private class PlumeParallelWriter( - driver: IDriver, - cpg: Cpg, - baseLogger: Logger = LoggerFactory.getLogger(classOf[CpgPass]) -) extends Runnable { - - case class DiffGraphAndKeyPool(diffGraph: Option[DiffGraph], keyPool: Option[KeyPool]) - - private val queue = new LinkedBlockingQueue[DiffGraphAndKeyPool] - - def enqueue(diffGraph: Option[DiffGraph], keyPool: Option[KeyPool]): Unit = { - queue.put(DiffGraphAndKeyPool(diffGraph, keyPool)) - } - - override def run(): Unit = { - try { - var terminate = false - while (!terminate) { - queue.take() match { - case DiffGraphAndKeyPool(Some(diffGraph), keyPool) => - val appliedDiffGraph = DiffGraph.Applier.applyDiff(diffGraph, cpg, keyPool = keyPool) - // Reflect changes in driver - driver.bulkTx(appliedDiffGraph) - case DiffGraphAndKeyPool(None, _) => - baseLogger.debug("Shutting down WriterThread") - terminate = true - case _ => - } - } - } catch { - case exception: InterruptedException => - baseLogger.warn("Interrupted WriterThread", exception) - } - } -} diff --git a/src/main/scala/com/github/plume/oss/passes/parallel/PlumeReachingDefPass.scala b/src/main/scala/com/github/plume/oss/passes/reachingdef/PlumeReachingDefPass.scala similarity index 85% rename from src/main/scala/com/github/plume/oss/passes/parallel/PlumeReachingDefPass.scala rename to src/main/scala/com/github/plume/oss/passes/reachingdef/PlumeReachingDefPass.scala index 8766297c..0604d28b 100644 --- a/src/main/scala/com/github/plume/oss/passes/parallel/PlumeReachingDefPass.scala +++ b/src/main/scala/com/github/plume/oss/passes/reachingdef/PlumeReachingDefPass.scala @@ -1,8 +1,8 @@ -package com.github.plume.oss.passes.parallel +package com.github.plume.oss.passes.reachingdef import com.github.plume.oss.drivers.IDriver -import com.github.plume.oss.passes.PlumeCpgPassBase -import com.github.plume.oss.passes.forkjoin.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore +import com.github.plume.oss.passes.PlumeForkJoinParallelCpgPass.forkJoinSerializeAndStore +import com.github.plume.oss.passes.base.PlumeCpgPassBase import io.joern.dataflowengineoss.passes.reachingdef.ReachingDefPass import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.Method diff --git a/src/test/scala/com/github/plume/oss/DiffTests.scala b/src/test/scala/com/github/plume/oss/DiffTests.scala index 2690c26d..518f83fb 100644 --- a/src/test/scala/com/github/plume/oss/DiffTests.scala +++ b/src/test/scala/com/github/plume/oss/DiffTests.scala @@ -67,7 +67,7 @@ class DiffTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { driver.propertyFromNodes(NodeTypes.TYPE, PropertyNames.FULL_NAME).size shouldBe 7 driver.propertyFromNodes(NodeTypes.NAMESPACE_BLOCK, PropertyNames.FULL_NAME).size shouldBe 3 driver.propertyFromNodes(NodeTypes.METHOD_PARAMETER_IN, PropertyNames.FULL_NAME).size shouldBe 9 - driver.propertyFromNodes(NodeTypes.LOCAL, PropertyNames.FULL_NAME).size shouldBe 7 + driver.propertyFromNodes(NodeTypes.LOCAL, PropertyNames.FULL_NAME).size shouldBe 5 val List(barM, fooM) = driver .propertyFromNodes(NodeTypes.TYPE_DECL, PropertyNames.FULL_NAME, PropertyNames.IS_EXTERNAL) @@ -98,7 +98,7 @@ class DiffTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { driver.propertyFromNodes(NodeTypes.TYPE, PropertyNames.FULL_NAME).size shouldBe 7 driver.propertyFromNodes(NodeTypes.NAMESPACE_BLOCK, PropertyNames.FULL_NAME).size shouldBe 3 driver.propertyFromNodes(NodeTypes.METHOD_PARAMETER_IN, PropertyNames.FULL_NAME).size shouldBe 9 - driver.propertyFromNodes(NodeTypes.LOCAL, PropertyNames.FULL_NAME).size shouldBe 7 + driver.propertyFromNodes(NodeTypes.LOCAL, PropertyNames.FULL_NAME).size shouldBe 5 val List(barM, fooM) = driver .propertyFromNodes(NodeTypes.TYPE_DECL, PropertyNames.FULL_NAME, PropertyNames.IS_EXTERNAL) diff --git a/src/test/scala/com/github/plume/oss/querying/CfgTests.scala b/src/test/scala/com/github/plume/oss/querying/CfgTests.scala index e0bedd57..79fec3aa 100644 --- a/src/test/scala/com/github/plume/oss/querying/CfgTests.scala +++ b/src/test/scala/com/github/plume/oss/querying/CfgTests.scala @@ -26,7 +26,7 @@ class CfgTests extends Jimple2CpgFixture { """.stripMargin "should find that sink is control dependent on condition" in { - val controllers = cpg.call("sink").controlledBy.isCall.toSet + val controllers = cpg.call("sink").controlledBy.isCall.toSetMutable controllers.map(_.code) should contain("y >= 10") controllers.map(_.code) should contain("x >= 5") } @@ -41,13 +41,7 @@ class CfgTests extends Jimple2CpgFixture { // } "should find sink(x) is dominated by `x < 5` and `y < 10`" in { - cpg.call("sink").dominatedBy.isCall.code.toSet shouldBe Set( - "x >= 5", - "this = this", - "x = @parameter0", - "y >= 10", - "y = @parameter1" - ) + cpg.call("sink").dominatedBy.isCall.code.toSetMutable shouldBe Set("x >= 5", "y >= 10") } // "should find that println post dominates correct nodes" in { diff --git a/src/test/scala/com/github/plume/oss/querying/ConstructorInvocationTests.scala b/src/test/scala/com/github/plume/oss/querying/ConstructorInvocationTests.scala index a65c4d09..f8c625f8 100644 --- a/src/test/scala/com/github/plume/oss/querying/ConstructorInvocationTests.scala +++ b/src/test/scala/com/github/plume/oss/querying/ConstructorInvocationTests.scala @@ -97,7 +97,7 @@ class ConstructorInvocationTests extends Jimple2CpgFixture { "it should create joint `alloc` and `init` calls for a constructor invocation in a vardecl" in { cpg.typeDecl.name("Bar").method.name("test1").l match { case List(method) => - val List(_: Local, assign: Call, init: Call, _: Local, _: Call, _: Return) = + val List(_: Local, _: Local, assign: Call, init: Call, _: Call, _: Return) = method.astChildren.isBlock.astChildren.l assign.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString @@ -116,15 +116,15 @@ class ConstructorInvocationTests extends Jimple2CpgFixture { init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString init.typeFullName shouldBe "void" init.signature shouldBe "void(int,int)" - init.code shouldBe "Bar(4, 2)" + init.code shouldBe "$stack1.Bar(4, 2)" init.argument.size shouldBe 3 val List(obj: Identifier, initArg1: Literal, initArg2: Literal) = init.argument.l obj.order shouldBe 0 obj.argumentIndex shouldBe 0 - obj.name shouldBe "this" + obj.name shouldBe "$stack1" obj.typeFullName shouldBe "Bar" - obj.code shouldBe "this" + obj.code shouldBe "$stack1" initArg1.code shouldBe "4" initArg2.code shouldBe "2" @@ -135,16 +135,8 @@ class ConstructorInvocationTests extends Jimple2CpgFixture { "it should create joint `alloc` and `init` calls for a constructor invocation in an assignment" in { cpg.typeDecl.name("Bar").method.name("test2").l match { case List(method) => - val List( - _: Local, - paramAssign: Call, - _: Local, - assign: Call, - init: Call, - _: Local, - _: Call, - _: Return - ) = method.astChildren.isBlock.astChildren.l + val List(_: Local, _: Local, assign: Call, init: Call, _: Call, _: Return) = + method.astChildren.isBlock.astChildren.l assign.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString assign.name shouldBe Operators.assignment @@ -162,15 +154,15 @@ class ConstructorInvocationTests extends Jimple2CpgFixture { init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString init.typeFullName shouldBe "void" init.signature shouldBe "void(int,int)" - init.code shouldBe "Bar(4, 2)" + init.code shouldBe "$stack1.Bar(4, 2)" init.argument.size shouldBe 3 val List(obj: Identifier, initArg1: Literal, initArg2: Literal) = init.argument.l obj.order shouldBe 0 obj.argumentIndex shouldBe 0 - obj.name shouldBe "this" + obj.name shouldBe "$stack1" obj.typeFullName shouldBe "Bar" - obj.code shouldBe "this" + obj.code shouldBe "$stack1" initArg1.code shouldBe "4" initArg2.code shouldBe "2" @@ -181,14 +173,8 @@ class ConstructorInvocationTests extends Jimple2CpgFixture { "it should create `alloc` and `init` calls in a block for complex assignments" in { cpg.typeDecl.name("Bar").method.name("test3").l match { case List(method) => - val List( - assignParam: Call, - allocAssign: Call, - init: Call, - assign: Call, - _: Call, - indexAccess: Call - ) = method.call.l + val List(allocAssign: Call, init: Call, assign: Call, _: Call, indexAccess: Call) = + method.call.l val List(arrayAccess: Call, temp: Identifier) = assign.argument.l temp.name shouldBe "$stack1" temp.typeFullName shouldBe "Bar" @@ -210,14 +196,14 @@ class ConstructorInvocationTests extends Jimple2CpgFixture { init.methodFullName shouldBe "Bar.:void(int)" init.callOut.head.fullName shouldBe "Bar.:void(int)" init.signature shouldBe "void(int)" - init.code shouldBe "Bar(42)" + init.code shouldBe "$stack1.Bar(42)" init.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString init.argument.size shouldBe 2 val List(receiver: Identifier, initArg1: Literal) = init.argument.l receiver.order shouldBe 0 receiver.argumentIndex shouldBe 0 - receiver.name shouldBe "this" + receiver.name shouldBe "$stack1" receiver.typeFullName shouldBe "Bar" initArg1.code shouldBe "42" @@ -229,14 +215,7 @@ class ConstructorInvocationTests extends Jimple2CpgFixture { "it should create only `init` call for direct invocation using `this`" in { cpg.typeDecl.name("Bar").method.fullNameExact("Bar.:void(int,int)").l match { case List(method) => - val List( - assignThis: Call, - assignParam1: Call, - assignParam2: Call, - assignAddition: Call, - init: Call, - addition: Call - ) = method.call.l + val List(assignAddition: Call, init: Call, addition: Call) = method.call.l init.name shouldBe "" init.methodFullName shouldBe "Bar.:void(int)" @@ -267,20 +246,12 @@ class ConstructorInvocationTests extends Jimple2CpgFixture { "it should create only `init` call for direct invocation using `super`" in { cpg.typeDecl.name("Bar").method.fullNameExact("Bar.:void(int)").l match { case List(method) => - val List(assignThis: Call, paramAssign: Call, alloc: Call) = method.call.l + val List(alloc: Call) = method.call.l alloc.name shouldBe "" alloc.methodFullName shouldBe "Foo.:void(int)" alloc.dispatchType shouldBe DispatchTypes.STATIC_DISPATCH.toString alloc.typeFullName shouldBe "void" alloc.signature shouldBe "void(int)" - - val List(thisNode: Identifier, thisParam: Identifier) = assignThis.astChildren.l - thisNode.name shouldBe "this" - thisNode.typeFullName shouldBe "Bar" - thisNode.argumentIndex shouldBe 0 - thisNode.order shouldBe 0 - thisNode.code shouldBe "this" - alloc.argument(1).code shouldBe "x" case res => fail(s"Expected Bar constructor but found $res") diff --git a/src/test/scala/com/github/plume/oss/querying/LocalTests.scala b/src/test/scala/com/github/plume/oss/querying/LocalTests.scala index bae29d33..0bcf075b 100644 --- a/src/test/scala/com/github/plume/oss/querying/LocalTests.scala +++ b/src/test/scala/com/github/plume/oss/querying/LocalTests.scala @@ -26,11 +26,11 @@ class LocalTests extends Jimple2CpgFixture { x.name shouldBe "$stack3" x.code shouldBe "java.lang.Integer $stack3" x.typeFullName shouldBe "java.lang.Integer" - x.order shouldBe 2 + x.order shouldBe 1 y.name shouldBe "y" y.code shouldBe "java.lang.Integer y" y.typeFullName shouldBe "java.lang.Integer" - y.order shouldBe 4 + y.order shouldBe 3 } } diff --git a/src/test/scala/com/github/plume/oss/querying/MetaDataTests.scala b/src/test/scala/com/github/plume/oss/querying/MetaDataTests.scala index 9bb366cf..10e81c97 100644 --- a/src/test/scala/com/github/plume/oss/querying/MetaDataTests.scala +++ b/src/test/scala/com/github/plume/oss/querying/MetaDataTests.scala @@ -19,8 +19,8 @@ class MetaDataTests extends Jimple2CpgFixture { "should not have any incoming or outgoing edges" in { cpg.metaData.size shouldBe 1 - cpg.metaData.in.l shouldBe List() - cpg.metaData.out.l shouldBe List() + cpg.metaData.in().l shouldBe List() + cpg.metaData.out().l shouldBe List() } } diff --git a/src/test/scala/com/github/plume/oss/querying/MethodParameterTests.scala b/src/test/scala/com/github/plume/oss/querying/MethodParameterTests.scala index 74d2b62c..5d6fd060 100644 --- a/src/test/scala/com/github/plume/oss/querying/MethodParameterTests.scala +++ b/src/test/scala/com/github/plume/oss/querying/MethodParameterTests.scala @@ -16,7 +16,7 @@ class MethodParameterTests extends Jimple2CpgFixture { """.stripMargin "should return exactly three parameters with correct fields" in { - cpg.parameter.filter(_.method.name == "foo").name.toSet shouldBe Set("this", "param1", "param2") + cpg.parameter.filter(_.method.name == "foo").name.toSetMutable shouldBe Set("this", "param1", "param2") val List(t) = cpg.parameter.filter(_.method.name == "foo").name("this").l t.code shouldBe "this" diff --git a/src/test/scala/com/github/plume/oss/querying/StaticCallGraphTests.scala b/src/test/scala/com/github/plume/oss/querying/StaticCallGraphTests.scala index d269b98a..f264452e 100644 --- a/src/test/scala/com/github/plume/oss/querying/StaticCallGraphTests.scala +++ b/src/test/scala/com/github/plume/oss/querying/StaticCallGraphTests.scala @@ -20,44 +20,41 @@ class StaticCallGraphTests extends Jimple2CpgFixture { """ "should find that add is called by main" in { - cpg.method.name("add").caller.name.toSet shouldBe Set("main") + cpg.method.name("add").caller.name.toSetMutable shouldBe Set("main") } "should find that main calls add and others" in { - // out is a static field but it represents an object whose println call is dynamic - cpg.method.name("main").callee.name.filterNot(_.startsWith("")).toSet shouldBe Set( + cpg.method.name("main").callee.name.filterNot(_.startsWith("")).toSetMutable shouldBe Set( "add", "println" ) } "should find a set of outgoing calls for main" in { - cpg.method.name("main").call.code.toSet shouldBe + cpg.method.name("main").call.code.toSetMutable shouldBe Set( - "println($stack3)", "add(3, 3)", - "argc = @parameter0", + "$stack2.println($stack3)", "$stack2 = java.lang.System.out", - "argv = @parameter1", "$stack3 = add(3, 3)", "java.lang.System.out" ) } "should find one callsite for add" in { - cpg.method.name("add").callIn.code.toSet shouldBe Set("add(3, 3)") + cpg.method.name("add").callIn.code.toSetMutable shouldBe Set("add(3, 3)") } "should find that argument '1+2' is passed to parameter 'x'" in { - cpg.parameter.name("x").argument.code.toSet shouldBe Set("3") + cpg.parameter.name("x").argument.code.toSetMutable shouldBe Set("3") } "should allow traversing from argument to formal parameter" in { - cpg.argument.parameter.name.toSet should not be empty + cpg.argument.parameter.name.toSetMutable should not be empty } "should allow traversing from argument to call" in { - cpg.method.name("add").callIn.argument.inCall.name.toSet shouldBe Set("add") + cpg.method.name("add").callIn.argument.inCall.name.toSetMutable shouldBe Set("add") } } diff --git a/src/test/scala/com/github/plume/oss/testfixtures/Jimple2CpgFixture.scala b/src/test/scala/com/github/plume/oss/testfixtures/Jimple2CpgFixture.scala index 91c38dd5..01bdfe11 100644 --- a/src/test/scala/com/github/plume/oss/testfixtures/Jimple2CpgFixture.scala +++ b/src/test/scala/com/github/plume/oss/testfixtures/Jimple2CpgFixture.scala @@ -3,8 +3,8 @@ package com.github.plume.oss.testfixtures import com.github.plume.oss.{Jimple2Cpg, PlumeStatistics} import com.github.plume.oss.drivers.OverflowDbDriver import com.github.plume.oss.JavaCompiler.compileJava +import io.joern.x2cpg.testfixtures.{CodeToCpgFixture, LanguageFrontend} import io.shiftleft.codepropertygraph.Cpg -import io.shiftleft.semanticcpg.testfixtures.{CodeToCpgFixture, LanguageFrontend} import org.slf4j.LoggerFactory import java.io.{File, PrintWriter} @@ -13,8 +13,8 @@ import scala.util.Using class PlumeFrontend extends LanguageFrontend { - private val logger = LoggerFactory.getLogger(classOf[PlumeFrontend]) - val driver: OverflowDbDriver = new OverflowDbDriver(dataFlowCacheFile = None) + private val logger = LoggerFactory.getLogger(classOf[PlumeFrontend]) + val driver: OverflowDbDriver = new OverflowDbDriver(dataFlowCacheFile = None) override val fileSuffix: String = ".java" override def execute(sourceCodeFile: File): Cpg = { diff --git a/src/test/scala/com/github/plume/oss/testfixtures/PlumeDriverFixture.scala b/src/test/scala/com/github/plume/oss/testfixtures/PlumeDriverFixture.scala index 032cd597..c0cd531a 100644 --- a/src/test/scala/com/github/plume/oss/testfixtures/PlumeDriverFixture.scala +++ b/src/test/scala/com/github/plume/oss/testfixtures/PlumeDriverFixture.scala @@ -1,16 +1,16 @@ package com.github.plume.oss.testfixtures import com.github.plume.oss.drivers.IDriver -import com.github.plume.oss.passes.forkjoin.PlumeForkJoinParallelCpgPass.DiffGraphBuilder import io.shiftleft.codepropertygraph.generated.NodeTypes._ +import io.shiftleft.codepropertygraph.generated.PropertyNames._ import io.shiftleft.codepropertygraph.generated.nodes._ import io.shiftleft.codepropertygraph.generated.{Cpg, DispatchTypes, EdgeTypes} import io.shiftleft.passes.{DiffGraph, IntervalKeyPool} -import io.shiftleft.codepropertygraph.generated.PropertyNames._ import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll} -import overflowdb.{BatchedUpdate, DetachedNodeData, DetachedNodeGeneric, Node} +import overflowdb.BatchedUpdate.DiffGraphBuilder +import overflowdb.{BatchedUpdate, DetachedNodeGeneric, Node} import scala.jdk.CollectionConverters.IteratorHasAsScala import scala.language.postfixOps