diff --git a/CHANGELOG.md b/CHANGELOG.md index d347b86a..b2f0e2f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [1.1.5] - 2022-03-18 + +### Changed + +- Only adding changed files to Soot to improve performance. +- Simplified `PlumeStatistics` file related changes to only those class/methods changed. +- `PlumeDynamicCallLinker` now extends `SimpleCpgPass`. + ## [1.1.4] - 2022-03-18 ### Added diff --git a/build.sbt b/build.sbt index 114f83d9..ed47cee3 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,7 @@ name := "Plume" inThisBuild( List( organization := "com.github.plume-oss", - version := "1.1.4", + version := "1.1.5", scalaVersion := "2.13.7", crossScalaVersions := Seq("2.13.7", "3.1.1"), resolvers ++= Seq( diff --git a/src/main/scala/com/github/plume/oss/Jimple2Cpg.scala b/src/main/scala/com/github/plume/oss/Jimple2Cpg.scala index 4c6209c9..c7045680 100644 --- a/src/main/scala/com/github/plume/oss/Jimple2Cpg.scala +++ b/src/main/scala/com/github/plume/oss/Jimple2Cpg.scala @@ -111,31 +111,22 @@ class Jimple2Cpg { logger.debug(s"Source files are: $sourceFileNames") // Load classes into Soot - val allSootClasses = loadClassesIntoSoot(sourceFileNames) if (sootOnlyBuild) return cpg - val codeToProcess = new PlumeDiffPass(sourceFileNames, driver).createAndApply() + val codeToProcess = new PlumeDiffPass(sourceFileNames, driver).createAndApply() + val allSootClasses = loadClassesIntoSoot(codeToProcess) // Record statistics for experimental purposes - PlumeStatistics.setMetric(PlumeStatistics.PROGRAM_CLASSES, allSootClasses.size) - PlumeStatistics.setMetric( - PlumeStatistics.PROGRAM_METHODS, - allSootClasses.map(_.getMethodCount).sum - ) if (codeToProcess.isEmpty) { logger.info("No files have changed since last update. Exiting...") return cpg } else { - val numChangedMethods = codeToProcess - .map(getQualifiedClassPath) - .map(Scene.v().getSootClass) - .map(_.getMethodCount) - .sum + val numChangedMethods = allSootClasses.map(_.getMethodCount).sum logger.info( s"Processing ${codeToProcess.size} new or changed program files " + s"($numChangedMethods new or changed methods)" ) - PlumeStatistics.setMetric(PlumeStatistics.CHANGED_CLASSES, codeToProcess.size) - PlumeStatistics.setMetric(PlumeStatistics.CHANGED_METHODS, numChangedMethods) + PlumeStatistics.setMetric(PlumeStatistics.PROGRAM_CLASSES, allSootClasses.size) + PlumeStatistics.setMetric(PlumeStatistics.PROGRAM_METHODS, numChangedMethods) } // After the diff pass any changed types are removed. Remaining types should be black listed to avoid duplicates @@ -197,7 +188,7 @@ class Jimple2Cpg { ).distinct } - private def loadClassesIntoSoot(sourceFileNames: List[String]): Seq[SootClass] = { + private def loadClassesIntoSoot(sourceFileNames: Seq[String]): Seq[SootClass] = { val sootClasses = sourceFileNames .map(getQualifiedClassPath) .map { cp => diff --git a/src/main/scala/com/github/plume/oss/PlumeStatistics.scala b/src/main/scala/com/github/plume/oss/PlumeStatistics.scala index 2b5b1e55..85f395a1 100644 --- a/src/main/scala/com/github/plume/oss/PlumeStatistics.scala +++ b/src/main/scala/com/github/plume/oss/PlumeStatistics.scala @@ -11,7 +11,7 @@ object PlumeStatistics extends Enumeration { type PlumeStatistic = Value val TIME_OPEN_DRIVER, TIME_CLOSE_DRIVER, TIME_EXTRACTION, TIME_REACHABLE_BY_QUERYING, - CHANGED_CLASSES, CHANGED_METHODS, PROGRAM_CLASSES, PROGRAM_METHODS = Value + PROGRAM_CLASSES, PROGRAM_METHODS = Value private val statistics: mutable.Map[PlumeStatistic, Long] = PlumeStatistics.values.map((_, 0L)).to(collection.mutable.Map) diff --git a/src/main/scala/com/github/plume/oss/drivers/GremlinDriver.scala b/src/main/scala/com/github/plume/oss/drivers/GremlinDriver.scala index 38eec386..90116cf5 100644 --- a/src/main/scala/com/github/plume/oss/drivers/GremlinDriver.scala +++ b/src/main/scala/com/github/plume/oss/drivers/GremlinDriver.scala @@ -356,7 +356,7 @@ abstract class GremlinDriver(txMax: Int = 50) extends IDriver { }).collect { case x: String => x }.foreach { dstFullName => dstNodeMap.get(dstFullName) match { case Some(dstId: Any) if !exists(srcId, dstId.toString.toLong, edgeType) => - g.V(typedNodeId(srcId)).addE(edgeType).to(__.V(dstId)).iterate() + g().V(typedNodeId(srcId)).addE(edgeType).to(__.V(dstId)).iterate() case _ => } } @@ -378,7 +378,7 @@ abstract class GremlinDriver(txMax: Int = 50) extends IDriver { if (dstFullName != null) { methodFullNameToNode.get(dstFullName) match { case Some(dstId) if !exists(srcId, dstId.toString.toLong, EdgeTypes.CALL) => - g.V(typedNodeId(srcId)).addE(EdgeTypes.CALL).to(__.V(dstId)).iterate() + g().V(typedNodeId(srcId)).addE(EdgeTypes.CALL).to(__.V(dstId)).iterate() case _ => } } diff --git a/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala b/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala index 073ce6a4..c4a09945 100644 --- a/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala +++ b/src/main/scala/com/github/plume/oss/drivers/OverflowDbDriver.scala @@ -361,7 +361,7 @@ final case class OverflowDbDriver( results .map { x => x.table } .foreach { t: ResultTable => - t.keys.foreach { n: StoredNode => + t.keys().foreach { n: StoredNode => t.get(n) match { case Some(v) => tab.put(n.id(), v.map(SerialReachableByResult.apply)) diff --git a/src/main/scala/com/github/plume/oss/passes/PlumeConcurrentCpgPass.scala b/src/main/scala/com/github/plume/oss/passes/PlumeConcurrentCpgPass.scala index 43e17575..e77a8854 100644 --- a/src/main/scala/com/github/plume/oss/passes/PlumeConcurrentCpgPass.scala +++ b/src/main/scala/com/github/plume/oss/passes/PlumeConcurrentCpgPass.scala @@ -122,7 +122,7 @@ object PlumeConcurrentWriter { class PlumeConcurrentWriter( driver: IDriver, cpg: Cpg, - baseLogger: Logger = LoggerFactory.getLogger(classOf[CpgPass]), + baseLogger: Logger = LoggerFactory.getLogger(classOf[PlumeConcurrentWriter]), keyPool: Option[KeyPool] = None, mdc: java.util.Map[String, String], setDiffT: Int => Int diff --git a/src/main/scala/com/github/plume/oss/passes/callgraph/PlumeDynamicCallLinker.scala b/src/main/scala/com/github/plume/oss/passes/callgraph/PlumeDynamicCallLinker.scala index dc3fcc7b..5ba6935b 100644 --- a/src/main/scala/com/github/plume/oss/passes/callgraph/PlumeDynamicCallLinker.scala +++ b/src/main/scala/com/github/plume/oss/passes/callgraph/PlumeDynamicCallLinker.scala @@ -3,7 +3,7 @@ package com.github.plume.oss.passes.callgraph import io.shiftleft.codepropertygraph.Cpg import io.shiftleft.codepropertygraph.generated.nodes.{Call, Method, TypeDecl} import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EdgeTypes, PropertyNames} -import io.shiftleft.passes.{CpgPass, DiffGraph} +import io.shiftleft.passes.SimpleCpgPass import io.shiftleft.semanticcpg.language._ import org.slf4j.{Logger, LoggerFactory} import overflowdb.{NodeDb, NodeRef} @@ -22,7 +22,7 @@ import scala.jdk.CollectionConverters.{CollectionHasAsScala, IteratorHasAsScala} * SAFEDISPATCH: Securing C++ Virtual Calls from Memory Corruption Attacks. * 10.14722/ndss.2014.23287. */ -class PlumeDynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { +class PlumeDynamicCallLinker(cpg: Cpg) extends SimpleCpgPass(cpg) { import PlumeDynamicCallLinker._ // Used to track potential method candidates for a given method fullname. Since our method full names contain the type @@ -47,10 +47,9 @@ class PlumeDynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { /** Main method of enhancement - to be implemented by child class */ - override def run(): Iterator[DiffGraph] = { - val dstGraph = DiffGraph.newBuilder + override def run(dstGraph: DiffGraphBuilder): Unit = { // Perform early stopping in the case of no virtual calls - if (!cpg.call.exists(_.dispatchType == DispatchTypes.DYNAMIC_DISPATCH)) return Iterator() + if (!cpg.call.exists(_.dispatchType == DispatchTypes.DYNAMIC_DISPATCH)) return else initMaps() // ValidM maps class C and method name N to the set of // func ptrs implementing N for C and its subclasses @@ -75,8 +74,6 @@ class PlumeDynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { throw new RuntimeException(exception) } } - - Iterator(dstGraph.build()) } /** Recursively returns all the sub-types of the given type declaration. Does not account for circular hierarchies. @@ -118,7 +115,7 @@ class PlumeDynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { } } - private def linkDynamicCall(call: Call, dstGraph: DiffGraph.Builder): Unit = { + private def linkDynamicCall(call: Call, dstGraph: DiffGraphBuilder): Unit = { validM.get(call.methodFullName) match { case Some(tgts) => val callsOut = call.callOut.fullName.toSetImmutable @@ -129,7 +126,7 @@ class PlumeDynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { cpg.method.fullNameExact(destMethod).headOption } if (tgtM.isDefined && !callsOut.contains(tgtM.get.fullName)) { - dstGraph.addEdgeInOriginal(call, tgtM.get, EdgeTypes.CALL) + dstGraph.addEdge(call, tgtM.get, EdgeTypes.CALL) } else { fallbackToStaticResolution(call, dstGraph) } @@ -141,9 +138,9 @@ class PlumeDynamicCallLinker(cpg: Cpg) extends CpgPass(cpg) { /** In the case where the method isn't an internal method and cannot be resolved by crawling TYPE_DECL nodes it can be * resolved from the map of external methods. */ - private def fallbackToStaticResolution(call: Call, dstGraph: DiffGraph.Builder): Unit = { + private def fallbackToStaticResolution(call: Call, dstGraph: DiffGraphBuilder): Unit = { methodMap.get(call.methodFullName) match { - case Some(tgtM) => dstGraph.addEdgeInOriginal(call, tgtM, EdgeTypes.CALL) + case Some(tgtM) => dstGraph.addEdge(call, tgtM, EdgeTypes.CALL) case None => printLinkingError(call) } }