Better temp storage handling (#245)

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [Unreleased]
+
+### Changed
+
+- Delete the unpacking dir after clean and generating a new one as needed if a successive analysis occurs.
+- Making the reference DB `None` by default to avoid accidentally generating large amounts of stored graphs.
+
 ## [1.1.2] - 2022-03-16
 
 ### Added

src/main/scala/com/github/plume/oss/Jimple2Cpg.scala

@@ -44,7 +44,7 @@ object Jimple2Cpg {
     * @return the correctly formatted class path.
     */
   def getQualifiedClassPath(filename: String): String = {
-    val codePath = ProgramHandlingUtil.TEMP_DIR
+    val codePath = ProgramHandlingUtil.getUnpackingDir
     val codeDir: String = if (codePath.toFile.isDirectory) {
       codePath.toAbsolutePath.normalize.toString
     } else {
@@ -68,7 +68,7 @@ class Jimple2Cpg {
   /** Creates a CPG from Jimple.
     *
     * @param rawSourceCodePath  The path to the Jimple code or code that can be transformed into Jimple.
-    * @param outputPath         The path to store the CPG. If `outputPath` is `None`, the CPG is created in-memory.
+    * @param referenceGraphOutputPath         The path to store the CPG. If `outputPath` is `None`, the CPG is created in-memory.
     * @param driver             The driver used to interact with the backend database.
     * @param sootOnlyBuild      (Experimental) Used to determine how many resources are used when only loading files
     *                           into Soot.
@@ -77,7 +77,7 @@ class Jimple2Cpg {
     */
   def createCpg(
       rawSourceCodePath: String,
-      outputPath: Option[String] = Option(JFile.createTempFile("plume-", ".odb").getAbsolutePath),
+      referenceGraphOutputPath: Option[String] = None,
       driver: IDriver = new OverflowDbDriver(),
       sootOnlyBuild: Boolean = false
   ): Cpg = PlumeStatistics.time(
@@ -96,7 +96,7 @@ class Jimple2Cpg {
         }
 
         configureSoot()
-        val cpg = newEmptyCpg(outputPath)
+        val cpg = newEmptyCpg(referenceGraphOutputPath)
 
         val metaDataKeyPool = new IncrementalKeyPool(1, 100, driver.idInterval(1, 100))
         val typesKeyPool    = new IncrementalKeyPool(101, 2_000_100, driver.idInterval(101, 2_000_100))
@@ -257,7 +257,7 @@ class Jimple2Cpg {
     Options.v().set_app(false)
     Options.v().set_whole_program(false)
     // make sure classpath is configured correctly
-    Options.v().set_soot_classpath(ProgramHandlingUtil.TEMP_DIR.toString)
+    Options.v().set_soot_classpath(ProgramHandlingUtil.getUnpackingDir.toAbsolutePath.toString)
     Options.v().set_prepend_classpath(true)
     // keep debugging info
     Options.v().set_keep_line_number(true)

src/main/scala/com/github/plume/oss/util/ProgramHandlingUtil.scala

@@ -1,6 +1,7 @@
 package com.github.plume.oss.util
 
 import io.joern.x2cpg.SourceFiles
+import org.apache.commons.io.FileUtils
 import org.objectweb.asm.ClassReader.SKIP_CODE
 import org.objectweb.asm.{ClassReader, ClassVisitor, Opcodes}
 import org.slf4j.LoggerFactory
@@ -19,7 +20,7 @@ object ProgramHandlingUtil {
 
   /** The temporary directory used to unpack class files to.
     */
-  val TEMP_DIR: Path = Files.createTempDirectory("plume")
+  private var TEMP_DIR: Option[Path] = Some(Files.createTempDirectory("plume"))
 
   logger.debug(s"Using temporary folder at $TEMP_DIR")
 
@@ -29,6 +30,7 @@ object ProgramHandlingUtil {
     * @return the list of class files at their new locations.
     */
   def moveClassFiles(files: List[String]): List[String] = {
+    val dir                      = getUnpackingDir
     var destPath: Option[String] = None
 
     sealed class ClassPathVisitor extends ClassVisitor(Opcodes.ASM8) {
@@ -40,7 +42,7 @@ object ProgramHandlingUtil {
           superName: String,
           interfaces: Array[String]
       ): Unit = {
-        destPath = Some(TEMP_DIR.toAbsolutePath.toString + File.separator + name + ".class")
+        destPath = Some(dir.toAbsolutePath.toString + File.separator + name + ".class")
       }
     }
 
@@ -61,12 +63,26 @@ object ProgramHandlingUtil {
     }
   }
 
+  /** Returns the temporary directory used to unpack and analyze projects in.
+    * @return the path pointing to the directory.
+    */
+  def getUnpackingDir: Path = {
+    TEMP_DIR match {
+      case None =>
+        val p = Files.createTempDirectory("plume")
+        TEMP_DIR = Some(p)
+        p
+      case Some(dir) => dir
+    }
+  }
+
   /** Unzips a ZIP file into a sequence of files. All files unpacked are deleted at the end of CPG construction.
     *
     * @param zf             The ZIP file to extract.
     * @param sourceCodePath The project root path to unpack to.
     */
   def unzipArchive(zf: ZipFile, sourceCodePath: String): Try[Seq[String]] = scala.util.Try {
+    val dir = getUnpackingDir
     Using.resource(zf) { zip: ZipFile =>
       // Copy zipped files across
       zip
@@ -77,11 +93,9 @@ object ProgramHandlingUtil {
           val sourceCodePathFile = new File(sourceCodePath)
           // Handle the case if the input source code path is an archive itself
           val destFile = if (sourceCodePathFile.isDirectory) {
-            new File(TEMP_DIR.toAbsolutePath.toString + File.separator + entry.getName)
+            new File(dir.toAbsolutePath.toString + File.separator + entry.getName)
           } else {
-            new File(
-              TEMP_DIR.toAbsolutePath.toString + File.separator + entry.getName
-            )
+            new File(dir.toAbsolutePath.toString + File.separator + entry.getName)
           }
           // dirName accounts for nested directories as a result of JAR package structure
           val dirName = destFile.getAbsolutePath
@@ -129,8 +143,6 @@ object ProgramHandlingUtil {
 
   /** Removes all files in the temporary unpacking directory.
     */
-  def clean(): Unit = {
-    Files.walk(TEMP_DIR).filter(Files.isRegularFile(_)).forEach(p => p.toFile.delete())
-  }
+  def clean(): Unit = FileUtils.deleteDirectory(getUnpackingDir.toFile)
 
 }

src/test/scala/com/github/plume/oss/unpacking/JarUnpackingTests.scala

@@ -30,7 +30,7 @@ class JarUnpackingTests extends AnyWordSpec with Matchers with BeforeAndAfterAll
     Try(getClass.getResource("/unpacking")) match {
       case Success(x) =>
         val fs = File(x.getPath).toDirectory.walk.toSeq
-        val cs = File(ProgramHandlingUtil.TEMP_DIR.toString).toDirectory.walk.toSeq
+        val cs = File(ProgramHandlingUtil.getUnpackingDir.toString).toDirectory.walk.toSeq
         fs.filter(_.name.contains(".jar")).map(_.name).toList shouldBe List("HelloWorld.jar")
         cs.count(_.name.contains(".class")) shouldBe 0
       case Failure(x: Throwable) =>