Added support for Java source code (#251)

* Added support for Java source code

* Added option to disable source parser

* Updated GSQL client URL

.github/workflows/ci.yml

@@ -28,7 +28,7 @@ jobs:
         run: sbt coverage "NeoTest / test"
       - name: Run TigerGraph integration tests
         run: |
-          curl https://docs.tigergraph.com/tigergraph-server/current/gsql-shell/_attachments/gsql_client.jar --output gsql_client.jar
+          curl https://docs.tigergraph.com/tigergraph-server/3.5/gsql-shell/_attachments/gsql_client_server353.jar --output gsql_client.jar
           export GSQL_HOME=`pwd`/gsql_client.jar
           sbt coverage "TgTest / test"
 

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [1.2.7] - 2022-08-29
+
+### Added
+
+- Support for Java source file via Soot's source parser. This can be disabled via 
+`Jimple2Cpg.createCpg(parseJavaSource)`.
+
 ## [1.2.6] - 2022-05-31
 
 ### Changed

src/main/scala/com/github/plume/oss/Jimple2Cpg.scala

@@ -20,7 +20,7 @@ import soot.options.Options
 import soot.{G, PhaseOptions, Scene, SootClass}
 
 import java.io.{File => JFile}
-import java.nio.file.Paths
+import java.nio.file.{Files, Paths, StandardCopyOption}
 import scala.jdk.CollectionConverters.EnumerationHasAsScala
 
 object Jimple2Cpg {
@@ -41,7 +41,6 @@ object Jimple2Cpg {
     }
     filename
       .replace(codeDir + JFile.separator, "")
-      .replace(".class", "")
       .replace(JFile.separator, ".")
   }
 }
@@ -68,6 +67,7 @@ class Jimple2Cpg {
       rawSourceCodePath: String,
       referenceGraphOutputPath: Option[String] = None,
       driver: IDriver = new OverflowDbDriver(),
+      parseJavaSource: Boolean = true,
       sootOnlyBuild: Boolean = false
   ): Cpg = PlumeStatistics.time(
     PlumeStatistics.TIME_EXTRACTION, {
@@ -96,15 +96,26 @@ class Jimple2Cpg {
             driver.idInterval(30_001_001, Long.MaxValue)
           )
 
-        val sourceFileExtensions  = Set(".class", ".jimple")
+        val classFileExtensions   = Set(".class", ".jimple")
+        val sourceFileExtensions  = if (parseJavaSource) Set(".java") else Set.empty[String]
         val archiveFileExtensions = Set(".jar", ".war")
         // Load source files and unpack archives if necessary
         val sourceFileNames = if (sourceTarget == sourceCodeDir) {
           // Load all source files in a directory
-          loadSourceFiles(sourceCodeDir, sourceFileExtensions, archiveFileExtensions)
+          loadSourceFiles(
+            sourceCodeDir,
+            classFileExtensions,
+            archiveFileExtensions,
+            sourceFileExtensions
+          )
         } else {
           // Load single file that was specified
-          loadSourceFiles(sourceTarget, sourceFileExtensions, archiveFileExtensions)
+          loadSourceFiles(
+            sourceTarget,
+            classFileExtensions,
+            archiveFileExtensions,
+            sourceFileExtensions
+          )
         }
 
         logger.info(s"Loading ${sourceFileNames.size} program files")
@@ -180,21 +191,39 @@ class Jimple2Cpg {
     */
   private def loadSourceFiles(
       sourceCodePath: String,
-      sourceFileExtensions: Set[String],
-      archiveFileExtensions: Set[String]
+      classFileExtensions: Set[String],
+      archiveFileExtensions: Set[String],
+      sourceFileExtensions: Set[String]
   ): List[String] = {
     (
       extractSourceFilesFromArchive(sourceCodePath, archiveFileExtensions) ++
-        moveClassFiles(SourceFiles.determine(Set(sourceCodePath), sourceFileExtensions))
+        moveClassFiles(SourceFiles.determine(Set(sourceCodePath), classFileExtensions)) ++
+        moveSourceFiles(sourceFileExtensions, sourceCodePath)
     ).distinct
   }
 
+  private def moveSourceFiles(sourceFileExtensions: Set[String], basePath: String): List[String] = {
+    val tempPath = ProgramHandlingUtil.getUnpackingDir
+    SourceFiles
+      .determine(Set(basePath), sourceFileExtensions)
+      .map(f => {
+        val newPath = new JFile(tempPath.toFile, f.stripPrefix(basePath))
+        newPath.getParentFile.mkdirs()
+        Files.copy(Paths.get(f), newPath.toPath, StandardCopyOption.REPLACE_EXISTING)
+        newPath.getAbsolutePath
+      })
+  }
+
   private def loadClassesIntoSoot(sourceFileNames: Seq[String]): Seq[SootClass] = {
     val sootClasses = sourceFileNames
       .map(getQualifiedClassPath)
       .map { cp =>
-        Scene.v().addBasicClass(cp)
-        Scene.v().loadClassAndSupport(cp)
+        val cpNoSuffix = cp.stripSuffix(".java").stripSuffix(".class")
+        Scene.v().addBasicClass(cpNoSuffix)
+        if (cp.endsWith(".class"))
+          Scene.v().loadClassAndSupport(cpNoSuffix)
+        else
+          Scene.v().getSootClass(cpNoSuffix)
       }
     Scene.v().loadNecessaryClasses()
     sootClasses

src/main/scala/com/github/plume/oss/passes/base/AstCreationPass.scala

@@ -23,7 +23,10 @@ class AstCreationPass(
   override def runOnPart(builder: DiffGraphBuilder, part: String): Unit = {
     val qualifiedClassName = Jimple2Cpg.getQualifiedClassPath(part)
     try {
-      val sootClass = Scene.v().loadClassAndSupport(qualifiedClassName)
+      val cNameNoSuff = qualifiedClassName.stripSuffix(".class").stripSuffix(".java")
+      val sootClass =
+        if (qualifiedClassName.contains(".class")) Scene.v().loadClassAndSupport(cNameNoSuff)
+        else Scene.v().getSootClass(cNameNoSuff)
       sootClass.setApplicationClass()
       val localDiff = new io.joern.jimple2cpg.passes.AstCreator(part, sootClass, global).createAst()
       builder.absorb(localDiff)

src/test/resources/source_support/Bar.java

@@ -0,0 +1,7 @@
+public class Bar {
+
+    public static int test(int a) {
+        return a + 1;
+    }
+
+}

src/test/resources/source_support/Foo.java

@@ -0,0 +1,7 @@
+public class Foo {
+
+    public static void main(String[] args) {
+        Bar.test(2);
+    }
+
+}

src/test/scala/com/github/plume/oss/DiffTests.scala

@@ -55,7 +55,7 @@ class DiffTests extends AnyWordSpec with Matchers with BeforeAndAfterAll {
     driver.clear()
     driver.close()
     driver = new OverflowDbDriver(storage)
-    cpg = Some(new Jimple2Cpg().createCpg(sandboxDir.getAbsolutePath, driver = driver))
+    cpg = Some(new Jimple2Cpg().createCpg(sandboxDir.getAbsolutePath, driver = driver, parseJavaSource = false))
     sandboxDir.listFiles().foreach(_.delete())
   }
 
@@ -138,7 +138,7 @@ class DiffTests extends AnyWordSpec with Matchers with BeforeAndAfterAll {
     rewriteFileContents(bar, bar1)
     rewriteFileContents(foo, foo2)
     JavaCompiler.compileJava(foo, bar)
-    cpg = Some(new Jimple2Cpg().createCpg(sandboxDir.getAbsolutePath, driver = driver))
+    cpg = Some(new Jimple2Cpg().createCpg(sandboxDir.getAbsolutePath, driver = driver, parseJavaSource = false))
 
     // Check the correct numbers of nodes are present
     driver.propertyFromNodes(NodeTypes.TYPE_DECL).size shouldBe 7

src/test/scala/com/github/plume/oss/SourceSupportTests.scala

@@ -0,0 +1,63 @@
+package com.github.plume.oss
+
+import com.github.plume.oss.drivers.OverflowDbDriver
+import io.shiftleft.codepropertygraph.generated.{Cpg, NodeTypes, PropertyNames}
+import org.scalatest.BeforeAndAfterAll
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpec
+import org.slf4j.LoggerFactory
+
+import java.io.File
+import java.nio.file.{Files, Paths}
+
+class SourceSupportTests extends AnyWordSpec with Matchers with BeforeAndAfterAll {
+
+  private val logger = LoggerFactory.getLogger(classOf[SourceSupportTests])
+
+  private def getTestResource(dir: String): File = {
+    val resourceURL = getClass.getResource(dir)
+    if (resourceURL == null) throw new NullPointerException("Unable to obtain test resource")
+    new File(resourceURL.getFile)
+  }
+
+  private val foo: File = getTestResource("/source_support/Foo.java")
+  private val bar: File = getTestResource("/source_support/Bar.java")
+
+  private var driver           = new OverflowDbDriver()
+  private var cpg: Option[Cpg] = None
+  private val storage          = Some("./cpg_test.odb")
+  private val sandboxDir: File = Files.createTempDirectory("plume").toFile
+
+  override def beforeAll(): Unit = {
+    Paths.get(storage.get).toFile.delete()
+    driver.clear()
+    driver.close()
+    driver = new OverflowDbDriver(storage)
+    cpg = Some(new Jimple2Cpg().createCpg(foo.getParent, driver = driver))
+    sandboxDir.listFiles().foreach(_.delete())
+  }
+
+  override def afterAll(): Unit = {
+    driver.clear()
+    driver.close()
+    Paths.get(storage.get).toFile.delete()
+    driver.cacheConfig.dataFlowCacheFile match {
+      case Some(jsonFile) => new File(jsonFile.toFile.getAbsolutePath + ".lz4").delete()
+      case None           =>
+    }
+  }
+
+  "should accept Java source files" in {
+    val List(barM, fooM) = driver
+      .propertyFromNodes(NodeTypes.TYPE_DECL, PropertyNames.NAME, PropertyNames.FULL_NAME, PropertyNames.IS_EXTERNAL)
+      .filter(!_.getOrElse(PropertyNames.IS_EXTERNAL, true).toString.toBoolean)
+      .sortWith { case (x, y) =>
+        x(PropertyNames.FULL_NAME).toString < y(PropertyNames.FULL_NAME).toString
+      }
+    fooM.get(PropertyNames.FULL_NAME) shouldBe Some("Foo")
+    barM.get(PropertyNames.FULL_NAME) shouldBe Some("Bar")
+    fooM.get(PropertyNames.IS_EXTERNAL) shouldBe Some(false)
+    barM.get(PropertyNames.IS_EXTERNAL) shouldBe Some(false)
+  }
+
+}