diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2a08c98..d20f263a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: run: sbt coverage "NeoTest / test" - name: Run TigerGraph integration tests run: | - curl https://docs.tigergraph.com/tigergraph-server/current/gsql-shell/_attachments/gsql_client.jar --output gsql_client.jar + curl https://docs.tigergraph.com/tigergraph-server/3.5/gsql-shell/_attachments/gsql_client_server353.jar --output gsql_client.jar export GSQL_HOME=`pwd`/gsql_client.jar sbt coverage "TgTest / test" diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f0f8c18..7ac330a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [1.2.7] - 2022-08-29 + +### Added + +- Support for Java source file via Soot's source parser. This can be disabled via +`Jimple2Cpg.createCpg(parseJavaSource)`. + ## [1.2.6] - 2022-05-31 ### Changed diff --git a/src/main/scala/com/github/plume/oss/Jimple2Cpg.scala b/src/main/scala/com/github/plume/oss/Jimple2Cpg.scala index 3f0b15fe..dcb623d3 100644 --- a/src/main/scala/com/github/plume/oss/Jimple2Cpg.scala +++ b/src/main/scala/com/github/plume/oss/Jimple2Cpg.scala @@ -20,7 +20,7 @@ import soot.options.Options import soot.{G, PhaseOptions, Scene, SootClass} import java.io.{File => JFile} -import java.nio.file.Paths +import java.nio.file.{Files, Paths, StandardCopyOption} import scala.jdk.CollectionConverters.EnumerationHasAsScala object Jimple2Cpg { @@ -41,7 +41,6 @@ object Jimple2Cpg { } filename .replace(codeDir + JFile.separator, "") - .replace(".class", "") .replace(JFile.separator, ".") } } @@ -68,6 +67,7 @@ class Jimple2Cpg { rawSourceCodePath: String, referenceGraphOutputPath: Option[String] = None, driver: IDriver = new OverflowDbDriver(), + parseJavaSource: Boolean = true, sootOnlyBuild: Boolean = false ): Cpg = PlumeStatistics.time( PlumeStatistics.TIME_EXTRACTION, { @@ -96,15 +96,26 @@ class Jimple2Cpg { driver.idInterval(30_001_001, Long.MaxValue) ) - val sourceFileExtensions = Set(".class", ".jimple") + val classFileExtensions = Set(".class", ".jimple") + val sourceFileExtensions = if (parseJavaSource) Set(".java") else Set.empty[String] val archiveFileExtensions = Set(".jar", ".war") // Load source files and unpack archives if necessary val sourceFileNames = if (sourceTarget == sourceCodeDir) { // Load all source files in a directory - loadSourceFiles(sourceCodeDir, sourceFileExtensions, archiveFileExtensions) + loadSourceFiles( + sourceCodeDir, + classFileExtensions, + archiveFileExtensions, + sourceFileExtensions + ) } else { // Load single file that was specified - loadSourceFiles(sourceTarget, sourceFileExtensions, archiveFileExtensions) + loadSourceFiles( + sourceTarget, + classFileExtensions, + archiveFileExtensions, + sourceFileExtensions + ) } logger.info(s"Loading ${sourceFileNames.size} program files") @@ -180,21 +191,39 @@ class Jimple2Cpg { */ private def loadSourceFiles( sourceCodePath: String, - sourceFileExtensions: Set[String], - archiveFileExtensions: Set[String] + classFileExtensions: Set[String], + archiveFileExtensions: Set[String], + sourceFileExtensions: Set[String] ): List[String] = { ( extractSourceFilesFromArchive(sourceCodePath, archiveFileExtensions) ++ - moveClassFiles(SourceFiles.determine(Set(sourceCodePath), sourceFileExtensions)) + moveClassFiles(SourceFiles.determine(Set(sourceCodePath), classFileExtensions)) ++ + moveSourceFiles(sourceFileExtensions, sourceCodePath) ).distinct } + private def moveSourceFiles(sourceFileExtensions: Set[String], basePath: String): List[String] = { + val tempPath = ProgramHandlingUtil.getUnpackingDir + SourceFiles + .determine(Set(basePath), sourceFileExtensions) + .map(f => { + val newPath = new JFile(tempPath.toFile, f.stripPrefix(basePath)) + newPath.getParentFile.mkdirs() + Files.copy(Paths.get(f), newPath.toPath, StandardCopyOption.REPLACE_EXISTING) + newPath.getAbsolutePath + }) + } + private def loadClassesIntoSoot(sourceFileNames: Seq[String]): Seq[SootClass] = { val sootClasses = sourceFileNames .map(getQualifiedClassPath) .map { cp => - Scene.v().addBasicClass(cp) - Scene.v().loadClassAndSupport(cp) + val cpNoSuffix = cp.stripSuffix(".java").stripSuffix(".class") + Scene.v().addBasicClass(cpNoSuffix) + if (cp.endsWith(".class")) + Scene.v().loadClassAndSupport(cpNoSuffix) + else + Scene.v().getSootClass(cpNoSuffix) } Scene.v().loadNecessaryClasses() sootClasses diff --git a/src/main/scala/com/github/plume/oss/passes/base/AstCreationPass.scala b/src/main/scala/com/github/plume/oss/passes/base/AstCreationPass.scala index 61822921..4107612c 100644 --- a/src/main/scala/com/github/plume/oss/passes/base/AstCreationPass.scala +++ b/src/main/scala/com/github/plume/oss/passes/base/AstCreationPass.scala @@ -23,7 +23,10 @@ class AstCreationPass( override def runOnPart(builder: DiffGraphBuilder, part: String): Unit = { val qualifiedClassName = Jimple2Cpg.getQualifiedClassPath(part) try { - val sootClass = Scene.v().loadClassAndSupport(qualifiedClassName) + val cNameNoSuff = qualifiedClassName.stripSuffix(".class").stripSuffix(".java") + val sootClass = + if (qualifiedClassName.contains(".class")) Scene.v().loadClassAndSupport(cNameNoSuff) + else Scene.v().getSootClass(cNameNoSuff) sootClass.setApplicationClass() val localDiff = new io.joern.jimple2cpg.passes.AstCreator(part, sootClass, global).createAst() builder.absorb(localDiff) diff --git a/src/test/resources/source_support/Bar.java b/src/test/resources/source_support/Bar.java new file mode 100644 index 00000000..8751a7f4 --- /dev/null +++ b/src/test/resources/source_support/Bar.java @@ -0,0 +1,7 @@ +public class Bar { + + public static int test(int a) { + return a + 1; + } + +} \ No newline at end of file diff --git a/src/test/resources/source_support/Foo.java b/src/test/resources/source_support/Foo.java new file mode 100644 index 00000000..100506a9 --- /dev/null +++ b/src/test/resources/source_support/Foo.java @@ -0,0 +1,7 @@ +public class Foo { + + public static void main(String[] args) { + Bar.test(2); + } + +} \ No newline at end of file diff --git a/src/test/scala/com/github/plume/oss/DiffTests.scala b/src/test/scala/com/github/plume/oss/DiffTests.scala index 4d2598c7..81fd1523 100644 --- a/src/test/scala/com/github/plume/oss/DiffTests.scala +++ b/src/test/scala/com/github/plume/oss/DiffTests.scala @@ -55,7 +55,7 @@ class DiffTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { driver.clear() driver.close() driver = new OverflowDbDriver(storage) - cpg = Some(new Jimple2Cpg().createCpg(sandboxDir.getAbsolutePath, driver = driver)) + cpg = Some(new Jimple2Cpg().createCpg(sandboxDir.getAbsolutePath, driver = driver, parseJavaSource = false)) sandboxDir.listFiles().foreach(_.delete()) } @@ -138,7 +138,7 @@ class DiffTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { rewriteFileContents(bar, bar1) rewriteFileContents(foo, foo2) JavaCompiler.compileJava(foo, bar) - cpg = Some(new Jimple2Cpg().createCpg(sandboxDir.getAbsolutePath, driver = driver)) + cpg = Some(new Jimple2Cpg().createCpg(sandboxDir.getAbsolutePath, driver = driver, parseJavaSource = false)) // Check the correct numbers of nodes are present driver.propertyFromNodes(NodeTypes.TYPE_DECL).size shouldBe 7 diff --git a/src/test/scala/com/github/plume/oss/SourceSupportTests.scala b/src/test/scala/com/github/plume/oss/SourceSupportTests.scala new file mode 100644 index 00000000..904ebe39 --- /dev/null +++ b/src/test/scala/com/github/plume/oss/SourceSupportTests.scala @@ -0,0 +1,63 @@ +package com.github.plume.oss + +import com.github.plume.oss.drivers.OverflowDbDriver +import io.shiftleft.codepropertygraph.generated.{Cpg, NodeTypes, PropertyNames} +import org.scalatest.BeforeAndAfterAll +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec +import org.slf4j.LoggerFactory + +import java.io.File +import java.nio.file.{Files, Paths} + +class SourceSupportTests extends AnyWordSpec with Matchers with BeforeAndAfterAll { + + private val logger = LoggerFactory.getLogger(classOf[SourceSupportTests]) + + private def getTestResource(dir: String): File = { + val resourceURL = getClass.getResource(dir) + if (resourceURL == null) throw new NullPointerException("Unable to obtain test resource") + new File(resourceURL.getFile) + } + + private val foo: File = getTestResource("/source_support/Foo.java") + private val bar: File = getTestResource("/source_support/Bar.java") + + private var driver = new OverflowDbDriver() + private var cpg: Option[Cpg] = None + private val storage = Some("./cpg_test.odb") + private val sandboxDir: File = Files.createTempDirectory("plume").toFile + + override def beforeAll(): Unit = { + Paths.get(storage.get).toFile.delete() + driver.clear() + driver.close() + driver = new OverflowDbDriver(storage) + cpg = Some(new Jimple2Cpg().createCpg(foo.getParent, driver = driver)) + sandboxDir.listFiles().foreach(_.delete()) + } + + override def afterAll(): Unit = { + driver.clear() + driver.close() + Paths.get(storage.get).toFile.delete() + driver.cacheConfig.dataFlowCacheFile match { + case Some(jsonFile) => new File(jsonFile.toFile.getAbsolutePath + ".lz4").delete() + case None => + } + } + + "should accept Java source files" in { + val List(barM, fooM) = driver + .propertyFromNodes(NodeTypes.TYPE_DECL, PropertyNames.NAME, PropertyNames.FULL_NAME, PropertyNames.IS_EXTERNAL) + .filter(!_.getOrElse(PropertyNames.IS_EXTERNAL, true).toString.toBoolean) + .sortWith { case (x, y) => + x(PropertyNames.FULL_NAME).toString < y(PropertyNames.FULL_NAME).toString + } + fooM.get(PropertyNames.FULL_NAME) shouldBe Some("Foo") + barM.get(PropertyNames.FULL_NAME) shouldBe Some("Bar") + fooM.get(PropertyNames.IS_EXTERNAL) shouldBe Some(false) + barM.get(PropertyNames.IS_EXTERNAL) shouldBe Some(false) + } + +}