Caught Plume up to Latest Changes Incl. Data Flow Analysis Fix (#206)

* Caught Plume up to latest changes

* Fixed definition error

* Added more defaults

* Handling defaults when writing

* Standardized calls to getPropertyDefault

* Added more defaults

* Fixed missed defaults causing issues

* Removed line numbers from those that dont use em

* Fixed some default issue again..

* Removed unnecessary plugins

* Removed deprecated gradle calls

* Removed properties not handled by tigergraph

* TigerGraph bug fixed

* Re-added neptune coverage

* Added stacktrace to see what's up with windows

* Let's hope Java 16 path works in windows

* Let's try this plugin

* Let's see if gradle upgrade fixes it

* Fixed scalatests running on Gradle 7.2

* Let's see if this plugin works

CHANGELOG.md

@@ -9,11 +9,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Changed
 
-- Upgrade ShiftLeft dependencies to 1.3.236
+- Upgrade ShiftLeft dependencies to 1.3.314
+- Upgrade Gradle to 7.2  
 - Removed `Binding` vertices
 - Can now handle new type arguments API of domain classes
 - `Extractor::project` now takes an optional boolean to disable reaching defs calculation
 - `Extractor::projectReachingDefs` now calculates reaching defs separately
+- Removed `IDriver::getProgramStructure` as it's not used by the core extractor
+- `VertexMapper` now handles new default property system 
 
 ## [0.5.12] - 2021-07-30
 

cpgconv/build.gradle

@@ -11,6 +11,7 @@ repositories {
 dependencies {
     implementation "io.shiftleft:codepropertygraph_2.13:$shiftleftVersion"
     implementation "io.shiftleft:semanticcpg_2.13:$shiftleftVersion"
+    implementation "io.shiftleft:dataflowengineoss_2.13:$shiftleftVersion"
 }
 
 group = "io.github.plume-oss"

cpgconv/src/main/scala/io/github/plume/oss/Traversals.scala

@@ -3,7 +3,8 @@ package io.github.plume.oss
 import java.util
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.{EdgeTypes, nodes}
-import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, File, MetaData, NamespaceBlock, StoredNode, TypeDecl}
+import io.shiftleft.codepropertygraph.generated.nodes.{AstNode, File, MetaData, Method, NamespaceBlock, StoredNode, TypeDecl}
+import io.shiftleft.dataflowengineoss.passes.reachingdef.{ReachingDefProblem, ReachingDefTransferFunction}
 import io.shiftleft.semanticcpg.language._
 import overflowdb.{Edge, Graph}
 
@@ -76,4 +77,13 @@ object Traversals {
     nodesToDelete.foreach(v => graph.remove(v))
   }
 
+  def maxNumberOfDefsFromAMethod(graph: Graph): Int = {
+    Cpg(graph).method.map(ReachingDefProblem.create)
+      .map(_.transferFunction.asInstanceOf[ReachingDefTransferFunction].gen.foldLeft(0)(_ + _._2.size))
+      .maxOption match {
+      case Some(value) => value
+      case None => 0
+    }
+  }
+
 }

gradle.properties

@@ -10,7 +10,7 @@ neo4jDriverVersion=4.2.0
 tigerGraphVersion=3.1.0
 khttpVersion=1.0.0
 jacksonVersion=2.12.0
-shiftleftVersion=1.3.236
+shiftleftVersion=1.3.314
 sootVersion=4.2.1
 lz4Version=1.7.1
 cache2kVersion=2.0.0.Final

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,5 @@
-#Mon Dec 21 21:42:58 CET 2020
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.7.1-all.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip
 zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists

gradlew

@@ -82,6 +82,7 @@ esac
 
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
+
 # Determine the Java command to use to start the JVM.
 if [ -n "$JAVA_HOME" ] ; then
     if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
@@ -129,6 +130,7 @@ fi
 if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
     JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath

gradlew.bat

@@ -40,7 +40,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome
 
 set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
+if "%ERRORLEVEL%" == "0" goto execute
 
 echo.
 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
@@ -54,7 +54,7 @@ goto fail
 set JAVA_HOME=%JAVA_HOME:"=%
 set JAVA_EXE=%JAVA_HOME%/bin/java.exe
 
-if exist "%JAVA_EXE%" goto init
+if exist "%JAVA_EXE%" goto execute
 
 echo.
 echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
@@ -64,28 +64,14 @@ echo location of your Java installation.
 
 goto fail
 
-:init
-@rem Get command-line arguments, handling Windows variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-
 :execute
 @rem Setup the command line
 
 set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 
+
 @rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
 
 :end
 @rem End local scope for the variables with windows NT shell

plume/build.gradle

@@ -6,14 +6,12 @@ buildscript {
 
 plugins {
     id "java"
-    id "maven"
     id "maven-publish"
     id "jacoco"
     id "org.jetbrains.dokka" version "$dokkaVersion"
     id "com.eden.orchidPlugin" version "$orchidVersion"
     id "com.avast.gradle.docker-compose" version "$avastDockerVersion"
     id "org.jetbrains.kotlin.jvm" version "$kotlinVersion"
-    id "com.jfrog.bintray" version "$bintrayVersion"
 }
 
 repositories {
@@ -37,13 +35,13 @@ dependencies {
     implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:$kotlinxVersion"
 
     // Logging
-    compile "org.apache.logging.log4j:log4j-api:$log4jVersion"
-    compile "org.apache.logging.log4j:log4j-core:$log4jVersion"
-    compile "org.apache.logging.log4j:log4j-slf4j-impl:$log4jVersion"
-    compile "org.slf4j:jul-to-slf4j:$slf4jVersion" // JUL bridge
-    compile "org.slf4j:jcl-over-slf4j:$slf4jVersion" // Apache Commons Logging (JCL) bridge
-    compile "org.slf4j:log4j-over-slf4j:$slf4jVersion" // log4j1.2 bridge
-    compile "me.tongfei:progressbar:$progressBarVersion" // simple progress bar
+    implementation "org.apache.logging.log4j:log4j-api:$log4jVersion"
+    implementation "org.apache.logging.log4j:log4j-core:$log4jVersion"
+    implementation "org.apache.logging.log4j:log4j-slf4j-impl:$log4jVersion"
+    implementation "org.slf4j:jul-to-slf4j:$slf4jVersion" // JUL bridge
+    implementation "org.slf4j:jcl-over-slf4j:$slf4jVersion" // Apache Commons Logging (JCL) bridge
+    implementation "org.slf4j:log4j-over-slf4j:$slf4jVersion" // log4j1.2 bridge
+    implementation "me.tongfei:progressbar:$progressBarVersion" // simple progress bar
 
     // Core dependencies
     implementation "org.apache.tinkerpop:gremlin-core:$gremlinVersion"
@@ -66,10 +64,10 @@ dependencies {
     subProject project(":cpgconv")
 
     // Orchid Docs
-    orchidRuntime "io.github.javaeden.orchid:OrchidDocs:$orchidVersion"
-    orchidRuntime "io.github.javaeden.orchid:OrchidKotlindoc:$orchidVersion"
-    orchidRuntime "io.github.javaeden.orchid:OrchidPluginDocs:$orchidVersion"
-    orchidRuntime "io.github.javaeden.orchid:OrchidGithub:$orchidVersion"
+    orchidRuntimeOnly "io.github.javaeden.orchid:OrchidDocs:$orchidVersion"
+    orchidRuntimeOnly "io.github.javaeden.orchid:OrchidKotlindoc:$orchidVersion"
+    orchidRuntimeOnly "io.github.javaeden.orchid:OrchidPluginDocs:$orchidVersion"
+    orchidRuntimeOnly "io.github.javaeden.orchid:OrchidGithub:$orchidVersion"
 
     // Testing
     testImplementation "org.junit.jupiter:junit-jupiter-engine:$junitVersion"
@@ -202,11 +200,11 @@ jacoco { toolVersion = "$jacocoVersion" }
 
 jacocoTestReport {
     reports {
-        xml.enabled true
+        xml.required.set(true)
         xml.destination file("${buildDir}/reports/jacoco/report.xml")
-        html.enabled true
+        html.required.set(true)
         html.destination file("${buildDir}/reports/jacoco")
-        csv.enabled false
+        csv.required.set(false)
     }
     executionData test, extractorTest, tinkerGraphIntTest, overflowDbIntTest, janusGraphIntTest, tigerGraphIntTest,
             neo4jIntTest, neptuneIntTest