From 502a638907a9a46182e599178b602864fb65a7e7 Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Tue, 15 Feb 2022 21:12:15 +0200 Subject: [PATCH] Fixed some TigerGraphDriver bugs (#230) --- CHANGELOG.md | 10 +++- .../github/plume/oss/drivers/IDriver.scala | 10 ++++ .../plume/oss/drivers/TigerGraphDriver.scala | 50 +++++++++++++++++-- 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f6be0b7..e60be3fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [1.0.11] - 2022-02-15 + +### Fixed + +- `TigerGraphDriver` default transaction limit was 3 instead of 30 seconds. +- AST linking in `TigerGraphDriver` did not escape `[]` or `_` but now does. +- Diversified error handling on exceptions on `TigerGraphDriver` HTTP requests. + ## [1.0.10] - 2022-02-15 ### Added -- `Jimple2Cpg::createCpg` can now enable an experimental "Soot only" build +- `Jimple2Cpg::createCpg` can now enable an experimental "Soot only" build. ### Fixed diff --git a/src/main/scala/com/github/plume/oss/drivers/IDriver.scala b/src/main/scala/com/github/plume/oss/drivers/IDriver.scala index 98d76a72..14868d5d 100644 --- a/src/main/scala/com/github/plume/oss/drivers/IDriver.scala +++ b/src/main/scala/com/github/plume/oss/drivers/IDriver.scala @@ -3,6 +3,7 @@ package com.github.plume.oss.drivers import io.shiftleft.codepropertygraph.generated.nodes.{AbstractNode, NewNode, StoredNode} import io.shiftleft.codepropertygraph.generated.{EdgeTypes, NodeTypes, PropertyNames} import io.shiftleft.passes.AppliedDiffGraph +import org.slf4j.LoggerFactory import scala.collection.mutable @@ -10,6 +11,8 @@ import scala.collection.mutable */ trait IDriver extends AutoCloseable { + private val logger = LoggerFactory.getLogger(IDriver.getClass) + /** Will return true if the database is connected, false if otherwise. */ def isConnected: Boolean @@ -82,6 +85,7 @@ trait IDriver extends AutoCloseable { def buildInterproceduralEdges(): Unit = { initMaps() astLinker() + logger.info("Linking call graph") staticCallLinker() dynamicCallLinker() clearMaps() @@ -95,6 +99,7 @@ trait IDriver extends AutoCloseable { */ protected def astLinker(): Unit = { // Link NAMESPACE and NAMESPACE_BLOCK + logger.info("Linking NAMESPACE and NAMESPACE_BLOCK nodes by REF edges") linkAstNodes( srcLabels = List(NodeTypes.NAMESPACE_BLOCK), edgeType = EdgeTypes.REF, @@ -103,6 +108,7 @@ trait IDriver extends AutoCloseable { dstNodeType = NodeTypes.NAMESPACE ) // Create REF edges between TYPE and TYPE_DECL + logger.info("Linking TYPE and TYPE_DECL nodes by REF edges") linkAstNodes( srcLabels = List(NodeTypes.TYPE), edgeType = EdgeTypes.REF, @@ -112,6 +118,7 @@ trait IDriver extends AutoCloseable { ) // Create EVAL_TYPE edges from nodes of various types // to TYPE + logger.info("Linking TYPE and AST nodes by EVAL_TYPE edges") linkAstNodes( srcLabels = List( NodeTypes.METHOD_PARAMETER_IN, @@ -134,6 +141,7 @@ trait IDriver extends AutoCloseable { ) // Create REF edges from METHOD_REFs to // METHOD + logger.info("Linking METHOD_REFs and METHOD nodes by REF edges") linkAstNodes( srcLabels = List(NodeTypes.METHOD_REF), edgeType = EdgeTypes.REF, @@ -143,6 +151,7 @@ trait IDriver extends AutoCloseable { ) // Create INHERITS_FROM nodes from TYPE_DECL // nodes to TYPE + logger.info("Linking INHERITS_FROM and TYPE_DECL nodes by INHERITS_FROM edges") linkAstNodes( srcLabels = List(NodeTypes.TYPE_DECL), edgeType = EdgeTypes.INHERITS_FROM, @@ -152,6 +161,7 @@ trait IDriver extends AutoCloseable { ) // Create ALIAS_OF edges from TYPE_DECL nodes to // TYPE + logger.info("Linking TYPE_DECL and TYPE_DECL nodes by ALIAS_OF edges") linkAstNodes( srcLabels = List(NodeTypes.TYPE_DECL), edgeType = EdgeTypes.ALIAS_OF, diff --git a/src/main/scala/com/github/plume/oss/drivers/TigerGraphDriver.scala b/src/main/scala/com/github/plume/oss/drivers/TigerGraphDriver.scala index d78460a6..72af36e8 100644 --- a/src/main/scala/com/github/plume/oss/drivers/TigerGraphDriver.scala +++ b/src/main/scala/com/github/plume/oss/drivers/TigerGraphDriver.scala @@ -57,7 +57,7 @@ final class TigerGraphDriver( case _: Exception => false } - override def clear(): Unit = NodeTypes.ALL.forEach { nodeType => + override def clear(): Unit = NODES_IN_SCHEMA.foreach { nodeType => Try(delete(s"graph/cpg/delete_by_type/vertices/${nodeType}_")) } @@ -292,13 +292,15 @@ final class TigerGraphDriver( dstFullNameKey: String, dstNodeType: String ): Unit = { + def escape(raw: String): String = + raw.replace("[]", "\\[\\]").replace("_", "\\_") val endpoint = s"query/cpg/link_ast_${edgeType.toLowerCase}_${dstFullNameKey.toLowerCase}" dstNodeMap.foreach { case (key, id) => get( endpoint, srcLabels.map { x => ("src_labels", s"${x}_") - } :+ ("dst_value", s"%$key%") :+ ("dst", id.toString) :+ ("dst.type", s"${dstNodeType}_") + } :+ ("dst_value", s"%${escape(key)}%") :+ ("dst", id.toString) :+ ("dst.type", s"${dstNodeType}_") ) } } @@ -343,7 +345,7 @@ final class TigerGraphDriver( ] ) = { response.body match { - case Left(e) => throw e + case Left(e: ResponseException[_, _]) => throw unpackUnboxingException(e) case Right(body) => if (body.error) { throw new IOException( @@ -355,6 +357,17 @@ final class TigerGraphDriver( } } + private def unpackUnboxingException(e: ResponseException[String, circe.Error]): Exception = { + e match { + case HttpError(body, statusCode) => + logger.error(s"HTTP Error $statusCode: $body") + e + case DeserializationException(body, error) => + logger.error(s"Failed to deserialize response: $body. $error") + e + } + } + private def get( endpoint: String, params: Seq[(String, String)] @@ -507,7 +520,7 @@ object TigerGraphDriver { /** Default timeout for HTTP requests. */ - private val DEFAULT_TIMEOUT = 30 * 100 + private val DEFAULT_TIMEOUT = 30 * 1000 /** Default maximum number of transactions to bundle in a single transaction */ @@ -598,6 +611,35 @@ object TigerGraphDriver { fromCheck && toCheck } + private def NODES_IN_SCHEMA: Seq[String] = Seq( + MetaData.Label, + File.Label, + Method.Label, + MethodParameterIn.Label, + MethodParameterOut.Label, + MethodReturn.Label, + Modifier.Label, + Type.Label, + TypeDecl.Label, + TypeParameter.Label, + TypeArgument.Label, + Member.Label, + Namespace.Label, + NamespaceBlock.Label, + Literal.Label, + Call.Label, + Local.Label, + Identifier.Label, + FieldIdentifier.Label, + Return.Label, + Block.Label, + MethodRef.Label, + TypeRef.Label, + JumpTarget.Label, + ControlStructure.Label, + Unknown.Label + ) + /** Edges as a schema string. Each edge is prepended with "_" to escape * reserved words. */