Skip to content

Commit

Permalink
Python: merge async with non-async code (#1673)
Browse files Browse the repository at this point in the history 
* merge function def and async function def

* merge for and async for

* prepare to merge with and async with

* new makeBlock

* make handleDecoratorList private

* rename WithASTLocation -> WithLocation

* fix python grammar

* Added very simple test

---------

Co-authored-by: Christian Banse <[email protected]>
  • Loading branch information
@maximiliankaul @oxisto
maximiliankaul and oxisto authored Sep 6, 2024
1 parent e131920 commit 3ed3526
Show file tree
Hide file tree
Showing 5 changed files with 163 additions and 127 deletions.
124 changes: 84 additions & 40 deletions cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/Python.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ interface Python {
* `ast.stmt` [AST.BaseStmt] and `ast.expr` [AST.BaseExpr] nodes have extra location
* properties as implemented here.
*/
interface WithASTLocation { // TODO make the fields accessible `by lazy`
interface WithLocation { // TODO make the fields accessible `by lazy`
val pyObject: PyObject

/** Maps to the `lineno` filed from Python's ast. */
Expand Down Expand Up @@ -156,28 +156,51 @@ interface Python {
* | | Continue
* ```
*/
sealed class BaseStmt(pyObject: PyObject) : AST(pyObject), WithASTLocation
sealed class BaseStmt(pyObject: PyObject) : AST(pyObject), WithLocation

/**
* Several classes are duplicated in the python AST for async and non-async variants. This
* interface is a common interface for those AST classes.
*/
interface AsyncOrNot : WithLocation

/** This interface denotes that this is an "async" node. */
interface IsAsync : AsyncOrNot

/**
* ast.FunctionDef and ast.AsyncFunctionDef are not related according to the Python syntax.
* However, they are so similar, that we make use of this interface to avoid a lot of
* duplicate code.
*/
interface NormalOrAsyncFunctionDef : AsyncOrNot {
val name: String
val args: arguments
val body: kotlin.collections.List<BaseStmt>
val decorator_list: kotlin.collections.List<BaseExpr>
val returns: BaseExpr?
val type_comment: String?
}

/**
* ```
* ast.FunctionDef = class FunctionDef(stmt)
* | FunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)
* ```
*/
class FunctionDef(pyObject: PyObject) : BaseStmt(pyObject) {
val name: String by lazy { "name" of pyObject }
class FunctionDef(pyObject: PyObject) : BaseStmt(pyObject), NormalOrAsyncFunctionDef {
override val name: String by lazy { "name" of pyObject }

val args: arguments by lazy { "args" of pyObject }
override val args: arguments by lazy { "args" of pyObject }

val body: kotlin.collections.List<BaseStmt> by lazy { "body" of pyObject }
override val body: kotlin.collections.List<BaseStmt> by lazy { "body" of pyObject }

val decorator_list: kotlin.collections.List<BaseExpr> by lazy {
override val decorator_list: kotlin.collections.List<BaseExpr> by lazy {
"decorator_list" of pyObject
}

val returns: BaseExpr? by lazy { "returns" of pyObject }
override val returns: BaseExpr? by lazy { "returns" of pyObject }

val type_comment: String? by lazy { "type_comment" of pyObject }
override val type_comment: String? by lazy { "type_comment" of pyObject }
}

/**
Expand All @@ -186,20 +209,21 @@ interface Python {
* | AsyncFunctionDef(identifier name, arguments args, stmt* body, expr* decorator_list, expr? returns, string? type_comment)
* ```
*/
class AsyncFunctionDef(pyObject: PyObject) : BaseStmt(pyObject) {
val name: String by lazy { "name" of pyObject }
class AsyncFunctionDef(pyObject: PyObject) :
BaseStmt(pyObject), NormalOrAsyncFunctionDef, IsAsync {
override val name: String by lazy { "name" of pyObject }

val args: arguments by lazy { "args" of pyObject }
override val args: arguments by lazy { "args" of pyObject }

val body: kotlin.collections.List<BaseStmt> by lazy { "body" of pyObject }
override val body: kotlin.collections.List<BaseStmt> by lazy { "body" of pyObject }

val decorator_list: kotlin.collections.List<BaseExpr> by lazy {
override val decorator_list: kotlin.collections.List<BaseExpr> by lazy {
"decorator_list" of pyObject
}

val returns: BaseExpr? by lazy { "returns" of pyObject }
override val returns: BaseExpr? by lazy { "returns" of pyObject }

val type_comment: String? by lazy { "type_comment" of pyObject }
override val type_comment: String? by lazy { "type_comment" of pyObject }
}

/**
Expand Down Expand Up @@ -281,18 +305,30 @@ interface Python {
val simple: Long by lazy { "simple" of pyObject }
}

/**
* ast.For and ast.AsyncFor are not related according to the Python syntax. However, they
* are so similar, that we make use of this interface to avoid a lot of duplicate code.
*/
interface NormalOrAsyncFor : AsyncOrNot {
val target: BaseExpr
val iter: BaseExpr
val body: kotlin.collections.List<BaseStmt>
val orelse: kotlin.collections.List<BaseStmt>
val type_comment: String?
}

/**
* ```
* ast.For = class For(stmt)
* | For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
* ```
*/
class For(pyObject: PyObject) : BaseStmt(pyObject) {
val target: BaseExpr by lazy { "target" of pyObject }
val iter: BaseExpr by lazy { "iter" of pyObject }
val body: kotlin.collections.List<BaseStmt> by lazy { "body" of pyObject }
val orelse: kotlin.collections.List<BaseStmt> by lazy { "orelse" of pyObject }
val type_comment: String? by lazy { "type_comment" of pyObject }
class For(pyObject: PyObject) : BaseStmt(pyObject), NormalOrAsyncFor {
override val target: BaseExpr by lazy { "target" of pyObject }
override val iter: BaseExpr by lazy { "iter" of pyObject }
override val body: kotlin.collections.List<BaseStmt> by lazy { "body" of pyObject }
override val orelse: kotlin.collections.List<BaseStmt> by lazy { "orelse" of pyObject }
override val type_comment: String? by lazy { "type_comment" of pyObject }
}

/**
Expand All @@ -301,12 +337,12 @@ interface Python {
* | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment)
* ```
*/
class AsyncFor(pyObject: PyObject) : BaseStmt(pyObject) {
val target: BaseExpr by lazy { "target" of pyObject }
val iter: BaseExpr by lazy { "iter" of pyObject }
val body: kotlin.collections.List<BaseStmt> by lazy { "body" of pyObject }
val orelse: kotlin.collections.List<BaseStmt> by lazy { "orelse" of pyObject }
val type_comment: String? by lazy { "type_comment" of pyObject }
class AsyncFor(pyObject: PyObject) : BaseStmt(pyObject), NormalOrAsyncFor, IsAsync {
override val target: BaseExpr by lazy { "target" of pyObject }
override val iter: BaseExpr by lazy { "iter" of pyObject }
override val body: kotlin.collections.List<BaseStmt> by lazy { "body" of pyObject }
override val orelse: kotlin.collections.List<BaseStmt> by lazy { "orelse" of pyObject }
override val type_comment: String? by lazy { "type_comment" of pyObject }
}

/**
Expand All @@ -333,16 +369,26 @@ interface Python {
val orelse: kotlin.collections.List<BaseStmt> by lazy { "orelse" of pyObject }
}

/**
* ast.With and ast.AsyncWith are not related according to the Python syntax. However, they
* are so similar, that we make use of this interface to avoid a lot of duplicate code.
*/
interface NormalOrAsyncWith : AsyncOrNot {
val items: kotlin.collections.List<withitem>
val body: kotlin.collections.List<BaseStmt>
val type_comment: String?
}

/**
* ```
* ast.With = class With(stmt)
* | With(withitem* items, stmt* body, string? type_comment)
* ```
*/
class With(pyObject: PyObject) : BaseStmt(pyObject) {
val items: withitem by lazy { "items" of pyObject }
val body: kotlin.collections.List<BaseStmt> by lazy { "body" of pyObject }
val type_comment: String? by lazy { "type_comment" of pyObject }
class With(pyObject: PyObject) : BaseStmt(pyObject), NormalOrAsyncWith {
override val items: kotlin.collections.List<withitem> by lazy { "items" of pyObject }
override val body: kotlin.collections.List<BaseStmt> by lazy { "body" of pyObject }
override val type_comment: String? by lazy { "type_comment" of pyObject }
}

/**
Expand All @@ -351,12 +397,10 @@ interface Python {
* | AsyncWith(withitem* items, stmt* body, string? type_comment)
* ```
*/
class AsyncWith(pyObject: PyObject) : BaseStmt(pyObject) {
val target: BaseExpr by lazy { "target" of pyObject }
val iter: BaseExpr by lazy { "iter" of pyObject }
val body: kotlin.collections.List<BaseStmt> by lazy { "body" of pyObject }
val orelse: kotlin.collections.List<BaseStmt> by lazy { "orelse" of pyObject }
val type_comment: String? by lazy { "type_comment" of pyObject }
class AsyncWith(pyObject: PyObject) : BaseStmt(pyObject), NormalOrAsyncWith, IsAsync {
override val items: kotlin.collections.List<withitem> by lazy { "items" of pyObject }
override val body: kotlin.collections.List<BaseStmt> by lazy { "body" of pyObject }
override val type_comment: String? by lazy { "type_comment" of pyObject }
}

/**
Expand Down Expand Up @@ -505,7 +549,7 @@ interface Python {
*
* ast.expr = class expr(AST)
*/
sealed class BaseExpr(pyObject: PyObject) : AST(pyObject), WithASTLocation
sealed class BaseExpr(pyObject: PyObject) : AST(pyObject), WithLocation

/**
* ```
Expand Down Expand Up @@ -1233,7 +1277,7 @@ interface Python {
* | arg(identifier arg, expr? annotation, string? type_comment)
* ```
*/
class arg(pyObject: PyObject) : AST(pyObject), WithASTLocation {
class arg(pyObject: PyObject) : AST(pyObject), WithLocation {
val arg: String by lazy { "arg" of pyObject }
val annotation: BaseExpr? by lazy { "annotation" of pyObject }
val type_comment: String? by lazy { "type_comment" of pyObject }
Expand Down
2 changes: 1 addition & 1 deletion ...python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ class PythonLanguageFrontend(language: Language<PythonLanguageFrontend>, ctx: Tr
}

override fun locationOf(astNode: Python.AST.AST): PhysicalLocation? {
return if (astNode is Python.AST.WithASTLocation) {
return if (astNode is Python.AST.WithLocation) {
PhysicalLocation(
uri,
Region(
Expand Down
Loading

0 comments on commit 3ed3526

Please sign in to comment.