Merge branch 'main' into feature/automated-graph-doc

.github/DISCUSSION_TEMPLATE/dev-meetings.yml

@@ -0,0 +1,33 @@
+title: "Meeting minutes for XX.XX.2023"
+labels: ["minutes"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        This discussion contains the public minutes of the dev meeting conducted on the date in the title. 
+        It can be created ahead of time (ideally at least a week before) to fill the initial agenda and 
+        list of PRs to discuss.
+  - type: textarea
+    id: agenda
+    attributes:
+      label: Agenda
+      description: "What is/was the general of the meeting?"
+      value: |
+        1. Open PRs
+        2. 
+        3.
+    validations:
+      required: true
+  - type: textarea
+    id: prs
+    attributes:
+      label: Discussed PRs
+      description: "Which PRs should be discussed / where discussed in the meeting?"
+      value: |
+        - [ ] #
+        - [ ] #
+        - [ ] #
+        ...
+    validations:
+      required: true
+

.github/workflows/build.yml

@@ -7,24 +7,22 @@ on:
       - main
     tags:
       - v*.**
+    paths-ignore:
+      - "docs/**"
   pull_request:
     types: [opened, synchronize, reopened]
+    paths-ignore:
+      - "docs/**"
 
 jobs:
   build-cpgo-osx:
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of SonarQube analysis
       - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.18
-      - uses: actions/setup-java@v3
+        uses: actions/setup-go@v4
         with:
-          distribution: "zulu"
-          java-version: "17"
+          go-version: "1.20"
       - name: Build
         run: |
           cd cpg-language-go/src/main/golang
@@ -43,7 +41,7 @@ jobs:
           path: cpg-language-go/src/main/resources/libcpgo-arm64.dylib
 
   build:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, linux, x64, faster]
     needs: build-cpgo-osx
     steps:
       - uses: actions/checkout@v3
@@ -55,18 +53,17 @@ jobs:
         with:
           distribution: "zulu"
           java-version: "17"
-          cache: "gradle"
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
       - uses: actions/setup-node@v3
         with:
-          node-version: "16"
+          node-version: "18"
       - name: Setup neo4j
         run: |
-          docker run -d --env NEO4J_AUTH=neo4j/password -p7474:7474 -p7687:7687 neo4j
+          docker run -d --env NEO4J_AUTH=neo4j/password -p7474:7474 -p7687:7687 neo4j || true
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: 1.18
       - name: Cache SonarCloud packages
@@ -92,7 +89,9 @@ jobs:
       - name: Install JEP
         run: |
           pip3 install jep==$(grep "black.ninia:jep" gradle/libs.versions.toml | grep -o -E "[0-9]\d*(\.[0-9]\d*)*")
-          find /opt/hostedtoolcache/Python/ -name libjep.so -exec sudo cp '{}' /usr/lib/ \;
+          if [ -d "/opt/hostedtoolcache/Python" ]; then
+            find /opt/hostedtoolcache/Python/ -name libjep.so -exec sudo cp '{}' /usr/lib/ \;
+          fi
       - name: Install pycodestyle
         run: |
           pip3 install pycodestyle
@@ -111,41 +110,80 @@ jobs:
         run: |
           if [ "$SONAR_TOKEN" != "" ]
           then
-            ./gradlew --no-daemon --parallel -Pversion=$VERSION -Pexperimental -PenableJavaFrontend=true -PenableGoFrontend=true -PenablePythonFrontend=true -PenableLLVMFrontend=true -PenableTypeScriptFrontend=true -Pintegration spotlessCheck -x spotlessApply build -x distZip -x distTar sonar \
+            ./gradlew --no-daemon --parallel -Pversion=$VERSION -Pintegration spotlessCheck -x spotlessApply build -x distZip -x distTar testCodeCoverageReport sonar \
             -Dsonar.projectKey=Fraunhofer-AISEC_cpg \
             -Dsonar.organization=fraunhofer-aisec \
             -Dsonar.host.url=https://sonarcloud.io \
             -Dsonar.login=$SONAR_TOKEN
           else
-            ./gradlew --no-daemon --parallel -Pversion=$VERSION -Pexperimental -PenableJavaFrontend=true -PenableGoFrontend=true -PenablePythonFrontend=true -PenableLLVMFrontend=true -PenableTypeScriptFrontend=true -Pintegration spotlessCheck -x spotlessApply build -x distZip -x distTar
+            ./gradlew --no-daemon --parallel -Pversion=$VERSION -Pintegration spotlessCheck -x spotlessApply build -x distZip -x distTar
           fi
         id: build
         env:
           VERSION: ${{ env.version }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Archive test reports
+      - name: Prepare test and coverage reports
+        if: ${{ always() }}
+        run: |
+          zip reports.zip **/build/reports/**/** || true
+      - name: Archive test and coverage reports
         if: ${{ always() }}
         uses: actions/upload-artifact@v3
         with:
-          name: test
-          path: "**/build/reports/tests"
-      - name: Publish
+          name: reports
+          path: reports.zip
+      - name: Publish to Maven Central
         if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, 'beta') && !contains(github.ref, 'alpha')
         run: |
           export ORG_GRADLE_PROJECT_signingKey=`echo ${{ secrets.GPG_PRIVATE_KEY }} | base64 -d`
-          ./gradlew --no-daemon -Dorg.gradle.internal.publish.checksums.insecure=true --parallel -Pversion=$VERSION -PenableJavaFrontend=true -PenableGoFrontend=true -PenablePythonFrontend=true -PenableLLVMFrontend=true -PenableTypeScriptFrontend=true publish dokkaHtmlMultiModule
+          ./gradlew --no-daemon -Dorg.gradle.internal.publish.checksums.insecure=true --parallel -Pversion=$VERSION -PenableJavaFrontend=true  -PenableCXXFrontend=true -PenableGoFrontend=true -PenablePythonFrontend=true -PenableLLVMFrontend=true -PenableTypeScriptFrontend=true publishToSonatype closeSonatypeStagingRepository
         env:
           VERSION: ${{ env.version }}
           ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.GPG_PASSWORD }}
           ORG_GRADLE_PROJECT_mavenCentralUsername: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
           ORG_GRADLE_PROJECT_mavenCentralPassword: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
-      - name: Publish JavaDoc
+      - name: Download old dokka versions (version)
         if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, 'beta') && !contains(github.ref, 'alpha')
+        run: |
+          # make sure the previousDocs folder exists
+          mkdir -p previousDocs && cd previousDocs
+          # retrieve the previous documentation folders for each published version (this also includes "main")
+          wget -O - https://github.com/Fraunhofer-AISEC/cpg/archive/gh-pages.tar.gz | tar -xz --strip=2 cpg-gh-pages/dokka || echo "No dokka directory present. Will continue as if nothing happened"
+      - name: Download old dokka versions (main)
+        if: github.ref == 'refs/heads/main'
+        run: |
+          # make sure the previousDocs folder exists
+          mkdir -p previousDocs && cd previousDocs
+          # retrieve the previous documentation folders for each published version (this also includes "main")
+          wget -O - https://github.com/Fraunhofer-AISEC/cpg/archive/gh-pages.tar.gz | tar -xz --strip=2 cpg-gh-pages/dokka || echo "No dokka directory present. Will continue as if nothing happened"
+          # in order to avoid duplicate mains, remove the "main" version from the previous versions
+          rm -rf main
+      - name: Build JavaDoc
+        if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, 'beta') && !contains(github.ref, 'alpha') || github.ref == 'refs/heads/main'
+        run: |
+          ./gradlew --no-daemon -Pversion=$VERSION dokkaHtmlMultiModule
+        env:
+          VERSION: ${{ env.version }}
+      - name: Publish JavaDoc (version)
+        if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, 'beta') && !contains(github.ref, 'alpha')
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: build/dokkaCustomMultiModuleOutput/${{ env.version }}
+          target-folder: dokka/${{ env.version }}
+      - name: Publish JavaDoc (version as main)
+        if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, 'beta') && !contains(github.ref, 'alpha')
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: build/dokkaCustomMultiModuleOutput/${{ env.version }}
+          target-folder: dokka/main
+      - name: Publish JavaDoc (main)
+        if: github.ref == 'refs/heads/main'
         uses: JamesIves/github-pages-deploy-action@v4
         with:
-          folder: build/dokkaCustomMultiModuleOutput
-      - name: "Create Release"
+          folder: build/dokkaCustomMultiModuleOutput/main
+          target-folder: dokka/main
+      - name: Create Release
         if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, 'beta') && !contains(github.ref, 'alpha')
         id: create_release
         uses: softprops/action-gh-release@v1

.github/workflows/docs.yml

@@ -0,0 +1,47 @@
+name: docs
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - docs/**
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - docs/**
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    if: github.event.repository.fork == false
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install python3
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.x
+      - name: Cache
+        uses: actions/cache@v3
+        with:
+          key: ${{ github.ref }}
+          path: .cache
+      - name: Install Material for MkDocs
+        run: |
+          pip install mkdocs-material pillow cairosvg
+          pip install -r docs/mkdocs-material-plugins.txt
+      - name: Build
+        run: cd docs && mkdocs build --clean --config-file mkdocs.yaml -d site -v
+      - name: Publish main
+        if: github.ref == 'refs/heads/main'
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: docs/site
+      - name: Publish version
+        if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, 'beta') && !contains(github.ref, 'alpha')
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: docs/site
+          clean-exclude: dokka/**

CONTRIBUTING.md

@@ -79,7 +79,7 @@ On some edges, we want to store additional information (e.g., if a `EOG` node is
 /** The list of function parameters. */
 @Relationship(value = "PARAMETERS", direction = Relationship.Direction.OUTGOING)
 @field:SubGraph("AST")
-var parameterEdges = mutableListOf<PropertyEdge<ParamVariableDeclaration>>()
+var parameterEdges = mutableListOf<PropertyEdge<ParameterDeclaration>>()
 
 /** Virtual property for accessing [parameterEdges] without property edges. */
 var parameters by PropertyEdgeDelegate(FunctionDeclaration::parameterEdges)
@@ -116,6 +116,10 @@ override fun hashCode() = Objects.hash(super.hashCode(), constructor, arguments)
 
 Before we can accept a pull request from you, you'll need to sign a Contributor License Agreement (CLA). It is an automated process and you only need to do it once.
 
+:warning:
+We are currently discussing the implementation of a Contributor License Agreement (CLA). Unfortunately, we cannot merge external pull requests until this issue is resolved.
+:warning:
+
 To enable us to quickly review and accept your pull requests, always create one pull request per issue and link the issue in the pull request.
 Never merge multiple requests in one unless they have the same root cause. Be sure your code is formatted correctly using the respective formatting task.
 Keep code changes as small as possible. 

README.md

@@ -10,14 +10,15 @@ A code property graph (CPG) is a representation of source code in form of a labe
 
 This library uses [Eclipse CDT](https://www.eclipse.org/cdt/) for parsing C/C++ source code [JavaParser](https://javaparser.org/) for parsing Java. In contrast to compiler AST generators, both are "forgiving" parsers that can cope with incomplete or even semantically incorrect source code. That makes it possible to analyze source code even without being able to compile it (due to missing dependencies or minor syntax errors). Furthermore, it uses [LLVM](https://llvm.org) through the [javacpp](https://github.com/bytedeco/javacpp) project to parse LLVM IR. Note that the LLVM IR parser is *not* forgiving, i.e., the LLVM IR code needs to be at least considered valid by LLVM. The necessary native libraries are shipped by the javacpp project for most platforms.
 
-
 ## Specifications
 
 In order to improve some formal aspects of our library, we created several specifications of our core concepts. Currently, the following specifications exist:
-* [Dataflow Graph](./cpg-core/specifications/dfg.md)
-* [Language and Language Frontend](./cpg-core/specifications/language.md)
+* [Dataflow Graph](https://fraunhofer-aisec.github.io/cpg/CPG/specs/dfg/)
+* [Evaluation Order Graph](https://fraunhofer-aisec.github.io/cpg/CPG/specs/eog/)
+* [Graph Model in neo4j](https://fraunhofer-aisec.github.io/cpg/CPG/specs/graph/)
+* [Language and Language Frontend](https://fraunhofer-aisec.github.io/cpg/CPG/impl/language/)
 
-We aim to provide more specifications over time and also include them in a new generated documentation site.
+We aim to provide more specifications over time.
 
 ## Usage
 
@@ -113,9 +114,10 @@ val translationConfig = TranslationConfiguration
 
 ### Experimental Languages
 
-Some languages, such as Golang are experimental and depend on other native libraries. Therefore, they are not included as gradle submodules by default.
-To include them as submodules simply toggle them on in your local `gradle.properties` file by setting the value of the properties to `true` e.g., (`enableGoFrontend=true`).
-We provide a sample file [here](./gradle.properties.example).
+Some languages, such as Golang are experimental and depend on other native libraries. Therefore, they are not included in the `cpg-core` module but have separate gradle submodules.
+C/CPP and Java are currently required by some of the modules (e.g. `cpg-analysis`) and thus, disabling them can lead to compile errors!
+To include the desired submodules simply toggle them on in your local `gradle.properties` file by setting the value of the properties to `true` e.g., (`enableGoFrontend=true`).
+We provide a sample file with all languages switched on [here](./gradle.properties.example).
 Instead of manually editing the `gradle.properties` file, you can also use the `configure_frontends.sh` script, which edits the properties for you.
 
 #### Golang