diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f462b3b482..f2495bc709 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -133,6 +133,17 @@ jobs: with: name: reports path: reports.zip + - name: Generate graph schema + if: github.ref == 'refs/heads/main' + run: | + mkdir cpg-neo4j/build/schema + ./gradlew :cpg-neo4j:run --args="--schema ./build/schema/graph.md" + - name: Publish graph schema (main) + if: github.ref == 'refs/heads/main' + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: cpg-neo4j/build/schema + target-folder: CPG/specs - name: Publish to Maven Central if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, 'beta') && !contains(github.ref, 'alpha') run: | diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/LocationConverter.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/LocationConverter.kt index 97530c9f44..e33b242872 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/LocationConverter.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/LocationConverter.kt @@ -30,11 +30,20 @@ import de.fraunhofer.aisec.cpg.sarif.Region import java.net.URI import org.neo4j.ogm.typeconversion.CompositeAttributeConverter +interface CpgCompositeConverter : CompositeAttributeConverter { + /** + * Determines to which properties and their types the received value will be split in the neo4j + * representation. The type is the first element in the pair and the property name is the second + * one. + */ + val graphSchema: List> +} + /** - * This class converts a [PhysicalLocation] into the the necessary composite attributes when - * persisting a node into a Neo4J graph database. + * This class converts a [PhysicalLocation] into the necessary composite attributes when persisting + * a node into a Neo4J graph database. */ -class LocationConverter : CompositeAttributeConverter { +class LocationConverter : CpgCompositeConverter { override fun toGraphProperties(value: PhysicalLocation?): Map { val properties: MutableMap = HashMap() if (value != null) { @@ -47,6 +56,16 @@ class LocationConverter : CompositeAttributeConverter { return properties } + override val graphSchema: List> + get() = + listOf( + Pair("String", ARTIFACT), + Pair("int", START_LINE), + Pair("int", END_LINE), + Pair("int", START_COLUMN), + Pair("int", END_COLUMN) + ) + override fun toEntityAttribute(value: Map?): PhysicalLocation? { return try { val startLine = toInt(value?.get(START_LINE)) ?: return null diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/NameConverter.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/NameConverter.kt index 3c7b16e607..464fd04340 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/NameConverter.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/NameConverter.kt @@ -27,7 +27,6 @@ package de.fraunhofer.aisec.cpg.helpers.neo4j import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.parseName -import org.neo4j.ogm.typeconversion.CompositeAttributeConverter /** * This converter can be used in a Neo4J session to persist the [Name] class into its components: @@ -37,7 +36,7 @@ import org.neo4j.ogm.typeconversion.CompositeAttributeConverter * * Additionally, it converts the aforementioned Neo4J attributes in a node back into a [Name]. */ -class NameConverter : CompositeAttributeConverter { +class NameConverter : CpgCompositeConverter { companion object { const val FIELD_FULL_NAME = "fullName" @@ -61,14 +60,22 @@ class NameConverter : CompositeAttributeConverter { // For reasons such as backwards compatibility and the fact that Neo4J likes to display // nodes in the UI with a "name" field as default, we also persist the full name (aka - // the - // toString() representation) as "name" + // the toString() representation) as "name" map[FIELD_NAME] = value.toString() } return map } + override val graphSchema: List> + get() = + listOf( + Pair("String", FIELD_FULL_NAME), + Pair("String", FIELD_LOCAL_NAME), + Pair("String", FIELD_NAME), + Pair("String", FIELD_NAME_DELIMITER) + ) + override fun toEntityAttribute(value: MutableMap): Name { return parseName(value[FIELD_FULL_NAME].toString(), value[FIELD_NAME_DELIMITER].toString()) } diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt index 7adc5ed7c2..155245900a 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt @@ -449,6 +449,7 @@ class Application : Callable { printSchema(mutuallyExclusiveParameters.files) return EXIT_SUCCESS } + if (mutuallyExclusiveParameters.listPasses) { log.info("List of passes:") passList.iterator().forEach { log.info("- $it") } @@ -456,6 +457,7 @@ class Application : Callable { log.info("End of list. Stopping.") return EXIT_SUCCESS } + val translationConfiguration = setupTranslationConfiguration() val startTime = System.currentTimeMillis() diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Schema.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Schema.kt index 6047667e48..65a57f84ca 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Schema.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Schema.kt @@ -26,6 +26,7 @@ package de.fraunhofer.aisec.cpg_vis_neo4j import de.fraunhofer.aisec.cpg.graph.Node +import de.fraunhofer.aisec.cpg.helpers.neo4j.CpgCompositeConverter import java.io.File import java.io.PrintWriter import java.lang.reflect.ParameterizedType @@ -75,6 +76,22 @@ class Schema { * MutableMap>> */ private val inheritedRels: MutableMap>> = mutableMapOf() + + /** + * Relationships newly defined in this specific entity. Saves + * MutableMap>> + */ + private val inherentProperties: MutableMap>> = + mutableMapOf() + + /** + * Relationships inherited from a parent in the inheritance hierarchy. A node with this label + * can have this relationship if it is non-nullable. Saves + * MutableMap>> + */ + private val inheritedProperties: MutableMap>> = + mutableMapOf() + /** * Relationships defined by children in the inheritance hierarchy. A node with this label can * have this relationship also has the label of the defining child entity. Saves @@ -97,48 +114,65 @@ class Schema { Node::class.java.isAssignableFrom(it.underlyingClass) && !it.isRelationshipEntity } // Node to filter for, filter out what is not explicitly a - entities.forEach { - if (it in entities) { - val superC = it.directSuperclass() + entities.forEach { entity -> + val superC = entity.directSuperclass() - hierarchy[it] = - Pair( - if (superC in entities) superC else null, - it.directSubclasses() - .filter { it in entities } - .distinct() // Filter out duplicates - ) - } + hierarchy[entity] = + Pair( + if (superC in entities) superC else null, + entity + .directSubclasses() + .filter { it in entities } + .distinct() // Filter out duplicates + ) } // node in neo4j - entities.forEach { - val key = meta.schema.findNode(it.neo4jName()) - allRels[it.neo4jName() ?: it.underlyingClass.simpleName] = + entities.forEach { classInfo -> + val key = meta.schema.findNode(classInfo.neo4jName()) + allRels[classInfo.neo4jName() ?: classInfo.underlyingClass.simpleName] = key.relationships().entries.map { Pair(it.key, it.value.type()) }.toSet() } // Complements the hierarchy and relationship information for abstract classes completeSchema(allRels, hierarchy, nodeClassInfo) - // Searches for all relationships backed by a class field to know which relationships are - // newly defined in the - // entity class - entities.forEach { - val entity = it + // Searches for all relationships and properties backed by a class field to know which + // of them are newly defined in the entity class + entities.forEach { entity -> val fields = entity.relationshipFields().filter { it.field.declaringClass == entity.underlyingClass } fields.forEach { relationshipFields.put(Pair(entity, it.name), it) } - val name = it.neo4jName() ?: it.underlyingClass.simpleName - allRels[name]?.let { + val name = entity.neo4jName() ?: entity.underlyingClass.simpleName + allRels[name]?.let { relationPair -> inherentRels[name] = - it.filter { - val rel = it.first - fields.any { it.name.equals(rel) } - } - .toSet() + relationPair.filter { rel -> fields.any { it.name.equals(rel.first) } }.toSet() + } + + entity.propertyFields().forEach { property -> + val persistedField = + if ( + property.hasCompositeConverter() && + property.compositeConverter is CpgCompositeConverter + ) { + (property.compositeConverter as CpgCompositeConverter).graphSchema + } else { + listOf>( + Pair(property.field.type.simpleName, property.name) + ) + } + + if (property.field.declaringClass == entity.underlyingClass) { + inherentProperties + .computeIfAbsent(name) { mutableSetOf() } + .addAll(persistedField) + } else { + inheritedProperties + .computeIfAbsent(name) { mutableSetOf() } + .addAll(persistedField) + } } } @@ -154,8 +188,8 @@ class Schema { allRels.forEach { childrensRels[it.key] = it.value - .subtract(inheritedRels[it.key] ?: emptyList()) - .subtract(inherentRels[it.key] ?: emptyList()) + .subtract(inheritedRels[it.key] ?: emptySet()) + .subtract(inherentRels[it.key] ?: emptySet()) } println() } @@ -189,8 +223,10 @@ class Schema { it.neo4jName() ?: it.underlyingClass.simpleName, hierarchy[it] ?.second - ?.flatMap { - relCanHave[it.neo4jName() ?: it.underlyingClass.simpleName] ?: setOf() + ?.flatMap { classInfo -> + relCanHave[ + classInfo.neo4jName() ?: classInfo.underlyingClass.simpleName] + ?: setOf() } ?.toSet() ?: setOf() @@ -210,10 +246,14 @@ class Schema { } } + /** + * Prints a section for every entity with a list of labels (e.g. superclasses), a list of + * relationships, a dropdown with inherited relationships, a list of properties and a dropdown + * with inherited properties. + * + * Generates links between the boxes. + */ private fun printEntities(classInfo: ClassInfo, out: PrintWriter) { - // TODO print a section for every entity. List of relationships not inherent. List of rel - // inherent with result node. try to get links into relationship and target. - // TODO subsection with inherent relationships. val entityLabel = toLabel(classInfo) out.println("## $entityLabel") @@ -243,15 +283,13 @@ class Schema { hierarchy[classInfo]?.second?.let { if (it.isNotEmpty()) { - it.forEach { + it.forEach { classInfo -> out.print( getBoxWithClass( "child", - "[${toLabel(it)}](#${toAnchorLink("e"+toLabel(it))})" + "[${toLabel(classInfo)}](#${toAnchorLink("e"+toLabel(classInfo))})" ) ) - // out.println("click ${toLabel(it)} href - // \"#${toAnchorLink(toLabel(it))}\"") } out.println() } @@ -261,7 +299,7 @@ class Schema { if (inherentRels.isNotEmpty() && inheritedRels.isNotEmpty()) { out.println("### Relationships") - noLabelDups(inherentRels[entityLabel])?.forEach { + removeLabelDuplicates(inherentRels[entityLabel])?.forEach { out.println( getBoxWithClass( "relationship", @@ -269,43 +307,66 @@ class Schema { ) ) } - noLabelDups(inheritedRels[entityLabel])?.forEach { - var inherited = it - var current = classInfo - var baseClass: ClassInfo? = null - while (baseClass == null) { - inherentRels[toLabel(current)]?.let { - if (it.any { it.second.equals(inherited.second) }) { - baseClass = current + + if (inheritedRels[entityLabel]?.isNotEmpty() == true) { + out.println("
Inherited Relationships") + out.println() + removeLabelDuplicates(inheritedRels[entityLabel])?.forEach { inherited -> + var current = classInfo + var baseClass: ClassInfo? = null + while (baseClass == null) { + inherentRels[toLabel(current)]?.let { rels -> + if (rels.any { it.second == inherited.second }) { + baseClass = current + } } + hierarchy[current]?.first?.let { current = it } } - hierarchy[current]?.first?.let { current = it } - } - out.println( - getBoxWithClass( - "inherited-relationship", - "[${it.second}](#${toConcatName(toLabel(baseClass)+it.second)})" + out.println( + getBoxWithClass( + "inherited-relationship", + "[${inherited.second}](#${toConcatName(toLabel(baseClass) + inherited.second)})" + ) ) - ) + } + out.println("
") + out.println() } - noLabelDups(inherentRels[entityLabel])?.forEach { + removeLabelDuplicates(inherentRels[entityLabel])?.forEach { printRelationships(classInfo, it, out) } } + if (inherentProperties.isNotEmpty() && inheritedProperties.isNotEmpty()) { + out.println("### Properties") + + removeLabelDuplicates(inherentProperties[entityLabel])?.forEach { + out.println("${it.second} : ${it.first}") + out.println() + } + if (inheritedProperties[entityLabel]?.isNotEmpty() == true) { + out.println("
Inherited Properties") + removeLabelDuplicates(inheritedProperties[entityLabel])?.forEach { + out.println("${it.second} : ${it.first}") + out.println() + } + out.println("
") + out.println() + } + } + hierarchy[classInfo]?.second?.forEach { printEntities(it, out) } } - private fun noLabelDups(list: Set>?): Set>? { + private fun removeLabelDuplicates( + list: Set>? + ): Set>? { if (list == null) return null return list .map { it.second } .distinct() - .map { - val label = it - list.first { it.second == label } - } + .map { label -> list.first { it.second == label } } .toSet() } @@ -351,7 +412,7 @@ class Schema { .filterIsInstance() .map { it.rawType } val baseClass: Type? = getNestedBaseType(type) - var multiplicity = getNestedMultiplicity(type) + val multiplicity = getNestedMultiplicity(type) var targetClassInfo: ClassInfo? = null if (baseClass != null) { @@ -375,12 +436,12 @@ class Schema { private fun getNestedMultiplicity(type: Type): Boolean { if (type is ParameterizedType) { - if ( + return if ( type.rawType.typeName.substringBeforeLast(".") == "java.util" ) { // listOf(List::class).contains(type.rawType) - return true + true } else { - return type.actualTypeArguments.any { getNestedMultiplicity(it) } + type.actualTypeArguments.any { getNestedMultiplicity(it) } } } return false