diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index f462b3b482..f2495bc709 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -133,6 +133,17 @@ jobs:
with:
name: reports
path: reports.zip
+ - name: Generate graph schema
+ if: github.ref == 'refs/heads/main'
+ run: |
+ mkdir cpg-neo4j/build/schema
+ ./gradlew :cpg-neo4j:run --args="--schema ./build/schema/graph.md"
+ - name: Publish graph schema (main)
+ if: github.ref == 'refs/heads/main'
+ uses: JamesIves/github-pages-deploy-action@v4
+ with:
+ folder: cpg-neo4j/build/schema
+ target-folder: CPG/specs
- name: Publish to Maven Central
if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, 'beta') && !contains(github.ref, 'alpha')
run: |
diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/LocationConverter.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/LocationConverter.kt
index 97530c9f44..e33b242872 100644
--- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/LocationConverter.kt
+++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/LocationConverter.kt
@@ -30,11 +30,20 @@ import de.fraunhofer.aisec.cpg.sarif.Region
import java.net.URI
import org.neo4j.ogm.typeconversion.CompositeAttributeConverter
+interface CpgCompositeConverter : CompositeAttributeConverter {
+ /**
+ * Determines to which properties and their types the received value will be split in the neo4j
+ * representation. The type is the first element in the pair and the property name is the second
+ * one.
+ */
+ val graphSchema: List>
+}
+
/**
- * This class converts a [PhysicalLocation] into the the necessary composite attributes when
- * persisting a node into a Neo4J graph database.
+ * This class converts a [PhysicalLocation] into the necessary composite attributes when persisting
+ * a node into a Neo4J graph database.
*/
-class LocationConverter : CompositeAttributeConverter {
+class LocationConverter : CpgCompositeConverter {
override fun toGraphProperties(value: PhysicalLocation?): Map {
val properties: MutableMap = HashMap()
if (value != null) {
@@ -47,6 +56,16 @@ class LocationConverter : CompositeAttributeConverter {
return properties
}
+ override val graphSchema: List>
+ get() =
+ listOf(
+ Pair("String", ARTIFACT),
+ Pair("int", START_LINE),
+ Pair("int", END_LINE),
+ Pair("int", START_COLUMN),
+ Pair("int", END_COLUMN)
+ )
+
override fun toEntityAttribute(value: Map?): PhysicalLocation? {
return try {
val startLine = toInt(value?.get(START_LINE)) ?: return null
diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/NameConverter.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/NameConverter.kt
index 3c7b16e607..464fd04340 100644
--- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/NameConverter.kt
+++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/helpers/neo4j/NameConverter.kt
@@ -27,7 +27,6 @@ package de.fraunhofer.aisec.cpg.helpers.neo4j
import de.fraunhofer.aisec.cpg.graph.Name
import de.fraunhofer.aisec.cpg.graph.parseName
-import org.neo4j.ogm.typeconversion.CompositeAttributeConverter
/**
* This converter can be used in a Neo4J session to persist the [Name] class into its components:
@@ -37,7 +36,7 @@ import org.neo4j.ogm.typeconversion.CompositeAttributeConverter
*
* Additionally, it converts the aforementioned Neo4J attributes in a node back into a [Name].
*/
-class NameConverter : CompositeAttributeConverter {
+class NameConverter : CpgCompositeConverter {
companion object {
const val FIELD_FULL_NAME = "fullName"
@@ -61,14 +60,22 @@ class NameConverter : CompositeAttributeConverter {
// For reasons such as backwards compatibility and the fact that Neo4J likes to display
// nodes in the UI with a "name" field as default, we also persist the full name (aka
- // the
- // toString() representation) as "name"
+ // the toString() representation) as "name"
map[FIELD_NAME] = value.toString()
}
return map
}
+ override val graphSchema: List>
+ get() =
+ listOf(
+ Pair("String", FIELD_FULL_NAME),
+ Pair("String", FIELD_LOCAL_NAME),
+ Pair("String", FIELD_NAME),
+ Pair("String", FIELD_NAME_DELIMITER)
+ )
+
override fun toEntityAttribute(value: MutableMap): Name {
return parseName(value[FIELD_FULL_NAME].toString(), value[FIELD_NAME_DELIMITER].toString())
}
diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt
index 7adc5ed7c2..155245900a 100644
--- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt
+++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt
@@ -449,6 +449,7 @@ class Application : Callable {
printSchema(mutuallyExclusiveParameters.files)
return EXIT_SUCCESS
}
+
if (mutuallyExclusiveParameters.listPasses) {
log.info("List of passes:")
passList.iterator().forEach { log.info("- $it") }
@@ -456,6 +457,7 @@ class Application : Callable {
log.info("End of list. Stopping.")
return EXIT_SUCCESS
}
+
val translationConfiguration = setupTranslationConfiguration()
val startTime = System.currentTimeMillis()
diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Schema.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Schema.kt
index 6047667e48..65a57f84ca 100644
--- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Schema.kt
+++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Schema.kt
@@ -26,6 +26,7 @@
package de.fraunhofer.aisec.cpg_vis_neo4j
import de.fraunhofer.aisec.cpg.graph.Node
+import de.fraunhofer.aisec.cpg.helpers.neo4j.CpgCompositeConverter
import java.io.File
import java.io.PrintWriter
import java.lang.reflect.ParameterizedType
@@ -75,6 +76,22 @@ class Schema {
* MutableMap>>
*/
private val inheritedRels: MutableMap>> = mutableMapOf()
+
+ /**
+ * Relationships newly defined in this specific entity. Saves
+ * MutableMap>>
+ */
+ private val inherentProperties: MutableMap>> =
+ mutableMapOf()
+
+ /**
+ * Relationships inherited from a parent in the inheritance hierarchy. A node with this label
+ * can have this relationship if it is non-nullable. Saves
+ * MutableMap>>
+ */
+ private val inheritedProperties: MutableMap>> =
+ mutableMapOf()
+
/**
* Relationships defined by children in the inheritance hierarchy. A node with this label can
* have this relationship also has the label of the defining child entity. Saves
@@ -97,48 +114,65 @@ class Schema {
Node::class.java.isAssignableFrom(it.underlyingClass) && !it.isRelationshipEntity
} // Node to filter for, filter out what is not explicitly a
- entities.forEach {
- if (it in entities) {
- val superC = it.directSuperclass()
+ entities.forEach { entity ->
+ val superC = entity.directSuperclass()
- hierarchy[it] =
- Pair(
- if (superC in entities) superC else null,
- it.directSubclasses()
- .filter { it in entities }
- .distinct() // Filter out duplicates
- )
- }
+ hierarchy[entity] =
+ Pair(
+ if (superC in entities) superC else null,
+ entity
+ .directSubclasses()
+ .filter { it in entities }
+ .distinct() // Filter out duplicates
+ )
}
// node in neo4j
- entities.forEach {
- val key = meta.schema.findNode(it.neo4jName())
- allRels[it.neo4jName() ?: it.underlyingClass.simpleName] =
+ entities.forEach { classInfo ->
+ val key = meta.schema.findNode(classInfo.neo4jName())
+ allRels[classInfo.neo4jName() ?: classInfo.underlyingClass.simpleName] =
key.relationships().entries.map { Pair(it.key, it.value.type()) }.toSet()
}
// Complements the hierarchy and relationship information for abstract classes
completeSchema(allRels, hierarchy, nodeClassInfo)
- // Searches for all relationships backed by a class field to know which relationships are
- // newly defined in the
- // entity class
- entities.forEach {
- val entity = it
+ // Searches for all relationships and properties backed by a class field to know which
+ // of them are newly defined in the entity class
+ entities.forEach { entity ->
val fields =
entity.relationshipFields().filter {
it.field.declaringClass == entity.underlyingClass
}
fields.forEach { relationshipFields.put(Pair(entity, it.name), it) }
- val name = it.neo4jName() ?: it.underlyingClass.simpleName
- allRels[name]?.let {
+ val name = entity.neo4jName() ?: entity.underlyingClass.simpleName
+ allRels[name]?.let { relationPair ->
inherentRels[name] =
- it.filter {
- val rel = it.first
- fields.any { it.name.equals(rel) }
- }
- .toSet()
+ relationPair.filter { rel -> fields.any { it.name.equals(rel.first) } }.toSet()
+ }
+
+ entity.propertyFields().forEach { property ->
+ val persistedField =
+ if (
+ property.hasCompositeConverter() &&
+ property.compositeConverter is CpgCompositeConverter
+ ) {
+ (property.compositeConverter as CpgCompositeConverter).graphSchema
+ } else {
+ listOf>(
+ Pair(property.field.type.simpleName, property.name)
+ )
+ }
+
+ if (property.field.declaringClass == entity.underlyingClass) {
+ inherentProperties
+ .computeIfAbsent(name) { mutableSetOf() }
+ .addAll(persistedField)
+ } else {
+ inheritedProperties
+ .computeIfAbsent(name) { mutableSetOf() }
+ .addAll(persistedField)
+ }
}
}
@@ -154,8 +188,8 @@ class Schema {
allRels.forEach {
childrensRels[it.key] =
it.value
- .subtract(inheritedRels[it.key] ?: emptyList())
- .subtract(inherentRels[it.key] ?: emptyList())
+ .subtract(inheritedRels[it.key] ?: emptySet())
+ .subtract(inherentRels[it.key] ?: emptySet())
}
println()
}
@@ -189,8 +223,10 @@ class Schema {
it.neo4jName() ?: it.underlyingClass.simpleName,
hierarchy[it]
?.second
- ?.flatMap {
- relCanHave[it.neo4jName() ?: it.underlyingClass.simpleName] ?: setOf()
+ ?.flatMap { classInfo ->
+ relCanHave[
+ classInfo.neo4jName() ?: classInfo.underlyingClass.simpleName]
+ ?: setOf()
}
?.toSet()
?: setOf()
@@ -210,10 +246,14 @@ class Schema {
}
}
+ /**
+ * Prints a section for every entity with a list of labels (e.g. superclasses), a list of
+ * relationships, a dropdown with inherited relationships, a list of properties and a dropdown
+ * with inherited properties.
+ *
+ * Generates links between the boxes.
+ */
private fun printEntities(classInfo: ClassInfo, out: PrintWriter) {
- // TODO print a section for every entity. List of relationships not inherent. List of rel
- // inherent with result node. try to get links into relationship and target.
- // TODO subsection with inherent relationships.
val entityLabel = toLabel(classInfo)
out.println("## $entityLabel")
@@ -243,15 +283,13 @@ class Schema {
hierarchy[classInfo]?.second?.let {
if (it.isNotEmpty()) {
- it.forEach {
+ it.forEach { classInfo ->
out.print(
getBoxWithClass(
"child",
- "[${toLabel(it)}](#${toAnchorLink("e"+toLabel(it))})"
+ "[${toLabel(classInfo)}](#${toAnchorLink("e"+toLabel(classInfo))})"
)
)
- // out.println("click ${toLabel(it)} href
- // \"#${toAnchorLink(toLabel(it))}\"")
}
out.println()
}
@@ -261,7 +299,7 @@ class Schema {
if (inherentRels.isNotEmpty() && inheritedRels.isNotEmpty()) {
out.println("### Relationships")
- noLabelDups(inherentRels[entityLabel])?.forEach {
+ removeLabelDuplicates(inherentRels[entityLabel])?.forEach {
out.println(
getBoxWithClass(
"relationship",
@@ -269,43 +307,66 @@ class Schema {
)
)
}
- noLabelDups(inheritedRels[entityLabel])?.forEach {
- var inherited = it
- var current = classInfo
- var baseClass: ClassInfo? = null
- while (baseClass == null) {
- inherentRels[toLabel(current)]?.let {
- if (it.any { it.second.equals(inherited.second) }) {
- baseClass = current
+
+ if (inheritedRels[entityLabel]?.isNotEmpty() == true) {
+ out.println("Inherited Relationships
")
+ out.println()
+ removeLabelDuplicates(inheritedRels[entityLabel])?.forEach { inherited ->
+ var current = classInfo
+ var baseClass: ClassInfo? = null
+ while (baseClass == null) {
+ inherentRels[toLabel(current)]?.let { rels ->
+ if (rels.any { it.second == inherited.second }) {
+ baseClass = current
+ }
}
+ hierarchy[current]?.first?.let { current = it }
}
- hierarchy[current]?.first?.let { current = it }
- }
- out.println(
- getBoxWithClass(
- "inherited-relationship",
- "[${it.second}](#${toConcatName(toLabel(baseClass)+it.second)})"
+ out.println(
+ getBoxWithClass(
+ "inherited-relationship",
+ "[${inherited.second}](#${toConcatName(toLabel(baseClass) + inherited.second)})"
+ )
)
- )
+ }
+ out.println(" ")
+ out.println()
}
- noLabelDups(inherentRels[entityLabel])?.forEach {
+ removeLabelDuplicates(inherentRels[entityLabel])?.forEach {
printRelationships(classInfo, it, out)
}
}
+ if (inherentProperties.isNotEmpty() && inheritedProperties.isNotEmpty()) {
+ out.println("### Properties")
+
+ removeLabelDuplicates(inherentProperties[entityLabel])?.forEach {
+ out.println("${it.second} : ${it.first}")
+ out.println()
+ }
+ if (inheritedProperties[entityLabel]?.isNotEmpty() == true) {
+ out.println("Inherited Properties
")
+ removeLabelDuplicates(inheritedProperties[entityLabel])?.forEach {
+ out.println("${it.second} : ${it.first}")
+ out.println()
+ }
+ out.println(" ")
+ out.println()
+ }
+ }
+
hierarchy[classInfo]?.second?.forEach { printEntities(it, out) }
}
- private fun noLabelDups(list: Set>?): Set>? {
+ private fun removeLabelDuplicates(
+ list: Set>?
+ ): Set>? {
if (list == null) return null
return list
.map { it.second }
.distinct()
- .map {
- val label = it
- list.first { it.second == label }
- }
+ .map { label -> list.first { it.second == label } }
.toSet()
}
@@ -351,7 +412,7 @@ class Schema {
.filterIsInstance()
.map { it.rawType }
val baseClass: Type? = getNestedBaseType(type)
- var multiplicity = getNestedMultiplicity(type)
+ val multiplicity = getNestedMultiplicity(type)
var targetClassInfo: ClassInfo? = null
if (baseClass != null) {
@@ -375,12 +436,12 @@ class Schema {
private fun getNestedMultiplicity(type: Type): Boolean {
if (type is ParameterizedType) {
- if (
+ return if (
type.rawType.typeName.substringBeforeLast(".") == "java.util"
) { // listOf(List::class).contains(type.rawType)
- return true
+ true
} else {
- return type.actualTypeArguments.any { getNestedMultiplicity(it) }
+ type.actualTypeArguments.any { getNestedMultiplicity(it) }
}
}
return false