more cleaning and fixing

WebFuzzing · Dec 18, 2024 · a572f05 · a572f05
1 parent 5c5a4e4
commit a572f05
Show file tree

Hide file tree

Showing 6 changed files with 137 additions and 65 deletions.
diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/resource/ResourceCluster.kt
@@ -127,16 +127,16 @@ class ResourceCluster {
      * @return existing rows of the table based on the specified [tableName]
      */
     fun getDataInDb(tableName: String) : MutableList<DataRowDto>?{
-        val found = dataInDB.filterKeys { k-> k.equals(tableName, ignoreCase = true) }.keys
-        if (found.isEmpty()) return null
-        Lazy.assert{found.size == 1}
-        return dataInDB.getValue(found.first())
+        val key = SqlActionUtils.getTableKey(dataInDB.keys, tableName)
+            ?: return null
+        return dataInDB.getValue(key)
     }
 
     /**
      * @return table class based on specified [name]
      */
-    fun getTableByName(name : String) = tables.keys.find { it.equals(name, ignoreCase = true) }?.run { tables[this] }
+    private fun getTableByName(name : String) =
+        SqlActionUtils.getTableKey(tables.keys, name)?.run { tables[this] }
 
 
     /**

diff --git a/...main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt b/...main/kotlin/org/evomaster/core/problem/rest/resource/dependency/ResourceRelatedToTable.kt
@@ -5,6 +5,7 @@ import org.evomaster.core.sql.SQLKey
 import org.evomaster.core.problem.rest.param.BodyParam
 import org.evomaster.core.problem.api.param.Param
 import org.evomaster.core.problem.util.inference.model.MatchedInfo
+import org.evomaster.core.sql.SqlActionUtils
 
 /**
  * related info between resource and tables
@@ -58,8 +59,14 @@ class ResourceRelatedToTable(val key: String) {
 
     fun updateActionRelatedToTable(verb : String, dto: SqlExecutionsDto, existingTables : Set<String>) : Boolean{
 
-        val tables = mutableListOf<String>().plus(dto.deletedData).plus(dto.updatedData.keys).plus(dto.insertedData.keys).plus(dto.queriedData.keys)
-                .filter { existingTables.contains(it) || existingTables.any { e->e.toLowerCase() == it.toLowerCase() }}.toHashSet()
+        val tables = mutableListOf<String>()
+            .plus(dto.deletedData)
+            .plus(dto.updatedData.keys)
+            .plus(dto.insertedData.keys)
+            .plus(dto.queriedData.keys)
+            .filter { x ->
+                existingTables.any { x.equals(it, true) }
+            }.toHashSet()
 
         if (tables.isEmpty()) return false
 
@@ -74,7 +81,7 @@ class ResourceRelatedToTable(val key: String) {
             actionToTables[verb]!!.add(access)
         }
 
-        access.updateTableWithFields(dto.deletedData.map { Pair(it, mutableSetOf<String>()) }.toMap(), SQLKey.DELETE)
+        access.updateTableWithFields(dto.deletedData.associateWith { mutableSetOf<String>() }, SQLKey.DELETE)
         access.updateTableWithFields(dto.insertedData, SQLKey.INSERT)
         access.updateTableWithFields(dto.queriedData, SQLKey.SELECT)
         access.updateTableWithFields(dto.updatedData, SQLKey.UPDATE)
@@ -83,15 +90,23 @@ class ResourceRelatedToTable(val key: String) {
     }
 
     fun getConfirmedDirectTables() : Set<String>{
-        return derivedMap.keys.filter { t-> confirmedSet.any { it.key.equals(t, ignoreCase = true) && it.value } }.toHashSet()
+        return derivedMap.keys
+            .filter { t->
+                confirmedSet.any { it.key.equals(t, ignoreCase = true) && it.value }
+            }.toHashSet()
     }
 
 
-    fun findBestTableForParam(tables: Set<String>, simpleP2Table : SimpleParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : Pair<Set<String>, Double>? {
-        val map = simpleP2Table.derivedMap.filter { tables.any { t-> t.equals(it.key, ignoreCase = true) } }
-                .map {
+    fun findBestTableForParam(
+        tables: Set<String>,
+        simpleP2Table : SimpleParamRelatedToTable,
+        onlyConfirmedColumn : Boolean = false
+    ) : Pair<Set<String>, Double>? {
+        val map = simpleP2Table.derivedMap
+            .filter { d -> tables.any { t-> SqlActionUtils.isMatchingTableName(d.key, t) } }
+            .map {
                     Pair(it.key, it.value.similarity)
-                }.toMap()
+            }.toMap()
         if(map.isEmpty()) return null
         val best = map.asSequence().sortedBy { it.value }.last().value
         return Pair(map.filter { it.value == best }.keys, best)
@@ -100,11 +115,18 @@ class ResourceRelatedToTable(val key: String) {
     /**
      * return Pair.first is name of table, Pair.second is column of Table
      */
-    fun getSimpleParamToSpecifiedTable(table: String, simpleP2Table : SimpleParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : Pair<String, String>? {
-        simpleP2Table.derivedMap.filter { table.equals(it.key, ignoreCase = true) }.let { map->
-            if (map.isEmpty()) return null
-            else return Pair(table, map.values.first().targetMatched)
-        }
+    fun getSimpleParamToSpecifiedTable(
+        table: String,
+        simpleP2Table : SimpleParamRelatedToTable,
+        onlyConfirmedColumn : Boolean = false
+    ) : Pair<String, String>? {
+
+        return simpleP2Table.derivedMap
+            .filter { table.equals(it.key, ignoreCase = true) }
+            .let { map->
+                if (map.isEmpty())  null
+                else  Pair(table, map.values.first().targetMatched)
+            }
     }
 
 
@@ -115,7 +137,9 @@ class ResourceRelatedToTable(val key: String) {
      */
     fun findBestTableForParam(tables: Set<String>, bodyP2Table : BodyParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : MutableMap<String, Pair<Set<String>, Double>>?{
         val fmap = bodyP2Table.fieldsMap
-                .filter { it.value.derivedMap.any { m-> tables.any { t-> t.equals(m.key, ignoreCase = true) } } }
+                .filter {
+                    it.value.derivedMap.any { m-> tables.any { t-> t.equals(m.key, ignoreCase = true) } }
+                }
         if(fmap.isEmpty()) return null
 
         val result : MutableMap<String, Pair<Set<String>, Double>> = mutableMapOf()
@@ -134,7 +158,9 @@ class ResourceRelatedToTable(val key: String) {
      */
     fun getBodyParamToSpecifiedTable(table:String, bodyP2Table : BodyParamRelatedToTable, onlyConfirmedColumn : Boolean = false) : Pair<String, Map<String, String>>? {
         val fmap = bodyP2Table.fieldsMap
-                .filter { it.value.derivedMap.any { m->m.key.toLowerCase() == table.toLowerCase() } }
+                .filter {
+                    it.value.derivedMap.any { m->m.key.toLowerCase() == table.toLowerCase() }
+                }
         if(fmap.isEmpty()) return null
         else{
             return Pair(table, fmap.map { f-> Pair(f.key, f.value.getRelatedColumn(table)!!.first())}.toMap())
@@ -148,7 +174,7 @@ class ResourceRelatedToTable(val key: String) {
      */
     fun getBodyParamToSpecifiedTable(table:String, bodyP2Table : BodyParamRelatedToTable, fieldName : String, onlyConfirmedColumn : Boolean = false) : Pair<String, Pair<String, String>>? {
         val fmap = bodyP2Table.fieldsMap[fieldName]?: return null
-        fmap.derivedMap.filter { it.key.toLowerCase() == table.toLowerCase() }.let {
+        fmap.derivedMap.filter { SqlActionUtils.isMatchingTableName(it.key, table)}.let {
             if (it.isEmpty()) return null
             else return Pair(table, Pair(fieldName, it.values.first().targetMatched))
         }
@@ -165,9 +191,10 @@ class ResourceRelatedToTable(val key: String) {
     }
 
     private fun getTablesInDerivedMap(input : String) : List<MatchedInfo>{
-        return derivedMap.values.flatMap {
-            it.filter { m-> m.input.toLowerCase() == input.toLowerCase()}
-        }
+        return derivedMap.values
+            .flatMap {
+                it.filter { m-> m.input.toLowerCase() == input.toLowerCase()}
+             }
     }
 
 }
@@ -232,7 +259,13 @@ abstract class ParamRelatedToTable (
 
     val confirmedColumn : MutableSet<String> = mutableSetOf()
 
-    open fun getRelatedColumn(table: String) : Set<String>?  = derivedMap.filterKeys { it.equals(table, ignoreCase = true) }.values.firstOrNull().run { if (this == null) null else setOf(this.targetMatched)  }
+    open fun getRelatedColumn(table: String) : Set<String>?  =
+        derivedMap.filterKeys { SqlActionUtils.isMatchingTableName(it, table)}
+            .values.firstOrNull()
+            .run {
+                if (this == null) null
+                else setOf(this.targetMatched)
+            }
 }
 
 /**
@@ -254,23 +287,31 @@ class BodyParamRelatedToTable(key: String, val referParam: Param): ParamRelatedT
      */
     val fieldsMap : MutableMap<String, ParamFieldRelatedToTable> = mutableMapOf()
 
-    override fun getRelatedColumn(table: String) : Set<String>? = fieldsMap.values.filter { it.derivedMap.any {  m-> m.key.toLowerCase() == table.toLowerCase() }}.run {
-        if (this.isEmpty()) return null
-        else
-            this.map { f->f.derivedMap.filterKeys { k->k.toLowerCase() == table.toLowerCase() }.asSequence().first().value.targetMatched}.toHashSet()
-    }
-
+    override fun getRelatedColumn(table: String) : Set<String>? =
+        fieldsMap.values
+            .filter {
+                it.derivedMap.any {  m-> SqlActionUtils.isMatchingTableName(m.key, table)}
+            }.run {
+                 if (this.isEmpty())
+                     return null
+                 else
+                    this.map { f->
+                        f.derivedMap.filterKeys { k -> SqlActionUtils.isMatchingTableName(k, table) }
+                            .asSequence().first().value.targetMatched
+                    }.toHashSet()
+            }
 }
 /**
  * related info between a field of BodyParam and a table
  */
 class ParamFieldRelatedToTable(key: String) : ParamRelatedToTable(key){
 
     override fun getRelatedColumn(table: String) :  Set<String>? {
-        derivedMap.filter { it.key.toLowerCase() == table.toLowerCase() }.let {
-            if(it.isEmpty()) return null
-            else return it.values.map {m-> m.targetMatched}.toSet()
-        }
+        return derivedMap.filter { SqlActionUtils.isMatchingTableName(it.key, table)}
+            .let {
+                if(it.isEmpty()) null
+                else  it.values.map {m-> m.targetMatched}.toSet()
+            }
     }
 }
 

diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlActionUtils.kt
@@ -495,7 +495,57 @@ object SqlActionUtils {
      * @return a list of dbactions from [sqlActions] whose related table is [tableName]
      */
     fun findDbActionsByTableName(sqlActions: List<SqlAction>, tableName : String) : List<SqlAction>{
-        return sqlActions.filter { it.table.name.equals(tableName, ignoreCase = true) }
+        return sqlActions.filter { it.table.name.equals(tableName, ignoreCase = true)
+                || it.table.name.endsWith(".$tableName", true)}
     }
 
+
+    /**
+     * Are the 2 names matching? This ignore case.
+     * The first [fullName] is a full qualifying name, including schema.
+     * The second [name] "might" be simple, or full qualifying.
+     */
+    fun isMatchingTableName(fullName: String, name: String) : Boolean{
+
+        if(fullName.equals(name, ignoreCase = true)){
+            return true
+        }
+
+        if(name.contains(".")){
+            return false
+        }
+
+        return fullName.endsWith(".$name", true)
+    }
+
+    /**
+     * Given a set of table ids, returns the one matching the given table name.
+     * There are at least 2 problems handled here:
+     * 1) case sensitiveness
+     * 2) keys having full names (eg, including schemas and possibly catalog) whereas input only having the name.
+     *    this latter case is not a problem if names are unique
+     */
+    fun getTableKey(keys: Set<String>, tableName: String) : String?{
+        /*
+         * SQL is not case sensitivity, table/column must ignore case sensitivity.
+         * No, this is not really true...
+         * Usually, names are lowered-cased by the DB, unless quoted in "":
+         * https://docs.aws.amazon.com/dms/latest/sql-server-to-aurora-postgresql-migration-playbook/chap-sql-server-aurora-pg.sql.casesensitivity.html#:~:text=By%20default%2C%20SQL%20Server%20names,names%20in%20lowercase%20for%20PostgreSQL.
+         *
+         */
+        val tableNameKey = keys.find { tableName.equals(it, ignoreCase = true) }
+        if (!tableName.contains(".") &&  tableNameKey == null){
+            //input name might be without schema, so check for partial match
+            val candidates = keys.filter { it.endsWith(".${tableName}", true) }
+            if(candidates.size > 1){
+                throw IllegalArgumentException("Ambiguity." +
+                        " More than one candidate of table called $tableName." +
+                        " Values: ${candidates.joinToString(", ")}")
+            }
+            if(candidates.size == 1){
+                return candidates[0]
+            }
+        }
+        return tableNameKey
+    }
 }
diff --git a/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt b/core/src/main/kotlin/org/evomaster/core/sql/SqlInsertBuilder.kt
@@ -180,7 +180,7 @@ class SqlInsertBuilder(
 
         for (fk in tableDto.foreignKeys) {
 
-            val tableKey = getTableKey(tableToColumns.keys, fk.targetTable)
+            val tableKey = SqlActionUtils.getTableKey(tableToColumns.keys, fk.targetTable)
 
             if(tableKey == null || tableToColumns[tableKey] == null) {
                 throw IllegalArgumentException("Foreign key for non-existent table ${fk.targetTable}")
@@ -624,43 +624,21 @@ class SqlInsertBuilder(
      *
      * quotes "" can be used to force case-sensitivity
      */
-    fun isTable(tableName: String) =  getTableKey(tables.keys, tableName) != null
+    fun isTable(tableName: String) =  SqlActionUtils.getTableKey(tables.keys, tableName) != null
 
     private fun <T>  getValueByTableNameKey(map: Map<String, T>, tableName: String) : T?{
 
-        val key = getTableKey(map.keys, tableName)
+        val key = SqlActionUtils.getTableKey(map.keys, tableName)
         return map[key]
     }
 
-    private fun getTableKey(keys: Set<String>, tableName: String) : String?{
-        /*
-         * SQL is not case sensitivity, table/column must ignore case sensitivity.
-         * No, this is not really true...
-         * Usually, names are lowered-cased by the DB, unless quoted in "":
-         * https://docs.aws.amazon.com/dms/latest/sql-server-to-aurora-postgresql-migration-playbook/chap-sql-server-aurora-pg.sql.casesensitivity.html#:~:text=By%20default%2C%20SQL%20Server%20names,names%20in%20lowercase%20for%20PostgreSQL.
-         *
-         */
-        val tableNameKey = keys.find { tableName.equals(it, ignoreCase = true) }
-        if (!tableName.contains(".") &&  tableNameKey == null){
-            //input name might be without schema, so check for partial match
-            val candidates = keys.filter { it.endsWith(".${tableName}", true) }
-            if(candidates.size > 1){
-                throw IllegalArgumentException("Ambiguity." +
-                        " More than one candidate of table called $tableName." +
-                        " Values: ${candidates.joinToString(", ")}")
-            }
-            if(candidates.size == 1){
-                return candidates[0]
-            }
-        }
-        return tableNameKey
-    }
+
 
     fun getTable(tableName: String, useExtraConstraints: Boolean): Table {
 
         val data = if (useExtraConstraints) extendedTables else tables
 
-        val tableNameKey = getTableKey(data.keys, tableName)
+        val tableNameKey = SqlActionUtils.getTableKey(data.keys, tableName)
             ?: throw IllegalArgumentException("No table called $tableName")
 
         return data[tableNameKey] ?: throw IllegalArgumentException("No table called $tableName")

diff --git a/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt b/core/src/main/kotlin/org/evomaster/core/sql/schema/Table.kt
@@ -7,6 +7,9 @@ import org.evomaster.dbconstraint.TableConstraint
  * Should be immutable
  */
 data class Table(
+        /**
+         * This usually would be fully qualified, ie, including schema
+         */
         val name: String,
 
         val columns: Set<Column>,

diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/resource/ResourceNodeWithDbTest.kt
@@ -69,7 +69,7 @@ class ResourceNodeWithDbTest {
         assertEquals(6, cluster.getCluster().size)
 
         // table in db
-        assertTrue(cluster.getTableInfo().keys.containsAll(setOf("RFOO", "RBAR", "RXYZ")))
+        assertTrue(cluster.getTableInfo().keys.containsAll(setOf("PUBLIC.RFOO", "PUBLIC.RBAR", "PUBLIC.RXYZ")))
 
         // data in db
         assertEquals(2, cluster.getDataInDb("RFOO")?.size)
@@ -79,7 +79,7 @@ class ResourceNodeWithDbTest {
         val rfooNode = cluster.getResourceNode("/v3/api/rfoo")
         assertNotNull(rfooNode)
         rfooNode!!.resourceToTable.apply {
-            assertTrue(derivedMap.keys.contains("RFOO"))
+            assertTrue(derivedMap.keys.contains("PUBLIC.RFOO"), "Keys: ${derivedMap.keys.joinToString(", ")}")
             assertEquals(1, paramToTable.size)
             assertTrue(paramToTable.values.first() is BodyParamRelatedToTable)
             (paramToTable.values.first() as BodyParamRelatedToTable).apply {