From 21494b59695ddc70bf74dd90a7c4ad112a81524d Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Wed, 20 Sep 2023 16:21:32 +0200 Subject: [PATCH 1/5] PRESIDECMS-2713 use faster Lucee 6 dbinfo when available https://presidecms.atlassian.net/browse/PRESIDECMS-2713 --- system/services/database/DbInfoService.cfc | 23 +++++++++++++++---- .../presideObjects/SqlSchemaSynchronizer.cfc | 12 +++++----- .../presideObjects/SqlSchemaVersioning.cfc | 2 +- tests/server.json | 3 +++ 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/system/services/database/DbInfoService.cfc b/system/services/database/DbInfoService.cfc index c981432dd8..c39d24e030 100644 --- a/system/services/database/DbInfoService.cfc +++ b/system/services/database/DbInfoService.cfc @@ -7,6 +7,7 @@ component { // CONSTRUCTOR public any function init() { + variables.hasModernDbInfo = structKeyExists( getTagData( "CF", "DBINFO" ).attributes, "filter" ); // Lucee 6 return this; } @@ -21,17 +22,31 @@ component { public query function getTableInfo( required string tableName, required string dsn ) { var table = ""; + var attrCol = { + type="tables", + name="table", + pattern="#arguments.tableName#", + datasource="#arguments.dsn#" + } + if ( variables.hasModernDbInfo ){ + attrCol.filter = "TABLE"; // this is much faster + } - dbinfo type="tables" name="table" pattern="#arguments.tableName#" datasource="#arguments.dsn#"; + dbinfo attributeCollection=attrCol; return table; } - public query function getTableColumns( required string tableName, required string dsn ) { + public query function getTableColumns( required string tableName, required string dsn, boolean detail=false ) { var columns = ""; + var attrCol = { + type: ( variables.hasModernDbInfo && !detail ) ? "columns_minimal" : "columns", + name: "columns", + table: arguments.tableName, + datasource: arguments.dsn + } - dbinfo type="columns" name="columns" table=arguments.tableName datasource=arguments.dsn; - + dbinfo attributeCollection=attrCol; return columns; } diff --git a/system/services/presideObjects/SqlSchemaSynchronizer.cfc b/system/services/presideObjects/SqlSchemaSynchronizer.cfc index 702fdeb739..fa6b71795d 100644 --- a/system/services/presideObjects/SqlSchemaSynchronizer.cfc +++ b/system/services/presideObjects/SqlSchemaSynchronizer.cfc @@ -102,7 +102,7 @@ component { } _syncForeignKeys( objects ); - for( dsn in dsns ){ + for( var dsn in dsns ){ _setDatabaseObjectVersion( entityType = "db" , entityName = "db" @@ -167,7 +167,7 @@ component { ) { var adapter = _getAdapter( dsn = arguments.dsn ); var tableExists = _tableExists( tableName=arguments.tableName, dsn=arguments.dsn ); - var tableColumns = tableExists ? valueList( _getTableColumns( tableName=arguments.tableName, dsn=arguments.dsn ).COLUMN_NAME ) : ""; + var tableColumns = tableExists ? QueryColumnData( _getTableColumns( tableName=arguments.tableName, dsn=arguments.dsn ), "COLUMN_NAME" ) : []; var columnSql = ""; var colName = ""; var column = ""; @@ -188,7 +188,7 @@ component { for( colName in ListToArray( arguments.dbFieldList ) ) { colMeta = arguments.properties[ colName ]; - if( listFindNoCase( tableColumns, colName ) && _skipSync( colMeta ) ) { + if( _skipSync( colMeta ) && ArrayContainsNoCase( tableColumns, colName ) ) { continue; } column = sql.columns[ colName ] = StructNew(); @@ -346,7 +346,7 @@ component { , required boolean skipSync ) { - var columnsFromDb = _getTableColumns( tableName=arguments.tableName, dsn=arguments.dsn ); + var columnsFromDb = _getTableColumns( tableName=arguments.tableName, dsn=arguments.dsn, detail=true ); var indexesFromDb = _getTableIndexes( tableName=arguments.tableName, dsn=arguments.dsn ); var dbColumnNames = ValueList( columnsFromDb.column_name ); var colsSql = arguments.generatedSql.columns; @@ -538,9 +538,9 @@ component { for( keyName in keys ){ key = keys[ keyName ]; if ( key.fk_table eq arguments.foreignTableName and key.fk_column eq arguments.foreignColumnName ) { - sql = adapter.getDropForeignKeySql( tableName = key.fk_table, foreignKeyName = keyName ); + dropSql = adapter.getDropForeignKeySql( tableName = key.fk_table, foreignKeyName = keyName ); - _runSql( sql = sql, dsn = arguments.dsn ); + _runSql( sql = dropSql, dsn = arguments.dsn ); } } } diff --git a/system/services/presideObjects/SqlSchemaVersioning.cfc b/system/services/presideObjects/SqlSchemaVersioning.cfc index f4cd1f5eb8..2382a484ab 100644 --- a/system/services/presideObjects/SqlSchemaVersioning.cfc +++ b/system/services/presideObjects/SqlSchemaVersioning.cfc @@ -21,7 +21,7 @@ component singleton=true { var versionRecords = ""; var versionRecord = ""; - for( dsn in dsns ){ + for( dsn in arguments.dsns ){ _checkVersionsTableExistance( dsn = dsn ); versionRecords = _runSql( sql = "select entity_type, entity_name, parent_entity_name, version_hash from _preside_generated_entity_versions order by entity_type, parent_entity_name" diff --git a/tests/server.json b/tests/server.json index 47a5bf6c03..d9f931e14e 100644 --- a/tests/server.json +++ b/tests/server.json @@ -8,6 +8,9 @@ "port":"9999" } }, + "app":{ + "cfengine": "lucee@6.0.0-SNAPSHOT+556" + }, "JVM":{ "heapSize":"2048", "minHeapSize":"1024" From 0a9a2767d0c4de24f218ebe1fbc4f46aaf8f6e29 Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Thu, 21 Sep 2023 09:10:31 +0200 Subject: [PATCH 2/5] optimize column retrieval --- system/services/database/DbInfoService.cfc | 6 ++- .../presideObjects/SqlSchemaSynchronizer.cfc | 42 +++++++++++++++++-- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/system/services/database/DbInfoService.cfc b/system/services/database/DbInfoService.cfc index c39d24e030..d934ebf8db 100644 --- a/system/services/database/DbInfoService.cfc +++ b/system/services/database/DbInfoService.cfc @@ -40,7 +40,7 @@ component { public query function getTableColumns( required string tableName, required string dsn, boolean detail=false ) { var columns = ""; var attrCol = { - type: ( variables.hasModernDbInfo && !detail ) ? "columns_minimal" : "columns", + type: ( variables.hasModernDbInfo && !arguments.detail ) ? "columns_minimal" : "columns", name: "columns", table: arguments.tableName, datasource: arguments.dsn @@ -106,4 +106,8 @@ component { return constraints; } + + public boolean function hasModernDbInfo(){ + return variables.hasModernDbInfo; + } } \ No newline at end of file diff --git a/system/services/presideObjects/SqlSchemaSynchronizer.cfc b/system/services/presideObjects/SqlSchemaSynchronizer.cfc index fa6b71795d..f0d1f0ade9 100644 --- a/system/services/presideObjects/SqlSchemaSynchronizer.cfc +++ b/system/services/presideObjects/SqlSchemaSynchronizer.cfc @@ -51,6 +51,7 @@ component { dbVersion.append( obj.sql.table.version ); } } + StructDelete( variables, "columnCache" ); // cache no longer required, release memory dbVersion.sort( "text" ); dbVersion = Hash( dbVersion.toList() ); if ( ( versions.db.db ?: "" ) neq dbVersion ) { @@ -167,7 +168,7 @@ component { ) { var adapter = _getAdapter( dsn = arguments.dsn ); var tableExists = _tableExists( tableName=arguments.tableName, dsn=arguments.dsn ); - var tableColumns = tableExists ? QueryColumnData( _getTableColumns( tableName=arguments.tableName, dsn=arguments.dsn ), "COLUMN_NAME" ) : []; + var tableColumns = tableExists ? _getTableColumns( tableName=arguments.tableName, dsn=arguments.dsn ) : []; var columnSql = ""; var colName = ""; var column = ""; @@ -346,7 +347,7 @@ component { , required boolean skipSync ) { - var columnsFromDb = _getTableColumns( tableName=arguments.tableName, dsn=arguments.dsn, detail=true ); + var columnsFromDb = _getTableColumnsWithForeignKeys( tableName=arguments.tableName, dsn=arguments.dsn, detail=true ); var indexesFromDb = _getTableIndexes( tableName=arguments.tableName, dsn=arguments.dsn ); var dbColumnNames = ValueList( columnsFromDb.column_name ); var colsSql = arguments.generatedSql.columns; @@ -702,7 +703,30 @@ component { return variables._tableCache[ arguments.dsn ][ arguments.tableName ] ?: false; } - private query function _getTableColumns() { + private array function _getTableColumns() { + try { + if ( _getDbInfoService().hasModernDbInfo() ){ + // with the faster version, we can fetch the entire schema in one call. + if ( !StructKeyExists( variables, "columnCache" )){ + variables.columnCache = _getTableColumnCache( argumentCollection = arguments ); + } + if ( StructKeyExists( variables.columnCache, arguments.tableName ) ) { + return variables.columnCache[ arguments.tableName ]; + } else { + return []; + } + } + var tableColumns = _getDbInfoService().getTableColumns( argumentCollection = arguments ); + return QueryColumnData( tableColumns, "COLUMN_NAME" ); + } catch( any e ) { + if ( e.message contains "there is no table that match the following pattern" ) { + return []; + } + rethrow; + } + } + + private query function _getTableColumnsWithForeignKeys() { try { return _getDbInfoService().getTableColumns( argumentCollection = arguments ); } catch( any e ) { @@ -771,6 +795,18 @@ component { } } + private struct function _getTableColumnCache(){ + var srcColumns = _getDbInfoService().getTableColumns( tableName="%", dsn=arguments.dsn, detail=false ); + var columns = {}; + loop query=srcColumns { + if ( !StructKeyExists( columns, srcColumns.table_name ) ) { + columns[ srcColumns.table_name ] = []; + } + ArrayAppend( columns[ srcColumns.table_name ], srcColumns.column_name ); + } + return columns; + } + // GETTERS AND SETTERS From 786a784b9be3e0426208e564224e9d2bd4b78a9f Mon Sep 17 00:00:00 2001 From: Zac Spitzer Date: Mon, 25 Sep 2023 11:14:19 +0200 Subject: [PATCH 3/5] revert CI to run on default lucee version (i.e. not 6) --- tests/server.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/server.json b/tests/server.json index d9f931e14e..47a5bf6c03 100644 --- a/tests/server.json +++ b/tests/server.json @@ -8,9 +8,6 @@ "port":"9999" } }, - "app":{ - "cfengine": "lucee@6.0.0-SNAPSHOT+556" - }, "JVM":{ "heapSize":"2048", "minHeapSize":"1024" From 96fbedeff6e4af60703d2150e0e80da89e9e5c2d Mon Sep 17 00:00:00 2001 From: Dominic Watson Date: Mon, 25 Sep 2023 13:55:13 +0100 Subject: [PATCH 4/5] [twgit] Init feature 'feature-PRESIDECMS-2713_use-faster-dbinfo-where-available'. From 25c8d6401f3b7f09bfa27e3af5efaf0ef617ae2a Mon Sep 17 00:00:00 2001 From: Dominic Watson Date: Mon, 25 Sep 2023 13:59:59 +0100 Subject: [PATCH 5/5] PRESIDECMS-2713 coding style cleanse --- system/services/database/DbInfoService.cfc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/system/services/database/DbInfoService.cfc b/system/services/database/DbInfoService.cfc index d934ebf8db..eeca0d975b 100644 --- a/system/services/database/DbInfoService.cfc +++ b/system/services/database/DbInfoService.cfc @@ -23,10 +23,10 @@ component { public query function getTableInfo( required string tableName, required string dsn ) { var table = ""; var attrCol = { - type="tables", - name="table", - pattern="#arguments.tableName#", - datasource="#arguments.dsn#" + type = "tables" + , name = "table" + , pattern = "#arguments.tableName#" + , datasource = "#arguments.dsn#" } if ( variables.hasModernDbInfo ){ attrCol.filter = "TABLE"; // this is much faster @@ -40,10 +40,10 @@ component { public query function getTableColumns( required string tableName, required string dsn, boolean detail=false ) { var columns = ""; var attrCol = { - type: ( variables.hasModernDbInfo && !arguments.detail ) ? "columns_minimal" : "columns", - name: "columns", - table: arguments.tableName, - datasource: arguments.dsn + type = ( variables.hasModernDbInfo && !arguments.detail ) ? "columns_minimal" : "columns" + , name = "columns" + , table = arguments.tableName + , datasource = arguments.dsn } dbinfo attributeCollection=attrCol;