From d14d587cb2cab55cc0b1b92d79d0b30f12807b42 Mon Sep 17 00:00:00 2001 From: Cody Fincher <204685+cofin@users.noreply.github.com> Date: Fri, 24 May 2024 16:52:33 -0500 Subject: [PATCH] feat: `mysql` readiness and collector udpates (#443) * feat: mysql updates * chore: ignore ERA001 for a line * chore: bump versions * feat: add all tables for loading data * fix: updated mysql queries --- .pre-commit-config.yaml | 2 +- .../mysql/sql/5.7/resource_groups.sql | 30 +- .../mysql/sql/base/resource_groups.sql | 28 +- src/dma/cli/main.py | 1 + src/dma/collector/query_managers.py | 25 +- .../sql/canonical/ddls-mysql-collection.sql | 242 ++++ .../canonical/ddls-postgres-collection.sql | 2 +- .../sql/canonical/readiness-check.sql | 2 +- .../sql/sources/mysql/collection-config.sql | 1080 ++++++++--------- .../sources/mysql/collection-process_list.sql | 30 +- .../mysql/collection-resource-groups.sql | 35 + src/dma/collector/sql/sources/mysql/init.sql | 25 +- src/dma/collector/workflows/base.py | 2 + .../readiness_check/_mysql/__init__.py | 44 - .../readiness_check/_mysql/cloudsql_checks.py | 0 .../readiness_check/_mysql/constants.py | 10 + .../readiness_check/_mysql/helpers.py | 34 + .../workflows/readiness_check/_mysql/main.py | 171 +++ .../readiness_check/_postgres/__init__.py | 44 - .../workflows/readiness_check/base.py | 11 + src/dma/lib/db/adapters/asyncmy.py | 2 + tests/docker-compose.yml | 12 + 22 files changed, 1159 insertions(+), 673 deletions(-) create mode 100644 src/dma/collector/sql/canonical/ddls-mysql-collection.sql create mode 100644 src/dma/collector/sql/sources/mysql/collection-resource-groups.sql delete mode 100644 src/dma/collector/workflows/readiness_check/_mysql/cloudsql_checks.py create mode 100644 src/dma/collector/workflows/readiness_check/_mysql/constants.py create mode 100644 src/dma/collector/workflows/readiness_check/_mysql/helpers.py create mode 100644 src/dma/collector/workflows/readiness_check/_mysql/main.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7cdfd83b..dcd827c3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: # Ruff replaces black, flake8, autoflake and isort - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: "v0.4.4" # make sure this is always consistent with hatch configs + rev: "v0.4.5" # make sure this is always consistent with hatch configs hooks: - id: ruff args: [--config, ./pyproject.toml] diff --git a/scripts/collector/mysql/sql/5.7/resource_groups.sql b/scripts/collector/mysql/sql/5.7/resource_groups.sql index fe66b48a..fe8ec9e9 100644 --- a/scripts/collector/mysql/sql/5.7/resource_groups.sql +++ b/scripts/collector/mysql/sql/5.7/resource_groups.sql @@ -1,17 +1,17 @@ --- name: collector-mysql-resource-groups +-- name: collector-mysql-5-resource-groups select concat(char(34), @PKEY, char(34)) as pkey, - concat(char(34), @DMA_SOURCE_ID, char(34)) as dma_source_id, - concat(char(34), @DMA_MANUAL_ID, char(34)) as dma_manual_id, - concat(char(34), src.resource_group_name, char(34)) as resource_group_name, - concat(char(34), src.resource_group_type, char(34)) as resource_group_type, - concat(char(34), src.resource_group_enabled, char(34)) as resource_group_enabled, - concat(char(34), src.vcpu_ids, char(34)) as vcpu_ids, - concat(char(34), src.thread_priority, char(34)) as thread_priority + concat(char(34), @DMA_SOURCE_ID, char(34)) as dma_source_id, + concat(char(34), @DMA_MANUAL_ID, char(34)) as dma_manual_id, + concat(char(34), src.resource_group_name, char(34)) as resource_group_name, + concat(char(34), src.resource_group_type, char(34)) as resource_group_type, + concat(char(34), src.resource_group_enabled, char(34)) as resource_group_enabled, + concat(char(34), src.vcpu_ids, char(34)) as vcpu_ids, + concat(char(34), src.thread_priority, char(34)) as thread_priority from ( - select 'Unsupported Version Placeholder' as resource_group_type, - 0 as resource_group_enabled, - 'Placeholder Value' as resource_group_name, - '' as vcpu_ids, - 0 as thread_priority - limit 0 - ) src; + select 'Unsupported Version Placeholder' as resource_group_type, + 0 as resource_group_enabled, + 'Placeholder Value' as resource_group_name, + '' as vcpu_ids, + 0 as thread_priority + limit 0 + ) src; diff --git a/scripts/collector/mysql/sql/base/resource_groups.sql b/scripts/collector/mysql/sql/base/resource_groups.sql index 50d7aeb4..a5d49754 100644 --- a/scripts/collector/mysql/sql/base/resource_groups.sql +++ b/scripts/collector/mysql/sql/base/resource_groups.sql @@ -1,17 +1,17 @@ -- name: collector-mysql-resource-groups select concat(char(34), @PKEY, char(34)) as pkey, - concat(char(34), @DMA_SOURCE_ID, char(34)) as dma_source_id, - concat(char(34), @DMA_MANUAL_ID, char(34)) as dma_manual_id, - concat(char(34), src.resource_group_name, char(34)) as resource_group_name, - concat(char(34), src.resource_group_type, char(34)) as resource_group_type, - concat(char(34), src.resource_group_enabled, char(34)) as resource_group_enabled, - concat(char(34), src.vcpu_ids, char(34)) as vcpu_ids, - concat(char(34), src.thread_priority, char(34)) as thread_priority + concat(char(34), @DMA_SOURCE_ID, char(34)) as dma_source_id, + concat(char(34), @DMA_MANUAL_ID, char(34)) as dma_manual_id, + concat(char(34), src.resource_group_name, char(34)) as resource_group_name, + concat(char(34), src.resource_group_type, char(34)) as resource_group_type, + concat(char(34), src.resource_group_enabled, char(34)) as resource_group_enabled, + concat(char(34), src.vcpu_ids, char(34)) as vcpu_ids, + concat(char(34), src.thread_priority, char(34)) as thread_priority from ( - select rg.resource_group_type as resource_group_type, - rg.resource_group_enabled as resource_group_enabled, - rg.resource_group_name as resource_group_name, - rg.vcpu_ids as vcpu_ids, - rg.thread_priority as thread_priority - from information_schema.resource_groups rg - ) src; + select rg.resource_group_type as resource_group_type, + rg.resource_group_enabled as resource_group_enabled, + rg.resource_group_name as resource_group_name, + rg.vcpu_ids as vcpu_ids, + rg.thread_priority as thread_priority + from information_schema.resource_groups rg + ) src; diff --git a/src/dma/cli/main.py b/src/dma/cli/main.py index 6b41bfbd..f1d1c20a 100644 --- a/src/dma/cli/main.py +++ b/src/dma/cli/main.py @@ -344,6 +344,7 @@ async def _readiness_check( console.print(Padding("", 1, expand=True)) console.rule("Processing collected data.", align="left") workflow.print_summary() + workflow.dump_database(working_path) await async_engine.dispose() diff --git a/src/dma/collector/query_managers.py b/src/dma/collector/query_managers.py index 5ba6268b..c05b76ab 100644 --- a/src/dma/collector/query_managers.py +++ b/src/dma/collector/query_managers.py @@ -289,21 +289,40 @@ def get_collection_queries(self) -> set[str]: if self.db_version is None: msg = "Database Version was not set. Ensure the initialization step complete successfully." raise ApplicationError(msg) - major_version = int(self.db_version[:2]) - version_prefix = "base" if major_version > 5.8 else "5.6" + major_version = int(self.db_version[:1]) + version_prefix = "base" if major_version > 5.8 else "5" return { f"collection_mysql_{version_prefix}_resource_groups", + f"collection_mysql_{version_prefix}_process_list", "collection_mysql_config", "collection_mysql_data_types", "collection_mysql_database_details", "collection_mysql_engines", "collection_mysql_plugins", - "collection_mysql_process_list", "collection_mysql_schema_objects", "collection_mysql_table_details", "collection_mysql_users", } + def get_collection_filenames(self) -> dict[str, str]: + if self.db_version is None: + msg = "Database Version was not set. Ensure the initialization step complete successfully." + raise ApplicationError(msg) + major_version = int(self.db_version[:1]) + version_prefix = "base" if major_version > 5.8 else "5" + return { + f"collection_mysql_{version_prefix}_resource_groups": "mysql_resource_groups", + f"collection_mysql_{version_prefix}_process_list": "mysql_process_list", + "collection_mysql_config": "mysql_config", + "collection_mysql_data_types": "mysql_data_types", + "collection_mysql_database_details": "mysql_database_details", + "collection_mysql_engines": "mysql_engines", + "collection_mysql_plugins": "mysql_plugins", + "collection_mysql_schema_objects": "mysql_schema_objects", + "collection_mysql_table_details": "mysql_table_details", + "collection_mysql_users": "mysql_users", + } + class OracleCollectionQueryManager(CollectionQueryManager): def __init__( diff --git a/src/dma/collector/sql/canonical/ddls-mysql-collection.sql b/src/dma/collector/sql/canonical/ddls-mysql-collection.sql new file mode 100644 index 00000000..ddfc35b7 --- /dev/null +++ b/src/dma/collector/sql/canonical/ddls-mysql-collection.sql @@ -0,0 +1,242 @@ +-- name: ddl-collection-scripts-02! +create or replace table collection_mysql_config ( + pkey varchar, + dma_source_id varchar, + dma_manual_id varchar, + variable_category varchar, + variable_name varchar, + variable_value varchar + ); + +create or replace table collection_mysql_users ( + pkey varchar, + dma_source_id varchar, + dma_manual_id varchar, + user_host varchar, + user_count numeric + ); + +drop view if exists collection_resource_groups; + +create or replace table collection_mysql_5_resource_groups ( + pkey varchar, + dma_source_id varchar, + dma_manual_id varchar, + resource_group_name varchar, + resource_group_type varchar, + resource_group_enabled varchar, + vcpu_ids varchar, + thread_priority varchar + ); + +create or replace table collection_mysql_base_resource_groups ( + pkey varchar, + dma_source_id varchar, + dma_manual_id varchar, + resource_group_name varchar, + resource_group_type varchar, + resource_group_enabled varchar, + vcpu_ids varchar, + thread_priority varchar + ); + +create or replace view collection_resource_groups as +select pkey, + dma_source_id, + dma_manual_id, + resource_group_name, + resource_group_type, + resource_group_enabled, + vcpu_ids, + thread_priority +from collection_mysql_base_resource_groups +union all +select pkey, + dma_source_id, + dma_manual_id, + resource_group_name, + resource_group_type, + resource_group_enabled, + vcpu_ids, + thread_priority +from collection_mysql_5_resource_groups; + +create or replace table collection_mysql_config ( + pkey varchar, + dma_source_id varchar, + dma_manual_id varchar, + variable_category varchar, + variable_name varchar, + variable_value varchar + ); + +create or replace table collection_mysql_data_types ( + pkey varchar, + dma_source_id varchar, + dma_manual_id varchar, + table_catalog varchar, + table_schema varchar, + table_name varchar, + data_type varchar, + data_type_count numeric + ); + +create or replace table collection_mysql_database_details ( + pkey varchar, + dma_source_id varchar, + dma_manual_id varchar, + table_schema varchar, + total_table_count numeric, + innodb_table_count numeric, + non_innodb_table_count numeric, + total_row_count numeric, + innodb_table_row_count numeric, + non_innodb_table_row_count numeric, + total_data_size_bytes numeric, + innodb_data_size_bytes numeric, + non_innodb_data_size_bytes numeric, + total_index_size_bytes numeric, + innodb_index_size_bytes numeric, + non_innodb_index_size_bytes numeric, + total_size_bytes numeric, + innodb_total_size_bytes numeric, + non_innodb_total_size_bytes numeric, + total_index_count numeric, + innodb_index_count numeric, + non_innodb_index_count numeric + ); + +create or replace table collection_mysql_engines ( + pkey varchar, + dma_source_id varchar, + dma_manual_id varchar, + engine_name varchar, + engine_support varchar, + engine_transactions varchar, + engine_xa varchar, + engine_savepoints varchar, + engine_comment varchar + ); + +create or replace table collection_mysql_plugins ( + pkey varchar, + dma_source_id varchar, + dma_manual_id varchar, + plugin_name varchar, + plugin_version varchar, + plugin_status varchar, + plugin_type varchar, + plugin_type_version varchar, + plugin_library varchar, + plugin_library_version varchar, + plugin_author varchar, + plugin_description varchar, + plugin_license varchar, + load_option varchar + ); + +drop view if exists collection_mysql_process_list; + +create or replace table collection_mysql_base_process_list ( + pkey varchar, + dma_source_id varchar, + dma_manual_id varchar, + process_id numeric, + process_host varchar, + process_db varchar, + process_command varchar, + process_time numeric, + process_state varchar + ); + +create or replace table collection_mysql_5_process_list ( + pkey varchar, + dma_source_id varchar, + dma_manual_id varchar, + process_id numeric, + process_host varchar, + process_db varchar, + process_command varchar, + process_time numeric, + process_state varchar + ); + +create or replace view collection_mysql_process_list as +select pkey, + dma_source_id, + dma_manual_id, + process_id, + process_host, + process_db, + process_command, + process_time, + process_state +from collection_mysql_base_process_list +union all +select pkey, + dma_source_id, + dma_manual_id, + process_id, + process_host, + process_db, + process_command, + process_time, + process_state +from collection_mysql_5_process_list; + +create or replace table collection_mysql_schema_details ( + pkey varchar, + dma_source_id varchar, + dma_manual_id varchar, + table_schema varchar, + table_name varchar, + table_engine varchar, + table_rows numeric, + data_length numeric, + index_length numeric, + is_compressed numeric, + is_partitioned numeric, + partition_count numeric, + index_count numeric, + fulltext_index_count numeric, + is_encrypted numeric, + spatial_index_count numeric, + has_primary_key numeric, + row_format varchar, + table_type varchar + ); + +create or replace table collection_mysql_schema_objects ( + pkey varchar, + dma_source_id varchar, + dma_manual_id varchar, + object_catalog varchar, + object_schema varchar, + object_category varchar, + object_type varchar, + object_owner_schema varchar, + object_owner varchar, + object_name varchar + ); + +create or replace table collection_mysql_table_details ( + pkey varchar, + dma_source_id varchar, + dma_manual_id varchar, + table_schema varchar, + table_name varchar, + table_engine varchar, + table_rows numeric, + data_length numeric, + index_length numeric, + is_compressed numeric, + is_partitioned numeric, + partition_count numeric, + index_count numeric, + fulltext_index_count numeric, + is_encrypted numeric, + spatial_index_count numeric, + has_primary_key numeric, + row_format varchar, + table_type varchar + ); diff --git a/src/dma/collector/sql/canonical/ddls-postgres-collection.sql b/src/dma/collector/sql/canonical/ddls-postgres-collection.sql index debe28c9..73b983e8 100644 --- a/src/dma/collector/sql/canonical/ddls-postgres-collection.sql +++ b/src/dma/collector/sql/canonical/ddls-postgres-collection.sql @@ -1,4 +1,4 @@ --- name: ddl-postgres-01-collection-scripts! +-- name: ddl-collection-scripts-01! create or replace table collection_postgres_12_database_details( pkey VARCHAR, dma_source_id VARCHAR, diff --git a/src/dma/collector/sql/canonical/readiness-check.sql b/src/dma/collector/sql/canonical/readiness-check.sql index 05235619..7aa0fec4 100644 --- a/src/dma/collector/sql/canonical/readiness-check.sql +++ b/src/dma/collector/sql/canonical/readiness-check.sql @@ -1,4 +1,4 @@ --- name: ddl-01-readiness-check-ddl! +-- name: ddl-readiness-check-01-ddl! create or replace table database_summary( collection_key varchar, database_name varchar, diff --git a/src/dma/collector/sql/sources/mysql/collection-config.sql b/src/dma/collector/sql/sources/mysql/collection-config.sql index ba776076..7fa2a2a9 100644 --- a/src/dma/collector/sql/sources/mysql/collection-config.sql +++ b/src/dma/collector/sql/sources/mysql/collection-config.sql @@ -1,555 +1,555 @@ -- name: collection-mysql-config select distinct @PKEY as pkey, - @DMA_SOURCE_ID as dma_source_id, - @DMA_MANUAL_ID as dma_manual_id, - src.variable_category as variable_category, - src.variable_name as variable_name, - src.variable_value as variable_value + @DMA_SOURCE_ID as dma_source_id, + @DMA_MANUAL_ID as dma_manual_id, + src.variable_category as variable_category, + src.variable_name as variable_name, + src.variable_value as variable_value from ( - select 'ALL_VARIABLES' as variable_category, - variable_name, - variable_value + select 'ALL_VARIABLES' as variable_category, + variable_name, + variable_value + from ( + select variable_name, + variable_value from ( - select variable_name, - variable_value - from ( - select upper(variable_name) as variable_name, - variable_value - from performance_schema.global_variables - union - select upper(variable_name), - variable_value - from performance_schema.session_variables - where variable_name not in ( - select variable_name - from performance_schema.global_variables - ) - ) a - where a.variable_name not in ('FT_BOOLEAN_SYNTAX') - and a.variable_name not like '%PUBLIC_KEY' - and a.variable_name not like '%PRIVATE_KEY' - ) all_vars + select upper(variable_name) as variable_name, + variable_value + from performance_schema.global_variables + union + select upper(variable_name), + variable_value + from performance_schema.session_variables + where variable_name not in ( + select variable_name + from performance_schema.global_variables + ) + ) a + where a.variable_name not in ('FT_BOOLEAN_SYNTAX') + and a.variable_name not like '%PUBLIC_KEY' + and a.variable_name not like '%PRIVATE_KEY' + ) all_vars + union + select 'GLOBAL_STATUS' as variable_category, + variable_name, + variable_value + from ( + select upper(variable_name) as variable_name, + variable_value + from performance_schema.global_status a + where a.variable_name not in ('FT_BOOLEAN_SYNTAX') + and a.variable_name not like '%PUBLIC_KEY' + and a.variable_name not like '%PRIVATE_KEY' + ) global_status + union + select 'CALCULATED_METRIC' as variable_category, + variable_name, + variable_value + from ( + select 'IS_MARIADB' as variable_name, + if(upper(gv.variable_value) like '%MARIADB%', 1, 0) as variable_value + from performance_schema.global_variables gv + where gv.variable_name = 'VERSION' union - select 'GLOBAL_STATUS' as variable_category, - variable_name, - variable_value + select 'TABLE_SIZE' as variable_name, + total_data_size_bytes as variable_value from ( - select upper(variable_name) as variable_name, - variable_value - from performance_schema.global_status a - where a.variable_name not in ('FT_BOOLEAN_SYNTAX') - and a.variable_name not like '%PUBLIC_KEY' - and a.variable_name not like '%PRIVATE_KEY' - ) global_status + select sum(data_length) as total_data_size_bytes + from ( + select t.table_schema as table_schema, + t.table_name as table_name, + t.table_rows as table_rows, + t.DATA_LENGTH as DATA_LENGTH, + t.INDEX_LENGTH as INDEX_LENGTH, + t.DATA_LENGTH + t.INDEX_LENGTH as total_length, + t.ROW_FORMAT as row_format, + t.TABLE_TYPE as table_type, + t.ENGINE as table_engine, + if(pks.table_name is not null, 1, 0) as has_primary_key + from information_schema.TABLES t + left join ( + select table_schema, + TABLE_NAME + from information_schema.statistics + where table_schema not in ( + 'mysql', + 'information_schema', + 'performance_schema', + 'sys' + ) + group by table_schema, + TABLE_NAME, + index_name + having SUM( + if( + non_unique = 0 + and NULLABLE != 'YES', + 1, + 0 + ) + ) = count(*) + ) pks on ( + t.table_schema = pks.table_schema + and t.TABLE_NAME = pks.TABLE_NAME + ) + where t.table_schema not in ( + 'mysql', + 'information_schema', + 'performance_schema', + 'sys' + ) + ) user_tables + ) data_summary union - select 'CALCULATED_METRIC' as variable_category, - variable_name, - variable_value + select 'TABLE_NO_INNODB_SIZE' as variable_name, + non_innodb_data_size_bytes as variable_value from ( - select 'IS_MARIADB' as variable_name, - if(upper(gv.variable_value) like '%MARIADB%', 1, 0) as variable_value - from performance_schema.global_variables gv - where gv.variable_name = 'VERSION' - union - select 'TABLE_SIZE' as variable_name, - total_data_size_bytes as variable_value - from ( - select sum(data_length) as total_data_size_bytes - from ( - select t.table_schema as table_schema, - t.table_name as table_name, - t.table_rows as table_rows, - t.DATA_LENGTH as DATA_LENGTH, - t.INDEX_LENGTH as INDEX_LENGTH, - t.DATA_LENGTH + t.INDEX_LENGTH as total_length, - t.ROW_FORMAT as row_format, - t.TABLE_TYPE as table_type, - t.ENGINE as table_engine, - if(pks.table_name is not null, 1, 0) as has_primary_key - from information_schema.TABLES t - left join ( - select table_schema, - TABLE_NAME - from information_schema.statistics - where table_schema not in ( - 'mysql', - 'information_schema', - 'performance_schema', - 'sys' - ) - group by table_schema, - TABLE_NAME, - index_name - having SUM( - if( - non_unique = 0 - and NULLABLE != 'YES', - 1, - 0 - ) - ) = count(*) - ) pks on ( - t.table_schema = pks.table_schema - and t.TABLE_NAME = pks.TABLE_NAME - ) - where t.table_schema not in ( - 'mysql', - 'information_schema', - 'performance_schema', - 'sys' - ) - ) user_tables - ) data_summary - union - select 'TABLE_NO_INNODB_SIZE' as variable_name, - non_innodb_data_size_bytes as variable_value - from ( - select sum( - if(upper(table_engine) != 'INNODB', data_length, 0) - ) as non_innodb_data_size_bytes - from ( - select t.table_schema as table_schema, - t.table_name as table_name, - t.table_rows as table_rows, - t.DATA_LENGTH as DATA_LENGTH, - t.INDEX_LENGTH as INDEX_LENGTH, - t.DATA_LENGTH + t.INDEX_LENGTH as total_length, - t.ROW_FORMAT as row_format, - t.TABLE_TYPE as table_type, - t.ENGINE as table_engine, - if(pks.table_name is not null, 1, 0) as has_primary_key - from information_schema.TABLES t - left join ( - select table_schema, - TABLE_NAME - from information_schema.statistics - where table_schema not in ( - 'mysql', - 'information_schema', - 'performance_schema', - 'sys' - ) - group by table_schema, - TABLE_NAME, - index_name - having SUM( - if( - non_unique = 0 - and NULLABLE != 'YES', - 1, - 0 - ) - ) = count(*) - ) pks on ( - t.table_schema = pks.table_schema - and t.TABLE_NAME = pks.TABLE_NAME - ) - where t.table_schema not in ( - 'mysql', - 'information_schema', - 'performance_schema', - 'sys' - ) - ) user_tables - ) data_summary - union - select 'TABLE_INNODB_SIZE' as variable_name, - innodb_data_size_bytes as variable_value - from ( - select sum( - if(upper(table_engine) = 'INNODB', data_length, 0) - ) as innodb_data_size_bytes - from ( - select t.table_schema as table_schema, - t.table_name as table_name, - t.table_rows as table_rows, - t.DATA_LENGTH as DATA_LENGTH, - t.INDEX_LENGTH as INDEX_LENGTH, - t.DATA_LENGTH + t.INDEX_LENGTH as total_length, - t.ROW_FORMAT as row_format, - t.TABLE_TYPE as table_type, - t.ENGINE as table_engine, - if(pks.table_name is not null, 1, 0) as has_primary_key - from information_schema.TABLES t - left join ( - select table_schema, - TABLE_NAME - from information_schema.statistics - where table_schema not in ( - 'mysql', - 'information_schema', - 'performance_schema', - 'sys' - ) - group by table_schema, - TABLE_NAME, - index_name - having SUM( - if( - non_unique = 0 - and NULLABLE != 'YES', - 1, - 0 - ) - ) = count(*) - ) pks on ( - t.table_schema = pks.table_schema - and t.TABLE_NAME = pks.TABLE_NAME - ) - where t.table_schema not in ( - 'mysql', - 'information_schema', - 'performance_schema', - 'sys' - ) - ) user_tables - ) data_summary - union - select 'TABLE_COUNT' as variable_name, - total_table_count as variable_value - from ( - select count(table_name) as total_table_count - from ( - select t.table_schema as table_schema, - t.table_name as table_name, - t.table_rows as table_rows, - t.DATA_LENGTH as DATA_LENGTH, - t.INDEX_LENGTH as INDEX_LENGTH, - t.DATA_LENGTH + t.INDEX_LENGTH as total_length, - t.ROW_FORMAT as row_format, - t.TABLE_TYPE as table_type, - t.ENGINE as table_engine, - if(pks.table_name is not null, 1, 0) as has_primary_key - from information_schema.TABLES t - left join ( - select table_schema, - TABLE_NAME - from information_schema.statistics - where table_schema not in ( - 'mysql', - 'information_schema', - 'performance_schema', - 'sys' - ) - group by table_schema, - TABLE_NAME, - index_name - having SUM( - if( - non_unique = 0 - and NULLABLE != 'YES', - 1, - 0 - ) - ) = count(*) - ) pks on ( - t.table_schema = pks.table_schema - and t.TABLE_NAME = pks.TABLE_NAME - ) - where t.table_schema not in ( - 'mysql', - 'information_schema', - 'performance_schema', - 'sys' - ) - ) user_tables - ) data_summary - union - select 'TABLE_NO_INNODB_COUNT' as variable_name, - non_innodb_table_count as variable_value - from ( - select sum(if(upper(table_engine) != 'INNODB', 1, 0)) as non_innodb_table_count - from ( - select t.table_schema as table_schema, - t.table_name as table_name, - t.table_rows as table_rows, - t.DATA_LENGTH as DATA_LENGTH, - t.INDEX_LENGTH as INDEX_LENGTH, - t.DATA_LENGTH + t.INDEX_LENGTH as total_length, - t.ROW_FORMAT as row_format, - t.TABLE_TYPE as table_type, - t.ENGINE as table_engine, - if(pks.table_name is not null, 1, 0) as has_primary_key - from information_schema.TABLES t - left join ( - select table_schema, - TABLE_NAME - from information_schema.statistics - where table_schema not in ( - 'mysql', - 'information_schema', - 'performance_schema', - 'sys' - ) - group by table_schema, - TABLE_NAME, - index_name - having SUM( - if( - non_unique = 0 - and NULLABLE != 'YES', - 1, - 0 - ) - ) = count(*) - ) pks on ( - t.table_schema = pks.table_schema - and t.TABLE_NAME = pks.TABLE_NAME - ) - where t.table_schema not in ( - 'mysql', - 'information_schema', - 'performance_schema', - 'sys' - ) - ) user_tables - ) data_summary - union - select 'TABLE_INNODB_COUNT' as variable_name, - innodb_table_count as variable_value - from ( - select sum(if(upper(table_engine) = 'INNODB', 1, 0)) as innodb_table_count - from ( - select t.table_schema as table_schema, - t.table_name as table_name, - t.table_rows as table_rows, - t.DATA_LENGTH as DATA_LENGTH, - t.INDEX_LENGTH as INDEX_LENGTH, - t.DATA_LENGTH + t.INDEX_LENGTH as total_length, - t.ROW_FORMAT as row_format, - t.TABLE_TYPE as table_type, - t.ENGINE as table_engine, - if(pks.table_name is not null, 1, 0) as has_primary_key - from information_schema.TABLES t - left join ( - select table_schema, - TABLE_NAME - from information_schema.statistics - where table_schema not in ( - 'mysql', - 'information_schema', - 'performance_schema', - 'sys' - ) - group by table_schema, - TABLE_NAME, - index_name - having SUM( - if( - non_unique = 0 - and NULLABLE != 'YES', - 1, - 0 - ) - ) = count(*) - ) pks on ( - t.table_schema = pks.table_schema - and t.TABLE_NAME = pks.TABLE_NAME - ) - where t.table_schema not in ( - 'mysql', - 'information_schema', - 'performance_schema', - 'sys' - ) - ) user_tables - ) data_summary - union - select 'TABLE_NO_PK_COUNT' as variable_name, - total_tables_without_primary_key as variable_value - from ( - select sum(if(has_primary_key = 0, 1, 0)) as total_tables_without_primary_key - from ( - select t.table_schema as table_schema, - t.table_name as table_name, - t.table_rows as table_rows, - t.DATA_LENGTH as DATA_LENGTH, - t.INDEX_LENGTH as INDEX_LENGTH, - t.DATA_LENGTH + t.INDEX_LENGTH as total_length, - t.ROW_FORMAT as row_format, - t.TABLE_TYPE as table_type, - t.ENGINE as table_engine, - if(pks.table_name is not null, 1, 0) as has_primary_key - from information_schema.TABLES t - left join ( - select table_schema, - TABLE_NAME - from information_schema.statistics - where table_schema not in ( - 'mysql', - 'information_schema', - 'performance_schema', - 'sys' - ) - group by table_schema, - TABLE_NAME, - index_name - having SUM( - if( - non_unique = 0 - and NULLABLE != 'YES', - 1, - 0 - ) - ) = count(*) - ) pks on ( - t.table_schema = pks.table_schema - and t.TABLE_NAME = pks.TABLE_NAME - ) - where t.table_schema not in ( - 'mysql', - 'information_schema', - 'performance_schema', - 'sys' - ) - ) user_tables - ) data_summary - union - select 'MYSQLX_PLUGIN' as variable_name, - p.mysqlx_plugin_enabled as variable_value - from ( - select if(agg.mysqlx_plugin > 0, 1, 0) as mysqlx_plugin_enabled - from ( - select sum( - if( - upper(p.plugin_name) like '%MYSQLX%', - 1, - 0 - ) - ) as mysqlx_plugin - from ( - select p.plugin_name as plugin_name, - p.PLUGIN_STATUS - from information_schema.PLUGINS p - ) p - ) agg - ) p - union - select 'MEMCACHED_PLUGIN' as variable_name, - p.memcached_plugin_enabled as variable_value + select sum( + if(upper(table_engine) != 'INNODB', data_length, 0) + ) as non_innodb_data_size_bytes + from ( + select t.table_schema as table_schema, + t.table_name as table_name, + t.table_rows as table_rows, + t.DATA_LENGTH as DATA_LENGTH, + t.INDEX_LENGTH as INDEX_LENGTH, + t.DATA_LENGTH + t.INDEX_LENGTH as total_length, + t.ROW_FORMAT as row_format, + t.TABLE_TYPE as table_type, + t.ENGINE as table_engine, + if(pks.table_name is not null, 1, 0) as has_primary_key + from information_schema.TABLES t + left join ( + select table_schema, + TABLE_NAME + from information_schema.statistics + where table_schema not in ( + 'mysql', + 'information_schema', + 'performance_schema', + 'sys' + ) + group by table_schema, + TABLE_NAME, + index_name + having SUM( + if( + non_unique = 0 + and NULLABLE != 'YES', + 1, + 0 + ) + ) = count(*) + ) pks on ( + t.table_schema = pks.table_schema + and t.TABLE_NAME = pks.TABLE_NAME + ) + where t.table_schema not in ( + 'mysql', + 'information_schema', + 'performance_schema', + 'sys' + ) + ) user_tables + ) data_summary + union + select 'TABLE_INNODB_SIZE' as variable_name, + innodb_data_size_bytes as variable_value + from ( + select sum( + if(upper(table_engine) = 'INNODB', data_length, 0) + ) as innodb_data_size_bytes + from ( + select t.table_schema as table_schema, + t.table_name as table_name, + t.table_rows as table_rows, + t.DATA_LENGTH as DATA_LENGTH, + t.INDEX_LENGTH as INDEX_LENGTH, + t.DATA_LENGTH + t.INDEX_LENGTH as total_length, + t.ROW_FORMAT as row_format, + t.TABLE_TYPE as table_type, + t.ENGINE as table_engine, + if(pks.table_name is not null, 1, 0) as has_primary_key + from information_schema.TABLES t + left join ( + select table_schema, + TABLE_NAME + from information_schema.statistics + where table_schema not in ( + 'mysql', + 'information_schema', + 'performance_schema', + 'sys' + ) + group by table_schema, + TABLE_NAME, + index_name + having SUM( + if( + non_unique = 0 + and NULLABLE != 'YES', + 1, + 0 + ) + ) = count(*) + ) pks on ( + t.table_schema = pks.table_schema + and t.TABLE_NAME = pks.TABLE_NAME + ) + where t.table_schema not in ( + 'mysql', + 'information_schema', + 'performance_schema', + 'sys' + ) + ) user_tables + ) data_summary + union + select 'TABLE_COUNT' as variable_name, + total_table_count as variable_value + from ( + select count(table_name) as total_table_count + from ( + select t.table_schema as table_schema, + t.table_name as table_name, + t.table_rows as table_rows, + t.DATA_LENGTH as DATA_LENGTH, + t.INDEX_LENGTH as INDEX_LENGTH, + t.DATA_LENGTH + t.INDEX_LENGTH as total_length, + t.ROW_FORMAT as row_format, + t.TABLE_TYPE as table_type, + t.ENGINE as table_engine, + if(pks.table_name is not null, 1, 0) as has_primary_key + from information_schema.TABLES t + left join ( + select table_schema, + TABLE_NAME + from information_schema.statistics + where table_schema not in ( + 'mysql', + 'information_schema', + 'performance_schema', + 'sys' + ) + group by table_schema, + TABLE_NAME, + index_name + having SUM( + if( + non_unique = 0 + and NULLABLE != 'YES', + 1, + 0 + ) + ) = count(*) + ) pks on ( + t.table_schema = pks.table_schema + and t.TABLE_NAME = pks.TABLE_NAME + ) + where t.table_schema not in ( + 'mysql', + 'information_schema', + 'performance_schema', + 'sys' + ) + ) user_tables + ) data_summary + union + select 'TABLE_NO_INNODB_COUNT' as variable_name, + non_innodb_table_count as variable_value + from ( + select sum(if(upper(table_engine) != 'INNODB', 1, 0)) as non_innodb_table_count + from ( + select t.table_schema as table_schema, + t.table_name as table_name, + t.table_rows as table_rows, + t.DATA_LENGTH as DATA_LENGTH, + t.INDEX_LENGTH as INDEX_LENGTH, + t.DATA_LENGTH + t.INDEX_LENGTH as total_length, + t.ROW_FORMAT as row_format, + t.TABLE_TYPE as table_type, + t.ENGINE as table_engine, + if(pks.table_name is not null, 1, 0) as has_primary_key + from information_schema.TABLES t + left join ( + select table_schema, + TABLE_NAME + from information_schema.statistics + where table_schema not in ( + 'mysql', + 'information_schema', + 'performance_schema', + 'sys' + ) + group by table_schema, + TABLE_NAME, + index_name + having SUM( + if( + non_unique = 0 + and NULLABLE != 'YES', + 1, + 0 + ) + ) = count(*) + ) pks on ( + t.table_schema = pks.table_schema + and t.TABLE_NAME = pks.TABLE_NAME + ) + where t.table_schema not in ( + 'mysql', + 'information_schema', + 'performance_schema', + 'sys' + ) + ) user_tables + ) data_summary + union + select 'TABLE_INNODB_COUNT' as variable_name, + innodb_table_count as variable_value + from ( + select sum(if(upper(table_engine) = 'INNODB', 1, 0)) as innodb_table_count + from ( + select t.table_schema as table_schema, + t.table_name as table_name, + t.table_rows as table_rows, + t.DATA_LENGTH as DATA_LENGTH, + t.INDEX_LENGTH as INDEX_LENGTH, + t.DATA_LENGTH + t.INDEX_LENGTH as total_length, + t.ROW_FORMAT as row_format, + t.TABLE_TYPE as table_type, + t.ENGINE as table_engine, + if(pks.table_name is not null, 1, 0) as has_primary_key + from information_schema.TABLES t + left join ( + select table_schema, + TABLE_NAME + from information_schema.statistics + where table_schema not in ( + 'mysql', + 'information_schema', + 'performance_schema', + 'sys' + ) + group by table_schema, + TABLE_NAME, + index_name + having SUM( + if( + non_unique = 0 + and NULLABLE != 'YES', + 1, + 0 + ) + ) = count(*) + ) pks on ( + t.table_schema = pks.table_schema + and t.TABLE_NAME = pks.TABLE_NAME + ) + where t.table_schema not in ( + 'mysql', + 'information_schema', + 'performance_schema', + 'sys' + ) + ) user_tables + ) data_summary + union + select 'TABLE_NO_PK_COUNT' as variable_name, + total_tables_without_primary_key as variable_value + from ( + select sum(if(has_primary_key = 0, 1, 0)) as total_tables_without_primary_key + from ( + select t.table_schema as table_schema, + t.table_name as table_name, + t.table_rows as table_rows, + t.DATA_LENGTH as DATA_LENGTH, + t.INDEX_LENGTH as INDEX_LENGTH, + t.DATA_LENGTH + t.INDEX_LENGTH as total_length, + t.ROW_FORMAT as row_format, + t.TABLE_TYPE as table_type, + t.ENGINE as table_engine, + if(pks.table_name is not null, 1, 0) as has_primary_key + from information_schema.TABLES t + left join ( + select table_schema, + TABLE_NAME + from information_schema.statistics + where table_schema not in ( + 'mysql', + 'information_schema', + 'performance_schema', + 'sys' + ) + group by table_schema, + TABLE_NAME, + index_name + having SUM( + if( + non_unique = 0 + and NULLABLE != 'YES', + 1, + 0 + ) + ) = count(*) + ) pks on ( + t.table_schema = pks.table_schema + and t.TABLE_NAME = pks.TABLE_NAME + ) + where t.table_schema not in ( + 'mysql', + 'information_schema', + 'performance_schema', + 'sys' + ) + ) user_tables + ) data_summary + union + select 'MYSQLX_PLUGIN' as variable_name, + p.mysqlx_plugin_enabled as variable_value + from ( + select if(agg.mysqlx_plugin > 0, 1, 0) as mysqlx_plugin_enabled + from ( + select sum( + if( + upper(p.plugin_name) like '%MYSQLX%', + 1, + 0 + ) + ) as mysqlx_plugin from ( - select if(agg.memcached_plugin > 0, 1, 0) as memcached_plugin_enabled - from ( - select sum( - if( - upper(p.plugin_name) like '%MEMCACHED%', - 1, - 0 - ) - ) as memcached_plugin - from ( - select p.plugin_name as plugin_name, - p.PLUGIN_STATUS - from information_schema.PLUGINS p - ) p - ) agg - ) p - union - select 'CLONE_PLUGIN' as variable_name, - p.clone_plugin_enabled as variable_value + select p.plugin_name as plugin_name, + p.PLUGIN_STATUS + from information_schema.PLUGINS p + ) p + ) agg + ) p + union + select 'MEMCACHED_PLUGIN' as variable_name, + p.memcached_plugin_enabled as variable_value + from ( + select if(agg.memcached_plugin > 0, 1, 0) as memcached_plugin_enabled + from ( + select sum( + if( + upper(p.plugin_name) like '%MEMCACHED%', + 1, + 0 + ) + ) as memcached_plugin from ( - select if(agg.clone_plugin > 0, 1, 0) as clone_plugin_enabled - from ( - select sum( - if( - upper(p.plugin_name) like '%CLONE%', - 1, - 0 - ) - ) as clone_plugin - from ( - select p.plugin_name as plugin_name, - p.PLUGIN_STATUS - from information_schema.PLUGINS p - ) p - ) agg - ) p - union - select 'KEYRING_PLUGIN' as variable_name, - p.keyring_plugin_enabled as variable_value + select p.plugin_name as plugin_name, + p.PLUGIN_STATUS + from information_schema.PLUGINS p + ) p + ) agg + ) p + union + select 'CLONE_PLUGIN' as variable_name, + p.clone_plugin_enabled as variable_value + from ( + select if(agg.clone_plugin > 0, 1, 0) as clone_plugin_enabled + from ( + select sum( + if( + upper(p.plugin_name) like '%CLONE%', + 1, + 0 + ) + ) as clone_plugin from ( - select if(agg.keyring_plugin > 0, 1, 0) as keyring_plugin_enabled - from ( - select sum( - if( - upper(p.plugin_name) like '%KEYRING%', - 1, - 0 - ) - ) as keyring_plugin - from ( - select p.plugin_name as plugin_name, - p.PLUGIN_STATUS - from information_schema.PLUGINS p - ) p - ) agg - ) p - union - select 'VALIDATE_PASSWORD_PLUGIN' as variable_name, - p.validate_password_plugin_enabled as variable_value + select p.plugin_name as plugin_name, + p.PLUGIN_STATUS + from information_schema.PLUGINS p + ) p + ) agg + ) p + union + select 'KEYRING_PLUGIN' as variable_name, + p.keyring_plugin_enabled as variable_value + from ( + select if(agg.keyring_plugin > 0, 1, 0) as keyring_plugin_enabled + from ( + select sum( + if( + upper(p.plugin_name) like '%KEYRING%', + 1, + 0 + ) + ) as keyring_plugin from ( - select if(agg.validate_password_plugin > 0, 1, 0) as validate_password_plugin_enabled - from ( - select sum( - if( - upper(p.plugin_name) like '%VALIDATE_PASSWORD%', - 1, - 0 - ) - ) as validate_password_plugin - from ( - select p.plugin_name as plugin_name, - p.PLUGIN_STATUS - from information_schema.PLUGINS p - ) p - ) agg - ) p - union - select 'THREAD_POOL_PLUGIN' as variable_name, - p.thread_pool_plugin_enabled as variable_value + select p.plugin_name as plugin_name, + p.PLUGIN_STATUS + from information_schema.PLUGINS p + ) p + ) agg + ) p + union + select 'VALIDATE_PASSWORD_PLUGIN' as variable_name, + p.validate_password_plugin_enabled as variable_value + from ( + select if(agg.validate_password_plugin > 0, 1, 0) as validate_password_plugin_enabled + from ( + select sum( + if( + upper(p.plugin_name) like '%VALIDATE_PASSWORD%', + 1, + 0 + ) + ) as validate_password_plugin from ( - select if(agg.thread_pool_plugin > 0, 1, 0) as thread_pool_plugin_enabled - from ( - select sum( - if( - upper(p.plugin_name) like '%THREAD_POOL%', - 1, - 0 - ) - ) as thread_pool_plugin - from ( - select p.plugin_name as plugin_name, - p.PLUGIN_STATUS - from information_schema.PLUGINS p - ) p - ) agg - ) p - union - select 'FIREWALL_PLUGIN' as variable_name, - p.firewall_plugin_enabled as variable_value + select p.plugin_name as plugin_name, + p.PLUGIN_STATUS + from information_schema.PLUGINS p + ) p + ) agg + ) p + union + select 'THREAD_POOL_PLUGIN' as variable_name, + p.thread_pool_plugin_enabled as variable_value + from ( + select if(agg.thread_pool_plugin > 0, 1, 0) as thread_pool_plugin_enabled + from ( + select sum( + if( + upper(p.plugin_name) like '%THREAD_POOL%', + 1, + 0 + ) + ) as thread_pool_plugin from ( - select if(agg.firewall_plugin > 0, 1, 0) as firewall_plugin_enabled - from ( - select sum( - if( - upper(p.plugin_name) like '%FIREWALL%', - 1, - 0 - ) - ) as firewall_plugin - from ( - select p.plugin_name as plugin_name, - p.PLUGIN_STATUS - from information_schema.PLUGINS p - ) p - ) agg - ) p - union - select 'VERSION_NUM' as variable_name, + select p.plugin_name as plugin_name, + p.PLUGIN_STATUS + from information_schema.PLUGINS p + ) p + ) agg + ) p + union + select 'FIREWALL_PLUGIN' as variable_name, + p.firewall_plugin_enabled as variable_value + from ( + select if(agg.firewall_plugin > 0, 1, 0) as firewall_plugin_enabled + from ( + select sum( if( - version() rlike '^[0-9]+\.[0-9]+\.[0-9]+$' = 1, - version(), - SUBSTRING_INDEX(VERSION(), '.', 2) || '.0' - ) as variable_value - ) calculated_metrics - ) src; + upper(p.plugin_name) like '%FIREWALL%', + 1, + 0 + ) + ) as firewall_plugin + from ( + select p.plugin_name as plugin_name, + p.PLUGIN_STATUS + from information_schema.PLUGINS p + ) p + ) agg + ) p + union + select 'VERSION_NUM' as variable_name, + if( + version() rlike '^[0-9]+\.[0-9]+\.[0-9]+$' = 1, + version(), + concat(SUBSTRING_INDEX(VERSION(), '.', 2), '.0') + ) as variable_value + ) calculated_metrics + ) src; diff --git a/src/dma/collector/sql/sources/mysql/collection-process_list.sql b/src/dma/collector/sql/sources/mysql/collection-process_list.sql index 95be20ac..b5ad5455 100644 --- a/src/dma/collector/sql/sources/mysql/collection-process_list.sql +++ b/src/dma/collector/sql/sources/mysql/collection-process_list.sql @@ -1,11 +1,23 @@ --- name: collection-mysql-process-list +-- name: collection-mysql-base-process-list select @PKEY as pkey, - @DMA_SOURCE_ID as dma_source_id, - @DMA_MANUAL_ID as dma_manual_id, - id as process_id, - HOST as process_host, - db as process_db, - command as process_command, - TIME as process_time, - state as process_state + @DMA_SOURCE_ID as dma_source_id, + @DMA_MANUAL_ID as dma_manual_id, + id as process_id, + HOST as process_host, + db as process_db, + command as process_command, + TIME as process_time, + state as process_state +from performance_schema.processlist; + +-- name: collection-mysql-5-process-list +select @PKEY as pkey, + @DMA_SOURCE_ID as dma_source_id, + @DMA_MANUAL_ID as dma_manual_id, + id as process_id, + HOST as process_host, + db as process_db, + command as process_command, + TIME as process_time, + state as process_state from information_schema.processlist; diff --git a/src/dma/collector/sql/sources/mysql/collection-resource-groups.sql b/src/dma/collector/sql/sources/mysql/collection-resource-groups.sql new file mode 100644 index 00000000..edd7e8c2 --- /dev/null +++ b/src/dma/collector/sql/sources/mysql/collection-resource-groups.sql @@ -0,0 +1,35 @@ +-- name: collection-mysql-base-resource-groups +select concat(char(34), @PKEY, char(34)) as pkey, + concat(char(34), @DMA_SOURCE_ID, char(34)) as dma_source_id, + concat(char(34), @DMA_MANUAL_ID, char(34)) as dma_manual_id, + concat(char(34), src.resource_group_name, char(34)) as resource_group_name, + concat(char(34), src.resource_group_type, char(34)) as resource_group_type, + concat(char(34), src.resource_group_enabled, char(34)) as resource_group_enabled, + concat(char(34), src.vcpu_ids, char(34)) as vcpu_ids, + concat(char(34), src.thread_priority, char(34)) as thread_priority +from ( + select rg.resource_group_type as resource_group_type, + rg.resource_group_enabled as resource_group_enabled, + rg.resource_group_name as resource_group_name, + rg.vcpu_ids as vcpu_ids, + rg.thread_priority as thread_priority + from information_schema.resource_groups rg + ) src; + +-- name: collection-mysql-5-resource-groups +select concat(char(34), @PKEY, char(34)) as pkey, + concat(char(34), @DMA_SOURCE_ID, char(34)) as dma_source_id, + concat(char(34), @DMA_MANUAL_ID, char(34)) as dma_manual_id, + concat(char(34), src.resource_group_name, char(34)) as resource_group_name, + concat(char(34), src.resource_group_type, char(34)) as resource_group_type, + concat(char(34), src.resource_group_enabled, char(34)) as resource_group_enabled, + concat(char(34), src.vcpu_ids, char(34)) as vcpu_ids, + concat(char(34), src.thread_priority, char(34)) as thread_priority +from ( + select 'Unsupported Version Placeholder' as resource_group_type, + 0 as resource_group_enabled, + 'Placeholder Value' as resource_group_name, + '' as vcpu_ids, + 0 as thread_priority + limit 0 + ) src; diff --git a/src/dma/collector/sql/sources/mysql/init.sql b/src/dma/collector/sql/sources/mysql/init.sql index a193da69..a79cdabe 100644 --- a/src/dma/collector/sql/sources/mysql/init.sql +++ b/src/dma/collector/sql/sources/mysql/init.sql @@ -1 +1,24 @@ -select @@server_uuid as server_uuid; +-- name: init-get-db-version$ +select if( + version() rlike '^[0-9]+\.[0-9]+\.[0-9]+$' = 1, + version(), + concat(SUBSTRING_INDEX(VERSION(), '.', 2), '.0') + ) as db_version; + +-- name: init-get-execution-id$ +select concat( + 'mysql_', + a.db_version, + '_', + DATE_FORMAT(SYSDATE(), '%Y%m%d%H%i%s') + ) as execution_id +from ( + select if( + version() rlike '^[0-9]+\.[0-9]+\.[0-9]+$' = 1, + version(), + concat(SUBSTRING_INDEX(VERSION(), '.', 2), '.0') + ) as db_version + ) a; + +-- name: init-get-source-id$ +select @@server_uuid as source_id; diff --git a/src/dma/collector/workflows/base.py b/src/dma/collector/workflows/base.py index 0d7ad3a0..915e356b 100644 --- a/src/dma/collector/workflows/base.py +++ b/src/dma/collector/workflows/base.py @@ -47,8 +47,10 @@ def import_to_table(self, data: dict[str, list[dict]]) -> None: self.local_db.execute( f"insert into {table_name}({', '.join(column_name for column_name in column_names)}) select {', '.join(column_name for column_name in column_names)} from obj_{table_name}" # noqa: S608 ) + self.local_db.execute(f"drop view obj_{table_name}") def dump_database(self, export_path: Path, delimiter: str = "|") -> None: """Export the entire database with DDLs and data as CSV""" self.local_db.execute(f"export database '{export_path!s}' (format csv, delimiter '{delimiter}')") + self.console.print(f"Database exported to '{export_path!s}'") diff --git a/src/dma/collector/workflows/readiness_check/_mysql/__init__.py b/src/dma/collector/workflows/readiness_check/_mysql/__init__.py index b8777e66..9d48db4f 100644 --- a/src/dma/collector/workflows/readiness_check/_mysql/__init__.py +++ b/src/dma/collector/workflows/readiness_check/_mysql/__init__.py @@ -1,45 +1 @@ from __future__ import annotations - -from typing import TYPE_CHECKING - -import duckdb -from rich.table import Table - -if TYPE_CHECKING: - import duckdb - from rich.console import Console - - from dma.collector.query_managers import CanonicalQueryManager - - -def print_summary_mysql( - console: Console, - local_db: duckdb.DuckDBPyConnection, - manager: CanonicalQueryManager, -) -> None: - """Print Summary of the Migration Readiness Assessment.""" - summary_table = Table(show_edge=False, width=80) - print_database_details(console=console, local_db=local_db, manager=manager) - console.print(summary_table) - - -def print_database_details( - console: Console, - local_db: duckdb.DuckDBPyConnection, - manager: CanonicalQueryManager, -) -> None: - """Print Summary of the Migration Readiness Assessment.""" - calculated_metrics = local_db.sql( - """ - select variable_category, variable_name, variable_value - from collection_mysql_config - """, - ).fetchall() - count_table = Table(show_edge=False, width=80) - count_table.add_column("Variable Category", justify="right", style="green") - count_table.add_column("Variable", justify="right", style="green") - count_table.add_column("Value", justify="right", style="green") - - for row in calculated_metrics: - count_table.add_row(*[str(col) for col in row]) - console.print(count_table) diff --git a/src/dma/collector/workflows/readiness_check/_mysql/cloudsql_checks.py b/src/dma/collector/workflows/readiness_check/_mysql/cloudsql_checks.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/dma/collector/workflows/readiness_check/_mysql/constants.py b/src/dma/collector/workflows/readiness_check/_mysql/constants.py new file mode 100644 index 00000000..8ab5a303 --- /dev/null +++ b/src/dma/collector/workflows/readiness_check/_mysql/constants.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from typing import Final + +RDS_MINOR_VERSION_SUPPORT_MAP: Final[dict[float, int]] = {} +DB_TYPE_MAP: Final[dict[float, str]] = { + 5.6: "MYSQL_5_6", + 5.7: "MYSQL_5_7", + 8.0: "MYSQL_8_0", +} diff --git a/src/dma/collector/workflows/readiness_check/_mysql/helpers.py b/src/dma/collector/workflows/readiness_check/_mysql/helpers.py new file mode 100644 index 00000000..b12fc2a8 --- /dev/null +++ b/src/dma/collector/workflows/readiness_check/_mysql/helpers.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import re +from typing import cast + +mysql_version_regex = re.compile(r"(?P\d+\.\d+(\.\d+)?).*") + + +def get_db_minor_version(db_version: str) -> int: + version_match = mysql_version_regex.match(db_version) + if not version_match: + return -1 + version = version_match.group("version") + split = version.split(".") + if db_version.startswith("5"): + if len(split) > 2: + return int(split[2]) + elif len(split) > 1: + return int(split[1]) + return -1 + + +def get_db_major_version(db_version: str) -> float: + index = db_version.find("beta") + if index != -1: + db_version = db_version[:index] + ".0" + + split = db_version.split(".") + db_version = ".".join(split[:2]) if db_version.startswith("5") else ".".join(split[:1]) + try: + val = float(db_version) + except ValueError: + val = cast("float", -1) + return val diff --git a/src/dma/collector/workflows/readiness_check/_mysql/main.py b/src/dma/collector/workflows/readiness_check/_mysql/main.py new file mode 100644 index 00000000..19b5709a --- /dev/null +++ b/src/dma/collector/workflows/readiness_check/_mysql/main.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from rich.table import Table + +from dma.collector.workflows.readiness_check._mysql.constants import ( + DB_TYPE_MAP, + RDS_MINOR_VERSION_SUPPORT_MAP, +) +from dma.collector.workflows.readiness_check._mysql.helpers import get_db_major_version +from dma.collector.workflows.readiness_check.base import ( + ReadinessCheck, + ReadinessCheckExecutor, + ReadinessCheckTargetConfig, +) + +if TYPE_CHECKING: + from rich.console import Console + + from dma.types import MySQLVariants + + +@dataclass +class MySQLReadinessCheckTargetConfig(ReadinessCheckTargetConfig): + db_variant: MySQLVariants + supported_plugins: set[str] + minimum_supported_rds_major_version: float + db_version_map: dict[float, str] = field(default_factory=lambda: DB_TYPE_MAP) + rds_minor_version_support_map: dict[float, int] = field(default_factory=lambda: RDS_MINOR_VERSION_SUPPORT_MAP) + + +MYSQL_RULE_CONFIGURATIONS: list[MySQLReadinessCheckTargetConfig] = [ + MySQLReadinessCheckTargetConfig( + db_type="MYSQL", + db_variant="CLOUDSQL", + minimum_supported_major_version=5.6, + minimum_supported_rds_major_version=8.0, + maximum_supported_major_version=8, + supported_plugins=set(), + ), +] + + +class MySQLReadinessCheckExecutor(ReadinessCheckExecutor): + db_version: str + + def __init__( + self, + console: Console, + readiness_check: ReadinessCheck, + rule_config: list[MySQLReadinessCheckTargetConfig] | None = None, + ) -> None: + self.rule_config = rule_config or MYSQL_RULE_CONFIGURATIONS + super().__init__(console=console, readiness_check=readiness_check) + + def execute(self) -> None: + """Execute postgres checks""" + self._check_version() + self._check_plugins() + + def _check_version(self) -> None: + rule_code = "DATABASE_VERSION" + + detected_major_version = get_db_major_version(self.db_version) + for c in self.rule_config: + if ( + detected_major_version not in c.db_version_map + or detected_major_version < c.minimum_supported_major_version + ): + self.save_rule_result( + c.db_variant, + rule_code, + "ERROR", + f"Replication from source database server ({self.db_version}) is not supported", + ) + else: + self.save_rule_result( + c.db_variant, + rule_code, + "PASS", + f"Version {self.db_version} is supported. Please ensure that you selected a version that meets or exceeds version {detected_major_version!s}.", + ) + + def _check_plugins(self) -> None: + rule_code = "PLUGINS" + result = self.local_db.sql("select distinct plugin_name from collection_mysql_plugins").fetchmany() + plugins = {row[0] for row in result} + for c in self.rule_config: + unsupported_plugins = plugins.difference(c.supported_plugins) + for unsupported_plugin in unsupported_plugins: + self.save_rule_result( + c.db_variant, + rule_code, + "ERROR", + f"Unsupported plugin: {unsupported_plugin} is not supported on this instance", + ) + if len(unsupported_plugins) == 0: + self.save_rule_result( + c.db_variant, + rule_code, + "PASS", + "All utilized plugins are supported.", + ) + + def print_summary(self) -> None: + """Print Summary of the Migration Readiness Assessment.""" + # self._print_database_details() # noqa: ERA001 + self._print_readiness_check_summary() + + def _print_database_details( + self, + ) -> None: + """Print Summary of the Migration Readiness Assessment.""" + results = self.local_db.sql( + """ + select metric_category, metric_name, metric_value + from collection_mysql_database_details + """, + ).fetchall() + count_table = Table(min_width=80) + count_table.add_column("Variable Category", justify="right", style="green") + count_table.add_column("Variable", justify="right", style="green") + count_table.add_column("Value", justify="right", style="green") + for row in results: + count_table.add_row(*[str(col) for col in row]) + self.console.print(count_table) + + def _print_readiness_check_summary(self) -> None: + """Print Summary of the Migration Readiness Assessment.""" + db_variants: set[MySQLVariants] = {"CLOUDSQL"} + + def table_for_target(migration_target: MySQLVariants) -> None: + results = self.local_db.execute( + """ + select severity, rule_code, info + from readiness_check_summary + where migration_target = ? + """, + [migration_target], + ).fetchall() + count_table = Table( + min_width=80, title=f"{migration_target} Compatibility", leading=5, title_justify="left" + ) + count_table.add_column("Severity", justify="right") + count_table.add_column("Rule Code", justify="left") + count_table.add_column("Info", justify="left") + + for row in results: + count_table.add_row( + f"[bold green]{row[0]}[/]" + if row[0] == "PASS" + else f"[bold yellow]{row[0]}[/]" + if row[0] == "WARNING" + else f"[bold red]{row[0]}[/]", + f"[bold]{row[1]}[/]", + row[2], + ) + self.console.print(count_table) + + for v in db_variants: + table_for_target(v) + + # helper methods + def _is_rds(self) -> bool: + result = self.local_db.sql(""" + select case when a.cnt > 0 then true else false end as is_rds + from (select count() as cnt from collection_mysql_plugins where plugin_name='rdsadmin') a + """).fetchone() + return bool(result[0] > 0) if result is not None else False diff --git a/src/dma/collector/workflows/readiness_check/_postgres/__init__.py b/src/dma/collector/workflows/readiness_check/_postgres/__init__.py index a0d7cc34..9d48db4f 100644 --- a/src/dma/collector/workflows/readiness_check/_postgres/__init__.py +++ b/src/dma/collector/workflows/readiness_check/_postgres/__init__.py @@ -1,45 +1 @@ from __future__ import annotations - -from typing import TYPE_CHECKING - -import duckdb -from rich.table import Table - -if TYPE_CHECKING: - import duckdb - from rich.console import Console - - from dma.collector.query_managers import CanonicalQueryManager - - -def print_summary_postgres( - console: Console, - local_db: duckdb.DuckDBPyConnection, - manager: CanonicalQueryManager, -) -> None: - """Print Summary of the Migration Readiness Assessment.""" - summary_table = Table(show_edge=False, width=80) - print_database_details(console=console, local_db=local_db, manager=manager) - console.print(summary_table) - - -def print_database_details( - console: Console, - local_db: duckdb.DuckDBPyConnection, - manager: CanonicalQueryManager, -) -> None: - """Print Summary of the Migration Readiness Assessment.""" - calculated_metrics = local_db.sql( - """ - select metric_category, metric_name, metric_value - from collection_postgres_calculated_metrics - """, - ).fetchall() - count_table = Table(show_edge=False, width=80) - count_table.add_column("Variable Category", justify="right", style="green") - count_table.add_column("Variable", justify="right", style="green") - count_table.add_column("Value", justify="right", style="green") - - for row in calculated_metrics: - count_table.add_row(*[str(col) for col in row]) - console.print(count_table) diff --git a/src/dma/collector/workflows/readiness_check/base.py b/src/dma/collector/workflows/readiness_check/base.py index ad7f2500..b79d1a63 100644 --- a/src/dma/collector/workflows/readiness_check/base.py +++ b/src/dma/collector/workflows/readiness_check/base.py @@ -60,6 +60,17 @@ def execute_readiness_check(self) -> None: console=self.console, ) self.executor.execute() + elif self.db_type == "MYSQL": + # lazy loaded to help with circular import issues + from dma.collector.workflows.readiness_check._mysql.main import ( # noqa: PLC0415 + MySQLReadinessCheckExecutor, + ) + + self.executor = MySQLReadinessCheckExecutor( + readiness_check=self, + console=self.console, + ) + self.executor.execute() else: msg = f"{self.db_type} is not implemented." raise ApplicationError(msg) diff --git a/src/dma/lib/db/adapters/asyncmy.py b/src/dma/lib/db/adapters/asyncmy.py index 56444a62..49ed045b 100644 --- a/src/dma/lib/db/adapters/asyncmy.py +++ b/src/dma/lib/db/adapters/asyncmy.py @@ -42,6 +42,8 @@ async def select_value(self, conn, _query_name, sql, parameters): async with self.select_cursor(conn, _query_name, sql, parameters) as cur: await cur.execute(sql) result = await cur.fetchone() + if isinstance(result, dict): + return result[next(iter(result))] if result else None return result[0] if result else None @asynccontextmanager diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 2fc2bdb2..a1ce0619 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -5,6 +5,18 @@ services: - "5427:5432" # use a non-standard port here environment: POSTGRES_PASSWORD: super-secret + postgres16: + image: postgres:latest + ports: + - "5427:5432" # use a non-standard port here + environment: + POSTGRES_PASSWORD: super-secret + postgres14: + image: postgres:latest + ports: + - "5427:5432" # use a non-standard port here + environment: + POSTGRES_PASSWORD: super-secret mysql8: image: mysql:latest ports: