feat: mysql readiness and collector udpates (#443)

* feat: mysql updates

* chore: ignore ERA001 for a line

* chore: bump versions

* feat: add all tables for loading data

* fix: updated mysql queries

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
 
   # Ruff replaces black, flake8, autoflake and isort
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.4.4" # make sure this is always consistent with hatch configs
+    rev: "v0.4.5" # make sure this is always consistent with hatch configs
     hooks:
       - id: ruff
         args: [--config, ./pyproject.toml]

scripts/collector/mysql/sql/5.7/resource_groups.sql

@@ -1,17 +1,17 @@
--- name: collector-mysql-resource-groups
+-- name: collector-mysql-5-resource-groups
 select concat(char(34), @PKEY, char(34)) as pkey,
-    concat(char(34), @DMA_SOURCE_ID, char(34)) as dma_source_id,
-    concat(char(34), @DMA_MANUAL_ID, char(34)) as dma_manual_id,
-    concat(char(34), src.resource_group_name, char(34)) as resource_group_name,
-    concat(char(34), src.resource_group_type, char(34)) as resource_group_type,
-    concat(char(34), src.resource_group_enabled, char(34)) as resource_group_enabled,
-    concat(char(34), src.vcpu_ids, char(34)) as vcpu_ids,
-    concat(char(34), src.thread_priority, char(34)) as thread_priority
+  concat(char(34), @DMA_SOURCE_ID, char(34)) as dma_source_id,
+  concat(char(34), @DMA_MANUAL_ID, char(34)) as dma_manual_id,
+  concat(char(34), src.resource_group_name, char(34)) as resource_group_name,
+  concat(char(34), src.resource_group_type, char(34)) as resource_group_type,
+  concat(char(34), src.resource_group_enabled, char(34)) as resource_group_enabled,
+  concat(char(34), src.vcpu_ids, char(34)) as vcpu_ids,
+  concat(char(34), src.thread_priority, char(34)) as thread_priority
 from (
-        select 'Unsupported Version Placeholder' as resource_group_type,
-            0 as resource_group_enabled,
-            'Placeholder Value' as resource_group_name,
-            '' as vcpu_ids,
-            0 as thread_priority
-        limit 0
-    ) src;
+    select 'Unsupported Version Placeholder' as resource_group_type,
+      0 as resource_group_enabled,
+      'Placeholder Value' as resource_group_name,
+      '' as vcpu_ids,
+      0 as thread_priority
+    limit 0
+  ) src;

scripts/collector/mysql/sql/base/resource_groups.sql

@@ -1,17 +1,17 @@
 -- name: collector-mysql-resource-groups
 select concat(char(34), @PKEY, char(34)) as pkey,
-    concat(char(34), @DMA_SOURCE_ID, char(34)) as dma_source_id,
-    concat(char(34), @DMA_MANUAL_ID, char(34)) as dma_manual_id,
-    concat(char(34), src.resource_group_name, char(34)) as resource_group_name,
-    concat(char(34), src.resource_group_type, char(34)) as resource_group_type,
-    concat(char(34), src.resource_group_enabled, char(34)) as resource_group_enabled,
-    concat(char(34), src.vcpu_ids, char(34)) as vcpu_ids,
-    concat(char(34), src.thread_priority, char(34)) as thread_priority
+  concat(char(34), @DMA_SOURCE_ID, char(34)) as dma_source_id,
+  concat(char(34), @DMA_MANUAL_ID, char(34)) as dma_manual_id,
+  concat(char(34), src.resource_group_name, char(34)) as resource_group_name,
+  concat(char(34), src.resource_group_type, char(34)) as resource_group_type,
+  concat(char(34), src.resource_group_enabled, char(34)) as resource_group_enabled,
+  concat(char(34), src.vcpu_ids, char(34)) as vcpu_ids,
+  concat(char(34), src.thread_priority, char(34)) as thread_priority
 from (
-        select rg.resource_group_type as resource_group_type,
-            rg.resource_group_enabled as resource_group_enabled,
-            rg.resource_group_name as resource_group_name,
-            rg.vcpu_ids as vcpu_ids,
-            rg.thread_priority as thread_priority
-        from information_schema.resource_groups rg
-    ) src;
+    select rg.resource_group_type as resource_group_type,
+      rg.resource_group_enabled as resource_group_enabled,
+      rg.resource_group_name as resource_group_name,
+      rg.vcpu_ids as vcpu_ids,
+      rg.thread_priority as thread_priority
+    from information_schema.resource_groups rg
+  ) src;

src/dma/cli/main.py

@@ -344,6 +344,7 @@ async def _readiness_check(
             console.print(Padding("", 1, expand=True))
             console.rule("Processing collected data.", align="left")
             workflow.print_summary()
+            workflow.dump_database(working_path)
         await async_engine.dispose()
 
 

src/dma/collector/query_managers.py

@@ -289,21 +289,40 @@ def get_collection_queries(self) -> set[str]:
         if self.db_version is None:
             msg = "Database Version was not set.  Ensure the initialization step complete successfully."
             raise ApplicationError(msg)
-        major_version = int(self.db_version[:2])
-        version_prefix = "base" if major_version > 5.8 else "5.6"
+        major_version = int(self.db_version[:1])
+        version_prefix = "base" if major_version > 5.8 else "5"
         return {
             f"collection_mysql_{version_prefix}_resource_groups",
+            f"collection_mysql_{version_prefix}_process_list",
             "collection_mysql_config",
             "collection_mysql_data_types",
             "collection_mysql_database_details",
             "collection_mysql_engines",
             "collection_mysql_plugins",
-            "collection_mysql_process_list",
             "collection_mysql_schema_objects",
             "collection_mysql_table_details",
             "collection_mysql_users",
         }
 
+    def get_collection_filenames(self) -> dict[str, str]:
+        if self.db_version is None:
+            msg = "Database Version was not set.  Ensure the initialization step complete successfully."
+            raise ApplicationError(msg)
+        major_version = int(self.db_version[:1])
+        version_prefix = "base" if major_version > 5.8 else "5"
+        return {
+            f"collection_mysql_{version_prefix}_resource_groups": "mysql_resource_groups",
+            f"collection_mysql_{version_prefix}_process_list": "mysql_process_list",
+            "collection_mysql_config": "mysql_config",
+            "collection_mysql_data_types": "mysql_data_types",
+            "collection_mysql_database_details": "mysql_database_details",
+            "collection_mysql_engines": "mysql_engines",
+            "collection_mysql_plugins": "mysql_plugins",
+            "collection_mysql_schema_objects": "mysql_schema_objects",
+            "collection_mysql_table_details": "mysql_table_details",
+            "collection_mysql_users": "mysql_users",
+        }
+
 
 class OracleCollectionQueryManager(CollectionQueryManager):
     def __init__(

src/dma/collector/sql/canonical/ddls-mysql-collection.sql

@@ -0,0 +1,242 @@
+-- name: ddl-collection-scripts-02!
+create or replace table collection_mysql_config (
+    pkey varchar,
+    dma_source_id varchar,
+    dma_manual_id varchar,
+    variable_category varchar,
+    variable_name varchar,
+    variable_value varchar
+  );
+
+create or replace table collection_mysql_users (
+    pkey varchar,
+    dma_source_id varchar,
+    dma_manual_id varchar,
+    user_host varchar,
+    user_count numeric
+  );
+
+drop view if exists collection_resource_groups;
+
+create or replace table collection_mysql_5_resource_groups (
+    pkey varchar,
+    dma_source_id varchar,
+    dma_manual_id varchar,
+    resource_group_name varchar,
+    resource_group_type varchar,
+    resource_group_enabled varchar,
+    vcpu_ids varchar,
+    thread_priority varchar
+  );
+
+create or replace table collection_mysql_base_resource_groups (
+    pkey varchar,
+    dma_source_id varchar,
+    dma_manual_id varchar,
+    resource_group_name varchar,
+    resource_group_type varchar,
+    resource_group_enabled varchar,
+    vcpu_ids varchar,
+    thread_priority varchar
+  );
+
+create or replace view collection_resource_groups as
+select pkey,
+  dma_source_id,
+  dma_manual_id,
+  resource_group_name,
+  resource_group_type,
+  resource_group_enabled,
+  vcpu_ids,
+  thread_priority
+from collection_mysql_base_resource_groups
+union all
+select pkey,
+  dma_source_id,
+  dma_manual_id,
+  resource_group_name,
+  resource_group_type,
+  resource_group_enabled,
+  vcpu_ids,
+  thread_priority
+from collection_mysql_5_resource_groups;
+
+create or replace table collection_mysql_config (
+    pkey varchar,
+    dma_source_id varchar,
+    dma_manual_id varchar,
+    variable_category varchar,
+    variable_name varchar,
+    variable_value varchar
+  );
+
+create or replace table collection_mysql_data_types (
+    pkey varchar,
+    dma_source_id varchar,
+    dma_manual_id varchar,
+    table_catalog varchar,
+    table_schema varchar,
+    table_name varchar,
+    data_type varchar,
+    data_type_count numeric
+  );
+
+create or replace table collection_mysql_database_details (
+    pkey varchar,
+    dma_source_id varchar,
+    dma_manual_id varchar,
+    table_schema varchar,
+    total_table_count numeric,
+    innodb_table_count numeric,
+    non_innodb_table_count numeric,
+    total_row_count numeric,
+    innodb_table_row_count numeric,
+    non_innodb_table_row_count numeric,
+    total_data_size_bytes numeric,
+    innodb_data_size_bytes numeric,
+    non_innodb_data_size_bytes numeric,
+    total_index_size_bytes numeric,
+    innodb_index_size_bytes numeric,
+    non_innodb_index_size_bytes numeric,
+    total_size_bytes numeric,
+    innodb_total_size_bytes numeric,
+    non_innodb_total_size_bytes numeric,
+    total_index_count numeric,
+    innodb_index_count numeric,
+    non_innodb_index_count numeric
+  );
+
+create or replace table collection_mysql_engines (
+    pkey varchar,
+    dma_source_id varchar,
+    dma_manual_id varchar,
+    engine_name varchar,
+    engine_support varchar,
+    engine_transactions varchar,
+    engine_xa varchar,
+    engine_savepoints varchar,
+    engine_comment varchar
+  );
+
+create or replace table collection_mysql_plugins (
+    pkey varchar,
+    dma_source_id varchar,
+    dma_manual_id varchar,
+    plugin_name varchar,
+    plugin_version varchar,
+    plugin_status varchar,
+    plugin_type varchar,
+    plugin_type_version varchar,
+    plugin_library varchar,
+    plugin_library_version varchar,
+    plugin_author varchar,
+    plugin_description varchar,
+    plugin_license varchar,
+    load_option varchar
+  );
+
+drop view if exists collection_mysql_process_list;
+
+create or replace table collection_mysql_base_process_list (
+    pkey varchar,
+    dma_source_id varchar,
+    dma_manual_id varchar,
+    process_id numeric,
+    process_host varchar,
+    process_db varchar,
+    process_command varchar,
+    process_time numeric,
+    process_state varchar
+  );
+
+create or replace table collection_mysql_5_process_list (
+    pkey varchar,
+    dma_source_id varchar,
+    dma_manual_id varchar,
+    process_id numeric,
+    process_host varchar,
+    process_db varchar,
+    process_command varchar,
+    process_time numeric,
+    process_state varchar
+  );
+
+create or replace view collection_mysql_process_list as
+select pkey,
+  dma_source_id,
+  dma_manual_id,
+  process_id,
+  process_host,
+  process_db,
+  process_command,
+  process_time,
+  process_state
+from collection_mysql_base_process_list
+union all
+select pkey,
+  dma_source_id,
+  dma_manual_id,
+  process_id,
+  process_host,
+  process_db,
+  process_command,
+  process_time,
+  process_state
+from collection_mysql_5_process_list;
+
+create or replace table collection_mysql_schema_details (
+    pkey varchar,
+    dma_source_id varchar,
+    dma_manual_id varchar,
+    table_schema varchar,
+    table_name varchar,
+    table_engine varchar,
+    table_rows numeric,
+    data_length numeric,
+    index_length numeric,
+    is_compressed numeric,
+    is_partitioned numeric,
+    partition_count numeric,
+    index_count numeric,
+    fulltext_index_count numeric,
+    is_encrypted numeric,
+    spatial_index_count numeric,
+    has_primary_key numeric,
+    row_format varchar,
+    table_type varchar
+  );
+
+create or replace table collection_mysql_schema_objects (
+    pkey varchar,
+    dma_source_id varchar,
+    dma_manual_id varchar,
+    object_catalog varchar,
+    object_schema varchar,
+    object_category varchar,
+    object_type varchar,
+    object_owner_schema varchar,
+    object_owner varchar,
+    object_name varchar
+  );
+
+create or replace table collection_mysql_table_details (
+    pkey varchar,
+    dma_source_id varchar,
+    dma_manual_id varchar,
+    table_schema varchar,
+    table_name varchar,
+    table_engine varchar,
+    table_rows numeric,
+    data_length numeric,
+    index_length numeric,
+    is_compressed numeric,
+    is_partitioned numeric,
+    partition_count numeric,
+    index_count numeric,
+    fulltext_index_count numeric,
+    is_encrypted numeric,
+    spatial_index_count numeric,
+    has_primary_key numeric,
+    row_format varchar,
+    table_type varchar
+  );

src/dma/collector/sql/canonical/ddls-postgres-collection.sql

@@ -1,4 +1,4 @@
--- name: ddl-postgres-01-collection-scripts!
+-- name: ddl-collection-scripts-01!
 create or replace table collection_postgres_12_database_details(
     pkey VARCHAR,
     dma_source_id VARCHAR,

src/dma/collector/sql/canonical/readiness-check.sql

@@ -1,4 +1,4 @@
--- name: ddl-01-readiness-check-ddl!
+-- name: ddl-readiness-check-01-ddl!
 create or replace table database_summary(
     collection_key varchar,
     database_name varchar,