diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1f6318c4..df4cd6fb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.3.14 +current_version = 4.3.15 commit = False tag = False diff --git a/scripts/collector/oracle/collect-data.sh b/scripts/collector/oracle/collect-data.sh index efb3a6d4..2c9943f4 100755 --- a/scripts/collector/oracle/collect-data.sh +++ b/scripts/collector/oracle/collect-data.sh @@ -16,7 +16,7 @@ ### Setup directories needed for execution ############################################################################# -OpVersion="4.3.14" +OpVersion="4.3.15" LOCALE=$(echo $LANG | cut -d '.' -f 1) export LANG=C @@ -85,10 +85,12 @@ fi ${SQLPLUS} -s /nolog << EOF connect ${connectString} -@${SQL_DIR}/op_set_sql_env.sql +@${SQL_DIR}/op_set_sql_env.sql set pagesize 0 lines 400 feedback off verify off heading off echo off timing off time off +column vname new_value v_name noprint +select min(object_name) as vname from dba_objects where object_name in ('V\$INSTANCE', 'GV\$INSTANCE'); select 'DMAFILETAG~'|| i.version||'|'||substr(replace(i.version,'.',''),0,3)||'_'||'${OpVersion}_'||i.host_name||'_'||d.name||'_'||i.instance_name||'_'||to_char(sysdate,'MMDDRRHH24MISS')||'~' -from v\$instance i, v\$database d; +from ( SELECT version, host_name, instance_name FROM &&v_name WHERE instance_number = (SELECT min(instance_number) FROM &&v_name) ) i, v\$database d; exit; EOF } @@ -157,7 +159,7 @@ function compressOpFiles { V_FILE_TAG=$1 V_ERR_TAG="" echo "" -echo "Archiving output files" +echo "Archiving output files with tag ${V_FILE_TAG}" CURRENT_WORKING_DIR=$(pwd) cp ${LOG_DIR}/opdb__${V_FILE_TAG}_errors.log ${OUTPUT_DIR}/opdb__${V_FILE_TAG}_errors.log if [ -f VERSION.txt ]; then @@ -184,7 +186,17 @@ locale > ${OUTPUT_DIR}/opdb__${V_FILE_TAG}_locale.txt echo "ZIPFILE: " $ZIPFILE >> ${OUTPUT_DIR}/opdb__defines__${V_FILE_TAG}.csv cd ${OUTPUT_DIR} -ls -1 opdb*${V_FILE_TAG}.csv opdb*${V_FILE_TAG}*.log opdb*${V_FILE_TAG}*.txt > opdb__manifest__${V_FILE_TAG}.txt +if [ -f opdb__manifest__${V_FILE_TAG}.txt ]; +then + rm opdb__manifest__${V_FILE_TAG}.txt +fi + +for file in $(ls -1 opdb*${V_FILE_TAG}.csv opdb*${V_FILE_TAG}*.log opdb*${V_FILE_TAG}*.txt) +do + MD5=$(md5sum $file | cut -d ' ' -f 1) + echo "${DBTYPE}|${MD5}|${file}" >> opdb__manifest__${V_FILE_TAG}.txt +done + if [ ! "${ZIP}" = "" ] then $ZIP $ZIPFILE opdb*${V_FILE_TAG}.csv opdb*${V_FILE_TAG}*.log opdb*${V_FILE_TAG}*.txt diff --git a/scripts/collector/oracle/sql/extracts/archlogs.sql b/scripts/collector/oracle/sql/extracts/archlogs.sql index 60c7dac9..a0bb8098 100644 --- a/scripts/collector/oracle/sql/extracts/archlogs.sql +++ b/scripts/collector/oracle/sql/extracts/archlogs.sql @@ -25,7 +25,8 @@ SELECT '&&v_host' thread# AS thread_num, dest_id, count(1) AS CNT, - round(sum(blocks * block_size)/1024/1024) as mbytes + round(sum(blocks * block_size)/1024/1024) as mbytes, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM gv$archived_log WHERE first_time >= trunc(sysdate) - '&&dtrange' GROUP BY trunc(first_time), thread#, to_char(first_time, 'HH24'), dest_id diff --git a/scripts/collector/oracle/sql/extracts/awr/awrhistcmdtypes.sql b/scripts/collector/oracle/sql/extracts/awr/awrhistcmdtypes.sql index f4a4281e..15c747eb 100644 --- a/scripts/collector/oracle/sql/extracts/awr/awrhistcmdtypes.sql +++ b/scripts/collector/oracle/sql/extracts/awr/awrhistcmdtypes.sql @@ -60,6 +60,7 @@ GROUP BY '&&v_host' b.command_type, aa.name) SELECT pkey , con_id , hh24 , command_type , cnt , avg_buffer_gets , avg_elasped_time , avg_rows_processed , avg_executions , avg_cpu_time , avg_iowait , avg_clwait , - avg_apwait , avg_ccwait , avg_plsexec_time, command_name + avg_apwait , avg_ccwait , avg_plsexec_time, command_name, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vcmdtype; spool off diff --git a/scripts/collector/oracle/sql/extracts/awr/awrhistosstat.sql b/scripts/collector/oracle/sql/extracts/awr/awrhistosstat.sql index f1d9c68f..d05d9a82 100644 --- a/scripts/collector/oracle/sql/extracts/awr/awrhistosstat.sql +++ b/scripts/collector/oracle/sql/extracts/awr/awrhistosstat.sql @@ -102,6 +102,7 @@ GROUP BY '&&v_host' stat_name) SELECT pkey , dbid , instance_number , hh24 , stat_name , hh24_total_secs , cumulative_value , avg_value , mode_value , median_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100 , - min_value , max_value , sum_value , count + min_value , max_value , sum_value , count, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vossummary; spool off diff --git a/scripts/collector/oracle/sql/extracts/awr/awrhistsysmetrichist.sql b/scripts/collector/oracle/sql/extracts/awr/awrhistsysmetrichist.sql index 8d9ebdb0..2dac0d88 100644 --- a/scripts/collector/oracle/sql/extracts/awr/awrhistsysmetrichist.sql +++ b/scripts/collector/oracle/sql/extracts/awr/awrhistsysmetrichist.sql @@ -65,6 +65,7 @@ ORDER BY hsm.dbid, TO_CHAR(hsm.begin_time, 'hh24')) SELECT pkey , dbid , instance_number , hour , metric_name , metric_unit , avg_value , mode_value , median_value , min_value , max_value , - sum_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100 + sum_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vsysmetric; spool off diff --git a/scripts/collector/oracle/sql/extracts/awr/awrhistsysmetricsumm.sql b/scripts/collector/oracle/sql/extracts/awr/awrhistsysmetricsumm.sql index 57734100..fc9acbce 100644 --- a/scripts/collector/oracle/sql/extracts/awr/awrhistsysmetricsumm.sql +++ b/scripts/collector/oracle/sql/extracts/awr/awrhistsysmetricsumm.sql @@ -81,6 +81,7 @@ vsysmetricsummperhour as ( ) SELECT pkey , dbid , instance_number , hour , metric_name , metric_unit , avg_value , mode_value , median_value , min_value , max_value , - sum_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100 + sum_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vsysmetricsummperhour; spool off diff --git a/scripts/collector/oracle/sql/extracts/awr/awrsnapdetails.sql b/scripts/collector/oracle/sql/extracts/awr/awrsnapdetails.sql index 8bf87b1f..402a3a49 100644 --- a/scripts/collector/oracle/sql/extracts/awr/awrsnapdetails.sql +++ b/scripts/collector/oracle/sql/extracts/awr/awrsnapdetails.sql @@ -55,6 +55,7 @@ WHERE startup_time = lag_startup_time GROUP BY '&&v_host' || '_' || '&&v_dbname' || '_' || '&&v_hora', dbid, instance_number, hour) SELECT pkey , dbid , instance_number , hour , min_snap_id , max_snap_id , min_begin_interval_time , max_begin_interval_time , cnt , sum_snaps_diff_secs , avg_snaps_diff_secs , median_snaps_diff_secs , - mode_snaps_diff_secs , min_snaps_diff_secs , max_snaps_diff_secs + mode_snaps_diff_secs , min_snaps_diff_secs , max_snaps_diff_secs, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vawrsnap; spool off diff --git a/scripts/collector/oracle/sql/extracts/awr/dbahistsysstat.sql b/scripts/collector/oracle/sql/extracts/awr/dbahistsysstat.sql index 010b1404..82a505ad 100644 --- a/scripts/collector/oracle/sql/extracts/awr/dbahistsysstat.sql +++ b/scripts/collector/oracle/sql/extracts/awr/dbahistsysstat.sql @@ -93,6 +93,7 @@ GROUP BY stat_name) SELECT pkey , dbid , instance_number , hour , stat_name , cnt , avg_value , mode_value , median_value , min_value , max_value , - sum_value , perc50 , perc75 , perc90 , perc95 , perc100 + sum_value , perc50 , perc75 , perc90 , perc95 , perc100, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vsysstat; spool off diff --git a/scripts/collector/oracle/sql/extracts/awr/dbahistsystimemodel.sql b/scripts/collector/oracle/sql/extracts/awr/dbahistsystimemodel.sql index 1ee5255a..5d89989d 100644 --- a/scripts/collector/oracle/sql/extracts/awr/dbahistsystimemodel.sql +++ b/scripts/collector/oracle/sql/extracts/awr/dbahistsystimemodel.sql @@ -71,6 +71,7 @@ GROUP BY stat_name) SELECT pkey , dbid , instance_number , hour , stat_name , cnt , avg_value , mode_value , median_value , min_value , max_value , - sum_value , perc50 , perc75 , perc90 , perc95 , perc100 + sum_value , perc50 , perc75 , perc90 , perc95 , perc100, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vtimemodel; spool off diff --git a/scripts/collector/oracle/sql/extracts/awr/ioevents.sql b/scripts/collector/oracle/sql/extracts/awr/ioevents.sql index 64d2e2f4..b28dacea 100644 --- a/scripts/collector/oracle/sql/extracts/awr/ioevents.sql +++ b/scripts/collector/oracle/sql/extracts/awr/ioevents.sql @@ -97,7 +97,8 @@ SELECT pkey , dbid , instance_number , hour , wait_class , event_name , time_wa_us_delta_value_P95, tot_waits_delta_value_P100 , tot_tout_delta_value_P100 , - time_wa_us_delta_value_P100 + time_wa_us_delta_value_P100, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vfev; spool off column hour clear diff --git a/scripts/collector/oracle/sql/extracts/awr/iofunction.sql b/scripts/collector/oracle/sql/extracts/awr/iofunction.sql index 377ff2c4..b0a108bb 100644 --- a/scripts/collector/oracle/sql/extracts/awr/iofunction.sql +++ b/scripts/collector/oracle/sql/extracts/awr/iofunction.sql @@ -187,7 +187,8 @@ SELECT pkey , dbid , instance_number , hour , function_name , total_reads_mb_P100 , total_reads_req_P100 , total_writes_mb_P100 , - total_write_req_P100 + total_write_req_P100, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM viof; spool off column hour clear diff --git a/scripts/collector/oracle/sql/extracts/awr/sourceconn.sql b/scripts/collector/oracle/sql/extracts/awr/sourceconn.sql index 28b0b246..c2887ba7 100644 --- a/scripts/collector/oracle/sql/extracts/awr/sourceconn.sql +++ b/scripts/collector/oracle/sql/extracts/awr/sourceconn.sql @@ -61,7 +61,8 @@ group by '&&v_host' &v_machine, scmd.command_name) SELECT pkey , dbid , instance_number , hour , program , - module , machine , command_name , cnt + module , machine , command_name , cnt, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vsrcconn order by hour; spool off diff --git a/scripts/collector/oracle/sql/extracts/awr/sqlstats.sql b/scripts/collector/oracle/sql/extracts/awr/sqlstats.sql index 488d6fc7..6a2c200a 100644 --- a/scripts/collector/oracle/sql/extracts/awr/sqlstats.sql +++ b/scripts/collector/oracle/sql/extracts/awr/sqlstats.sql @@ -71,7 +71,8 @@ SELECT pkey , con_id , dbid , instance_number , force_matching_signature , sql_i physical_read_bytes_total , physical_write_bytes_total , io_offload_elig_bytes_total , io_interconnect_bytes_total , optimized_physical_reads_total , cell_uncompressed_bytes_total , io_offload_return_bytes_total , direct_writes_total , perc_exec_finished , avg_rows , avg_disk_reads , avg_buffer_gets , avg_cpu_time_us , avg_elapsed_us , avg_iowait_us , - avg_clwait_us , avg_apwait_us , avg_ccwait_us , avg_plsexec_us , avg_javexec_us + avg_clwait_us , avg_apwait_us , avg_ccwait_us , avg_plsexec_us , avg_javexec_us, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vsqlstat WHERE rownum < 300; spool off diff --git a/scripts/collector/oracle/sql/extracts/awr/sqlstats111.sql b/scripts/collector/oracle/sql/extracts/awr/sqlstats111.sql index 3c4ca4cb..0e45b04b 100644 --- a/scripts/collector/oracle/sql/extracts/awr/sqlstats111.sql +++ b/scripts/collector/oracle/sql/extracts/awr/sqlstats111.sql @@ -72,7 +72,8 @@ SELECT pkey , con_id , dbid , instance_number , force_matching_signature , sql_i optimized_physical_reads_total , cell_uncompressed_bytes_total , io_offload_return_bytes_total , direct_writes_total , perc_exec_finished , avg_rows , avg_disk_reads , avg_buffer_gets , avg_cpu_time_us , avg_elapsed_us , avg_iowait_us , - avg_clwait_us , avg_apwait_us , avg_ccwait_us , avg_plsexec_us , avg_javexec_us + avg_clwait_us , avg_apwait_us , avg_ccwait_us , avg_plsexec_us , avg_javexec_us, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vsqlstat WHERE rownum < 300; spool off diff --git a/scripts/collector/oracle/sql/extracts/backups.sql b/scripts/collector/oracle/sql/extracts/backups.sql index 46fcc647..32c29451 100644 --- a/scripts/collector/oracle/sql/extracts/backups.sql +++ b/scripts/collector/oracle/sql/extracts/backups.sql @@ -25,7 +25,8 @@ SELECT '&&v_host' input_type, round(sum(elapsed_seconds)) AS elapsed_seconds, round(sum(input_bytes)/1024/1024) AS mbytes_in, - round(sum(output_bytes)/1024/1024) AS mbytes_out + round(sum(output_bytes)/1024/1024) AS mbytes_out, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM v$rman_backup_job_details a WHERE start_time >= trunc(sysdate) - '&&dtrange' GROUP BY trunc(start_time), input_type, &v_a_con_id diff --git a/scripts/collector/oracle/sql/extracts/columntypes.sql b/scripts/collector/oracle/sql/extracts/columntypes.sql index b34c65b6..26d4e0e8 100644 --- a/scripts/collector/oracle/sql/extracts/columntypes.sql +++ b/scripts/collector/oracle/sql/extracts/columntypes.sql @@ -208,8 +208,9 @@ SELECT c.VARCHAR2_COL_COUNT , c.XMLTYPE_COL_COUNT , c.UNDEFINED_COL_COUNT , - c.USER_DEFINED_COL_COUNT - , NVL(s.bytes,0) as bytes + c.USER_DEFINED_COL_COUNT , + NVL(s.bytes,0) as bytes , + '&v_dma_source_id' AS DMA_SOURCE_ID FROM coltypes c LEFT JOIN segs s ON c.con_id = s.con_id and s.owner = c.owner and s.segment_name = c.table_name ORDER BY 1,2,3,4 diff --git a/scripts/collector/oracle/sql/extracts/compressbytype.sql b/scripts/collector/oracle/sql/extracts/compressbytype.sql index cf211dee..eac3240f 100644 --- a/scripts/collector/oracle/sql/extracts/compressbytype.sql +++ b/scripts/collector/oracle/sql/extracts/compressbytype.sql @@ -92,7 +92,8 @@ WITH vcompresstype AS ( --HAVING TRUNC(SUM(gbytes)) > 0 ) SELECT pkey , con_id , owner , basic , oltp , query_low , - query_high , archive_low , archive_high , total_gb + query_high , archive_low , archive_high , total_gb, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vcompresstype ORDER BY total_gb DESC; spool off diff --git a/scripts/collector/oracle/sql/extracts/cpucoresusage.sql b/scripts/collector/oracle/sql/extracts/cpucoresusage.sql index 78854e9b..154794e2 100644 --- a/scripts/collector/oracle/sql/extracts/cpucoresusage.sql +++ b/scripts/collector/oracle/sql/extracts/cpucoresusage.sql @@ -27,6 +27,7 @@ SELECT '&&v_host' cpu_socket_count FROM dba_cpu_usage_statistics ORDER BY timestamp) -SELECT pkey , dt , cpu_count , cpu_core_count , cpu_socket_count +SELECT pkey , dt , cpu_count , cpu_core_count , cpu_socket_count, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vcpursc; spool off diff --git a/scripts/collector/oracle/sql/extracts/dataguard.sql b/scripts/collector/oracle/sql/extracts/dataguard.sql index 9f137218..63e3cd7c 100644 --- a/scripts/collector/oracle/sql/extracts/dataguard.sql +++ b/scripts/collector/oracle/sql/extracts/dataguard.sql @@ -53,7 +53,8 @@ FROM gv$archive_dest a WHERE destination IS NOT NULL) SELECT pkey , con_id , inst_id , log_archive_config , dest_id , dest_name , destination , status , target , schedule , register , alternate , - transmit_mode , affirm , valid_role , verify + transmit_mode , affirm , valid_role , verify, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vodg; spool off diff --git a/scripts/collector/oracle/sql/extracts/datatypes.sql b/scripts/collector/oracle/sql/extracts/datatypes.sql index 887d98a4..6e5986d6 100644 --- a/scripts/collector/oracle/sql/extracts/datatypes.sql +++ b/scripts/collector/oracle/sql/extracts/datatypes.sql @@ -47,6 +47,7 @@ GROUP BY '&&v_host' avg_col_len ) SELECT pkey , con_id , owner , data_type , cnt, - data_length, data_precision, data_scale, avg_col_len, distinct_table_count + data_length, data_precision, data_scale, avg_col_len, distinct_table_count, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vdtype; spool off diff --git a/scripts/collector/oracle/sql/extracts/dbfeatures.sql b/scripts/collector/oracle/sql/extracts/dbfeatures.sql index 15ee4e84..5893c21f 100644 --- a/scripts/collector/oracle/sql/extracts/dbfeatures.sql +++ b/scripts/collector/oracle/sql/extracts/dbfeatures.sql @@ -33,6 +33,7 @@ FROM &v_tblprefix._feature_usage_statistics a WHERE dbid = &&v_dbid ORDER BY name) SELECT pkey , con_id , name , currently_used , detected_usages , - total_samples , first_usage , last_usage , aux_count + total_samples , first_usage , last_usage , aux_count, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vdbf; spool off diff --git a/scripts/collector/oracle/sql/extracts/dbhwmarkstatistics.sql b/scripts/collector/oracle/sql/extracts/dbhwmarkstatistics.sql index 43d5df31..bce5d605 100644 --- a/scripts/collector/oracle/sql/extracts/dbhwmarkstatistics.sql +++ b/scripts/collector/oracle/sql/extracts/dbhwmarkstatistics.sql @@ -27,6 +27,7 @@ SELECT '&&v_host' &v_a_con_id AS con_id FROM &v_tblprefix._high_water_mark_statistics a ORDER BY description) -SELECT pkey , description , highwater , last_value, con_id +SELECT pkey , description , highwater , last_value, con_id, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vhwmst; spool off diff --git a/scripts/collector/oracle/sql/extracts/dbinstances.sql b/scripts/collector/oracle/sql/extracts/dbinstances.sql index cead50d2..920516e6 100644 --- a/scripts/collector/oracle/sql/extracts/dbinstances.sql +++ b/scripts/collector/oracle/sql/extracts/dbinstances.sql @@ -30,6 +30,7 @@ SELECT '&&v_host' instance_role FROM gv$instance ) SELECT pkey , inst_id , instance_name , host_name , - version , status , database_status , instance_role + version , status , database_status , instance_role, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vdbinst; spool off diff --git a/scripts/collector/oracle/sql/extracts/dblinks.sql b/scripts/collector/oracle/sql/extracts/dblinks.sql index 84f7235b..2f9fddfb 100644 --- a/scripts/collector/oracle/sql/extracts/dblinks.sql +++ b/scripts/collector/oracle/sql/extracts/dblinks.sql @@ -32,6 +32,7 @@ GROUP BY '&&v_host' || '_' || '&&v_hora', &v_a_con_id , owner) -SELECT pkey , con_id , owner , count +SELECT pkey , con_id , owner , count, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vdbl; spool off diff --git a/scripts/collector/oracle/sql/extracts/dbobjectnames.sql b/scripts/collector/oracle/sql/extracts/dbobjectnames.sql index f8a9572e..4e0df8c2 100644 --- a/scripts/collector/oracle/sql/extracts/dbobjectnames.sql +++ b/scripts/collector/oracle/sql/extracts/dbobjectnames.sql @@ -76,7 +76,8 @@ SELECT pkey , object_type , editionable , lines, - status + status, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vdbobj a; spool off diff --git a/scripts/collector/oracle/sql/extracts/dbobjects.sql b/scripts/collector/oracle/sql/extracts/dbobjects.sql index 98e9181c..3249b423 100644 --- a/scripts/collector/oracle/sql/extracts/dbobjects.sql +++ b/scripts/collector/oracle/sql/extracts/dbobjects.sql @@ -59,7 +59,8 @@ SELECT pkey , owner , object_type , editionable , - count + count , + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vdbobj a; spool off diff --git a/scripts/collector/oracle/sql/extracts/dbparameters.sql b/scripts/collector/oracle/sql/extracts/dbparameters.sql index 95dbb19d..7a264229 100644 --- a/scripts/collector/oracle/sql/extracts/dbparameters.sql +++ b/scripts/collector/oracle/sql/extracts/dbparameters.sql @@ -30,7 +30,8 @@ SELECT '&&v_host' isdefault FROM gv$system_parameter a ORDER BY 2,3 ) -SELECT pkey , inst_id , con_id , name , value , default_value , isdefault +SELECT pkey , inst_id , con_id , name , value , default_value , isdefault, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vparam; spool off column default_value clear diff --git a/scripts/collector/oracle/sql/extracts/dbsummary.sql b/scripts/collector/oracle/sql/extracts/dbsummary.sql index 57bbad80..60b565a0 100644 --- a/scripts/collector/oracle/sql/extracts/dbsummary.sql +++ b/scripts/collector/oracle/sql/extracts/dbsummary.sql @@ -112,6 +112,6 @@ SELECT pkey , dbid , db_name , cdb , db_version , db_fullversion , log_mode , fo db_long_size_gb , dg_database_role , dg_protection_mode , dg_protection_level, db_size_temp_allocated_gb, db_size_redo_allocated_gb, ebs_owner, siebel_owner, psft_owner, rds_flag, oci_autonomous_flag, dbms_cloud_pkg_installed, - apex_installed, sap_owner, db_unique_name + apex_installed, sap_owner, db_unique_name, '&v_dma_source_id' AS DMA_SOURCE_ID FROM vdbsummary; spool off diff --git a/scripts/collector/oracle/sql/extracts/defines.sql b/scripts/collector/oracle/sql/extracts/defines.sql index 8b87987f..a7f66d4c 100644 --- a/scripts/collector/oracle/sql/extracts/defines.sql +++ b/scripts/collector/oracle/sql/extracts/defines.sql @@ -38,8 +38,10 @@ prompt v_io_function_sql = &v_io_function_sql prompt v_is_container = &v_is_container prompt v_max_snapid = &v_max_snapid prompt v_min_snapid = &v_min_snapid -prompt v_max_snaptime = &v_max_snaptime -prompt v_min_snaptime = &v_min_snaptime prompt v_tblprefix = &v_tblprefix prompt v_umf_test = &v_umf_test +prompt Applies to STATSPACK collections only: +prompt v_max_snaptime = &v_max_snaptime +prompt v_min_snaptime = &v_min_snaptime +prompt v_dma_source_id = &v_dma_source_id spool off diff --git a/scripts/collector/oracle/sql/extracts/dtlsourcecode.sql b/scripts/collector/oracle/sql/extracts/dtlsourcecode.sql index 177f3156..d18049c5 100644 --- a/scripts/collector/oracle/sql/extracts/dtlsourcecode.sql +++ b/scripts/collector/oracle/sql/extracts/dtlsourcecode.sql @@ -74,6 +74,7 @@ GROUP BY pkey, name, TYPE) SELECT pkey , con_id , owner, name , type , sum_nr_lines , qt_objs , - sum_nr_lines_w_utl , sum_nr_lines_w_dbms , count_exec_im , count_dbms_sql , sum_nr_lines_w_dbms_utl , sum_count_total + sum_nr_lines_w_utl , sum_nr_lines_w_dbms , count_exec_im , count_dbms_sql , sum_nr_lines_w_dbms_utl , sum_count_total, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vsrc; spool off diff --git a/scripts/collector/oracle/sql/extracts/exttab.sql b/scripts/collector/oracle/sql/extracts/exttab.sql index 50a9dd0f..a0b72dc9 100644 --- a/scripts/collector/oracle/sql/extracts/exttab.sql +++ b/scripts/collector/oracle/sql/extracts/exttab.sql @@ -24,6 +24,7 @@ SELECT '&&v_host' &v_a_con_id as con_id, owner, table_name, type_owner, type_name, default_directory_owner, default_directory_name FROM &v_tblprefix._external_tables a) SELECT pkey , con_id , owner , table_name , type_owner , type_name , - default_directory_owner , default_directory_name + default_directory_owner , default_directory_name, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vexttab; spool off diff --git a/scripts/collector/oracle/sql/extracts/idxpertable.sql b/scripts/collector/oracle/sql/extracts/idxpertable.sql index c84f1828..6d0898b5 100644 --- a/scripts/collector/oracle/sql/extracts/idxpertable.sql +++ b/scripts/collector/oracle/sql/extracts/idxpertable.sql @@ -34,6 +34,7 @@ SELECT pkey, round(100*ratio_to_report(count(table_name)) over ()) idx_perc FROM vrawidx GROUP BY pkey, con_id, idx_cnt) -SELECT pkey , con_id , tab_count , idx_cnt , idx_perc +SELECT pkey , con_id , tab_count , idx_cnt , idx_perc, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vcidx; spool off diff --git a/scripts/collector/oracle/sql/extracts/indextypedtl.sql b/scripts/collector/oracle/sql/extracts/indextypedtl.sql index f38cb1bf..e8138036 100644 --- a/scripts/collector/oracle/sql/extracts/indextypedtl.sql +++ b/scripts/collector/oracle/sql/extracts/indextypedtl.sql @@ -59,7 +59,8 @@ SELECT pkey , join_index, custom_index_type, table_name, - index_name + index_name, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vidxtype; spool off COLUMN PARTITIONED CLEAR diff --git a/scripts/collector/oracle/sql/extracts/indextypes.sql b/scripts/collector/oracle/sql/extracts/indextypes.sql index 6db8b964..bf2267a3 100644 --- a/scripts/collector/oracle/sql/extracts/indextypes.sql +++ b/scripts/collector/oracle/sql/extracts/indextypes.sql @@ -73,7 +73,8 @@ SELECT pkey , visibility, join_index, custom_index_type, - cnt + cnt, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vidxtype; spool off COLUMN PARTITIONED CLEAR diff --git a/scripts/collector/oracle/sql/extracts/lobsizing.sql b/scripts/collector/oracle/sql/extracts/lobsizing.sql index 9eb397ca..bb2b873c 100644 --- a/scripts/collector/oracle/sql/extracts/lobsizing.sql +++ b/scripts/collector/oracle/sql/extracts/lobsizing.sql @@ -166,7 +166,8 @@ SELECT '&&v_host' seg_bytes, table_avg_lob_bytes, partition_avg_lob_bytes, - subpartition_avg_lob_bytes + subpartition_avg_lob_bytes, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM lobdata; spool off diff --git a/scripts/collector/oracle/sql/extracts/mviewtypes.sql b/scripts/collector/oracle/sql/extracts/mviewtypes.sql index 5761c88b..2432e69c 100644 --- a/scripts/collector/oracle/sql/extracts/mviewtypes.sql +++ b/scripts/collector/oracle/sql/extracts/mviewtypes.sql @@ -48,7 +48,8 @@ SELECT '&&v_host' refresh_mode, refresh_method, fast_refreshable, - compile_state + compile_state, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM mvinfo; spool off COLUMN UPDATABLE CLEAR diff --git a/scripts/collector/oracle/sql/extracts/opatch.sql b/scripts/collector/oracle/sql/extracts/opatch.sql index 0b4afac9..0361262d 100644 --- a/scripts/collector/oracle/sql/extracts/opatch.sql +++ b/scripts/collector/oracle/sql/extracts/opatch.sql @@ -68,6 +68,7 @@ SELECT '&&v_host' || '&&v_dbname' || '_' || '&&v_hora' AS pkey, - patch_id, unique_patch_id, patch_type, applied_date, patch_descr, bug_descr + patch_id, unique_patch_id, patch_type, applied_date, patch_descr, bug_descr, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vopatch; spool off diff --git a/scripts/collector/oracle/sql/extracts/opkeylog.sql b/scripts/collector/oracle/sql/extracts/opkeylog.sql index f37f5f71..1f9939c0 100644 --- a/scripts/collector/oracle/sql/extracts/opkeylog.sql +++ b/scripts/collector/oracle/sql/extracts/opkeylog.sql @@ -27,7 +27,8 @@ select '&&v_host' '&&v_dbname' db_name, '&&v_inst' instance_name, '&&v_hora' collection_time, &&v_dbid db_id, null "CMNT" from dual) select pkey , opscriptversion , db_version , hostname - , db_name , instance_name , collection_time , db_id , CMNT + , db_name , instance_name , collection_time , db_id , CMNT, + '&v_dma_source_id' AS DMA_SOURCE_ID from vop; spool off COLUMN DB_NAME CLEAR diff --git a/scripts/collector/oracle/sql/extracts/pdbsinfo.sql b/scripts/collector/oracle/sql/extracts/pdbsinfo.sql index 3f90f6ac..e32f4c97 100644 --- a/scripts/collector/oracle/sql/extracts/pdbsinfo.sql +++ b/scripts/collector/oracle/sql/extracts/pdbsinfo.sql @@ -77,7 +77,8 @@ mem_stats AS ( LEFT OUTER JOIN pdb_pga p ON (s.con_id = p.con_id AND s.inst_id = p.inst_id) ) -SELECT i.*, m.sga_allocated_bytes, m.pga_used_bytes, m.pga_allocated_bytes, m.pga_max_bytes +SELECT i.*, m.sga_allocated_bytes, m.pga_used_bytes, m.pga_allocated_bytes, m.pga_max_bytes, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vpdbinfo i LEFT OUTER JOIN mem_stats m ON i.con_id = m.con_id; spool off diff --git a/scripts/collector/oracle/sql/extracts/pdbsopenmode.sql b/scripts/collector/oracle/sql/extracts/pdbsopenmode.sql index f08ee9ef..acc719b5 100644 --- a/scripts/collector/oracle/sql/extracts/pdbsopenmode.sql +++ b/scripts/collector/oracle/sql/extracts/pdbsopenmode.sql @@ -27,6 +27,7 @@ SELECT '&&v_host' total_size / 1024 / 1024 / 1024 TOTAL_GB, con_uid FROM v$pdbs ) -SELECT pkey , con_id , name , open_mode , TOTAL_GB, con_uid +SELECT pkey , con_id , name , open_mode , TOTAL_GB, con_uid, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vpdbmode; spool off diff --git a/scripts/collector/oracle/sql/extracts/sourcecode.sql b/scripts/collector/oracle/sql/extracts/sourcecode.sql index a5f4d9e2..9d6eba5f 100644 --- a/scripts/collector/oracle/sql/extracts/sourcecode.sql +++ b/scripts/collector/oracle/sql/extracts/sourcecode.sql @@ -72,6 +72,7 @@ GROUP BY pkey, owner, TYPE) SELECT pkey , con_id , owner , type , sum_nr_lines , qt_objs , - sum_nr_lines_w_utl , sum_nr_lines_w_dbms , count_exec_im , count_dbms_sql , sum_nr_lines_w_dbms_utl , sum_count_total + sum_nr_lines_w_utl , sum_nr_lines_w_dbms , count_exec_im , count_dbms_sql , sum_nr_lines_w_dbms_utl , sum_count_total, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vsrc; spool off diff --git a/scripts/collector/oracle/sql/extracts/statspack/awrhistcmdtypes.sql b/scripts/collector/oracle/sql/extracts/statspack/awrhistcmdtypes.sql index 84dccdb4..87cfac9d 100644 --- a/scripts/collector/oracle/sql/extracts/statspack/awrhistcmdtypes.sql +++ b/scripts/collector/oracle/sql/extracts/statspack/awrhistcmdtypes.sql @@ -180,7 +180,8 @@ GROUP BY '&&v_host' ) SELECT pkey , con_id AS sp_con_id, hh24 , command_type , cnt , avg_buffer_gets , avg_elapsed_time , avg_rows_processed , avg_executions , avg_cpu_time , avg_iowait , avg_clwait , - avg_apwait , avg_ccwait , avg_plsexec_time, command_name + avg_apwait , avg_ccwait , avg_plsexec_time, command_name, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vcmdtype; spool off COLUMN sp_con_id CLEAR diff --git a/scripts/collector/oracle/sql/extracts/statspack/awrhistosstat.sql b/scripts/collector/oracle/sql/extracts/statspack/awrhistosstat.sql index 486e3a08..39360f6e 100644 --- a/scripts/collector/oracle/sql/extracts/statspack/awrhistosstat.sql +++ b/scripts/collector/oracle/sql/extracts/statspack/awrhistosstat.sql @@ -103,7 +103,8 @@ GROUP BY '&&v_host' stat_name) SELECT pkey, dbid, instance_number, hh24, stat_name, hh24_total_secs , cumulative_value, avg_value, mode_value, median_value, PERC50, PERC75, PERC90, PERC95, PERC100, - min_value, max_value, sum_value, count + min_value, max_value, sum_value, count, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vossummary; spool off diff --git a/scripts/collector/oracle/sql/extracts/statspack/awrhistsysmetrichist.sql b/scripts/collector/oracle/sql/extracts/statspack/awrhistsysmetrichist.sql index 34c39730..c4c63764 100644 --- a/scripts/collector/oracle/sql/extracts/statspack/awrhistsysmetrichist.sql +++ b/scripts/collector/oracle/sql/extracts/statspack/awrhistsysmetrichist.sql @@ -75,7 +75,8 @@ ORDER BY hsm.dbid, TO_CHAR(dhsnap.snap_time, 'hh24')) SELECT pkey , dbid , instance_number , hour , metric_name , metric_unit , avg_value , mode_value , median_value , min_value , max_value , - sum_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100 + sum_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vsysmetric; spool off COLUMN HOUR CLEAR diff --git a/scripts/collector/oracle/sql/extracts/statspack/awrhistsysmetricsumm.sql b/scripts/collector/oracle/sql/extracts/statspack/awrhistsysmetricsumm.sql index b52b13e4..b1cc66b8 100644 --- a/scripts/collector/oracle/sql/extracts/statspack/awrhistsysmetricsumm.sql +++ b/scripts/collector/oracle/sql/extracts/statspack/awrhistsysmetricsumm.sql @@ -104,7 +104,8 @@ vsysmetricsummperhour as ( ) SELECT pkey , dbid , instance_number , hour , metric_name , metric_unit , avg_value , mode_value , median_value , min_value , max_value , - sum_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100 + sum_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vsysmetricsummperhour; spool off COLUMN HOUR CLEAR diff --git a/scripts/collector/oracle/sql/extracts/statspack/awrsnapdetails.sql b/scripts/collector/oracle/sql/extracts/statspack/awrsnapdetails.sql index 316b2f3b..75c2ea25 100644 --- a/scripts/collector/oracle/sql/extracts/statspack/awrsnapdetails.sql +++ b/scripts/collector/oracle/sql/extracts/statspack/awrsnapdetails.sql @@ -69,7 +69,8 @@ FROM ( GROUP BY '&&v_host' || '_' || '&&v_dbname' || '_' || '&&v_hora', dbid, instance_number, hour) SELECT pkey , dbid , instance_number , hour , min_snap_id , max_snap_id , min_begin_interval_time , max_begin_interval_time , cnt , sum_snaps_diff_secs , avg_snaps_diff_secs , median_snaps_diff_secs , - mode_snaps_diff_secs , min_snaps_diff_secs , max_snaps_diff_secs + mode_snaps_diff_secs , min_snaps_diff_secs , max_snaps_diff_secs, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vawrsnap; spool off diff --git a/scripts/collector/oracle/sql/extracts/statspack/dbahistsysstat.sql b/scripts/collector/oracle/sql/extracts/statspack/dbahistsysstat.sql index 0c3e023b..c96c63ff 100644 --- a/scripts/collector/oracle/sql/extracts/statspack/dbahistsysstat.sql +++ b/scripts/collector/oracle/sql/extracts/statspack/dbahistsysstat.sql @@ -87,7 +87,8 @@ GROUP BY name) SELECT pkey , dbid , instance_number , hour , stat_name , cnt , avg_value , mode_value , median_value , min_value , max_value , - sum_value , perc50 , perc75 , perc90 , perc95 , perc100 + sum_value , perc50 , perc75 , perc90 , perc95 , perc100, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vsysstat; spool off diff --git a/scripts/collector/oracle/sql/extracts/statspack/dbahistsystimemodel.sql b/scripts/collector/oracle/sql/extracts/statspack/dbahistsystimemodel.sql index be268ffa..f1986e6e 100644 --- a/scripts/collector/oracle/sql/extracts/statspack/dbahistsystimemodel.sql +++ b/scripts/collector/oracle/sql/extracts/statspack/dbahistsystimemodel.sql @@ -75,7 +75,8 @@ GROUP BY stat_name) SELECT pkey, dbid, instance_number, hour, stat_name, cnt, avg_value, mode_value, median_value, min_value, max_value, - sum_value, perc50, perc75, perc90, perc95, perc100 + sum_value, perc50, perc75, perc90, perc95, perc100, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vtimemodel; spool off diff --git a/scripts/collector/oracle/sql/extracts/statspack/ioevents.sql b/scripts/collector/oracle/sql/extracts/statspack/ioevents.sql index 6798faf9..b0e4269b 100644 --- a/scripts/collector/oracle/sql/extracts/statspack/ioevents.sql +++ b/scripts/collector/oracle/sql/extracts/statspack/ioevents.sql @@ -98,7 +98,8 @@ SELECT pkey , dbid , instance_number , hour , wait_class , event_name , time_wa_us_delta_value_P95, tot_waits_delta_value_P100 , tot_tout_delta_value_P100 , - time_wa_us_delta_value_P100 + time_wa_us_delta_value_P100, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vfev; spool off column hour clear diff --git a/scripts/collector/oracle/sql/extracts/statspack/iofunction.sql b/scripts/collector/oracle/sql/extracts/statspack/iofunction.sql index 0580f9e0..1d5ee252 100644 --- a/scripts/collector/oracle/sql/extracts/statspack/iofunction.sql +++ b/scripts/collector/oracle/sql/extracts/statspack/iofunction.sql @@ -190,7 +190,8 @@ SELECT pkey , dbid , instance_number , hour , function_name , total_reads_mb_P100 , total_reads_req_P100 , total_writes_mb_P100 , - total_write_req_P100 + total_write_req_P100, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM viof; spool off column hour clear diff --git a/scripts/collector/oracle/sql/extracts/statspack/sqlstats.sql b/scripts/collector/oracle/sql/extracts/statspack/sqlstats.sql index 7834d3e5..1a26aa25 100644 --- a/scripts/collector/oracle/sql/extracts/statspack/sqlstats.sql +++ b/scripts/collector/oracle/sql/extracts/statspack/sqlstats.sql @@ -203,7 +203,8 @@ SELECT pkey , con_id AS sp_con_id , dbid , instance_number , force_matching_sign physical_read_bytes_total , physical_write_bytes_total , io_offload_elig_bytes_total , io_interconnect_bytes_total , optimized_physical_reads_total , cell_uncompressed_bytes_total , io_offload_return_bytes_total , direct_writes_total , perc_exec_finished , avg_rows , avg_disk_reads , avg_buffer_gets , avg_cpu_time_us , avg_elapsed_us , avg_iowait_us , - avg_clwait_us , avg_apwait_us , avg_ccwait_us , avg_plsexec_us , avg_javexec_us + avg_clwait_us , avg_apwait_us , avg_ccwait_us , avg_plsexec_us , avg_javexec_us, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vsqlstat WHERE rownum < 300; spool off diff --git a/scripts/collector/oracle/sql/extracts/tableconstraints.sql b/scripts/collector/oracle/sql/extracts/tableconstraints.sql index ae129cef..5253cdf9 100644 --- a/scripts/collector/oracle/sql/extracts/tableconstraints.sql +++ b/scripts/collector/oracle/sql/extracts/tableconstraints.sql @@ -71,6 +71,7 @@ GROUP BY '&&v_host' owner, table_name) SELECT pkey , con_id , owner ,table_name , pk , uk , ck , - ri , vwck , vwro , hashexpr , suplog , total_cons + ri , vwck , vwro , hashexpr , suplog , total_cons, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vnopk; spool off diff --git a/scripts/collector/oracle/sql/extracts/tablesnopk.sql b/scripts/collector/oracle/sql/extracts/tablesnopk.sql index 5b5e2de7..d5ae16fe 100644 --- a/scripts/collector/oracle/sql/extracts/tablesnopk.sql +++ b/scripts/collector/oracle/sql/extracts/tablesnopk.sql @@ -70,6 +70,7 @@ GROUP BY '&&v_host' con_id, owner) SELECT pkey , con_id , owner , pk , uk , ck , - ri , vwck , vwro , hashexpr , suplog , num_tables , total_cons + ri , vwck , vwro , hashexpr , suplog , num_tables , total_cons, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vnopk; spool off diff --git a/scripts/collector/oracle/sql/extracts/tabletypedtl.sql b/scripts/collector/oracle/sql/extracts/tabletypedtl.sql index 3540a7a0..d610c78d 100644 --- a/scripts/collector/oracle/sql/extracts/tabletypedtl.sql +++ b/scripts/collector/oracle/sql/extracts/tabletypedtl.sql @@ -137,7 +137,8 @@ SELECT '&&v_host' p.partitioning_type, p.subpartitioning_type, p.partition_count, - sp.cnt AS subpartition_count + sp.cnt AS subpartition_count, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM tblinfo a LEFT OUTER JOIN &v_tblprefix._part_tables p ON a.owner = p.owner diff --git a/scripts/collector/oracle/sql/extracts/tabletypes.sql b/scripts/collector/oracle/sql/extracts/tabletypes.sql index aaea0e68..8171cdc2 100644 --- a/scripts/collector/oracle/sql/extracts/tabletypes.sql +++ b/scripts/collector/oracle/sql/extracts/tabletypes.sql @@ -153,7 +153,8 @@ SELECT '&&v_host' clustered_table, table_count, object_table, - xml_table + xml_table, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM tblinfo; spool off COLUMN TEMPORARY CLEAR diff --git a/scripts/collector/oracle/sql/extracts/triggers.sql b/scripts/collector/oracle/sql/extracts/triggers.sql index 3d30d373..a2f02083 100644 --- a/scripts/collector/oracle/sql/extracts/triggers.sql +++ b/scripts/collector/oracle/sql/extracts/triggers.sql @@ -60,6 +60,7 @@ SELECT '&&v_host' trigger_type, triggering_event, base_object_type, - trigger_count + trigger_count, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM trginfo; spool off diff --git a/scripts/collector/oracle/sql/extracts/usedspacedetails.sql b/scripts/collector/oracle/sql/extracts/usedspacedetails.sql index 0d3eb70c..02d91970 100644 --- a/scripts/collector/oracle/sql/extracts/usedspacedetails.sql +++ b/scripts/collector/oracle/sql/extracts/usedspacedetails.sql @@ -35,6 +35,7 @@ SELECT '&&v_host' || '_' || '&&v_hora', &v_a_con_id , owner, segment_type ) -SELECT pkey , con_id , owner , segment_type , GB +SELECT pkey , con_id , owner , segment_type , GB, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vused; spool off diff --git a/scripts/collector/oracle/sql/extracts/users.sql b/scripts/collector/oracle/sql/extracts/users.sql new file mode 100644 index 00000000..3a5f15f8 --- /dev/null +++ b/scripts/collector/oracle/sql/extracts/users.sql @@ -0,0 +1,31 @@ +/* +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +spool &outputdir/opdb__users__&v_tag + +WITH vuser AS ( +SELECT '&&v_host' + || '_' + || '&&v_dbname' + || '_' + || '&&v_hora' AS pkey, + &v_a_con_id AS con_id, + username +FROM &v_tblprefix._users a +ORDER BY username) +SELECT pkey , con_id, username, + '&v_dma_source_id' AS DMA_SOURCE_ID +FROM vuser; +spool off diff --git a/scripts/collector/oracle/sql/extracts/usrsegatt.sql b/scripts/collector/oracle/sql/extracts/usrsegatt.sql index 0d4279b3..f8c3ade3 100644 --- a/scripts/collector/oracle/sql/extracts/usrsegatt.sql +++ b/scripts/collector/oracle/sql/extracts/usrsegatt.sql @@ -31,6 +31,7 @@ WITH vuseg AS ( AND owner NOT IN @&EXTRACTSDIR/exclude_schemas.sql ) -SELECT pkey , con_id , owner , segment_name , segment_type , tablespace_name +SELECT pkey , con_id , owner , segment_name , segment_type , tablespace_name, + '&v_dma_source_id' AS DMA_SOURCE_ID FROM vuseg; spool off diff --git a/scripts/collector/oracle/sql/op_collect.sql b/scripts/collector/oracle/sql/op_collect.sql index 727637f2..717ab517 100644 --- a/scripts/collector/oracle/sql/op_collect.sql +++ b/scripts/collector/oracle/sql/op_collect.sql @@ -52,6 +52,7 @@ prompt set termout &TERMOUTOFF @&EXTRACTSDIR/defines.sql @&EXTRACTSDIR/archlogs.sql +@&EXTRACTSDIR/users.sql @&EXTRACTSDIR/backups.sql @&EXTRACTSDIR/columntypes.sql --@&EXTRACTSDIR/compressbytype.sql diff --git a/scripts/collector/oracle/sql/op_collect_init.sql b/scripts/collector/oracle/sql/op_collect_init.sql index ea48ba11..d06405f9 100644 --- a/scripts/collector/oracle/sql/op_collect_init.sql +++ b/scripts/collector/oracle/sql/op_collect_init.sql @@ -79,6 +79,7 @@ column min_snaptime new_value v_min_snaptime noprint column max_snapid new_value v_max_snapid noprint column max_snaptime new_value v_max_snaptime noprint column umf_test new_value v_umf_test noprint +column p_dma_source_id new_value v_dma_source_id noprint column p_dbid new_value v_dbid noprint column p_tblprefix new_value v_tblprefix noprint column p_is_container new_value v_is_container noprint @@ -108,39 +109,24 @@ FROM v$database / -select RTRIM(SUBSTR('&v_tag',INSTR('&v_tag','_',1,5)+1), '.csv') horanc from dual; +SELECT RTRIM(SUBSTR('&v_tag',INSTR('&v_tag','_',1,5)+1), '.csv') horanc from dual; SELECT substr(replace(version,'.',''),0,3) dbversion -from v$instance +FROM v$instance +/ + +column vname new_value v_name noprint +SELECT min(object_name) AS vname +FROM dba_objects +WHERE object_name IN ('V$INSTANCE', 'GV$INSTANCE'); + +SELECT lower(i.host_name||'_'||d.db_unique_name||'_'||d.dbid) AS p_dma_source_id +FROM ( + SELECT version, host_name + FROM &&v_name + WHERE instance_number = (SELECT min(instance_number) FROM &&v_name) ) i, v$database d / -/* -WITH control_params AS -( -SELECT 'dba' as tblprefix, - 0 as is_container, - '''N/A''' as editionable_col, - '112' as this_version, - 'op_collect_nopluggable_info.sql' as do_pluggable, - '''N/A''' as db_container_col -FROM DUAL -UNION -SELECT 'cdb' as tblprefix, - 1 as is_container, - 'EDITIONABLE' as editionable_col, - 'OTHER' as this_version, - 'op_collect_pluggable_info.sql' as do_pluggable, - 'cdb' as db_container_col -FROM DUAL -) -SELECT tblprefix AS p_tblprefix, - is_container AS p_is_container, - editionable_col AS p_editionable_col, - do_pluggable AS p_dopluggable, - db_container_col as p_db_container_col -FROM control_params WHERE ('&v_dbversion' = '112' AND this_version = '&v_dbversion') - OR ('&v_dbversion' != '112' AND this_version = 'OTHER') -*/ var lv_tblprefix VARCHAR2(3); var lv_is_container NUMBER; @@ -413,4 +399,4 @@ FROM DUAL; column CON_ID &v_h_con_id set numwidth 48 - +column v_dma_source_id format a100 diff --git a/scripts/collector/sqlserver/dmaSQLServerPerfmonDataset.ps1 b/scripts/collector/sqlserver/dmaSQLServerPerfmonDataset.ps1 index fcb2e779..0acc3961 100644 --- a/scripts/collector/sqlserver/dmaSQLServerPerfmonDataset.ps1 +++ b/scripts/collector/sqlserver/dmaSQLServerPerfmonDataset.ps1 @@ -50,7 +50,11 @@ param ( [Parameter( Mandatory=$False, HelpMessage="The pkey value for the final perfmon combined file" - )][string]$pkey=$null + )][string]$pkey=$null, + [Parameter( + Mandatory=$False, + HelpMessage="The dma_source_id value for the final perfmon combined file" + )][string]$dmaSourceId=$null ) Import-Module $PSScriptRoot\dmaCollectorCommonFunctions.psm1 @@ -438,7 +442,8 @@ param( [string]$perfmonOutDir, [string]$perfmonOutFile, [string]$pkey, - [string]$logFile + [string]$logFile, + [string]$dmaSourceId ) $outputDir = $PSScriptRoot + "\" + $perfmonOutDir @@ -464,7 +469,7 @@ param( foreach($file in Get-ChildItem -Path $env:TEMP\*$dataSet*.csv) { $tempFileName = 'PKEY_' + (Split-Path $file -leaf) Get-Content -Path $file | ForEach-Object { - '"' + $pkey + '"|' + $_ + '"' + $pkey + '"|' + $_ + '|"' + $dmaSourceId + '"' } | Out-File -FilePath $env:TEMP\$tempFileName -Encoding utf8 } } else { @@ -477,23 +482,23 @@ param( WriteLog -logLocation $outputDir\$perfmonLogFile -logMessage "Concatenating and adding header to perfmon files..." -logOperation "BOTH" if ($fileExists) { - $tempContent = '"PKEY"|"COLLECTION_TIME"|"AVAILABLE_MBYTES"|"PHYSICALDISK_AVG_DISK_BYTES_READ"|"PHYSICALDISK_AVG_DISK_BYTES_WRITE"|"PHYSICALDISK_AVG_DISK_BYTES_READ_SEC"|"PHYSICALDISK_AVG_DISK_BYTES_WRITE_SEC"|"PHYSICALDISK_DISK_READS_SEC"|"PHYSICALDISK_DISK_WRITES_SEC"|"PROCESSOR_IDLE_TIME_PCT"|"PROCESSOR_TOTAL_TIME_PCT"|"PROCESSOR_FREQUENCY"|"PROCESSOR_QUEUE_LENGTH"|"BUFFER_CACHE_HIT_RATIO"|"CHECKPOINT_PAGES_SEC"|"FREE_LIST_STALLS_SEC"|"PAGE_LIFE_EXPECTANCY"|"PAGE_LOOKUPS_SEC"|"PAGE_READS_SEC"|"PAGE_WRITES_SEC"|"USER_CONNECTION_COUNT"|"MEMORY_GRANTS_PENDING"|"TARGET_SERVER_MEMORY_KB"|"TOTAL_SERVER_MEMORY_KB"|"BATCH_REQUESTS_SEC"' + $tempContent = '"PKEY"|"COLLECTION_TIME"|"AVAILABLE_MBYTES"|"PHYSICALDISK_AVG_DISK_BYTES_READ"|"PHYSICALDISK_AVG_DISK_BYTES_WRITE"|"PHYSICALDISK_AVG_DISK_BYTES_READ_SEC"|"PHYSICALDISK_AVG_DISK_BYTES_WRITE_SEC"|"PHYSICALDISK_DISK_READS_SEC"|"PHYSICALDISK_DISK_WRITES_SEC"|"PROCESSOR_IDLE_TIME_PCT"|"PROCESSOR_TOTAL_TIME_PCT"|"PROCESSOR_FREQUENCY"|"PROCESSOR_QUEUE_LENGTH"|"BUFFER_CACHE_HIT_RATIO"|"CHECKPOINT_PAGES_SEC"|"FREE_LIST_STALLS_SEC"|"PAGE_LIFE_EXPECTANCY"|"PAGE_LOOKUPS_SEC"|"PAGE_READS_SEC"|"PAGE_WRITES_SEC"|"USER_CONNECTION_COUNT"|"MEMORY_GRANTS_PENDING"|"TARGET_SERVER_MEMORY_KB"|"TOTAL_SERVER_MEMORY_KB"|"BATCH_REQUESTS_SEC"|"DMA_SOURCE_ID"' Set-Content -Path $outputDir\$outputFileName -Value $tempContent -Encoding utf8 ((Get-Content -Path $env:TEMP\PKEY_*$dataSet*.csv -Raw ) -replace ',','|') | Add-Content -Encoding utf8 -Path $outputDir\$outputFileName } else { - $tempContent = '"PKEY"|"COLLECTION_TIME"|"AVAILABLE_MBYTES"|"PHYSICALDISK_AVG_DISK_BYTES_READ"|"PHYSICALDISK_AVG_DISK_BYTES_WRITE"|"PHYSICALDISK_AVG_DISK_BYTES_READ_SEC"|"PHYSICALDISK_AVG_DISK_BYTES_WRITE_SEC"|"PHYSICALDISK_DISK_READS_SEC"|"PHYSICALDISK_DISK_WRITES_SEC"|"PROCESSOR_IDLE_TIME_PCT"|"PROCESSOR_TOTAL_TIME_PCT"|"PROCESSOR_FREQUENCY"|"PROCESSOR_QUEUE_LENGTH"|"BUFFER_CACHE_HIT_RATIO"|"CHECKPOINT_PAGES_SEC"|"FREE_LIST_STALLS_SEC"|"PAGE_LIFE_EXPECTANCY"|"PAGE_LOOKUPS_SEC"|"PAGE_READS_SEC"|"PAGE_WRITES_SEC"|"USER_CONNECTION_COUNT"|"MEMORY_GRANTS_PENDING"|"TARGET_SERVER_MEMORY_KB"|"TOTAL_SERVER_MEMORY_KB"|"BATCH_REQUESTS_SEC"' + $tempContent = '"PKEY"|"COLLECTION_TIME"|"AVAILABLE_MBYTES"|"PHYSICALDISK_AVG_DISK_BYTES_READ"|"PHYSICALDISK_AVG_DISK_BYTES_WRITE"|"PHYSICALDISK_AVG_DISK_BYTES_READ_SEC"|"PHYSICALDISK_AVG_DISK_BYTES_WRITE_SEC"|"PHYSICALDISK_DISK_READS_SEC"|"PHYSICALDISK_DISK_WRITES_SEC"|"PROCESSOR_IDLE_TIME_PCT"|"PROCESSOR_TOTAL_TIME_PCT"|"PROCESSOR_FREQUENCY"|"PROCESSOR_QUEUE_LENGTH"|"BUFFER_CACHE_HIT_RATIO"|"CHECKPOINT_PAGES_SEC"|"FREE_LIST_STALLS_SEC"|"PAGE_LIFE_EXPECTANCY"|"PAGE_LOOKUPS_SEC"|"PAGE_READS_SEC"|"PAGE_WRITES_SEC"|"USER_CONNECTION_COUNT"|"MEMORY_GRANTS_PENDING"|"TARGET_SERVER_MEMORY_KB"|"TOTAL_SERVER_MEMORY_KB"|"BATCH_REQUESTS_SEC"|"DMA_SOURCE_ID"' Set-Content -Path $outputDir\$outputFileName -Value $tempContent -Encoding utf8 $tempDate = Get-Date -Format "MM/dd/yyyy HH:mm:ss.fff" - $tempContent = '"' + $pkey + '"|"' + $tempDate + '"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"' + $tempContent = '"' + $pkey + '"|"' + $tempDate + '"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"' + $dmaSourceId + '"' Add-Content -Path $outputDir\$outputFileName -Value $tempContent -Encoding utf8 $futureDate = (Get-Date).AddMinutes(1) $tempDate = $futureDate.ToString("MM/dd/yyyy HH:mm:ss.fff") - $tempContent = '"' + $pkey + '"|"' + $tempDate + '"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"' + $tempContent = '"' + $pkey + '"|"' + $tempDate + '"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"' + $dmaSourceId + '"' Add-Content -Path $outputDir\$outputFileName -Value $tempContent -Encoding utf8 } @@ -501,8 +506,13 @@ param( WriteLog -logLocation $outputDir\$perfmonLogFile -logMessage "Clean up Temp File area..." -logOperation "BOTH" Remove-Item -Path $env:TEMP\*$dataSet*.csv } + WriteLog -logLocation $outputDir\$perfmonLogFile -logMessage " " -logOperation "FILE" + WriteLog -logLocation $foldername\$logFile -logMessage "DMA Source Id: $dmaSourceId " -logOperation "FILE" + WriteLog -logLocation $outputDir\$perfmonLogFile -logMessage " " -logOperation "FILE" WriteLog -logLocation $outputDir\$perfmonLogFile -logMessage "Collecting current state of perfmon dataset: $dataset..." -logOperation "BOTH" logman.exe query -n $dataset | out-string | Add-Content -Encoding utf8 -Path $outputDir\$perfmonLogFile + + } function CreateEmptyFile @@ -512,12 +522,16 @@ function CreateEmptyFile [string]$perfmonOutDir, [string]$perfmonOutFile, [string]$pkey, - [string]$logFile + [string]$logFile, + [string]$dmaSourceId ) $outputDir = $PSScriptRoot + "\" + $perfmonOutDir $outputFileName = $perfmonOutFile WriteLog -logLocation $outputDir\$perfmonLogFile -logMessage "Creating an empty Google DMA SQL Server Perfmon Counter Data Set..." -logOperation "BOTH" + WriteLog -logLocation $outputDir\$perfmonLogFile -logMessage " " -logOperation "FILE" + WriteLog -logLocation $foldername\$logFile -logMessage "DMA Source Id: $dmaSourceId " -logOperation "FILE" + WriteLog -logLocation $outputDir\$perfmonLogFile -logMessage " " -logOperation "FILE" if (!(Test-Path -PathType container $outputDir)) { WriteLog -logLocation $outputDir\$perfmonLogFile -logMessage " " -logOperation "BOTH" @@ -525,16 +539,16 @@ function CreateEmptyFile $null = New-Item -ItemType Directory -Path $outputDir } - $tempContent = '"PKEY"|"COLLECTION_TIME"|"AVAILABLE_MBYTES"|"PHYSICALDISK_AVG_DISK_BYTES_READ"|"PHYSICALDISK_AVG_DISK_BYTES_WRITE"|"PHYSICALDISK_AVG_DISK_BYTES_READ_SEC"|"PHYSICALDISK_AVG_DISK_BYTES_WRITE_SEC"|"PHYSICALDISK_DISK_READS_SEC"|"PHYSICALDISK_DISK_WRITES_SEC"|"PROCESSOR_IDLE_TIME_PCT"|"PROCESSOR_TOTAL_TIME_PCT"|"PROCESSOR_FREQUENCY"|"PROCESSOR_QUEUE_LENGTH"|"BUFFER_CACHE_HIT_RATIO"|"CHECKPOINT_PAGES_SEC"|"FREE_LIST_STALLS_SEC"|"PAGE_LIFE_EXPECTANCY"|"PAGE_LOOKUPS_SEC"|"PAGE_READS_SEC"|"PAGE_WRITES_SEC"|"USER_CONNECTION_COUNT"|"MEMORY_GRANTS_PENDING"|"TARGET_SERVER_MEMORY_KB"|"TOTAL_SERVER_MEMORY_KB"|"BATCH_REQUESTS_SEC"' + $tempContent = '"PKEY"|"COLLECTION_TIME"|"AVAILABLE_MBYTES"|"PHYSICALDISK_AVG_DISK_BYTES_READ"|"PHYSICALDISK_AVG_DISK_BYTES_WRITE"|"PHYSICALDISK_AVG_DISK_BYTES_READ_SEC"|"PHYSICALDISK_AVG_DISK_BYTES_WRITE_SEC"|"PHYSICALDISK_DISK_READS_SEC"|"PHYSICALDISK_DISK_WRITES_SEC"|"PROCESSOR_IDLE_TIME_PCT"|"PROCESSOR_TOTAL_TIME_PCT"|"PROCESSOR_FREQUENCY"|"PROCESSOR_QUEUE_LENGTH"|"BUFFER_CACHE_HIT_RATIO"|"CHECKPOINT_PAGES_SEC"|"FREE_LIST_STALLS_SEC"|"PAGE_LIFE_EXPECTANCY"|"PAGE_LOOKUPS_SEC"|"PAGE_READS_SEC"|"PAGE_WRITES_SEC"|"USER_CONNECTION_COUNT"|"MEMORY_GRANTS_PENDING"|"TARGET_SERVER_MEMORY_KB"|"TOTAL_SERVER_MEMORY_KB"|"BATCH_REQUESTS_SEC"|"DMA_SOURCE_ID"' Set-Content $env:TEMP\emptyStrings.csv -Value $tempContent -Encoding utf8 $tempDate = Get-Date -Format "MM/dd/yyyy HH:mm:ss.fff" - $tempContent = '"' + $pkey + '"|"' + $tempDate + '"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"' + $tempContent = '"' + $pkey + '"|"' + $tempDate + '"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"' + $dmaSourceId + '"' Add-Content $env:TEMP\emptyStrings.csv -Value $tempContent -Encoding utf8 $futureDate = (Get-Date).AddMinutes(1) $tempDate = $futureDate.ToString("MM/dd/yyyy HH:mm:ss.fff") - $tempContent = '"' + $pkey + '"|"' + $tempDate + '"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"' + $tempContent = '"' + $pkey + '"|"' + $tempDate + '"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"0"|"' + $dmaSourceId + '"' Add-Content $env:TEMP\emptyStrings.csv -Value $tempContent -Encoding utf8 WriteLog -logLocation $outputDir\$perfmonLogFile -logMessage "Concatenating and adding header to perfmon files to $outputFileName ..." -logOperation "BOTH" @@ -575,9 +589,9 @@ if ($operation.ToLower() -eq "create") { } elseif ($operation.ToLower() -eq "delete") { DeleteDMAPerfmonDataSet -dataSet $datasetName -perfmonOutDir $perfmonOutDir -logFile $perfmonLogFile } elseif ($operation.ToLower() -eq "collect") { - CollectDMAPerfmonDataSet -dataSet $datasetName -perfmonOutDir $perfmonOutDir -perfmonOutFile $perfmonOutFile -pkey $pkey -logFile $perfmonLogFile + CollectDMAPerfmonDataSet -dataSet $datasetName -perfmonOutDir $perfmonOutDir -perfmonOutFile $perfmonOutFile -pkey $pkey -dmaSourceId $dmaSourceId -logFile $perfmonLogFile } elseif ($operation.ToLower() -eq "createemptyfile") { - CreateEmptyFile -dataSet $datasetName -perfmonOutDir $perfmonOutDir -perfmonOutFile $perfmonOutFile -pkey $pkey -logFile $perfmonLogFile + CreateEmptyFile -dataSet $datasetName -perfmonOutDir $perfmonOutDir -perfmonOutFile $perfmonOutFile -pkey $pkey -dmaSourceId $dmaSourceId -logFile $perfmonLogFile } else { Write-Output "Operation $operation specified is invalid" } \ No newline at end of file diff --git a/scripts/collector/sqlserver/instanceReview.ps1 b/scripts/collector/sqlserver/instanceReview.ps1 index 62ef2ca4..96f750c1 100644 --- a/scripts/collector/sqlserver/instanceReview.ps1 +++ b/scripts/collector/sqlserver/instanceReview.ps1 @@ -70,6 +70,7 @@ if ([string]::IsNullorEmpty($serverName)) { $inputServerName = $serverName $obj = sqlcmd -S $serverName -i sql\foldername.sql -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v database=$database | findstr /v /c:"---" $dbNameArray = @(sqlcmd -S $serverName -i sql\getDBList.sql -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v database=$database) + $dmaSourceId = @(sqlcmd -S $serverName -i sql\getDmaSourceId.sql -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1) if ([string]$database -ne "all") { $validDBObj = sqlcmd -S $serverName -i sql\checkValidDatabase.sql -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v database=$database | findstr /v /c:"-" $countValidDBs = $validDBObj @@ -85,6 +86,7 @@ if ([string]::IsNullorEmpty($serverName)) { WriteLog -logMessage "Retrieving Metadata Information from $serverName" -logOperation "MESSAGE" $obj = sqlcmd -S $serverName -i sql\foldername.sql -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v database=$database | findstr /v /c:"---" $dbNameArray = @(sqlcmd -S $serverName -i sql\getDBList.sql -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v database=$database) + $dmaSourceId = @(sqlcmd -S $serverName -i sql\getDmaSourceId.sql -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1) if ([string]$database -ne "all") { $validDBObj = sqlcmd -S $serverName -i sql\checkValidDatabase.sql -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v database=$database | findstr /v /c:"-" $countValidDBs = $validDBObj @@ -116,8 +118,9 @@ if ([string]$database -eq "all") { $instancename = $values[3] $current_ts = $values[4] $pkey = $values[5] +$dmaSourceId = $dmaSourceId[0] -$op_version = "4.3.14" +$op_version = "4.3.15" if ($ignorePerfmon -eq "true") { $perfCounterLabel = "NoPerfCounter" @@ -162,6 +165,9 @@ $PSVersionTable | out-string | Add-Content -Encoding utf8 -Path $foldername\$log WriteLog -logLocation $foldername\$logFile -logMessage "Output Encoding Table" -logOperation "FILE" $OutputEncoding | out-string | Add-Content -Encoding utf8 -Path $foldername\$logFile +WriteLog -logLocation $foldername\$logFile -logMessage "DMA Source Id: $dmaSourceId " -logOperation "FILE" +WriteLog -logLocation $foldername\$logFile -logMessage " " -logOperation "FILE" + WriteLog -logLocation $foldername\$logFile -logMessage "Execution Variables List" -logOperation "FILE" WriteLog -logLocation $foldername\$logFile -logMessage " " -logOperation "FILE" WriteLog -logLocation $foldername\$logFile -logMessage "serverName = $inputServerName " -logOperation "FILE" @@ -212,56 +218,56 @@ foreach ($dbNameList in $dbNameArray) { } WriteLog -logLocation $foldername\$logFile -logMessage "Retriving SQL Server Installed Components..." -logOperation "BOTH" - sqlcmd -S $serverName -i sql\componentsInstalled.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey -s"|" | findstr /v /c:"---" > $foldername\$compFileName + sqlcmd -S $serverName -i sql\componentsInstalled.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey dmaSourceId=$dmaSourceId -s"|" | findstr /v /c:"---" > $foldername\$compFileName WriteLog -logLocation $foldername\$logFile -logMessage "Retriving SQL Server Properties..." -logOperation "BOTH" - sqlcmd -S $serverName -i sql\serverProperties.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey -s"|" | findstr /v /c:"---" > $foldername\$srvFileName + sqlcmd -S $serverName -i sql\serverProperties.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey dmaSourceId=$dmaSourceId -s"|" | findstr /v /c:"---" > $foldername\$srvFileName WriteLog -logLocation $foldername\$logFile -logMessage "Retriving SQL Server CloudSQL Unsupported Flag Info..." -logOperation "BOTH" - sqlcmd -S $serverName -i sql\dbServerUnsupportedFlags.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey -s"|" | findstr /v /c:"---" > $foldername\$dbServerFlags + sqlcmd -S $serverName -i sql\dbServerUnsupportedFlags.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey dmaSourceId=$dmaSourceId -s"|" | findstr /v /c:"---" > $foldername\$dbServerFlags WriteLog -logLocation $foldername\$logFile -logMessage "Retriving SQL Server Features in Use Info..." -logOperation "BOTH" - sqlcmd -S $serverName -i sql\dbServerFeatures.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey -s"|" | findstr /v /c:"---" > $foldername\$blockingFeatures + sqlcmd -S $serverName -i sql\dbServerFeatures.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey dmaSourceId=$dmaSourceId -s"|" | findstr /v /c:"---" > $foldername\$blockingFeatures WriteLog -logLocation $foldername\$logFile -logMessage "Retriving SQL Server Linked Server Info..." -logOperation "BOTH" - sqlcmd -S $serverName -i sql\linkedServers.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey -s"|" | findstr /v /c:"---" > $foldername\$linkedServers + sqlcmd -S $serverName -i sql\linkedServers.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey dmaSourceId=$dmaSourceId -s"|" | findstr /v /c:"---" > $foldername\$linkedServers WriteLog -logLocation $foldername\$logFile -logMessage "Retriving SQL Server Cluster Node Info..." -logOperation "BOTH" - sqlcmd -S $serverName -i sql\dbClusterNodes.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey -s"|" | findstr /v /c:"---" > $foldername\$dbClusterNodes + sqlcmd -S $serverName -i sql\dbClusterNodes.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey dmaSourceId=$dmaSourceId -s"|" | findstr /v /c:"---" > $foldername\$dbClusterNodes WriteLog -logLocation $foldername\$logFile -logMessage "Retriving SQL Server DBCC Trace Info..." -logOperation "BOTH" - sqlcmd -S $serverName -i sql\dbccTraceFlags.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey -s"|" | findstr /v /c:"---" > $foldername\$dbccTraceFlg + sqlcmd -S $serverName -i sql\dbccTraceFlags.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey dmaSourceId=$dmaSourceId -s"|" | findstr /v /c:"---" > $foldername\$dbccTraceFlg WriteLog -logLocation $foldername\$logFile -logMessage "Retriving SQL Server Disk Volume Info..." -logOperation "BOTH" - sqlcmd -S $serverName -i sql\diskVolumeInfo.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey -s"|" | findstr /v /c:"---" > $foldername\$diskVolumeInfo + sqlcmd -S $serverName -i sql\diskVolumeInfo.sql -d master -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -v pkey=$pkey dmaSourceId=$dmaSourceId -s"|" | findstr /v /c:"---" > $foldername\$diskVolumeInfo ### First establish headers for the collection files which could execute against multiple databases in the instance -Set-Content -Path $foldername\$objectList -Encoding utf8 -Value "PKEY|database_name|schema_name|object_name|object_type|object_type_desc|object_count|lines_of_code|associated_table_name" -Set-Content -Path $foldername\$tableList -Encoding utf8 -Value "PKEY|database_name|schema_name|table_name|partition_count|is_memory_optimized|temporal_type|is_external|lock_escalation|is_tracked_by_cdc|text_in_row_limit|is_replicated|row_count|data_compression|total_space_mb|used_space_mb|unused_space_mb" -Set-Content -Path $foldername\$indexList -Encoding utf8 -Value "PKEY|database_name|schema_name|table_name|index_name|index_type|is_primary_key|is_unique|fill_factor|allow_page_locks|has_filter|data_compression|data_compression_desc|is_partitioned|count_key_ordinal|count_partition_ordinal|count_is_included_column|total_space_mb" -Set-Content -Path $foldername\$columnDatatypes -Encoding utf8 -Value "PKEY|database_name|schema_name|table_name|datatype|max_length|precision|scale|is_computed|is_filestream|is_masked|encryption_type|is_sparse|rule_object_id|column_count" -Set-Content -Path $foldername\$userConnectionList -Encoding utf8 -Value "PKEY|database_name|is_user_process|host_name|program_name|login_name|num_reads|num_writes|last_read|last_write|reads|logical_reads|writes|client_interface_name|nt_domain|nt_user_name|client_net_address|local_net_address" -Set-Content -Path $foldername\$dbsizes -Encoding utf8 -Value "PKEY|database_name|type_desc|current_size_mb" +Set-Content -Path $foldername\$objectList -Encoding utf8 -Value "PKEY|database_name|schema_name|object_name|object_type|object_type_desc|object_count|lines_of_code|associated_table_name|dma_source_id" +Set-Content -Path $foldername\$tableList -Encoding utf8 -Value "PKEY|database_name|schema_name|table_name|partition_count|is_memory_optimized|temporal_type|is_external|lock_escalation|is_tracked_by_cdc|text_in_row_limit|is_replicated|row_count|data_compression|total_space_mb|used_space_mb|unused_space_mb|dma_source_id" +Set-Content -Path $foldername\$indexList -Encoding utf8 -Value "PKEY|database_name|schema_name|table_name|index_name|index_type|is_primary_key|is_unique|fill_factor|allow_page_locks|has_filter|data_compression|data_compression_desc|is_partitioned|count_key_ordinal|count_partition_ordinal|count_is_included_column|total_space_mb|dma_source_id" +Set-Content -Path $foldername\$columnDatatypes -Encoding utf8 -Value "PKEY|database_name|schema_name|table_name|datatype|max_length|precision|scale|is_computed|is_filestream|is_masked|encryption_type|is_sparse|rule_object_id|column_count|dma_source_id" +Set-Content -Path $foldername\$userConnectionList -Encoding utf8 -Value "PKEY|database_name|is_user_process|host_name|program_name|login_name|num_reads|num_writes|last_read|last_write|reads|logical_reads|writes|client_interface_name|nt_domain|nt_user_name|client_net_address|local_net_address|dma_source_id" +Set-Content -Path $foldername\$dbsizes -Encoding utf8 -Value "PKEY|database_name|type_desc|current_size_mb|dma_source_id" ### Iterate through collections that could execute against multiple databases in the instance foreach ($databaseName in $dbNameArray) { WriteLog -logLocation $foldername\$logFile -logMessage "Retriving SQL Server Object Info for Database $databaseName ..." -logOperation "BOTH" - sqlcmd -S $serverName -i sql\objectList.sql -d $databaseName -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v pkey=$pkey database=$databaseName -s"|" | findstr /v /c:"---" | Add-Content -Path $foldername\$objectList -Encoding utf8 + sqlcmd -S $serverName -i sql\objectList.sql -d $databaseName -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v pkey=$pkey database=$databaseName dmaSourceId=$dmaSourceId -s"|" | findstr /v /c:"---" | Add-Content -Path $foldername\$objectList -Encoding utf8 WriteLog -logLocation $foldername\$logFile -logMessage "Retriving SQL Server Table Info for Database $databaseName ..." -logOperation "BOTH" - sqlcmd -S $serverName -i sql\tableList.sql -d $databaseName -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v pkey=$pkey database=$databaseName -s"|" | findstr /v /c:"---" | Add-Content -Path $foldername\$tableList -Encoding utf8 + sqlcmd -S $serverName -i sql\tableList.sql -d $databaseName -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v pkey=$pkey database=$databaseName dmaSourceId=$dmaSourceId -s"|" | findstr /v /c:"---" | Add-Content -Path $foldername\$tableList -Encoding utf8 WriteLog -logLocation $foldername\$logFile -logMessage "Retriving SQL Server Index Info for Database $databaseName ..." -logOperation "BOTH" - sqlcmd -S $serverName -i sql\indexList.sql -d $databaseName -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v pkey=$pkey database=$databaseName -s"|" | findstr /v /c:"---" | Add-Content -Path $foldername\$indexList -Encoding utf8 + sqlcmd -S $serverName -i sql\indexList.sql -d $databaseName -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v pkey=$pkey database=$databaseName dmaSourceId=$dmaSourceId -s"|" | findstr /v /c:"---" | Add-Content -Path $foldername\$indexList -Encoding utf8 WriteLog -logLocation $foldername\$logFile -logMessage "Retriving SQL Server Column Datatype Info for Database $databaseName ..." -logOperation "BOTH" - sqlcmd -S $serverName -i sql\columnDatatypes.sql -d $databaseName -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v pkey=$pkey database=$databaseName -s"|" | findstr /v /c:"---" | Add-Content -Path $foldername\$columnDatatypes -Encoding utf8 + sqlcmd -S $serverName -i sql\columnDatatypes.sql -d $databaseName -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v pkey=$pkey database=$databaseName dmaSourceId=$dmaSourceId -s"|" | findstr /v /c:"---" | Add-Content -Path $foldername\$columnDatatypes -Encoding utf8 WriteLog -logLocation $foldername\$logFile -logMessage "Retriving SQL Server User Connection Info for Database $databaseName ..." -logOperation "BOTH" - sqlcmd -S $serverName -i sql\userConnectionInfo.sql -d $databaseName -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v pkey=$pkey database=$databaseName -s"|" | findstr /v /c:"---" | Add-Content -Path $foldername\$userConnectionList -Encoding utf8 + sqlcmd -S $serverName -i sql\userConnectionInfo.sql -d $databaseName -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v pkey=$pkey database=$databaseName dmaSourceId=$dmaSourceId -s"|" | findstr /v /c:"---" | Add-Content -Path $foldername\$userConnectionList -Encoding utf8 WriteLog -logLocation $foldername\$logFile -logMessage "Retriving SQL Server Database Size Info for Database $databaseName ..." -logOperation "BOTH" - sqlcmd -S $serverName -i sql\dbSizes.sql -d $databaseName -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v pkey=$pkey database=$databaseName -s"|" | findstr /v /c:"---" | Add-Content -Path $foldername\$dbsizes -Encoding utf8 + sqlcmd -S $serverName -i sql\dbSizes.sql -d $databaseName -U $collectionUserName -P $collectionUserPass -l 30 -W -m 1 -u -h-1 -v pkey=$pkey database=$databaseName dmaSourceId=$dmaSourceId -s"|" | findstr /v /c:"---" | Add-Content -Path $foldername\$dbsizes -Encoding utf8 } # Pull perfmon file if we are running from same server. Generate empty file if running on remote server @@ -270,27 +276,28 @@ foreach ($databaseName in $dbNameArray) { if ($ignorePerfmon -eq "true") { WriteLog -logLocation $foldername\$logFile -logMessage "Skipping Perfmon Information..." -logOperation "FILE" if (($instancename -eq "MSSQLSERVER") -and ([string]$env:computername.toUpper() -ne [string]$machinename.toUpper())) { - .\dmaSQLServerPerfmonDataset.ps1 -operation createemptyfile -perfmonOutDir $foldername -perfmonOutFile $perfMonOutput -pkey $pkey + .\dmaSQLServerPerfmonDataset.ps1 -operation createemptyfile -perfmonOutDir $foldername -perfmonOutFile $perfMonOutput -pkey $pkey -dmaSourceId $dmaSourceId } else { - .\dmaSQLServerPerfmonDataset.ps1 -operation createemptyfile -managedInstanceName $instancename -perfmonOutDir $foldername -perfmonOutFile $perfMonOutput -pkey $pkey + .\dmaSQLServerPerfmonDataset.ps1 -operation createemptyfile -managedInstanceName $instancename -perfmonOutDir $foldername -perfmonOutFile $perfMonOutput -pkey $pkey -dmaSourceId $dmaSourceId } } else { WriteLog -logLocation $foldername\$logFile -logMessage "Retrieving Perfmon Information..." -logOperation "FILE" if (($instancename -eq "MSSQLSERVER") -and ([string]$env:computername.toUpper() -eq [string]$machinename.toUpper())) { - .\dmaSQLServerPerfmonDataset.ps1 -operation collect -perfmonOutDir $foldername -perfmonOutFile $perfMonOutput -pkey $pkey + .\dmaSQLServerPerfmonDataset.ps1 -operation collect -perfmonOutDir $foldername -perfmonOutFile $perfMonOutput -pkey $pkey -dmaSourceId $dmaSourceId } elseif (($instancename -ne "MSSQLSERVER") -and ([string]$env:computername.toUpper() -eq [string]$machinename.toUpper())) { - .\dmaSQLServerPerfmonDataset.ps1 -operation collect -managedInstanceName $instancename -perfmonOutDir $foldername -perfmonOutFile $perfMonOutput -pkey $pkey + .\dmaSQLServerPerfmonDataset.ps1 -operation collect -managedInstanceName $instancename -perfmonOutDir $foldername -perfmonOutFile $perfMonOutput -pkey $pkey -dmaSourceId $dmaSourceId } elseif (($instancename -eq "MSSQLSERVER") -and ([string]$env:computername.toUpper() -ne [string]$machinename.toUpper())) { - .\dmaSQLServerPerfmonDataset.ps1 -operation createemptyfile -perfmonOutDir $foldername -perfmonOutFile $perfMonOutput -pkey $pkey + .\dmaSQLServerPerfmonDataset.ps1 -operation createemptyfile -perfmonOutDir $foldername -perfmonOutFile $perfMonOutput -pkey $pkey -dmaSourceId $dmaSourceId } elseif (($instancename -ne "MSSQLSERVER") -and ([string]$env:computername.toUpper() -ne [string]$machinename.toUpper())) { - .\dmaSQLServerPerfmonDataset.ps1 -operation createemptyfile -managedInstanceName $instancename -perfmonOutDir $foldername -perfmonOutFile $perfMonOutput -pkey $pkey + .\dmaSQLServerPerfmonDataset.ps1 -operation createemptyfile -managedInstanceName $instancename -perfmonOutDir $foldername -perfmonOutFile $perfMonOutput -pkey $pkey -dmaSourceId $dmaSourceId } } WriteLog -logLocation $foldername\$logFile -logMessage "Remove special characters and UTF8 BOM from extracted files..." -logOperation "BOTH" foreach($file in Get-ChildItem -Path $foldername\*.csv) { $inputFile = Split-Path -Leaf $file - ((Get-Content -Path $foldername\$inputFile) -join "`n") + "`n" | Set-Content -NoNewLine -Encoding utf8 -Force -Path $foldername\$inputFile + #((Get-Content -Path $foldername\$inputFile) -join "`n") + "`n" | Set-Content -NoNewLine -Encoding utf8 -Force -Path $foldername\$inputFile + ((Get-Content -Path $foldername\$inputFile) -join "`n") + "`n" | Set-Content -Encoding utf8 -Force -Path $foldername\$inputFile $utf8 = New-Object System.Text.UTF8Encoding $false $fileContent = Get-Content $foldername\$inputFile -Raw Set-Content -Value $utf8.GetBytes($fileContent) -Encoding Byte -Path $foldername\$inputFile -Force @@ -347,4 +354,4 @@ if ($powerShellVersion -ge 5) { WriteLog -logLocation $foldername\$logFile -logMessage "Collection Complete..." -logOperation "MESSAGE" } -Exit 0 +Exit 0 \ No newline at end of file diff --git a/scripts/collector/sqlserver/sql/columnDatatypes.sql b/scripts/collector/sqlserver/sql/columnDatatypes.sql index cd8134dd..ab0477fa 100644 --- a/scripts/collector/sqlserver/sql/columnDatatypes.sql +++ b/scripts/collector/sqlserver/sql/columnDatatypes.sql @@ -17,14 +17,18 @@ limitations under the License. SET NOCOUNT ON; SET LANGUAGE us_english; + DECLARE @PKEY AS VARCHAR(256) DECLARE @CLOUDTYPE AS VARCHAR(256) DECLARE @ASSESSMENT_DATABSE_NAME AS VARCHAR(256) DECLARE @PRODUCT_VERSION AS INTEGER DECLARE @validDB AS INTEGER +DECLARE @DMA_SOURCE_ID AS VARCHAR(256) SELECT @PKEY = N'$(pkey)'; SELECT @ASSESSMENT_DATABSE_NAME = N'$(database)'; +SELECT @DMA_SOURCE_ID = N'$(dmaSourceId)'; + IF @ASSESSMENT_DATABSE_NAME = 'all' SELECT @ASSESSMENT_DATABSE_NAME = '%' SELECT @PRODUCT_VERSION = CONVERT(INTEGER, PARSENAME(CONVERT(nvarchar, SERVERPROPERTY('productversion')), 4)); @@ -234,7 +238,7 @@ BEGIN END -SELECT @PKEY as PKEY, a.* from #columnDatatypes a; +SELECT @PKEY as PKEY, a.*, @DMA_SOURCE_ID as dma_source_id from #columnDatatypes a; IF OBJECT_ID('tempdb..#columnDatatypes') IS NOT NULL DROP TABLE #columnDatatypes; \ No newline at end of file diff --git a/scripts/collector/sqlserver/sql/componentsInstalled.sql b/scripts/collector/sqlserver/sql/componentsInstalled.sql index ff541c50..1308b876 100644 --- a/scripts/collector/sqlserver/sql/componentsInstalled.sql +++ b/scripts/collector/sqlserver/sql/componentsInstalled.sql @@ -17,9 +17,12 @@ limitations under the License. SET nocount ON; SET language us_english; DECLARE @PKEY AS VARCHAR(256) -SELECT @PKEY = N'$(pkey)'; +DECLARE @DMA_SOURCE_ID AS VARCHAR(256) DECLARE @TABLE_PERMISSION_COUNT AS INTEGER +SELECT @PKEY = N'$(pkey)'; +SELECT @DMA_SOURCE_ID = N'$(dmaSourceId)'; + /* need to record table permissions in order to determine if we can run certain server level queryies as some tables are not available in managed instances */ @@ -620,12 +623,12 @@ BEGIN CATCH END CATCH /* -------------------------------------------------------------------------------------------------------------*/ SELECT @PKEY AS PKEY, - physicalsrvername AS 'Physical Server Name' /*Display finding*/ - , - servername AS 'SQL Instance Name', - servicename AS 'SQL Server Services', - servicestatus AS 'Current Service Service Status', - statusdatetime AS 'Date/Time Service Status Checked' + physicalsrvername AS 'physical_server_name', + servername AS 'sql_instance_name', + servicename AS 'sql_server_services', + servicestatus AS 'current_service_status', + statusdatetime AS 'status_date_time', + @DMA_SOURCE_ID AS 'dma_source_id' FROM #servicesservicestatus /* -------------------------------------------------------------------------------------------------------------*/ diff --git a/scripts/collector/sqlserver/sql/dbClusterNodes.sql b/scripts/collector/sqlserver/sql/dbClusterNodes.sql index ec116589..086f4d17 100644 --- a/scripts/collector/sqlserver/sql/dbClusterNodes.sql +++ b/scripts/collector/sqlserver/sql/dbClusterNodes.sql @@ -17,9 +17,13 @@ limitations under the License. SET NOCOUNT ON; SET LANGUAGE us_english; + DECLARE @PKEY AS VARCHAR(256) -SELECT @PKEY = N'$(pkey)'; DECLARE @PRODUCT_VERSION AS INTEGER +DECLARE @DMA_SOURCE_ID AS VARCHAR(256) + +SELECT @PKEY = N'$(pkey)'; +SELECT @DMA_SOURCE_ID = N'$(dmaSourceId)'; SELECT @PRODUCT_VERSION = CONVERT(INTEGER, PARSENAME(CONVERT(nvarchar, SERVERPROPERTY('productversion')), 4)); IF OBJECT_ID('tempdb..#clusterNodesTable') IS NOT NULL @@ -70,7 +74,7 @@ BEGIN CATCH SUBSTRING(CONVERT(nvarchar,ERROR_MESSAGE()),1,512) as error_message END CATCH -SELECT @PKEY as PKEY, NodeName AS node_name, status, status_description from #clusterNodesTable; +SELECT @PKEY as PKEY, NodeName AS node_name, status, status_description, @DMA_SOURCE_ID as dma_source_id from #clusterNodesTable; IF OBJECT_ID('tempdb..#clusterNodesTable') IS NOT NULL DROP TABLE #clusterNodesTable; \ No newline at end of file diff --git a/scripts/collector/sqlserver/sql/dbServerFeatures.sql b/scripts/collector/sqlserver/sql/dbServerFeatures.sql index 78b0b4e1..96353a52 100644 --- a/scripts/collector/sqlserver/sql/dbServerFeatures.sql +++ b/scripts/collector/sqlserver/sql/dbServerFeatures.sql @@ -21,8 +21,10 @@ SET LANGUAGE us_english; DECLARE @PKEY AS VARCHAR(256) DECLARE @CLOUDTYPE AS VARCHAR(256) DECLARE @PRODUCT_VERSION AS INTEGER +DECLARE @DMA_SOURCE_ID AS VARCHAR(256) SELECT @PKEY = N'$(pkey)'; +SELECT @DMA_SOURCE_ID = N'$(dmaSourceId)'; SELECT @PRODUCT_VERSION = CONVERT(INTEGER, PARSENAME(CONVERT(nvarchar, SERVERPROPERTY('productversion')), 4)); SELECT @CLOUDTYPE = 'NONE' IF UPPER(@@VERSION) LIKE '%AZURE%' @@ -141,4 +143,4 @@ BEGIN CATCH exec('INSERT INTO #FeaturesEnabled VALUES (''Policy Based Management'', ''No'', ''0'')') END CATCH -SELECT @PKEY as PKEY, * FROM #FeaturesEnabled; \ No newline at end of file +SELECT @PKEY as PKEY, f.*, @DMA_SOURCE_ID as dma_source_id FROM #FeaturesEnabled f; \ No newline at end of file diff --git a/scripts/collector/sqlserver/sql/dbServerUnsupportedFlags.sql b/scripts/collector/sqlserver/sql/dbServerUnsupportedFlags.sql index 985b6789..000a3e96 100644 --- a/scripts/collector/sqlserver/sql/dbServerUnsupportedFlags.sql +++ b/scripts/collector/sqlserver/sql/dbServerUnsupportedFlags.sql @@ -19,15 +19,19 @@ SET NOCOUNT ON; SET LANGUAGE us_english; DECLARE @PKEY AS VARCHAR(256) DECLARE @PRODUCT_VERSION AS INTEGER +DECLARE @DMA_SOURCE_ID AS VARCHAR(256) + SELECT @PRODUCT_VERSION = CONVERT(INTEGER, PARSENAME(CONVERT(nvarchar, SERVERPROPERTY('productversion')), 4)); SELECT @PKEY = N'$(pkey)'; +SELECT @DMA_SOURCE_ID = N'$(dmaSourceId)'; SELECT @PKEY as PKEY, lower(name) as flag_name, value, value_in_use, - SUBSTRING(description, 1, 256) AS description + SUBSTRING(description, 1, 256) AS description, + @DMA_SOURCE_ID as dma_source_id FROM sys.configurations WHERE diff --git a/scripts/collector/sqlserver/sql/dbSizes.sql b/scripts/collector/sqlserver/sql/dbSizes.sql index 854ca565..dde02472 100644 --- a/scripts/collector/sqlserver/sql/dbSizes.sql +++ b/scripts/collector/sqlserver/sql/dbSizes.sql @@ -17,14 +17,18 @@ limitations under the License. SET NOCOUNT ON; SET LANGUAGE us_english; + DECLARE @PKEY AS VARCHAR(256) DECLARE @CLOUDTYPE AS VARCHAR(256) DECLARE @ASSESSMENT_DATABSE_NAME AS VARCHAR(256) DECLARE @PRODUCT_VERSION AS INTEGER DECLARE @validDB AS INTEGER +DECLARE @DMA_SOURCE_ID AS VARCHAR(256) SELECT @PKEY = N'$(pkey)'; SELECT @ASSESSMENT_DATABSE_NAME = N'$(database)'; +SELECT @DMA_SOURCE_ID = N'$(dmaSourceId)'; + IF @ASSESSMENT_DATABSE_NAME = 'all' SELECT @ASSESSMENT_DATABSE_NAME = '%' SELECT @PRODUCT_VERSION = CONVERT(INTEGER, PARSENAME(CONVERT(nvarchar, SERVERPROPERTY('productversion')), 4)); @@ -45,7 +49,7 @@ BEGIN IF @validDB <> 0 BEGIN SELECT - @PKEY as PKEY, sizing.* + @PKEY as PKEY, sizing.*, @DMA_SOURCE_ID as dma_source_id FROM( SELECT db_name() AS database_name, diff --git a/scripts/collector/sqlserver/sql/dbccTraceFlags.sql b/scripts/collector/sqlserver/sql/dbccTraceFlags.sql index 83ecf417..e7f50895 100644 --- a/scripts/collector/sqlserver/sql/dbccTraceFlags.sql +++ b/scripts/collector/sqlserver/sql/dbccTraceFlags.sql @@ -17,8 +17,12 @@ limitations under the License. SET NOCOUNT ON; SET LANGUAGE us_english; + DECLARE @PKEY AS VARCHAR(256) +DECLARE @DMA_SOURCE_ID AS VARCHAR(256) + SELECT @PKEY = N'$(pkey)'; +SELECT @DMA_SOURCE_ID = N'$(dmaSourceId)'; IF OBJECT_ID('tempdb..#dbccTraceTable') IS NOT NULL DROP TABLE #dbccTraceTable; @@ -32,7 +36,7 @@ CREATE TABLE #dbccTraceTable ( INSERT INTO #dbccTraceTable exec('dbcc tracestatus()'); -SELECT @PKEY as PKEY, a.* from #dbccTraceTable a; +SELECT @PKEY as PKEY, a.*, @DMA_SOURCE_ID as dma_source_id from #dbccTraceTable a; IF OBJECT_ID('tempdb..#dbccTraceTable') IS NOT NULL DROP TABLE #dbccTraceTable; \ No newline at end of file diff --git a/scripts/collector/sqlserver/sql/diskVolumeInfo.sql b/scripts/collector/sqlserver/sql/diskVolumeInfo.sql index beca6fa2..53d904b0 100644 --- a/scripts/collector/sqlserver/sql/diskVolumeInfo.sql +++ b/scripts/collector/sqlserver/sql/diskVolumeInfo.sql @@ -17,13 +17,17 @@ limitations under the License. SET NOCOUNT ON; SET LANGUAGE us_english; + DECLARE @PKEY AS VARCHAR(256) DECLARE @CLOUDTYPE AS VARCHAR(256) DECLARE @PRODUCT_VERSION AS INTEGER +DECLARE @DMA_SOURCE_ID AS VARCHAR(256) SELECT @PKEY = N'$(pkey)'; +SELECT @DMA_SOURCE_ID = N'$(dmaSourceId)'; SELECT @PRODUCT_VERSION = CONVERT(INTEGER, PARSENAME(CONVERT(nvarchar, SERVERPROPERTY('productversion')), 4)); -SELECT @CLOUDTYPE = 'NONE' +SELECT @CLOUDTYPE = 'NONE'; + IF UPPER(@@VERSION) LIKE '%AZURE%' SELECT @CLOUDTYPE = 'AZURE' @@ -80,7 +84,7 @@ IF @CLOUDTYPE = 'AZURE' '''' as cluster_block_size FROM sum_sizes'); -SELECT @PKEY as PKEY, a.* from #gcpDMADiskVolumeInfo a; +SELECT @PKEY as PKEY, a.*, @DMA_SOURCE_ID as dma_source_id from #gcpDMADiskVolumeInfo a; IF OBJECT_ID('tempdb..#gcpDMADiskVolumeInfo') IS NOT NULL DROP TABLE #gcpDMADiskVolumeInfo; \ No newline at end of file diff --git a/scripts/collector/sqlserver/sql/getDmaSourceId.sql b/scripts/collector/sqlserver/sql/getDmaSourceId.sql new file mode 100644 index 00000000..b6e62ccc --- /dev/null +++ b/scripts/collector/sqlserver/sql/getDmaSourceId.sql @@ -0,0 +1,28 @@ +/* +Copyright 2023 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +*/ + +SET NOCOUNT ON; +SET LANGUAGE us_english; +DECLARE @MACHINE_NAME AS VARCHAR(256) + +IF CHARINDEX('\', @@SERVERNAME)-1 = -1 + SELECT @MACHINE_NAME = UPPER(@@SERVERNAME) +ELSE + SELECT @MACHINE_NAME = UPPER(SUBSTRING(CONVERT(nvarchar, @@SERVERNAME),1,CHARINDEX('\', CONVERT(nvarchar, @@SERVERNAME))-1)) +SELECT @MACHINE_NAME + '_' + replace(service_broker_guid,'-','') + '_' + COALESCE(CONVERT(nvarchar, SERVERPROPERTY('InstanceName')), 'MSSQLSERVER') +FROM sys.databases +WHERE [name] = N'msdb'; \ No newline at end of file diff --git a/scripts/collector/sqlserver/sql/indexList.sql b/scripts/collector/sqlserver/sql/indexList.sql index c048b9c9..53e0ef49 100644 --- a/scripts/collector/sqlserver/sql/indexList.sql +++ b/scripts/collector/sqlserver/sql/indexList.sql @@ -17,14 +17,18 @@ limitations under the License. SET NOCOUNT ON; SET LANGUAGE us_english; + DECLARE @PKEY AS VARCHAR(256) DECLARE @CLOUDTYPE AS VARCHAR(256) DECLARE @ASSESSMENT_DATABSE_NAME AS VARCHAR(256) DECLARE @PRODUCT_VERSION AS INTEGER DECLARE @validDB AS INTEGER +DECLARE @DMA_SOURCE_ID AS VARCHAR(256) SELECT @PKEY = N'$(pkey)'; SELECT @ASSESSMENT_DATABSE_NAME = N'$(database)'; +SELECT @DMA_SOURCE_ID = N'$(dmaSourceId)'; + IF @ASSESSMENT_DATABSE_NAME = 'all' SELECT @ASSESSMENT_DATABSE_NAME = '%' SELECT @PRODUCT_VERSION = CONVERT(INTEGER, PARSENAME(CONVERT(nvarchar, SERVERPROPERTY('productversion')), 4)); @@ -131,7 +135,7 @@ BEGIN END -SELECT @PKEY as PKEY, a.* from #indexList a; +SELECT @PKEY as PKEY, a.*, @DMA_SOURCE_ID as dma_source_id from #indexList a; IF OBJECT_ID('tempdb..#indexList') IS NOT NULL DROP TABLE #indexList; \ No newline at end of file diff --git a/scripts/collector/sqlserver/sql/linkedServers.sql b/scripts/collector/sqlserver/sql/linkedServers.sql index 51cf7fcb..3995192b 100644 --- a/scripts/collector/sqlserver/sql/linkedServers.sql +++ b/scripts/collector/sqlserver/sql/linkedServers.sql @@ -17,11 +17,14 @@ limitations under the License. --Linked Servers SET NOCOUNT ON; SET LANGUAGE us_english; + DECLARE @PKEY AS VARCHAR(256) DECLARE @CLOUDTYPE AS VARCHAR(256) DECLARE @PRODUCT_VERSION AS INTEGER +DECLARE @DMA_SOURCE_ID AS VARCHAR(256) SELECT @PKEY = N'$(pkey)'; +SELECT @DMA_SOURCE_ID = N'$(dmaSourceId)'; SELECT @PRODUCT_VERSION = CONVERT(INTEGER, PARSENAME(CONVERT(nvarchar, SERVERPROPERTY('productversion')), 4)); SELECT @CLOUDTYPE = 'NONE' IF UPPER(@@VERSION) LIKE '%AZURE%' @@ -45,7 +48,7 @@ IF @CLOUDTYPE = 'NONE' where is_linked = 1 GROUP BY product; -SELECT @PKEY as PKEY, a.* from #LinkedServersData a; +SELECT @PKEY as PKEY, a.*, @DMA_SOURCE_ID as DMA_SOURCE_ID from #LinkedServersData a; IF OBJECT_ID('tempdb..#LinkedServersData') IS NOT NULL DROP TABLE #LinkedServersData; diff --git a/scripts/collector/sqlserver/sql/objectList.sql b/scripts/collector/sqlserver/sql/objectList.sql index df6c1846..548eaecc 100644 --- a/scripts/collector/sqlserver/sql/objectList.sql +++ b/scripts/collector/sqlserver/sql/objectList.sql @@ -17,14 +17,18 @@ limitations under the License. SET NOCOUNT ON; SET LANGUAGE us_english; + DECLARE @PKEY AS VARCHAR(256) DECLARE @CLOUDTYPE AS VARCHAR(256) DECLARE @ASSESSMENT_DATABSE_NAME AS VARCHAR(256) DECLARE @PRODUCT_VERSION AS INTEGER DECLARE @validDB AS INTEGER +DECLARE @DMA_SOURCE_ID AS VARCHAR(256) SELECT @PKEY = N'$(pkey)'; SELECT @ASSESSMENT_DATABSE_NAME = N'$(database)'; +SELECT @DMA_SOURCE_ID = N'$(dmaSourceId)'; + IF @ASSESSMENT_DATABSE_NAME = 'all' SELECT @ASSESSMENT_DATABSE_NAME = '%' SELECT @PRODUCT_VERSION = CONVERT(INTEGER, PARSENAME(CONVERT(nvarchar, SERVERPROPERTY('productversion')), 4)); @@ -242,7 +246,7 @@ BEGIN END -SELECT @PKEY as PKEY, a.* from #objectList a; +SELECT @PKEY as PKEY, a.*, @DMA_SOURCE_ID as dma_source_id from #objectList a; IF OBJECT_ID('tempdb..#objectList') IS NOT NULL DROP TABLE #objectList; \ No newline at end of file diff --git a/scripts/collector/sqlserver/sql/serverProperties.sql b/scripts/collector/sqlserver/sql/serverProperties.sql index f1894239..352e3cef 100644 --- a/scripts/collector/sqlserver/sql/serverProperties.sql +++ b/scripts/collector/sqlserver/sql/serverProperties.sql @@ -23,8 +23,10 @@ DECLARE @CLOUDTYPE AS VARCHAR(256) DECLARE @PRODUCT_VERSION AS INTEGER DECLARE @TABLE_PERMISSION_COUNT AS INTEGER DECLARE @MACHINENAME AS VARCHAR(256) +DECLARE @DMA_SOURCE_ID AS VARCHAR(256) SELECT @PKEY = N'$(pkey)'; +SELECT @DMA_SOURCE_ID = N'$(dmaSourceId)'; SELECT @PRODUCT_VERSION = CONVERT(INTEGER, PARSENAME(CONVERT(nvarchar, SERVERPROPERTY('productversion')), 4)); SELECT @CLOUDTYPE = 'NONE' IF UPPER(@@VERSION) LIKE '%AZURE%' @@ -437,7 +439,7 @@ BEGIN exec('INSERT INTO #serverProperties VALUES (''MaintenancePlansEnabled'', CONVERT(varchar,0))'); END; -SELECT @PKEY as PKEY, a.* FROM #serverProperties a; +SELECT @PKEY as PKEY, a.*, @DMA_SOURCE_ID as dma_source_id FROM #serverProperties a; IF OBJECT_ID('tempdb..#serverProperties') IS NOT NULL DROP TABLE #serverProperties; diff --git a/scripts/collector/sqlserver/sql/tableList.sql b/scripts/collector/sqlserver/sql/tableList.sql index 5abd108c..a21dba29 100644 --- a/scripts/collector/sqlserver/sql/tableList.sql +++ b/scripts/collector/sqlserver/sql/tableList.sql @@ -17,14 +17,18 @@ limitations under the License. SET NOCOUNT ON; SET LANGUAGE us_english; + DECLARE @PKEY AS VARCHAR(256) DECLARE @CLOUDTYPE AS VARCHAR(256) DECLARE @ASSESSMENT_DATABSE_NAME AS VARCHAR(256) DECLARE @PRODUCT_VERSION AS INTEGER DECLARE @validDB AS INTEGER +DECLARE @DMA_SOURCE_ID AS VARCHAR(256) SELECT @PKEY = N'$(pkey)'; SELECT @ASSESSMENT_DATABSE_NAME = N'$(database)'; +SELECT @DMA_SOURCE_ID = N'$(dmaSourceId)'; + IF @ASSESSMENT_DATABSE_NAME = 'all' SELECT @ASSESSMENT_DATABSE_NAME = '%' SELECT @PRODUCT_VERSION = CONVERT(INTEGER, PARSENAME(CONVERT(nvarchar, SERVERPROPERTY('productversion')), 4)); @@ -271,7 +275,7 @@ BEGIN END -SELECT @PKEY as PKEY, a.* from #tableList a; +SELECT @PKEY as PKEY, a.*, @DMA_SOURCE_ID as dma_source_id from #tableList a; IF OBJECT_ID('tempdb..#tableList') IS NOT NULL DROP TABLE #tableList; \ No newline at end of file diff --git a/scripts/collector/sqlserver/sql/userConnectionInfo.sql b/scripts/collector/sqlserver/sql/userConnectionInfo.sql index 90db85c1..9bba1284 100644 --- a/scripts/collector/sqlserver/sql/userConnectionInfo.sql +++ b/scripts/collector/sqlserver/sql/userConnectionInfo.sql @@ -17,14 +17,18 @@ limitations under the License. SET NOCOUNT ON; SET LANGUAGE us_english; + DECLARE @PKEY AS VARCHAR(256) DECLARE @CLOUDTYPE AS VARCHAR(256) DECLARE @ASSESSMENT_DATABSE_NAME AS VARCHAR(256) DECLARE @PRODUCT_VERSION AS INTEGER DECLARE @validDB AS INTEGER +DECLARE @DMA_SOURCE_ID AS VARCHAR(256) SELECT @PKEY = N'$(pkey)'; SELECT @ASSESSMENT_DATABSE_NAME = N'$(database)'; +SELECT @DMA_SOURCE_ID = N'$(dmaSourceId)'; + IF @ASSESSMENT_DATABSE_NAME = 'all' SELECT @ASSESSMENT_DATABSE_NAME = '%' SELECT @PRODUCT_VERSION = CONVERT(INTEGER, PARSENAME(CONVERT(nvarchar, SERVERPROPERTY('productversion')), 4)); @@ -134,7 +138,7 @@ BEGIN END -SELECT @PKEY as PKEY, a.* from #connectionInfo a; +SELECT @PKEY as PKEY, a.*, @DMA_SOURCE_ID as dma_source_id from #connectionInfo a; IF OBJECT_ID('tempdb..#connectionInfo') IS NOT NULL DROP TABLE #connectionInfo; \ No newline at end of file diff --git a/scripts/masker/dma-collection-masker b/scripts/masker/dma-collection-masker index 1edae126..bbd23a19 100755 --- a/scripts/masker/dma-collection-masker +++ b/scripts/masker/dma-collection-masker @@ -32,7 +32,9 @@ __all__ = [ "main", "run_masker", ] -__version__ = "4.3.5" + +__version__ = "4.3.15" + logger = logging.getLogger(__name__) here = Path(__file__).parent @@ -60,6 +62,7 @@ DATA_MASK_CONFIG = [ DataMaskRule(mask_type="Schema Name", table_name="triggers", column_position=2, missing_ok=True), DataMaskRule(mask_type="Schema Name", table_name="usedspacedetails", column_position=2, missing_ok=True), DataMaskRule(mask_type="Schema Name", table_name="usrsegatt", column_position=2, missing_ok=True), + DataMaskRule(mask_type="Schema Name", table_name="users", column_position=2, missing_ok=True), ]