feat: dma_source_id addition. New Oracle extract & additional MSSQL…

… adjustments. (#291)

* Add extract for user names.wq

* Add UNIQ_ID column, add username extract.

* Version bump

* Add hash to manifest file.

* Rename v_uniq_id to DMA_SOURCE_ID.

* Standardize column name used as source for v_dma_source_id.

* Modify column name to provent NOPRITN issue.

* Sqlserver 4.3.15 feature (#290)

* fix: folder name fixes & minimum privs for SQL Server (#289)

* add additional permissions for sql 2022

* add os and server memory usage

* Add Azure fixes for sys_dm_os_process_memory

* Fix FolderName errors

* foldername fix

* further foldername refinements

* Strip Slash from foldername

* Update for proper machine name

* Update docs for new 2022 permissions

* Remove csv from sqlserver build process

* Bump Version

---------

Co-authored-by: Shane Borden <[email protected]>

* Initial changes for dma_source_id

* add dmaSourceId to perfmon collector

* Update format for dma_source_id

---------

Co-authored-by: Shane Borden <[email protected]>

* add dma_source_id to linked server query

* Sync version# prior to running bump2version.

* Set version number with bump2version.

---------

Co-authored-by: Shane Borden <[email protected]>
Co-authored-by: Shane Borden <[email protected]>

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.3.14
+current_version = 4.3.15
 commit = False
 tag = False
 

scripts/collector/oracle/collect-data.sh

@@ -16,7 +16,7 @@
 
 ### Setup directories needed for execution
 #############################################################################
-OpVersion="4.3.14"
+OpVersion="4.3.15"
 
 LOCALE=$(echo $LANG | cut -d '.' -f 1)
 export LANG=C
@@ -85,10 +85,12 @@ fi
 
 ${SQLPLUS} -s /nolog << EOF
 connect ${connectString}
-@${SQL_DIR}/op_set_sql_env.sql 
+@${SQL_DIR}/op_set_sql_env.sql
 set pagesize 0 lines 400 feedback off verify off heading off echo off timing off time off
+column vname new_value v_name noprint
+select min(object_name) as vname from dba_objects where object_name in ('V\$INSTANCE', 'GV\$INSTANCE');
 select 'DMAFILETAG~'|| i.version||'|'||substr(replace(i.version,'.',''),0,3)||'_'||'${OpVersion}_'||i.host_name||'_'||d.name||'_'||i.instance_name||'_'||to_char(sysdate,'MMDDRRHH24MISS')||'~'
-from v\$instance i, v\$database d;
+from ( SELECT version, host_name, instance_name FROM &&v_name WHERE instance_number = (SELECT min(instance_number) FROM &&v_name) ) i, v\$database d;
 exit;
 EOF
 }
@@ -157,7 +159,7 @@ function compressOpFiles  {
 V_FILE_TAG=$1
 V_ERR_TAG=""
 echo ""
-echo "Archiving output files"
+echo "Archiving output files with tag ${V_FILE_TAG}"
 CURRENT_WORKING_DIR=$(pwd)
 cp ${LOG_DIR}/opdb__${V_FILE_TAG}_errors.log ${OUTPUT_DIR}/opdb__${V_FILE_TAG}_errors.log
 if [ -f VERSION.txt ]; then
@@ -184,7 +186,17 @@ locale > ${OUTPUT_DIR}/opdb__${V_FILE_TAG}_locale.txt
 echo "ZIPFILE: " $ZIPFILE >> ${OUTPUT_DIR}/opdb__defines__${V_FILE_TAG}.csv
 
 cd ${OUTPUT_DIR}
-ls -1  opdb*${V_FILE_TAG}.csv opdb*${V_FILE_TAG}*.log opdb*${V_FILE_TAG}*.txt > opdb__manifest__${V_FILE_TAG}.txt	
+if [ -f opdb__manifest__${V_FILE_TAG}.txt ];
+then
+  rm opdb__manifest__${V_FILE_TAG}.txt
+fi
+
+for file in $(ls -1  opdb*${V_FILE_TAG}.csv opdb*${V_FILE_TAG}*.log opdb*${V_FILE_TAG}*.txt)
+do
+ MD5=$(md5sum $file | cut -d ' ' -f 1)
+ echo "${DBTYPE}|${MD5}|${file}"  >> opdb__manifest__${V_FILE_TAG}.txt
+done
+
 if [ ! "${ZIP}" = "" ]
 then
   $ZIP $ZIPFILE  opdb*${V_FILE_TAG}.csv opdb*${V_FILE_TAG}*.log opdb*${V_FILE_TAG}*.txt

scripts/collector/oracle/sql/extracts/archlogs.sql

@@ -25,7 +25,8 @@ SELECT '&&v_host'
        thread# AS thread_num, 
        dest_id, 
        count(1) AS CNT, 
-       round(sum(blocks * block_size)/1024/1024) as mbytes
+       round(sum(blocks * block_size)/1024/1024) as mbytes,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM gv$archived_log
 WHERE first_time >= trunc(sysdate) - '&&dtrange'
 GROUP BY trunc(first_time), thread#, to_char(first_time, 'HH24'), dest_id

scripts/collector/oracle/sql/extracts/awr/awrhistcmdtypes.sql

@@ -60,6 +60,7 @@ GROUP  BY '&&v_host'
           b.command_type, aa.name)
 SELECT pkey , con_id , hh24 , command_type , cnt , avg_buffer_gets , avg_elasped_time ,
        avg_rows_processed , avg_executions , avg_cpu_time , avg_iowait , avg_clwait ,
-       avg_apwait , avg_ccwait , avg_plsexec_time, command_name
+       avg_apwait , avg_ccwait , avg_plsexec_time, command_name,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vcmdtype;
 spool off

scripts/collector/oracle/sql/extracts/awr/awrhistosstat.sql

@@ -102,6 +102,7 @@ GROUP  BY '&&v_host'
           stat_name)
 SELECT pkey , dbid , instance_number , hh24 , stat_name , hh24_total_secs ,
        cumulative_value , avg_value , mode_value , median_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100 ,
-	     min_value , max_value , sum_value , count
+	     min_value , max_value , sum_value , count,
+	       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vossummary;
 spool off

scripts/collector/oracle/sql/extracts/awr/awrhistsysmetrichist.sql

@@ -65,6 +65,7 @@ ORDER  BY hsm.dbid,
           TO_CHAR(hsm.begin_time, 'hh24'))
 SELECT pkey , dbid , instance_number , hour , metric_name ,
        metric_unit , avg_value , mode_value , median_value , min_value , max_value ,
-	   sum_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100
+	   sum_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100,
+	       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vsysmetric;
 spool off

scripts/collector/oracle/sql/extracts/awr/awrhistsysmetricsumm.sql

@@ -81,6 +81,7 @@ vsysmetricsummperhour as (
 )
 SELECT pkey , dbid , instance_number , hour , metric_name ,
        metric_unit , avg_value , mode_value , median_value , min_value , max_value ,
-	   sum_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100
+	   sum_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100,
+	       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vsysmetricsummperhour;
 spool off

scripts/collector/oracle/sql/extracts/awr/awrsnapdetails.sql

@@ -55,6 +55,7 @@ WHERE startup_time = lag_startup_time
 GROUP BY '&&v_host' || '_' || '&&v_dbname' || '_' || '&&v_hora', dbid, instance_number, hour)
 SELECT pkey , dbid , instance_number , hour , min_snap_id , max_snap_id , min_begin_interval_time ,
        max_begin_interval_time , cnt , sum_snaps_diff_secs , avg_snaps_diff_secs , median_snaps_diff_secs ,
-       mode_snaps_diff_secs , min_snaps_diff_secs , max_snaps_diff_secs
+       mode_snaps_diff_secs , min_snaps_diff_secs , max_snaps_diff_secs,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vawrsnap;
 spool off

scripts/collector/oracle/sql/extracts/awr/dbahistsysstat.sql

@@ -93,6 +93,7 @@ GROUP BY
           stat_name)
 SELECT pkey , dbid , instance_number , hour , stat_name , cnt ,
        avg_value , mode_value , median_value , min_value , max_value ,
-	   sum_value , perc50 , perc75 , perc90 , perc95 , perc100
+	   sum_value , perc50 , perc75 , perc90 , perc95 , perc100,
+	       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vsysstat;
 spool off

scripts/collector/oracle/sql/extracts/awr/dbahistsystimemodel.sql

@@ -71,6 +71,7 @@ GROUP BY
       stat_name)
 SELECT pkey , dbid , instance_number , hour , stat_name , cnt ,
        avg_value , mode_value , median_value , min_value , max_value ,
-	   sum_value , perc50 , perc75 , perc90 , perc95 , perc100
+	   sum_value , perc50 , perc75 , perc90 , perc95 , perc100,
+	       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vtimemodel;
 spool off

scripts/collector/oracle/sql/extracts/awr/ioevents.sql

@@ -97,7 +97,8 @@ SELECT pkey , dbid , instance_number , hour , wait_class , event_name ,
        time_wa_us_delta_value_P95,
        tot_waits_delta_value_P100 ,
        tot_tout_delta_value_P100 ,
-       time_wa_us_delta_value_P100
+       time_wa_us_delta_value_P100,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vfev;
 spool off
 column hour clear

scripts/collector/oracle/sql/extracts/awr/iofunction.sql

@@ -187,7 +187,8 @@ SELECT pkey , dbid , instance_number , hour , function_name ,
        total_reads_mb_P100 ,
        total_reads_req_P100 ,
        total_writes_mb_P100 ,
-       total_write_req_P100
+       total_write_req_P100,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM viof;
 spool off
 column hour clear

scripts/collector/oracle/sql/extracts/awr/sourceconn.sql

@@ -61,7 +61,8 @@ group by '&&v_host'
        &v_machine,
        scmd.command_name)
 SELECT pkey , dbid , instance_number , hour , program ,
-       module , machine , command_name , cnt
+       module , machine , command_name , cnt,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vsrcconn
 order by hour;
 spool off

scripts/collector/oracle/sql/extracts/awr/sqlstats.sql

@@ -71,7 +71,8 @@ SELECT pkey , con_id , dbid , instance_number , force_matching_signature , sql_i
        physical_read_bytes_total , physical_write_bytes_total , io_offload_elig_bytes_total , io_interconnect_bytes_total ,
        optimized_physical_reads_total , cell_uncompressed_bytes_total , io_offload_return_bytes_total , direct_writes_total ,
        perc_exec_finished , avg_rows , avg_disk_reads , avg_buffer_gets , avg_cpu_time_us , avg_elapsed_us , avg_iowait_us ,
-       avg_clwait_us , avg_apwait_us , avg_ccwait_us , avg_plsexec_us , avg_javexec_us
+       avg_clwait_us , avg_apwait_us , avg_ccwait_us , avg_plsexec_us , avg_javexec_us,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vsqlstat
 WHERE rownum < 300;
 spool off

scripts/collector/oracle/sql/extracts/awr/sqlstats111.sql

@@ -72,7 +72,8 @@ SELECT pkey , con_id , dbid , instance_number , force_matching_signature , sql_i
        optimized_physical_reads_total , cell_uncompressed_bytes_total , io_offload_return_bytes_total , 
        direct_writes_total ,
        perc_exec_finished , avg_rows , avg_disk_reads , avg_buffer_gets , avg_cpu_time_us , avg_elapsed_us , avg_iowait_us ,
-       avg_clwait_us , avg_apwait_us , avg_ccwait_us , avg_plsexec_us , avg_javexec_us
+       avg_clwait_us , avg_apwait_us , avg_ccwait_us , avg_plsexec_us , avg_javexec_us,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vsqlstat
 WHERE rownum < 300;
 spool off

scripts/collector/oracle/sql/extracts/backups.sql

@@ -25,7 +25,8 @@ SELECT '&&v_host'
        input_type, 
        round(sum(elapsed_seconds)) AS elapsed_seconds, 
        round(sum(input_bytes)/1024/1024) AS mbytes_in, 
-       round(sum(output_bytes)/1024/1024) AS mbytes_out
+       round(sum(output_bytes)/1024/1024) AS mbytes_out,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM v$rman_backup_job_details a
 WHERE start_time >= trunc(sysdate) - '&&dtrange'
 GROUP BY trunc(start_time), input_type, &v_a_con_id

scripts/collector/oracle/sql/extracts/columntypes.sql

@@ -208,8 +208,9 @@ SELECT
     c.VARCHAR2_COL_COUNT                         ,
     c.XMLTYPE_COL_COUNT                          ,
     c.UNDEFINED_COL_COUNT                        ,
-    c.USER_DEFINED_COL_COUNT
-    , NVL(s.bytes,0) as bytes
+    c.USER_DEFINED_COL_COUNT                     ,
+    NVL(s.bytes,0) as bytes                      ,
+    '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM  coltypes c 
 LEFT JOIN segs s ON c.con_id = s.con_id and s.owner = c.owner and s.segment_name = c.table_name 
 ORDER BY 1,2,3,4

scripts/collector/oracle/sql/extracts/compressbytype.sql

@@ -92,7 +92,8 @@ WITH vcompresstype AS (
      --HAVING TRUNC(SUM(gbytes)) > 0
      )
 SELECT pkey , con_id , owner , basic , oltp , query_low ,
-       query_high , archive_low , archive_high , total_gb
+       query_high , archive_low , archive_high , total_gb,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vcompresstype
 ORDER BY total_gb DESC;
 spool off

scripts/collector/oracle/sql/extracts/cpucoresusage.sql

@@ -27,6 +27,7 @@ SELECT '&&v_host'
        cpu_socket_count
 FROM   dba_cpu_usage_statistics
 ORDER  BY timestamp)
-SELECT pkey , dt , cpu_count , cpu_core_count , cpu_socket_count
+SELECT pkey , dt , cpu_count , cpu_core_count , cpu_socket_count,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vcpursc;
 spool off

scripts/collector/oracle/sql/extracts/dataguard.sql

@@ -53,7 +53,8 @@ FROM gv$archive_dest a
 WHERE destination IS NOT NULL)
 SELECT pkey , con_id , inst_id , log_archive_config , dest_id , dest_name , destination , status ,
        target , schedule , register , alternate ,
-       transmit_mode , affirm , valid_role , verify
+       transmit_mode , affirm , valid_role , verify,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vodg;
 spool off
 

scripts/collector/oracle/sql/extracts/datatypes.sql

@@ -47,6 +47,7 @@ GROUP  BY '&&v_host'
           avg_col_len
 )
 SELECT pkey , con_id , owner , data_type , cnt,
-       data_length, data_precision, data_scale, avg_col_len, distinct_table_count
+       data_length, data_precision, data_scale, avg_col_len, distinct_table_count,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vdtype;
 spool off

scripts/collector/oracle/sql/extracts/dbfeatures.sql

@@ -33,6 +33,7 @@ FROM   &v_tblprefix._feature_usage_statistics a
 WHERE dbid = &&v_dbid
 ORDER  BY name)
 SELECT pkey , con_id , name , currently_used , detected_usages ,
-       total_samples , first_usage , last_usage , aux_count
+       total_samples , first_usage , last_usage , aux_count,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vdbf;
 spool off

scripts/collector/oracle/sql/extracts/dbhwmarkstatistics.sql

@@ -27,6 +27,7 @@ SELECT '&&v_host'
        &v_a_con_id AS con_id
 FROM   &v_tblprefix._high_water_mark_statistics a
 ORDER  BY description)
-SELECT pkey , description , highwater , last_value, con_id
+SELECT pkey , description , highwater , last_value, con_id,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vhwmst;
 spool off

scripts/collector/oracle/sql/extracts/dbinstances.sql

@@ -30,6 +30,7 @@ SELECT '&&v_host'
        instance_role
 FROM   gv$instance )
 SELECT pkey , inst_id , instance_name , host_name ,
-       version , status , database_status , instance_role
+       version , status , database_status , instance_role,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vdbinst;
 spool off

scripts/collector/oracle/sql/extracts/dblinks.sql

@@ -32,6 +32,7 @@ GROUP BY '&&v_host'
        || '_'
        || '&&v_hora',
        &v_a_con_id , owner)
-SELECT pkey , con_id , owner , count
+SELECT pkey , con_id , owner , count,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vdbl;
 spool off

scripts/collector/oracle/sql/extracts/dbobjectnames.sql

@@ -76,7 +76,8 @@ SELECT pkey ,
        object_type , 
        editionable ,
        lines,
-       status
+       status,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vdbobj a;
 spool off
 

scripts/collector/oracle/sql/extracts/dbobjects.sql

@@ -59,7 +59,8 @@ SELECT pkey ,
        owner , 
        object_type , 
        editionable ,
-       count  
+       count  ,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vdbobj a;
 spool off
 

scripts/collector/oracle/sql/extracts/dbparameters.sql

@@ -30,7 +30,8 @@ SELECT '&&v_host'
        isdefault
 FROM   gv$system_parameter a
 ORDER  BY 2,3 )
-SELECT pkey , inst_id , con_id , name , value , default_value , isdefault
+SELECT pkey , inst_id , con_id , name , value , default_value , isdefault,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vparam;
 spool off
 column default_value clear

scripts/collector/oracle/sql/extracts/dbsummary.sql

@@ -112,6 +112,6 @@ SELECT pkey , dbid , db_name , cdb , db_version , db_fullversion , log_mode , fo
 	   db_long_size_gb , dg_database_role , dg_protection_mode , dg_protection_level, 
            db_size_temp_allocated_gb, db_size_redo_allocated_gb,
            ebs_owner, siebel_owner, psft_owner, rds_flag, oci_autonomous_flag, dbms_cloud_pkg_installed,
-           apex_installed, sap_owner, db_unique_name
+           apex_installed, sap_owner, db_unique_name, '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vdbsummary;
 spool off

scripts/collector/oracle/sql/extracts/defines.sql

@@ -38,8 +38,10 @@ prompt v_io_function_sql  = &v_io_function_sql
 prompt v_is_container     = &v_is_container 
 prompt v_max_snapid       = &v_max_snapid 
 prompt v_min_snapid       = &v_min_snapid 
-prompt v_max_snaptime     = &v_max_snaptime
-prompt v_min_snaptime     = &v_min_snaptime
 prompt v_tblprefix        = &v_tblprefix 
 prompt v_umf_test         = &v_umf_test 
+prompt Applies to STATSPACK collections only: 
+prompt v_max_snaptime     = &v_max_snaptime
+prompt v_min_snaptime     = &v_min_snaptime
+prompt v_dma_source_id          = &v_dma_source_id
 spool off

scripts/collector/oracle/sql/extracts/dtlsourcecode.sql

@@ -74,6 +74,7 @@ GROUP  BY pkey,
           name,
           TYPE)
 SELECT pkey , con_id , owner, name , type , sum_nr_lines , qt_objs ,
-       sum_nr_lines_w_utl , sum_nr_lines_w_dbms , count_exec_im , count_dbms_sql , sum_nr_lines_w_dbms_utl , sum_count_total
+       sum_nr_lines_w_utl , sum_nr_lines_w_dbms , count_exec_im , count_dbms_sql , sum_nr_lines_w_dbms_utl , sum_count_total,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vsrc;
 spool off

scripts/collector/oracle/sql/extracts/exttab.sql

@@ -24,6 +24,7 @@ SELECT '&&v_host'
        &v_a_con_id as con_id, owner, table_name, type_owner, type_name, default_directory_owner, default_directory_name
 FROM &v_tblprefix._external_tables a)
 SELECT pkey , con_id , owner , table_name , type_owner , type_name ,
-       default_directory_owner , default_directory_name
+       default_directory_owner , default_directory_name,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vexttab;
 spool off

scripts/collector/oracle/sql/extracts/idxpertable.sql

@@ -34,6 +34,7 @@ SELECT pkey,
        round(100*ratio_to_report(count(table_name)) over ()) idx_perc
 FROM vrawidx
 GROUP BY pkey, con_id, idx_cnt)
-SELECT pkey , con_id , tab_count , idx_cnt , idx_perc
+SELECT pkey , con_id , tab_count , idx_cnt , idx_perc,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vcidx;
 spool off

scripts/collector/oracle/sql/extracts/indextypedtl.sql

@@ -59,7 +59,8 @@ SELECT pkey ,
        join_index,
        custom_index_type,
        table_name,
-       index_name
+       index_name,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vidxtype;
 spool off
 COLUMN PARTITIONED CLEAR

scripts/collector/oracle/sql/extracts/indextypes.sql

@@ -73,7 +73,8 @@ SELECT pkey ,
        visibility,                      
        join_index,
        custom_index_type,
-       cnt
+       cnt,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM vidxtype;
 spool off
 COLUMN PARTITIONED CLEAR

scripts/collector/oracle/sql/extracts/lobsizing.sql

@@ -166,7 +166,8 @@ SELECT  '&&v_host'
        seg_bytes,
        table_avg_lob_bytes,
        partition_avg_lob_bytes,
-       subpartition_avg_lob_bytes
+       subpartition_avg_lob_bytes,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM lobdata;
 spool off
 

scripts/collector/oracle/sql/extracts/mviewtypes.sql

@@ -48,7 +48,8 @@ SELECT '&&v_host'
        refresh_mode,
        refresh_method,
        fast_refreshable,
-       compile_state
+       compile_state,
+       '&v_dma_source_id' AS DMA_SOURCE_ID
 FROM  mvinfo;
 spool off
 COLUMN UPDATABLE CLEAR