From ae03713cee04a9cf221dcd62dcb382f6170dd73e Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 9 Aug 2023 15:31:11 +0530 Subject: [PATCH 01/40] feat: update ONTAP metric document Fixes #2008 --- cmd/tools/generate/counter.go | 26 +- cmd/tools/generate/counter.yaml | 12 + conf/zapiperf/cdot/9.8.0/fcvi.yaml | 2 +- docs/ontap-metrics.md | 572 +++++++++++++++++++++++++++-- 4 files changed, 571 insertions(+), 41 deletions(-) diff --git a/cmd/tools/generate/counter.go b/cmd/tools/generate/counter.go index e330e8adc..a10dab613 100644 --- a/cmd/tools/generate/counter.go +++ b/cmd/tools/generate/counter.go @@ -43,6 +43,21 @@ var ( "read_latency_hist": {}, "write_latency_hist": {}, } + // Excludes these Rest gaps from logs + excludeLogRestCounters = []string{ + "smb2_", + "ontaps3_svm_", + "nvmf_rdma_port_", + "nvmf_tcp_port_", + "netstat_", + "external_service_op_", + "fabricpool_average_latency", + "fabricpool_get_throughput_bytes", + "fabricpool_put_throughput_bytes", + "fabricpool_stats", + "fabricpool_throughput_ops", + "iw_", + } ) type Counters struct { @@ -509,8 +524,17 @@ func generateCounterTemplate(counters map[string]Counter, client *rest.Client) { // Print such counters which are missing Rest mapping if len(counter.APIs) == 1 { if counter.APIs[0].API == "ZAPI" { + isPrint := true + for _, substring := range excludeLogRestCounters { + if strings.HasPrefix(counter.Name, substring) { + isPrint = false + break + } + } //missing Rest Mapping - fmt.Printf("Missing %s mapping for %v \n", "REST", counter) + if isPrint { + fmt.Printf("Missing %s mapping for %v \n", "REST", counter) + } } } diff --git a/cmd/tools/generate/counter.yaml b/cmd/tools/generate/counter.yaml index 2af11a9e4..1be133f45 100644 --- a/cmd/tools/generate/counter.yaml +++ b/cmd/tools/generate/counter.yaml @@ -241,6 +241,18 @@ counters: ONTAPCounter: Harvest generated Template: conf/zapi/cdot/9.8.0/sensor.yaml + - Name: fabricpool_average_latency + Description: This counter is deprecated.Average latencies executed during various phases of command execution. The execution-start latency represents the average time taken to start executing a operation. The request-prepare latency represent the average time taken to prepare the commplete request that needs to be sent to the server. The send latency represents the average time taken to send requests to the server. The execution-start-to-send-complete represents the average time taken to send a operation out since its execution started. The execution-start-to-first-byte-received represent the average time taken to to receive the first byte of a response since the command's request execution started. These counters can be used to identify performance bottlenecks within the object store client module. + + - Name: fabricpool_get_throughput_bytes + Description: This counter is deprecated. Counter that indicates the throughput for GET command in bytes per second. + + - Name: fabricpool_put_throughput_bytes + Description: This counter is deprecated. Counter that indicates the throughput for PUT command in bytes per second. + + - Name: fabricpool_stats + Description: This counter is deprecated. Counter that indicates the number of object store operations sent, and their success and failure counts. The objstore_client_op_name array indicate the operation name such as PUT, GET, etc. The objstore_client_op_stats_name array contain the total number of operations, their success and failure counter for each operation. + - Name: metadata_collector_api_time Description: amount of time to collect data from monitored cluster object APIs: diff --git a/conf/zapiperf/cdot/9.8.0/fcvi.yaml b/conf/zapiperf/cdot/9.8.0/fcvi.yaml index db9c97b19..c2def7e60 100644 --- a/conf/zapiperf/cdot/9.8.0/fcvi.yaml +++ b/conf/zapiperf/cdot/9.8.0/fcvi.yaml @@ -6,7 +6,7 @@ object: fcvi instance_key: uuid counters: - - fw_SyStatDiscardFrames => firmwares_systat_discard_frames + - fw_SyStatDiscardFrames => firmware_systat_discard_frames - fw_invalid_crc => firmware_invalid_crc_count - fw_invalid_xmit_words => firmware_invalid_transmit_word_count - fw_link_failure => firmware_link_failure_count diff --git a/docs/ontap-metrics.md b/docs/ontap-metrics.md index bc98ec99b..61ee2b149 100644 --- a/docs/ontap-metrics.md +++ b/docs/ontap-metrics.md @@ -7,8 +7,8 @@ These can be generated on demand by running `bin/harvest grafana metrics`. See - More information about ONTAP REST performance counters can be found [here](https://docs.netapp.com/us-en/ontap-pcmap-9121/index.html). ``` -Creation Date : 2023-May-03 -ONTAP Version: 9.12.1 +Creation Date : 2023-Aug-09 +ONTAP Version: 9.13.1 ``` ## Understanding the structure @@ -921,11 +921,11 @@ This histogram holds the latency values for requests of this operation to the sp ### fabricpool_average_latency -Note This counter is deprecated and will be removed in a future release. Average latencies executed during various phases of command execution. The execution-start latency represents the average time taken to start executing a operation. The request-prepare latency represent the average time taken to prepare the commplete request that needs to be sent to the server. The send latency represents the average time taken to send requests to the server. The execution-start-to-send-complete represents the average time taken to send a operation out since its execution started. The execution-start-to-first-byte-received represent the average time taken to to receive the first byte of a response since the command's request execution started. These counters can be used to identify performance bottlenecks within the object store client module. +This counter is deprecated.Average latencies executed during various phases of command execution. The execution-start latency represents the average time taken to start executing a operation. The request-prepare latency represent the average time taken to prepare the commplete request that needs to be sent to the server. The send latency represents the average time taken to send requests to the server. The execution-start-to-send-complete represents the average time taken to send a operation out since its execution started. The execution-start-to-first-byte-received represent the average time taken to to receive the first byte of a response since the command's request execution started. These counters can be used to identify performance bottlenecks within the object store client module. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| ZAPI | `perf-object-get-instances object_store_client_op` | `average_latency`
Unit: microsec
Type: average,no-zero-values
Base: ops | conf/zapiperf/cdot/9.8.0/object_store_client_op.yaml | +| ZAPI | `perf-object-get-instances object_store_client_op` | `average_latency`
Unit:
Type:
Base: | conf/zapiperf/cdot/9.8.0/object_store_client_op.yaml | ### fabricpool_cloud_bin_op_latency_average @@ -950,29 +950,29 @@ Cloud bin operation counters. ### fabricpool_get_throughput_bytes -Note This counter is deprecated and will be removed in a future release. Counter that indicates the throughput for GET command in bytes per second. +This counter is deprecated. Counter that indicates the throughput for GET command in bytes per second. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| ZAPI | `perf-object-get-instances object_store_client_op` | `get_throughput_bytes`
Unit: b_per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/object_store_client_op.yaml | +| ZAPI | `perf-object-get-instances object_store_client_op` | `get_throughput_bytes`
Unit:
Type:
Base: | conf/zapiperf/cdot/9.8.0/object_store_client_op.yaml | ### fabricpool_put_throughput_bytes -Note This counter is deprecated and will be removed in a future release. Counter that indicates the throughput for PUT command in bytes per second. +This counter is deprecated. Counter that indicates the throughput for PUT command in bytes per second. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| ZAPI | `perf-object-get-instances object_store_client_op` | `put_throughput_bytes`
Unit: b_per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/object_store_client_op.yaml | +| ZAPI | `perf-object-get-instances object_store_client_op` | `put_throughput_bytes`
Unit:
Type:
Base: | conf/zapiperf/cdot/9.8.0/object_store_client_op.yaml | ### fabricpool_stats -Note This counter is deprecated and will be removed in a future release. Counter that indicates the number of object store operations sent, and their success and failure counts. The objstore_client_op_name array indicate the operation name such as PUT, GET, etc. The objstore_client_op_stats_name array contain the total number of operations, their success and failure counter for each operation. +This counter is deprecated. Counter that indicates the number of object store operations sent, and their success and failure counts. The objstore_client_op_name array indicate the operation name such as PUT, GET, etc. The objstore_client_op_stats_name array contain the total number of operations, their success and failure counter for each operation. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| ZAPI | `perf-object-get-instances object_store_client_op` | `stats`
Unit: none
Type: delta,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/object_store_client_op.yaml | +| ZAPI | `perf-object-get-instances object_store_client_op` | `stats`
Unit:
Type:
Base: | conf/zapiperf/cdot/9.8.0/object_store_client_op.yaml | ### fabricpool_throughput_ops @@ -1584,6 +1584,76 @@ Number of write operations | ZAPI | `perf-object-get-instances fcp_port` | `write_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/fcp.yaml | +### fcvi_firmware_invalid_crc_count + +Firmware reported invalid CRC count + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/fcvi` | `firmware.invalid_crc_count`
Unit: none
Type: delta
Base: | conf/restperf/9.12.0/fcvi.yaml | +| ZAPI | `perf-object-get-instances fcvi` | `fw_invalid_crc`
Unit: none
Type: delta
Base: | conf/zapiperf/cdot/9.8.0/fcvi.yaml | + + +### fcvi_firmware_invalid_transmit_word_count + +Firmware reported invalid transmit word count + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/fcvi` | `firmware.invalid_transmit_word_count`
Unit: none
Type: delta
Base: | conf/restperf/9.12.0/fcvi.yaml | +| ZAPI | `perf-object-get-instances fcvi` | `fw_invalid_xmit_words`
Unit: none
Type: delta
Base: | conf/zapiperf/cdot/9.8.0/fcvi.yaml | + + +### fcvi_firmware_link_failure_count + +Firmware reported link failure count + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/fcvi` | `firmware.link_failure_count`
Unit: none
Type: delta
Base: | conf/restperf/9.12.0/fcvi.yaml | +| ZAPI | `perf-object-get-instances fcvi` | `fw_link_failure`
Unit: none
Type: delta
Base: | conf/zapiperf/cdot/9.8.0/fcvi.yaml | + + +### fcvi_firmware_loss_of_signal_count + +Firmware reported loss of signal count + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/fcvi` | `firmware.loss_of_signal_count`
Unit: none
Type: delta
Base: | conf/restperf/9.12.0/fcvi.yaml | +| ZAPI | `perf-object-get-instances fcvi` | `fw_loss_of_signal`
Unit: none
Type: delta
Base: | conf/zapiperf/cdot/9.8.0/fcvi.yaml | + + +### fcvi_firmware_loss_of_sync_count + +Firmware reported loss of sync count + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/fcvi` | `firmware.loss_of_sync_count`
Unit: none
Type: delta
Base: | conf/restperf/9.12.0/fcvi.yaml | +| ZAPI | `perf-object-get-instances fcvi` | `fw_loss_of_sync`
Unit: none
Type: delta
Base: | conf/zapiperf/cdot/9.8.0/fcvi.yaml | + + +### fcvi_firmware_systat_discard_frames + +Firmware reported SyStatDiscardFrames value + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/fcvi` | `firmware.systat.discard_frames`
Unit: none
Type: delta
Base: | conf/restperf/9.12.0/fcvi.yaml | +| ZAPI | `perf-object-get-instances fcvi` | `fw_SyStatDiscardFrames`
Unit: none
Type: delta
Base: | conf/zapiperf/cdot/9.8.0/fcvi.yaml | + + +### fcvi_hard_reset_count + +Number of times hard reset of FCVI adapter got issued. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/fcvi` | `hard_reset_count`
Unit: none
Type: delta
Base: | conf/restperf/9.12.0/fcvi.yaml | +| ZAPI | `perf-object-get-instances fcvi` | `hard_reset_cnt`
Unit: none
Type: delta
Base: | conf/zapiperf/cdot/9.8.0/fcvi.yaml | + + ### fcvi_rdma_write_avg_latency Average RDMA write I/O latency. @@ -1614,6 +1684,16 @@ RDMA write throughput in bytes per second. | ZAPI | `perf-object-get-instances fcvi` | `rdma_write_throughput`
Unit: b_per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/fcvi.yaml | +### fcvi_soft_reset_count + +Number of times soft reset of FCVI adapter got issued. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/fcvi` | `soft_reset_count`
Unit: none
Type: delta
Base: | conf/restperf/9.12.0/fcvi.yaml | +| ZAPI | `perf-object-get-instances fcvi` | `soft_reset_cnt`
Unit: none
Type: delta
Base: | conf/zapiperf/cdot/9.8.0/fcvi.yaml | + + ### flashcache_accesses External cache accesses per second @@ -2322,6 +2402,42 @@ Amount of data written to the storage system in bytes | ZAPI | `perf-object-get-instances iscsi_lif` | `write_data`
Unit: b_per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/iscsi_lif.yaml | +### iw_avg_latency + +Average RDMA I/O latency. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances iwarp` | `iw_avg_latency`
Unit: microsec
Type: average
Base: iw_ops | conf/zapiperf/cdot/9.8.0/iwarp.yaml | + + +### iw_ops + +Number of RDMA I/Os issued. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances iwarp` | `iw_ops`
Unit: none
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/iwarp.yaml | + + +### iw_read_ops + +Number of RDMA read I/Os issued. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances iwarp` | `iw_read_ops`
Unit: none
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/iwarp.yaml | + + +### iw_write_ops + +Number of RDMA write I/Os issued. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances iwarp` | `iw_write_ops`
Unit: none
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/iwarp.yaml | + + ### lif_recv_data Number of bytes received per second @@ -5643,6 +5759,204 @@ Number of write operations | ZAPI | `perf-object-get-instances nvmf_fc_lif` | `write_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.10.1/nvmf_lif.yaml | +### nvmf_rdma_port_avg_latency + +Average latency for NVMF operations + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_rdma_port` | `avg_latency`
Unit: microsec
Type: average
Base: total_ops | conf/zapiperf/cdot/9.8.0/nvmf_rdma_port.yaml | + + +### nvmf_rdma_port_avg_other_latency + +Average latency for operations other than read, write, compare or caw + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_rdma_port` | `avg_other_latency`
Unit: microsec
Type: average
Base: other_ops | conf/zapiperf/cdot/9.8.0/nvmf_rdma_port.yaml | + + +### nvmf_rdma_port_avg_read_latency + +Average latency for read operations + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_rdma_port` | `avg_read_latency`
Unit: microsec
Type: average
Base: read_ops | conf/zapiperf/cdot/9.8.0/nvmf_rdma_port.yaml | + + +### nvmf_rdma_port_avg_write_latency + +Average latency for write operations + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_rdma_port` | `avg_write_latency`
Unit: microsec
Type: average
Base: write_ops | conf/zapiperf/cdot/9.8.0/nvmf_rdma_port.yaml | + + +### nvmf_rdma_port_other_ops + +Number of operations that are not read, write, compare or caw. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_rdma_port` | `other_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/nvmf_rdma_port.yaml | + + +### nvmf_rdma_port_read_data + +Amount of data read from the storage system + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_rdma_port` | `read_data`
Unit: b_per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/nvmf_rdma_port.yaml | + + +### nvmf_rdma_port_read_ops + +Number of read operations + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_rdma_port` | `read_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/nvmf_rdma_port.yaml | + + +### nvmf_rdma_port_total_data + +Amount of NVMF traffic to and from the storage system + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_rdma_port` | `total_data`
Unit: b_per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/nvmf_rdma_port.yaml | + + +### nvmf_rdma_port_total_ops + +Total number of operations. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_rdma_port` | `total_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/nvmf_rdma_port.yaml | + + +### nvmf_rdma_port_write_data + +Amount of data written to the storage system + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_rdma_port` | `write_data`
Unit: b_per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/nvmf_rdma_port.yaml | + + +### nvmf_rdma_port_write_ops + +Number of write operations + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_rdma_port` | `write_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/nvmf_rdma_port.yaml | + + +### nvmf_tcp_port_avg_latency + +Average latency for NVMF operations + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_tcp_port` | `avg_latency`
Unit: microsec
Type: average
Base: total_ops | conf/zapiperf/cdot/9.8.0/nvmf_tcp_port.yaml | + + +### nvmf_tcp_port_avg_other_latency + +Average latency for operations other than read, write, compare or caw + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_tcp_port` | `avg_other_latency`
Unit: microsec
Type: average
Base: other_ops | conf/zapiperf/cdot/9.8.0/nvmf_tcp_port.yaml | + + +### nvmf_tcp_port_avg_read_latency + +Average latency for read operations + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_tcp_port` | `avg_read_latency`
Unit: microsec
Type: average
Base: read_ops | conf/zapiperf/cdot/9.8.0/nvmf_tcp_port.yaml | + + +### nvmf_tcp_port_avg_write_latency + +Average latency for write operations + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_tcp_port` | `avg_write_latency`
Unit: microsec
Type: average
Base: write_ops | conf/zapiperf/cdot/9.8.0/nvmf_tcp_port.yaml | + + +### nvmf_tcp_port_other_ops + +Number of operations that are not read, write, compare or caw. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_tcp_port` | `other_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/nvmf_tcp_port.yaml | + + +### nvmf_tcp_port_read_data + +Amount of data read from the storage system + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_tcp_port` | `read_data`
Unit: b_per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/nvmf_tcp_port.yaml | + + +### nvmf_tcp_port_read_ops + +Number of read operations + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_tcp_port` | `read_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/nvmf_tcp_port.yaml | + + +### nvmf_tcp_port_total_data + +Amount of NVMF traffic to and from the storage system + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_tcp_port` | `total_data`
Unit: b_per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/nvmf_tcp_port.yaml | + + +### nvmf_tcp_port_total_ops + +Total number of operations. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_tcp_port` | `total_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/nvmf_tcp_port.yaml | + + +### nvmf_tcp_port_write_data + +Amount of data written to the storage system + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_tcp_port` | `write_data`
Unit: b_per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/nvmf_tcp_port.yaml | + + +### nvmf_tcp_port_write_ops + +Number of write operations + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZAPI | `perf-object-get-instances nvmf_tcp_port` | `write_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/nvmf_tcp_port.yaml | + + ### ontaps3_logical_used_size Specifies the bucket logical used size up to this point. @@ -9492,6 +9806,7 @@ Total number of current active connections | API | Endpoint | Metric | Template | |--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/svm_vscan` | `connections_active`
Unit: none
Type: raw
Base: | conf/restperf/9.13.0/vscan_svm.yaml | | ZAPI | `perf-object-get-instances offbox_vscan` | `connections_active`
Unit: none
Type: raw
Base: | conf/zapiperf/cdot/9.8.0/vscan_svm.yaml | @@ -9501,6 +9816,7 @@ Average dispatch latency | API | Endpoint | Metric | Template | |--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/svm_vscan` | `dispatch.latency`
Unit: microsec
Type: average
Base: dispatch.requests | conf/restperf/9.13.0/vscan_svm.yaml | | ZAPI | `perf-object-get-instances offbox_vscan` | `dispatch_latency`
Unit: microsec
Type: average
Base: dispatch_latency_base | conf/zapiperf/cdot/9.8.0/vscan_svm.yaml | @@ -9510,6 +9826,7 @@ Average scan latency | API | Endpoint | Metric | Template | |--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/svm_vscan` | `scan.latency`
Unit: microsec
Type: average
Base: scan.requests | conf/restperf/9.13.0/vscan_svm.yaml | | ZAPI | `perf-object-get-instances offbox_vscan` | `scan_latency`
Unit: microsec
Type: average
Base: scan_latency_base | conf/zapiperf/cdot/9.8.0/vscan_svm.yaml | @@ -9519,6 +9836,7 @@ Total number of scan notifications received by the dispatcher per second | API | Endpoint | Metric | Template | |--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/svm_vscan` | `scan.notification_received_rate`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.13.0/vscan_svm.yaml | | ZAPI | `perf-object-get-instances offbox_vscan` | `scan_noti_received_rate`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/vscan_svm.yaml | @@ -9528,6 +9846,7 @@ Total number of scan requests sent to the Vscanner per second | API | Endpoint | Metric | Template | |--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/svm_vscan` | `scan.request_dispatched_rate`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.13.0/vscan_svm.yaml | | ZAPI | `perf-object-get-instances offbox_vscan` | `scan_request_dispatched_rate`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/vscan_svm.yaml | @@ -9627,7 +9946,7 @@ Used space threshold size, in percentage, for the automatic growth of the volume | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `autosize.grow_threshold` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `autosize.grow_threshold` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-autosize-attributes.grow-threshold-percent` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9637,7 +9956,7 @@ Maximum size in bytes up to which a volume grows automatically. This size cannot | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `autosize.maximum` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `autosize.maximum` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-autosize-attributes.maximum-size` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9657,7 +9976,7 @@ Total usable size of the volume, in bytes. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.filesystem_size` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.filesystem_size` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.filesystem-size` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9691,6 +10010,176 @@ volume_inode_files_used / volume_inode_total | ZAPI | `NA` | `Harvest generated` | conf/zapi/cdot/9.8.0/volume.yaml | +### volume_nfs_access_latency + +Average time for the WAFL filesystem to process NFS protocol access requests to the volume; not including NFS protocol request processing or network communication time which will also be included in client observed NFS request latency. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.access_latency`
Unit: microsec
Type: average
Base: nfs.access_ops | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_access_latency`
Unit: microsec
Type: average
Base: nfs_access_ops | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_access_ops + +Number of NFS accesses per second to the volume. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.access_ops`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_access_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_getattr_latency + +Average time for the WAFL filesystem to process NFS protocol getattr requests to the volume; not including NFS protocol request processing or network communication time which will also be included in client observed NFS request latency. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.getattr_latency`
Unit: microsec
Type: average
Base: nfs.getattr_ops | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_getattr_latency`
Unit: microsec
Type: average
Base: nfs_getattr_ops | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_getattr_ops + +Number of NFS getattr per second to the volume. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.getattr_ops`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_getattr_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_lookup_latency + +Average time for the WAFL filesystem to process NFS protocol lookup requests to the volume; not including NFS protocol request processing or network communication time which will also be included in client observed NFS request latency. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.lookup_latency`
Unit: microsec
Type: average
Base: nfs.lookup_ops | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_lookup_latency`
Unit: microsec
Type: average
Base: nfs_lookup_ops | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_lookup_ops + +Number of NFS lookups per second to the volume. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.lookup_ops`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_lookup_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_other_latency + +Average time for the WAFL filesystem to process other NFS operations to the volume; not including NFS protocol request processing or network communication time which will also be included in client observed NFS request latency + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.other_latency`
Unit: microsec
Type: average
Base: nfs.other_ops | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_other_latency`
Unit: microsec
Type: average
Base: nfs_other_ops | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_other_ops + +Number of other NFS operations per second to the volume + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.other_ops`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_other_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_punch_hole_latency + +Average time for the WAFL filesystem to process NFS protocol hole-punch requests to the volume. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.punch_hole_latency`
Unit: microsec
Type: average
Base: nfs.punch_hole_ops | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_punch_hole_latency`
Unit: microsec
Type: average
Base: nfs_punch_hole_ops | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_punch_hole_ops + +Number of NFS hole-punch requests per second to the volume. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.punch_hole_ops`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_punch_hole_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_read_latency + +Average time for the WAFL filesystem to process NFS protocol read requests to the volume; not including NFS protocol request processing or network communication time which will also be included in client observed NFS request latency + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.read_latency`
Unit: microsec
Type: average
Base: nfs.read_ops | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_read_latency`
Unit: microsec
Type: average
Base: nfs_read_ops | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_read_ops + +Number of NFS read operations per second from the volume + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.read_ops`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_read_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_setattr_latency + +Average time for the WAFL filesystem to process NFS protocol setattr requests to the volume. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.setattr_latency`
Unit: microsec
Type: average
Base: nfs.setattr_ops | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_setattr_latency`
Unit: microsec
Type: average
Base: nfs_setattr_ops | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_setattr_ops + +Number of NFS setattr requests per second to the volume. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.setattr_ops`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_setattr_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_total_ops + +Number of total NFS operations per second to the volume. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.total_ops`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_total_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_write_latency + +Average time for the WAFL filesystem to process NFS protocol write requests to the volume; not including NFS protocol request processing or network communication time, which will also be included in client observed NFS request latency + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.write_latency`
Unit: microsec
Type: average
Base: nfs.write_ops | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_write_latency`
Unit: microsec
Type: average
Base: nfs_write_ops | conf/zapiperf/cdot/9.8.0/volume.yaml | + + +### volume_nfs_write_ops + +Number of NFS write operations per second to the volume + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/volume` | `nfs.write_ops`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.12.0/volume.yaml | +| ZAPI | `perf-object-get-instances volume` | `nfs_write_ops`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/volume.yaml | + + ### volume_other_latency Average latency in microseconds for the WAFL filesystem to process other operations to the volume; not including request processing or network communication time @@ -9727,7 +10216,7 @@ Reserved space for overwrites, in bytes. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.overwrite_reserve` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.overwrite_reserve` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.overwrite-reserve` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9737,7 +10226,7 @@ Overwrite logical reserve space used, in bytes. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.overwrite_reserve_used` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.overwrite_reserve_used` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.overwrite-reserve-used` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9837,7 +10326,7 @@ Total provisioned size. The default size is equal to the minimum size of 20MB, i | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.size` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.size` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.size` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9847,7 +10336,7 @@ The available space, in bytes. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.available` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.available` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.size-available` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9857,7 +10346,7 @@ Total size of AFS, excluding snap-reserve, in bytes. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.afs_total` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.afs_total` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.size-total` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9867,7 +10356,7 @@ The virtual space used (includes volume reserves) before storage efficiency, in | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.used` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.used` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.size-used` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9877,7 +10366,7 @@ percentage of utilized storage space in a volume relative to its total capacity | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.percent_used` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.percent_used` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.percentage-size-used` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9887,7 +10376,7 @@ Number of Snapshot copies in the volume. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `snapshot_count` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `snapshot_count` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-snapshot-attributes.snapshot-count` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9897,7 +10386,7 @@ Size available for Snapshot copies within the Snapshot copy reserve, in bytes. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.snapshot.reserve_available` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.snapshot.reserve_available` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.snapshot-reserve-available` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9907,7 +10396,7 @@ The space that has been set aside as a reserve for Snapshot copy usage, in perce | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.snapshot.reserve_percent` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.snapshot.reserve_percent` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.percentage-snapshot-reserve` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9917,7 +10406,7 @@ Size in the volume that has been set aside as a reserve for Snapshot copy usage, | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.snapshot.reserve_size` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.snapshot.reserve_size` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.snapshot-reserve-size` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9937,7 +10426,7 @@ Percentage of snapshot reserve size that has been used. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.snapshot.space_used_percent` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.snapshot.space_used_percent` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.percentage-snapshot-reserve-used` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9947,7 +10436,7 @@ Available space for Snapshot copies from snap-reserve, in bytes. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.size_available_for_snapshots` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.size_available_for_snapshots` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.size-available-for-snapshots` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9957,7 +10446,7 @@ The total space used by Snapshot copies in the volume, in bytes. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.snapshot.used` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.snapshot.used` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.size-used-by-snapshots` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9967,7 +10456,7 @@ Size that should be available for the volume, irrespective of available size in | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.expected_available` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.expected_available` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.expected-available` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9977,7 +10466,7 @@ The amount of space available in this volume with storage efficiency space consi | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.logical_space.available` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.logical_space.available` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.logical-available` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9987,7 +10476,7 @@ SUM of (physical-used, shared_refs, compression_saved_in_plane0, vbn_zero, futur | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.logical_space.used` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.logical_space.used` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.logical-used` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -9997,7 +10486,7 @@ The virtual space used by AFS alone (includes volume reserves) and along with st | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.logical_space.used_by_afs` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.logical_space.used_by_afs` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.logical-used-by-afs` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -10007,7 +10496,7 @@ Size that is logically used across all Snapshot copies in the volume, in bytes. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.logical_space.used_by_snapshots` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.logical_space.used_by_snapshots` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.logical-used-by-snapshots` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -10017,7 +10506,7 @@ SUM of (physical-used, shared_refs, compression_saved_in_plane0, vbn_zero, futur | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.logical_space.used_percent` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.logical_space.used_percent` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.logical-used-percent` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -10027,7 +10516,7 @@ Size that is physically used in the volume, in bytes. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.physical_used` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.physical_used` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.physical-used` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -10037,7 +10526,7 @@ Size that is physically used in the volume, as a percentage. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/storage/volumes` | `space.physical_used_percent` | conf/rest/9.12.0/volume.yaml | +| REST | `api/storage/volumes` | `space.physical_used_percent` | conf/rest/9.9.0/volume.yaml | | ZAPI | `volume-get-iter` | `volume-attributes.volume-space-attributes.physical-used-percent` | conf/zapi/cdot/9.8.0/volume.yaml | @@ -10087,42 +10576,47 @@ Average scan latency | API | Endpoint | Metric | Template | |--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/vscan` | `scan.latency`
Unit: microsec
Type: average
Base: scan.requests | conf/restperf/9.13.0/vscan.yaml | | ZAPI | `perf-object-get-instances offbox_vscan_server` | `scan_latency`
Unit: microsec
Type: average
Base: scan_latency_base | conf/zapiperf/cdot/9.8.0/vscan.yaml | ### vscan_scan_request_dispatched_rate -Total number of scan requests sent to the Vscanner per second +Total number of scan requests sent to the scanner per second | API | Endpoint | Metric | Template | |--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/vscan` | `scan.request_dispatched_rate`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.13.0/vscan.yaml | | ZAPI | `perf-object-get-instances offbox_vscan_server` | `scan_request_dispatched_rate`
Unit: per_sec
Type: rate
Base: | conf/zapiperf/cdot/9.8.0/vscan.yaml | ### vscan_scanner_stats_pct_cpu_used -Percentage CPU utilization on scanner +Percentage CPU utilization on scanner calculated over the last 15 seconds. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/vscan` | `scanner.stats_percent_cpu_used`
Unit: none
Type: raw
Base: | conf/restperf/9.13.0/vscan.yaml | | ZAPI | `perf-object-get-instances offbox_vscan_server` | `scanner_stats_pct_cpu_used`
Unit: none
Type: raw
Base: | conf/zapiperf/cdot/9.8.0/vscan.yaml | ### vscan_scanner_stats_pct_mem_used -Percentage RAM utilization on scanner +Percentage RAM utilization on scanner calculated over the last 15 seconds. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/vscan` | `scanner.stats_percent_mem_used`
Unit: none
Type: raw
Base: | conf/restperf/9.13.0/vscan.yaml | | ZAPI | `perf-object-get-instances offbox_vscan_server` | `scanner_stats_pct_mem_used`
Unit: none
Type: raw
Base: | conf/zapiperf/cdot/9.8.0/vscan.yaml | ### vscan_scanner_stats_pct_network_used -Percentage network utilization on scanner +Percentage network utilization on scanner calculated for the last 15 seconds. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| +| REST | `api/cluster/counter/tables/vscan` | `scanner.stats_percent_network_used`
Unit: none
Type: raw
Base: | conf/restperf/9.13.0/vscan.yaml | | ZAPI | `perf-object-get-instances offbox_vscan_server` | `scanner_stats_pct_network_used`
Unit: none
Type: raw
Base: | conf/zapiperf/cdot/9.8.0/vscan.yaml | From 1df85ff62bc0f3f93e1beab42f37dbe7bb880f2e Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 9 Aug 2023 20:28:22 +0530 Subject: [PATCH 02/40] ci: remove apt-get update --- jenkins/artifacts/jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins/artifacts/jenkinsfile b/jenkins/artifacts/jenkinsfile index b21e935c8..d64b89bfd 100644 --- a/jenkins/artifacts/jenkinsfile +++ b/jenkins/artifacts/jenkinsfile @@ -65,7 +65,6 @@ pipeline { stage('Download Prerequisites') { steps { sh ''' - apt-get update apt-get install -y rpm apt-get install -y net-tools apt install -y git-all From 7468e8ac650b3f420ac0bcbf034d5d0e3c4bc3cb Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Thu, 10 Aug 2023 02:50:21 -0400 Subject: [PATCH 03/40] =?UTF-8?q?feat:=20Harvest=20SVM=20dashboard=20shoul?= =?UTF-8?q?d=20include=20latency=20heatmap=20panels=20NFS=E2=80=A6=20(#226?= =?UTF-8?q?8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Harvest SVM dashboard should include latency heatmap panels NFSv4 and NFSv4.1 Fixes #2248 * feat: Harvest SVM dashboard should include latency heatmap panels NFSv4 and NFSv4.1 Fixes #2248 * feat: changed the from time in dashboard --------- Co-authored-by: hardikl --- conf/restperf/9.12.0/nfsv4.yaml | 3 + conf/restperf/9.12.0/nfsv4_1.yaml | 3 + conf/zapiperf/cdot/9.8.0/nfsv4.yaml | 3 + conf/zapiperf/cdot/9.8.0/nfsv4_1.yaml | 3 + go.mod | 1 + go.sum | 28 +- grafana/dashboards/cmode/svm.json | 554 +++++++++++++++++++++++--- vendor/modules.txt | 2 + 8 files changed, 526 insertions(+), 71 deletions(-) diff --git a/conf/restperf/9.12.0/nfsv4.yaml b/conf/restperf/9.12.0/nfsv4.yaml index fe89a1647..05c9fc9c3 100644 --- a/conf/restperf/9.12.0/nfsv4.yaml +++ b/conf/restperf/9.12.0/nfsv4.yaml @@ -58,6 +58,7 @@ counters: - putrootfh.total => putrootfh_total - read.average_latency => read_avg_latency - read.total => read_total + - read_latency_histogram => read_latency_hist - readdir.average_latency => readdir_avg_latency - readdir.total => readdir_total - readlink.average_latency => readlink_avg_latency @@ -82,6 +83,7 @@ counters: - setclientid.total => setclientid_total - setclientid_confirm.average_latency => setclientid_confirm_avg_latency - setclientid_confirm.total => setclientid_confirm_total + - total.latency_histogram => latency_hist - total.read_throughput => read_throughput - total.throughput => throughput - total.write_throughput => write_throughput @@ -90,6 +92,7 @@ counters: - verify.total => verify_total - write.average_latency => write_avg_latency - write.total => write_total + - write_latency_histogram => write_latency_hist override: - access.total: rate diff --git a/conf/restperf/9.12.0/nfsv4_1.yaml b/conf/restperf/9.12.0/nfsv4_1.yaml index 3f023aa82..923d13e6b 100644 --- a/conf/restperf/9.12.0/nfsv4_1.yaml +++ b/conf/restperf/9.12.0/nfsv4_1.yaml @@ -82,6 +82,7 @@ counters: - putrootfh.total => putrootfh_total - read.average_latency => read_avg_latency - read.total => read_total + - read_latency_histogram => read_latency_hist - readdir.average_latency => readdir_avg_latency - readdir.total => readdir_total - readlink.average_latency => readlink_avg_latency @@ -108,6 +109,7 @@ counters: - setattr.total => setattr_total - test_stateid.average_latency => test_stateid_avg_latency - test_stateid.total => test_stateid_total + - total.latency_histogram => latency_hist - total.read_throughput => read_throughput - total.throughput => write_throughput - total.write_throughput => throughput @@ -118,6 +120,7 @@ counters: - want_delegation.total => want_delegation_total - write.average_latency => write_avg_latency - write.total => write_total + - write_latency_histogram => write_latency_hist export_options: instance_keys: diff --git a/conf/zapiperf/cdot/9.8.0/nfsv4.yaml b/conf/zapiperf/cdot/9.8.0/nfsv4.yaml index c78bab55c..bf0ceabc9 100644 --- a/conf/zapiperf/cdot/9.8.0/nfsv4.yaml +++ b/conf/zapiperf/cdot/9.8.0/nfsv4.yaml @@ -37,6 +37,7 @@ counters: - lookup_total - lookupp_avg_latency - lookupp_total + - nfs4_latency_hist => latency_hist - nfs4_read_throughput => read_throughput - nfs4_throughput => throughput - nfs4_write_throughput => write_throughput @@ -59,6 +60,7 @@ counters: - putrootfh_avg_latency - putrootfh_total - read_avg_latency + - read_latency_hist - read_total - readdir_avg_latency - readdir_total @@ -88,6 +90,7 @@ counters: - verify_avg_latency - verify_total - write_avg_latency + - write_latency_hist - write_total override: diff --git a/conf/zapiperf/cdot/9.8.0/nfsv4_1.yaml b/conf/zapiperf/cdot/9.8.0/nfsv4_1.yaml index c1fd14254..7cb286d20 100644 --- a/conf/zapiperf/cdot/9.8.0/nfsv4_1.yaml +++ b/conf/zapiperf/cdot/9.8.0/nfsv4_1.yaml @@ -63,6 +63,7 @@ counters: - lookup_total - lookupp_avg_latency - lookupp_total + - nfs41_latency_hist => latency_hist - nfs41_read_throughput => read_throughput - nfs41_throughput => throughput - nfs41_write_throughput => write_throughput @@ -83,6 +84,7 @@ counters: - putrootfh_avg_latency - putrootfh_total - read_avg_latency + - read_latency_hist - read_total - readdir_avg_latency - readdir_total @@ -116,6 +118,7 @@ counters: - want_delegation_avg_latency - want_delegation_total - write_avg_latency + - write_latency_hist - write_total export_options: diff --git a/go.mod b/go.mod index d4f641769..cbae6e2b1 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/mattn/go-runewidth v0.0.14 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect github.com/rivo/uniseg v0.4.4 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tidwall/match v1.1.1 // indirect diff --git a/go.sum b/go.sum index 08a6d81e9..38d118fb8 100644 --- a/go.sum +++ b/go.sum @@ -12,7 +12,6 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= @@ -37,8 +36,8 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -73,17 +72,12 @@ github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:Om github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= -github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y= -github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY= -github.com/shirou/gopsutil/v3 v3.23.6 h1:5y46WPI9QBKBbK7EEccUPNXpJpNrvPuTD0O2zHEHT08= -github.com/shirou/gopsutil/v3 v3.23.6/go.mod h1:j7QX50DrXYggrpN30W0Mo+I4/8U2UUIQrnrhqUeWrAU= github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -104,12 +98,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.15.0 h1:5n/pM+v3r5ujuNl4YLZLsQ+UE5jlkLVm7jMzT5Mpolw= github.com/tidwall/gjson v1.15.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -136,22 +127,11 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/grafana/dashboards/cmode/svm.json b/grafana/dashboards/cmode/svm.json index 4fbe367f2..88e6a104c 100644 --- a/grafana/dashboards/cmode/svm.json +++ b/grafana/dashboards/cmode/svm.json @@ -71,7 +71,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1681989179502, + "iteration": 1691508761583, "links": [ { "asDropdown": true, @@ -5321,7 +5321,7 @@ "h": 5, "w": 8, "x": 0, - "y": 161 + "y": 7 }, "id": 39, "options": { @@ -5395,7 +5395,7 @@ "h": 5, "w": 8, "x": 8, - "y": 161 + "y": 7 }, "id": 50, "interval": null, @@ -5494,7 +5494,7 @@ "h": 5, "w": 8, "x": 16, - "y": 161 + "y": 7 }, "id": 46, "interval": null, @@ -5584,7 +5584,7 @@ "h": 5, "w": 4, "x": 0, - "y": 166 + "y": 12 }, "id": 48, "options": { @@ -5649,7 +5649,7 @@ "h": 5, "w": 4, "x": 4, - "y": 166 + "y": 12 }, "id": 47, "options": { @@ -5723,7 +5723,7 @@ "h": 5, "w": 4, "x": 8, - "y": 166 + "y": 12 }, "id": 150, "interval": null, @@ -5822,7 +5822,7 @@ "h": 5, "w": 4, "x": 12, - "y": 166 + "y": 12 }, "id": 151, "interval": null, @@ -5921,7 +5921,7 @@ "h": 5, "w": 4, "x": 16, - "y": 166 + "y": 12 }, "id": 152, "interval": null, @@ -6020,7 +6020,7 @@ "h": 5, "w": 4, "x": 20, - "y": 166 + "y": 12 }, "id": 153, "interval": null, @@ -6095,7 +6095,7 @@ "h": 8, "w": 8, "x": 0, - "y": 171 + "y": 17 }, "heatmap": {}, "hideZeroBuckets": true, @@ -6164,7 +6164,7 @@ "h": 8, "w": 8, "x": 8, - "y": 171 + "y": 17 }, "heatmap": {}, "hideZeroBuckets": true, @@ -6233,7 +6233,7 @@ "h": 8, "w": 8, "x": 16, - "y": 171 + "y": 17 }, "heatmap": {}, "hideZeroBuckets": true, @@ -6336,7 +6336,7 @@ "h": 8, "w": 8, "x": 0, - "y": 179 + "y": 25 }, "id": 51, "options": { @@ -6428,7 +6428,7 @@ "h": 8, "w": 8, "x": 8, - "y": 179 + "y": 25 }, "id": 53, "options": { @@ -6528,7 +6528,7 @@ "h": 8, "w": 8, "x": 16, - "y": 179 + "y": 25 }, "id": 42, "options": { @@ -6629,7 +6629,7 @@ "h": 8, "w": 12, "x": 0, - "y": 187 + "y": 33 }, "id": 52, "options": { @@ -6724,7 +6724,7 @@ "h": 8, "w": 12, "x": 12, - "y": 187 + "y": 33 }, "id": 54, "options": { @@ -6811,7 +6811,7 @@ "h": 5, "w": 8, "x": 0, - "y": 196 + "y": 8 }, "id": 154, "options": { @@ -6885,7 +6885,7 @@ "h": 5, "w": 8, "x": 8, - "y": 196 + "y": 8 }, "id": 155, "interval": null, @@ -6984,7 +6984,7 @@ "h": 5, "w": 8, "x": 16, - "y": 196 + "y": 8 }, "id": 166, "interval": null, @@ -7074,7 +7074,7 @@ "h": 5, "w": 4, "x": 0, - "y": 201 + "y": 13 }, "id": 158, "options": { @@ -7139,7 +7139,7 @@ "h": 5, "w": 4, "x": 4, - "y": 201 + "y": 13 }, "id": 159, "options": { @@ -7213,7 +7213,7 @@ "h": 5, "w": 4, "x": 8, - "y": 201 + "y": 13 }, "id": 160, "interval": null, @@ -7312,7 +7312,7 @@ "h": 5, "w": 4, "x": 12, - "y": 201 + "y": 13 }, "id": 161, "interval": null, @@ -7411,7 +7411,7 @@ "h": 5, "w": 4, "x": 16, - "y": 201 + "y": 13 }, "id": 162, "interval": null, @@ -7510,7 +7510,7 @@ "h": 5, "w": 4, "x": 20, - "y": 201 + "y": 13 }, "id": 163, "interval": null, @@ -7566,6 +7566,213 @@ ], "type": "stat" }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 18 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 256, + "interval": "6m", + "legend": { + "show": false + }, + "maxDataPoints": 25, + "reverseYBuckets": false, + "targets": [ + { + "exemplar": false, + "expr": "sum(increase(svm_nfs_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$TopNfsv4Latency\",nfsv=\"v4\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "NFSv4 Latency Heatmap", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null, + "width": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 18 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 257, + "interval": "6m", + "legend": { + "show": false + }, + "maxDataPoints": 25, + "reverseYBuckets": false, + "targets": [ + { + "exemplar": false, + "expr": "sum(increase(svm_nfs_read_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$TopNfsv4Latency\",nfsv=\"v4\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "NFSv4 Read Latency Heatmap", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null, + "width": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 18 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 258, + "interval": "6m", + "legend": { + "show": false + }, + "maxDataPoints": 25, + "reverseYBuckets": false, + "targets": [ + { + "exemplar": false, + "expr": "sum(increase(svm_nfs_write_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$TopNfsv4Latency\",nfsv=\"v4\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "NFSv4 Write Latency Heatmap", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null, + "width": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, { "datasource": "${DS_PROMETHEUS}", "fieldConfig": { @@ -7624,7 +7831,7 @@ "h": 8, "w": 8, "x": 0, - "y": 206 + "y": 26 }, "id": 145, "options": { @@ -7723,7 +7930,7 @@ "h": 8, "w": 8, "x": 8, - "y": 206 + "y": 26 }, "id": 146, "options": { @@ -7817,7 +8024,7 @@ "h": 8, "w": 8, "x": 16, - "y": 206 + "y": 26 }, "id": 147, "options": { @@ -7920,7 +8127,7 @@ "h": 8, "w": 12, "x": 0, - "y": 214 + "y": 34 }, "id": 157, "options": { @@ -8020,7 +8227,7 @@ "h": 8, "w": 12, "x": 12, - "y": 214 + "y": 34 }, "id": 149, "options": { @@ -8111,7 +8318,7 @@ "h": 5, "w": 8, "x": 0, - "y": 223 + "y": 9 }, "id": 164, "options": { @@ -8185,7 +8392,7 @@ "h": 5, "w": 8, "x": 8, - "y": 223 + "y": 9 }, "id": 165, "interval": null, @@ -8284,7 +8491,7 @@ "h": 5, "w": 8, "x": 16, - "y": 223 + "y": 9 }, "id": 156, "interval": null, @@ -8374,7 +8581,7 @@ "h": 5, "w": 4, "x": 0, - "y": 228 + "y": 14 }, "id": 140, "options": { @@ -8439,7 +8646,7 @@ "h": 5, "w": 4, "x": 4, - "y": 228 + "y": 14 }, "id": 167, "options": { @@ -8513,7 +8720,7 @@ "h": 5, "w": 4, "x": 8, - "y": 228 + "y": 14 }, "id": 168, "interval": null, @@ -8612,7 +8819,7 @@ "h": 5, "w": 4, "x": 12, - "y": 228 + "y": 14 }, "id": 169, "interval": null, @@ -8711,7 +8918,7 @@ "h": 5, "w": 4, "x": 16, - "y": 228 + "y": 14 }, "id": 170, "interval": null, @@ -8810,7 +9017,7 @@ "h": 5, "w": 4, "x": 20, - "y": 228 + "y": 14 }, "id": 171, "interval": null, @@ -8866,6 +9073,213 @@ ], "type": "stat" }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 19 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 253, + "interval": "6m", + "legend": { + "show": false + }, + "maxDataPoints": 25, + "reverseYBuckets": false, + "targets": [ + { + "exemplar": false, + "expr": "sum(increase(svm_nfs_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$TopNfsv41Latency\",nfsv=\"v4.1\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "NFSv4.1 Latency Heatmap", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null, + "width": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 19 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 254, + "interval": "6m", + "legend": { + "show": false + }, + "maxDataPoints": 25, + "reverseYBuckets": false, + "targets": [ + { + "exemplar": false, + "expr": "sum(increase(svm_nfs_read_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$TopNfsv41Latency\",nfsv=\"v4.1\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "NFSv4.1 Read Latency Heatmap", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null, + "width": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 19 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 255, + "interval": "6m", + "legend": { + "show": false + }, + "maxDataPoints": 25, + "reverseYBuckets": false, + "targets": [ + { + "exemplar": false, + "expr": "sum(increase(svm_nfs_write_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$TopNfsv41Latency\",nfsv=\"v4.1\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "NFSv4.1 Write Latency Heatmap", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null, + "width": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, { "datasource": "${DS_PROMETHEUS}", "fieldConfig": { @@ -8924,7 +9338,7 @@ "h": 8, "w": 8, "x": 0, - "y": 233 + "y": 27 }, "id": 172, "options": { @@ -9023,7 +9437,7 @@ "h": 8, "w": 8, "x": 8, - "y": 233 + "y": 27 }, "id": 173, "options": { @@ -9122,7 +9536,7 @@ "h": 8, "w": 8, "x": 16, - "y": 233 + "y": 27 }, "id": 174, "options": { @@ -9228,7 +9642,7 @@ "h": 8, "w": 12, "x": 0, - "y": 241 + "y": 35 }, "id": 175, "options": { @@ -9328,7 +9742,7 @@ "h": 8, "w": 12, "x": 12, - "y": 241 + "y": 35 }, "id": 176, "options": { @@ -14653,6 +15067,52 @@ "skipUrlSync": false, "sort": 0, "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(svm_nfs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopNfsv41Latency", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(svm_nfs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*svm=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(svm_nfs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopNfsv4Latency", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(svm_nfs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*svm=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" } ] }, @@ -14676,5 +15136,5 @@ "timezone": "", "title": "ONTAP: SVM", "uid": "", - "version": 22 + "version": 23 } diff --git a/vendor/modules.txt b/vendor/modules.txt index 2d0d5086e..0cdca1e01 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -56,6 +56,8 @@ github.com/power-devops/perfstat # github.com/rivo/uniseg v0.4.4 ## explicit; go 1.18 github.com/rivo/uniseg +# github.com/rogpeppe/go-internal v1.11.0 +## explicit; go 1.19 # github.com/rs/zerolog v1.30.0 ## explicit; go 1.15 github.com/rs/zerolog From b4578efc9d469ccb3bea3dc6a4003548151d470c Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Thu, 10 Aug 2023 11:06:10 -0400 Subject: [PATCH 04/40] refactor: reduce asup log noise --- cmd/poller/collector/asup.go | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/cmd/poller/collector/asup.go b/cmd/poller/collector/asup.go index 42841295b..38fd28c90 100644 --- a/cmd/poller/collector/asup.go +++ b/cmd/poller/collector/asup.go @@ -15,7 +15,6 @@ import ( "github.com/shirou/gopsutil/v3/host" "github.com/shirou/gopsutil/v3/mem" "github.com/shirou/gopsutil/v3/process" - "golang.org/x/sys/unix" "os" "os/exec" "path" @@ -291,7 +290,6 @@ func attachMemory(msg *Payload) { // This is similar to ps -ww -eo user,pid,ppid,stime,nlwp,rss,cmd processes, err := process.Processes() if err != nil { - logging.Get().Error().Err(err).Msg("Unable to get processes") return } @@ -301,24 +299,13 @@ func attachMemory(msg *Payload) { for _, p := range processes { name, err := p.Name() if err != nil { - if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.ENOENT) { - // we do not have permissions. That's OK, skip this one - } else { - logging.Get().Error().Err(err).Msg("Unable to get name") - } continue } pp := Process{ Pid: p.Pid, } memInfo, err := p.MemoryInfo() - if err != nil { - if errors.Is(err, unix.EPERM) { - // we do not have permissions. That's OK, skip this one - } else { - logging.Get().Error().Err(err).Msg("Unable to get memory") - } - } else { + if err == nil { pp.RssBytes = memInfo.RSS } @@ -333,7 +320,7 @@ func attachMemory(msg *Payload) { cmdline, err := p.Cmdline() if err != nil { - logging.Get().Error().Err(err).Msg("Unable to get cmdline") + continue } else { pp.Cmdline = cmdline } @@ -342,27 +329,19 @@ func attachMemory(msg *Payload) { } username, err := p.Username() - if err != nil { - logging.Get().Error().Err(err).Msg("Unable to get username") - } else { + if err == nil { pp.User = username } ppid, err := p.Ppid() - if err != nil { - logging.Get().Error().Err(err).Msg("Unable to get parent pid") - } else { + if err == nil { pp.Ppid = ppid } ctime, err := p.CreateTime() - if err != nil { - logging.Get().Error().Err(err).Msg("Unable to get createTime") - } else { + if err == nil { pp.Ctime = ctime } threads, err := p.NumThreads() - if err != nil { - logging.Get().Error().Err(err).Msg("Unable to get numThreads") - } else { + if err == nil { pp.Threads = threads } msg.Platform.Processes = append(msg.Platform.Processes, pp) From 86294163eeae362e3e543ec7ac1adeca22e86b31 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Thu, 10 Aug 2023 11:00:02 -0400 Subject: [PATCH 05/40] test: ensure dashboard time is now-3h --- cmd/tools/grafana/dashboard_test.go | 21 +++++++++++++++++++ grafana/dashboards/cmode/compliance.json | 2 +- .../cmode/data_protection_snapshot.json | 2 +- grafana/dashboards/cmode/disk.json | 2 +- grafana/dashboards/cmode/fsa.json | 2 +- grafana/dashboards/cmode/headroom.json | 2 +- grafana/dashboards/cmode/health.json | 2 +- grafana/dashboards/cmode/lun.json | 2 +- grafana/dashboards/cmode/mcc_cluster.json | 2 +- grafana/dashboards/cmode/namespace.json | 2 +- grafana/dashboards/cmode/nfs4storePool.json | 2 +- grafana/dashboards/cmode/nfs_clients.json | 2 +- grafana/dashboards/cmode/node.json | 2 +- grafana/dashboards/cmode/qtree.json | 2 +- grafana/dashboards/cmode/quotaReport.json | 2 +- grafana/dashboards/cmode/s3ObjectStorage.json | 2 +- grafana/dashboards/cmode/security.json | 2 +- grafana/dashboards/cmode/volume.json | 2 +- grafana/dashboards/cmode/workload.json | 2 +- .../dashboards/storagegrid/fabricpool.json | 2 +- grafana/dashboards/storagegrid/overview.json | 2 +- grafana/dashboards/storagegrid/tenant.json | 2 +- 22 files changed, 42 insertions(+), 21 deletions(-) diff --git a/cmd/tools/grafana/dashboard_test.go b/cmd/tools/grafana/dashboard_test.go index 91c307326..db7a51629 100644 --- a/cmd/tools/grafana/dashboard_test.go +++ b/cmd/tools/grafana/dashboard_test.go @@ -1212,3 +1212,24 @@ func writeSorted(t *testing.T, path string, sorted []byte) string { create.Close() return dest } + +func TestDashboardTime(t *testing.T) { + visitDashboards(dashboards, func(path string, data []byte) { + checkDashboardTime(t, path, data) + }) +} + +func checkDashboardTime(t *testing.T, path string, data []byte) { + dashPath := shortPath(path) + from := gjson.GetBytes(data, "time.from") + to := gjson.GetBytes(data, "time.to") + + fromWant := "now-3h" + toWant := "now" + if from.String() != fromWant { + t.Errorf("dashboard=%s time.from got=%s want=%s", dashPath, from.String(), fromWant) + } + if to.String() != toWant { + t.Errorf("dashboard=%s time.to got=%s want=%s", dashPath, to.String(), toWant) + } +} diff --git a/grafana/dashboards/cmode/compliance.json b/grafana/dashboards/cmode/compliance.json index 17d67e639..0801ee387 100644 --- a/grafana/dashboards/cmode/compliance.json +++ b/grafana/dashboards/cmode/compliance.json @@ -136,7 +136,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/cmode/data_protection_snapshot.json b/grafana/dashboards/cmode/data_protection_snapshot.json index ef67f8251..00719d08b 100644 --- a/grafana/dashboards/cmode/data_protection_snapshot.json +++ b/grafana/dashboards/cmode/data_protection_snapshot.json @@ -1586,7 +1586,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/cmode/disk.json b/grafana/dashboards/cmode/disk.json index 902ee5e3c..e2815844d 100644 --- a/grafana/dashboards/cmode/disk.json +++ b/grafana/dashboards/cmode/disk.json @@ -2756,7 +2756,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/cmode/fsa.json b/grafana/dashboards/cmode/fsa.json index 937eb7ac4..9faa5a3a7 100644 --- a/grafana/dashboards/cmode/fsa.json +++ b/grafana/dashboards/cmode/fsa.json @@ -1665,7 +1665,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/cmode/headroom.json b/grafana/dashboards/cmode/headroom.json index 78c3aa275..63e192ff4 100644 --- a/grafana/dashboards/cmode/headroom.json +++ b/grafana/dashboards/cmode/headroom.json @@ -2576,7 +2576,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-3h", "to": "now" }, "timepicker": {}, diff --git a/grafana/dashboards/cmode/health.json b/grafana/dashboards/cmode/health.json index a671b695c..ef693f763 100644 --- a/grafana/dashboards/cmode/health.json +++ b/grafana/dashboards/cmode/health.json @@ -3244,7 +3244,7 @@ ] }, "time": { - "from": "now-24h", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/cmode/lun.json b/grafana/dashboards/cmode/lun.json index e2068b63e..5d08e5a5f 100644 --- a/grafana/dashboards/cmode/lun.json +++ b/grafana/dashboards/cmode/lun.json @@ -5359,7 +5359,7 @@ ] }, "time": { - "from": "now-1h", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/cmode/mcc_cluster.json b/grafana/dashboards/cmode/mcc_cluster.json index 506950393..5e21b3790 100644 --- a/grafana/dashboards/cmode/mcc_cluster.json +++ b/grafana/dashboards/cmode/mcc_cluster.json @@ -3780,7 +3780,7 @@ ] }, "time": { - "from": "now-30m", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/cmode/namespace.json b/grafana/dashboards/cmode/namespace.json index 62421d5a5..46ce473bb 100644 --- a/grafana/dashboards/cmode/namespace.json +++ b/grafana/dashboards/cmode/namespace.json @@ -1295,7 +1295,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/cmode/nfs4storePool.json b/grafana/dashboards/cmode/nfs4storePool.json index 280503458..97ddcc551 100644 --- a/grafana/dashboards/cmode/nfs4storePool.json +++ b/grafana/dashboards/cmode/nfs4storePool.json @@ -2115,7 +2115,7 @@ ] }, "time": { - "from": "now-1h", + "from": "now-3h", "to": "now" }, "timepicker": {}, diff --git a/grafana/dashboards/cmode/nfs_clients.json b/grafana/dashboards/cmode/nfs_clients.json index a5e46a825..0ac6dc5a9 100644 --- a/grafana/dashboards/cmode/nfs_clients.json +++ b/grafana/dashboards/cmode/nfs_clients.json @@ -558,7 +558,7 @@ ] }, "time": { - "from": "now-30m", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/cmode/node.json b/grafana/dashboards/cmode/node.json index d00c4fd61..b551b8aae 100644 --- a/grafana/dashboards/cmode/node.json +++ b/grafana/dashboards/cmode/node.json @@ -4998,7 +4998,7 @@ ] }, "time": { - "from": "now-24h", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/cmode/qtree.json b/grafana/dashboards/cmode/qtree.json index f4e7dbd05..0930a45e9 100644 --- a/grafana/dashboards/cmode/qtree.json +++ b/grafana/dashboards/cmode/qtree.json @@ -1120,7 +1120,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/cmode/quotaReport.json b/grafana/dashboards/cmode/quotaReport.json index 8236ceb2d..75770cb25 100644 --- a/grafana/dashboards/cmode/quotaReport.json +++ b/grafana/dashboards/cmode/quotaReport.json @@ -761,7 +761,7 @@ ] }, "time": { - "from": "now-5m", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/cmode/s3ObjectStorage.json b/grafana/dashboards/cmode/s3ObjectStorage.json index e226f629e..a36fe1dc6 100644 --- a/grafana/dashboards/cmode/s3ObjectStorage.json +++ b/grafana/dashboards/cmode/s3ObjectStorage.json @@ -1003,7 +1003,7 @@ ] }, "time": { - "from": "now-5m", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/cmode/security.json b/grafana/dashboards/cmode/security.json index 8240a5105..70857e7b9 100644 --- a/grafana/dashboards/cmode/security.json +++ b/grafana/dashboards/cmode/security.json @@ -4856,7 +4856,7 @@ ] }, "time": { - "from": "now-1m", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/cmode/volume.json b/grafana/dashboards/cmode/volume.json index cf815eb6a..cb98c96a4 100644 --- a/grafana/dashboards/cmode/volume.json +++ b/grafana/dashboards/cmode/volume.json @@ -6965,7 +6965,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/cmode/workload.json b/grafana/dashboards/cmode/workload.json index 83e2aa605..b10bb54f5 100644 --- a/grafana/dashboards/cmode/workload.json +++ b/grafana/dashboards/cmode/workload.json @@ -2470,7 +2470,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-3h", "to": "now" }, "timepicker": { diff --git a/grafana/dashboards/storagegrid/fabricpool.json b/grafana/dashboards/storagegrid/fabricpool.json index d0a32a4ee..f6593b90d 100644 --- a/grafana/dashboards/storagegrid/fabricpool.json +++ b/grafana/dashboards/storagegrid/fabricpool.json @@ -1941,7 +1941,7 @@ ] }, "time": { - "from": "now-24h", + "from": "now-3h", "to": "now" }, "timepicker": {}, diff --git a/grafana/dashboards/storagegrid/overview.json b/grafana/dashboards/storagegrid/overview.json index fd84cb4ff..c7c849c76 100644 --- a/grafana/dashboards/storagegrid/overview.json +++ b/grafana/dashboards/storagegrid/overview.json @@ -2516,7 +2516,7 @@ ] }, "time": { - "from": "now-6h", + "from": "now-3h", "to": "now" }, "timepicker": {}, diff --git a/grafana/dashboards/storagegrid/tenant.json b/grafana/dashboards/storagegrid/tenant.json index ca55545b2..e9d7a04b4 100644 --- a/grafana/dashboards/storagegrid/tenant.json +++ b/grafana/dashboards/storagegrid/tenant.json @@ -570,7 +570,7 @@ ] }, "time": { - "from": "now-30m", + "from": "now-3h", "to": "now" }, "timepicker": {}, From 0ff1e6d63c9571f5b237cc900a47c9396f80d0f8 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Thu, 10 Aug 2023 13:08:30 -0400 Subject: [PATCH 06/40] refactor: increase max log file size from 5MB to 10MB --- docs/configure-harvest-basic.md | 38 ++++++++++++++++----------------- pkg/logging/logger.go | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/configure-harvest-basic.md b/docs/configure-harvest-basic.md index 3eaa693d4..cf5b5699f 100644 --- a/docs/configure-harvest-basic.md +++ b/docs/configure-harvest-basic.md @@ -4,25 +4,25 @@ The main configuration file, `harvest.yml`, consists of the following sections, All pollers are defined in `harvest.yml`, the main configuration file of Harvest, under the section `Pollers`. -| parameter | type | description | default | -|------------------------|------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------| -| Poller name (header) | **required** | Poller name, user-defined value | | -| `datacenter` | **required** | Datacenter name, user-defined value | | -| `addr` | required by some collectors | IPv4 or FQDN of the target system | | -| `collectors` | **required** | List of collectors to run for this poller | | -| `exporters` | **required** | List of exporter names from the `Exporters` section. Note: this should be the name of the exporter (e.g. `prometheus1`), not the value of the `exporter` key (e.g. `Prometheus`) | | -| `auth_style` | required by Zapi* collectors | Either `basic_auth` or `certificate_auth` See [authentication](#authentication) for details | `basic_auth` | -| `username`, `password` | required if `auth_style` is `basic_auth` | | | -| `ssl_cert`, `ssl_key` | optional if `auth_style` is `certificate_auth` | Absolute paths to SSL (client) certificate and key used to authenticate with the target system.

If not provided, the poller will look for `.key` and `.pem` in `$HARVEST_HOME/cert/`.

To create certificates for ONTAP systems, see [using certificate authentication](prepare-cdot-clusters.md#using-certificate-authentication) | | -| `use_insecure_tls` | optional, bool | If true, disable TLS verification when connecting to ONTAP cluster | false | -| `credentials_file` | optional, string | Path to a yaml file that contains cluster credentials. The file should have the same shape as `harvest.yml`. See [here](configure-harvest-basic.md#credentials-file) for examples. Path can be relative to `harvest.yml` or absolute. | | -| `credentials_script` | optional, section | Section that defines how Harvest should fetch credentials via external script. See [here](configure-harvest-basic.md#credentials-script) for details. | | -| `tls_min_version` | optional, string | Minimum TLS version to use when connecting to ONTAP cluster: One of tls10, tls11, tls12 or tls13 | Platform decides | -| `labels` | optional, list of key-value pairs | Each of the key-value pairs will be added to a poller's metrics. Details [below](configure-harvest-basic.md#labels) | | -| `log_max_bytes` | | Maximum size of the log file before it will be rotated | `5_242_880` (5 MB) | -| `log_max_files` | | Number of rotated log files to keep | `5` | -| `log` | optional, list of collector names | Matching collectors log their ZAPI request/response | | -| `prefer_zapi` | optional, bool | Use the ZAPI API if the cluster supports it, otherwise allow Harvest to choose REST or ZAPI, whichever is appropriate to the ONTAP version. See [rest-strategy](https://github.com/NetApp/harvest/blob/main/docs/architecture/rest-strategy.md) for details. | | +| parameter | type | description | default | +|------------------------|------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------| +| Poller name (header) | **required** | Poller name, user-defined value | | +| `datacenter` | **required** | Datacenter name, user-defined value | | +| `addr` | required by some collectors | IPv4 or FQDN of the target system | | +| `collectors` | **required** | List of collectors to run for this poller | | +| `exporters` | **required** | List of exporter names from the `Exporters` section. Note: this should be the name of the exporter (e.g. `prometheus1`), not the value of the `exporter` key (e.g. `Prometheus`) | | +| `auth_style` | required by Zapi* collectors | Either `basic_auth` or `certificate_auth` See [authentication](#authentication) for details | `basic_auth` | +| `username`, `password` | required if `auth_style` is `basic_auth` | | | +| `ssl_cert`, `ssl_key` | optional if `auth_style` is `certificate_auth` | Absolute paths to SSL (client) certificate and key used to authenticate with the target system.

If not provided, the poller will look for `.key` and `.pem` in `$HARVEST_HOME/cert/`.

To create certificates for ONTAP systems, see [using certificate authentication](prepare-cdot-clusters.md#using-certificate-authentication) | | +| `use_insecure_tls` | optional, bool | If true, disable TLS verification when connecting to ONTAP cluster | false | +| `credentials_file` | optional, string | Path to a yaml file that contains cluster credentials. The file should have the same shape as `harvest.yml`. See [here](configure-harvest-basic.md#credentials-file) for examples. Path can be relative to `harvest.yml` or absolute. | | +| `credentials_script` | optional, section | Section that defines how Harvest should fetch credentials via external script. See [here](configure-harvest-basic.md#credentials-script) for details. | | +| `tls_min_version` | optional, string | Minimum TLS version to use when connecting to ONTAP cluster: One of tls10, tls11, tls12 or tls13 | Platform decides | +| `labels` | optional, list of key-value pairs | Each of the key-value pairs will be added to a poller's metrics. Details [below](configure-harvest-basic.md#labels) | | +| `log_max_bytes` | | Maximum size of the log file before it will be rotated | `10 MB` | +| `log_max_files` | | Number of rotated log files to keep | `5` | +| `log` | optional, list of collector names | Matching collectors log their ZAPI request/response | | +| `prefer_zapi` | optional, bool | Use the ZAPI API if the cluster supports it, otherwise allow Harvest to choose REST or ZAPI, whichever is appropriate to the ONTAP version. See [rest-strategy](https://github.com/NetApp/harvest/blob/main/docs/architecture/rest-strategy.md) for details. | | ## Defaults diff --git a/pkg/logging/logger.go b/pkg/logging/logger.go index 545909358..33a400b80 100644 --- a/pkg/logging/logger.go +++ b/pkg/logging/logger.go @@ -19,7 +19,7 @@ const ( defaultLogLevel = zerolog.InfoLevel defaultConsoleLoggingEnabled = true defaultFileLoggingEnabled = false // false to avoid opening many file descriptors for same log file - DefaultLogMaxMegaBytes = 5 // 5 MB + DefaultLogMaxMegaBytes = 10 // 10 MB DefaultLogMaxBackups = 5 DefaultLogMaxAge = 7 ) From 1c6af51a0f8a03911d8fb095deab394f9886628a Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Thu, 10 Aug 2023 16:23:07 +0530 Subject: [PATCH 07/40] feat: add cpu_firmware_release to Cluster dashboard --- conf/rest/9.12.0/node.yaml | 2 ++ conf/zapi/cdot/9.8.0/node.yaml | 3 ++- grafana/dashboards/cmode/cluster.json | 17 ++++++++++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/conf/rest/9.12.0/node.yaml b/conf/rest/9.12.0/node.yaml index 470882b63..9b11de92d 100644 --- a/conf/rest/9.12.0/node.yaml +++ b/conf/rest/9.12.0/node.yaml @@ -5,6 +5,7 @@ object: node counters: - ^^name => node + - ^controller.cpu.firmware_release => cpu_firmware_release - ^controller.failed_fan.message.message => failed_fan_message - ^controller.failed_power_supply.message.message => failed_power_message - ^controller.over_temperature => over_temperature @@ -39,6 +40,7 @@ export_options: instance_keys: - node instance_labels: + - cpu_firmware_release - healthy - location - max_aggr_size diff --git a/conf/zapi/cdot/9.8.0/node.yaml b/conf/zapi/cdot/9.8.0/node.yaml index daac41d55..0f34d442f 100644 --- a/conf/zapi/cdot/9.8.0/node.yaml +++ b/conf/zapi/cdot/9.8.0/node.yaml @@ -6,7 +6,7 @@ object: node counters: node-details-info: - ^^node => node -# - ^cpu-firmware-release + - ^cpu-firmware-release => cpu_firmware_release - ^env-failed-fan-message => failed_fan_message - ^env-failed-power-supply-message => failed_power_message - ^env-over-temperature => over_temperature @@ -51,6 +51,7 @@ export_options: instance_keys: - node instance_labels: + - cpu_firmware_release - healthy - location - max_aggr_size diff --git a/grafana/dashboards/cmode/cluster.json b/grafana/dashboards/cmode/cluster.json index d8f550a7a..0b98961c1 100644 --- a/grafana/dashboards/cmode/cluster.json +++ b/grafana/dashboards/cmode/cluster.json @@ -71,7 +71,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1687422677620, + "iteration": 1691664436025, "links": [ { "asDropdown": true, @@ -1449,10 +1449,21 @@ "serial", "version", "model", - "Value #B" + "Value #B", + "cpu_firmware_release" ] } } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "cpu_firmware_release": "CPU Firmware Release" + } + } } ], "type": "table" @@ -4315,5 +4326,5 @@ "timezone": "", "title": "ONTAP: Cluster", "uid": "", - "version": 19 + "version": 20 } From e0448baf3617110ea4474ec3f602ae934660fe3b Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Fri, 11 Aug 2023 12:18:53 +0530 Subject: [PATCH 08/40] doc: update metric generate step command --- .github/ISSUE_TEMPLATE/release-template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/release-template.md b/.github/ISSUE_TEMPLATE/release-template.md index aee261ac0..ce3be7f02 100644 --- a/.github/ISSUE_TEMPLATE/release-template.md +++ b/.github/ISSUE_TEMPLATE/release-template.md @@ -46,7 +46,7 @@ go run pkg/changelog/main.go --title $RELEASE --highlights releaseHighlights_$RE #### Update Metrics Documentation ```bash -bin/harvest generate metrics +bin/harvest generate metrics --poller POLLERNAME ``` - [ ] Make sure docs look good and open a PR for review with `docs/ontap-metrics.md` changes From 40a073db322692260985fae988916d11515e9fd8 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Fri, 11 Aug 2023 12:23:15 +0530 Subject: [PATCH 09/40] fix: make poller mandatory for metrics generation cmd --- cmd/tools/generate/generate.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/tools/generate/generate.go b/cmd/tools/generate/generate.go index 9241a125d..20f3a57fc 100644 --- a/cmd/tools/generate/generate.go +++ b/cmd/tools/generate/generate.go @@ -396,6 +396,8 @@ func init() { flags := metricCmd.PersistentFlags() flags.StringVarP(&opts.Poller, "poller", "p", "sar", "name of poller, e.g. 10.193.48.154") + _ = metricCmd.MarkPersistentFlagRequired("poller") + dFlags.IntVarP(&opts.loglevel, "loglevel", "l", 2, "logging level (0=trace, 1=debug, 2=info, 3=warning, 4=error, 5=critical)", ) From e54b835bdf7a1a0f43a24649e56ac34f1fc1e976 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Fri, 11 Aug 2023 08:00:35 -0400 Subject: [PATCH 10/40] refactor: code cleanup --- cmd/collectors/ems/ems.go | 5 +---- cmd/poller/poller.go | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cmd/collectors/ems/ems.go b/cmd/collectors/ems/ems.go index d7b5ba516..530558e19 100644 --- a/cmd/collectors/ems/ems.go +++ b/cmd/collectors/ems/ems.go @@ -138,10 +138,7 @@ func (e *Ems) InitMatrix() error { } func (e *Ems) LoadPlugin(kind string, _ *plugin.AbstractPlugin) plugin.Plugin { - switch kind { - default: - e.Logger.Warn().Str("kind", kind).Msg("no ems plugin found ") - } + e.Logger.Warn().Str("kind", kind).Msg("no ems plugin found") return nil } diff --git a/cmd/poller/poller.go b/cmd/poller/poller.go index 83967dfa5..f6f937810 100644 --- a/cmd/poller/poller.go +++ b/cmd/poller/poller.go @@ -501,7 +501,7 @@ func (p *Poller) Run() { } } - // only zeroLog when numbers have changes, since hopefully that happens rarely + // only log when there are changes, which we expect to be infrequent if upc != upCollectors || upe != upExporters { logger.Info().Msgf("updated status, up collectors: %d (of %d), up exporters: %d (of %d)", upc, len(p.collectors), upe, len(p.exporters)) } From 4e6ac586b39d443c92bef46be0321e657b443762 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Fri, 11 Aug 2023 18:12:34 +0530 Subject: [PATCH 11/40] fix: disable netconnections in Rest by default --- conf/rest/default.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/rest/default.yaml b/conf/rest/default.yaml index 74fef53f9..05c758b9f 100644 --- a/conf/rest/default.yaml +++ b/conf/rest/default.yaml @@ -19,7 +19,7 @@ objects: Health: health.yaml Lun: lun.yaml Namespace: namespace.yaml - NetConnections: netconnections.yaml +# NetConnections: netconnections.yaml # NetPort: netport.yaml NetRoute: netroute.yaml # NFSClients: nfs_clients.yaml From b81c82d1c8d8edfdd94d38143b1ca6bf2478d752 Mon Sep 17 00:00:00 2001 From: Rahul Date: Fri, 11 Aug 2023 21:44:16 +0530 Subject: [PATCH 12/40] refactor: add cp command in dashboard sort test (#2278) * refactor: add cp command in dashboard sort test --- cmd/tools/grafana/dashboard_test.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cmd/tools/grafana/dashboard_test.go b/cmd/tools/grafana/dashboard_test.go index db7a51629..b1ede5cf6 100644 --- a/cmd/tools/grafana/dashboard_test.go +++ b/cmd/tools/grafana/dashboard_test.go @@ -946,7 +946,7 @@ func checkConnectNullValues(t *testing.T, path string, data []byte) { func TestPanelChildPanels(t *testing.T) { visitDashboards( - []string{"../../../grafana/dashboards/cmode", "../../../grafana/dashboards/storagegrid"}, + dashboards, func(path string, data []byte) { checkPanelChildPanels(t, shortPath(path), data) }) @@ -1182,8 +1182,9 @@ func TestDashboardKeysAreSorted(t *testing.T) { }) if string(sorted) != string(data) { sortedPath := writeSorted(t, path, sorted) - t.Errorf("dashboard=%s should have sorted keys but does not. Sorted version created at path=%s", - path, sortedPath) + path = "grafana/dashboards/" + path + t.Errorf("dashboard=%s should have sorted keys but does not. Sorted version created at path=%s. Run cp %s %s", + path, sortedPath, sortedPath, path) } }) } @@ -1209,7 +1210,11 @@ func writeSorted(t *testing.T, path string, sorted []byte) string { t.Errorf("failed to write sorted json to file=%s err=%v", dest, err) return "" } - create.Close() + err = create.Close() + if err != nil { + t.Errorf("failed to close file=%s err=%v", dest, err) + return "" + } return dest } From 9ff68687775fba3111ffb97bd88cc2c7151afe78 Mon Sep 17 00:00:00 2001 From: Rahul Date: Fri, 11 Aug 2023 21:55:39 +0530 Subject: [PATCH 13/40] feat: add workload panels in workload dashboard (#2100) * feat: add workload panels in workload dashboard * feat: add workload panels in workload dashboard * feat: workload dashboard * feat: workload dashboard * feat: workload dashboard * feat: workload dashboard * feat: workload dashboard changes * feat: workload dashboard changes * feat: make workload_class configurable * feat: add workload panels * feat: add workload panels * feat: add workload panels * feat: add workload panels * feat: add workload panels * feat: add workload panels * feat: add workload panels * feat: add service center view * feat: add service center view * feat: add service center view * feat: add service center view * feat: add comments for detail workload * feat: add comments for detail workload * feat: address review comments * feat: address review comments * feat: address review comments * feat: address review comments * feat: remove workload class from templates --------- Co-authored-by: Chris Grindstaff --- cmd/collectors/restperf/restperf.go | 190 +- cmd/collectors/zapiperf/zapiperf.go | 176 +- cmd/tools/generate/counter.yaml | 14 - conf/rest/9.12.0/qos_workload.yaml | 15 + conf/rest/default.yaml | 1 + conf/restperf/9.12.0/workload.yaml | 2 + conf/restperf/9.12.0/workload_detail.yaml | 1 + .../9.12.0/workload_detail_volume.yaml | 3 +- conf/restperf/9.12.0/workload_volume.yaml | 4 +- conf/restperf/default.yaml | 6 +- conf/zapi/cdot/9.8.0/qos_workload.yaml | 18 + conf/zapi/default.yaml | 1 + conf/zapiperf/cdot/9.8.0/workload.yaml | 1 + conf/zapiperf/cdot/9.8.0/workload_detail.yaml | 1 + .../cdot/9.8.0/workload_detail_volume.yaml | 3 +- conf/zapiperf/cdot/9.8.0/workload_volume.yaml | 4 +- conf/zapiperf/default.yaml | 12 +- grafana/dashboards/cmode/lun.json | 66 +- grafana/dashboards/cmode/volume.json | 114 +- grafana/dashboards/cmode/workload.json | 6020 ++++++++++++++--- integration/test/dashboard_json_test.go | 3 +- 21 files changed, 5369 insertions(+), 1286 deletions(-) create mode 100644 conf/rest/9.12.0/qos_workload.yaml create mode 100644 conf/zapi/cdot/9.8.0/qos_workload.yaml diff --git a/cmd/collectors/restperf/restperf.go b/cmd/collectors/restperf/restperf.go index 3e63571f1..121140c90 100644 --- a/cmd/collectors/restperf/restperf.go +++ b/cmd/collectors/restperf/restperf.go @@ -19,6 +19,7 @@ import ( "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/set" + "github.com/netapp/harvest/v2/pkg/tree/node" "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" "path" @@ -28,9 +29,11 @@ import ( ) const ( - latencyIoReqd = 10 - BILLION = 1_000_000_000 - arrayKeyToken = "#" + latencyIoReqd = 10 + BILLION = 1_000_000_000 + arrayKeyToken = "#" + objWorkloadClass = "user_defined|system_defined" + objWorkloadVolumeClass = "autovolume" ) var qosQuery = "api/cluster/counter/tables/qos" @@ -39,6 +42,8 @@ var qosDetailQuery = "api/cluster/counter/tables/qos_detail" var qosDetailVolumeQuery = "api/cluster/counter/tables/qos_detail_volume" var qosWorkloadQuery = "api/storage/qos/workloads" +var workloadDetailMetrics = []string{"resource_latency", "service_time_latency"} + var qosQueries = map[string]string{ qosQuery: qosQuery, qosVolumeQuery: qosVolumeQuery, @@ -171,6 +176,36 @@ func (r *RestPerf) InitMatrix() error { return nil } +// load workload_class or use defaultValue +func (r *RestPerf) loadWorkloadClassQuery(defaultValue string) string { + + var x *node.Node + + name := "workload_class" + + if x = r.Params.GetChildS(name); x != nil { + v := x.GetAllChildContentS() + if len(v) == 0 { + r.Logger.Debug(). + Str("name", name). + Str("defaultValue", defaultValue). + Send() + return defaultValue + } + s := strings.Join(v, "|") + r.Logger.Debug(). + Str("name", name). + Str("value", s). + Send() + return s + } + r.Logger.Debug(). + Str("name", name). + Str("defaultValue", defaultValue). + Send() + return defaultValue +} + // load an int parameter or use defaultValue func (r *RestPerf) loadParamInt(name string, defaultValue int) int { @@ -549,25 +584,27 @@ func (r *RestPerf) processWorkLoadCounter() (map[string]*matrix.Matrix, error) { return nil, errs.New(errs.ErrMissingParam, "resource_map") } else { for _, x := range resourceMap.GetChildren() { - name := x.GetNameS() - resource := x.GetContentS() + for _, wm := range workloadDetailMetrics { + name := x.GetNameS() + wm + resource := x.GetContentS() - if m := mat.GetMetric(name); m != nil { - continue - } - if m, err := mat.NewMetricFloat64(name, "resource_latency"); err != nil { - return nil, err - } else { - r.perfProp.counterInfo[name] = &counter{ - name: "resource_latency", - description: "", - counterType: r.perfProp.counterInfo[service.GetName()].counterType, - unit: r.perfProp.counterInfo[service.GetName()].unit, - denominator: "ops", + if m := mat.GetMetric(name); m != nil { + continue } - m.SetLabel("resource", resource) + if m, err := mat.NewMetricFloat64(name, wm); err != nil { + return nil, err + } else { + r.perfProp.counterInfo[name] = &counter{ + name: wm, + description: "", + counterType: r.perfProp.counterInfo[service.GetName()].counterType, + unit: r.perfProp.counterInfo[service.GetName()].unit, + denominator: "ops", + } + m.SetLabel("resource", resource) - r.Logger.Debug().Str("name", name).Str("resource", resource).Msg("added workload latency metric") + r.Logger.Debug().Str("name", name).Str("resource", resource).Msg("added workload latency metric") + } } } } @@ -611,16 +648,15 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) { func (r *RestPerf) pollData(startTime time.Time, perfRecords []rest.PerfRecord) (map[string]*matrix.Matrix, error) { var ( - count, numRecords uint64 - apiD, parseD time.Duration - err error - instanceKeys []string - resourceLatency *matrix.Metric // for workload* objects - skips int - instIndex int - ts float64 - prevMat *matrix.Matrix - curMat *matrix.Matrix + count uint64 + apiD, parseD time.Duration + err error + instanceKeys []string + skips int + instIndex int + ts float64 + prevMat *matrix.Matrix + curMat *matrix.Matrix ) prevMat = r.Matrix[r.Object] @@ -683,12 +719,12 @@ func (r *RestPerf) pollData(startTime time.Time, perfRecords []rest.PerfRecord) } } + var layer = "" // latency layer (resource) for workloads + // special case for these two objects // we need to process each latency layer for each instance/counter if isWorkloadDetailObject(r.Prop.Query) { - layer := "" // latency layer (resource) for workloads - // example instanceKey : umeng-aff300-02:test-wid12022.CPU_dblade i := strings.Index(instanceKey, ":") instanceKey = instanceKey[i+1:] @@ -703,11 +739,14 @@ func (r *RestPerf) pollData(startTime time.Time, perfRecords []rest.PerfRecord) return true } - if resourceLatency = curMat.GetMetric(layer); resourceLatency == nil { - r.Logger.Trace(). - Str("layer", layer). - Msg("Resource-latency metric missing in cache") - return true + for _, wm := range workloadDetailMetrics { + mLayer := layer + wm + if l := curMat.GetMetric(mLayer); l == nil { + r.Logger.Trace(). + Str("layer", layer). + Msg("Resource-latency metric missing in cache") + return true + } } } @@ -770,27 +809,61 @@ func (r *RestPerf) pollData(startTime time.Time, perfRecords []rest.PerfRecord) if ok { // special case for workload_detail if isWorkloadDetailObject(r.Prop.Query) { - if name == "wait_time" || name == "service_time" { - if err := resourceLatency.AddValueString(instance, f.value); err != nil { - r.Logger.Error(). - Stack(). - Err(err). - Str("name", name). - Str("value", f.value). - Msg("Add resource-latency failed") - } else { - r.Logger.Trace(). - Str("name", name). - Str("value", f.value). - Msg("Add resource-latency") - count++ + for _, wm := range workloadDetailMetrics { + // "visits" are ignored. This counter is only used to set properties of ops counter + if name == "visits" { + continue + } + wMetric := curMat.GetMetric(layer + wm) + if wm == "resource_latency" && (name == "wait_time" || name == "service_time") { + if err := wMetric.AddValueString(instance, f.value); err != nil { + r.Logger.Error(). + Stack(). + Err(err). + Str("name", name). + Str("value", f.value). + Msg("Add resource_latency failed") + } else { + r.Logger.Trace(). + Str("name", name). + Str("value", f.value). + Msg("Add resource_latency") + count++ + } + continue + } else if wm == "service_time_latency" && name == "service_time" { + if err = wMetric.SetValueString(instance, f.value); err != nil { + r.Logger.Error(). + Stack(). + Err(err). + Str("name", name). + Str("value", f.value). + Msg("Add service_time_latency failed") + } else { + r.Logger.Trace(). + Str("name", name). + Str("value", f.value). + Msg("Add service_time_latency") + count++ + } + } else if wm == "wait_time_latency" && name == "wait_time" { + if err = wMetric.SetValueString(instance, f.value); err != nil { + r.Logger.Error(). + Stack(). + Err(err). + Str("name", name). + Str("value", f.value). + Msg("Add wait_time_latency failed") + } else { + r.Logger.Trace(). + Str("name", name). + Str("value", f.value). + Msg("Add wait_time_latency") + count++ + } } - continue - } - // "visits" are ignored. This counter is only used to set properties of ops counter - if name == "visits" { - continue } + continue } else { if f.isArray { labels := strings.Split(f.label, ",") @@ -917,7 +990,6 @@ func (r *RestPerf) pollData(startTime time.Time, perfRecords []rest.PerfRecord) r.Logger.Error().Err(err).Msg("Failed to set timestamp") } - numRecords += 1 return true }) } @@ -933,7 +1005,7 @@ func (r *RestPerf) pollData(startTime time.Time, perfRecords []rest.PerfRecord) _ = r.Metadata.LazySetValueInt64("api_time", "data", apiD.Microseconds()) _ = r.Metadata.LazySetValueInt64("parse_time", "data", parseD.Microseconds()) _ = r.Metadata.LazySetValueUint64("metrics", "data", count) - _ = r.Metadata.LazySetValueUint64("instances", "data", numRecords) + _ = r.Metadata.LazySetValueUint64("instances", "data", uint64(len(curMat.GetInstances()))) r.AddCollectCount(count) // skip calculating from delta if no data from previous poll @@ -1255,9 +1327,9 @@ func (r *RestPerf) PollInstance() (map[string]*matrix.Matrix, error) { fields = "*" dataQuery = qosWorkloadQuery if r.Prop.Query == qosVolumeQuery || r.Prop.Query == qosDetailVolumeQuery { - filter = append(filter, "workload-class=autovolume|user_defined|system_defined") + filter = append(filter, "workload_class="+r.loadWorkloadClassQuery(objWorkloadVolumeClass)) } else { - filter = append(filter, "workload-class=user_defined|system_defined") + filter = append(filter, "workload_class="+r.loadWorkloadClassQuery(objWorkloadClass)) } } diff --git a/cmd/collectors/zapiperf/zapiperf.go b/cmd/collectors/zapiperf/zapiperf.go index 3585a4325..35bf4926e 100644 --- a/cmd/collectors/zapiperf/zapiperf.go +++ b/cmd/collectors/zapiperf/zapiperf.go @@ -60,9 +60,13 @@ const ( objWorkloadDetail = "workload_detail" objWorkloadVolume = "workload_volume" objWorkloadDetailVolume = "workload_detail_volume" + objWorkloadClass = "user_defined|system_defined" + objWorkloadVolumeClass = "autovolume" BILLION = 1_000_000_000 ) +var workloadDetailMetrics = []string{"resource_latency", "service_time_latency"} + type ZapiPerf struct { *zapi.Zapi // provides: AbstractCollector, Client, Object, Query, TemplateFn, TemplateType object string @@ -173,6 +177,36 @@ func (z *ZapiPerf) loadParamStr(name, defaultValue string) string { return defaultValue } +// load workload_class or use defaultValue +func (z *ZapiPerf) loadWorkloadClassQuery(defaultValue string) string { + + var x *node.Node + + name := "workload_class" + + if x = z.Params.GetChildS(name); x != nil { + v := x.GetAllChildContentS() + if len(v) == 0 { + z.Logger.Debug(). + Str("name", name). + Str("defaultValue", defaultValue). + Send() + return defaultValue + } + s := strings.Join(v, "|") + z.Logger.Debug(). + Str("name", name). + Str("value", s). + Send() + return s + } + z.Logger.Debug(). + Str("name", name). + Str("defaultValue", defaultValue). + Send() + return defaultValue +} + // load an int parameter or use defaultValue func (z *ZapiPerf) loadParamInt(name string, defaultValue int) int { @@ -199,10 +233,9 @@ func (z *ZapiPerf) loadParamInt(name string, defaultValue int) int { func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) { var ( - instanceKeys []string - resourceLatency *matrix.Metric // for workload* objects - err error - skips int + instanceKeys []string + err error + skips int ) z.Logger.Trace().Msg("updating data cache") @@ -338,12 +371,12 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) { key := i.GetChildContentS(z.instanceKey) + var layer = "" // latency layer (resource) for workloads + // special case for these two objects // we need to process each latency layer for each instance/counter if z.Query == objWorkloadDetail || z.Query == objWorkloadDetailVolume { - layer := "" // latency layer (resource) for workloads - if x := strings.Split(key, "."); len(x) == 2 { key = x[0] layer = x[1] @@ -354,11 +387,14 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) { continue } - if resourceLatency = curMat.GetMetric(layer); resourceLatency == nil { - z.Logger.Warn(). - Str("layer", layer). - Msg("Resource-latency metric missing in cache") - continue + for _, wm := range workloadDetailMetrics { + mLayer := layer + wm + if l := curMat.GetMetric(mLayer); l == nil { + z.Logger.Warn(). + Str("layer", mLayer). + Msg("metric missing in cache") + continue + } } } @@ -475,29 +511,68 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) { // special case for workload_detail if z.Query == objWorkloadDetail || z.Query == objWorkloadDetailVolume { - if name == "wait_time" || name == "service_time" { - if err := resourceLatency.AddValueString(instance, value); err != nil { - z.Logger.Error(). - Stack(). - Err(err). - Str("name", name). - Str("value", value). - Int("instIndex", instIndex). - Msg("Add resource-latency failed") - } else { - z.Logger.Trace(). - Str("name", name). - Str("value", value). - Int("instIndex", instIndex). - Msg("Add resource-latency") - count++ + for _, wm := range workloadDetailMetrics { + // "visits" are ignored + if name == "visits" { + continue + } + + wMetric := curMat.GetMetric(layer + wm) + + if wm == "resource_latency" && (name == "wait_time" || name == "service_time") { + if err := wMetric.AddValueString(instance, value); err != nil { + z.Logger.Error(). + Err(err). + Str("name", name). + Str("value", value). + Int("instIndex", instIndex). + Msg("Add resource_latency failed") + } else { + z.Logger.Trace(). + Str("name", name). + Str("value", value). + Int("instIndex", instIndex). + Msg("Add resource_latency") + count++ + } + continue + } else if wm == "service_time_latency" && name == "service_time" { + if err = wMetric.SetValueString(instance, value); err != nil { + z.Logger.Error(). + Err(err). + Str("name", name). + Str("value", value). + Int("instIndex", instIndex). + Msg("Add service_time_latency failed") + } else { + z.Logger.Trace(). + Int("instIndex", instIndex). + Str("name", name). + Str("value", value). + Int("instIndex", instIndex). + Msg("Add service_time_latency") + count++ + } + } else if wm == "wait_time_latency" && name == "wait_time" { + if err = wMetric.SetValueString(instance, value); err != nil { + z.Logger.Error(). + Err(err). + Str("name", name). + Str("value", value). + Int("instIndex", instIndex). + Msg("Add wait_time_latency failed") + } else { + z.Logger.Trace(). + Int("instIndex", instIndex). + Str("name", name). + Str("value", value). + Int("instIndex", instIndex). + Msg("Add wait_time_latency") + count++ + } } - continue - } - // "visits" are ignored - if name == "visits" { - continue } + continue } // store as scalar metric @@ -1038,23 +1113,26 @@ func (z *ZapiPerf) PollCounter() (map[string]*matrix.Matrix, error) { return nil, errs.New(errs.ErrMissingParam, "resource_map") } else { for _, x := range resourceMap.GetChildren() { - name := x.GetNameS() - resource := x.GetContentS() + for _, wm := range workloadDetailMetrics { - if m := mat.GetMetric(name); m != nil { - oldMetrics.Remove(name) - continue - } - if m, err := mat.NewMetricFloat64(name, "resource_latency"); err != nil { - return nil, err - } else { - m.SetLabel("resource", resource) - m.SetProperty(service.GetProperty()) - // base counter is the ops of the same resource - m.SetComment("ops") + name := x.GetNameS() + wm + resource := x.GetContentS() - oldMetrics.Remove(name) - z.Logger.Debug().Msgf("+ [%s] (=> %s) added workload latency metric", name, resource) + if m := mat.GetMetric(name); m != nil { + oldMetrics.Remove(name) + continue + } + if m, err := mat.NewMetricFloat64(name, wm); err != nil { + return nil, err + } else { + m.SetLabel("resource", resource) + m.SetProperty(service.GetProperty()) + // base counter is the ops of the same resource + m.SetComment("ops") + + oldMetrics.Remove(name) + z.Logger.Debug().Msgf("+ [%s] (=> %s) added workload latency metric", name, resource) + } } } } @@ -1323,9 +1401,9 @@ func (z *ZapiPerf) PollInstance() (map[string]*matrix.Matrix, error) { queryElem := request.NewChildS("query", "") infoElem := queryElem.NewChildS("qos-workload-info", "") if z.Query == objWorkloadVolume || z.Query == objWorkloadDetailVolume { - infoElem.NewChildS("workload-class", "autovolume|user_defined|system_defined") + infoElem.NewChildS("workload-class", z.loadWorkloadClassQuery(objWorkloadVolumeClass)) } else { - infoElem.NewChildS("workload-class", "user_defined|system_defined") + infoElem.NewChildS("workload-class", z.loadWorkloadClassQuery(objWorkloadClass)) } instancesAttr = "attributes-list" diff --git a/cmd/tools/generate/counter.yaml b/cmd/tools/generate/counter.yaml index 1be133f45..f11369684 100644 --- a/cmd/tools/generate/counter.yaml +++ b/cmd/tools/generate/counter.yaml @@ -487,20 +487,6 @@ counters: Template: conf/zapiperf/9.12.0/workload_detail.yaml Unit: microseconds - - Name: qos_detail_volume_resource_latency - Description: average latency for volume on Data ONTAP subsystems - APIs: - - API: REST - Endpoint: api/cluster/counter/tables/qos_detail_volume - ONTAPCounter: Harvest generated - Template: conf/restperf/9.12.0/workload_detail_volume.yaml - Unit: microseconds - - API: ZAPI - Endpoint: perf-object-get-instances workload_detail_volume - ONTAPCounter: Harvest generated - Template: conf/zapiperf/9.12.0/workload_detail_volume.yaml - Unit: microseconds - - Name: quota_disk_limit Description: Maximum amount of disk space, in kilobytes, allowed for the quota target (hard disk space limit). The value is -1 if the limit is unlimited. diff --git a/conf/rest/9.12.0/qos_workload.yaml b/conf/rest/9.12.0/qos_workload.yaml new file mode 100644 index 000000000..c4d2ca723 --- /dev/null +++ b/conf/rest/9.12.0/qos_workload.yaml @@ -0,0 +1,15 @@ +name: QosWorkload +query: api/storage/qos/workloads +object: qos_workload + +counters: + - ^^uuid => uuid + - ^name => workload + - ^workload_class => class + +export_options: + instance_keys: + - uuid + instance_labels: + - class + - workload \ No newline at end of file diff --git a/conf/rest/default.yaml b/conf/rest/default.yaml index 05c758b9f..22b1b9d74 100644 --- a/conf/rest/default.yaml +++ b/conf/rest/default.yaml @@ -29,6 +29,7 @@ objects: OntapS3Policy: ontap_s3_policy.yaml QosPolicyAdaptive: qos_policy_adaptive.yaml QosPolicyFixed: qos_policy_fixed.yaml + QosWorkload: qos_workload.yaml Qtree: qtree.yaml Security: security.yaml SecurityAccount: security_account.yaml diff --git a/conf/restperf/9.12.0/workload.yaml b/conf/restperf/9.12.0/workload.yaml index ba598908d..92587264a 100644 --- a/conf/restperf/9.12.0/workload.yaml +++ b/conf/restperf/9.12.0/workload.yaml @@ -6,6 +6,7 @@ object: qos # recommended to use large interval, since workload objects are expensive client_timeout: 1m30s + schedule: - counter: 1200s - instance: 600s @@ -17,6 +18,7 @@ counters: - concurrency - latency - ops + - other_ops - read_data - read_io_type_percent => read_io_type - read_latency diff --git a/conf/restperf/9.12.0/workload_detail.yaml b/conf/restperf/9.12.0/workload_detail.yaml index 104d5898e..bc1faeade 100644 --- a/conf/restperf/9.12.0/workload_detail.yaml +++ b/conf/restperf/9.12.0/workload_detail.yaml @@ -7,6 +7,7 @@ object: qos_detail # recommended to use large interval, since workload objects are expensive client_timeout: 1m30s + schedule: - counter: 1200s - instance: 600s diff --git a/conf/restperf/9.12.0/workload_detail_volume.yaml b/conf/restperf/9.12.0/workload_detail_volume.yaml index 7ab7ac16d..a5c57ce1f 100644 --- a/conf/restperf/9.12.0/workload_detail_volume.yaml +++ b/conf/restperf/9.12.0/workload_detail_volume.yaml @@ -2,10 +2,11 @@ # object provides latency breakdown per service or delay center per volume name: WorkloadDetailVolume query: api/cluster/counter/tables/qos_detail_volume -object: qos_detail_volume +object: qos_detail # recommended to use large interval, since workload objects are expensive client_timeout: 1m30s + schedule: - counter: 1200s - instance: 600s diff --git a/conf/restperf/9.12.0/workload_volume.yaml b/conf/restperf/9.12.0/workload_volume.yaml index e06f54afa..f628bb359 100644 --- a/conf/restperf/9.12.0/workload_volume.yaml +++ b/conf/restperf/9.12.0/workload_volume.yaml @@ -4,10 +4,11 @@ name: WorkloadVolume query: api/cluster/counter/tables/qos_volume -object: qos_volume +object: qos # recommended to use large interval, since workload objects are expensive client_timeout: 1m30s + schedule: - counter: 1200s - instance: 600s @@ -15,6 +16,7 @@ schedule: counters: - ^^uuid + - concurrency - latency - ops - read_data diff --git a/conf/restperf/default.yaml b/conf/restperf/default.yaml index 58ebc6f0c..b3d59f1eb 100644 --- a/conf/restperf/default.yaml +++ b/conf/restperf/default.yaml @@ -52,8 +52,10 @@ objects: Vscan: vscan.yaml VscanSVM: vscan_svm.yaml -# Uncomment to collect workload/QOS counters. They are disabled by default because they typically slow down data collection due to a high number of metrics. +# Uncomment to collect workload/QOS counters. # Workload: workload.yaml -# WorkloadDetail: workload_detail.yaml # WorkloadVolume: workload_volume.yaml + +# The following workload templates may slow down data collection due to a high number of metrics. +# WorkloadDetail: workload_detail.yaml # WorkloadDetailVolume: workload_detail_volume.yaml \ No newline at end of file diff --git a/conf/zapi/cdot/9.8.0/qos_workload.yaml b/conf/zapi/cdot/9.8.0/qos_workload.yaml new file mode 100644 index 000000000..e003c068e --- /dev/null +++ b/conf/zapi/cdot/9.8.0/qos_workload.yaml @@ -0,0 +1,18 @@ +name: QosWorkload +query: qos-workload-get-iter +object: qos_workload + +counters: + qos-workload-info: + - ^^workload-uuid => uuid + - ^workload-class => class + - ^workload-name => workload + +collect_only_labels: true + +export_options: + instance_keys: + - uuid + instance_labels: + - class + - workload \ No newline at end of file diff --git a/conf/zapi/default.yaml b/conf/zapi/default.yaml index caf17fa86..0477410f1 100644 --- a/conf/zapi/default.yaml +++ b/conf/zapi/default.yaml @@ -20,6 +20,7 @@ objects: NtpServer: ntpserver.yaml QosPolicyAdaptive: qos_policy_adaptive.yaml QosPolicyFixed: qos_policy_fixed.yaml + QosWorkload: qos_workload.yaml Qtree: qtree.yaml Security: security.yaml SecurityAccount: security_account.yaml diff --git a/conf/zapiperf/cdot/9.8.0/workload.yaml b/conf/zapiperf/cdot/9.8.0/workload.yaml index 037e12cb2..d561a725a 100644 --- a/conf/zapiperf/cdot/9.8.0/workload.yaml +++ b/conf/zapiperf/cdot/9.8.0/workload.yaml @@ -20,6 +20,7 @@ counters: - instance_uuid - latency - ops + - other_ops - read_data - read_io_type - read_latency diff --git a/conf/zapiperf/cdot/9.8.0/workload_detail.yaml b/conf/zapiperf/cdot/9.8.0/workload_detail.yaml index c7fcd28dd..f79d4188b 100644 --- a/conf/zapiperf/cdot/9.8.0/workload_detail.yaml +++ b/conf/zapiperf/cdot/9.8.0/workload_detail.yaml @@ -9,6 +9,7 @@ instance_key: name # recommended to use a large interval, since workload objects are expensive client_timeout: 1m30s + schedule: - counter: 1200s - instance: 600s diff --git a/conf/zapiperf/cdot/9.8.0/workload_detail_volume.yaml b/conf/zapiperf/cdot/9.8.0/workload_detail_volume.yaml index 081991924..8482b3c27 100644 --- a/conf/zapiperf/cdot/9.8.0/workload_detail_volume.yaml +++ b/conf/zapiperf/cdot/9.8.0/workload_detail_volume.yaml @@ -2,12 +2,13 @@ # object provides latency breakdown per service or delay center per volume name: WorkloadDetailVolume query: workload_detail_volume -object: qos_detail_volume +object: qos_detail instance_key: name # recommended to use a large interval, since workload objects are expensive client_timeout: 1m30s + schedule: - counter: 1200s - instance: 600s diff --git a/conf/zapiperf/cdot/9.8.0/workload_volume.yaml b/conf/zapiperf/cdot/9.8.0/workload_volume.yaml index 4291e041c..e26f938d3 100644 --- a/conf/zapiperf/cdot/9.8.0/workload_volume.yaml +++ b/conf/zapiperf/cdot/9.8.0/workload_volume.yaml @@ -4,7 +4,7 @@ name: WorkloadVolume query: workload_volume -object: qos_volume +object: qos # recommended to use large interval, since workload objects are expensive client_timeout: 1m30s @@ -16,10 +16,12 @@ schedule: instance_key: name counters: + - concurrency - instance_name - instance_uuid - latency - ops + - other_ops - read_data - read_io_type - read_latency diff --git a/conf/zapiperf/default.yaml b/conf/zapiperf/default.yaml index ae68ea82f..8eaad696b 100644 --- a/conf/zapiperf/default.yaml +++ b/conf/zapiperf/default.yaml @@ -59,8 +59,10 @@ objects: Vscan: vscan.yaml VscanSVM: vscan_svm.yaml -# Uncomment to collect workload/QOS counters. They are disabled by default because they typically slow down data collection due to a high number of metrics. -# Workload: workload.yaml -# WorkloadDetail: workload_detail.yaml -# WorkloadVolume: workload_volume.yaml -# WorkloadDetailVolume: workload_detail_volume.yaml \ No newline at end of file +# Uncomment to collect workload/QOS counters. +# Workload: workload.yaml +# WorkloadVolume: workload_volume.yaml + +# The following workload templates may slow down data collection due to a high number of metrics. +# WorkloadDetail: workload_detail.yaml +# WorkloadDetailVolume: workload_detail_volume.yaml \ No newline at end of file diff --git a/grafana/dashboards/cmode/lun.json b/grafana/dashboards/cmode/lun.json index 5d08e5a5f..896335bc6 100644 --- a/grafana/dashboards/cmode/lun.json +++ b/grafana/dashboards/cmode/lun.json @@ -2409,7 +2409,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyNetwork\",resource=\"network\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyNetwork\",resource=\"network\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -2500,7 +2500,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyThrottle\",resource=\"throttle\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyThrottle\",resource=\"throttle\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -2591,7 +2591,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyMin\",resource=\"qos_min\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyMin\",resource=\"qos_min\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -2681,7 +2681,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyCluster\",resource=\"cluster\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyCluster\",resource=\"cluster\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -2771,7 +2771,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyBackend\",resource=\"backend\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyBackend\",resource=\"backend\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -2862,7 +2862,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyCP\",resource=\"cp\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyCP\",resource=\"cp\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -2953,7 +2953,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencySuspend\",resource=\"suspend\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencySuspend\",resource=\"suspend\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}} ", "refId": "A" @@ -3044,7 +3044,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyCloud\",resource=\"cloud\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyCloud\",resource=\"cloud\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}} ", "refId": "A" @@ -3134,7 +3134,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyFrontend\",resource=\"frontend\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyFrontend\",resource=\"frontend\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -3224,7 +3224,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyNvlog\",resource=\"nvlog\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyNvlog\",resource=\"nvlog\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}} ", "refId": "A" @@ -3314,7 +3314,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyDisk\",resource=\"disk\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopQosDetailVolumeResourceLatencyDisk\",resource=\"disk\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -5107,7 +5107,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"network\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"network\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -5117,7 +5117,7 @@ "name": "TopQosDetailVolumeResourceLatencyNetwork", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"network\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"network\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -5130,7 +5130,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"throttle\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"throttle\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -5140,7 +5140,7 @@ "name": "TopQosDetailVolumeResourceLatencyThrottle", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"throttle\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"throttle\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -5153,7 +5153,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"frontend\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"frontend\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -5163,7 +5163,7 @@ "name": "TopQosDetailVolumeResourceLatencyFrontend", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"frontend\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"frontend\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -5176,7 +5176,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cluster\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cluster\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -5186,7 +5186,7 @@ "name": "TopQosDetailVolumeResourceLatencyCluster", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cluster\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cluster\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -5199,7 +5199,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"backend\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"backend\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -5209,7 +5209,7 @@ "name": "TopQosDetailVolumeResourceLatencyBackend", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"backend\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"backend\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -5222,7 +5222,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"disk\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"disk\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -5232,7 +5232,7 @@ "name": "TopQosDetailVolumeResourceLatencyDisk", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"disk\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"disk\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -5245,7 +5245,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"suspend\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"suspend\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -5255,7 +5255,7 @@ "name": "TopQosDetailVolumeResourceLatencySuspend", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"suspend\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"suspend\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -5268,7 +5268,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cloud\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cloud\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -5278,7 +5278,7 @@ "name": "TopQosDetailVolumeResourceLatencyCloud", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cloud\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cloud\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -5291,7 +5291,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"nvlog\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"nvlog\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -5301,7 +5301,7 @@ "name": "TopQosDetailVolumeResourceLatencyNvlog", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"nvlog\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"nvlog\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -5314,7 +5314,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"qos_min\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"qos_min\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -5324,7 +5324,7 @@ "name": "TopQosDetailVolumeResourceLatencyMin", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"qos_min\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"qos_min\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -5337,7 +5337,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cp\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cp\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -5347,7 +5347,7 @@ "name": "TopQosDetailVolumeResourceLatencyCP", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cp\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cp\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, diff --git a/grafana/dashboards/cmode/volume.json b/grafana/dashboards/cmode/volume.json index cb98c96a4..29d209b28 100644 --- a/grafana/dashboards/cmode/volume.json +++ b/grafana/dashboards/cmode/volume.json @@ -2619,7 +2619,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_volume_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSReadLatency\"})", + "expr": "topk($TopResources, qos_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSReadLatency\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -2709,7 +2709,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_volume_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSWriteLatency\"})", + "expr": "topk($TopResources, qos_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSWriteLatency\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -2799,7 +2799,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_volume_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSReadOps\"})", + "expr": "topk($TopResources, qos_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSReadOps\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -2889,7 +2889,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_volume_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSWriteOps\"})", + "expr": "topk($TopResources, qos_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSWriteOps\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -2979,7 +2979,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSReadData\"})", + "expr": "topk($TopResources, qos_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSReadData\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -3069,7 +3069,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSWriteData\"})", + "expr": "topk($TopResources, qos_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSWriteData\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -3160,7 +3160,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_volume_sequential_reads{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSSequentialReads\"})", + "expr": "topk($TopResources, qos_sequential_reads{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSSequentialReads\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -3251,7 +3251,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_volume_sequential_writes{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSSequentialWrites\"})", + "expr": "topk($TopResources, qos_sequential_writes{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSSequentialWrites\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -3373,7 +3373,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSNetwork\",resource=\"network\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSNetwork\",resource=\"network\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -3464,7 +3464,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSThrottle\",resource=\"throttle\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSThrottle\",resource=\"throttle\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -3555,7 +3555,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSMin\",resource=\"qos_min\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSMin\",resource=\"qos_min\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -3645,7 +3645,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSCluster\",resource=\"cluster\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSCluster\",resource=\"cluster\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -3735,7 +3735,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSBackend\",resource=\"backend\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSBackend\",resource=\"backend\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -3826,7 +3826,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSCP\",resource=\"cp\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSCP\",resource=\"cp\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -3917,7 +3917,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSSuspend\",resource=\"suspend\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSSuspend\",resource=\"suspend\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}} ", "refId": "A" @@ -4008,7 +4008,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSCloud\",resource=\"cloud\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSCloud\",resource=\"cloud\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}} ", "refId": "A" @@ -4098,7 +4098,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSfrontend\",resource=\"frontend\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSfrontend\",resource=\"frontend\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -4188,7 +4188,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSNvlog\",resource=\"nvlog\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSNvlog\",resource=\"nvlog\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}} ", "refId": "A" @@ -4278,7 +4278,7 @@ "targets": [ { "exemplar": false, - "expr": "topk($TopResources, qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSDisk\",resource=\"disk\"})", + "expr": "topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeQOSDisk\",resource=\"disk\",volume!=\"\"})", "interval": "", "legendFormat": "{{svm}} / {{workload}} / {{volume}}", "refId": "A" @@ -6253,7 +6253,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"frontend\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"frontend\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6263,7 +6263,7 @@ "name": "TopVolumeQOSfrontend", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"frontend\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"frontend\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6276,7 +6276,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"network\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"network\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6286,7 +6286,7 @@ "name": "TopVolumeQOSNetwork", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"network\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"network\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6299,7 +6299,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cluster\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cluster\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6309,7 +6309,7 @@ "name": "TopVolumeQOSCluster", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cluster\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cluster\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6322,7 +6322,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"disk\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"disk\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6332,7 +6332,7 @@ "name": "TopVolumeQOSDisk", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"disk\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"disk\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6345,7 +6345,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"suspend\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"suspend\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6355,7 +6355,7 @@ "name": "TopVolumeQOSSuspend", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"suspend\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"suspend\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6368,7 +6368,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cloud\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cloud\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6378,7 +6378,7 @@ "name": "TopVolumeQOSCloud", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cloud\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cloud\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6391,7 +6391,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"nvlog\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"nvlog\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6401,7 +6401,7 @@ "name": "TopVolumeQOSNvlog", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"nvlog\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"nvlog\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6414,7 +6414,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"throttle\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"throttle\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6424,7 +6424,7 @@ "name": "TopVolumeQOSThrottle", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"throttle\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"throttle\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6437,7 +6437,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"backend\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"backend\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6447,7 +6447,7 @@ "name": "TopVolumeQOSBackend", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"backend\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"backend\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6460,7 +6460,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_volume_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6470,7 +6470,7 @@ "name": "TopVolumeQOSReadLatency", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_volume_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6483,7 +6483,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6493,7 +6493,7 @@ "name": "TopVolumeQOSReadData", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6506,7 +6506,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_volume_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6516,7 +6516,7 @@ "name": "TopVolumeQOSReadOps", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_volume_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6529,7 +6529,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_volume_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6539,7 +6539,7 @@ "name": "TopVolumeQOSWriteLatency", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_volume_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6552,7 +6552,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6562,7 +6562,7 @@ "name": "TopVolumeQOSWriteData", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6575,7 +6575,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_volume_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6585,7 +6585,7 @@ "name": "TopVolumeQOSWriteOps", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_volume_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6598,7 +6598,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_volume_sequential_reads{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_sequential_reads{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6608,7 +6608,7 @@ "name": "TopVolumeQOSSequentialReads", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_volume_sequential_reads{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_sequential_reads{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6621,7 +6621,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_volume_sequential_writes{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_sequential_writes{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6631,7 +6631,7 @@ "name": "TopVolumeQOSSequentialWrites", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_volume_sequential_writes{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_sequential_writes{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6644,7 +6644,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"qos_min\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"qos_min\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6654,7 +6654,7 @@ "name": "TopVolumeQOSMin", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"qos_min\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"qos_min\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -6667,7 +6667,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cp\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cp\",volume!=\"\"}[${__range}])))", "description": null, "error": null, "hide": 2, @@ -6677,7 +6677,7 @@ "name": "TopVolumeQOSCP", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cp\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\",resource=\"cp\",volume!=\"\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, diff --git a/grafana/dashboards/cmode/workload.json b/grafana/dashboards/cmode/workload.json index b10bb54f5..df590395d 100644 --- a/grafana/dashboards/cmode/workload.json +++ b/grafana/dashboards/cmode/workload.json @@ -59,7 +59,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1679501540219, + "iteration": 1690889588293, "links": [ { "asDropdown": true, @@ -94,14 +94,14 @@ { "datasource": "${DS_PROMETHEUS}", "gridPos": { - "h": 4, + "h": 6, "w": 24, "x": 0, "y": 1 }, "id": 107, "options": { - "content": "This dashboard requires Workload/QOS counter templates. They are disabled by default because they typically slow down data collection due to a high number of metrics. Two actions are required to use this dashboard:
\n1. If you are using the ZapiPerf collector, then you must enable the Workload/QOS counters in $HARVEST/conf/zapiperf/default.yaml.
\n2. If you are using the RestPerf Collector, then you must enable the Workload/QOS counters in $HARVEST/conf/restperf/default.yaml.
\n\nMore information about [ONTAP Performance](https://kb.netapp.com/Advice_and_Troubleshooting/Data_Storage_Software/ONTAP_OS/ONTAP_9_Performance_-_Resolution_Guide).", + "content": "This dashboard requires Workload/QOS counter templates. They are disabled by default because they typically slow down data collection due to a high number of metrics. Two actions are required to use this dashboard:
\n1. If you are using the ZapiPerf collector, then you must enable the Workload/QOS counters in $HARVEST/conf/zapiperf/default.yaml.
\n2. If you are using the RestPerf Collector, then you must enable the Workload/QOS counters in $HARVEST/conf/restperf/default.yaml.
\n3. To access the panels under `Highlights` and `Workload Read IO Type`, please enable the `workload.yaml` and `workload_volume.yaml` templates. If you're interested in viewing panels within the `Service Center` and `Latency Breakdown`, enable the `workload_detail.yaml` and `workload_detail_volume.yaml` templates.\n\n\nMore information about [ONTAP Performance](https://kb.netapp.com/Advice_and_Troubleshooting/Data_Storage_Software/ONTAP_OS/ONTAP_9_Performance_-_Resolution_Guide).", "mode": "markdown" }, "pluginVersion": "8.1.8", @@ -109,20 +109,21 @@ }, { "collapsed": false, - "datasource": "${DS_PROMETHEUS}", + "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 5 + "y": 7 }, - "id": 123, + "id": 161, "panels": [], - "title": "Latency", + "title": "Highlights", "type": "row" }, { "datasource": "${DS_PROMETHEUS}", + "description": "This is the rate of this workload's read operations that completed during the measurement interval.", "fieldConfig": { "defaults": { "color": { @@ -156,112 +157,8 @@ "mode": "off" } }, - "decimals": 2, "mappings": [], "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "µs" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 103, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.1.8", - "targets": [ - { - "exemplar": false, - "expr": "(avg(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\"}) by (resource))", - "interval": "", - "legendFormat": "{{resource}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Latency by Resources", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -271,7 +168,7 @@ } ] }, - "unit": "percent" + "unit": "iops" }, "overrides": [] }, @@ -279,9 +176,9 @@ "h": 10, "w": 12, "x": 0, - "y": 16 + "y": 8 }, - "id": 104, + "id": 163, "options": { "legend": { "calcs": [ @@ -300,8 +197,7 @@ "targets": [ { "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSBackend\",resource=\"backend\"}) by (workload)/ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSBackend\",resource=\"backend\"}))", - "instant": false, + "expr": "topk($TopResources, qos_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSReadOps\"})", "interval": "", "legendFormat": "{{workload}}", "refId": "A" @@ -309,11 +205,12 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources Workloads by Latency from backend", + "title": "Top $TopResources Workloads by Read IOPS", "type": "timeseries" }, { "datasource": "${DS_PROMETHEUS}", + "description": "This is the workload's write operations that completed during the measurement interval; measured per second.", "fieldConfig": { "defaults": { "color": { @@ -347,20 +244,7 @@ "mode": "off" } }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" - } - }, - "type": "special" - } - ], - "max": 100, + "mappings": [], "min": 0, "thresholds": { "mode": "absolute", @@ -371,7 +255,7 @@ } ] }, - "unit": "percent" + "unit": "iops" }, "overrides": [] }, @@ -379,9 +263,9 @@ "h": 10, "w": 12, "x": 12, - "y": 16 + "y": 8 }, - "id": 105, + "id": 164, "options": { "legend": { "calcs": [ @@ -400,8 +284,7 @@ "targets": [ { "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSFrontend\",resource=\"frontend\"}) by (workload)) / ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSFrontend\",resource=\"frontend\"})", - "instant": false, + "expr": "topk($TopResources, qos_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSWriteOps\"})", "interval": "", "legendFormat": "{{workload}}", "refId": "A" @@ -409,12 +292,12 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources Workloads by Latency from frontend", + "title": "Top $TopResources Workloads by Write IOPS", "type": "timeseries" }, { "datasource": "${DS_PROMETHEUS}", - "description": "Represents the cables and adapters with which clustered nodes are physically connected. If the cluster interconnect component is in contention, it means high wait time for I/O requests at the cluster interconnect is impacting the latency of one or more workloads.", + "description": "This is the workload's other operations that completed during the measurement interval measured per second.", "fieldConfig": { "defaults": { "color": { @@ -448,20 +331,7 @@ "mode": "off" } }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" - } - }, - "type": "special" - } - ], - "max": 100, + "mappings": [], "min": 0, "thresholds": { "mode": "absolute", @@ -469,14 +339,10 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "percent" + "unit": "iops" }, "overrides": [] }, @@ -484,9 +350,9 @@ "h": 10, "w": 12, "x": 0, - "y": 26 + "y": 18 }, - "id": 110, + "id": 166, "options": { "legend": { "calcs": [ @@ -505,8 +371,7 @@ "targets": [ { "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCluster\",resource=\"cluster\"}) by (workload)) / ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCluster\",resource=\"cluster\"})", - "instant": false, + "expr": "topk($TopResources, qos_other_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSOtherOps\"})", "interval": "", "legendFormat": "{{workload}}", "refId": "A" @@ -514,11 +379,12 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources Workloads by Latency from cluster", + "title": "Top $TopResources Workloads by Other IOPS", "type": "timeseries" }, { "datasource": "${DS_PROMETHEUS}", + "description": "Workload operations executed per second.", "fieldConfig": { "defaults": { "color": { @@ -552,20 +418,7 @@ "mode": "off" } }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" - } - }, - "type": "special" - } - ], - "max": 100, + "mappings": [], "min": 0, "thresholds": { "mode": "absolute", @@ -573,14 +426,10 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "percent" + "unit": "iops" }, "overrides": [] }, @@ -588,9 +437,9 @@ "h": 10, "w": 12, "x": 12, - "y": 26 + "y": 18 }, - "id": 111, + "id": 165, "options": { "legend": { "calcs": [ @@ -609,8 +458,7 @@ "targets": [ { "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCP\",resource=\"cp\"}) by (workload)) / ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCP\",resource=\"cp\"})", - "instant": false, + "expr": "topk($TopResources, qos_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSOps\"})", "interval": "", "legendFormat": "{{workload}}", "refId": "A" @@ -618,11 +466,12 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources Workloads by Latency from cp", + "title": "Top $TopResources Workloads by Total IOPS", "type": "timeseries" }, { "datasource": "${DS_PROMETHEUS}", + "description": "This is the amount of data read per second from the filer by the workload.", "fieldConfig": { "defaults": { "color": { @@ -656,20 +505,7 @@ "mode": "off" } }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" - } - }, - "type": "special" - } - ], - "max": 100, + "mappings": [], "min": 0, "thresholds": { "mode": "absolute", @@ -677,14 +513,10 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "percent" + "unit": "Bps" }, "overrides": [] }, @@ -692,9 +524,9 @@ "h": 10, "w": 12, "x": 0, - "y": 36 + "y": 28 }, - "id": 114, + "id": 167, "options": { "legend": { "calcs": [ @@ -713,8 +545,7 @@ "targets": [ { "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSDisk\",resource=\"disk\"}) by (workload)) / ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSDisk\",resource=\"disk\"})", - "instant": false, + "expr": "topk($TopResources, qos_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSReadData\"})", "interval": "", "legendFormat": "{{workload}}", "refId": "A" @@ -722,12 +553,12 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources Workloads by Latency from disk", + "title": "Top $TopResources Workloads by Read Throughput", "type": "timeseries" }, { "datasource": "${DS_PROMETHEUS}", - "description": "Represents the wait time of I/O requests by the external networking protocols on the cluster. The wait time is time spent waiting for transfer ready transactions to finish before the cluster can respond to an I/O request. If the network component is in contention, it means high wait time at the protocol layer is impacting the latency of one or more workloads.", + "description": "This is the amount of data written per second to the filer by the workload.", "fieldConfig": { "defaults": { "color": { @@ -761,20 +592,7 @@ "mode": "off" } }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" - } - }, - "type": "special" - } - ], - "max": 100, + "mappings": [], "min": 0, "thresholds": { "mode": "absolute", @@ -782,14 +600,10 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "percent" + "unit": "Bps" }, "overrides": [] }, @@ -797,9 +611,9 @@ "h": 10, "w": 12, "x": 12, - "y": 36 + "y": 28 }, - "id": 108, + "id": 168, "options": { "legend": { "calcs": [ @@ -818,8 +632,7 @@ "targets": [ { "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSNetwork\",resource=\"network\"}) by (workload)) / ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSNetwork\",resource=\"network\"})", - "instant": false, + "expr": "topk($TopResources, qos_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSwriteData\"})", "interval": "", "legendFormat": "{{workload}}", "refId": "A" @@ -827,11 +640,12 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources Workloads by Latency from network", + "title": "Top $TopResources Workloads by Write Throughput", "type": "timeseries" }, { "datasource": "${DS_PROMETHEUS}", + "description": "This is the average response time for read requests that were initiated by the workload.", "fieldConfig": { "defaults": { "color": { @@ -865,20 +679,7 @@ "mode": "off" } }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" - } - }, - "type": "special" - } - ], - "max": 100, + "mappings": [], "min": 0, "thresholds": { "mode": "absolute", @@ -886,14 +687,10 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "percent" + "unit": "µs" }, "overrides": [] }, @@ -901,9 +698,9 @@ "h": 10, "w": 12, "x": 0, - "y": 46 + "y": 38 }, - "id": 116, + "id": 169, "options": { "legend": { "calcs": [ @@ -922,8 +719,7 @@ "targets": [ { "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSNVLog\",resource=\"nvlog\"}) by (workload)) / ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSNVLog\",resource=\"nvlog\"})", - "instant": false, + "expr": "topk($TopResources, qos_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSReadLatency\"})", "interval": "", "legendFormat": "{{workload}}", "refId": "A" @@ -931,11 +727,12 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources Workloads by Latency from nvlog", + "title": "Top $TopResources Workloads by Average Read Latency", "type": "timeseries" }, { "datasource": "${DS_PROMETHEUS}", + "description": "This is the average response time for write requests that were initiated by the workload.", "fieldConfig": { "defaults": { "color": { @@ -969,20 +766,7 @@ "mode": "off" } }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" - } - }, - "type": "special" - } - ], - "max": 100, + "mappings": [], "min": 0, "thresholds": { "mode": "absolute", @@ -990,14 +774,10 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "percent" + "unit": "µs" }, "overrides": [] }, @@ -1005,9 +785,9 @@ "h": 10, "w": 12, "x": 12, - "y": 46 + "y": 38 }, - "id": 112, + "id": 171, "options": { "legend": { "calcs": [ @@ -1026,8 +806,7 @@ "targets": [ { "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSSuspend\",resource=\"suspend\"}) by (workload)) / ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSSuspend\",resource=\"suspend\"})", - "instant": false, + "expr": "topk($TopResources, qos_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSWriteLatency\"})", "interval": "", "legendFormat": "{{workload}}", "refId": "A" @@ -1035,12 +814,12 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources Workloads by Latency from WAFL suspend", + "title": "Top $TopResources Workloads by Average Write Latency", "type": "timeseries" }, { "datasource": "${DS_PROMETHEUS}", - "description": "Represents the throughput maximum (peak) setting of the storage Quality of Service (QoS) policy group assigned to the workload. If the policy group component is in contention, it means all workloads in the policy group are being throttled by the set throughput limit, which is impacting the latency of one or more of those workloads.", + "description": "This is the average response time for requests that were initiated by the workload.", "fieldConfig": { "defaults": { "color": { @@ -1074,20 +853,7 @@ "mode": "off" } }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" - } - }, - "type": "special" - } - ], - "max": 100, + "mappings": [], "min": 0, "thresholds": { "mode": "absolute", @@ -1095,14 +861,10 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "percent" + "unit": "µs" }, "overrides": [] }, @@ -1110,9 +872,9 @@ "h": 10, "w": 12, "x": 0, - "y": 56 + "y": 48 }, - "id": 109, + "id": 170, "options": { "legend": { "calcs": [ @@ -1131,8 +893,7 @@ "targets": [ { "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSThrottle\",resource=\"throttle\"}) by (workload)) / ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSThrottle\",resource=\"throttle\"})", - "instant": false, + "expr": "topk($TopResources, qos_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSLatency\"})", "interval": "", "legendFormat": "{{workload}}", "refId": "A" @@ -1140,12 +901,12 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources Workloads by Latency from Qos throttle", + "title": "Top $TopResources Workloads by Average Latency", "type": "timeseries" }, { "datasource": "${DS_PROMETHEUS}", - "description": "Represents the latency to a workload that is being caused by QoS throughput minimum (expected) setting assigned to other workloads. If the QoS minimum set on certain workloads use the majority of the bandwidth to guarantee the promised throughput, other workloads will be throttled and see more latency.", + "description": "This is the average number of concurrent requests for the workload.", "fieldConfig": { "defaults": { "color": { @@ -1179,20 +940,7 @@ "mode": "off" } }, - "decimals": 1, - "mappings": [ - { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" - } - }, - "type": "special" - } - ], - "max": 100, + "mappings": [], "min": 0, "thresholds": { "mode": "absolute", @@ -1200,14 +948,10 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, - "unit": "percent" + "unit": "locale" }, "overrides": [] }, @@ -1215,9 +959,9 @@ "h": 10, "w": 12, "x": 12, - "y": 56 + "y": 48 }, - "id": 113, + "id": 173, "options": { "legend": { "calcs": [ @@ -1236,8 +980,7 @@ "targets": [ { "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSQosMin\",resource=\"qos_min\"}) by (workload)) / ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSQosMin\",resource=\"qos_min\"})", - "instant": false, + "expr": "topk($TopResources, qos_concurrency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSConcurrency\"})", "interval": "", "legendFormat": "{{workload}}", "refId": "A" @@ -1245,879 +988,5034 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources Workloads by Latency from qos_min", + "title": "Top $TopResources Workloads by Concurrency", "type": "timeseries" }, { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 175, + "panels": [ + { + "datasource": "${DS_PROMETHEUS}", + "description": "This is the percentage of read requests served from bamboo_ssd component", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 172, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSReadIOTypeBamboo\", metric=\"bamboo_ssd\"})", + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Read IO Type bamboo_ssd", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This is the percentage of read requests served from cache", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 176, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single" } }, - "decimals": 1, - "mappings": [ + "pluginVersion": "8.1.8", + "targets": [ { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" + "exemplar": false, + "expr": "topk($TopResources, qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSReadIOTypeCache\", metric=\"cache\"})", + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Read IO Type cache", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This is the percentage of read requests served from cloud", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "type": "special" + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 68 + }, + "id": 177, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSReadIOTypeCloud\", metric=\"cloud\"})", + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" } ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Read IO Type cloud", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This is the percentage of read requests served from cloud_s2c", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 - } - ] + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 68 }, - "unit": "percent" + "id": 178, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSReadIOTypeClouds2c\", metric=\"cloud_s2c\"})", + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Read IO Type cloud_s2c", + "type": "timeseries" }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 66 - }, - "id": 118, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" + { + "datasource": "${DS_PROMETHEUS}", + "description": "This is the percentage of read requests served from disk", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 78 + }, + "id": 179, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSReadIOTypeDisk\", metric=\"disk\"})", + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } ], - "displayMode": "table", - "placement": "bottom" + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Read IO Type disk", + "type": "timeseries" }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.1.8", - "targets": [ { - "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSFlexcacheSpinhi\",resource=\"flexcache_spinhi\"}) by (workload)) / ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSFlexcacheSpinhi\",resource=\"flexcache_spinhi\"})", - "instant": false, - "interval": "", - "legendFormat": "{{workload}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Top $TopResources Workloads by Latency from flexcache_spinhi", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Represents the software component in the cluster involved with I/O processing between the cluster and the cloud tier on which user data is stored. If the cloud latency component is in contention, it means that a large amount of reads from volumes that are hosted on the cloud tier are impacting the latency of one or more workloads.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "datasource": "${DS_PROMETHEUS}", + "description": "This is the percentage of read requests served from ext_cache", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 78 + }, + "id": 180, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSReadIOTypeExtCache\", metric=\"ext_cache\"})", + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Read IO Type ext_cache", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This is the percentage of read requests served from fc_miss", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 88 + }, + "id": 181, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single" } }, - "decimals": 1, - "mappings": [ + "pluginVersion": "8.1.8", + "targets": [ { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" + "exemplar": false, + "expr": "topk($TopResources, qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSReadIOTypeFcmiss\", metric=\"fc_miss\"})", + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Read IO Type fc_miss", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This is the percentage of read requests served from hya_cache", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "type": "special" + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 88 + }, + "id": 182, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSReadIOTypeHyaCache\", metric=\"hya_cache\"})", + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" } ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Read IO Type hya_cache", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This is the percentage of read requests served from hya_hdd", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 - } - ] + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 66 - }, - "id": 115, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 98 + }, + "id": 183, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSReadIOTypeHyahdd\", metric=\"hya_hdd\"})", + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } ], - "displayMode": "table", - "placement": "bottom" + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Read IO Type hya_hdd", + "type": "timeseries" }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.1.8", - "targets": [ { - "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCloud\",resource=\"cloud\"}) by (workload)) / ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCloud\",resource=\"cloud\"})", - "instant": false, - "interval": "", - "legendFormat": "{{workload}}", - "refId": "A" + "datasource": "${DS_PROMETHEUS}", + "description": "This is the percentage of read requests served from hya_non_cache", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 98 + }, + "id": 184, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSReadIOTypeHyaNon\", metric=\"hya_non_cache\"})", + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Read IO Type hya_non_cache", + "type": "timeseries" } ], - "timeFrom": null, - "timeShift": null, - "title": "Top $TopResources Workloads by Latency from cloud", - "type": "timeseries" + "title": "Read IO Type", + "type": "row" }, { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 186, + "panels": [ + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 187, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "(avg(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}) by (resource))", + "interval": "", + "legendFormat": "{{resource}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Service Latency by Resources", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents the delays in the network layer of ONTAP.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 70 + }, + "id": 189, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single" } }, - "decimals": 1, - "mappings": [ + "pluginVersion": "8.1.8", + "targets": [ { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" + "exemplar": false, + "expr": "100 * topk($TopResources, (qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSServiceFrontend\",resource=\"frontend\"}/on() group_left sum(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"frontend\"})))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Service Time from frontend", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents the delays in the data/WAFL layer of ONTAP.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "type": "special" + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 70 + }, + "id": 191, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSServiceBackend\",resource=\"backend\"}/on() group_left sum(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"backend\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" } ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Service Time from backend", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents delays caused by the cluster switches, cables, and adapters which physically connect clustered nodes. \n\nIf the cluster interconnect component is in contention, it means high wait time for I/O requests at the cluster interconnect is impacting the latency of one or more workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 - } - ] + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 193, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } }, - "unit": "percent" + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSServiceCluster\",resource=\"cluster\"}/on() group_left sum(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"cluster\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Service Time from cluster", + "type": "timeseries" }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 120, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents delays due to buffered write flushes, called consistency points (cp).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 195, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSServiceCP\",resource=\"cp\"}/on() group_left sum(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"cp\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } ], - "displayMode": "table", - "placement": "bottom" + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Service Time from cp", + "type": "timeseries" }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.1.8", - "targets": [ { - "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCOP\",resource=\"cop\"}) by (workload)) / ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSCOP\",resource=\"cop\"})", - "instant": false, - "interval": "", - "legendFormat": "{{workload}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Top $TopResources Workloads by Latency from cop", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "datasource": "${DS_PROMETHEUS}", + "description": "Represents slowness due to attached hard drives or solid state drives.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 90 + }, + "id": 197, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSServiceDisk\",resource=\"disk\"}/on() group_left sum(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"disk\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Service Time from disk", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "`Note:` Typically these latencies only apply to SAN not NAS.\n\nRepresents the wait time of I/O requests by the external networking protocols on the cluster. The wait time is time spent waiting for transfer ready transactions to finish before the cluster can respond to an I/O request. If the network component is in contention, it means high wait time at the protocol layer is impacting the latency of one or more workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 90 + }, + "id": 199, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single" } }, - "decimals": 1, - "mappings": [ + "pluginVersion": "8.1.8", + "targets": [ { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSServiceNetwork\",resource=\"network\"}/on() group_left sum(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"network\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Service Time from network", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents delays due to mirroring writes to the NVRAM/NVLOG memory and to the HA partner NVRAM/NVLOG memory.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "type": "special" + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 100 + }, + "id": 201, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSServiceNVLog\",resource=\"nvlog\"}/on() group_left sum(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"nvlog\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" } ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Service Time from nvlog", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents delays due to operations suspending on a delay mechanism. Typically this is diagnosed by NetApp Support.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 - } - ] + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 76 - }, - "id": 117, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 100 + }, + "id": 203, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSServiceSuspend\",resource=\"suspend\"}/on() group_left sum(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"suspend\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } ], - "displayMode": "table", - "placement": "bottom" + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Service Time from WAFL suspend", + "type": "timeseries" }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.1.8", - "targets": [ { - "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSFlexcacheRAL\",resource=\"flexcache_ral\"}) by (workload)) / ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSFlexcacheRAL\",resource=\"flexcache_ral\"})", - "instant": false, - "interval": "", - "legendFormat": "{{workload}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Top $TopResources Workloads by Latency from flexcache_ral", - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "datasource": "${DS_PROMETHEUS}", + "description": "Represents the throughput maximum (ceiling) setting of the storage Quality of Service (QoS) policy group assigned to the workload. If the policy group component is in contention, it means all workloads in the policy group are being throttled by the set throughput limit, which is impacting the latency of one or more of those workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 110 + }, + "id": 205, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSServiceThrottle\",resource=\"throttle\"}/on() group_left sum(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"throttle\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Service Time from Qos throttle", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents the latency to a workload that is being caused by QoS throughput floor (expected) setting assigned to other workloads. If the QoS floor set on certain workloads use the majority of the bandwidth to guarantee the promised throughput, other workloads will be throttled and see more latency.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 110 + }, + "id": 207, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single" } }, - "decimals": 1, - "mappings": [ + "pluginVersion": "8.1.8", + "targets": [ { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSServiceQosMin\",resource=\"qos_min\"}/on() group_left sum(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"qos_min\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Service Time from qos_min", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents the software component in the cluster involved with I/O processing between the cluster and the cloud tier on which user data is stored. If the cloud latency component is in contention, it means that a large amount of reads from volumes that are hosted on the cloud tier are impacting the latency of one or more workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - "type": "special" + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 120 + }, + "id": 209, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSServiceCloud\",resource=\"cloud\"}/on() group_left sum(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"cloud\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" } ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Service Time from cloud", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 - } - ] + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 120 + }, + "id": 211, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } }, - "unit": "percent" + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSServiceSyncRepl\",resource=\"sync_repl\"}/on() group_left sum(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"sync_repl\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Service Time from sync_repl", + "type": "timeseries" }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 86 - }, - "id": 119, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 130 + }, + "id": 213, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSServiceFlexcacheRal\",resource=\"flexcache_ral\"}/on() group_left sum(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"flexcache_ral\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } ], - "displayMode": "table", - "placement": "bottom" + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Service Time from flexcache_ral", + "type": "timeseries" }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.1.8", - "targets": [ { - "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSSyncRepl\",resource=\"sync_repl\"}) by (workload)) / ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSSyncRepl\",resource=\"sync_repl\"})", - "instant": false, - "interval": "", - "legendFormat": "{{workload}}", - "refId": "A" + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 130 + }, + "id": 215, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSServiceFlexcacheSpinhi\",resource=\"flexcache_spinhi\"}/on() group_left sum(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"flexcache_spinhi\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Service Time from flexcache_spinhi", + "type": "timeseries" } ], - "timeFrom": null, - "timeShift": null, - "title": "Top $TopResources Workloads by Latency from sync_repl", - "type": "timeseries" + "title": "Service Center", + "type": "row" }, { + "collapsed": true, "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 127, + "panels": [ + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 129, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "(avg(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}) by (resource))", + "interval": "", + "legendFormat": "{{resource}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency by Resources", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents the delays in the network layer of ONTAP.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 133, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single" } }, - "decimals": 1, - "mappings": [ + "pluginVersion": "8.1.8", + "targets": [ { - "options": { - "match": "null+nan", - "result": { - "index": 0, - "text": "0%" + "exemplar": false, + "expr": "100 * topk($TopResources, (qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSFrontend\",resource=\"frontend\"}/on() group_left sum(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"frontend\"})))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Latency from frontend", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents the delays in the data/WAFL layer of ONTAP.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "type": "special" + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 131, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSBackend\",resource=\"backend\"}/on() group_left sum(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"backend\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" } ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Latency from backend", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents delays caused by the cluster switches, cables, and adapters which physically connect clustered nodes. \n\nIf the cluster interconnect component is in contention, it means high wait time for I/O requests at the cluster interconnect is impacting the latency of one or more workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 - } - ] + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 135, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSCluster\",resource=\"cluster\"}/on() group_left sum(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"cluster\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Latency from cluster", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents delays due to buffered write flushes, called consistency points (cp).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 137, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSCP\",resource=\"cp\"}/on() group_left sum(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"cp\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Latency from cp", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents slowness due to attached hard drives or solid state drives.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 139, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSDisk\",resource=\"disk\"}/on() group_left sum(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"disk\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Latency from disk", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "`Note:` Typically these latencies only apply to SAN not NAS.\n\nRepresents the wait time of I/O requests by the external networking protocols on the cluster. The wait time is time spent waiting for transfer ready transactions to finish before the cluster can respond to an I/O request. If the network component is in contention, it means high wait time at the protocol layer is impacting the latency of one or more workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 141, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSNetwork\",resource=\"network\"}/on() group_left sum(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"network\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Latency from network", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents delays due to mirroring writes to the NVRAM/NVLOG memory and to the HA partner NVRAM/NVLOG memory.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 143, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSNVLog\",resource=\"nvlog\"}/on() group_left sum(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"nvlog\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Latency from nvlog", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents delays due to operations suspending on a delay mechanism. Typically this is diagnosed by NetApp Support.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 145, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSSuspend\",resource=\"suspend\"}/on() group_left sum(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"suspend\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Latency from WAFL suspend", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents the throughput maximum (ceiling) setting of the storage Quality of Service (QoS) policy group assigned to the workload. If the policy group component is in contention, it means all workloads in the policy group are being throttled by the set throughput limit, which is impacting the latency of one or more of those workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 52 + }, + "id": 147, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSThrottle\",resource=\"throttle\"}/on() group_left sum(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"throttle\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Latency from Qos throttle", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents the latency to a workload that is being caused by QoS throughput floor (expected) setting assigned to other workloads. If the QoS floor set on certain workloads use the majority of the bandwidth to guarantee the promised throughput, other workloads will be throttled and see more latency.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 52 + }, + "id": 149, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } }, - "unit": "percent" + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSQosMin\",resource=\"qos_min\"}/on() group_left sum(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"qos_min\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Latency from qos_min", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Represents the software component in the cluster involved with I/O processing between the cluster and the cloud tier on which user data is stored. If the cloud latency component is in contention, it means that a large amount of reads from volumes that are hosted on the cloud tier are impacting the latency of one or more workloads.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 153, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSCloud\",resource=\"cloud\"}/on() group_left sum(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"cloud\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Latency from cloud", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 159, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSSyncRepl\",resource=\"sync_repl\"}/on() group_left sum(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"sync_repl\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Latency from sync_repl", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 157, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSFlexcacheRal\",resource=\"flexcache_ral\"}/on() group_left sum(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"flexcache_ral\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Latency from flexcache_ral", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0%" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 151, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "100 * topk($TopResources, qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$TopQOSFlexcacheSpinhi\",resource=\"flexcache_spinhi\"}/on() group_left sum(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",resource=\"flexcache_spinhi\"}))", + "instant": false, + "interval": "", + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Workloads by Latency from flexcache_spinhi", + "type": "timeseries" + } + ], + "title": "Latency Breakdown", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 30, + "style": "dark", + "tags": [ + "harvest", + "ontap", + "cdot" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 2, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(volume_labels{system_type!=\"7mode\"}, datacenter)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "Datacenter", + "options": [], + "query": { + "query": "label_values(volume_labels{system_type!=\"7mode\"}, datacenter)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(volume_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\"}, cluster)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "Cluster", + "options": [], + "query": { + "query": "label_values(volume_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\"}, cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "Workload Class", + "multi": true, + "name": "WorkloadClass", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "user_defined", + "value": "user_defined" + }, + { + "selected": false, + "text": "system_defined", + "value": "system_defined" + }, + { + "selected": false, + "text": "autovolume", + "value": "autovolume" + } + ], + "query": "user_defined,system_defined,autovolume", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(qos_workload_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", class=~\"$WorkloadClass\"}, workload)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Workload", + "options": [], + "query": { + "query": "label_values(qos_workload_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", class=~\"$WorkloadClass\"}, workload)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "5", + "value": "5" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "TopResources", + "options": [ + { + "selected": false, + "text": "1", + "value": "1" + }, + { + "selected": false, + "text": "2", + "value": "2" + }, + { + "selected": false, + "text": "3", + "value": "3" + }, + { + "selected": false, + "text": "4", + "value": "4" + }, + { + "selected": true, + "text": "5", + "value": "5" + }, + { + "selected": false, + "text": "6", + "value": "6" + }, + { + "selected": false, + "text": "8", + "value": "8" + }, + { + "selected": false, + "text": "10", + "value": "10" + }, + { + "selected": false, + "text": "15", + "value": "15" + }, + { + "selected": false, + "text": "25", + "value": "25" + }, + { + "selected": false, + "text": "50", + "value": "50" + }, + { + "selected": false, + "text": "100", + "value": "100" + }, + { + "selected": false, + "text": "250", + "value": "250" + }, + { + "selected": false, + "text": "500", + "value": "500" + } + ], + "query": "1,2,3,4,5,6,8,10,15,25,50,100,250,500", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"backend\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSBackend", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"backend\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"frontend\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSFrontend", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"frontend\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"throttle\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSThrottle", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"throttle\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"network\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSNetwork", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"network\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"cluster\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSCluster", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"cluster\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"cp\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSCP", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"cp\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"qos_min\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSQosMin", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"qos_min\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"suspend\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSSuspend", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"suspend\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"disk\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSDisk", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"disk\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"cloud\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSCloud", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"cloud\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"nvlog\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSNVLog", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"nvlog\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"flexcache_ral\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSFlexcacheRal", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"flexcache_ral\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"flexcache_spinhi\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSFlexcacheSpinhi", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"flexcache_spinhi\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"sync_repl\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSSyncRepl", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"sync_repl\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSReadOps", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSWriteOps", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_other_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSOtherOps", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_other_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSOps", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSReadData", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSwriteData", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSReadLatency", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_read_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSWriteLatency", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSLatency", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_concurrency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSConcurrency", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_concurrency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}[${__range}])))", + "refId": "StandardVariableQuery" }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 86 + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" }, - "id": 121, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"bamboo_ssd\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSReadIOTypeBamboo", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"bamboo_ssd\"}[${__range}])))", + "refId": "StandardVariableQuery" }, - "tooltip": { - "mode": "single" - } + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" }, - "pluginVersion": "8.1.8", - "targets": [ - { - "exemplar": false, - "expr": "100 * topk($TopResources, sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSWaflAdmission\",resource=\"admission\"}) by (workload)) / ignoring(workload) group_left sum(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$TopVolumeQOSWaflAdmission\",resource=\"admission\"})", - "instant": false, - "interval": "", - "legendFormat": "{{workload}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Top $TopResources Workloads by Latency from admission", - "type": "timeseries" - } - ], - "refresh": "", - "schemaVersion": 30, - "style": "dark", - "tags": [ - "harvest", - "ontap", - "cdot" - ], - "templating": { - "list": [ { - "current": { - "selected": false, - "text": "Prometheus", - "value": "Prometheus" - }, + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"cache\"}[${__range}])))", "description": null, "error": null, "hide": 2, - "includeAll": false, - "label": "Data Source", - "multi": false, - "name": "DS_PROMETHEUS", + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSReadIOTypeCache", "options": [], - "query": "prometheus", + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"cache\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, "refresh": 2, - "regex": "", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, - "type": "datasource" + "sort": 0, + "type": "query" }, { "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(volume_labels{system_type!=\"7mode\"}, datacenter)", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"cloud\"}[${__range}])))", "description": null, "error": null, - "hide": 0, - "includeAll": false, + "hide": 2, + "includeAll": true, "label": null, "multi": true, - "name": "Datacenter", + "name": "TopQOSReadIOTypeCloud", "options": [], "query": { - "query": "label_values(volume_labels{system_type!=\"7mode\"}, datacenter)", + "query": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"cloud\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": "", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "sort": 0, + "type": "query" }, { "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(volume_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\"}, cluster)", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"cloud_s2c\"}[${__range}])))", "description": null, "error": null, - "hide": 0, - "includeAll": false, + "hide": 2, + "includeAll": true, "label": null, "multi": true, - "name": "Cluster", + "name": "TopQOSReadIOTypeClouds2c", "options": [], "query": { - "query": "label_values(volume_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\"}, cluster)", + "query": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"cloud_s2c\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": "", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "sort": 0, + "type": "query" }, { "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, workload)", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"disk\"}[${__range}])))", "description": null, "error": null, - "hide": 0, + "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "Workload", + "name": "TopQOSReadIOTypeDisk", "options": [], "query": { - "query": "label_values(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}, workload)", + "query": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"disk\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": "", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "sort": 0, + "type": "query" }, { "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}, volume)", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"ext_cache\"}[${__range}])))", "description": null, "error": null, - "hide": 0, + "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "Volume", + "name": "TopQOSReadIOTypeExtCache", "options": [], "query": { - "query": "label_values(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\"}, volume)", + "query": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"ext_cache\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": "", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false + "sort": 0, + "type": "query" }, { "allValue": null, - "current": { - "selected": false, - "text": "5", - "value": "5" + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"fc_miss\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopQOSReadIOTypeFcmiss", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"fc_miss\"}[${__range}])))", + "refId": "StandardVariableQuery" }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"hya_cache\"}[${__range}])))", "description": null, "error": null, - "hide": 0, - "includeAll": false, + "hide": 2, + "includeAll": true, "label": null, - "multi": false, - "name": "TopResources", - "options": [ - { - "selected": false, - "text": "1", - "value": "1" - }, - { - "selected": false, - "text": "2", - "value": "2" - }, - { - "selected": false, - "text": "3", - "value": "3" - }, - { - "selected": false, - "text": "4", - "value": "4" - }, - { - "selected": true, - "text": "5", - "value": "5" - }, - { - "selected": false, - "text": "6", - "value": "6" - }, - { - "selected": false, - "text": "8", - "value": "8" - }, - { - "selected": false, - "text": "10", - "value": "10" - }, - { - "selected": false, - "text": "15", - "value": "15" - }, - { - "selected": false, - "text": "25", - "value": "25" - }, - { - "selected": false, - "text": "50", - "value": "50" - }, - { - "selected": false, - "text": "100", - "value": "100" - }, - { - "selected": false, - "text": "250", - "value": "250" - }, - { - "selected": false, - "text": "500", - "value": "500" - } - ], - "query": "1,2,3,4,5,6,8,10,15,25,50,100,250,500", - "queryValue": "", + "multi": true, + "name": "TopQOSReadIOTypeHyaCache", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"hya_cache\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, - "type": "custom" + "sort": 0, + "type": "query" }, { "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"backend\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"hya_hdd\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSBackend", + "name": "TopQOSReadIOTypeHyahdd", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"backend\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"hya_hdd\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2126,21 +6024,21 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"frontend\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"hya_non_cache\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSFrontend", + "name": "TopQOSReadIOTypeHyaNon", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"frontend\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_read_io_type{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",metric=\"hya_non_cache\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2149,21 +6047,21 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"throttle\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"backend\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSThrottle", + "name": "TopQOSServiceBackend", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"throttle\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"backend\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2172,21 +6070,21 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"network\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"frontend\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSNetwork", + "name": "TopQOSServiceFrontend", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"network\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"frontend\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2195,21 +6093,21 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cluster\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"throttle\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSCluster", + "name": "TopQOSServiceThrottle", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cluster\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"throttle\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2218,21 +6116,21 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cp\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"network\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSCP", + "name": "TopQOSServiceNetwork", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cp\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"network\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2241,21 +6139,21 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"qos_min\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"cluster\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSQosMin", + "name": "TopQOSServiceCluster", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"qos_min\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"cluster\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2264,21 +6162,21 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"suspend\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"cp\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSSuspend", + "name": "TopQOSServiceCP", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"suspend\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"cp\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2287,21 +6185,21 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"disk\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"qos_min\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSDisk", + "name": "TopQOSServiceQosMin", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"disk\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"qos_min\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2310,21 +6208,21 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cloud\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"suspend\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSCloud", + "name": "TopQOSServiceSuspend", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cloud\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"suspend\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2333,21 +6231,21 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"nvlog\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"disk\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSNVLog", + "name": "TopQOSServiceDisk", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"nvlog\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"disk\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2356,21 +6254,21 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"flexcache_ral\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"cloud\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSFlexcacheRAL", + "name": "TopQOSServiceCloud", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"flexcache_ral\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"cloud\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2379,21 +6277,21 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"flexcache_spinhi\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"nvlog\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSFlexcacheSpinhi", + "name": "TopQOSServiceNVLog", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"flexcache_spinhi\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"nvlog\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2402,21 +6300,21 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"sync_repl\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"flexcache_ral\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSSyncRepl", + "name": "TopQOSServiceFlexcacheRal", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"sync_repl\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"flexcache_ral\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2425,21 +6323,21 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cop\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"flexcache_spinhi\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSCOP", + "name": "TopQOSServiceFlexcacheSpinhi", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"cop\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"flexcache_spinhi\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2448,21 +6346,21 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"admission\"}[${__range}])))", + "definition": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"sync_repl\"}[${__range}])))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, - "name": "TopVolumeQOSWaflAdmission", + "name": "TopQOSServiceSyncRepl", "options": [], "query": { - "query": "query_result(topk($TopResources, avg_over_time(qos_detail_volume_resource_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",volume=~\"$Volume\",resource=\"admission\"}[${__range}])))", + "query": "query_result(topk($TopResources, avg_over_time(qos_detail_service_time_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",workload=~\"$Workload\",resource=\"sync_repl\"}[${__range}])))", "refId": "StandardVariableQuery" }, "refresh": 2, - "regex": ".*volume=\\\"(.*?)\\\".*", + "regex": ".*workload=\\\"(.*?)\\\".*", "skipUrlSync": false, "sort": 0, "type": "query" @@ -2489,5 +6387,5 @@ "timezone": "", "title": "ONTAP: Workload", "uid": "", - "version": 1 + "version": 2 } diff --git a/integration/test/dashboard_json_test.go b/integration/test/dashboard_json_test.go index 68acf4bde..b23d7b488 100644 --- a/integration/test/dashboard_json_test.go +++ b/integration/test/dashboard_json_test.go @@ -76,7 +76,6 @@ var excludeCounters = []string{ "path_", "poller", "qos_detail_resource_latency", - "qos_detail_volume_resource_latency", "quota_disk_used_pct_disk_limit", "quota_files_used_pct_file_limit", "security_login", @@ -130,7 +129,7 @@ func TestJsonExpression(t *testing.T) { now := time.Now() // QoS counters have the longest schedule so check for them before checking for any of the other counters - precheckCounters := []string{"qos_read_data", "qos_volume_read_data"} + precheckCounters := []string{"qos_read_data"} for _, counter := range precheckCounters { if counterIsMissing(rest, counter, 7*time.Minute) { t.Fatalf("rest qos counters not found dur=%s", time.Since(now).Round(time.Millisecond).String()) From 1b91e21d69506e9f0619cfc9419f968a0192be50 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Fri, 11 Aug 2023 12:26:16 -0400 Subject: [PATCH 14/40] fix: grafana ask-for-token should retry at most 5 times --- cmd/tools/grafana/grafana.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/tools/grafana/grafana.go b/cmd/tools/grafana/grafana.go index eb9f7ef39..75be7bbf8 100644 --- a/cmd/tools/grafana/grafana.go +++ b/cmd/tools/grafana/grafana.go @@ -90,7 +90,7 @@ func adjustOptions() { func askForToken() { // ask for API token if not provided as arg and validate - if err := checkToken(opts, false); err != nil { + if err := checkToken(opts, false, 5); err != nil { fmt.Println(err) os.Exit(1) } @@ -716,15 +716,17 @@ func addPrefixToMetricNames(expr, prefix string) string { return expr } -func checkToken(opts *options, ignoreConfig bool) error { - - // @TODO check and handle expired API token +func checkToken(opts *options, ignoreConfig bool, tries int) error { var ( token, configPath, answer string err error ) + if tries == 0 { + return fmt.Errorf("no more attempts") + } + configPath = opts.config err = conf.LoadHarvestConfig(configPath) @@ -765,7 +767,7 @@ func checkToken(opts *options, ignoreConfig bool) error { msg := result["message"].(string) fmt.Printf("error connect: (%d - %s) %s\n", code, status, msg) opts.token = "" - return checkToken(opts, true) + return checkToken(opts, true, tries-1) } // ask user to save API key From d286bc0d910e84410b9c3b8f034167cf646b1e5d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 11:48:21 +0530 Subject: [PATCH 15/40] chore: update module github.com/tidwall/gjson to v1.16.0 (#2286) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 + vendor/github.com/tidwall/gjson/README.md | 1 + vendor/github.com/tidwall/gjson/SYNTAX.md | 1 + vendor/github.com/tidwall/gjson/gjson.go | 79 +++++++++++++++++------ vendor/modules.txt | 2 +- 6 files changed, 65 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index cbae6e2b1..f36a21fb4 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/rs/zerolog v1.30.0 github.com/shirou/gopsutil/v3 v3.23.7 github.com/spf13/cobra v1.7.0 - github.com/tidwall/gjson v1.15.0 + github.com/tidwall/gjson v1.16.0 github.com/tidwall/pretty v1.2.1 github.com/tidwall/sjson v1.2.5 github.com/zekroTJA/timedmap v1.5.1 diff --git a/go.sum b/go.sum index 38d118fb8..4040b3e08 100644 --- a/go.sum +++ b/go.sum @@ -103,6 +103,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.15.0 h1:5n/pM+v3r5ujuNl4YLZLsQ+UE5jlkLVm7jMzT5Mpolw= github.com/tidwall/gjson v1.15.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg= +github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= diff --git a/vendor/github.com/tidwall/gjson/README.md b/vendor/github.com/tidwall/gjson/README.md index c8db11f14..da2bad447 100644 --- a/vendor/github.com/tidwall/gjson/README.md +++ b/vendor/github.com/tidwall/gjson/README.md @@ -211,6 +211,7 @@ There are currently the following built-in modifiers: - `@tostr`: Converts json to a string. Wraps a json string. - `@fromstr`: Converts a string from json. Unwraps a json string. - `@group`: Groups arrays of objects. See [e4fc67c](https://github.com/tidwall/gjson/commit/e4fc67c92aeebf2089fabc7872f010e340d105db). +- `@dig`: Search for a value without providing its entire path. See [e8e87f2](https://github.com/tidwall/gjson/commit/e8e87f2a00dc41f3aba5631094e21f59a8cf8cbf). ### Modifier arguments diff --git a/vendor/github.com/tidwall/gjson/SYNTAX.md b/vendor/github.com/tidwall/gjson/SYNTAX.md index 9405514bb..6721d7f51 100644 --- a/vendor/github.com/tidwall/gjson/SYNTAX.md +++ b/vendor/github.com/tidwall/gjson/SYNTAX.md @@ -258,6 +258,7 @@ There are currently the following built-in modifiers: - `@tostr`: Converts json to a string. Wraps a json string. - `@fromstr`: Converts a string from json. Unwraps a json string. - `@group`: Groups arrays of objects. See [e4fc67c](https://github.com/tidwall/gjson/commit/e4fc67c92aeebf2089fabc7872f010e340d105db). +- `@dig`: Search for a value without providing its entire path. See [e8e87f2](https://github.com/tidwall/gjson/commit/e8e87f2a00dc41f3aba5631094e21f59a8cf8cbf). #### Modifier arguments diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go index e7e8d7e20..a1633be52 100644 --- a/vendor/github.com/tidwall/gjson/gjson.go +++ b/vendor/github.com/tidwall/gjson/gjson.go @@ -2754,6 +2754,7 @@ func execModifier(json, path string) (pathOut, res string, ok bool) { var parsedArgs bool switch pathOut[0] { case '{', '[', '"': + // json arg res := Parse(pathOut) if res.Exists() { args = squash(pathOut) @@ -2762,14 +2763,20 @@ func execModifier(json, path string) (pathOut, res string, ok bool) { } } if !parsedArgs { - idx := strings.IndexByte(pathOut, '|') - if idx == -1 { - args = pathOut - pathOut = "" - } else { - args = pathOut[:idx] - pathOut = pathOut[idx:] + // simple arg + i := 0 + for ; i < len(pathOut); i++ { + if pathOut[i] == '|' { + break + } + switch pathOut[i] { + case '{', '[', '"', '(': + s := squash(pathOut[i:]) + i += len(s) - 1 + } } + args = pathOut[:i] + pathOut = pathOut[i:] } } return pathOut, fn(json, args), true @@ -2789,19 +2796,24 @@ func unwrap(json string) string { // DisableModifiers will disable the modifier syntax var DisableModifiers = false -var modifiers = map[string]func(json, arg string) string{ - "pretty": modPretty, - "ugly": modUgly, - "reverse": modReverse, - "this": modThis, - "flatten": modFlatten, - "join": modJoin, - "valid": modValid, - "keys": modKeys, - "values": modValues, - "tostr": modToStr, - "fromstr": modFromStr, - "group": modGroup, +var modifiers map[string]func(json, arg string) string + +func init() { + modifiers = map[string]func(json, arg string) string{ + "pretty": modPretty, + "ugly": modUgly, + "reverse": modReverse, + "this": modThis, + "flatten": modFlatten, + "join": modJoin, + "valid": modValid, + "keys": modKeys, + "values": modValues, + "tostr": modToStr, + "fromstr": modFromStr, + "group": modGroup, + "dig": modDig, + } } // AddModifier binds a custom modifier command to the GJSON syntax. @@ -3435,3 +3447,30 @@ func escapeComp(comp string) string { } return comp } + +func parseRecursiveDescent(all []Result, parent Result, path string) []Result { + if res := parent.Get(path); res.Exists() { + all = append(all, res) + } + if parent.IsArray() || parent.IsObject() { + parent.ForEach(func(_, val Result) bool { + all = parseRecursiveDescent(all, val, path) + return true + }) + } + return all +} + +func modDig(json, arg string) string { + all := parseRecursiveDescent(nil, Parse(json), arg) + var out []byte + out = append(out, '[') + for i, res := range all { + if i > 0 { + out = append(out, ',') + } + out = append(out, res.Raw...) + } + out = append(out, ']') + return string(out) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 0cdca1e01..bf99fe05a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -82,7 +82,7 @@ github.com/spf13/cobra # github.com/spf13/pflag v1.0.5 ## explicit; go 1.12 github.com/spf13/pflag -# github.com/tidwall/gjson v1.15.0 +# github.com/tidwall/gjson v1.16.0 ## explicit; go 1.12 github.com/tidwall/gjson # github.com/tidwall/match v1.1.1 From 5162835d5be7df91c3c1833d47e8eba978bf1fbf Mon Sep 17 00:00:00 2001 From: Hardikl <83282894+Hardikl@users.noreply.github.com> Date: Mon, 14 Aug 2023 11:49:21 +0530 Subject: [PATCH 16/40] fix: handled when metric not found in plugin (#2281) --- cmd/poller/plugin/metricagent/metric_agent.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/poller/plugin/metricagent/metric_agent.go b/cmd/poller/plugin/metricagent/metric_agent.go index 16bb0cfea..7044c2694 100644 --- a/cmd/poller/plugin/metricagent/metric_agent.go +++ b/cmd/poller/plugin/metricagent/metric_agent.go @@ -59,6 +59,7 @@ func (a *MetricAgent) computeMetrics(m *matrix.Matrix) error { metric *matrix.Metric metricVal, firstMetricVal *matrix.Metric err error + metricNotFound []error ) // map values for compute_metric mapping rules @@ -96,8 +97,8 @@ func (a *MetricAgent) computeMetrics(m *matrix.Matrix) error { if metricVal != nil { v, _ = metricVal.GetValueFloat64(instance) } else { - a.Logger.Warn().Err(err).Str("metricName", r.metricNames[i]).Msg("computeMetrics: metric not found") - return nil + metricNotFound = append(metricNotFound, err) + break } } @@ -131,6 +132,9 @@ func (a *MetricAgent) computeMetrics(m *matrix.Matrix) error { a.Logger.Trace().Str("metricName", r.metric).Float64("metricValue", result).Msg("computeMetrics: new metric created") } } + if len(metricNotFound) > 0 { + a.Logger.Warn().Errs("computeMetrics: errors for metric not found", metricNotFound).Send() + } return nil } From 5eb42993460b53990ff36b3986d682333f69d2cd Mon Sep 17 00:00:00 2001 From: Hardikl <83282894+Hardikl@users.noreply.github.com> Date: Mon, 14 Aug 2023 11:49:36 +0530 Subject: [PATCH 17/40] feat: added table description for cluster compliance (#2269) * feat: added table description for cluster compliance * feat: handled review comments * feat: handled review comments --- conf/rest/9.10.0/volume.yaml | 2 +- conf/rest/9.9.0/volume.yaml | 2 +- grafana/dashboards/cmode/security.json | 58 +++++++++++++++++--------- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/conf/rest/9.10.0/volume.yaml b/conf/rest/9.10.0/volume.yaml index 31df04304..c45c7ad8c 100644 --- a/conf/rest/9.10.0/volume.yaml +++ b/conf/rest/9.10.0/volume.yaml @@ -15,7 +15,7 @@ counters: - ^is_svm_root => svm_root - ^snaplock.type => snaplock_type - ^snapshot_policy.name => snapshot_policy - - ^space.snapshot.autodelete.enabled => snapshot_autodelete + - ^space.snapshot.autodelete_enabled => snapshot_autodelete - ^state => state - ^style => style - ^type => type diff --git a/conf/rest/9.9.0/volume.yaml b/conf/rest/9.9.0/volume.yaml index 1ce2aeb55..d2c9a6665 100644 --- a/conf/rest/9.9.0/volume.yaml +++ b/conf/rest/9.9.0/volume.yaml @@ -13,7 +13,7 @@ counters: - ^is_svm_root => svm_root - ^snaplock.type => snaplock_type - ^snapshot_policy.name => snapshot_policy - - ^space.snapshot.autodelete.enabled => snapshot_autodelete + - ^space.snapshot.autodelete_enabled => snapshot_autodelete - ^state => state - ^style => style - ^type => type diff --git a/grafana/dashboards/cmode/security.json b/grafana/dashboards/cmode/security.json index 70857e7b9..4fed00338 100644 --- a/grafana/dashboards/cmode/security.json +++ b/grafana/dashboards/cmode/security.json @@ -2462,7 +2462,7 @@ "panels": [ { "datasource": "${DS_PROMETHEUS}", - "description": "❌ means this attribute is non-compliant", + "description": "❌ means this attribute is non-compliant. \n\n| Column | Compliant When | \n|---|---|\n| `Snapshot Policy` | All volumes have applied Snapshot policy | \n| `Snapshot Autodelete` | All volumes have enabled Snapshot autodelete |\n| `ARW Protection for SVMs` | All SVMs have enabled ARW protection |\n|`ARW Protection for Volumes`| All volumes have enabled ARW protection|\n| `Cluster Certificate Validity` | Cluster has active certificate(s) |\n| `Global FIPS`| Cluster has global FIPS enabled |\n| `Telnet` | Cluster has telnet disabled |\n| `Autosupport Https Transport` | Cluster uses HTTPS for autosupport |\n| `Default Admin User` | Default admin user is locked |\n| `Remote Shell` | Cluster's remote shell is disabled |\n| `MD5 in use` | Cluster does not use MD5 algorithm |\n| `Insecure SSH Settings` | Cluster has strong SSH server ciphers |\n| `Login Banner` | Cluster has enabled login banner |\n| `Network Time Protocol` | Cluster has configured three NTP servers |\n| `Cluster Peering` | Cluster peers use encryption |\n| `Notification Configured` | Cluster has configured destinations for notifications |\n| `Automatic Updates Configured`| Cluster has enabled automatic updates |", "fieldConfig": { "defaults": { "color": { @@ -2543,11 +2543,11 @@ "options": { "false": { "index": 0, - "text": "Disabled" + "text": "❌ Enabled" }, "true": { "index": 1, - "text": "❌ Enabled" + "text": "Disabled" } }, "type": "value" @@ -2884,6 +2884,16 @@ "to": 9999 }, "type": "range" + }, + { + "options": { + "match": "null", + "result": { + "index": 4, + "text": "Not Applicable" + } + }, + "type": "special" } ] }, @@ -3604,7 +3614,7 @@ }, { "exemplar": false, - "expr": "security_account_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", methods=~\".*password.*\",role_name=\"admin\", user_name=\"admin\"}", + "expr": "group by (datacenter,cluster,locked) (security_account_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", methods=~\".*password.*\",role_name=\"admin\", user_name=\"admin\",locked!=\"\"})", "format": "table", "hide": false, "instant": true, @@ -3819,7 +3829,12 @@ "Value #R", "Value #T", "auto_update_enabled", - "ArwStatus" + "ArwStatus", + "Value #J", + "Value #G", + "banner", + "insecured", + "Value #D" ] } } @@ -3859,26 +3874,29 @@ "methods": true }, "indexByName": { - "ArwStatus": 6, + "ArwStatus": 5, + "Value #G": 13, "Value #H": 2, - "Value #I": 13, - "Value #K": 17, - "Value #L": 14, - "Value #M": 15, - "Value #N": 16, + "Value #I": 17, + "Value #J": 14, + "Value #K": 21, + "Value #L": 18, + "Value #M": 19, + "Value #N": 20, "Value #P": 0, "Value #Q": 3, "Value #R": 4, - "Value #S": 5, - "asup_enabled": 10, - "certificateExpiryStatus": 7, - "certificateIssuerType": 18, + "Value #T": 15, + "asup_enabled": 9, + "auto_update_enabled": 16, + "banner": 12, + "certificateExpiryStatus": 6, + "certificateIssuerType": 22, "cluster": 1, - "datacenter": 19, - "fips_enabled": 8, - "locked": 11, - "rsh_enabled": 12, - "telnet_enabled": 9 + "fips_enabled": 7, + "locked": 10, + "rsh_enabled": 11, + "telnet_enabled": 8 }, "renameByName": { "ArwStatus": "ARW Protection for Volumes", From 4892c01128403baf091e6fabebcd4e1435c5ba68 Mon Sep 17 00:00:00 2001 From: Rahul Date: Mon, 14 Aug 2023 15:20:17 +0530 Subject: [PATCH 18/40] fix: nfs heatmap per cluster (#2273) * fix: nfs heatmap per cluster * fix: address review comments * fix: address review comments * fix: address review comments --- grafana/dashboards/cmode/svm.json | 1778 ++++++++++++++--------------- 1 file changed, 867 insertions(+), 911 deletions(-) diff --git a/grafana/dashboards/cmode/svm.json b/grafana/dashboards/cmode/svm.json index 88e6a104c..f009e2149 100644 --- a/grafana/dashboards/cmode/svm.json +++ b/grafana/dashboards/cmode/svm.json @@ -71,7 +71,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1691508761583, + "iteration": 1691772584861, "links": [ { "asDropdown": true, @@ -704,7 +704,7 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 7, "w": 8, "x": 0, "y": 9 @@ -810,7 +810,7 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 7, "w": 8, "x": 8, "y": 9 @@ -908,7 +908,7 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 7, "w": 8, "x": 16, "y": 9 @@ -966,7 +966,7 @@ "h": 1, "w": 24, "x": 0, - "y": 15 + "y": 16 }, "id": 79, "panels": [ @@ -1516,7 +1516,7 @@ "h": 1, "w": 24, "x": 0, - "y": 16 + "y": 17 }, "id": 88, "panels": [ @@ -2213,7 +2213,7 @@ "h": 1, "w": 24, "x": 0, - "y": 17 + "y": 18 }, "id": 98, "panels": [ @@ -3275,7 +3275,7 @@ "h": 1, "w": 24, "x": 0, - "y": 18 + "y": 19 }, "id": 110, "panels": [ @@ -4336,7 +4336,7 @@ "h": 1, "w": 24, "x": 0, - "y": 19 + "y": 20 }, "id": 124, "panels": [ @@ -5283,7 +5283,7 @@ "h": 1, "w": 24, "x": 0, - "y": 20 + "y": 21 }, "id": 37, "panels": [ @@ -5321,7 +5321,7 @@ "h": 5, "w": 8, "x": 0, - "y": 7 + "y": 22 }, "id": 39, "options": { @@ -5395,7 +5395,7 @@ "h": 5, "w": 8, "x": 8, - "y": 7 + "y": 22 }, "id": 50, "interval": null, @@ -5494,7 +5494,7 @@ "h": 5, "w": 8, "x": 16, - "y": 7 + "y": 22 }, "id": 46, "interval": null, @@ -5584,7 +5584,7 @@ "h": 5, "w": 4, "x": 0, - "y": 12 + "y": 27 }, "id": 48, "options": { @@ -5649,7 +5649,7 @@ "h": 5, "w": 4, "x": 4, - "y": 12 + "y": 27 }, "id": 47, "options": { @@ -5723,7 +5723,7 @@ "h": 5, "w": 4, "x": 8, - "y": 12 + "y": 27 }, "id": 150, "interval": null, @@ -5822,7 +5822,7 @@ "h": 5, "w": 4, "x": 12, - "y": 12 + "y": 27 }, "id": 151, "interval": null, @@ -5921,7 +5921,7 @@ "h": 5, "w": 4, "x": 16, - "y": 12 + "y": 27 }, "id": 152, "interval": null, @@ -6020,7 +6020,7 @@ "h": 5, "w": 4, "x": 20, - "y": 12 + "y": 27 }, "id": 153, "interval": null, @@ -6076,213 +6076,6 @@ ], "type": "stat" }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateRdYlGn", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_PROMETHEUS}", - "description": "", - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 17 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 246, - "interval": "6m", - "legend": { - "show": false - }, - "maxDataPoints": 25, - "reverseYBuckets": false, - "targets": [ - { - "exemplar": false, - "expr": "sum(increase(svm_nfs_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$TopNfsv3Latency\",nfsv=\"v3\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ le }}", - "refId": "A" - } - ], - "timeFrom": null, - "title": "NFSv3 Latency Heatmap", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "µs", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null, - "width": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateRdYlGn", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_PROMETHEUS}", - "description": "", - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 17 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 249, - "interval": "6m", - "legend": { - "show": false - }, - "maxDataPoints": 25, - "reverseYBuckets": false, - "targets": [ - { - "exemplar": false, - "expr": "sum(increase(svm_nfs_read_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$TopNfsv3ReadAvgLatency\",nfsv=\"v3\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ le }}", - "refId": "A" - } - ], - "timeFrom": null, - "title": "NFSv3 Read Latency Heatmap", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "µs", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null, - "width": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateRdYlGn", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_PROMETHEUS}", - "description": "", - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 17 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 250, - "interval": "6m", - "legend": { - "show": false - }, - "maxDataPoints": 25, - "reverseYBuckets": false, - "targets": [ - { - "exemplar": false, - "expr": "sum(increase(svm_nfs_write_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$TopNfsv3WriteAvgLatency\",nfsv=\"v3\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ le }}", - "refId": "A" - } - ], - "timeFrom": null, - "title": "NFSv3 Write Latency Heatmap", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "µs", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null, - "width": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, { "datasource": "${DS_PROMETHEUS}", "fieldConfig": { @@ -6336,7 +6129,7 @@ "h": 8, "w": 8, "x": 0, - "y": 25 + "y": 32 }, "id": 51, "options": { @@ -6428,7 +6221,7 @@ "h": 8, "w": 8, "x": 8, - "y": 25 + "y": 32 }, "id": 53, "options": { @@ -6528,7 +6321,7 @@ "h": 8, "w": 8, "x": 16, - "y": 25 + "y": 32 }, "id": 42, "options": { @@ -6629,7 +6422,7 @@ "h": 8, "w": 12, "x": 0, - "y": 33 + "y": 40 }, "id": 52, "options": { @@ -6724,7 +6517,7 @@ "h": 8, "w": 12, "x": 12, - "y": 33 + "y": 40 }, "id": 54, "options": { @@ -6773,7 +6566,7 @@ "h": 1, "w": 24, "x": 0, - "y": 21 + "y": 22 }, "id": 126, "panels": [ @@ -6811,7 +6604,7 @@ "h": 5, "w": 8, "x": 0, - "y": 8 + "y": 23 }, "id": 154, "options": { @@ -6885,7 +6678,7 @@ "h": 5, "w": 8, "x": 8, - "y": 8 + "y": 23 }, "id": 155, "interval": null, @@ -6984,7 +6777,7 @@ "h": 5, "w": 8, "x": 16, - "y": 8 + "y": 23 }, "id": 166, "interval": null, @@ -7074,7 +6867,7 @@ "h": 5, "w": 4, "x": 0, - "y": 13 + "y": 28 }, "id": 158, "options": { @@ -7139,7 +6932,7 @@ "h": 5, "w": 4, "x": 4, - "y": 13 + "y": 28 }, "id": 159, "options": { @@ -7213,7 +7006,7 @@ "h": 5, "w": 4, "x": 8, - "y": 13 + "y": 28 }, "id": 160, "interval": null, @@ -7312,7 +7105,7 @@ "h": 5, "w": 4, "x": 12, - "y": 13 + "y": 28 }, "id": 161, "interval": null, @@ -7411,7 +7204,7 @@ "h": 5, "w": 4, "x": 16, - "y": 13 + "y": 28 }, "id": 162, "interval": null, @@ -7510,7 +7303,7 @@ "h": 5, "w": 4, "x": 20, - "y": 13 + "y": 28 }, "id": 163, "interval": null, @@ -7566,213 +7359,6 @@ ], "type": "stat" }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateRdYlGn", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_PROMETHEUS}", - "description": "", - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 18 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 256, - "interval": "6m", - "legend": { - "show": false - }, - "maxDataPoints": 25, - "reverseYBuckets": false, - "targets": [ - { - "exemplar": false, - "expr": "sum(increase(svm_nfs_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$TopNfsv4Latency\",nfsv=\"v4\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ le }}", - "refId": "A" - } - ], - "timeFrom": null, - "title": "NFSv4 Latency Heatmap", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "µs", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null, - "width": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateRdYlGn", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_PROMETHEUS}", - "description": "", - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 18 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 257, - "interval": "6m", - "legend": { - "show": false - }, - "maxDataPoints": 25, - "reverseYBuckets": false, - "targets": [ - { - "exemplar": false, - "expr": "sum(increase(svm_nfs_read_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$TopNfsv4Latency\",nfsv=\"v4\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ le }}", - "refId": "A" - } - ], - "timeFrom": null, - "title": "NFSv4 Read Latency Heatmap", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "µs", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null, - "width": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateRdYlGn", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_PROMETHEUS}", - "description": "", - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 18 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 258, - "interval": "6m", - "legend": { - "show": false - }, - "maxDataPoints": 25, - "reverseYBuckets": false, - "targets": [ - { - "exemplar": false, - "expr": "sum(increase(svm_nfs_write_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$TopNfsv4Latency\",nfsv=\"v4\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ le }}", - "refId": "A" - } - ], - "timeFrom": null, - "title": "NFSv4 Write Latency Heatmap", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "µs", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null, - "width": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, { "datasource": "${DS_PROMETHEUS}", "fieldConfig": { @@ -7831,7 +7417,7 @@ "h": 8, "w": 8, "x": 0, - "y": 26 + "y": 33 }, "id": 145, "options": { @@ -7930,7 +7516,7 @@ "h": 8, "w": 8, "x": 8, - "y": 26 + "y": 33 }, "id": 146, "options": { @@ -8024,7 +7610,7 @@ "h": 8, "w": 8, "x": 16, - "y": 26 + "y": 33 }, "id": 147, "options": { @@ -8127,7 +7713,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 41 }, "id": 157, "options": { @@ -8148,7 +7734,7 @@ "targets": [ { "exemplar": false, - "expr": "{__name__=~\"svm_nfs_.+_avg_latency\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}", + "expr": "avg by(__name__) ({__name__=~\"svm_nfs_.+_avg_latency\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"})", "hide": false, "interval": "", "legendFormat": "{{__name__}}", @@ -8157,7 +7743,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources NFSv4 SVMs by Latency by Op Type", + "title": "NFSv4 SVMs by Latency by Op Type", "transformations": [ { "id": "renameByRegex", @@ -8227,7 +7813,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 41 }, "id": 149, "options": { @@ -8248,7 +7834,7 @@ "targets": [ { "exemplar": false, - "expr": "{__name__=~\"svm_nfs_.+_total\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}", + "expr": "sum by (__name__) ({__name__=~\"svm_nfs_.+_total\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"})", "hide": false, "interval": "", "legendFormat": "{{__name__}}", @@ -8257,7 +7843,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources NFSv4 SVMs by IOPs per Type", + "title": "NFSv4 SVMs by IOPs per Type", "transformations": [ { "id": "renameByRegex", @@ -8280,7 +7866,7 @@ "h": 1, "w": 24, "x": 0, - "y": 22 + "y": 23 }, "id": 130, "panels": [ @@ -8318,7 +7904,7 @@ "h": 5, "w": 8, "x": 0, - "y": 9 + "y": 24 }, "id": 164, "options": { @@ -8392,7 +7978,7 @@ "h": 5, "w": 8, "x": 8, - "y": 9 + "y": 24 }, "id": 165, "interval": null, @@ -8491,7 +8077,7 @@ "h": 5, "w": 8, "x": 16, - "y": 9 + "y": 24 }, "id": 156, "interval": null, @@ -8581,7 +8167,7 @@ "h": 5, "w": 4, "x": 0, - "y": 14 + "y": 29 }, "id": 140, "options": { @@ -8646,7 +8232,7 @@ "h": 5, "w": 4, "x": 4, - "y": 14 + "y": 29 }, "id": 167, "options": { @@ -8720,7 +8306,7 @@ "h": 5, "w": 4, "x": 8, - "y": 14 + "y": 29 }, "id": 168, "interval": null, @@ -8819,7 +8405,7 @@ "h": 5, "w": 4, "x": 12, - "y": 14 + "y": 29 }, "id": 169, "interval": null, @@ -8918,7 +8504,7 @@ "h": 5, "w": 4, "x": 16, - "y": 14 + "y": 29 }, "id": 170, "interval": null, @@ -9017,7 +8603,7 @@ "h": 5, "w": 4, "x": 20, - "y": 14 + "y": 29 }, "id": 171, "interval": null, @@ -9074,271 +8660,64 @@ "type": "stat" }, { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateRdYlGn", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", "datasource": "${DS_PROMETHEUS}", - "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 8, "x": 0, - "y": 19 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 253, - "interval": "6m", - "legend": { - "show": false - }, - "maxDataPoints": 25, - "reverseYBuckets": false, - "targets": [ - { - "exemplar": false, - "expr": "sum(increase(svm_nfs_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$TopNfsv41Latency\",nfsv=\"v4.1\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ le }}", - "refId": "A" - } - ], - "timeFrom": null, - "title": "NFSv4.1 Latency Heatmap", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "µs", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null, - "width": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateRdYlGn", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_PROMETHEUS}", - "description": "", - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 19 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 254, - "interval": "6m", - "legend": { - "show": false - }, - "maxDataPoints": 25, - "reverseYBuckets": false, - "targets": [ - { - "exemplar": false, - "expr": "sum(increase(svm_nfs_read_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$TopNfsv41Latency\",nfsv=\"v4.1\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ le }}", - "refId": "A" - } - ], - "timeFrom": null, - "title": "NFSv4.1 Read Latency Heatmap", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "µs", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null, - "width": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateRdYlGn", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_PROMETHEUS}", - "description": "", - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 19 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 255, - "interval": "6m", - "legend": { - "show": false - }, - "maxDataPoints": 25, - "reverseYBuckets": false, - "targets": [ - { - "exemplar": false, - "expr": "sum(increase(svm_nfs_write_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$TopNfsv41Latency\",nfsv=\"v4.1\"}[$__interval])) by (le)", - "format": "heatmap", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ le }}", - "refId": "A" - } - ], - "timeFrom": null, - "title": "NFSv4.1 Write Latency Heatmap", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "xBucketNumber": null, - "xBucketSize": null, - "yAxis": { - "decimals": null, - "format": "µs", - "logBase": 1, - "max": null, - "min": null, - "show": true, - "splitFactor": null, - "width": null - }, - "yBucketBound": "auto", - "yBucketNumber": null, - "yBucketSize": null - }, - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "µs" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 27 + "y": 34 }, "id": 172, "options": { @@ -9437,7 +8816,7 @@ "h": 8, "w": 8, "x": 8, - "y": 27 + "y": 34 }, "id": 173, "options": { @@ -9536,7 +8915,7 @@ "h": 8, "w": 8, "x": 16, - "y": 27 + "y": 34 }, "id": 174, "options": { @@ -9642,7 +9021,7 @@ "h": 8, "w": 12, "x": 0, - "y": 35 + "y": 42 }, "id": 175, "options": { @@ -9663,7 +9042,7 @@ "targets": [ { "exemplar": false, - "expr": "{__name__=~\"svm_nfs_.+_avg_latency\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}", + "expr": "avg by(__name__) ({__name__=~\"svm_nfs_.+_avg_latency\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"})", "hide": false, "interval": "", "legendFormat": "{{__name__}}", @@ -9672,7 +9051,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources NFSv4.1 SVMs by Latency by Op Type", + "title": "NFSv4.1 SVMs by Latency by Op Type", "transformations": [ { "id": "renameByRegex", @@ -9742,7 +9121,7 @@ "h": 8, "w": 12, "x": 12, - "y": 35 + "y": 42 }, "id": 176, "options": { @@ -9763,7 +9142,7 @@ "targets": [ { "exemplar": false, - "expr": "{__name__=~\"svm_nfs_.+_total\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}", + "expr": "sum by (__name__) ({__name__=~\"svm_nfs_.+_total\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"})", "hide": false, "interval": "", "legendFormat": "{{__name__}}", @@ -9772,7 +9151,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources NFSv4.1 SVMs by IOPs per Type", + "title": "NFSv4.1 SVMs by IOPs per Type", "transformations": [ { "id": "renameByRegex", @@ -9795,7 +9174,7 @@ "h": 1, "w": 24, "x": 0, - "y": 23 + "y": 24 }, "id": 56, "panels": [ @@ -10857,7 +10236,7 @@ "h": 1, "w": 24, "x": 0, - "y": 24 + "y": 25 }, "id": 132, "panels": [ @@ -11111,7 +10490,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 26 }, "id": 134, "panels": [ @@ -12048,7 +11427,7 @@ "h": 1, "w": 24, "x": 0, - "y": 26 + "y": 27 }, "id": 136, "panels": [ @@ -12873,7 +12252,7 @@ "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 28 }, "id": 138, "panels": [ @@ -13383,131 +12762,777 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 357 + "w": 12, + "x": 12, + "y": 357 + }, + "id": 239, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, volume_sis_compress_saved{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeSisCompressSaved\"})", + "interval": "", + "legendFormat": "{{volume}} - {{svm}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Compression Savings", + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 365 + }, + "id": 240, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "topk($TopResources, volume_sis_total_saved{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeSisTotalSaved\"})", + "interval": "", + "legendFormat": "{{volume}} - {{svm}}", + "refId": "A" + } + ], + "title": "Top $TopResources Volumes by Total Efficiency Savings", + "type": "timeseries" + } + ], + "title": "Volume Capacity Drilldown", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 266, + "panels": [ + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 30 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 274, + "interval": "6m", + "legend": { + "show": false + }, + "maxDataPoints": 25, + "repeat": null, + "reverseYBuckets": false, + "targets": [ + { + "exemplar": false, + "expr": "sum(increase(svm_nfs_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "NFSv3 Latency Heatmap", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null, + "width": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 30 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 282, + "interval": "6m", + "legend": { + "show": false + }, + "maxDataPoints": 25, + "repeat": null, + "reverseYBuckets": false, + "targets": [ + { + "exemplar": false, + "expr": "sum(increase(svm_nfs_read_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "NFSv3 Read Latency Heatmap", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null, + "width": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 30 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 284, + "interval": "6m", + "legend": { + "show": false + }, + "maxDataPoints": 25, + "repeat": null, + "reverseYBuckets": false, + "targets": [ + { + "exemplar": false, + "expr": "sum(increase(svm_nfs_write_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "NFSv3 Write Latency Heatmap", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null, + "width": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 38 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 292, + "interval": "6m", + "legend": { + "show": false + }, + "maxDataPoints": 25, + "repeat": null, + "reverseYBuckets": false, + "targets": [ + { + "exemplar": false, + "expr": "sum(increase(svm_nfs_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "NFSv4 Latency Heatmap", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null, + "width": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 38 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 294, + "interval": "6m", + "legend": { + "show": false + }, + "maxDataPoints": 25, + "repeat": null, + "reverseYBuckets": false, + "targets": [ + { + "exemplar": false, + "expr": "sum(increase(svm_nfs_read_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "NFSv4 Read Latency Heatmap", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null, + "width": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 38 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 296, + "interval": "6m", + "legend": { + "show": false + }, + "maxDataPoints": 25, + "repeat": null, + "reverseYBuckets": false, + "targets": [ + { + "exemplar": false, + "expr": "sum(increase(svm_nfs_write_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "NFSv4 Write Latency Heatmap", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null, + "width": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 46 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 304, + "interval": "6m", + "legend": { + "show": false + }, + "maxDataPoints": 25, + "repeat": null, + "reverseYBuckets": false, + "targets": [ + { + "exemplar": false, + "expr": "sum(increase(svm_nfs_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "NFSv4.1 Latency Heatmap", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null, + "width": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 46 }, - "id": 239, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 306, + "interval": "6m", + "legend": { + "show": false }, - "pluginVersion": "8.1.8", + "maxDataPoints": 25, + "repeat": null, + "reverseYBuckets": false, "targets": [ { "exemplar": false, - "expr": "topk($TopResources, volume_sis_compress_saved{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeSisCompressSaved\"})", + "expr": "sum(increase(svm_nfs_read_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, "interval": "", - "legendFormat": "{{volume}} - {{svm}}", + "intervalFactor": 1, + "legendFormat": "{{ le }}", "refId": "A" } ], - "title": "Top $TopResources Volumes by Compression Savings", - "type": "timeseries" + "timeFrom": null, + "title": "NFSv4.1 Read Latency Heatmap", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null, + "width": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null }, { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 2, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "description": "", "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 365 + "h": 8, + "w": 8, + "x": 16, + "y": 46 }, - "id": 240, - "options": { - "legend": { - "calcs": [ - "mean", - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 308, + "interval": "6m", + "legend": { + "show": false }, - "pluginVersion": "8.1.8", + "maxDataPoints": 25, + "repeat": null, + "reverseYBuckets": false, "targets": [ { "exemplar": false, - "expr": "topk($TopResources, volume_sis_total_saved{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeSisTotalSaved\"})", + "expr": "sum(increase(svm_nfs_write_latency_hist_bucket{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}[$__interval])) by (le)", + "format": "heatmap", + "instant": false, "interval": "", - "legendFormat": "{{volume}} - {{svm}}", + "intervalFactor": 1, + "legendFormat": "{{ le }}", "refId": "A" } ], - "title": "Top $TopResources Volumes by Total Efficiency Savings", - "type": "timeseries" + "timeFrom": null, + "title": "NFSv4.1 Write Latency Heatmap", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null, + "width": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null } ], - "title": "Volume Capacity Drilldown", + "repeat": "Cluster", + "title": "NFS Latency Heatmap $Cluster", "type": "row" } ], @@ -14217,29 +14242,6 @@ "sort": 0, "type": "query" }, - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(svm_nfs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}[${__range}])))", - "description": null, - "error": null, - "hide": 2, - "includeAll": true, - "label": null, - "multi": true, - "name": "TopNfsv3Latency", - "options": [], - "query": { - "query": "query_result(topk($TopResources, avg_over_time(svm_nfs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v3\"}[${__range}])))", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": ".*svm=\\\"(.*?)\\\".*", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, { "allValue": null, "current": {}, @@ -15067,52 +15069,6 @@ "skipUrlSync": false, "sort": 0, "type": "query" - }, - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(svm_nfs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}[${__range}])))", - "description": null, - "error": null, - "hide": 2, - "includeAll": true, - "label": null, - "multi": true, - "name": "TopNfsv41Latency", - "options": [], - "query": { - "query": "query_result(topk($TopResources, avg_over_time(svm_nfs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4.1\"}[${__range}])))", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": ".*svm=\\\"(.*?)\\\".*", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "allValue": null, - "current": {}, - "datasource": "${DS_PROMETHEUS}", - "definition": "query_result(topk($TopResources, avg_over_time(svm_nfs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}[${__range}])))", - "description": null, - "error": null, - "hide": 2, - "includeAll": true, - "label": null, - "multi": true, - "name": "TopNfsv4Latency", - "options": [], - "query": { - "query": "query_result(topk($TopResources, avg_over_time(svm_nfs_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",nfsv=\"v4\"}[${__range}])))", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": ".*svm=\\\"(.*?)\\\".*", - "skipUrlSync": false, - "sort": 0, - "type": "query" } ] }, @@ -15136,5 +15092,5 @@ "timezone": "", "title": "ONTAP: SVM", "uid": "", - "version": 23 + "version": 24 } From de3080b4bb65c31a353ade6907f8bfd817cbd858 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Mon, 14 Aug 2023 05:50:27 -0400 Subject: [PATCH 19/40] chore: bump go (#2285) chore: replace utils with builtins --- .github/workflows/go.yml | 2 +- .github/workflows/golangci-lint.yml | 2 +- Makefile | 2 +- cmd/collectors/ems/ems_test.go | 16 ++++---- cmd/tools/doctor/restZapiDiff.go | 6 +-- cmd/tools/grafana/dashboard_test.go | 6 +-- go.mod | 2 +- go.sum | 1 + integration/Jenkinsfile | 2 +- integration/go.mod | 18 ++++----- integration/go.sum | 29 ++++++------- jenkins/artifacts/jenkinsfile | 2 +- pkg/tree/node/node.go | 7 ++-- pkg/util/util.go | 63 ++++++++--------------------- 14 files changed, 66 insertions(+), 92 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 999df8661..454c5b15a 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -1,7 +1,7 @@ name: Build, Test, Lint License env: - GO_VERSION: "1.20.7" + GO_VERSION: "1.21.0" on: push: diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index e1079e065..09861e044 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -19,7 +19,7 @@ jobs: steps: - uses: actions/setup-go@v4 with: - go-version: '1.20.7' + go-version: '1.21.0' - uses: actions/checkout@v3 - name: golangci-lint uses: golangci/golangci-lint-action@v3 diff --git a/Makefile b/Makefile index 6dc30d128..bc9e3ce8c 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ ############################################################################### SHELL := /bin/bash GCC_EXISTS := $(shell which gcc) -REQUIRED_GO_VERSION := 1.20 +REQUIRED_GO_VERSION := 1.21 ifneq (, $(shell which go)) FOUND_GO_VERSION := $(shell go version | cut -d" " -f3 | cut -d"o" -f 2) CORRECT_GO_VERSION := $(shell expr `go version | cut -d" " -f3 | cut -d"o" -f 2` \>= ${REQUIRED_GO_VERSION}) diff --git a/cmd/collectors/ems/ems_test.go b/cmd/collectors/ems/ems_test.go index 379b2126f..d75f65b0e 100644 --- a/cmd/collectors/ems/ems_test.go +++ b/cmd/collectors/ems/ems_test.go @@ -7,9 +7,9 @@ import ( "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/tree" "github.com/netapp/harvest/v2/pkg/tree/node" - "github.com/netapp/harvest/v2/pkg/util" "github.com/rs/zerolog/log" "os" + "slices" "testing" "time" ) @@ -90,7 +90,7 @@ func (e *Ems) testBookendIssuingEms(t *testing.T, path string) { // Check and evaluate ems events var notGeneratedEmsNames []string for generatedEmsName, mx := range e.Matrix { - if util.Contains(issuingEmsNames, generatedEmsName) { + if slices.Contains(issuingEmsNames, generatedEmsName) { metr, ok := mx.GetMetrics()["events"] if !ok { t.Fatalf("Failed to get netric for Ems %s", generatedEmsName) @@ -129,7 +129,7 @@ func (e *Ems) testBookendResolvingEms(t *testing.T, path string) { // Check and evaluate ems events var notResolvedEmsNames []string for generatedEmsName, mx := range e.Matrix { - if util.Contains(issuingEmsNames, generatedEmsName) { + if slices.Contains(issuingEmsNames, generatedEmsName) { metr, ok := mx.GetMetrics()["events"] if !ok { t.Fatalf("Failed to get netric for Ems %s", generatedEmsName) @@ -169,7 +169,7 @@ func (e *Ems) testAutoResolvingEms(t *testing.T, path string) { // Check and evaluate ems events for generatedEmsName, mx := range e.Matrix { - if util.Contains(autoresolveEmsNames, generatedEmsName) { + if slices.Contains(autoresolveEmsNames, generatedEmsName) { if metr, ok := mx.GetMetrics()["events"]; ok { for _, instance := range mx.GetInstances() { // If value not exist for that instance or metric value 0 indicate ems hasn't been raised. @@ -191,7 +191,7 @@ func (e *Ems) testAutoResolvingEms(t *testing.T, path string) { e.updateMatrix(time.Now().Add(1 * time.Second)) // Check and evaluate bookend ems events got auto resolved successfully. for generatedEmsName, mx := range e.Matrix { - if util.Contains(autoresolveEmsNames, generatedEmsName) { + if slices.Contains(autoresolveEmsNames, generatedEmsName) { if metr, ok := mx.GetMetrics()["events"]; ok { for _, instance := range mx.GetInstances() { // If value not exist for that instance or metric value 0 indicate ems hasn't been raised. @@ -204,7 +204,7 @@ func (e *Ems) testAutoResolvingEms(t *testing.T, path string) { } } } - if util.Contains(notAutoResolvedEmsNames, "LUN.offline") { + if slices.Contains(notAutoResolvedEmsNames, "LUN.offline") { t.Fatalf("These Bookend Ems haven't been auto resolved: %s", notAutoResolvedEmsNames) } @@ -217,7 +217,7 @@ func (e *Ems) testAutoResolvingEms(t *testing.T, path string) { t.Fatalf("These Bookend Ems haven't been removed from cache: %s", "LUN.offline") } for generatedEmsName, mx := range e.Matrix { - if util.Contains(autoresolveEmsNames, generatedEmsName) { + if slices.Contains(autoresolveEmsNames, generatedEmsName) { if metr, ok := mx.GetMetrics()["events"]; ok { for _, instance := range mx.GetInstances() { // If value not exist for that instance or metric value 0 indicate ems hasn't been raised. @@ -230,7 +230,7 @@ func (e *Ems) testAutoResolvingEms(t *testing.T, path string) { } } } - if util.Contains(notAutoResolvedEmsNames, "monitor.fan.critical") { + if slices.Contains(notAutoResolvedEmsNames, "monitor.fan.critical") { t.Fatalf("These Bookend Ems haven't been auto resolved: %s", notAutoResolvedEmsNames) } } diff --git a/cmd/tools/doctor/restZapiDiff.go b/cmd/tools/doctor/restZapiDiff.go index 248787b18..3e949fb8d 100644 --- a/cmd/tools/doctor/restZapiDiff.go +++ b/cmd/tools/doctor/restZapiDiff.go @@ -2,7 +2,6 @@ package doctor import ( "fmt" - "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" "io" "log" @@ -11,6 +10,7 @@ import ( "os" "path/filepath" "regexp" + "slices" "strconv" "strings" "time" @@ -520,7 +520,7 @@ func labelValueDiff(label string, labelNames []string) { results = gjson.GetMany(data, prefixLabelsName...) } - if util.Contains([]string{"shelf_voltage_labels", "shelf_temperature_labels", "shelf_sensor_labels"}, label) { + if slices.Contains([]string{"shelf_voltage_labels", "shelf_temperature_labels", "shelf_sensor_labels"}, label) { keyIndexes = append(keyIndexes, IndexOf(finalLabelNames, "shelf")) keyIndexes = append(keyIndexes, IndexOf(finalLabelNames, "sensor_id")) dataCenterIndex = IndexOf(finalLabelNames, "datacenter") @@ -533,7 +533,7 @@ func labelValueDiff(label string, labelNames []string) { results = gjson.GetMany(data, prefixLabelsName...) } - if util.Contains([]string{"svm_labels", "security_login_labels"}, label) { + if slices.Contains([]string{"svm_labels", "security_login_labels"}, label) { keyIndexes = append(keyIndexes, IndexOf(finalLabelNames, "svm")) dataCenterIndex = IndexOf(finalLabelNames, "datacenter") results = gjson.GetMany(data, prefixLabelsName...) diff --git a/cmd/tools/grafana/dashboard_test.go b/cmd/tools/grafana/dashboard_test.go index b1ede5cf6..6b7fbd87d 100644 --- a/cmd/tools/grafana/dashboard_test.go +++ b/cmd/tools/grafana/dashboard_test.go @@ -2,12 +2,12 @@ package grafana import ( "fmt" - "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" "github.com/tidwall/pretty" "os" "path/filepath" "regexp" + "slices" "sort" "strconv" "strings" @@ -83,7 +83,7 @@ func checkThreshold(t *testing.T, path string, data []byte) { isThresholdSet = color.String() == th[0] && v.String() == th[1] } else if id == "custom.displayMode" && kind == "table" { v := propertiesN.Get("value") - if !util.Contains(expectedColorBackground[kind], v.String()) { + if !slices.Contains(expectedColorBackground[kind], v.String()) { t.Errorf("dashboard=%s panel=%s kind=%s expr=%s don't have correct displaymode expected %s found %s", path, panelTitle, kind, expr, expectedColorBackground[kind], v.String()) } else { isColorBackgroundSet = true @@ -94,7 +94,7 @@ func checkThreshold(t *testing.T, path string, data []byte) { if kind == "stat" { colorMode := value.Get("options.colorMode") - if !util.Contains(expectedColorBackground[kind], colorMode.String()) { + if !slices.Contains(expectedColorBackground[kind], colorMode.String()) { t.Errorf("dashboard=%s panel=%s kind=%s expr=%s don't have correct colorMode expected %s found %s", path, panelTitle, kind, expr, expectedColorBackground[kind], colorMode.String()) } else { isColorBackgroundSet = true diff --git a/go.mod b/go.mod index f36a21fb4..adddf84e0 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/netapp/harvest/v2 -go 1.20 +go 1.21 require ( dario.cat/mergo v1.0.0 diff --git a/go.sum b/go.sum index 4040b3e08..d2202a457 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,7 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= diff --git a/integration/Jenkinsfile b/integration/Jenkinsfile index 560020f23..2ff39f0a3 100644 --- a/integration/Jenkinsfile +++ b/integration/Jenkinsfile @@ -18,7 +18,7 @@ pipeline { environment { BUILD_ID="dontKillMe" JENKINS_NODE_COOKIE="dontKillMe" - GO_VERSION = "1.20.7" + GO_VERSION = "1.21.0" } stages { diff --git a/integration/go.mod b/integration/go.mod index e8321114e..a231c398c 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -1,15 +1,15 @@ module github.com/Netapp/harvest-automation -go 1.20 +go 1.21 replace github.com/netapp/harvest/v2 => ../ require ( github.com/carlmjohnson/requests v0.23.4 - github.com/netapp/harvest/v2 v2.0.0-20230404163343-f21b0c1a08ac - github.com/rs/zerolog v1.29.1 - github.com/tidwall/gjson v1.14.4 - golang.org/x/text v0.11.0 + github.com/netapp/harvest/v2 v2.0.0-20230811164902-1b91e21d6950 + github.com/rs/zerolog v1.30.0 + github.com/tidwall/gjson v1.16.0 + golang.org/x/text v0.12.0 ) require ( @@ -21,16 +21,16 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect - github.com/shirou/gopsutil/v3 v3.23.6 // indirect + github.com/shirou/gopsutil/v3 v3.23.7 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect - github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.10.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sys v0.11.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/integration/go.sum b/integration/go.sum index 1b4dc8607..44ae3d0d1 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -33,12 +33,12 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= -github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= +github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shirou/gopsutil/v3 v3.23.6 h1:5y46WPI9QBKBbK7EEccUPNXpJpNrvPuTD0O2zHEHT08= -github.com/shirou/gopsutil/v3 v3.23.6/go.mod h1:j7QX50DrXYggrpN30W0Mo+I4/8U2UUIQrnrhqUeWrAU= +github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= +github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -54,22 +54,23 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg= +github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -78,11 +79,11 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/jenkins/artifacts/jenkinsfile b/jenkins/artifacts/jenkinsfile index d64b89bfd..86430d77f 100644 --- a/jenkins/artifacts/jenkinsfile +++ b/jenkins/artifacts/jenkinsfile @@ -37,7 +37,7 @@ pipeline { jfrogImagePrefix = "netappdownloads.jfrog.io/oss-docker-harvest-production/harvest" jfrogRepo = "netappdownloads.jfrog.io" COMMIT_ID = sh(returnStdout: true, script: 'git rev-parse HEAD') - GO_VERSION = "1.20.7" + GO_VERSION = "1.21.0" } stages { diff --git a/pkg/tree/node/node.go b/pkg/tree/node/node.go index f9f1c14c0..16ac91ebb 100644 --- a/pkg/tree/node/node.go +++ b/pkg/tree/node/node.go @@ -10,6 +10,7 @@ import ( "fmt" "github.com/netapp/harvest/v2/pkg/util" "regexp" + "slices" "strings" ) @@ -326,7 +327,7 @@ func (n *Node) Merge(subtemplate *Node, skipOverwrite []string) { } else if mine == nil { n.AddChild(child) } else { - if mine.GetParent() != nil && util.Contains(skipOverwrite, mine.GetParent().GetNameS()) { + if mine.GetParent() != nil && slices.Contains(skipOverwrite, mine.GetParent().GetNameS()) { mine.SetContentS(mine.GetContentS() + "," + child.GetContentS()) } else { mine.SetContentS(child.GetContentS()) @@ -418,7 +419,7 @@ func (n *Node) SearchContent(prefix []string, paths [][]string) ([]string, bool) } //fmt.Printf(" -> current_path=%v \t new_path=%v\n", currentPath, newPath) for _, path := range paths { - if util.EqualStringSlice(newPath, path) { + if slices.Equal(newPath, path) { matches = append(matches, node.GetContentS()) //fmt.Println(" MATCH!") break @@ -451,7 +452,7 @@ func (n *Node) SearchChildren(path []string) []*Node { newPath = make([]string, len(currentPath)) copy(newPath, currentPath) } - if util.EqualStringSlice(newPath, path) { + if slices.Equal(newPath, path) { matches = append(matches, node) } else if len(newPath) < len(path) { for _, child := range node.GetChildren() { diff --git a/pkg/util/util.go b/pkg/util/util.go index 002936ccb..8c63b2fcb 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -15,6 +15,7 @@ import ( "net/url" "os" "regexp" + "slices" "sort" "strconv" "strings" @@ -40,25 +41,25 @@ func GetCollectorSlice() []string { } func MinLen(elements [][]string) int { - var min, i int - min = len(elements[0]) + var smallest, i int + smallest = len(elements[0]) for i = 1; i < len(elements); i++ { - if len(elements[i]) < min { - min = len(elements[i]) + if len(elements[i]) < smallest { + smallest = len(elements[i]) } } - return min + return smallest } func MaxLen(elements [][]string) int { - var max, i int - max = len(elements[0]) + var largest, i int + largest = len(elements[0]) for i = 1; i < len(elements); i++ { - if len(elements[i]) > max { - max = len(elements[i]) + if len(elements[i]) > largest { + largest = len(elements[i]) } } - return max + return largest } func AllSame(elements [][]string, k int) bool { @@ -71,19 +72,6 @@ func AllSame(elements [][]string, k int) bool { return true } -func EqualStringSlice(a, b []string) bool { - var i int - if len(a) != len(b) { - return false - } - for i = 0; i < len(a); i++ { - if a[i] != b[i] { - return false - } - } - return true -} - var pollerRegex = regexp.MustCompile(`poller\s+--poller\s+(.*?)\s`) var profRegex = regexp.MustCompile(`--profiling (\d+)`) var promRegex = regexp.MustCompile(`--promPort (\d+)`) @@ -127,15 +115,6 @@ func GetPollerStatuses() ([]PollerStatus, error) { return result, nil } -func Contains(s []string, e string) bool { - for _, a := range s { - if a == e { - return true - } - } - return false -} - func FindLocalIP() (string, error) { conn, err := net.Dial("udp", "1.1.1.1:80") if err != nil { @@ -310,28 +289,20 @@ func SumNumbers(s []float64) float64 { return total } +// Max returns 0 when passed an empty slice, slices.Max panics if input is empty +// This function can be removed once all callers are checked for empty slices func Max(input []float64) float64 { if len(input) > 0 { - max := input[0] - for _, v := range input { - if v > max { - max = v - } - } - return max + return slices.Max(input) } return 0 } +// Min returns 0 when passed an empty slice, slices.Min panics if input is empty +// This function can be removed once all callers are checked for empty slices func Min(input []float64) float64 { if len(input) > 0 { - min := input[0] - for _, v := range input { - if v < min { - min = v - } - } - return min + return slices.Min(input) } return 0 } From d5592f27fc13dc8a288bb5ac136893dc07281003 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Mon, 14 Aug 2023 14:09:44 +0530 Subject: [PATCH 20/40] doc: move troubleshoot docs to doc site --- docs/help/faq.md | 214 +++++++++++++++++++++++++++ docs/help/log-collection.md | 29 ++++ docs/help/troubleshooting.md | 271 +++++++++++++++++++++++++++++++++++ docs/troubleshooting.md | 3 - mkdocs.yml | 5 +- 5 files changed, 518 insertions(+), 4 deletions(-) create mode 100644 docs/help/faq.md create mode 100644 docs/help/log-collection.md create mode 100644 docs/help/troubleshooting.md delete mode 100644 docs/troubleshooting.md diff --git a/docs/help/faq.md b/docs/help/faq.md new file mode 100644 index 000000000..a84108b4e --- /dev/null +++ b/docs/help/faq.md @@ -0,0 +1,214 @@ +## How do I migrate from Harvest 1.6 to 2.0? +There currently is not a tool to migrate data from Harvest 1.6 to 2.0. The most common workaround is to run both, 1.6 and 2.0, in parallel. Run both, until the 1.6 data expires due to normal retention policy, and then fully cut over to 2.0. + +Technically, it’s possible to take a Graphite DB, extract the data, and send it to a Prometheus db, but it’s not an area we’ve invested in. +If you want to explore that option, check out the [promtool](https://github.com/prometheus/prometheus/issues/8280) which supports importing, but probably not worth the effort. + +## How do I share sensitive log files with NetApp? +Email them to [ng-harvest-files@netapp.com](mailto:ng-harvest-files@netapp.com) +This mail address is accessible to NetApp Harvest employees only. + +## Multi-tenancy +### Question +Is there a way to allow per SVM level user views? I need to offer 1 tenant per SVM. Can I limit visibility to specific SVMs? Is there an SVM dashboard available? + +### Answer +You can do this with Grafana. Harvest can provide the labels for SVMs. The pieces are there but need to be put together. + +Grafana templates support the $__user variable to make pre-selections and decisions. +You can use that + metadata mapping the user <-> SVM. With both of those you can build SVM specific dashboards. + +There is a German service provider who is doing this. They have service managers responsible for a set of customers – and only want to see the data/dashboards of their corresponding customers. + +## Harvest Authentication and Permissions +### Question +What permissions does Harvest need to talk to ONTAP? +### Answer +Permissions, authentication, role based security, and creating a Harvest user are covered [here](https://netapp.github.io/harvest/prepare-cdot-clusters/). + +## ONTAP counters are missing +### Question +How do I make Harvest collect additional ONTAP counters? + +### Answer +Instead of modifying the out-of-the-box templates in the `conf/` directory, it is better to create your own custom templates following these [instructions](https://github.com/NetApp/harvest/blob/main/cmd/collectors/zapiperf/README.md#creatingediting-subtemplates). + +## Capacity Metrics +### Question +How are capacity and other metrics calculated by Harvest? +### Answer +Each collector has its own way of collecting and post-processing metrics. Check the documentation of each individual collector (usually under section #Metrics). Capacity and hardware-related metrics are collected by the [Zapi collector](https://github.com/NetApp/harvest/tree/main/cmd/collectors/zapi/README.md#Metrics) which emits metrics as they are without any additional calculation. Performance metrics are collected by the [ZapiPerf collector](https://github.com/NetApp/harvest/tree/main/cmd/collectors/zapiperf/README.md#Metrics) and the final values are calculated from the delta of two consequent polls. + +## Tagging Volumes +### Question +How do I tag ONTAP volumes with metadata and surface that data in Harvest? +### Answer +See [volume tagging issue](https://github.com/NetApp/harvest/issues/209#issuecomment-879751733) and [volume tagging via sub-templates](https://github.com/NetApp/harvest/issues/309#issuecomment-882553771) + +## REST and Zapi Documentation +### Question +How do I relate ONTAP REST endpoints to ZAPI APIs and attributes? +### Answer +Please refer to the [ONTAPI to REST API mapping document](https://library.netapp.com/ecm/ecm_download_file/ECMLP2874886). + +## Sizing +How much disk space is required by Prometheus? + +This depends on the collectors you've added, # of nodes monitored, cardinality of labels, # instances, retention, ingest rate, etc. A good approximation is to curl your Harvest exporter and count the number of samples that it publishes and then feed that information into a Prometheus sizing formula. + +Prometheus stores an average of 1-2 bytes per sample. To plan the capacity of a Prometheus server, you can use the rough formula: needed_disk_space = retention_time_seconds * ingested_samples_per_second * bytes_per_sample + +A rough approximation is outlined https://devops.stackexchange.com/questions/9298/how-to-calculate-disk-space-required-by-prometheus-v2-2 + +## Topk usage in Grafana + +### Question +In Grafana, why do I see more results from topk than I asked for? + +### Answer +[Topk](https://prometheus.io/docs/prometheus/latest/querying/operators/#aggregation-operators) is one of Prometheus's out-of-the-box aggregation operators, and is used to calculate the **largest k elements by sample value**. + +Depending on the time range you select, Prometheus will often return more results than you asked for. That's because Prometheus is picking the topk for each time in the graph. In other words, different time series are the topk at different times in the graph. When you use a large duration, there are often many time series. + +This is a limitation of Prometheus and can be mitigated by: + +- reducing the time range to a smaller duration that includes fewer topk results - something like a five to ten minute range works well for most of Harvest's charts +- the panel's table shows the current topk rows and that data can be used to supplement the additional series shown in the charts + +Additional details: [here](https://stackoverflow.com/questions/38783424/prometheus-topk-returns-more-results-than-expected), [here](https://www.robustperception.io/graph-top-n-time-series-in-grafana), and [here](https://grafana.com/docs/grafana/latest/datasources/prometheus/) + +## Where are Harvest container images published? + +Harvest images are published to both NetApp's (cr.netapp.io) and Docker's (hub.docker.com) image registry. By default, cr.netapp.io is used. + +### How do I switch from DockerHub to NetApp's image registry (cr.netapp.io) or vice-versa? + +### Answer + +Replace all instances of `rahulguptajss/harvest:latest` with `cr.netapp.io/harvest:latest` + +- Edit your docker-compose file and make those replacements or regenerate the compose file using the `--image cr.netapp.io/harvest:latest` option) + +- Update any shell or Ansible scripts you have that are also using those images + +- After making these changes, you should stop your containers, pull new images, and restart. + +You can verify that you're using the cr.netapp.io images like so: + +**Before** + +``` +docker image ls -a +REPOSITORY TAG IMAGE ID CREATED SIZE +rahulguptajss/harvest latest 80061bbe1c2c 10 days ago 85.4MB <=== no prefix in the repository +prom/prometheus v2.33.1 e528f02c45a6 3 weeks ago 204MB column means from DockerHub +grafana/grafana 8.3.4 4a34578e4374 5 weeks ago 274MB +``` + +**Pull image from cr.netapp.io** + +``` +docker pull cr.netapp.io/harvest +Using default tag: latest +latest: Pulling from harvest +Digest: sha256:6ff88153812ebb61e9dd176182bf8a792cde847748c5654d65f4630e61b1f3ae +Status: Image is up to date for cr.netapp.io/harvest:latest +cr.netapp.io/harvest:latest +``` + +Notice that the `IMAGE ID` for both images are identical since the images are the same. + +``` +docker image ls -a +REPOSITORY TAG IMAGE ID CREATED SIZE +cr.netapp.io/harvest latest 80061bbe1c2c 10 days ago 85.4MB <== Harvest image from cr.netapp.io +rahulguptajss/harvest latest 80061bbe1c2c 10 days ago 85.4MB +prom/prometheus v2.33.1 e528f02c45a6 3 weeks ago 204MB +grafana/grafana 8.3.4 4a34578e4374 5 weeks ago 274MB +grafana/grafana latest 1d60b4b996ad 2 months ago 275MB +prom/prometheus latest c10e9cbf22cd 3 months ago 194MB +``` + +We can now remove the DockerHub pulled image + +``` +docker image rm rahulguptajss/harvest +Untagged: rahulguptajss/harvest:latest +Untagged: rahulguptajss/harvest@sha256:6ff88153812ebb61e9dd176182bf8a792cde847748c5654d65f4630e61b1f3ae + +docker image ls -a +REPOSITORY TAG IMAGE ID CREATED SIZE +cr.netapp.io/harvest latest 80061bbe1c2c 10 days ago 85.4MB +prom/prometheus v2.33.1 e528f02c45a6 3 weeks ago 204MB +grafana/grafana 8.3.4 4a34578e4374 5 weeks ago 274MB +``` + +## Ports + +### What ports does Harvest use? + +### Answer + + + +The default ports are shown in the following diagram. + +![h](https://user-images.githubusercontent.com/242252/172162362-49909154-623e-4feb-9e13-e9f9f75d2b69.svg) + +- Harvest's pollers use ZAPI or REST to communicate with ONTAP on port `443` +- Each poller exposes the Prometheus port defined in your `harvest.yml` file +- Prometheus scrapes each poller-exposed Prometheus port (`promPort1`, `promPort2`, `promPort3`) +- Prometheus's default port is `9090` +- Grafana's default port is `3000` + + +## Snapmirror_labels + +### Why do my snapmirror_labels have an empty source_node? + +### Answer + +Snapmirror relationships have a source and destination node. ONTAP however does not expose the source side of that relationship, only the destination side is returned via ZAPI/REST APIs. Because of that, the Prometheus metric named, `snapmirror_labels`, will have an empty `source_node` label. + +The dashboards show the correct value for `source_node` since we join multiple metrics in the Grafana panels to synthesize that information. + +In short: don't rely on the `snapmirror_labels` for `source_node` labels. If you need `source_node` you will need to do a similar join as the Snapmirror dashboard does. + +See https://github.com/NetApp/harvest/issues/1192 for more information and linked pull requests for REST and ZAPI. + +## NFS Clients Dashboard + +### Why do my NFS Clients Dashboard have no data? + +### Answer + +NFS Clients dashboard is only available through Rest Collector. This information is not available through Zapi. You must enable the Rest collector in your harvest.yml config and uncomment the nfs_clients.yaml section in your [default.yaml](https://github.com/NetApp/harvest/blob/main/conf/rest/default.yaml) file. + +**Note:** Enabling nfs_clients.yaml may slow down data collection. + +## File Analytics Dashboard + +### Why do my File Analytics Dashboard have no data? + +### Answer + +This dashboard requires ONTAP 9.8+ and the APIs are only available via REST. Please enable the REST collector in your harvest config. To collect and display usage data such as capacity analytics, you need to enable File System Analytics on a volume. Please see https://docs.netapp.com/us-en/ontap/task_nas_file_system_analytics_enable.html for more details. + +## Why do I have Volume Sis Stat panel empty in Volume dashboard? + +### Answer + +This panel requires ONTAP 9.12+ and the APIs are only available via REST. Enable the REST collector in your `harvest.yml` config. \ No newline at end of file diff --git a/docs/help/log-collection.md b/docs/help/log-collection.md new file mode 100644 index 000000000..7de19a790 --- /dev/null +++ b/docs/help/log-collection.md @@ -0,0 +1,29 @@ +# Harvest Logs Collection Guide + +This guide will help you collect logs for Harvest on various platforms. Follow the instructions specific to your platform. If you would like to share the collected logs with the Harvest team, feel free to email them to `ng-harvest-files@netapp.com`. + +## RPM, DEB, and Native Installations + +For RPM, DEB, and native installations, use the following command to create a compressed tar file containing the logs: + +```bash +tar -czvf harvest_logs.tar.gz -C /var/log harvest +``` + +This command will create a file named `harvest_logs.tar.gz` with the contents of the `/var/log/harvest` directory. + +## Docker Container + +For Docker containers, first, identify the container ID for your Harvest instance. Then, replace `` with the actual container ID in the following command: + +```bash +docker logs &> harvest_logs.txt && tar -czvf harvest_logs.tar.gz harvest_logs.txt +``` + +This command will create a file named `harvest_logs.tar.gz` containing the logs from the specified container. + +## NABox + +For NABox installations, refer to the NABox documentation on collecting logs: + +[NABox Troubleshooting - Collecting Logs](https://nabox.org/documentation/troubleshooting/#collecting-logs) \ No newline at end of file diff --git a/docs/help/troubleshooting.md b/docs/help/troubleshooting.md new file mode 100644 index 000000000..dcc37ae83 --- /dev/null +++ b/docs/help/troubleshooting.md @@ -0,0 +1,271 @@ +# Checklists for Harvest +A set of steps to go through when something goes wrong. + +## What version of ONTAP do you have? +Run the following, replacing `` with the poller from your `harvest.yaml` + +``` +./bin/harvest zapi -p show system +``` + +Copy and paste the output into your issue. Here's an example: +``` +./bin/harvest -p infinity show system +connected to infinity (NetApp Release 9.8P2: Tue Feb 16 03:49:46 UTC 2021) +[results] - * + [build-timestamp] - 1613447386 + [is-clustered] - true + [version] - NetApp Release 9.8P2: Tue Feb 16 03:49:46 UTC 2021 + [version-tuple] - * + [system-version-tuple] - * + [generation] - 9 + [major] - 8 + [minor] - 0 + +``` + + +## Install fails + +I tried to install and ... + +## How do I tell if Harvest is doing anything? + +You believe Harvest is installed fine, but it's not working. + +* Post the contents of your `harvest.yml` + +Try validating your `harvest.yml` with `yamllint` like so: `yamllint -d relaxed harvest.yml` +If you do not have yamllint installed, look [here](https://yamllint.readthedocs.io/en/stable/quickstart.html). + +There should be no **errors** - warnings like the following are fine: +``` +harvest.yml + 64:1 warning too many blank lines (3 > 0) (empty-lines) +``` + +* How did you start Harvest? + +* What do you see in `/var/log/harvest/*` + +* What does `ps aux | grep poller` show? + +* If you are using Prometheus, try hitting Harvest's Prometheus endpoint like so: + +`curl http://machine-this-is-running-harvest:prometheus-port-in-harvest-yaml/metrics` + +* Check file ownership (user/group) and file permissions of your templates, executable, etc in your Harvest home directory (`ls -la /opt/harvest/`) [See also](https://github.com/NetApp/harvest/issues/931#issuecomment-1083305441). + +## How do I start Harvest in debug mode? + +Use the `--debug` flag when starting a poller. In debug mode, the poller will only collect metrics, but not write to databases. Another useful flag is `--foreground`, in which case all log messages are written to the terminal. Note that you can only start one poller in foreground mode. + +Finally, you can use `--loglevel=1` or `--verbose`, if you want to see a lot of log messages. For even more, you can use `--loglevel=0` or `--trace`. + +Examples: + +``` +bin/harvest start $POLLER_NAME --foreground --debug --loglevel=0 +or +bin/harvest start $POLLER_NAME --loglevel=1 --collectors Zapi --objects Qtree +``` + +## How do I start Harvest in foreground mode? +See [How do I start Harvest in debug mode?](#how-do-i-start-harvest-in-debug-mode) + +## How do I start my poller with only one collector? + +Since a poller will start a large number of collectors (each collector-object pair is treated as a collector), it is often hard to find the issue you are looking for in the abundance of log messages. It might be therefore useful to start one single collector-object pair when troubleshooting. You can use the `--collectors` and `--objects` flags for that. For example, start only the ZapiPerf collector with the SystemNode object: + +`harvest start my_poller --collectors ZapiPerf --objects SystemNode` + +(To find to correct object name, check `conf/COLLECTOR/default.yaml` file of the collector). + +## Errors in the log file + +### Some of my clusters are not showing up in Grafana +The logs show these errors: +``` +context deadline exceeded (Client.Timeout or context cancellation while reading body) + +and then for each volume + +skipped instance [9c90facd-3730-48f1-b55c-afacc35c6dbe]: not found in cache +``` + +### Workarounds +> context deadline exceeded (Client.Timeout or context cancellation while reading body) + +means Harvest is timing out when talking to your cluster. This sometimes happens when you have a large number of resources (e.g. volumes). + +There are a few parameters that you can change to avoid this from happening. You can do this by editing the subtemplate of the resource affected. E.g. you can add the parameters in `conf/zapiperf/cdot/9.8.0/volume.yaml` or `conf/zapi/cdot/9.8.0/volume.yaml`. If the errors happen for most of the resources, you can add them in the main template of the collector (`conf/zapi/default.yaml` or `conf/zapiperf/default.yaml`) to apply them on all objects. + +#### `client_timeout` + +Increase the `client_timeout` value by adding a `client_timeout` line at the beginning of the template, like so: + +```yaml +# increase the timeout to 60 seconds +client_timeout: 60s +``` + +#### `batch_size` + +Decrease the `batch_size` value by adding a `batch_size` line at the beginning of the template. The default value of this parameter is `500`. By decreasing it, the collector will fetch less instances during each API request. Example: + +```yaml +# decrease number of instances to 200 for each API request +batch_size: 200 +``` + +#### `schedule` + +If nothing else helps, you can increase the data poll interval of the collector (default is `60s` for ZapiPerf and `180s` for Zapi). You can do this either by adding a `schedule` attribute to the template or, if it already exists, by changing the `- data` line. + +Example for ZapiPerf: + +```yaml +# increase data poll frequency to 2 minutes +schedule: + - counter: 1200s + - instance: 600s + - data: 120s +``` +Example for Zapi: + +```yaml +# increase data poll frequency to 5 minutes +schedule: + - instance: 600s + - data: 300s +``` + +## Prometheus HTTP Service Discovery doesn't work + +Some things to check: + +- Make sure the Harvest admin node is started via `bin/harvest admin start` and there are no errors printed to the console +- Make sure your [`harvest.yml`](https://github.com/NetApp/harvest/blob/main/cmd/exporters/prometheus/README.md#enable-http-service-discovery-in-harvest) includes a valid `Admin:` section +- Ensure `bin/harvest doctor` runs without error. If it does, include the output of `bin/harvest doctor --print` in Slack or your GitHub issue +- Ensure your `/etc/prometheus/prometheus.yml` has a scrape config with `http_sd_configs` and it points to the admin node's `ip:port` +- Ensure there are no errors in your poller logs (`/var/log/harvest`) related to the poller publishing its Prometheus port to the admin node. Something like this should help narrow it down: `grep -R -E "error.*poller.go" /var/log/harvest/` + * If you see errors like `dial udp 1.1.1.1:80: connect: network is unreachable`, make sure your machine has a default route setup for your main interface +- If the admin node is running, your `harvest.yml` includes the `Admin:` section, and your pollers are using the Prometheus exporter you should be able to curl the admin node endpoint for a list of running Harvest pollers like this: +``` +curl -s -k https://localhost:8887/api/v1/sd | jq . +[ + { + "targets": [ + ":12994" + ], + "labels": { + "__meta_poller": "F2240-127-26" + } + }, + { + "targets": [ + ":39000" + ], + "labels": { + "__meta_poller": "simple1" + } + } +] +``` + +## How do I run Harvest commands in NAbox? + +NAbox is a vApp running Alpine Linux and Docker. NAbox runs Harvest as a set of Docker containers. That means to execute Harvest commands on NAbox, you need to `exec` into the container by following these commands. + +1. [ssh into your NAbox instance](https://nabox.org/documentation/configuration/) + +2. Start bash in the Harvest container + +```bash +dc exec nabox-harvest2 bash +``` + +You should see no errors and your prompt will change to something like `root@nabox-harvest2:/app#` + +Below are examples of running Harvest commands against a cluster named `umeng-aff300-05-06`. Replace with your cluster name as appropriate. + +```bash +# inside container + +> cat /etc/issue +Debian GNU/Linux 10 \n \l + +> cd /netapp-harvest +bin/harvest version +harvest version 22.08.0-1 (commit 93db10a) (build date 2022-08-19T09:10:05-0400) linux/amd64 +checking GitHub for latest... you have the latest ✓ + +# harvest.yml is found at /conf/harvest.yml + +> bin/zapi --poller umeng-aff300-05-06 show system +connected to umeng-aff300-05-06 (NetApp Release 9.9.1P9X3: Tue Apr 19 19:05:24 UTC 2022) +[results] - * + [build-timestamp] - 1650395124 + [is-clustered] - true + [version] - NetApp Release 9.9.1P9X3: Tue Apr 19 19:05:24 UTC 2022 + [version-tuple] - * + [system-version-tuple] - * + [generation] - 9 + [major] - 9 + [minor] - 1 + +bin/zapi -p umeng-aff300-05-06 show data --api environment-sensors-get-iter --max 10000 > env-sensor.xml +``` + +The `env-sensor.xml` file will be written to the `/opt/packages/harvest2` directory on the host. + +If needed, you can `scp` that file off NAbox and [share](https://github.com/NetApp/harvest/wiki/FAQ#how-do-i-share-sensitive-log-files-with-netapp) it with the Harvest team. + +## Rest Collector Auth errors? + +If you are seeing errors like `User is not authorized` or `not authorized for that command` while using `Rest` Collector. Follow below steps to make sure permissions are set correctly. + +1. Verify that user has permissions for relevant authentication method. + +`security login show -vserver ROOT_VSERVER -user-or-group-name harvest2 -application http` + +image + + +2. Verify that user has read-only permissions to api. + +``` +security login role show -role harvest2-role +``` + +image + +3. Verify if an entry is present for following command. + +``` +vserver services web access show -role harvest2-role -name rest +``` + +If It is missing then add an entry with following commands + +``` +vserver services web access create -vserver umeng-aff300-01-02 -name rest -role harvest2-role +``` + +## Why do I have gaps in my dashboards? + +Here are possible reasons and things to check: + +* Prometheus `scrape_interval` found via (http://$promIP:9090/config) +* Prometheus log files +* Harvest collector scrape interval check your: + * `conf/zapi/default.yaml` - default for config is 3m + * `conf/zapiperf/default.yaml` - default of perf is 1m +* Check you poller logs for any errors or lag messages +* When using [VictoriaMetrics](https://discord.com/channels/855068651522490400/1087312484215566426/1087356045531303936), make sure your Prometheus exporter config includes `sort_labels: true`, since VictoriaMetrics will mark series stale if the label order changes between polls. + +## NABox + +For NABox installations, refer to the NABox documentation on troubleshooting: + +[NABox Troubleshooting](https://nabox.org/documentation/troubleshooting/) \ No newline at end of file diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md deleted file mode 100644 index e38dffac2..000000000 --- a/docs/troubleshooting.md +++ /dev/null @@ -1,3 +0,0 @@ -- [Troubleshooting Harvest](https://github.com/NetApp/harvest/wiki/Troubleshooting-Harvest) -- [FAQ](https://github.com/NetApp/harvest/wiki/FAQ) -- [NABox Troubleshooting](https://nabox.org/documentation/troubleshooting/) \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index c04d98ff8..7458f0184 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -39,7 +39,10 @@ nav: - Manage Harvest Pollers: 'manage-harvest.md' - Configure Harvest (advanced): 'configure-harvest-advanced.md' - Monitor Harvest: 'monitor-harvest.md' - - Troubleshoot: 'troubleshooting.md' + - Help: + - 'Troubleshooting': 'help/troubleshooting.md' + - 'FAQ': 'help/faq.md' + - 'Log Collection': 'help/log-collection.md' - Reference: - 'Plugins': 'plugins.md' - 'Matrix': 'resources/matrix.md' From a2605abef6071dcc8bce7e5360cbe2a6a6a316f8 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Mon, 14 Aug 2023 16:34:45 +0530 Subject: [PATCH 21/40] fix: match object name with zapiperf for cifs_vserver.yaml --- conf/restperf/9.12.0/{cifs_svm.yaml => cifs_vserver.yaml} | 2 +- conf/restperf/default.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename conf/restperf/9.12.0/{cifs_svm.yaml => cifs_vserver.yaml} (94%) diff --git a/conf/restperf/9.12.0/cifs_svm.yaml b/conf/restperf/9.12.0/cifs_vserver.yaml similarity index 94% rename from conf/restperf/9.12.0/cifs_svm.yaml rename to conf/restperf/9.12.0/cifs_vserver.yaml index 770272c54..a4b88a499 100644 --- a/conf/restperf/9.12.0/cifs_svm.yaml +++ b/conf/restperf/9.12.0/cifs_vserver.yaml @@ -1,5 +1,5 @@ -name: CIFSsvm +name: CIFSvserver query: api/cluster/counter/tables/svm_cifs object: svm_cifs diff --git a/conf/restperf/default.yaml b/conf/restperf/default.yaml index b3d59f1eb..d6c21d8d0 100644 --- a/conf/restperf/default.yaml +++ b/conf/restperf/default.yaml @@ -34,7 +34,7 @@ objects: # NFSv4Pool: nfsv4_pool.yaml # SVM-level metrics - CIFSsvm: cifs_svm.yaml + CIFSvserver: cifs_vserver.yaml CopyManager: copy_manager.yaml FcpLif: fcp_lif.yaml ISCSI: iscsi_lif.yaml From 10b94fc812e3e50d6748c2fd8acb7b62a776b712 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Mon, 14 Aug 2023 17:02:52 +0530 Subject: [PATCH 22/40] fix: add bin dir check before removing files --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bc9e3ce8c..355d75de7 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,9 @@ endif clean: header ## Cleanup the project binary (bin) folders @echo "Cleaning harvest files" - @if test -d bin; then ls -d ./bin/* | grep -v "asup" | xargs rm -f; fi + @if [ -d bin ]; then \ + ls -d ./bin/* | grep -v "asup" | xargs rm -f; \ + fi test: ## run tests @echo "Running tests" From bd0b0db795e5d5a2a0944e4e8f1d09b47aae23db Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Mon, 14 Aug 2023 17:07:16 +0530 Subject: [PATCH 23/40] doc: release 23.08 metric docs --- cmd/tools/generate/counter.go | 3 + docs/ontap-metrics.md | 164 +++++----------------------------- 2 files changed, 25 insertions(+), 142 deletions(-) diff --git a/cmd/tools/generate/counter.go b/cmd/tools/generate/counter.go index a10dab613..db892ec0f 100644 --- a/cmd/tools/generate/counter.go +++ b/cmd/tools/generate/counter.go @@ -40,8 +40,11 @@ var ( "read_latency_histogram": {}, "latency_histogram": {}, "nfsv3_latency_hist": {}, + "nfs4_latency_hist": {}, "read_latency_hist": {}, "write_latency_hist": {}, + "total.latency_histogram": {}, + "nfs41_latency_hist": {}, } // Excludes these Rest gaps from logs excludeLogRestCounters = []string{ diff --git a/docs/ontap-metrics.md b/docs/ontap-metrics.md index 61ee2b149..d5401ec73 100644 --- a/docs/ontap-metrics.md +++ b/docs/ontap-metrics.md @@ -7,7 +7,7 @@ These can be generated on demand by running `bin/harvest grafana metrics`. See - More information about ONTAP REST performance counters can be found [here](https://docs.netapp.com/us-en/ontap-pcmap-9121/index.html). ``` -Creation Date : 2023-Aug-09 +Creation Date : 2023-Aug-14 ONTAP Version: 9.13.1 ``` ## Understanding the structure @@ -7213,8 +7213,8 @@ This is the average number of concurrent requests for the workload. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos` | `concurrency`
Unit: none
Type: rate
Base: | conf/restperf/9.12.0/workload.yaml | -| ZAPI | `perf-object-get-instances workload` | `concurrency`
Unit: none
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload.yaml | +| REST | `api/cluster/counter/tables/qos_volume` | `concurrency`
Unit: none
Type: rate
Base: | conf/restperf/9.12.0/workload_volume.yaml | +| ZAPI | `perf-object-get-instances workload_volume` | `concurrency`
Unit: none
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | ### qos_detail_resource_latency @@ -7227,127 +7227,37 @@ average latency for workload on Data ONTAP subsystems | ZAPI | `perf-object-get-instances workload_detail` | `Harvest generated`
Unit: microseconds
Type:
Base: | conf/zapiperf/9.12.0/workload_detail.yaml | -### qos_detail_volume_resource_latency - -average latency for volume on Data ONTAP subsystems - -| API | Endpoint | Metric | Template | -|--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos_detail_volume` | `Harvest generated`
Unit: microseconds
Type:
Base: | conf/restperf/9.12.0/workload_detail_volume.yaml | -| ZAPI | `perf-object-get-instances workload_detail_volume` | `Harvest generated`
Unit: microseconds
Type:
Base: | conf/zapiperf/9.12.0/workload_detail_volume.yaml | - - ### qos_latency This is the average response time for requests that were initiated by the workload. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos` | `latency`
Unit: microsec
Type: average
Base: ops | conf/restperf/9.12.0/workload.yaml | -| ZAPI | `perf-object-get-instances workload` | `latency`
Unit: microsec
Type: average,no-zero-values
Base: ops | conf/zapiperf/cdot/9.8.0/workload.yaml | +| REST | `api/cluster/counter/tables/qos_volume` | `latency`
Unit: microsec
Type: average
Base: ops | conf/restperf/9.12.0/workload_volume.yaml | +| ZAPI | `perf-object-get-instances workload_volume` | `latency`
Unit: microsec
Type: average,no-zero-values
Base: ops | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | ### qos_ops -Workload operations executed per second. - -| API | Endpoint | Metric | Template | -|--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos` | `ops`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.12.0/workload.yaml | -| ZAPI | `perf-object-get-instances workload` | `ops`
Unit: per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload.yaml | - - -### qos_read_data - -This is the amount of data read per second from the filer by the workload. - -| API | Endpoint | Metric | Template | -|--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos` | `read_data`
Unit: b_per_sec
Type: rate
Base: | conf/restperf/9.12.0/workload.yaml | -| ZAPI | `perf-object-get-instances workload` | `read_data`
Unit: b_per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload.yaml | - - -### qos_read_io_type - -This is the percentage of read requests served from various components (such as buffer cache, ext_cache, disk, etc.). - -| API | Endpoint | Metric | Template | -|--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos` | `read_io_type_percent`
Unit: percent
Type: percent
Base: read_io_type_base | conf/restperf/9.12.0/workload.yaml | -| ZAPI | `perf-object-get-instances workload` | `read_io_type`
Unit: percent
Type: percent
Base: read_io_type_base | conf/zapiperf/cdot/9.8.0/workload.yaml | - - -### qos_read_latency - -This is the average response time for read requests that were initiated by the workload. - -| API | Endpoint | Metric | Template | -|--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos` | `read_latency`
Unit: microsec
Type: average
Base: read_ops | conf/restperf/9.12.0/workload.yaml | -| ZAPI | `perf-object-get-instances workload` | `read_latency`
Unit: microsec
Type: average,no-zero-values
Base: read_ops | conf/zapiperf/cdot/9.8.0/workload.yaml | - - -### qos_read_ops - -This is the rate of this workload's read operations that completed during the measurement interval. - -| API | Endpoint | Metric | Template | -|--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos` | `read_ops`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.12.0/workload.yaml | -| ZAPI | `perf-object-get-instances workload` | `read_ops`
Unit: per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload.yaml | - - -### qos_sequential_reads - -This is the percentage of reads, performed on behalf of the workload, that were sequential. - -| API | Endpoint | Metric | Template | -|--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos` | `sequential_reads_percent`
Unit: percent
Type: percent
Base: sequential_reads_base | conf/restperf/9.12.0/workload.yaml | -| ZAPI | `perf-object-get-instances workload` | `sequential_reads`
Unit: percent
Type: percent,no-zero-values
Base: sequential_reads_base | conf/zapiperf/cdot/9.8.0/workload.yaml | - - -### qos_sequential_writes - -This is the percentage of writes, performed on behalf of the workload, that were sequential. This counter is only available on platforms with more than 4GB of NVRAM. - -| API | Endpoint | Metric | Template | -|--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos` | `sequential_writes_percent`
Unit: percent
Type: percent
Base: sequential_writes_base | conf/restperf/9.12.0/workload.yaml | -| ZAPI | `perf-object-get-instances workload` | `sequential_writes`
Unit: percent
Type: percent,no-zero-values
Base: sequential_writes_base | conf/zapiperf/cdot/9.8.0/workload.yaml | - - -### qos_total_data - -This is the total amount of data read/written per second from/to the filer by the workload. - -| API | Endpoint | Metric | Template | -|--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos` | `total_data`
Unit: b_per_sec
Type: rate
Base: | conf/restperf/9.12.0/workload.yaml | -| ZAPI | `perf-object-get-instances workload` | `total_data`
Unit: b_per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload.yaml | - - -### qos_volume_latency - -This is the average response time for requests that were initiated by the workload. +This field is the workload's rate of operations that completed during the measurement interval; measured per second. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos_volume` | `latency`
Unit: microsec
Type: average
Base: ops | conf/restperf/9.12.0/workload_volume.yaml | -| ZAPI | `perf-object-get-instances workload_volume` | `latency`
Unit: microsec
Type: average,no-zero-values
Base: ops | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | +| REST | `api/cluster/counter/tables/qos_volume` | `ops`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.12.0/workload_volume.yaml | +| ZAPI | `perf-object-get-instances workload_volume` | `ops`
Unit: per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | -### qos_volume_ops +### qos_other_ops -This field is the workload's rate of operations that completed during the measurement interval; measured per second. +This is the rate of this workload's other operations that completed during the measurement interval. | API | Endpoint | Metric | Template | |--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos_volume` | `ops`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.12.0/workload_volume.yaml | -| ZAPI | `perf-object-get-instances workload_volume` | `ops`
Unit: per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | +| REST | `api/cluster/counter/tables/qos` | `other_ops`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.12.0/workload.yaml | +| ZAPI | `perf-object-get-instances workload_volume` | `other_ops`
Unit: per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | -### qos_volume_read_data +### qos_read_data This is the amount of data read per second from the filer by the workload. @@ -7357,7 +7267,7 @@ This is the amount of data read per second from the filer by the workload. | ZAPI | `perf-object-get-instances workload_volume` | `read_data`
Unit: b_per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | -### qos_volume_read_io_type +### qos_read_io_type This is the percentage of read requests served from various components (such as buffer cache, ext_cache, disk, etc.). @@ -7367,7 +7277,7 @@ This is the percentage of read requests served from various components (such as | ZAPI | `perf-object-get-instances workload_volume` | `read_io_type`
Unit: percent
Type: percent
Base: read_io_type_base | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | -### qos_volume_read_latency +### qos_read_latency This is the average response time for read requests that were initiated by the workload. @@ -7377,7 +7287,7 @@ This is the average response time for read requests that were initiated by the w | ZAPI | `perf-object-get-instances workload_volume` | `read_latency`
Unit: microsec
Type: average,no-zero-values
Base: read_ops | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | -### qos_volume_read_ops +### qos_read_ops This is the rate of this workload's read operations that completed during the measurement interval. @@ -7387,7 +7297,7 @@ This is the rate of this workload's read operations that completed during the me | ZAPI | `perf-object-get-instances workload_volume` | `read_ops`
Unit: per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | -### qos_volume_sequential_reads +### qos_sequential_reads This is the percentage of reads, performed on behalf of the workload, that were sequential. @@ -7397,7 +7307,7 @@ This is the percentage of reads, performed on behalf of the workload, that were | ZAPI | `perf-object-get-instances workload_volume` | `sequential_reads`
Unit: percent
Type: percent,no-zero-values
Base: sequential_reads_base | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | -### qos_volume_sequential_writes +### qos_sequential_writes This is the percentage of writes, performed on behalf of the workload, that were sequential. This counter is only available on platforms with more than 4GB of NVRAM. @@ -7407,7 +7317,7 @@ This is the percentage of writes, performed on behalf of the workload, that were | ZAPI | `perf-object-get-instances workload_volume` | `sequential_writes`
Unit: percent
Type: percent,no-zero-values
Base: sequential_writes_base | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | -### qos_volume_total_data +### qos_total_data This is the total amount of data read/written per second from/to the filer by the workload. @@ -7417,7 +7327,7 @@ This is the total amount of data read/written per second from/to the filer by th | ZAPI | `perf-object-get-instances workload_volume` | `total_data`
Unit: b_per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | -### qos_volume_write_data +### qos_write_data This is the amount of data written per second to the filer by the workload. @@ -7427,7 +7337,7 @@ This is the amount of data written per second to the filer by the workload. | ZAPI | `perf-object-get-instances workload_volume` | `write_data`
Unit: b_per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | -### qos_volume_write_latency +### qos_write_latency This is the average response time for write requests that were initiated by the workload. @@ -7437,7 +7347,7 @@ This is the average response time for write requests that were initiated by the | ZAPI | `perf-object-get-instances workload_volume` | `write_latency`
Unit: microsec
Type: average,no-zero-values
Base: write_ops | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | -### qos_volume_write_ops +### qos_write_ops This is the workload's write operations that completed during the measurement interval; measured per second. @@ -7447,36 +7357,6 @@ This is the workload's write operations that completed during the measurement in | ZAPI | `perf-object-get-instances workload_volume` | `write_ops`
Unit: per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload_volume.yaml | -### qos_write_data - -This is the amount of data written per second to the filer by the workload. - -| API | Endpoint | Metric | Template | -|--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos` | `write_data`
Unit: b_per_sec
Type: rate
Base: | conf/restperf/9.12.0/workload.yaml | -| ZAPI | `perf-object-get-instances workload` | `write_data`
Unit: b_per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload.yaml | - - -### qos_write_latency - -This is the average response time for write requests that were initiated by the workload. - -| API | Endpoint | Metric | Template | -|--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos` | `write_latency`
Unit: microsec
Type: average
Base: write_ops | conf/restperf/9.12.0/workload.yaml | -| ZAPI | `perf-object-get-instances workload` | `write_latency`
Unit: microsec
Type: average,no-zero-values
Base: write_ops | conf/zapiperf/cdot/9.8.0/workload.yaml | - - -### qos_write_ops - -This is the workload's write operations that completed during the measurement interval; measured per second. - -| API | Endpoint | Metric | Template | -|--------|----------|--------|---------| -| REST | `api/cluster/counter/tables/qos` | `write_ops`
Unit: per_sec
Type: rate
Base: | conf/restperf/9.12.0/workload.yaml | -| ZAPI | `perf-object-get-instances workload` | `write_ops`
Unit: per_sec
Type: rate,no-zero-values
Base: | conf/zapiperf/cdot/9.8.0/workload.yaml | - - ### qtree_cifs_ops Number of CIFS operations per second to the qtree From e320b582c718651557565ebbfcb43b756ba7b5e7 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Mon, 14 Aug 2023 08:33:00 -0400 Subject: [PATCH 24/40] chore: minor release issue fixes (#2293) --- .github/ISSUE_TEMPLATE/release-template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE/release-template.md b/.github/ISSUE_TEMPLATE/release-template.md index ce3be7f02..f3d0f32ae 100644 --- a/.github/ISSUE_TEMPLATE/release-template.md +++ b/.github/ISSUE_TEMPLATE/release-template.md @@ -20,6 +20,7 @@ git push origin release/$RELEASE ```bash RELEASE=23.02.0 git clone https://github.com/NetApp/harvest-metrics.git +cd harvest-metrics git checkout origin/HEAD git switch --create release/$RELEASE git push origin release/$RELEASE From 36a4403cbf28585c9d715212c6ba6c339e6069e7 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 16 Aug 2023 11:39:13 +0530 Subject: [PATCH 25/40] feat: add volume other latency to volume dashboard Fixes #2247 --- grafana/dashboards/cmode/volume.json | 258 +++++++++++++++++++-------- 1 file changed, 186 insertions(+), 72 deletions(-) diff --git a/grafana/dashboards/cmode/volume.json b/grafana/dashboards/cmode/volume.json index 29d209b28..ef144d091 100644 --- a/grafana/dashboards/cmode/volume.json +++ b/grafana/dashboards/cmode/volume.json @@ -71,7 +71,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1691050019799, + "iteration": 1692165657901, "links": [ { "asDropdown": true, @@ -1827,7 +1827,7 @@ } ] }, - "unit": "Bps" + "unit": "µs" }, "overrides": [] }, @@ -1837,7 +1837,7 @@ "x": 8, "y": 15 }, - "id": 31, + "id": 33, "options": { "legend": { "calcs": [ @@ -1855,16 +1855,15 @@ "pluginVersion": "8.1.8", "targets": [ { - "expr": "topk($TopResources, volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeReadThroughput\"})", + "expr": "topk($TopResources, volume_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeWriteLatency\"})", "interval": "", - "intervalFactor": 1, "legendFormat": "{{svm}} - {{volume}}", "refId": "A" } ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources Volumes by Read Throughput", + "title": "Top $TopResources Volumes by Write Latency", "transformations": [], "type": "timeseries" }, @@ -1918,7 +1917,7 @@ } ] }, - "unit": "iops" + "unit": "µs" }, "overrides": [] }, @@ -1928,7 +1927,7 @@ "x": 16, "y": 15 }, - "id": 32, + "id": 116, "options": { "legend": { "calcs": [ @@ -1946,7 +1945,8 @@ "pluginVersion": "8.1.8", "targets": [ { - "expr": "topk($TopResources, volume_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeReadIOPS\"})", + "exemplar": false, + "expr": "topk($TopResources, volume_other_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeOtherLatency\"})", "interval": "", "legendFormat": "{{svm}} - {{volume}}", "refId": "A" @@ -1954,7 +1954,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources Volumes by Read IOPs", + "title": "Top $TopResources Volumes by Other Latency", "transformations": [], "type": "timeseries" }, @@ -2008,7 +2008,7 @@ } ] }, - "unit": "µs" + "unit": "iops" }, "overrides": [] }, @@ -2018,7 +2018,7 @@ "x": 0, "y": 24 }, - "id": 33, + "id": 32, "options": { "legend": { "calcs": [ @@ -2036,7 +2036,7 @@ "pluginVersion": "8.1.8", "targets": [ { - "expr": "topk($TopResources, volume_write_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeWriteLatency\"})", + "expr": "topk($TopResources, volume_read_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeReadIOPS\"})", "interval": "", "legendFormat": "{{svm}} - {{volume}}", "refId": "A" @@ -2044,7 +2044,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources Volumes by Write Latency", + "title": "Top $TopResources Volumes by Read IOPs", "transformations": [], "type": "timeseries" }, @@ -2098,7 +2098,7 @@ } ] }, - "unit": "Bps" + "unit": "iops" }, "overrides": [] }, @@ -2108,7 +2108,7 @@ "x": 8, "y": 24 }, - "id": 34, + "id": 35, "options": { "legend": { "calcs": [ @@ -2126,7 +2126,7 @@ "pluginVersion": "8.1.8", "targets": [ { - "expr": "topk($TopResources, volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeWriteThroughput\"})", + "expr": "topk($TopResources, volume_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeWriteIOPS\"})", "interval": "", "legendFormat": "{{svm}} - {{volume}}", "refId": "A" @@ -2134,7 +2134,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources Volumes by Write Throughput", + "title": "Top $TopResources Volumes by Write IOPs", "transformations": [], "type": "timeseries" }, @@ -2198,7 +2198,7 @@ "x": 16, "y": 24 }, - "id": 35, + "id": 115, "options": { "legend": { "calcs": [ @@ -2216,7 +2216,8 @@ "pluginVersion": "8.1.8", "targets": [ { - "expr": "topk($TopResources, volume_write_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeWriteIOPS\"})", + "exemplar": false, + "expr": "topk($TopResources, volume_other_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeOtherIOPS\"})", "interval": "", "legendFormat": "{{svm}} - {{volume}}", "refId": "A" @@ -2224,7 +2225,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Top $TopResources Volumes by Write IOPs", + "title": "Top $TopResources Volumes by Other IOPs", "transformations": [], "type": "timeseries" }, @@ -2244,12 +2245,11 @@ "fillOpacity": 10, "gradientMode": "none", "hideFrom": { - "graph": false, "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "smooth", + "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -2270,22 +2270,26 @@ "mode": "absolute", "steps": [ { - "color": "semi-dark-blue", + "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] }, - "unit": "µs" + "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 9, - "w": 8, + "w": 12, "x": 0, "y": 33 }, - "id": 112, + "id": 31, "options": { "legend": { "calcs": [ @@ -2300,28 +2304,110 @@ "mode": "single" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "8.1.8", "targets": [ { - "exemplar": false, - "expr": "avg by(__name__) ({__name__=~\"volume_nfs_(.*)_latency\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", - "hide": false, - "instant": false, + "expr": "topk($TopResources, volume_read_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeReadThroughput\"})", "interval": "", - "legendFormat": "{{__name__}}", - "refId": "Read Throughput" + "intervalFactor": 1, + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" } ], - "title": "Volume Latency by Op Type", - "transformations": [ + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Volumes by Read Throughput", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.1.8", + "targets": [ { - "id": "renameByRegex", - "options": { - "regex": "volume_nfs_(.*)_latency", - "renamePattern": "$1" - } + "expr": "topk($TopResources, volume_write_data{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeWriteThroughput\"})", + "interval": "", + "legendFormat": "{{svm}} - {{volume}}", + "refId": "A" } ], + "timeFrom": null, + "timeShift": null, + "title": "Top $TopResources Volumes by Write Throughput", + "transformations": [], "type": "timeseries" }, { @@ -2371,21 +2457,23 @@ } ] }, - "unit": "iops" + "unit": "µs" }, "overrides": [] }, "gridPos": { "h": 9, - "w": 8, - "x": 8, - "y": 33 + "w": 12, + "x": 0, + "y": 42 }, - "id": 114, + "id": 112, "options": { "legend": { "calcs": [ - "sum" + "mean", + "lastNotNull", + "max" ], "displayMode": "table", "placement": "bottom" @@ -2398,19 +2486,20 @@ "targets": [ { "exemplar": false, - "expr": "sum by (__name__) ({__name__=~\"volume_nfs_(.*)_ops\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "expr": "avg by(__name__) ({__name__=~\"volume_nfs_(.*)_latency\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", "hide": false, + "instant": false, "interval": "", "legendFormat": "{{__name__}}", "refId": "Read Throughput" } ], - "title": "Volume IOPs per Type", + "title": "Volume Latency by Op Type", "transformations": [ { "id": "renameByRegex", "options": { - "regex": "volume_nfs_(.*)_ops", + "regex": "volume_nfs_(.*)_latency", "renamePattern": "$1" } } @@ -2433,11 +2522,12 @@ "fillOpacity": 10, "gradientMode": "none", "hideFrom": { + "graph": false, "legend": false, "tooltip": false, "viz": false }, - "lineInterpolation": "linear", + "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -2458,12 +2548,8 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "semi-dark-blue", "value": null - }, - { - "color": "red", - "value": 80 } ] }, @@ -2473,17 +2559,15 @@ }, "gridPos": { "h": 9, - "w": 8, - "x": 16, - "y": 33 + "w": 12, + "x": 12, + "y": 42 }, - "id": 115, + "id": 114, "options": { "legend": { "calcs": [ - "mean", - "lastNotNull", - "max" + "sum" ], "displayMode": "table", "placement": "bottom" @@ -2492,20 +2576,27 @@ "mode": "single" } }, - "pluginVersion": "8.1.8", + "pluginVersion": "8.0.6", "targets": [ { "exemplar": false, - "expr": "topk($TopResources, volume_other_ops{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$TopVolumeOtherIOPS\"})", + "expr": "sum by (__name__) ({__name__=~\"volume_nfs_(.*)_ops\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\"})", + "hide": false, "interval": "", - "legendFormat": "{{svm}} - {{volume}}", - "refId": "A" + "legendFormat": "{{__name__}}", + "refId": "Read Throughput" + } + ], + "title": "Volume IOPs per Type", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "volume_nfs_(.*)_ops", + "renamePattern": "$1" + } } ], - "timeFrom": null, - "timeShift": null, - "title": "Top $TopResources Volumes by Other IOPs", - "transformations": [], "type": "timeseries" } ], @@ -6961,6 +7052,29 @@ "skipUrlSync": false, "sort": 0, "type": "query" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "query_result(topk($TopResources, avg_over_time(volume_other_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "description": null, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "TopVolumeOtherLatency", + "options": [], + "query": { + "query": "query_result(topk($TopResources, avg_over_time(volume_other_latency{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",svm=~\"$SVM\",volume=~\"$Volume\"}[${__range}])))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": ".*volume=\\\"(.*?)\\\".*", + "skipUrlSync": false, + "sort": 0, + "type": "query" } ] }, @@ -6984,5 +7098,5 @@ "timezone": "", "title": "ONTAP: Volume", "uid": "", - "version": 14 + "version": 15 } From 32b450c77c87e6158ee62398c65459e97e824c0c Mon Sep 17 00:00:00 2001 From: hardikl Date: Wed, 16 Aug 2023 16:30:28 +0530 Subject: [PATCH 26/40] fix: renaming nfs panels in svm dashboard --- grafana/dashboards/cmode/svm.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/grafana/dashboards/cmode/svm.json b/grafana/dashboards/cmode/svm.json index f009e2149..ad1e67e76 100644 --- a/grafana/dashboards/cmode/svm.json +++ b/grafana/dashboards/cmode/svm.json @@ -7743,7 +7743,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "NFSv4 SVMs by Latency by Op Type", + "title": "NFSv4 Latency by Op Type", "transformations": [ { "id": "renameByRegex", @@ -7843,7 +7843,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "NFSv4 SVMs by IOPs per Type", + "title": "NFSv4 IOPs per Type", "transformations": [ { "id": "renameByRegex", @@ -9051,7 +9051,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "NFSv4.1 SVMs by Latency by Op Type", + "title": "NFSv4.1 Latency by Op Type", "transformations": [ { "id": "renameByRegex", @@ -9151,7 +9151,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "NFSv4.1 SVMs by IOPs per Type", + "title": "NFSv4.1 IOPs per Type", "transformations": [ { "id": "renameByRegex", From 83ede0603f9d75206bc8c01979b1b0a6eca6f41a Mon Sep 17 00:00:00 2001 From: Rahul Date: Wed, 16 Aug 2023 17:47:11 +0530 Subject: [PATCH 27/40] chore: release doc update (#2294) * chore: release doc update * chore: address review comments --- .github/ISSUE_TEMPLATE/release-template.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/release-template.md b/.github/ISSUE_TEMPLATE/release-template.md index f3d0f32ae..87305d6a2 100644 --- a/.github/ISSUE_TEMPLATE/release-template.md +++ b/.github/ISSUE_TEMPLATE/release-template.md @@ -16,6 +16,18 @@ git checkout origin/HEAD git switch --create release/$RELEASE git push origin release/$RELEASE ``` +- [ ] If any changes are made in the [harvest autosupport](https://github.com/NetApp/harvest-private/tree/main/harvest-asup) repository, please update the harvest-metrics repository `main` branch with the latest `asup_linux_amd64` binary. This binary can be generated using Jenkins with the following parameters: + +| Field | Value | +|-----------------------------|-----------------| +| VERSION | 23.02.0 | +| RELEASE | 1 | +| BRANCH | release/23.02.0 | +| ASUP_MAKE_TARGET | production | +| DOCKER_PUBLISH | false | +| RUN_TEST | true | +| OVERWRITE_DOCKER_LATEST_TAG | false | + - [ ] Create a release branch for the harvest-metrics repo like so: ```bash RELEASE=23.02.0 @@ -43,7 +55,6 @@ go run pkg/changelog/main.go --title $RELEASE --highlights releaseHighlights_$RE ``` - [ ] Open a PR against the release branch with the generated release notes for review - [ ] PR approval -- [ ] Update metrics repo if needed #### Update Metrics Documentation ```bash From 483451f932891937489ca295612bd25570718c42 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Wed, 16 Aug 2023 08:21:59 -0400 Subject: [PATCH 28/40] Revert "feat: Harvest should include a rename label_agent" This reverts commit a2713c40d78801cc6e0fa28fe9474c98f0ad9f94. --- cmd/poller/plugin/labelagent/label_agent.go | 19 ------------ .../plugin/labelagent/label_agent_test.go | 22 +------------- cmd/poller/plugin/labelagent/parse_rules.go | 29 ------------------- docs/plugins.md | 26 +---------------- pkg/matrix/instance.go | 4 --- 5 files changed, 2 insertions(+), 98 deletions(-) diff --git a/cmd/poller/plugin/labelagent/label_agent.go b/cmd/poller/plugin/labelagent/label_agent.go index 3efef6e2e..bd43e29a8 100644 --- a/cmd/poller/plugin/labelagent/label_agent.go +++ b/cmd/poller/plugin/labelagent/label_agent.go @@ -20,7 +20,6 @@ type LabelAgent struct { splitRegexRules []splitRegexRule splitPairsRules []splitPairsRule joinSimpleRules []joinSimpleRule - renameRules []renameRule replaceSimpleRules []replaceSimpleRule replaceRegexRules []replaceRegexRule excludeEqualsRules []excludeEqualsRule @@ -138,24 +137,6 @@ func (a *LabelAgent) joinSimple(matrix *matrix.Matrix) error { return nil } -// rename source label, if present, to target label -// if target label already exists overwrite it -func (a *LabelAgent) rename(matrix *matrix.Matrix) error { - for _, instance := range matrix.GetInstances() { - for _, r := range a.renameRules { - if old := instance.GetLabel(r.source); old != "" { - instance.SetLabel(r.target, old) - instance.DeleteLabel(r.source) - a.Logger.Trace(). - Str("source", r.source). - Str("target", r.target). - Msg("rename") - } - } - } - return nil -} - // replace in source label, if present, and add as new label func (a *LabelAgent) replaceSimple(matrix *matrix.Matrix) error { for _, instance := range matrix.GetInstances() { diff --git a/cmd/poller/plugin/labelagent/label_agent_test.go b/cmd/poller/plugin/labelagent/label_agent_test.go index c3b83b374..14f10ec1d 100644 --- a/cmd/poller/plugin/labelagent/label_agent_test.go +++ b/cmd/poller/plugin/labelagent/label_agent_test.go @@ -54,7 +54,7 @@ func newLabelAgent() *LabelAgent { // create metric "result", if label "state" matches regex then map to 1 else use default value "4" params.NewChildS("value_to_num_regex", "").NewChildS("", "result value ^test\\d+ ^error `4`") - // These both are mutually exclusive, and should honor the above one's filtered result. + // These both are mutually exclusive, and should honour the above one's filtered result. // exclude instance if label "volstate" has value "offline" params.NewChildS("exclude_equals", "").NewChildS("", "volstate `offline`") // include instance if label "voltype" has value "rw" @@ -64,9 +64,6 @@ func newLabelAgent() *LabelAgent { // exclude instance if label "volstatus" has value which starts with "stopped_" params.NewChildS("exclude_contains", "").NewChildS("", "volstatus `stop`") - // rename label named style to type - params.NewChildS("rename", "").NewChildS("", "style type") - abc := plugin.New("Test", nil, params, nil, "", nil) p := &LabelAgent{AbstractPlugin: abc} @@ -148,23 +145,6 @@ func TestJoinSimpleRule(t *testing.T) { } } -func TestRenameRule(t *testing.T) { - m := matrix.New("TestLabelAgent", "test", "test") - p := newLabelAgent() - - instance, _ := m.NewInstance("0") - instance.SetLabel("style", "aaa_X") - - _ = p.rename(m) - - if instance.GetLabel("type") != "aaa_X" { - t.Errorf("rename failed, label type got=[%s] want=[%s]", instance.GetLabel("type"), "aaa_X") - } - if instance.GetLabel("style") != "" { - t.Errorf("rename failed, style lable should not exist got=[%s] ", instance.GetLabel("style")) - } -} - func TestReplaceSimpleRule(t *testing.T) { m := matrix.New("TestLabelAgent", "test", "test") p := newLabelAgent() diff --git a/cmd/poller/plugin/labelagent/parse_rules.go b/cmd/poller/plugin/labelagent/parse_rules.go index fc9ab298e..d085db17d 100644 --- a/cmd/poller/plugin/labelagent/parse_rules.go +++ b/cmd/poller/plugin/labelagent/parse_rules.go @@ -46,8 +46,6 @@ func (a *LabelAgent) parseRules() int { a.parseSplitPairsRule(rule) case "join": a.parseJoinSimpleRule(rule) - case "rename": - a.parseRenameRule(rule) case "replace": a.parseReplaceSimpleRule(rule) case "replace_regex": @@ -102,11 +100,6 @@ func (a *LabelAgent) parseRules() int { a.actions = append(a.actions, a.joinSimple) count += len(a.joinSimpleRules) } - case "rename": - if len(a.renameRules) != 0 { - a.actions = append(a.actions, a.rename) - count += len(a.renameRules) - } case "replace": if len(a.replaceSimpleRules) != 0 { a.actions = append(a.actions, a.replaceSimple) @@ -285,28 +278,6 @@ func (a *LabelAgent) parseJoinSimpleRule(rule string) { a.Logger.Warn().Msgf("(join) rule has invalid format [%s]", rule) } -type renameRule struct { - source string - target string -} - -// example rule: -// style type -// if the label named `style` exists, rename that label to `type` -// metric_one{style="flex",vol="vol1"} becomes metric_one{type="flex",vol="vol1"} - -func (a *LabelAgent) parseRenameRule(rule string) { - if fields := strings.SplitN(rule, " ", 2); len(fields) == 2 { - r := renameRule{source: strings.TrimSpace(fields[0]), target: strings.TrimSpace(fields[1])} - a.Logger.Debug().Msgf("fields := %v", fields) - a.renameRules = append(a.renameRules, r) - a.addNewLabels([]string{r.target}) - a.Logger.Debug().Msgf("(rename) parsed rule [%v]", r) - return - } - a.Logger.Warn().Str("rule", rule).Msg("rename rule has invalid format") -} - type replaceSimpleRule struct { source string target string diff --git a/docs/plugins.md b/docs/plugins.md index deb075a36..0ae2719d8 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -278,30 +278,6 @@ join: # by joining their values with "_" ``` -## rename - -If the label named `SOURCE` exists, rename that label to `TARGET`. -If the `TARGET` label already exists, overwrite it. - -NOTE: Don't forget to update your `export_options` to include the `TARGET` label. - -Rule syntax: - -```yaml -rename: - - SOURCE TARGET -``` - -Example: - -```yaml -rename: - - style type -# this rule will rename the `style` label to `type` -# for example, metric_one{style="flex",vol="vol1"} 3 -# becomes metric_one{type="flex",vol="vol1"} 3 -``` - ## replace Substitute substring `OLD` with `NEW` in label `SOURCE` and store in `TARGET`. Note that target and source labels can be @@ -320,7 +296,7 @@ Example: ```yaml replace: - node node_short `node_` `` -# this rule will remove "node_" from all values of label +# this rule will just remove "node_" from all values of label # "node". E.g. if label is "node_jamaica1", it will rewrite it # as "jamaica1" ``` diff --git a/pkg/matrix/instance.go b/pkg/matrix/instance.go index 75357fdcf..0b168c333 100644 --- a/pkg/matrix/instance.go +++ b/pkg/matrix/instance.go @@ -35,10 +35,6 @@ func (i *Instance) ClearLabels() { i.labels = dict.New() } -func (i *Instance) DeleteLabel(key string) { - i.labels.Delete(key) -} - func (i *Instance) SetLabel(key, value string) { i.labels.Set(key, value) } From 1f638967af1dde85196c1cd0d3e1324107eba5d6 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Wed, 16 Aug 2023 10:52:47 -0400 Subject: [PATCH 29/40] refactor: keep at most 100 pids --- cmd/poller/collector/asup.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/cmd/poller/collector/asup.go b/cmd/poller/collector/asup.go index 38fd28c90..9deeafa8f 100644 --- a/cmd/poller/collector/asup.go +++ b/cmd/poller/collector/asup.go @@ -1,6 +1,7 @@ package collector import ( + "cmp" "context" "crypto/sha1" //nolint:gosec // used for sha1sum not for security "encoding/hex" @@ -19,7 +20,7 @@ import ( "os/exec" "path" "regexp" - "sort" + "slices" "time" ) @@ -347,11 +348,17 @@ func attachMemory(msg *Payload) { msg.Platform.Processes = append(msg.Platform.Processes, pp) } + // keep at most 100, ordered by size descending + if len(msg.Platform.Processes) > 100 { + slices.SortStableFunc(msg.Platform.Processes, func(a, b Process) int { + return cmp.Compare(b.RssBytes, a.RssBytes) + }) + msg.Platform.Processes = msg.Platform.Processes[:100] + } + // sort processes by pid - sort.Slice(msg.Platform.Processes, func(i, j int) bool { - a := msg.Platform.Processes[i] - b := msg.Platform.Processes[j] - return a.Pid < b.Pid + slices.SortStableFunc(msg.Platform.Processes, func(a, b Process) int { + return cmp.Compare(a.Pid, b.Pid) }) } From b24dc7657ae53aad5c9f7cd2659c3fd4ac4ed4b3 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 16 Aug 2023 22:11:49 +0530 Subject: [PATCH 30/40] doc: update permission doc --- docs/assets/prepare-ontap/ontap_user_sm_3.png | Bin 72431 -> 184367 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/docs/assets/prepare-ontap/ontap_user_sm_3.png b/docs/assets/prepare-ontap/ontap_user_sm_3.png index bbaf882a0f80c548c33ee86af7e3bd38de847ebc..0ebce9c19d6c4b8d687ae4c56b1c3e3f821e49c3 100644 GIT binary patch literal 184367 zcma&N1DGbuvM$`3wr$(CF>TwnZQHip)3$BfHl};p=0CIc+IR1B{&UaW-}ls0S&{ih zM7|Y?%E}6tlM#i1#DoL@0Dut}6IK8K0A&UM00sgF{VkaWr3(fCfUGhX5|R@a62g~r zur)EaG6nz;3r|i3Q&L<*4VmdWBgiKSA|$yd6(jRYg4X~z(^e8h#7}|;78r^NBcexD z5|%@7QWEZyr(g_=AV&oS{MG@?7)E}e*B3J0(Mvcvsd?4)@TlRLO~>{2ZkpS0*=DH= zFnWs}PmYoThA*8)0ej<8Dk?HEYd-)0XXMW)_pKebd8X_*0rmr+jRWSi;?eZXx6e-Y86gqvCVs;CUw{g)&`x0Ki@$VU-L710|lR zBRAoa%vj%xf0r7{ZQ*4hi}(g2*VqP>$|S`d8;-S|FERh*m(^T__*{^e|9KKr~{1 z=Av5^J(yMSTYkV*z(+p?Iat`hgI#2HU`&3dR_JPf%{-|40MR@=Xn$2bXfFUHc#u7O zMqz&?0va(q#^6eP&_f`1!B07GWSAj=@HqGce#<;*IY1?lWI+@;lzS#8+;-?KzxI5v zSpa6BuHf!o@-z@deVA1M9RMah(hIQAKFM8SH;ycrlm3le#8yOJh?RlVJxx2hO}L(b z5qSWTz=Hfnc@uKbh+s(Jp*%!6gGEjy-g0#1@Nxl`{B?O(xsNjWMN?)}X@P|~6Em_# zI8N*-5jxQtA+!R?xskbtQ`UQ&djJpU4;-Q4GD9hb7<5)r$iy&p|FHpN!x)2~^r{S9 zX#|sC)TojJm9!ZP+T0;R(+k}#GUS5H8XdXZ8jNQqG^ zP>Tz$f9L$(F6_bYAxfGEGiq~)?m+5})fV9u{GRkKoP_uF%8&qLM1iEfd~1?)5>Zk`5&>n3g_yBo;*W2Oii?S$)`v%r1X`R;Cd%%c>#B)urKs9J%iawUhi zGP`)a3~zq1&|1JyF23NjAho<%wN}7UATsALhiLX|xxMTmp<38hy_d9^RGie9v_)E9 zkxSA43~=G{CG7?FF8Ge>WQU0;onfkevV013(){@InBy4cxCZT%CQQB7QQ$4$3bh&a z2dXGFt~#a0tH!Y!t9o}6W8J<+viepNsivoTO^Z#VkD;N(KnsVN<4%e}$EZuBb;U0> zE>EsH?dYwrt{m^I51%(rXiB0m`O(%(KYKe@wX4~S>$SOQhJ~A`gIpOC844M)EFEfQ ztx;V!qrJR_7^ac7{U-hSQ2M9ZU;45A(fuTQRR|6UbqLA`ctkvek%VuBsKQmz z?I=~KG+{PTeM7>7*wGle7j14c`}t}ctDUReBq5S2v~6`t*5*Iwp(YMf>{GI%2L?rk zv%}uTAr6%sh7ZV5f|53`LGV@qr!wf1ZoBO)U|{U#lsnc=|Vz;+00 z^SG^*O_oVE-O(~+F8Km?iU-2yWYe_WZhQXZrSmxj6dAA|P!qh_pUJ23sdjsC*c@b? zz45CVZ_|H^U~{fz!TIEdFu35Npieu}W$CKqYI^6xXQrNqEy6lt;tBCd;_)%{IhFZx zVWY)xXJx)Be*NR;Tg&B_o1bgmN!~_ZhC}piz%5-UNa&@io9dz}rV=XNFzz(x z`@7g+{9`UUpE8azmxkT_2E$(6=K;aUM)^fWxlCE^st$~*WVxu!aLxtik$rjHaAiqB zii)NkW>!j;&x_;cjJN6gbYqTjRuxC7uZ=HGF>McSxvnzba@^wV4*J%r_x6*QWul_u z7Ts|Vw&k)7ZAYGyz9&>2NsIBV!Q;@4WPxO1DiR%BGp(h!6Uoo)e4=?Wj!8r{2g zm*VU-pPTl(hhwZ5w&@+CO{-R8*NX0e=E6(W4c&9jU!3Q*8#^`^4yP7Bbym0L57r%D zVn(tpUDsS~A3*P2o`RpYdeA&;zId-bBf);aZF5h0u-}*s7)~C%likWj;oe}Ld9*zr%~H)(gwJ*I zA%CuZt)D)$@ojxxphse2c@X(9y)<8E|Liclvfgfq0gLAQh_000COg~Y{wpA`)qjE!v^&1{`ygAr?g3n1*o)Exl; zP)YtAfZ__o*8l)OYvxL7PHNIpoQAg6wE9N22FA2*)^>mN0C2l;{!UsOJL%)QSzFmS za=P&l{#AnWclwW*ju8K^B2Jb(glf`q_(HZ0#`vtX^tAMZypZ_#_}mUgCY%bw-~WdH zzTzP?b8@odq@#0nb)|J>qP2A}rDNdW;Gm;tq+?{H`CWp>(cQ*L-;Kt`k?0>v{;5aU z*wN6z+|J3|)&~ENUVQ^wXD1#)!as)o&+i|88oQbQ%aV=b-_`nELApOVbPTlgbpI2~ z$=u}s0{fHm57=Mz`iD90Kgu}e%-xKw)P&8ge^dS2HC{$`W=8J6?EL?-{^jXEpvsQM z4nnrpzmZP7|GHU!ga1AAzXSiWss1mU3`}h7|8Dc&vi=SFrxKhB#*VgD&VNW$wlR0& z{q5@iB>wLx^?$*58JL;>2K~GAza!NCN5sEN|2sm~!TdK7`hQ&GW%%2|zsvqE&rSE| z=KotT{3Fu-lKu`gUPx}b{}Z;nkPT~7RsaC}0OG;|N^XD`+F<-CLs(w~o>wGcegTl< z4pcTwq8gW#7r7?Y_4>vtCTq<+)R`8NOcTta8C*{};1Okc1<-3)NnclOv2gKnSSl8- zkUJx9+gqMb9iCS?o+aXHYHA!#^WbJ6%s`lc(SG0giSiI;|CdWU7$UiIOq)!bjQ?lR ze<=#~TYI@?r+Tt^5XaSwu#jd*zWotd6|dM z+V}gJk7s8W^cUebmLq65g$h*a?9SB@EL}UtK-i+ol>g9uJ(tGWkKrJJF{W{S6AMV& z34pY;unh{FulHeWt1YVkfU)1c%puS>{7`lgdX&Fl5cZOf)Rmf$e38~P+l&LQ;wt`+ znD~MKnH;y3``!c2H2yM!ze>MX+C2L#J88Bh{-60kfB;Y1Bl91!_a_3X%|ROZ4>E9Z zx7hz93x6;mfJ`uf|DD-9N+42Fjgw+Z+}Do3@>=8J$?5ofiJV2BME(_S{$J=o8hJN( zSaw_5|HWXcaPJ_<(aCgvV`Z2+F=6^YD`7`OfImkfJw>Ch5E*dp&^5gyW?1o7@J~AZ zzaanU)5S2~lKrpxM0!C^CdT8keISQ|Us9)j*joG}`gD>?kzYsU_ z$%&ArD&WeCEIFS~?6=`aPV;|nZmlx#vp~`3MtM$5|7LF%{Cx|Yyys{CF9q=q!k3Pl zxe@yS6iayv z=Py3wU;w+ORe^=Se*BX%!06tGAU{+yE!o22jq-m<|F>?z|BL*u+lJ$|W&K}h{=bKs zmMr)wI_AoP*;(`7%>Q2v*w}##a^(d7fS|TW&wJM57-T6APKSsXAD4I8zJ7v%^}o#d z9a44>e?k?p7oBbbX#4>&#pJ7!0cynrIVD>as=1ElRqaErLVBXYXKooXWT2ANF)tR| zY*XXqCis182*~^iu3=yPi-YdfY%QJ4#wbuJuyHX_cQdyF`TkYI zB$FjE0vFjbus#lyy&j@>G$M2}NCm}UstQTCS!r=^L--KZL79|l=xYLD2U}*X2&Ln4 zJYIj$4l2&3^P8(bab$w1mbud3x9a6|xdDGLa^EgBUExbb#Du1JA}bV>3-bXSfiLo> zLjT*a(;FS`2lHabl>m7?`%A#>v^(?W$n~7zF=vTih$=70Afm9M*5X62lgAz4afb~F z;~NRGUz(1QyFRkX;7L-!sTa*V-8ircYExXCf?u*H<6KcSirMN-O9j=JB9W0`TQo+M z+#sUJJcmYp7zv~?$M4o@qxL7DB!ngaFhM7v)sY92eqA-qlAlB`= zu5|4QaaY0dl9}=F?Z5Ne`w;Dy#bh;RmB^Hy6@NQsPPKP}$1$L_1Ae;vS04jd@6vh- z!3XJr>gJ@xR=H-!55ZIb=7GL%Y!^)p#}!Na>zMF(R?O>Bimve|50yPp>AFus9q&9jwEgjH0zqW}sqtL%bTMki5`tx{Lz0R+ zLX`0Vc^fU4Lebd|$08rs6}p-Egza$d)cecSSzoapHY*L{Eo(T2Ch{r_3;h-j<3bH_CBxm1Id* zXVWv~C6>NtbS+Ky7_&4~XE2)o?4JYO^8h%xgMW_?ob+`l)fXIET$dJoGLKd%6;a;F zUNXZg-C>;TcQY?hD#b&T z5Ua7ttvW`m9iJr_)Wd^&pIfw`Cwr0admQyfX7>Dcm1TXJ61Ip8bVO6!uucj1{1nn1j^^@dx&S_QU%iHaGQ@Iwbb=KB@*a) z6h_wG83RG&d#qK16rV4ak&O;fa)lCl+iu$Tg!GM^y}UyyIQE7H-{W3K%nzP9VWka^eRqU_KjCBmB!S>^oXR)!$qWWj^G zsz?lCi}nwi#2a6!MijlziDqY#1v17dgLxUfzV}6`*(goJ`Y;@kI{lP_EuoY3;|h0c z%N%keiAG;M8%rHfZrkic^lG8Zxyw&l#zVGHQo;{4soN~O(vKvwfeWR$%0|PdRN$a7 zkB}UX69|8=YzhxsOOKwV=OT%3%TiJ-m+QuiJuFlXLn7@&hrq*gU99>QK)9Uh#)#AL zM7CdC1KYQ;p|z)W%SJ~&PZckTGfFG)B93dPH90%FP>%>@m|Jrsn{GIkWoWpuA}~lU zYYggsAxx^HmDS5(qQfx~X6=E?P!ly2jxEUfsM28(7LS&y4Zd9?i!^e(<;0id?LW~V!P>kZbnoh)W`aCz9bl-Vp4Vv?*>?eVCiuYC2~7_QkUfdtXg;%NmHu^Mf*0nS;6BUs6_tgPkL1drw;NaV~SV!E53^+gzS z!uY~Jd5GwphXvTbWfJhzD9xc2NO5!)mm->=cZP*IFq$iRU@#OH4_9ep?!Ab*_O|zs zSLpbLtt+P05%J<}X}azDhfj?GLkC4ique0bk`=K`XAAlGAT(ccL_Bq`g`jXP%vR=9 zAzEnBb)je|oEj;@YUP8Mm$PVHLP^tci;|3%yL@@PDZbX9n*rvuHz3$p;05UUx*8oL z8@%Nhx;G~!Pft)9 zfxQn14GdLfXU=fCafX^rq5G_x<60%Binxc zc8=%>zh?oSMI1U&S!CeU%?B>@)}&C(nJYFTdTvsF70_daPAJEoqaH%5AA{} zVy()p^Yu+v%SFt80vcJo z-tTILawzDs?{v1DcR#k+;Acz6-emLgY;?u#WQBJ4WZmQ-;k{QogNL(CLHW$FLFY{` zHMaOzp-P5&v3r?*cQRGMes~>QVN6TIc#kxQ!<{fb+1+Eyfos!7<@v(jErBn_?|NP9 z2XnlDNYG(Jv0X`1SxezJ440GSjS1hxlEkWkn76sQUyT<X@6rS^AjZQ&ifl4~Ef-w1TLy~3^pYBDMV}#Yq*U|NYv{iO%P2)zR zrU47)jqTjxR6G1&^DoD$!PBNldVS$!je8I9jxtk1kc^0~$hAj}HjhpIrI|<0ESJf^ z6+Q3fAi`-srCy(&A*O-hxV&0+1sW~4KrU@)vnD|OcMsr<4L7FMq&&QNA1k>7J!o*^ zz6aycEIu#Po(KG%_V=fB*ySyWKNtz{=0_)U*qkB8oF4b3-Td<4e*Njf99b(wyQQdozJ;ojsz!x9 zMIj*b$ss)3AqsdV$zds`&`5ITwig@`qCkvND55{?wi9>$`a-kq;(|jv*!hD^zn5l; z5=&=e0wZ6^&1{7c{Yfe#w$RjJPY}0=+~R!D^}WUqe{!>MgYkMGARfH>Ds+DzkrrKx zg;oOSK3NxSZdNrEAwslNMl9B1#X zMa9D-8R?nfwMrm%4z+=at@!R0De{Z5&^24wzAmH_15E{5uZv0OHz2eCg?6I^B^4J~ zK}<+bWxs0IihbV55*Xbt-A)oq4wGZSi8UI z;06{|wY#q)7b>HyDfF{Zm0%`|?>Tz%)Oa=$N-ddqH&Qb5T0)6HA8le_w7?5QKmF+W zB(qKmm3(OzOAKN!q$v6gU0Hey3p-RHgQrc7DX_J%tMqM6r(81gtt0iQKYD>(Wz!-H zf4DP0bg%;`CtLWgsOn#js*Pt7`rMY2w#7s-I$fa$$W#T!FX96q^lX1alN>nK>cShY zcUX;o694&Dn4;LP7theCq~1pN!$UKArurrk$_b)Mi+Bv%v9U$)%Ecd@&l+s=XSyow z==|je#t`}n2S5`~79|TgwBywEX2#FdZVC0Xb36+)DyDt92Z^;v(Y%t<@VeMN_j2fy zpT)!Z9%oegnmj%4))$#p8Y~b87L<5lsRlb%RJGL+k#aEfu>D{0+gWryXg>DZFuTs0s4V66q z`07Z!&>IU!E?G_Q-wJO;c&b!thX{jp7VR1F*bgPysAV99$>9#trBb0$?H7Lk$lXkh z<+_9&hN245->SJ6_kKzRw>S!J_=dKK6Fe-LUu<5}>^$3TZAh(N2h}rNOaXP(sP`jx&CmUm)D~g z*rumvqeYw>6@t&s*n)bPB)jysa9h0;f==4?QVW{oCr zA#2;%ie(ABE9u4Eh~PG~kF_}^iP9ZH{HwCGXR>U{zAW_?6-cHCa-%d>OBmAWEzv9C zrnu>WxkFxou!9LgsQjW059Xm0z9D>$27ZQ% zC6Sk*sQyV7ucMPWN3*#di~}7_ahfh=i`=>*VrL|@T)2^@;k0}lrR{V_wtauN=6q{_ zsc&nfKM12QaWU!P6Fc)fXHXAf0G;JpVnQLibA(`gOhvJp(*2h)hSAy`4)zGis^}X* z)NU-PU-`PDBGal6okk6KibTO*AHw$Ydtt3JS4Jku3K=ejoWlx!Y}vv^SH@u$5rqg- z>4$LJe$I5zA9p!Rm>AA{YcS_Al|r9Gkk|hdWA_2Ah&{&NT*~rB*z*?PokHCLT z1k_+qppnlQetLHCR;rJ~f-G$y22(GgrTw@d${&%qF32oMqaLaLg(U7QZiE3cr9Q=^ zJj;r*el`MLJV1I3+idX7pqk=A;v-dh{<}MVqJ4~WxD9w`tMr3v#DumxC@b<_3K}M* zs=-N-qg1)n5EF530sd2E zcq$0aMy^Pe{4$;dX`+ZoZuvf?b=!NC1yE#R!GC5EPJ1@*;_Riil3{Q6jP>!;>OOC#)_%sMzG^| zho0ChsKmPX(n!TJ*?%qe^zQikRdSdc)o0I8k7Qkq6j)c-xp}q`lCS`W!s3l|Y{ms|pyZPRAQWUGv{$u8;7Pv&B5^~HLtFbkl%*UfRKfIiG2Ub1Pf;QSua(R1bc z&`e$&Ij`W>%7I^7%{C1Z{wNMxNnBN&&A}5-VM|q%(!sJ}IUQ=$p+gxfkHXYx6?c~~90CUs0+}i55OUOKt{IjcdamaU! zDOe@h-KkBp0T`JDMfrftw(n?N4ka^# zGTxoS{tP5i(phBPt<9c(yJwzDec|qb1sI5tY~qMt&LYING+MbossV%Jd4EIq`jRa( zPWaqd7R@s1VgtD_^g+$`PAX)pXJmq!Nt#39^9Rp>pvW8FLvr(G29V;zE4qw%up?PSZ}G` zh{T}mCfmm=2VsDU?21r570d)m`m5ct6pctz*IjgJ+BuF4Cy)0(ZEoX|oj!iF)jSfF zUk)+9G|emKJ0{T+b%AGNb?Ds=bnTuW1u?ca``)~U=I2r8mMly){Z9daE?jMTt0)HpXNJbWWcMhfjTF6pmeLAp{JS9=IRX_ zN1ee!;@{UAx9s#4tA&&*+o2e7Gn)5mwl%4@gx?=UrmOAe${Qu(Mg``q_d~kCbsWqy zt~XBq{~NJj$%5&coyZwNo*EnxIQm$DF$+WVt`qB`%;u-z$eYIxUV8PJj* zM3;X`gtX@uML}RJQX?JPmckA?@Z21#LuWMt^=+dzGRLFi^loNTyr+jkbOkbae%QNd zM$+{-V9OLw0EpjD;7CZza*(TJ-_DPCBkpEIS&bp>Q$xEVVeb=kledA`0J@+rH&-L7 zcs2cEreRE5nM0P=qN&r~WWuIACiL!$x}Llt%!&sP@M@+rlw-fXr@f{$?*f6}H+*fr z52@+EVI3+lFts9;oo#{_?~B_tO$!ZSh&;kh5CE^R4(1u7mXr?5xAxjKG99ar>OxEm zhxfHav9lPFw@BesH(=kO^qYaF*}M=8TC7vqO7;27G4kR8I`=Tp!_%$gOe+acf*3BC z!|c5I_rOkV-!!sdV(vpi8(3K7nVe7b+fU%9 zBRlElDW;>pBRiGV6&)!^nLtD+NX+H8ae1`Z_w6CAsWnq6x0*FF@0Uyw_RAq{M<=a$ zW*Cmq5g;R$`~7^s^TqLA%4l%d!SEOV$HKPV5<(o0NYKQ|!EK$%N8u?tPWVL@5Ox>` zG#i_$)8#DkR0iuXeFNJMa{d*6XDejyU9neTGhT6(lwZxWgbEk>nf^Hf} z6@y^u?($*UkwD(nZAY6R!hTvjbsk6aV$9_t>3M$sp1~;%1-*QU0bmG9;AfVbnb^d% z=vd0V-EHgYT6GYK8DaSxLnak`ZN6xq@IIJ*H9Z&@v_8W;-a`G;B| zqlYwV-94ePaemc0i(-Gd2>OU%WtF+_FHe?TC$oZwFyev0<(mY3 z7u(SEOErO$@An_tA|06M47w}QqM#b>jBmAY(6P_wqLasjhc=!uYN1UpQeDnkXh!A* zr@^0TjJHqb5ablOl*BQ3FjQiU>+Bbm&+l#!8=?? zDY=s7*%}Naw7n*st0Kr_W5h5_f{m%Ca*qz&iajD<5_dVd3mpy}yn+0IXCuZnS z1FjH`MpxIJkuG$nHRgpGpt9Bw&`-wL`G33`>UH+_qc_27)ztfT0P*pB2Me;~tav$T zyu$>EPrHPWFJdGs$sOOnT8Hs`dE$L-qaV4jc{Nm(B#)swE~#A*MYZ+N|tNKMc&_bPxTy*$5CgvfO95<3iA-8q>n79(+mU(?15&eVxzWo96nw z%lhmzgfNeGB8CT!Cr%!0NhGd#5d@bYALQbUeSH@BkxWmt=SJy*CnJP&J13k*@X}MA=;+1B55QTq;ePfY!g2Nr8Js}6Me)P{g!wq<7y!PCvb49ab zd;ff!QTH2L4r+)4-307jefyi^d{iXKz>;=)Ug9yPqq?7Kohu>TAH>BuQv>%`t_RCD zd$sv~g~mz%z4)@JL(=6&T@Je`CljWL+fJ@^Y#8Pk?$>rrilO{&vxG$%c~VYm6K=cC zwRL5i*NX(%K;Rtw6LX5NN%}Sc0vtM;@R9VgR2MT`vD-9Wb(ejaW6%r(2}@w~_b)7A zl?K6tKG{ztsRlz-U9%aO;Sl2Htq18MtCGJex}&uj_M&CLVegIjsul?!R0Q%7OQgO$ zh7>pkY8X*d<4IEK{HAVEyzEtdt zeeBPbdtd7vp>DPcaMG@qVTFU%sZ2%zAr&tnT}OsXZImBiG5Ek{V19^-`)aatiC@55 zs3Zu~8J{3*avO8h@}Mo0d@rP<(aFNLUJ7~U#6Xi&y&9QyWtEE)Iu3|39pY9!d1tPnRn#ZaQaiJqN9G@nJd7k_;n4=+f z(g$vt!BGNi7}dO&bz^=rNl;GWa>t5NeT}VBj@^;fHq){4D3vGxnBgx+Wj0=y{Y#1n&WJie3H}XkyHtY z;{KbD0~AD;<>Jm;eEHcArz zv3)}Wd^9IBO)Dr96ogX$`)#@t^CcHby5{~1xG3V5H^TZT9v56rNN{GbL0C@F(%RWp zsoPFd0fv&%jbkXBc@Ge7t;~f%RO`r_p6Aom4#APdRw$3cwLLSs#ay8hp}08dG&EZp4d#GJ4rP_5%#J z@2xtAm*S7O3>RX1Iq42cr&`TWR@FS|+LCv{_c;_DlRTXwEqXjj>gfGhRn(mGWRhF+G}#MaoD=t*<7X zDg6u?1i@DbYgM&xAl)#-ychU^WO|tZyln^93LRS8gyt{CI-?c`w*;Xb8foZ$(-D?bNgt^_*KAPfh{w7;Q~@LoTV ziP8mm;cXblT{7mln@D@vN6JQ{#TNpKZ5%M#Ag=)x3q*836R^v$KRmcdn!_!?iFF0`IaZx`s=Hzgr?hsQwxB-F8{r3YC!H6s zc|_7j09&0>If9B;4L5SOvv8NXmxcu5M9(W)asII!rYY`PQbQhGzE0?s8N#I5S_=WNsNuo$1!whQ=@ z8La>cq)@_klsiohMLY35)O^(%&E)n6)xTbz+jSA*pQm5XN|fbo*n(6+C(InIhTC61 z9r+9z6>XJ2#SV$Zqf^6``ZR(r=x#&w*hGy;_I;eqBes*!i5MtAswyXdZdY?VW)ehK zuSuF|XE-HzX7mWG<$H9k?C+=$%eV|38oU;nxw*5G`@|un4ic}e(9r8ksS2|^Do*(t z36o8J*r7IG@J4(d?uAt-?u{(|_!VH^!aO(_>q@{V*I5eA`Q?N7Dj9?y=Oj3Njtos##Ml02@Bfvn!g3Yq=S7b%c$ZUySCd~mWxD!SnaR5MA(V@%J~@vK-=Uq`pa2?G6= z5in#9lq|sn@gTI)UW3>P+KqvWrzkWE7%FB8Hms8>yV=A1@nHfY3?fuTH2Fc$Ztwac zeQMq8c?AQ!b>cUen9Qisi0zF#jw*?VSvcT~UPi`2t zFk}Y-g-R<62Q!REiS3;0{aZ@uf&EN}orKAnzYO%VINF`fqVUL)@@i|*gc3I z`S+_otA50*1-2W2PkmMPA#GlxNcou|_;laqy<1LNZd3Fe^d za)7YLOttiEKtGZw7~>OV0=OJM6}jW|!TvvKo0oID!#hQ1uPd%LEAULY-(=R4mT6q|EQKww~3eJ{JiJ;)^{AJT~V5*mNxxFK`& zlT*t`CCUK8b}R4Xq`7lKVkDgjUFqx=F7TU+0_LOteR6Kvb43AEU`Kb@iO!|}+tnLL z?zBc&DXwnJY%^}yT3RsM_EqR-*T(#Qe+t;Ufz_r5Kw#)ph;JN8%++ZBAsVONAWzHw z6r2x#g-PG(B5g!P2ZT(P*l^g0iJq=QHRmA9wS5UGdmSvj+C_1(>2Btg%3kqIa{=p? zNyL8SeUnFmK1y*EQisD@bH{9zqobPQv#yp>5Q-%_mEq|VxCMo0I(RsAu}IzS+czi- zEW(6p!0EliA53n?$+mS?VBeq`X}w)vEQU@@2Q6Qdh2ZVq$+d}?2h&aW^6>Cs;Bf#O-I@#Yi$m6Ce>SEYN$ODY%;k~ zJ{-7jw1aOvb7_aMW@`F7zrp2MPYf4jQ99N5T{A_q<`JUBZr3HS#n|3S9T~Ft0pZsXDR86oEP)uS;zlkQt1LS=bq;8ICFf-D;Q2MY z!}urxnB3@;R*KW@-o*FfhbuAH@@2?}AN!tI7Lm6z+=kDb1>?>~;6r+PKPzbUUW>Or z>o|oayY;;|{w1yM9-WY4k=}#+h3)mhGl$}>8#q}hqUA(xG)1Dqn$3AyPD2C*1)PTB z>br~%!*LB4?ANP`MBn+-qHp2JizF)M?WfUQ(F*Boyp%&E>n!Af{pLmmLQ62FLUmB;hkUnAv$5f0Y>NswGi?L;R}5)HY+o05 z)Ye7`Z2Hn;xv+x;EZPP2vz%mk24H|eZDiXk9W;nf#>)k8rP6f~FOAprSOgJwol2Il zdg8aHSJO&K!5JCR*=}VaVb+_GNVpKg`P8>vaaVR#F7e5rq^{soo^syw6{BXv9&Lt# zY}Jshx0v6LziL?A3tZrIli>nIc!oyT!PPlRaAopz%Xr2{gpry7IOh#>_D1yIKwKz2 zDT1Z(nL1UpEzZZDD8<#2jgl1JKHxuCSryY@el9>p8=miu7<r3PZ zwQ@n~L5Gv< zqwSQ$wFMH+AfUoKwRV$O$YosI{YaU)=p{!RqCi~*Ky3~DIA&7l0Wo zm(d&{MvQfnW50fGG*ATFS4Zq({{F_pm9GtSE*^H(l^suV`%0N&a<%(}JjcToW&c#+ zq6&d#wnmkS#4eJ<3-X*DEQVOVBe5F4J+$3ml1Dpl+kE#?OAl3xf^AX!zA_?2+>be&e|VUJ!~mdz?Ukf=_rJ>+g>NAOK61#t1t zPVLrmaPriIhss(SDEAYYTsk>3+dTDWNB70V^X&xQ z?N)24raNE=$}??gF5;UZc##5K{w39Kd9@ilg4Zte!0>ntp~0__5Wk6BPdfeOU~jym zo8?58jBe&0pApXQO9zeR0mW(+I}&B?DMvOgqpls6nd4@$EStuL=?Z&V86b@-%^l6j zdr2uXs%i@>+QC|rA4hADm2F+;^LPLbSs5KQsfAs79#~fUihk3hdA-#SgKWWGK2ZGQ z2S%mUMVY^}^u?gTQm>Mo$Zib^Dp6EnVAv2Q2#Q{7SY%oKaGu!@|oL$n{-ilsN2jq}9z-_L%`6}L^ z>y6-jJgzPyl8Gz*A+#;DN0B?h_eSq2%+y-1_!#Hb$ptaHYvir4AzMpOxVSd0N5@ZV zsO|gv+eAO4E5MuZhRCv%1fnxVf_D#E@c2Xh=?>VpzAZl7nPhgE@$1*J!s`#iL0t*; z@Zy}>FynPR;pHpS9&~TT29t?z zv#H9V`wXeFCfCb<3$hwUc*~l60&Hapm$v;RK8*a^qCfwb)VJen+qNkBt#eCrgNeZ0r;JLr<>3KVl zYk2t3u6(&spJ0E9Xw;%}Jq;_~jG+a-I`{p2c)Pjb>M)Zxl-)xLN64m)^m8am)r8fg z>>gcr;v|7G{jvH@qg)lmM2sXW#sDwq&S)`wc4=-7!MW4zl1u1YGN+R#Rr=o_kFmi6 zIW(iGhOw)O4UBF^RF*9!BNBHcc?eJ}EuOw=Jysm$*ge;y7 z+^JC>!NrEes`z&n-?r;>>El?m$FD+CFLJ?o3K%T8Ae*ZiGkJJUh^N+LBkSKxZrK9F zj0qX%DA348Fr}RqY>N+LEaWntKvn6VyDSab-vbmwz5 zlPp3?yC}$t42}R*bw)-mURr(>PMw^vdCv*-?%s?(n9b0psXwL2`(eh93ZAseFYT2i z%y{g~$3_8%8<+fy8QG9-ce*L2vE=YLqoa~S`$m*Pb0U=gTR&!_}{=bs=ge$mwNd5`I!A(BAPU{!)G7H;n4B3 zc>CD_97{`#9M}}!-?jkm&1}?0FS}qZzIWnk?76p>;;qRC7#W+(Ld<6DKHxwXReq}8 zRJvW5Fm*A$_%I&uBr#yN8;6z+SDTF;ef{0p%nOTlokCp-Wv;YgNIQPnatyc7rI@Fi zBYLzC!l!eO;!tgKyxOy&67(pGqopzE-Z4}?-}p=XkVkSuo3Shrk&RZUgZQM^_HaYy ze_v`~uFXWj7o`|=cW1T1M}+$F;Kz9r#Ewcbe(*(&J8~MMo?U=nzo&_2Am7dS?bvr< zIGQx{Q3w6bVcz(7-BC36u*UZIT)gwyN+k$+U>cce6pWz`i6Lq|ZrOhtvtGYL`G9ZT zl-dr?>kSz2a2gR>M?Llo8FDQiUBF$p6iw+;YEYk_=+4W9jGKC#-xjpS*aEN4tdFnQ zq%j>SAhuL%$F!NNF=6ax1ov>j;q(%`{PR&9`|@rwFWk_xa}$i8wu`JJL$_q@ktCln zhfbpLU9+i;QbbP`-00icq6sylOE}Ms`3$-BZ;gOPKqK(i5D;Is{uJ;ZT%V386{L!t$v3c8md^2w?W`3}ZTHa*39?QcUv&Jf6#)qHI$NeLx@!@%x`So*TviHQ) z2Rq`4hrhLvD-4$8RlHZf!e~NfO}>x8PG;LO4?vdqA3=? zB^#q2X`_T0DQTJLJ?u?vUlNC~E`eAvXCiuaYl#WZ?#F9W7N`TXlubUJhWjUeiUnU% zc>~rscxVy@ApRJCPX|1>WqYNYC2?s10 zA3X&~+KTZ{--kC}9F1Q$?LpsOBQfcPN0cyQ_rAl33V)jAQqi?vd;C22ISMg)P@wTO z+iGBlOnByFeDLN&2#J0OEvY3sbNNBIQQ5~BACHA~x(ou!ipAX&$J`0XU4=TcUwXSVL-T>od zRk?1g@1On@rR12yg?Mv(SLM}4czZ8?xEkx|3bRc(l~Cz%M%bCvlUx!PRKj<4BuzGC z@?vb7IZU~mpD?;BK3zT&fevO$m$@c-w|ag5*57iL@+A#0e|HM%yuApI-q%HW)G@Y4 zTvaY!xd#JZT!SV-^opW|8AfOnFRb#9Kl;VM7nUG+C49lolPM~G>Wcr|56k9BS(@Ie zVBoYGxRxi~<*<_W=WvpVd6WFeONN~Dl?%w_%Kn~v>Rsl7X2k=iS;d=ORLDgMtG<{>P6bAV2)KcHRE>?8Z2Ht z7{9#P2V18PLgJj^j3>=K!~p;RKmbWZK~yq;LB*X_`CSK+aQAB)&?L~>FdpjagO z5VO1ncD2xU#}stg}SLvyg%Z$@p2ZG*!9bs$P-2Q(+V+jiX@UnfV9d(U6Pa;mOUz{*nc0*sF*(?=EnK_ux#>jEPq#T;j!234t>wT|G;sO)2-hIY-A&X@D~nF5VpGg2={oA+jmM?BZM{zV58{X>E4;jb<|y>qVf66uh1|^nEza3?JHama$Ocb z$daCYbq?1%S&+}+&R*fljn1yUaTI2Ji|}^7@bost^w&3WohF?`g#x3N)7YoKL>?L6 zbRdZ}o8tJ%_1Hs{4R~l0B8f@d9r-o0rB%oQ3S6FMaWh(gjKZw1?&MeMTJt~{qFf5n zTDe82Qm%M<>O7V#LE~s|eDiHAKAX=y*_E*`xOK=fZK&Cd?zg1<{HubiZU|Dzct^Al z6zjhQOEyxNu>ozm`GXre=6=7D`JtHLvH=+nrHhwsrZ8gxT6e9F*`I7B10ajaE|xH7 zoQUJQGL*H5hr0_FEqw`n?(BehGdE)CoOsmrrpp>7L{ZgV^h??*Rv>N2IG40)&2|bi zwxeASf9zXz5Z^6Wuda*rD1&NL((4Ce;>q-1$b81Cc+0Js_>weo`+%IO*odU#_@$mZ=+D;3B;^Dg>OFEi28M`si1SQmY;e+#V?6{ zRlk)Jo_usZg&EV(xqm}^@X=Jln{Y zl3^~T(V`|q{k?Ym4hl0?qJ2-9%gH!R;g3ocd$-{aV^4(1VwfrGHteA=V?WE)!3MFu zRl(zdrr}heV0@5=#?GcN;|sn=k@)D-PtdDtq^iopUD{}iDAF<(q(w3^Nz>B8Hmr3E ze7-tCxjG8;_hBA4y44C&W!7xihbG;EG3~4)_&S&?V{~;>|GFXq!n6qTw8ZA0PtjuLjH2uN_T)JrEd}nxQk39Spm$oZ z`@2{odCy6_c3*pH-dd}7d&$>F5a3huS>}ol*BLYd8Uc;Kb%21JSrKTC0|}YRT0=41 zuh8xziQJ=1g{gmqjMwRH>6dsQkwE9-5vF>oAQ{x<#VdAW2KhGYzZ{@EP>3?Gk*26x zu8=$gv5YWKr&aI#^WP=LXI&~6fBM}v93`zHbnt``?GYU6f@9>V$=+Dg*vfE=rxg** zWYhZbvxzq;VMcyI2|oE|J!(0Ld(ms5hnL4xMnQO75#$xUOXcpH$9#fjD!2p z*S})e<6mRo#1(YMWq~pRaf43t%Y$@@yN1)?C2n5n@>#wFCJo&&@rAFIxw~i?8OZM| zmAu4#78aLN2acx0fk7R`_nl6^MLuVlItm#$Sh<_32D3;+?K|3G!bh7_i~~88-EvbTBKq&5z^SCh#pJiGyz&)Um}1uD zW8k>xN=uWt1TuabJHyk{7B;1(*nj8@nf0zpYkzS20K7au7ELKlHYVfZ?Z;Rm@>4yD zG*evtjC*|r{G3Gu@mrH_DI=H(}*}jgNvbaaAQrHXH|O49lpp#zL3)ly=$BNA#-ovAC&y zD96neH#Myd6S~j}3iMK?w(d%xf}kO2c1_jDcm;AnKYEBlqk(?3gdmfQ!lb}DF0f*- zZd+P`NOI$NNzkMU1z2@Weu+Rw{1`hjP^koq64z(Qr_hmZOC*WM2h}HDlRgRE5aMis zgZuLl8R3qH2Kds2haVc%bLX>cl#44VXGUR-0|il1<}BS}8Rw@2r58u+a@|g%*(Q`2 z%w1VV+)9b29x}_&BFsm*eUrMxSLE|=x20u`xw0x!rl$Q$=@XuFj!9x zj$BbI15VI9bK%<4Xu$yRX?YjgU~G%z)JY?t5zq)+O$2037E8y7jvn}i`}8-TY(e+I zVcajfq5Dm(F=Rj&#JrG5MTz~2UX;CQ#q6h46J+I*@!bKm%gZhpdH!~G>Ew}9?q#I&D# zg~U3tbya&$9^(!ypL-ZVt(+l&&wZ(sU`>nI_|2J>he^YslABD$qAYzkBzdl*!EF-1 zB_2!SZ#(2E8i4EhSu&)VeCe`@5_s+lhCNsWE0ma2x{flg^}-Y_r4`FNAAV1!_W<0| zBO2YiwZYCu4&lW&=V9@xWALp*?Ys)QQ5dZkr*c{>8t9AoW2DW<_)f_p)85S;n|4sZ zf1m8J+}kJsGVcANMJ`q_x!nI4s0%S7%x7{^F`b0rUy)R&&mp9t+PrYbpe}R=)kV#N z_X(<{?8!{+{Oss%%#cT{DkU&>CKX}aDd4x_m)0}kc|SE zyxe>>WaboV*-;Zv&83Qh(IF1lbuu3X6mk@EPnRV0qzJY`8W`GtL}BPk>lm4lR*Vf( zaRj?C0i;-EGG4Z|vEns*1Ew5jvG|k}SXY!~PT;`_gD`ePAJnhsXV65=%UAa;De!ZN zw8e&nhiIX5mhX}mMvZSz3#Y}j;vKK7cH<5oV_r*mc^hc22S@eC10(yZ@-n$CI1b`s z3V{K5>Agj$UE4zm?p}HJKE{2yjjqvF;>q#L8C2XbkBm4PD$u`cLO@fydg(ST6%=erc99L(qaoLO=Q;>*z$92UBTFFRSxO5|b7jZh;aCl@M>7)_R z2xtUuJOqR%Bf|0n$1_pP_>&f9JlD$xZ6t7N=<tIqFIFpl$wMY`2|Ice^&%kH#?LoTI%0LNoY`U^SbyzI*OnD z5ud+2Q1RA&oZ1(mqvyh!bft<>%KjPYBWbb3tR+hkyJ+39^n>2$*rp)`K4p0Jtwnfq zMl7vP?UluK)iVEXPiua@< zF8Sob-_>*%N&lK5{e4$u?mP3KwlywQF@vwxM6o2NQ4Y3xfNCBGA%=xXo1Rmm0zcc4 zf$GRu8XgU8X(}&fVrSR? z0$oFV_tQ2!G~y;6My)aRf%X{t&ab~Qk2jY4BcIYHvFbsnjfx2%W#w=_?DlrJvws_P zIF^3OTre=^P0*F@1zvt)53B|mgfddb*v&I(IXKCUOb+%U=s9&JlNJLx*h;O>k3LI4 zFf}Y4=(?&ElU zaRM4Qw#BR^v5ckAO!@p4H$rgTsoWBEY~4VbNk#;HD6c)Lv0?il;P;!r-RjPT zUv4h;s6*?QgU48_3_?Xx8y(FtFC-CIT=+}nRQD_1OP;l=y6#f*?Lu`I3fAyV>aFc; ziKz>ZV(dNX@bhs)#})w;1ehtNM;6^`eX%r-mI?GRt|t41N~B5F6z1(YQ;fS>dE@)n z+M^)@r;A%MV=5qX%P{{XP-)GLQdfB|n~Rtq3#zX2B1Ia;YAPp}__IF5O(mQz0aku} zGlSGK;I`5dHe?lngb0KT+O^z>Sg!=NIuVnjd?O3y4JfF2es~j1et0trnP9-!05p%} zJYf=F>));?qlP}%eLd_9Yac=C+Ac)>s7y8dk^tz=4kYmM#82pjek$u5)}<{nY5+}N z(gEt4$>^TGfX5R3Qa>07K<6G8!GUm*ih}#ErWz2~t`1r?N>iK6< zGw}V7E0w@jF{QbLiPrw+9Y=7}fR<$J4Ob@h-_2i%8Q*QfS^E0lzHDm6TE_=gEO{o} z2xaY+cy6Mams4S#CvN+MnQTK#Ryi2UUZVEQD!nfgNw?5O(t(Ql`0b+PKw2nePa1yS z`Ya^Q$3Dhl_9nOld?B&YvR@!el}hN>a|~u2cl=k|a)vSWieW^$W?!+`Ld*4u$n%H8XX} zTFjv}%!5N(;IlJ(&6 z3`8_}2~SLJjov-msNmqwKQ)YufI__f;wBZaJCUv&J*a#l(?F8UMHj2(EB<(0dBpO?sOh@e&k$8U6a3v&MxO6?{{IDLe45Yg8`==CRBSX2^ z0NuW=#?z19iRo`XfGHD)GI+fsPB8G~Ctt2$O%(hU7{+<&&ueiiIhEtUxQKi>o}c z%7d$YE$lm$OAKGdgHN6_AectzY$o*;c6TL#z8ju^stgC6ZIVeeq8;{9dPgUgYWLaf=$Sh$S2Y87OS zMpU*H<#r<;*IdW`p$1*x0{)2y3hi#kgG0tXA%x|C^;an?1My!F>_F`+9zyx2OjeyP zMnJU58gl~A+L?+;JZRWiJ1f)nXlk;pjn71F3Mu%WUzB8`H*>Rsx4VP7E|T39JChMf z#`5m726s6Z3YgF;;5?cK+v4H(89cCkh>9Z-&49^0BjZs>7h}#8a8!H5h2p>Cn;BQ2 zz0_X3v$;F9^2Id)XAc)?srOQ;Ap=o1?eB_ox+gQ3#~B#|U@-4^4DS6CrP0)0FW`ZS zF-wrnpGFpF(aur%_Pv}i!}v^DP=xuBLk&7ho)n@OaEI$8o}IZL&wg?Md*Hi_P+6!8fOH=+v2Pq~=iDjusL|rh%&T3coHqtKyyfedq8?)GAg*WJKk316F(J9&GEuY!WPw7bW?-`Zca^g=AcWxLMhH9Z&GcEw}Y!DPI= z;yAXw*%x7qy&=z&eykc>ivmm62M^f21^SG`b}>R zoVR>@#fvLUgu_QusEL`QgmhKgDq|@c7-C6vIEloYJKa50ec>eo*9iI0Nh6>U&{x{?fUW>$SL!$Y~RCO_|_*;Y(1tM@ax1p=V!?^qQ zZfMNF$f5rqq+6 ztm&jVrm~+Qo9sEDC*87EGh;_oEX;(n-74Q-d}R@~ibl~}x8%?%a~s$vuD+Is{Y+lE)Txk|<9 zG`2bS2>SIHt8yGUF5^aeuwoBV0phWrN*prwC(xmFh|0J3U=lxVFYL*x&87@4oq&$f z5fE$Iv7?4CPfz~HdEx|r>hs4A`wt#hRsqc#&IXD6;TQ&UJTqA%U5vgrbs2wVV)82wQaExqZtKy4pFhCL+(ipCh}Q2-`4l8N zH?~2$!5?Dfm-nGdhbZ*#(_a1LQ87p2%^B04eMe|v1#@|&&iotlnj!Rc_wD8RN2dK)FiQI z-5D?Un^K7(x>J37a)0H8W9#((81v#1>JODF3s~uk@Dumbiu$YfyJA4krb>Bf z>W9nm@+bT0!NdU>*RU`{rm4}oWM@ns1A#l?wb6~yI?@ldz1*5l{- za*ib#t)I-^h}S0#Q0^6m^o~+~cMW;}{Ei{Q0vs8WRNuuE&Aki(BgN&Dlxr1ik1eMQF#X%DcyF>VHSA24D=e{k zDHb&;Pctl8eE{uSH&pJkZs`z&qsP8CG@vVus)_r1MIHQLo#|p}Q<1*;aXVgrdLPi5 zWeWK)$t%Moh2+q~=Qj4@A?dHKcol7^cdDTz!i?#2*U6o)jFWY9^}(sC zk9g}%#C>uDLb2-Vw_@b9czJG5k`ZX&p`$~adbw36sRQ@XY7hJIj~7WB6&}vrkt&Xs z00|U?0DfvCbpcAeu8h0)w(inNCvKquFn#7CS++J_%>Z_jzBgTAs+u`xxo-EX^XwUF zuBMkv9r={J^GO}~^PkOfOVu_Bq_57>1%EDFg^5lJ27-!qi(8KRFZnUHRH%$cw zH%R~cuTz1HuS}V@N$&fwlT@u*Nm|@EM3yU$Z2f(Ytr#P{Rr57i?+M>5Oft_1T0xGs zLmhea~Y|zZ*SW^Tn8RhCbpM2(fPg_UI&|e)(HI z8!=<%E|PDDk5>(a zpCmzF@@mF~%ZqnuT-mv5lE+)Ve)~>IP+{7;L&vM%t|WaSjziOO@SqBrRB$wT%M)gY4=bne9TtuLnKEJ@*>KK~As+ z0PySwQ(%8jUm|_`Tqc)Z)Jnej{#j+SSU!5?7P(j*ha+He!30<1u6@VftGjoUr1zds zjpJ?d(3f3woU!Uyew$4EVXky(bG^yguwFH}s^0@rKQ3DC)9(saW#U?i(r4uaRmQJf zBi3BjqsA?hU)4!ybzRxlu5paanY~NP`^m5$XUR1^E|NQL?XK6tc=_(vc~U7lSo&Vy zL1yUp_Wz#wQ~jpBWgOLi^3^z*y>hp-YgyN{?eaH=g;aSAz%dr?9wU&h?SMx zh+Z3S$n&ptS7AmcHMLlZ* zXPYh#v$D-&wR4XINPq-NHUUibVlHE|KC9HNS;0(aU~b10TygdmRg~z zacSw&6yAP@EXykq8Tm+f)gb|Ngy0HGap)+k>u^HqC2DkM`B& zr4h@eS>+(Jcr;1JdXf4G>!lnJ-uvn=dG5W1(hiy#T8DkSDo*3GeZhSj^EZpt#QGO6 zc9yHJXr8NqYx6j_c-N>1(!611BXmMVrz^a|kFELfRV@?vJwQlMr^;^+na6;WflH+6)!`DVFB`adV3mL*W=e#bv9K?n`x5k}SbZ!Yy=82G4EcGZ+Z`&df0rpP^%IeXNsFA}_-H@E6bn`n!U2+0^OKniH>;16W~QwhdbE(YhOCf_ zss@-Jdi+qL@B7q*!lslgX&4n?gghqmIs5~w6qR){{`09j<agU=$Y=f zY*g7)fyt}q-kYetDbl4&mnb95nlU9w+dFo>AtMAY^Vg-wh*vMwNx4c!fH-vIOlhpE z93iM%u|8hU+ZGu!XP>lD8|x69!MW)Sb^e&4wi12SR_I0TYD=SQ%F0nywJjTq@ADCJ z&ePS*##}YI)k0Sr1IJz!Y%q`j36Q|KAb?-KL}*3UEh}$*_><8Z=+L&Y)UH`2_vI5m zk3anUm%R4;G-=Sjf!gT*pTxvON$+bel-HlXLtfCuFaf&qJ5^U*UEaKceDLkx(x=yj z#u@bZ5rfPE1vt+|2r~Wzbj7+p-}tJdV?RHAUO*>wa?lS88}#Mou@}CPPX^yBoi1pi zX0Dp#zR;{$yG1jq?O7EzAeei(`RT^$el4p3&Xb`{p|=||L5sx;l9VWurYz72^ow<3 z_cqfUOdD=hlRZ@{$EZL;GgpX`CtsYP<5mq#oq&F-|4s6^_6a7DHf`A@O)H1X{P9~< zlW>i+ZB<`J4Smuq*jcmrd9d?Q#i7E2RTi?rz19kPgPV`^Zy^}J~=M?4i z{wJg4yU*1&vV`d~n5)lbL5J4i60GZUpR#m~5u?EK)=?GYTqeDHU1AnvAa1-0I6i-W zhTM2p8x_=q8lg;Q)u#CM`<3EfD_HicPpq2J{<2`wPIbEeUwP%(f0;V^Tz8p{Nxht? zkm9v>hs&(rp)H~&wsQEvCiA~;=`MY5?rykarTLp5{2)_IKG^+7>#jaQ+hD=;y)y9S zZ}d92U3&MtNXOgREN((|zZg?Zy6|DtCV8#Fa5b59vpEO7uD-|!ns6@+Rg;}KU!%uN zQ-2pXs!;7p^ZE1dMY3;SqA?RTqIDx9puo%Es&(7sn{asG2^p&BNrq;Vb2)KC6vTY51p)o3U))xmU-1NZSn7g$L%MGo0}(Y@TafBXen> z>OQnNR991m4_z&t+WjH-+*lV~Pd_*_8&$idM)e9>2lAb`h{NTBLlsRzg3Q?T zanAGu8cqXV`c^O2NcrH$#nR)-i=|1U+VacSPnqik*ISzUy$DsaU`eK*YN9}$efR3o zQT{jVX}zw|nqGDKY`K5`0e!~*Mcz<*)o-^LA&))ON427^Pyy1Vh7J0;ZD)f14OX*Z zsnS}7Y(w9xttMkti&?)L=*zpG{-zIfk&;4lQ_88}!M=OoyHmSujJ%{mqnDnUB(1Md zGa_0I-YphR+@^aftZh+2I`#Zm=KuMaI)kqxuME0f=da4hke4S)gANsC`$5&zaDCki z-T~$$0TLjAVoboyQR%bD-W54Bk*IEZA^l?27(0#;DqPiISV&R?^pkp zraGSj->Ahg;M^ZjfkfMy!SeZk*GrZ16ZCmIT3#BtQkv>~8s;bwB6Jb-l$HDC^Y5le za7ds${^lI@nHFk<8E%(Ku^J}&27%p5)v}!SK~L$_*0n$cPiJ_iz-h!E_c0th&v;+) zmiDb`$$y7bl!yQCCmFqXw<=`&7K?elqK17&0u9&0YlQrztB-2dOjHexyq)+KWMFtW zOV^jtEj#w<%4a`4A+C$HAZ5?DJYdtLVt~9jYL#4fWeaIkH&)<~`lomLNdH&I$mowY zs<{U}z3Z~1>z!5P!{@FxW~snw^S1pe-CQd5YX$1Nf8Ia(_!422UWG=1^5&N_jNk-tMmEwHisUZY`uq-3oH^ zkXhD#+0*!`sE9Xwq zFIc)+k~XACEnPi|uwX&5eEu3%{Fjv;sy4p*e|o*Eb_vW&?@3C_U9AaiB?xx#@|mG8 zj|j%Grp_kJU#9)i`Toe70rJN9^>SyQ-BL@noL%Sy6ik6!#=x2 z*}JY|a|6-_&2^}6mu;F>H_dO@x=#i@JX!bcPp)25(3Gj+FLQULNyBJADX*FyGZ!C_ z8FSal6&E!&CaFGt`U<)5e$~=Y2VhcGFOZGerk=+qyT!?mGq)Ri-?-^BkN^pgz_}uT zmBtm+k@9b2x5#fJ2dk6fw#K|plsbiu-?c~PEn2UW@Qb8Mi)h)Y+N0HULBb6^Kb5;5 zo+mAv*HQC;YR`Y`cKP|wWioq#E^5%X#UcFIX{=d2M^fN7VqE^ij zm9MCid0Fz_N5ds7I6$`SJ|saAzOr9m;NlvF$R8utOPe*X$U`@@khmIECFRf|nL2a1 z4F6)8{^mB54?i6(70XA+di4nqr2d9d^~G@Rg7q?Z$S~Qc&mwAT$T*$`&m?`Jic~f5 zK6ihoJ~K8+{rYk0pX-2({(YY8R0r98yEl}@E4E00nmUTqdsELVKh|;S3zd#5G%Q5c zt=%Lqel}0vBK7`MH$-|r@QM8U){atLSN`wWv(NY^_+#Q?efCY4X|vaA+fi!aDo}+D z3uNNFEpqSmEu~7;SlOpeohM9LDu1ZM@wyk3lQrA@jhfZWJ#>j`s=V{sT=jKQK}LSDNHs>jl(%2H z+f0VLrn+4(Q+st0M6I@w^6^`J+-7?2g|pKaE=?Gyd3S($}jWikb5_ z$Qy5eBSXh5m#f=XmDa5q$pQ6a_WQ(zGH%ocX?01A%>PUKXJ#ZYF^?NprZjCABVT?n zU*@Xz!OcCIO0_sWU+P$Wk(#>ta0$^t4ml|m~2!h>u-HOPZD8f zb#Jy@dF$))_fIe62(w)88(0)$OnG@_ORxJsk;nUWQuAo#Wao|@^2GB~<+dBE>vL;4 z*}hLrUR5Zs0;V4%S{GRCN=%n9z2_&Xw&nCWtK`jhhRGT@F;~-SwW|fn{dbO%+4EMb zfUTiw@%qWEIm_hJKU7e2cT1V_&jz&=rA{lY3NJ1^4M(u*QUip6tJiFn)+(6UzHO%r z`)Qsmo4QZ0gK$~1PMw$E|AE|nV+Xwl+$+toO%_8)@d6B*{!jWE#Vo`s@Yqzc-hlx`mUx58E-ejzqkHxtO`Ub z$zByK^Lg;3$eDozNT4JU(BDTqOCWf>zhD|me@BfUs3mFQTg_im9TWMcp#7tH;cFGa zEMn0|5t6E=b^#4C;M`$h3g$;&c<3@2dE+qg z(s!ZrG1%F)(;7Qec`7!KkbXCPujO^cdG6(PTGizg(o3{%sGb}@!9{8>*fXiIG?vAW znOU5tbD}SQI8i=(y^lnwqvF^K5i<1E-txq4JB)M8V70wguXe04c>@k`vN-rZ6D9t@ zVX0U#&}FJdzdV>=Mu1~!1j`*ScTjB`_h-C}owQhP?AgwkMetRphNetDUpiEYe&Pqo zRG(9+xtFLmM2A*21dfdn<}cl<%h1%?wn~bj#b86UI{q0ua<^>$cf4K`HH|r@ZmnZw z-miPa>oU~u>bLE`Emz>J_Q_6?N9EXO#Huqm*KXgsj%?q#PdTQmc_V*S*{@_wNTKcx zTlUC%|63&u^oKK5-!?GDN~#qxV*EXiBFUJZmB7E z-_qH1aosl+R3l`Y%v-iYp&ylhUE4-NbbUaW?Sk{V!$ob|cGYWoZL{iydTdN6=NdCQA+kpOSv05*8?1 z6M`+oO=_BK*t~cNQG1qk;sWKd_om2{Ce^k6Q0c5TFgK35L+0p&70gl9)=8^o4Xa4) zxC&Bf=mdH6(*;sjZ{!G;m7N4B)=w#>$NWPABv2X%;MM>w$4a`gUvJs+fv%7enXVuF z5aNCwq?4~PvPUP#5tFR<%sTC&W$1@XWT^a;lTUwitA$G?^*w|yk~dyLo!(NZMX1qkc;|JL&62Q25$b%~<(L{mjcTnz)wy1}JonUCO-Dc-K~hsE z3=ipj=YzKw>T$XRHHkA~@c$_${GE%Ndq`si|nO-F71oM-R{H%4LPx|aoH&Rw_ zNtUsrSL;JWgbG^Jhek@KPHu)vye{N;`mr&Z4?**n50%>O%E=DZM66KOPsYyOt>xBe ze3(?M6d(und7**|$KQBa7h|Z_LU`jyWf7r^M6%=sUH$E96LtCO1Zb3O(1jcwyH%Cj zZ|WlO(=d73auu8h=Pq=?Vh^0Nnkv+oy*5E+k4L|8cC;^Q-U;!$HNCu=c$m52pv)XU zRs!lp$a{lvUU057lna&m?W1MB`sWZCr`jO>O?0wx2JS5fQnRE+n+RF9D_JMv!5;x^ z{G>`t^?#<hssM5Oe)VwJn7R>a9xKeO%8#uY zsLr|%%ER}L&@_ajDljZ3AHTaq_c7*!GGM^v`W#YGUU~By{Z8Djaeef&Q9pGhA@Nhy zj`A~$M5?xolGz*f%j`#W5rruWj)8g}bQPzb2V8sqQBw{gVCv%Ot$nH5DA<^GHReGy zh2iuFKzNF0$BO#SWv&Cg5E3*F&z1O85BHbtb%p>Q{?pQ=VGVgzH8p}&qx(g5 z7GAw|sQNI$Ib$FJ5+H$+PrytERq&BD>KGi)D1mAj2Jh!N(-CHXc7Max1d%E(Kd5<* zT;KDP)Kq3fxGo^jXECX8qR-hp%NC;q+5r|NTGhkJgKj~{8H=BJwL;^hb< z=#B_=%?+U2xPR=7Pn6MrERaV(Ss?W)`kH5-g7}s_*Tcp=F-0$grut0gttOSQ(AxF% z;p!RoF^*=sNEDO(&N;+$-S_<3H&#}bWSF@;q$S4!WjuPaFM0y7YZ+rzUuVwOwr+-;7bQ*Fk0<#7wma-P6|9!ymYnD zRe_`C?UL|UewJq*?kd$)9Se@7+qSHALOnZoC&`rTyrh+`pd0h5+Bv?kvAJnhQpnFYBebzI0;sU8LT85)P!w{3Uu0 zRGT8UVwCxG?{dr>7z)l=v{CN=a-p=*38J|>(q!as3#4U}8pduj%yd~(T|R2k;7wiq z_}=6#>hS$fxwr3y#qL6pxN&qK$kWNOlaWz8lO51{N?M>bLHhHE2)4(&s)%vtgH( z3y`XnB3;+0Dz}*HLi-S)*Ea&LO=HeMo56rg`axW?Iu1l>oG}A&=!mZF)YY;-%#)6- zYa4-94Rs9u=$)6CeE0)%3W7DA(3`*MWXlyQoQak=T@8wld#I*rrm!sv{`kO58A4!y zZk44OlY1c^lK=^jKp_yoZxB?#p}ij0Ceqk+$ImOCCtxbj3NvieSWR5Lb+|q|Mwkb9 zyojXgJ<|AY&`h!V;u2J;yh%en$$I~(p_A8GB#@MIFRI=;!e|v7&=p+BgMdkR*xog^ z>D!jq82lEx`hZ0^xK~$e6{e;by(B>=(@_q$n{vTE649=LG3}SE{lIt-u)#Azx)wzH zh_^3I=u5D+aX_7pqh5r4ZsS6PeFan<$+kAm!94_r;O=e*2=49{Ah^3rkdUCk9fCUq zcXxNU;O=h!$;h2M^X9#OtzN9|K6G_e^{%hJEwu~(A>AMGK~N(jh^CtV#uGWtgD4FO zp%6)(Z3RpoJtVdRsk0sEXK$y=tGM6MjhY7GtW-Orz4eXlTR zmzX;C%WrqJEGBzloS<)CNe}Y(?;}rRgRd?7qfo>OTt>9gsv|gz`f*w}8ua@x@ir)h z24NYsZ*)iqTl>zI-(){^)|6|+qDMKtM57qal#sbUCCwJ4*?mohD6sq)UyyCvqM3B* zuHde&1S1q7U=g>&_Z=p6%G;xirVqr(lZ9%$+rQFsb%arlYNo=AIdLY;ti~QKJATBS z5j44KuwOZHbH9yA$tivS$6@yP4ln$XD#L7(5B!r6u-)Rx=zTjt%m(R1Ynn_`ApMD>t$H-k%5 z1$F*w>xFCo4GJvwX-k|BJNW#@j5msIdhIO`F>@ou9z&l#nGTSy(Z>s|SV}KZRQ^mc z(!%;!V#=^gQC`H3vf!wZJcr2gq9Yx^)RhjEN}C{x(+5`<+9=AC?tL5+N?JI7(#ZBP zV(%%+Qg2WTg8LfT=!!=^|I+wuD8!V5)p-YJrchm9SgQWx+p)|lxBkwqy9MO3DaTq% zdfn@dKE%%8H*o>RdnE)cPA^P2F8X+^(FBBqA1KrfYxmmyjWsJjh9EwDD6?8jp`o9S zmXjI{^h+X+IFDAfbnRDS&j+J)YP!vr2rIhV?7$__);Bo6U<{eu+O&O%KWt(LZ8wB4 z6Z?(rc1(A~l;w~JKt;BwF3WioeQ#j_;X^dL5AT{d+z}2F^Je6yu<=3>y~QS%%B(jA`&|sEHnPcfq z6ZAE3C|1KP&N%L`b%R02c*&vh4PQC-wIzbx3iVOBJj+TL(~Z3xAXQ zZo5z`eyngW(e&2WU2245Sm%ex!>wgL1=?V7slD%jin3~GD6B~WyjpOC9+_|1BBkum z{SZ%09bP#F2FU1TYNd}i1=!Yk8kTk@*YB_h=^5fkgbS(R@~tex3gpozpX}6+-zW2$ z8N{2(m#Vd3DoBd@!i3@abT6XjC>MMD6KiQY)VaWZx#kwwsHJnk;w)01#WfZP#4|IIT2b`*IT06G0!-u&aTy|T15eb#)OSgHpzmIvGKA` zMr_S5)Nz*dzWjNTEyo-_hI8EYZ9C<;?<}i3Vq5U~QMEO-mObdYV%5a`oDx|)?8@9lWESl9Zlt?N)acEgr zNpwOkImpSp>oHWSzQj5gy5#Jp!2UjF2GMRN-o#{lXiCKlE5B8PXNb3aIh#VJYB(Sk za|S+$^hP`VL1?h(4O$>e{pd=YYoAB)+Obz_)u`99;3E`qN!K-ER&V?n=Kf=UzQc?t z($=8i$ORU5Tv=XK~)U$sq6;t6qxtWT~me{$WkUjf;1-ntk9z`a}6YE z>Rb!(u6&;oGfiaAQ@g&^3%#B<`-^YKUkeX^PrzW|@&C^)N5}^WNcRp_o1+kl`7P>6O^Qv(R9$O@i~JfPr&1 zuG-6RYA=`*QDk?%9w4O%`(60y#LQ! z07gFs=3iW6>zZoQ?^fWH&IiXpK?p_>wpN6<^m;&unR`wsINJC^t+t2;)geF!4ReWJ zN{xc=Rt#&74J($z-WH!)%|ZfYs#n1Pqq_Dui|){Yv`ToHg_KYb>L?@BB4E5e_i?6r zUZ&(`5`aa+rf3M6^WClC$HpCxvW#9if=I5v)BG-0~RZQO&!E$%iS z^p<$T^`?P{qHoO%nPQBKg_1DQQJtKGOHU0Pi{Py3J9{zSj>ez9r9Zg1wEHv|6&B}E zSUIP9nx>3zHMTXhp@}Rg;9@3SL+Py!ytFhI=5riRS(@SMg#a zaAZHT2;_qvQ7>hTLj|Sh4MO34El+hkB{Q!Sn!2VUb5-^AZSrnM|`E zhgyC{mkV5NCS+$nQ40$Ni22UNeH_D&XI{o8m}&Equ@9w0P5gH3_SZl8H3Q~>BP%-Y?zONHSxPS$Z8x_5!q!486Te3Tm6YvWj7N-Q znT;Q)=8o4 zYQ{?}@Z7ahrmxEP3A}KLU!}G<@BQGKzaAo8r7JAIueJYd^JjLGDUZ#2F zvzyR7WjFo@>nmfa(k)j`o+`89Gi>av?`oc$X7oPyIcDMfMxlsNAUu3;E~2N&oPC-Z z_0)qOlh@MtSXdjOFe=90c{+7Kd-?aMPI*QNF!E$DI7*_jVP<5mu*%8O78Op_&)aEM zFbfSD?>nLrD#95;13U#Lgm7TG~&XdK4S3>Q8rDvVl`Zb2`~=f)up zL1w3i*3Jr7&DF$7U7R-q3DQw7ZC9?gJx9J{n!W!ikfF}hX7>O)2F#^5usW0fa?fMw z0BE~R%ZX}LZ4xilppK`1Dz7W)q!^M?4cP4`akJA3v6v)U#hTfn6}3 zJEP^(05$sRJ*0UVW72(Tyn~v*YZ@_@MuFF2iA>;#w#P1#*kRszfo< zOv+K}KwUTLUb1p`u(>CV(2yUA-62tM=%!bFn!bA1X>^33)v)MC3x?PDGP3V2cnvLH zz`aGa*Nq;Xfqn5Mw%Y>-^$ly`_7@A5oQ1L;nmXO9%kfHD@ zNYZlf4f}Yd#<28Cvw8v^tlL`$%D)o07Avb((3QJyG*4PA5i>p@?+jQ#K%#&p-bGFx zR)qCt%Sl|I>3B0{8+G1uM-%nJf`g^n3kUlDP`K@45B9SlHtx6%c1qQk??hC$nfnBZ zPrR|{{|0$@(7mzO%ExM=&rXMQ5e2Z(zLKt=;}6TwLx#1OW3g6&J}ob6nQ616)|f70 z#*~S_2#L3RG-0YE@>bj?tugWvoE785NX!#;5Lf|Df;JogRKO{kg97C}jSG&Okeh6F z>?4SZ?qZA+;9nYg=@wDC`68byTjIm1x(By`e<0kZ9xfd_vh+}FUiHI)m|B$Q8qY#a z1GpFDcWRu+!}+NY{ECfcd?|6E^WM*H0A>q$N|o-80eE2BXEoJ8{>y$9gOLf`+zOMD zNA}FPoqa@0&wwY*L|V?jHK=UoprbfrmfOgR3V;^8N&3#TI9e>X%w1Uk`SookwAYbb zvU=$E$T{T?>F!D>9}CsW%H41HSM=8JiM28!XAc5UB*%lOD9o&`fraYS3O5I=Ciccr zQ;Ifxs1<7~z-I^dNx-GTRv;BTn;ftc@GC?@tbMgyeT%MjT#{i9%D*bZgSR->r)P9- zW?v3-nUNdH&bsS0qrqk@bIGl@>&g1d=wo{MMwqNz+()}8GY|Dk^(yvr7LQ>%qgGfH zzjVlI%&TpQAQ(@KDG=+YCFt48XM>NTu9}tV-RSD2w$r;dz~VS6ATO#~r($0+9)9a? za~7z_L+7UZ_^wSKz{EURD+66d8m-iKwgq7`&sz=rJ8^ zi(c>-;pxt}28!eB`~?5Xz6b}@=0YM_qYsFE8(BgZ3vJqkzA=RdelnH;m_8vlLcSQm z;Rh6PGm-ER!%6gO-*4!PVc+`0{X8Q8KUBEmngFwoX6w5hdw)Qbs=tKYZH@+m{)bZOhwU@9@7< zdJfJxd>6#ZEy~q&cC5rL_v|}89ioJBr74cr0f8gHxJ67Yl(Kwk1HP6mH9kJC`rB*B z{T?w&P}vmo#~Em;Z``4&UG&#puYK|s-b!S4qqbv(xf9T(a)9G`Y8qL7dUQTc|2Q=N zw7U_Hi+i8yq$8*y*`k^7jo?9gWW0UcSI_aRKG6w0`F%pPmbI`9x978=<@6VWGIO2qojUD5A@#5T}M(WvMW?lE47g z&GthG1QisObNN0BnuiXBJE-9UDl5Mu9Y!8q!m3DK5z(j2&1!J? zHg&5kF4T6K=)41r?L*0>T&*7ai}7P!G9DHUEzV*?fl&cMIdvk+gZ`)MvAn@yYuFl{ zc1_l_9jlPXUtIoQ50Gn0K_r_kRWxEF71d7xa;KYTCj6TPmP(4)2t>(ZGF{sX0VC3M z^rfO~h#vw3eemU1uu;#7hb4{b=Zb3OkX23{E%Aft0^;rO(_5e5TWkA>Vr7fsDxGGK z?+=Pc(3e;gyie|M4Pj(tn198d04FdqgT*FTO~3x8UR*&HWGP+^FTDHJpTCaw1#+N& z@51)L;wUM^_`Lt=P8=-jjKBZlJ7q`1iD&_I?34Tm_lsL?2SP(0+YoqqdaeX{h5S=( zm!byl`NPq16b5|X7Q1PZ!oBNBF>B+la@gt)sv$00Mr`C%s4w8)@N8c?Bn_~CGcb+A zDP$_{eO~ze)xT;-f{{5;C=4@vAk7FaBT(#2VvfH-DWGT)mc-1J9Ef29gU*D@M^oU3 zal2C!5bkJ8Tj3`I34xN3ZJ%I7m?Gxz@16Wx&!>Z4wvmKjB$)|>VuDL6R6_E4E76Vt z4xr!LTEf8Tjs8z9{&CV?h^9cwjrg~sqeT2bdnRN4YQ^7=FVx@`W|&hLT{eG}2aLw> zYvIdw(h#c{`dtVVq~OjtTVvBTkiY&>fRrGpXQW=acL0+ADR7+(Y8~O=^BpwQ*YdVD zgV%qNGy1;M$4-BQuJaF}S5&F5kS=@Gc1*!3Kj1u-wVcjJG)oZjX zW|ZWQ;z4il6NeBh0DTr8aKf0cwqz9B@IMCUcdC|t@_w(Hs=72uH4T|V+fw2f8GK@><0V6vqun)@NimJ z{RVc#{=yUrs2jEm zd$F-=MhoSCZv8JOTtt9WN?hEE5DP&K`@QPOFW#nhWfD$#3vRbRI94@79M|~UWQU4V ztugj%OISza>$FoS+l*2^E>&NXtoAPO%{R{N%Q)sOdDRM-W~7;<-dho-oD_Pe?C6d5 z^47U;SDyDDF1yCpJaX`tmyVd$aS%lux2`O5rOZXlv5#h3^meWFNjEzLhs)wP536f* zOdl0`Ea^7!#~|0w8Aeu$0v|a>-@Dz#s5e$2n=<1?a%s1%xbAeddajHbX-AqEM_Loq z+N9WQ9?#Bhx7O_6TU%>GTX)TxEdA)4_xMt^&oN?S?Il#_9_JV-xKLTepE73?{qgpA zw%5>;<)uGH9q0{kpV^S&znN3^xfMt|N+l>hp1$F$v9`3k>F8Z3m95h5Uh-}lVJw#& zvOUy`962P6Gzx5XTXIq_GrC_stlZ~VjUOu4+jT#WU7Ou5!FwF#vF1+cOA6*nIU%&2 z;_F#zOg-LJjbx3aHw9Ira^EFIy|ww^2J@79boO-F!(vEtw!wBN%?78U<~pqex!UA% zr{+{fJiS?1@BZSexURb{0#%5?)MZg$+~K_bt=CCkp95jr?r?}V)Kcx?UUwTuMt<|? zgB+bZUGMBhA-%k_+ZSH7rE^b?@)aKCH2YESEqg(i_{&(IA5XQ-tQn4LEu+noR(!R9 zd*&+l=&`9*P|xyQ>|$LE?dlO7R6Gxr>*!gNtj1Sqo9f}2)}5GhjTU>$n^?Kuvf-1? zH&&d6Iqr;E5}7Q28aXuo+B|w=XLV`#ssb@cQa`u*W#rLTZ`(?6xvrK1FVaZfd7dj} ztwo<>rFY3_OB_GK8uuq3w03XO7^ryp?Ns|wkl^I-9(Gs$k_}|-l)0oG>XqJwCoZvy zF6STk z;#_w`8a*pateYh2tm}8LL0ZD|eQp+m{Jtr=>Eo_m2!HRJj)<9vsk|m7-RB>V94Hai z0@^4)Q}?+>8RsuX`A+OdjdokM{|I5z+=kE5u<7l}qvd90jd?*IjA?4Vcgk2p&9=2q z&h`waX>wI6Ya8oow0^z-0)OlW|6qPz6Q|C3?LoIf7WmJLd&1&FFcRx-5cN-9%|rc@ z-H6fER%j~;bm21U`CdWJy5D{NBy7LawtxJkvVzd)(e3BR^4cnp%Zs1QWbGG^#E%QB z{zNlPW&ABSj-=0d7T|VB{Yk!F`r|Ke*z^@|R$5A*9uoDFOY@1{t@Lh7=`HI{_M!O&KoND; zyE2#DkNeAG<*NZZL*)_5vb<>=^7+GsH^oesO}A^$^-N0w3C`HYsn7$;Z;hx9T)sB) zGhI&S<#SQ4#pLd)mRBL~3LJ=VO{o(tF*Z5Rc$~bXhx??4^NbSsS1n!aR{(JWSzBs# zYU*FoEd=G*+p0>d8hjZZ?HQH?QV`{H8E%=Ih%%&fMU(0>pZPRfV{);#6Tet$n|_VX z_u%WBZn#E zLy%rqZi~j&O(8+-3t`f%sx1sbMoOxUoNu~MsqE#&s#+0U_)R*l8LQLvyk8M31~@}| z3rqGYz!LXQKJef4KrRl9jg5tsH5@2HjvK^5r1{0$j4P_k9UKxa)e)i|S#?{gZCsDW z_)Lk_Opp69$V6vU`&cK7pNrJ=AJX}^OaD&d_2kRxd%0^9-Negi0TZt$`L@?YQdL9g ztdN)*#~)Y1rv@bKb{u`%B|{CqJi4xozu>wI`u3@8+4R;{vD!YE|0X~X;XJ0ILLpRv zRoTZUgq%%9R!&r`C|GRc?gtzIz$Ew%(=2skz{vg&HRo?t_GVC~T&vxWu0|r${YHi^FU9|}&eJGsBX8z6Tcu|$1?a5)G)w?OwF*%9_?fyx~*XQ-wRk_b3 zL6-s+8NZ+M_FU41udi>%+o|aOvMiMPZ|(ojhX5g-g&F?XC-Faiwi|?E3PquZF;fh~ z=T>@pe%?OY=yG^Gu9I$X*oH5Gi&pOv5)zthbhbRdzGlaL)nZo^$Wk}P5);!QZjmaS*=yBWUm`RN`~AIH+Y)x?mH5CGW3 zR9q{5JPI!=2mOPC-^yDbq+pQosbVPQe)Qv-iy~#V{ zP%C1pwL05H{}*NcqphGS0VmLZDTfyyEr0(kxv@YmuV$~?6<4Ji3sv*!f@o%5cEe7n zg+}LM9A-AM3xq`DKG>M68zaiqOIpw*paMH7Uqfh`mXwqvrV*2trkm-J*#k}=)!du1 zByQaufpNSYw{`=)ZIvDnUuAN1cRZ`PIy?67bvdU5`iq?liXV~uuBI?k7*Gzp-tQ7E zw0PEmVz(th?IUMnQ+Y$DL0qg>%F@zD=Aj4fJTG~3b>(EF2aii6r;mLfC*Cs3gSJaKSJ>G4to6Nt# zhJqK|_?F5!Kj#t7AC;msp!`YV<#z^=)tGE%v0Pti|DE4*mhS$rye-FauB-v$=0xO2 z%r&p;Nh}WQ$Ns1LOLAsr#m>&o9QVr|haN2Tw;KS8jpu1l9Ag_#{@-!OkFr zaMIJyp8=aPPv4{&qWk+Lxb4>c7Isv{4SSdDg=0(>tIdD-JbPrf{M_6mbv~SpQ!CX< zLcz~iYTk=nfyGEXgUC2us>6fbXSbM+iXh;XO5v~$tLWSjC2|v0Bn%w#2=&yCKalWPP);k1z%ak3(X6FwW%sXHNs^=~RAXcXzjh zj!r6YFIv6*{qrXj0W*q&jnfaHDl}-G)^8${Viu~ECZF3zrzpJVbxHmDhW^{*Nt1j$ zou)%TFjTMiW;o|y+v~G!p;GP_!=Mu(g{lud@a`uKyT(s<=agLRA-<8ftDf>UO~)LY zbzQa4{yhbX*)ofqYhI+Nsmnf($6{0gm*1kIh|_qSY2HCt1^?UZy>9hO%d}E3Cis0& z?6QNKnHmHyr~z!L;1p2?$Z#>&1UybrHY<%uTOQX316?SB zRut>%WjfTKU{T_0t(T-0ewa9VOU!#%5sd5jz3HD(*SD_5&O8PKhg%Fe1H*X1*0J+{ zFD^b*rrTt7{%%1F9P$nkm#yDS&FN%Wv#jMttBZ!4dx7chptyYNxNa@p^R#8|3c=`8 zVTtupEshZE1F|eKfWIAp`xX^SsCe5SmXN6v5x{Qci5i%Rm>T=(ygL*le98szIWrF2 z8Yz|ZUzxD~HxZB>;hSUGyz=je^g#?PX6LNOFVy+S=mKA^1C2(t$e3c(@WyO1N3z!m zbC{aMV3ME{k!zfg$Eg6#tQ~*ac}|zob;Vh|Op|Kbdpk~jFvYgjoZ?Ke7sn3KzrByh zmo7oqC5gEl${JDT2o@GLrGzO2x=Iz*LIeDBRYd1%9YpXpjf!6qb>gJQURFq$L1*ns zu{xvqT5AgwG_+)?Rt+N2;VtAM)_A9ctZW?Ts#{lBPyFF(OH=df%cB^r$$W)jFlRNd zm0uguG-MWfpvXEv;LH&M33&)(Hv+Ue<3F|*&`G8-T6+^rXCdEMhFsD&(V!M`|4I4v zyQJtzLA{8mKmKk!UZ9yRT$4;@Ih!xSy5d?UiAB`sejD593!HHH<7wL( zfYpis92{KC6H*J#PV1zkYgQF=$=mZ5mm>t3U~uNcfRdbA1=#T$0DCG!hhf>_SH@PR zi1yRwOF7cv_IgE_PWUh+Ts+ioevoX;%aAg%LVJ*gXl(k#!(e#b+Wx_wIRBh{Fl`~+ z82yMLi5_j7eh`A%9WyiF1hSQU{{_B(S_y#=xqg1S(A8e|4x3iIAjl<-1y-1!<@>{3 z)mYBqPRUDDp-jw&I37{RCMQ<=JU`%ch}Fu~8ht0oAmyfwV7fmc%yhTYVCPbRUG%3e z1j)cctmCY|18N137i#G{ay;~tYx@yq?RcIaq$3f*Cjz{OkMmqS0N7nrEtL|DiNpe7 z^aBx`;^TmaO#c8deToxs{0)UCju|a~CfH-*vP?$)lk4-uzkkj3WsG)Da4#%EKDY5^ zQiBO4AMkyunAlk3H@OS9u}_0c;h-<>hMoz`c1y>;`SRPtnTZ(fkW4bd9h{I9D~uqf zv$@-8fwJz8vy=b9$S7Robbl%zVOY?Rg#i5i=6K1J4Jn!3GTyRiQ2MZ@<7Aut-iM2R zgbP*t5!5Xlqt%MDny?WQ>iNXb_RvV9wPRaZM4joa-jBE$*p{;;%n-qm0uTL(pCGUS zW~tAC>xN_L{9<5=tzPa%atNK-CJ!C6N5#~IGsWt7jw%L7Byjcb=6$kl?%4g;Av-Pv zV=5EB+=cwT(V6{l^SfTnC_wGU71uR(gqE|^j06S9a|!vWTgt?epc{_i$#?S)U%ZH| zKmxs94VPNKby;2fLFf-k$R!Avhy8FDn8@#0n?$!(OcA#}jqAhAq#1!iF-KzM=Xk(u z)Z70Ha!HrL$PrD9Wa_s9Hc-zePD194hmL1CcY8va*^le<0$#9ziJK{dz zI#DR#5b#h-4ArFR?mIY}M8^G5Bw~jUj2LIf;2}G1g?rP5%4NMUGRi{hkjG(ZyslCp zIxFj){t`>8KKfoHb7P+z#`_HOw2cXc;s%i< z!|ji_!`BRY&GU3&zozDqUJ%m&!o35DHXM)U5zX&Y_&Kp3>_@1QgN{LBGgz) zMlzR!a8EiBw?2i^#2&6R)lnNA(k`gXd!lo-0SbU!`#zUXxP0Q$5Teq4pbhJ{M`k55 z?Ev#|&Lw9_OojmyVSd5wcjuerR8)IhvS2c0`8hcN8CumcorMzO#+_u#uyRD1I}ut) zcM#7AZ_gACg71@^UqYMbIIdg~2B2eL7|F19m~&N=Lhu9dqt<5qt%W|--5?1A%+yPC z8Yay?ZK-pJ266DZ95Kl?md*x_L}Eif3Qts@qF#rj59e_{czgWjjNYtbrx89mPL?s)>01@ne~ z!y67EW7U0^0EgWomRXh8?aV-k8%kS#;4QnSBm|iO+?@nYf+PU1er- zG8B{hzoP_$b&^hCz)DPRJS=N+AW-%^01PWVU(e~YcwCveulvL1fG8FPULeZG#s-V+ zDxY#>6R=CbKtu#$r}*jVDWaJ095pr*?lwLT!x3EvanJ9ogK>>-7%%1@W3``&M+rd8OoxKaF z2l2frfaDo^02|<=!YE7_mNT)&MyTv&{6E0c3B8@y&zs0U3gQ$nK79w8yhoZQtm{@N zx`L5nvP43Xy=?-ychuZGc&#KJefE1wCA>#8unn9 zK-m$vydm5SZYE)OjoC}toDWnAK}uRuX1R7ZHyLj(Yyw0-}!2V37t4O-}HmU_9K z*jxJvQEZ$xlyTuZkR~fDkf_(0!!Il>9DwHb^zwMBRH#JmkKuqfywyVJv_B~-@N_1! znX*}2-pbA8IU}Mn!vAoocZf1BoQEYYyod_FRv@@89K- z4b2Xo-KKG$;?jo6Y@8T}*QG=zoo^v=D@aqXxmbZUn=Q>fV%v*yeUm`x-pjQ8JO&tz zdIs;;K<%7oDI+gb(DBP)q6Bv|-@1~R5}=s_BG@>kP+>jr;79dG%&Ea52;dm6AZ*@* z5Jm-1XDUwA+3Fj8ghQ^%x}T^p6fyb$uZ7nYf|3B5xl<&400Px$$}OdMi-fn!j*)y9 z{iKssrtZk3-`<1h zI+W-l;iz=yRLIvc)G2=UH@I1xJp)?r`Qg`cCXnPR5C!hJAJy(g7^YqJ-~Amp$S zgq=(l+jB>Z_;A0Qu?1ver(mn~dusD%Sj!k3J?{0vA&DTwHz ztV%!e1 zZn_xX>!I{<5P}rM(e^E#*2nS2iZyv!RA3A0$4tawkt`dT>8=tCmAovpNhNPh5Y=I* z)Qsz2A?DB={=XL?g)V03@6-D}VFfGEDl#6I&~6oIr}NZ;vhuy=-58+$Q8 z!Fb>sc`wM(nFCV0@T}he0&v+ajNKNW;;HL5>rfGgrNh)xlxvE?Uw*|?{ntTN33b%T z!UYy$PEBVVP|LHLn+YCq8v?=veURnLjIG=H1)bNx1?21J25;WoG}`5b&u+pr!-P?t zl>}K=K|XkJ_-u)FJ`w}U1%9MCek7Ei08}qvRhP0I<0fCPfEqCgu->4CjBmz~#(tDT z0a^G>Ga?Z9!P|#;m7nM*YJ^$ax^H(dXs5_cg*VPy*s0DueIc-90BS3{ohtkPF*M{L z)VTaz3+iTur}rkZ!%y2#1nz)kO(#6r`_Ef3^0`@xYss$Hlg4xt;5pr{%N5l&j4E`KPx%K{D{xv)SsAo05aF$WMa8#Z1jSl zpB~kNSR)JOpyLq=62znieo^WH9{r==EqOp{N;?k10+1n`vdb&aI#u$XmD=-%J#*SW zM&ccMh4AOtSWI^i={A`zP=qjD|3!^841=-U=9^J1j7?34F2@9IFGQ3#f1yHNfmQlX z-VZlnP!jEFehHpXk$;QUfy!EXy1w01O~4SphG1- za@)^M2ovkg6#BFUl3m6!5u-K`!;};L6<-A2ZTvTx|8Myx_Wvoeo@`kbE&BD1BuFdg z1Fg`LFI9ll2C!!qGJ8Kb?Jrd2k97W~X^<*V6b*Ry@x{hhj8-*~DSQLGhjAMUZh&KP zUTn)1(;G@gR%0kD%p_c2bF?EyyKX95Wli^Ca{=x9?HQ>UADoLWc8v!69vW>P{ z>*zgK?rYMFbEU>pfE<8go}U~k+|i8q;8F?whlgc4@|oJSr^mTk9J`eM@qz#QQd9;E zBNtTxNCnDdiFNPX6@RYyii7!aOZ!UE&EguupBNUuuroc$y?*fY46QrCALV(skUWuuHS!Lr zBWWkhb@nujGw2FsdP>1m&l`xtNMPW{c=QmuFYdD`;u=K+?rh4-TVkKT2%C|5zVM)+ z>vK#>1d{WAZNi^7G1Vb<-)H&{+;?Bm4GbpL+}<#d`U~ix6RcZ9A;Fn1@HF`HvEb*K zd>qOeCZ!9@kP+Iz(|HF9$rH^~XMKw|pH$j5o~$sb!(FiETi z`dEwa@2ELTvCTQ)0>1KlSdqQe%E;J3xS%FAh|h|nu;e}QlI{AHP~lIM_}7!3*miC- zGhn6yErm(|#8X&1F*&;~`nM_Lp(o>>SbFxQ_k%tt!r}dA#Dy5R27%y~KT)JyKE?yx z{6f8CUc?_f2lCuu--FUM#^a?(H0&vW63Fmq@|ow)9+Fqm3x@%r0{rq+yLzI>>oM69ewi#xsL}%iHKA zLFCxI0y~hyEV>e7M*@_4DO!r=NrEqx9*t&ro=;E`rp2R>&+C>QosjUmFlar_;Bc&} zWnI5E(F@(?u1$2#cF0ywhwgSIY*`EttDyODAd+Q0TCf^L^J3E1V?ej=T1CjO&WrOW zN<;RVFS~%k%jt~V#Z^q5w1-D9TEJUJWg71JghaYE3txRVi|_F4!eI&~o=U;f#c!zf zBaRqHaOco%iHMAZ4?cw&(4+&HyIt3o9UmLqY43e|X#wCN)>Iv6-tZnT3~V zRY@kZyXzg)etkn_ZIzrt1qtIEDG2Jt3N;Vl*tdxP`VXEGulx=qT_Xx@xF$s%=x z*bxM!oVB!??HE_0DJvR~dmH8=H`jFVi)zZ&NTg5Pc{YdC^{k--ut8B-#Gd8b2IsG;#ytpkbAZ!ejNq^W6GWLDj4^4Ln$HBSyWWi0B74ZP%!bR8s4eDktZLc=RH zbB&sM-?580E9up+P4i91vr{|tb(L;4P(FSkI^y4kI;yi$QzvXL4tYA8DaJh{CFuZf zSVTp&cuO4suUc`+qsAN@8Uj~9ioSkY?EH|3!bZ#lv&Rv*_ZCLLrRrC(Z0n0nv`P_S zPG#%G<6NZVNd1>aU%Jc)%kl#{)(`J$PRk|IT(+kJXB9PDcr?on=QIvxN(~RhLwRb7 zhe~hRVoqE2sF9Y$%w%ezU>+n3ZkG51gF(4>yvt+#!1NoogBjn`1SOxt!Zua&TfK}c zL?_CP|hQ)iM1!cd5WVosO;0cP%+RT$MwmxSEvLVjJT5*3r#`iZC7Rj>+ z{#+1B#cg#UiVsnZtMdUNG3iIko20urzJ?e5^8UcK7v_5})u0^HI<*5#GEo->saPTy z{RGdgQ&!D-<5KQm1yG9Sxx?1BLHj-FF5m{yzcYb9NQwUk`OJQ3+{AR(;Oee@ z7PIyY2j^Q;!LKOeHP79hZuA>GqC6SJNPR+yDc9G9&A zjo5lFm_;lOO4#2Bxj8yM(MiGQ<%V~m+N5rT<`Dz=Kw$AEz8A?d*$qIM?!B$;Za&U< z(GQu?KerZHCD53wYdOQ9*a7dQ+di$j8^K^WG6-6!u#)Gd+`0)ZJ3M<0`^+U~v)1~2 zF~Ix0U90WUpU(d%6AxgooT;(qE@j|X{{#i+sbX-o_?4(Qez5q*$7i*ex{o>~d^a)( zM&A+SCUE{?E&S(YW<3U#gMTnDDkBy?TVqADn0ho$Rl*j9dYC_lXne_P3PYVfKBc`X zZDMtXPg;>&83_>8C{GesEt=qQ@`_5TS60vL=lSk;IeId~S=6Mab<&19U)Y7JE5GDC z9uA#D-k-JFjRO~LYw_1rt8pnddlpZ+TWUWI^Oth5C7#l`EzpZYBv&fTLze9y$Ixe( zRB-jsX-O)Ma}1|u-Ef$_ve^!GMAA|@zo7(=#fyYYht>xls4}k-FT^-$9O!6>6sh)B z_7|yInrwW>`q<`R#ce)S8qXsga`U-(&BP%PmwDUBxE(>nhJU#Z$$8u^4Htu%>>XPqc6`%Fu~V#0%~{C1vBBoXuJpk3Spd z5US~-BvzuAiacviU&S4NMc+En8_CgG0I;KxvpBwpY1SF$4`Ad{)I8`->D1E8j2k8F zgg$U_W~p9!u-x@DZGA(~mW+A;C5-NmU+a^_UdRndH>+bd;47*Dk594~9fTp$w;QiE z`a;AJs{MbCP!Okeff{W9LguA?AT@wLsJOd-+*#i{6_Lt1RT_WfEteDTW7v_UMQ~R#%;KD75%i zC=>BU1Sje2*312QCHmqZYUASeDu{+_01s|5Z&ZgYQt-JI@p*@H22)Z|pOD1iG>v}- z$F=&2%`%^i(C)q(WRJx#M<;JRfF#WvRxH#e6$>I}FdU4~U0g#-TErGKUoA<;8A7|J zzIck^9Q@>ZJABPJSuIH*`-#*pae!rEI75Uw1ez#;)iIjLC!H*k|0=G!UL!59@6$r1 zfsCnWc7*Zf=asyiJmKP=LoRFNlwV$$|1g|?JHh0Ha35R6R+O)r5xRYnw5+ZH?mHfn zHG~A|xPQgOWMO6*WJfi}W_! zUC-%|Y=3_o>>X*6(KIuxwhr?^#v#H`LowR;=6o@1q(7KRDG+BeG@I0XXEfzV>fkb4 zw#IOZc)|$E>RV&A79sE@FbGEabf~zHV?c{L_Q>Hy_xN>z7Xgg6`1ou2{)c4nV4ECe zOax5Sp6-*A6C3YYj7$H}^P`6)`JQ)Ba6*kuj&BoBUiEfIaTqrJ{QSBXPgfqj*Y+<{ z?1Q-vPg)N1HYP4n+H>^#jJ9(^>$fLKX`49{=~5v?tU}0~%$BCZ(QF8*haZJ(!_O*! zdS6{i@v_f_sMX}pyNtsx_)+kf3FnsY;j29BFP?d8D=Z1NwybGBR{Y2ue<<_D$k%gY zYIOQIil)&d{KPymn~wk0>ykJ+)DQ6a|2W!DESj;1Zpui^mU5(srFdoF4GY^v_jMvY3#2 zNa)8W$Fp>(9$x~$TezQ!{GWO2y75lEKJCaJez)KBC_7#ZyS}@-6m`+Z$nfTeI9}p# z+*a3LL`hp@7t_%}M79fUmzC+KZ-4b>pFH?#zd4YGYWo3Teb*0nXSnx1FbP|toRGl& z4BT1b1nL;_(O0}Ok2R%eGG6vz z>;1He@8zq;T@6jLha2R;1R84UF8WBHFQP_!_LTq- zRyqd2I7(a{O$C?Vb5tu)fzNQZPu!_X>?gs32mbTf2^(%m2pL+AIf-+j*6 z-*?#W{`0%8`2!}Ox#xM-z1I50!q@PWDa%-R#xnNtQe3LmCkcUKhRdzA_2akk>PB#u z?;V<7pyXr;I3dP3(?>Wo;>r<*68 zR*ScRIYvV@r(qf)LpCaKe~;lcgmjkRX_4)`APaKAS2nX2v-y^ zp-hrma@q53(-p~2vqmK^Py$Z(pq_U}CL1KG6ctI`)T}Kd8~U-GuP2v+v#fG0Y6!Kh zOmbzSjUpEBQ)1J`@e&dT^+}4$>@IZ&GMZXJ;;x&_RX@9kr|vz-=V4lvZ^x$wM|1UD z@whChTqVJl!#SRKhg=NhH{`aQuIw%+Z|3~ld1)=Ba@{2mk0D!~%V~cU5v9cb?&fZb zK*g|EGDWQXmWU~FG@W`d*LWQf6{K*ZSo(w|a1uqTNWC$>K(7-&QC53w0J z5?ZA&cpY`!#tQKEJ~)(W8{bYOCWq&Jac|JZV=4NxEn&>?6thLM!R=RA3?*J1Qt@h8 zbM_b_K4X=ZggO02+`7O(|JVBUJ#9%W>@#JVk0nRiAGLz4cj|u?CuY*C%Gsz>Q1QHH z#wvsjx|(i~GFr80kL`VeywGnqZ80(%V^}D*?7cv@{b7hl_x zA+xHYz}v1CPYku#&fcpT`7DPmfgEoiF+>OtC{7V0D)Fu^@>STnr5r?rMHMC^V#S(6 zy=RP1$>S{8aj4{?F;51$;!wc*zgLQc0?lUplB$M(I(tzL*4a_)?x60rLASmoSQ*ls zkb`*6jm-Yp@d`XIJ#NRZj=jPKyYRk{?|)ZAB$xr12)z#n7}jcDY|-9NY!q zL8O8d<+OhjJukvhc2UJ?w#IxW_dXb+>%Edu6o21UNW`u$eo1x>bMm*~5hg~kw7Q@# zm%LW+=AB}Uj@iAR7Pp`J(0S?pC5!TnLw#U{5aV#}rfl7%3n~+1P@muJMhX6liCNN% zFZ8OGuU}Rrc;X2eF|+_c9Tqk6w1n(uS@6Jy!pcw{_gSs zE}paJ2=@`$<&1Eub58r14W>Vt2gF%1Q~!4V1x||AMVi(QRF1#hMET9&wslA6q+-s52&HhiT zBjyPwo+<}qPrL6B?pinl=KtW?ApG4?OxdEOWaF6L=jDbO)J%J_H3;8Udwro8TP~fr z);bk5C7I>uuCAQNv7Mb^uXZz-o$PrO^!oy^FF$5VLkf=|1UNUm_;t(0h6pPUV*5^>nh}y}6-+DWgN;<`!1Qs$_%(e!!Z4_R_67n?so|hm($`lA z7Z+DWTRWnthzS6bA80FLy1JxEtVXfPM*bw58LpUVJob=ZztjNz#tLv36yPMx`N_+J ze*OOKJfy^cAm@L3sx|2VkcvxgI`EXG08na<*eKwZ8+dTNBjoVVsmQR2)GEu5 zXQI?7&avf|kb!~0LD9?0%Rv$F5xaq@7VC2JigX47>IGESje#F2r2xYLKueKLO-=PL ziHv-gr9k_P0s^G-#U;6e2j?xuPz2x0_2d$JQ&Z|#p<_96mmVqt38ZsAfGGv=lj`pD zuf>rm)-w730DSA#rqAzw$pdzjQDy*5t1coY#?3Ze&=K5xapUUL{PpYe$!7Px8PB`R z&3XTol@;>tj0|#fbMvJ}*pMW^>wno*G?ne}Xc}^cig2dt>_4N*%06=5T5<_q8jfkD zi#~^BJ(~$Z+08{H<*|zZ=%_hb2MbD4&yXOPO?#=~yIc@l00e&4e1GS2zPtEc=`#dC z%1-uXiTtikpaApU8KAkC0hFJT-f&9)h<=s^bb|)>&OzSAXde5S_XtlqpL)}xlkuSA zODORzBcAL|$Td8eDB?p)wN`PSDyvOCQ2@!kUrFXu0(x!TTy+>?A% zArc6aTlUAF7#u|BXmo_nrp!D67KDmEdCz2(=aFeNh2R8to}5H3{n38Ifj~~!FsJ2! z*kW#+I|A@>!<`8jfS&S~2lq?lzGb9(IScORSUZ7LP_3A6odXmj9^~ z?ViF1s2JucQ5;I!0y+>p3f&q919k*S=~1*`Fy2Xiwp@Hh#_l%Bmq;*Lvv3nHrs$0m zkZz-%6!5J^bFz6IoAc5NmSwg*xn|dTDyibOJ=W_*{YF9CUmbWKk0oWC9AJ?N%*tkb zw4AU`>bIuKX>=>CmksC8n)q1ix2UyhS|S-fH@p-7Iw&89>@BB|!UhkpM7sd$dpPH? z0GgJOUsZY)psC14X}5x~;cK-i?jHKAiA_fIm|#FYuXs}aeb;zH;ee<7vh zyL3}{EFr)=bGRKR20$_=uYPD0CNbtF_x2?+7S83_X7fH09XLF>}CG3 z_Zw?rXY;iZ31bqsF^GyTD)#%3HW5#h=mfLSq8`He1450G49DgyVEFRQ?sNr??Q-UZ zPQ*9HlG+tz)2E&2Ry=gwX0^1C=K%T8jVh0iR#JD=#nqKp2d|0$C0Tz^Kf_lQ^X!z? z^4ho$E{>=WPJB$_<48klBfZ*}VJTm{;?N*O_%xP78Grif2!}%e>zUf+FtWp0$Cf&8 z$V(`!A|B?#vhUalA$X8*p>=i&R9gt|1fBPt`97^V`tp%+Ed8-Puhmdm3{2`Xo~)jZ z^H-ulfOwO`?`k@7Q*q8>%=b-_=|uf>d)&DLN4EA*9huU%%=Yy`1ZBX@Xe1IorvJq< zve4<6`q1iX!2LBPAYeK;mQZ2302n*v?t8j)JZu>RO{6JCD*&0mZ3PZyIfA!(HtXo& z$0rWZsH^IF_hN$J+D%8%LhKouo$eS*8-IBPI4y7|C9f9OEVvA#|B*^qbL!89+LlM6 zi4-MeMQMfktaw{zv=8r;o~WdX_|+F$rLuZFU3bK1;Uk2Z#ACR6k7#eqUNiOwcY_?- z1s+YoS{G!2+-KpOdp!29h;v7UAoM1W=!q~6y}s{pFCEJpyv`?-?vU`a6ZdSm!-3z* ztDb0dZUC2{o&YwIg@6$A1+Lpvloo1PON&Z6Pz5E&MVFKg49q0R;MJCLdSbT`vqz85 z_LjFva;hcZ{*Hy|(*dsO>S$rm&UZf8kl?|yP8XLV?wr8u<4lrf;nIOXXnkMTr8Pc% ztz+6bqBEAbqdr>9>KljEZV(|?Q2n;%WBI2s<+JQ~_3Kt59X^gL?39f3 zOBQ-56{l=IF*8rPjZb_txQ3#}^6h_$%e>M^oa07l=;Y8K?X(ixVaUB0qb*o~ffF zK2pk|AUiwGpFe;07bfio6K$7$ll5Id4o#<;`T@764lxT2A!bg#wDZkJU1Xn^posI8 zd@xKCCG@Z!)avn#QD~X2+s?RaCo~g}JQ<;9BuDZyFNjQKz{C(U>DsG~Hc5Rf+v+v2 z9|BvJ2D{>63PEW7U#H#wlet4Lxil_?rybs3XcD|S-3ecf z6&z~XQe1|~9=0xQ#$`{5;{<1tS0*v*eF3&LPe6qdpP0Db=Q$#JcE0!-83~Oq>tML> zT;D0Ce=)_D+EWqg)@ap48GOmYC)3GChzT)$RqdtE);JD`S;7JN5f&hB8*&Yj@BgDJ zkr2Eh=k8t&duC!7bIa7r$D9I^v;>GIwk)D3(qPgiQDVb>&YZknGRMZ{Hv96-OnM@1 zjzeGYuz*OZ=c$|6UIL&E3! zT#gHKAb5NnH8nMTN^d`OVm7#zo8u@vD$gb5z#Qc)k&?_d#E)S<&^7{1?_54?ePxqY zG}3RhA5^$dsyIkHQ?W#Euqpjca9A34IX1o8F8?f#0_?w<=8fvRbcgz0(2L!6KVkdj z%B~o)&52(C_3}ZyoSg@aMU^hYU`HUEMjr<*d`E#P{9e zUx^|2J9akuUifHKT6q2J`L}t3z@-VF18@~5&J;7sbJ=i{{S-p-U2fdt78K> zg06v0_{AZ%uAK8M^97PsTLmDL*B&a~I0BL(8>Q#jvIydc?iV!gOV zrn+VeGz#?%(ZY1T9A#&ILBKj8@sG?&p~qEuXH2rgh=_8135+5f7h4_1D+#m^`B7a+ zA7CcB(Pn&+U$9Mr8)Zm@F1X%Axy3B8{H!)QlcqnJ)dTixHK+}vdt;b9Y+!|t%R89O z1F5vR*_XjM|7FP?gl7m1nXM&&x1fv7WPky8?q64%q11ci zU9S5-PtNTOQ4>2c0A&7EA6NDssOG1PLVH0F!s2+p&^5r!1vh$?hy*HLORAqQo6yN=_oM5j% zUay0AUQ^pal}F{18)}vRi;41{h2yAsD+ND1UZea|hDvzY9>LiBAep_H)C}VQX`=3= zscpO4rRoLz+}5EO*+fOG*2HzQ?~p}ZF;(A;i$PIopT6EKAmQTJ7dE$R+%r}b} zC!>)HHP*QLGo_}O6^OHIAbjPDa}K}y4sZNtSN~aIk8UV+l^;31#HXo0_aAyEnnz~n zWV2<0iSczvI%6w|UsqHIsJ(C52o#Ccy;jtRwu~?|h>=Tln zNAvpsxzpH)wyggt283h>^_BD8i{kiCZ!k!9Tn@*3ydt;`AR3q7_;#Xi@B4uH?yzX)as9D5)>M1H-Q`lArUTQiBz;#S+);JK5Lrd!t1< zl0~|8NFTosv=bc+OLh21=7Y&%*LNUJro&g(Dy$e@m(s*{*S<`>1G1VXB(XaSa4}?W zYPg9^r>35!R56kH)ql^uOs!I=U~s15Zlo1gj>A@`cd-vJ!psVOKHJ{Np2`!TNbb5I zp0AfjOUR-dHoCfD+@s~HQX!cZ$$NcVlYhqYzaql_9V3PaOeRf@^M7#TRUECAMXAy- z6GSDIbjiw!zF%_^tjf}X$To{o#*mJ=fXV*EH3x_gi!l|D6)9H(g3L~e>b|V^*0(?c%Z>xU-3<57~*e|@M zqt zW1R@Hx^feinZI_J-o7xo$DAjv+Q(JKG`pU@PC6rWXeX9`pzgTD=X~)4!bfwxe`&q$ zMj;Ncu4I*S)W1wQ>hmSu=Qm$y)yh2iS(VCi zEgePGCMW*`Ug2`YzQKERG)IdM5>hEm*OwU`aL+qJ(gE;!7@ZHkM&HLfG$kR~DB{Z( zTtK$I6z{MXPfYg5#ie#c)ljvpylr6o`E#m)2?@I77@y_ssp^nkp#g|JlPGVNpO=y$ z{6^70XP_V1a!LAd2{z{q7A9@Li%|& zpI$#nnr-8-<+W8KiaR@qYS<@PY2&irC5sJR<#4{v&Si>zQ^|PeeP~VYaY|NuSy|1u zGiK=NJfgLMln|%Zp9&=>P1mK@EM*l-7EyGbc!!`?t4pS>;`Tl zd#+9G+WGl7+Ku3`-qC+2k5lI;yD~vCaLri5Yupi>O8b!whLjDF$&zCFDAI@yd+3@$9v#*_b}Z2kIH}p>r1?w<*Msa`tw_bQhaM zH%*u>{T`mK5>Y9ATNmCXBTh5sVod(4ie!5{LefE8KGfss$AZU)}n)vtJ`j ztS3LyPlZxQHvRqfu$zwPE-2paqdt(@g8zqGojFTJAQKqX}Wo{;9#n7WcX~)JRlY2gN zsLAg|>z|E??6LE{+CDAUuYu6r-qNIW8?NHG*g^fxz__X@tKn@L;|7~*?o=NdgoM^6 z7a3{C05hgYZ7t18#?GAh=e>4Z`hkD%8QWq^CyxTA|MAg0yYt~*(!051Hx;@Z73%*4 z5c_><{hoKyfD^K{MK5&Q=iwlYu`?-_d&NEPCwJNF-@M*vy+8|gY~n(&*E6vH(i%m4 zd(Fl6gXd|S!*xxYSq(TEQGC=Yt!;8?NeD z+ORayd}BS?^FD2<-I#r`tHtO7g~;%hOR1$uwCfHB+ptfeV@h{5f`sr@!MHf+Vrx*; z4ogdSKRS?@-xBqhnVrA*bLgn|&0VJUl`Il&ZpwRo+JEd#Wpvm}2T^^c!Y1>&e?Yk+%``aF1Nge1P>8}uBYyo zTW|GNCl_gn58lHiq)A~#OWg3^B8G-r4;a+;dFdRYz4JV0;k#W~bzUhK9@$vCBLGus z$m5?3q;S}aEcn007a`$7x8qkzpbrf0|Aj&Y z&PTy@qS6^p9R%Yx3=c8LMBburmcrk~MpR-4JC6d?%ca=^!-f%?|H5lN9`(8AI-R|hT`CKnL7wRzQEzX1#jPYG zV~OeFf|gM`QY0^|b+FicVM%Q0ZOQd(_C@{C>I%ZDj1E2HH9GS#12$K27~{St#Q{bm zQt`QMNJj`C8fei`5s}Lo6T@d4xFPASZRBKS9m`J%sTmmshlNy3M=G5SCWl&X4`K9;r&;XQ zW84Hg%tDEOmF~vpG)tY488>RtsXxJH-!08LGJFSg^`iBobW^}YkXScr=MjH)zA^)Z zw8=QP7TAUD(l9urVezh#`mshMfEE~_?7E2O#&1g()8Cx*(~lLTLY_ux(aZ@^W;~I& zRY9lVAskE}t5#93h*m`$5kgYF-Q@g)Pr&CG$q!4?_7c(^i*6fFZj6CS(yo4}sHb6I zfX7KtrSBQ;j5nfNy%TJf%)M1FdtQh#&RM~J(TyV~UlO(`GhAk^KHBq=Ufci=pO$&< zJrDCi4KVpE+n_KkB09E8d7O}nh6ZmK^9m#y&;Z9Q@Z6a8L(zdb4556EjP@Oqb|aD; zhi_7t*;q<%QeG~;?S!RC?=d=KoLIsh=PYIoeP8s2J@wCuigKG%Bqp5%&{t{WU=Y4r zddE`1>$|8@B;{5OBJ$-kYzH6upTg)A{7=sp&M0i$<=k!O4Vq0VG5@hxfpGBG_8SP~ zYVr4ex`dhcz61-IX;|io)CDrc0x9vT?I*hm*H6!nYxNt2n8*fqELQp?SDirlZmnMAag)M7bTQ$gF(bz`QGRBd+rof5`q(z$nM zh=yiY20p;zU?6z<9aLoKDF1VlwTc535tKh$hs2>6pN~Tp>(zg3{4y}{_Y;~4Lt1+6 z^ezMS_yjG}OJ2LAFKTOwtG-XqUPu%K-p!HF~4$gC`cgVsnRJ`^hCL|R?Eu?(>=osotO4zhdnRcR44JV4wk9X z9_=_*?7y`AC1qlXmR6&L4> z^wlPR0>}-sCfrRN6Xry@Qxk0>%|;kIM7FmGr_!2~`4zNEZz6fbUrcwT_W7u!FyPMk zC;&ZVSk;)4RBIHrXmnp|0xOg3=@j@?D3%q`QuM~C=TR4u9AtC&zJ;*lQZR_s@kY3c^FDBCb9z*(B@R2Pzi06yR3?s~ z&1sn=nDHI!W7Qx4l1~u?b+KUaOK;=jXL$<0F?@nV}88VIv#A z*K2ozy!X$E0;~}4{ZSU!rQa7IMl+v+_FDw!wW_dMM2P%fvqXZ!OCCM35+C|IGT*p& z<7L6yh<;9P4S?_p81p7*gLOZ4#a;T)*xKC#_VzSWqyR17W0)>eb;siQ$tx#Qq$k}? zXPxBwcXtx!Xbht+*3VPYm*Z&d8rgiA>+fsPS@joC?aAnt&p2&N-`scBT6q%2{z;0# zWG;C6=I|vx2CL=T`nt(L3cGfV1H~7n4E_GwGf4ngtO3w*xy_58nc%_&j@Ask-4N+C+t=sua9O16yo^t$Bt_t#!j zECFv>>c5BPy2hZ4MX`&wyI5B$qjv|o<+8n#1`qlAqiPoS%g%lucWPe-B_;hp6pb|j z)8fGwsKDP)`DkhqK0pB{Tcb}N`oft3XF6OqV<<*`r-?iLLPB-*r(%nZK~>TYK{G)u(7=B|p`g9M!!XL>g2yCl*Qv&o{?wu)vGS8YOfl6JD^r zpnl)akDnBRneP!}D2CtA{MB~ctwKZypT@J=G!Bi#k*$jE^p9C7n~{Q$-8)k0@;0=(XZ!|K|u&wgoK0}X`?%< zfSO%eO6u7`IA!}eGZWLhJwJZIO98{z3UWX6H8m(F^KY10Scc|r0eE?T+l_qRerqIW z%Wo}#VCU?5MV~ymx~}Sir?H-=#4TG`Qb+Y2Jqt?-ar!tDY_FI0Vmb!r&B2GX#99|s*aaq7fX`=qh+FQX2&zx?W%`HUY4Z?2qI%=B~(mn>hc#Wpg zVECLPYB#>|7v^P#8LF&{DRJ|)88ZuCH6n&h_tK{;oC|YtoXnq%YTQhzwU^S`i+xSZ zSYr?ZzN@WV)m#}2mFT5HtKm$CBtTBP3}EaXIvdj?m2ZDTBrhiyJ)exLt2+bWxEWwE zcd-R%M8-Ts?0WMl906G9U^?Z(x zPESt*s$#@2EXvpyuajDb#~Zy|C#?YSGJ)-y zcP}vV2+Ms{g)c{p-D+Ro_kP~PIG-_gK2Mo4!>J7v0fD##bpZZ)pXIog<3vZYO(eUm zCW>@wmx9m$NDMiiby_iv3j|=}P{6t%24F?hUAA0qknv)kUyi9OF9YO`cSVhCE2NL} zGL6G=;={WzC4U6|m4crUAcS5L{$$8QKP^NSBC}-aefniEs#PTW-in`EEL1$xMmoee z+RLn{r$<(;itTInZ0Yg3@{p8PYan)8mU@4CJ7)##= z!Nn{f*iM#ob`lvNLeS9{AIu-<+jT4fR!U&mlYct5Nbwn^xiSW4rbUG67 zy%RWy1K@h1la<CXoc9nIhelLnOfOo1 z1R?ScAQ*sN;9G1pNiYHlla`@BiL2sRcIz*|6ePp{>JPE*5X83b=D6QrAvQqxiXMQ4 z)+dG-LkS@~bBvu?06m8cU#-?*KUJnYxUHLWSSa`bS1g8VOihYCTKJW%f_fy)x;e)q z{@R9o1mVHjv-hu4DCU@`=fFHPGM@NCE8p`~2Y*7s*(><6rn~2nGYw2e$gs}e3K{3+ zAqX>g4oltwi%0|z8{8wS06WzPWUwb3;5b@<_kvz~#FMziZkod*;9@yExUKWE$AZo> zH&pWC!P5r>#{#OU@&&99=iqOv0M5tNe8BxJ%VsC#2DH_DJ|Q7NNo)yT^x)%%4;jWB zfGrXoecA7i$|8Rdlhl>?J3&ki*~h$Lex_LfOuAx$N2VllX}*c`;ODbxBH zzm|v+cgGUXTDz-b8Q`)KtSgJ&R`xTriQse61L#?tzHbVnx01>jiOouH!p0BroFGHO zVbOabQeKB>z&&Oau*7r^!W7o16@ffEG0$-x)E@>s)>azT_*jI6YK^>`Jqxc2|JXj> z%9}Doz~!n=&baz#HC};eYRZf(?TfO(#M(h6lcN}HBPLGrWtUuJ@=a6$W@USklcfoX zQZYrOz*}|}$1VxjkeCf|8w~>Tg@$|Okr%cy`!ewToqH=60lO%2r&jn2Aci>4yv+-w z%B1IGd-DtM+c{_i*!U|Tz>2X!@@m$hVbF{42Jp&prb?y;w-w3bqoFo=r zNgY5=8S)9D`bI1fbxX|ChsKYNCOsN>$#bzHJm{0pR_ArTKmXYZrN1}QB&U%Vw7J&Y z6-Rl1NEr56x7LYj>qva{qLO@;ml+_QgeD!!|L*-{FLHNbHW_w~pG17sSCc1;((XKp zZ_{To@)*rymkVK?j$Y8-7lY}aIiwFx;CnKekM&7{G%Sd@-#62e!$72iYNHJn+)P~D zG((Nt2*o_}?v0UzFGxE1jr>UOI-XQ2tfar0rMd=g(XRv zRBW+Tagj~a5)k+p_C9Ns{VRc_T6rcyIFSTGFycaRe+v0*DSs`-!5=^IoRH<>ty^(b z%dTh=rR*FP@zM8faQ0`r(+?!J2P#Siz(A`#?qf8lE8ToD`KlBvE$jEBE5*srY;B2z zjR`$Rw@XeQyQ!rsl^f69@@^-3-86%qDDs@T1SxIoTtspO)n zC4KM&h!bA^5)m-|!O7<03`~)EM$%28Hf(a~aN@l=K3Z(S^kKDq4?7F_SOZTFK+gNa z#c|%tBm{v94{GLc@F}csuTEW;zk13IuZ^KAuE*@9Mx>WYV)7$Ccx%GOB3RB<4yir^ z28O`P__G?7J!xqmK#lHqf`P=Gtt+VdjpJ0vd7L%uEbP(Vniky@)0a40lzfY^)|ECaSm&TYuT zq@K0I*y?88v2NkF_?%lmmYzO|Js9+nK1@9qvHMs^S9#p-e|CxhCyfMQMJJQa4VNj) zLw@=sEOjLc_CS;=`5dq+WH@qULB3czY?RETvoeKVZoY)x=>||cp>enQUe;_G8UR0V zE^Rt|XUVXR2O@|*+Uff&BTk5fr4YDUlv0jXqSHmKv;2ks4T8=GtgJiv>zL<%b!D+| z_{x1=ijaK_o&3I;vy9yD^jpVmq`_9e-zF(~^rM!^%^XwFFr#=yn(9=G#moB20zmAN zkr;|Bs=eOGxqpz1NYvbC^D>}zk@D)d?MkXurn5=bP=(Zzg8rb?*u&=Q(0C-{!KQrl zelJ$!%DNHQ=~?7!?$rRGX73@P62fN6g=^I(McL=MEHF4 zQf&9g=klcl`{^dtDN?t)6~pc(`Lzdt&07#?WYK$vnU|%>Ak*(mhnIEoHJE^~au~w# zZ;C9oe~4hp;uO7C?KO^As=o`>4o=Kn^WRpvP$pehT?iPyWjeR;#cbZ>w72a?D1Nf% zb+CC;=rWhh6{d8wNhG(Hl+t zXJ~e+pe02ZX2Xz(i9t6e2sWShEfsN+a_DkGaJw(KW+}Ltg1e*(C(!S+*bhHk(J{i|}s$sjfTMJhrf%8y~yiouw*!D1FV{xJsu{bKi}oNPAIIQ!cP?~z$b!cjKn z$|#8f+z6Ep;J~$b2E%|`H##{Ll|#21k_yW~QZ^4M=GWE+ z%=xjISVBvAD6Qs`BGI^;rV(@y%#0{kC!y**&t++p*g+1F>*kZ6#{2?cXD;)Ui`B-7VvcfF9e){|(jJU3@>Txh zA>i{=uMf5xE}u#Q(=7VJFs{z_cDhzl9{4$iWZa5QAXY5ys&9Vvv5y7zK)(pMd!^0( zEcM;!;&7S{3DXDGg1bl{c9$+zhu-4l$G1p0*nywr-CuQH!3CZz|vNLwqq*VUEX_Z8+sn$0Ck(W05`i{-T6ziQ+MmPC;} zvnh9RTHC!y+to&GgG~~)3~-n%%P^is0RCLHM6(qGX!t@sR&p;K{S{(I0miY>Zpp5? z&ex@>69`ImK?kbQK`7Gw?qr)4Dgo52oP~j*x5@W<>Fxw^nQMQgjZBlnub)GFm@WCYMl@8s^uXzfAjnK`SIGY8yAI@c|bxZlA`Pr;r>(t zcN(AxphbMu@401}wZ_6&UU-f){WTH{tkKys1b57RsTif}&ebM8nxp;m!lPXjG`iEO z7uR?Gl1pRhbc;80aKEjmPS++zYqPaDVnz4}OwJU9gH~@6k48RkbRsQguAku6cDcM| zYv}-tw4(uD;)GU%t%jc;NvRUqm5F3p8MTW=&Ive zyTwecG)FQ|)FNlSz8gM&Cegc%F4*Z^pF8wiG@?Qe5m_KZFSAdf-IqeQm2|Qm_Ylgw>zLJ2M1Kk@;_T6kr(M1pt57Fv!Hu*lX-@z(pAjZ&Ykf~r?QR4Asy zUW+)&QxX*K&QA%J2(!CJQ5id=j&E^Tl-qvsXy3L1w1sGHg*S2WXgl*??+i!>x>zkS zDo-Bm{r&5myLJz00FOz`qpjk9g)k3m04dbPEDHDkf@}Zv{T}{q9ijzfUH?B1G~|8{ zhwHzOQ|xKr&!0ab*L}Qz5n5u@$$wQe|GHuif0rc!q(ZwF^xXe!iv}{|WXFp`S4ySO z79C!j{ZM+0Q4K(uOT)nHZc-y({S)9cE(Cc721&@6m{3g=>xBb7NFh?@-qTGs=DDt3y;W*_zaIXBT6!R}xjW$aTGI8OO~U{FCElalq_&FES|2(t z0ovUTnUEREwzC`&9~K5?Nq5yB4V!AKxe_?rFZZY};>>OPS8i6+)(8`+2KI{?zwFX> zbyZ+>2vo>XY8pY>aJ#s%J~?Mvrzh{tEiNmCe|)GoTF-C~!z=MQ9~@A}iuH(qXqHbrbQ|K_VvZm!bclh5<|-j$V^e@9wpBO! zMLG&1lHLFk;VPNyDh<|Z-oiOnCO+S^f4y^RUwwY1eBkOc*;MBbo#UCb^k8OPKi9jQ z^c%bae$e#bl{V9JKVm|#gW)Uo{A5D2`|9$=;q?}m;ATblN&B}3|L?g9uyFYU1EfZl zOk7?m@J*}{nsid`pyOa2dJ*GBen)<8>?i&MkC!mkz{t=lSv2+5ErT@pXZ!qk&-DR6 zh~`?>G^5`eGmiFN8Y`W#hSw7%7FLzH&s4NkmfGU?P^b!C8VZ`tlYeYcXRu__NsJl} zXX)xT(=?khzxEhMy{=J?6mb=C`Pm=*s0Qg#i9b1vsp)-y#pg5+tOKJZufi}*#U88I>j zD}%5cM8-Y!5IS;0EPFj}r`6-QonOCjKM@d_{wmVA*k65{KTd`5J%O(@ju?+KiMhY~ zg;uq=w|9YA?Yn8x#=G8kV1b=yPn7gS1%vZwI64SkO*wTG`qR|N zK}e!_vVY%UsH$j({BYHx7Um|rJc$6|rcXJzDZy`ldv4q!6G<#=&SS%^C?(!2Gn`sr zrTXI`R@%L4jUq%NWO6e39wf|px6PJyx=ZPhW$ZX!Kmg2{2-zMloIR*dQKSQT&y<-2 z!y5A~BRhj*RuFfzMSuZ31D^#S^nA|9Yd08b@;^NcZlQIHl82tN-UooqemCV2(Ai!F z2BSEcb^-SH<-la7lkM?-uWxocwRQ*HXF|8IMlTo}FzcvhxRwv^KIWTag)r;}&+jIU zXdo+2meLO&m@HEY+do>Lt)N}>&?;X#m|VuwT=_C-X*xv7Z#~M6iIt&`$Ng&s+ktXl zJ3S#hp@L@#dY-%tJ!`((?m9Xu=~Cy?Ebb<*hho%f2B-;e(4V;A-DsYG{IAYRcEaF>!{O!z@FTzPrA=o$%!9lD6;E$;wCzS~Qy zvQyIIa~#sCjl(fP)A5IIwO2!vhqqX0BhB{In#%f+DCCT|)mV{tAKiVY;y;H|4uM0} zhwfRod1A`j9i9!{U35V>=KOZ}t+iW{b~qiO9E~DNN*&zj&L#yCB>NG_ulY?b&cew? zuOn^^dKg5-lVYy#*hN>4cSLL8`$sNYuP!G2lBqeU2Y7$fvKUqqvff=_7bLII3~ZAu zywc=`_wFbfVCaCKCtY26JtQ6I_v^Gi+^C~(*LSRla9s)`b4zt#DKGr>Wa3KS?66qz ziEcjOxrje0CmOe+KJIG4Wx<<&Y|9xc#(Cv8UKKXuFDvb5C%BK1`vUflHgQ>P*xvGW zMgurawoZj*x^M>25jGw&ZA^ShO4{(J z^U~neo+IXW`TAOW0T?<)x9ytIXGk^|wHPP*42xy8iAbha(1n=OQb9Tk=w8Cz-<`Jr zn{wdVn)zn$O7{`h#V-Fv-CQD7MV~b4ew>fsZdv=wM$reMZU!(_BD-u95XQdWqO4r# zvt9M|5Ab&w&OJ`k9==O2>YmW?8t7XsH%jzjSftr$8f6T=H#EK|2UM|uj0UA<-aWnW z%oVx9f`r+#xcPI|N3G_b5U?oFz03DGJGDDF@YCbna&NpWZ@#r>xDw$ZkV9!t_u%eM znBRUT{9-9*HGHs_6VJCqa{ zL_O!fs-rZpMA^+>_q1n_@g23;B#ZsXGhrNUO>|WFPC^m-mh2G50epY-w(XqoUmZNxSm& z;`+F$(=jM~wfx%0q_-A*>{Uc-277AlSyF7idPA6PdriHY%}O7cfVTCIQx+nmc7@{1 zN^tvP1Fr1p_VyW+pJP%_=q;K&uH$O8&FBa|9$KeZU`KS$D}+h4O&t|u1#2ChYC_n` z5*_&^a*G`dF(nVt77geXJb6ByqXTrH4hIa9sk)@fG| zo%FP<9qo3mwKO)c=57V3si`yMR!HUm9ooip?LZ{S?r^)$8OR^-qCg&A->5wEUj)X= zKviWaG|Js$-TD~vj1wqkd`gMcN`;VSFRP9gV#qpLVJHr(*Kff6q7@5z>qk@H`OhFt_)7l1*+xRKO_!l;KZe~ijuw^ zS43nd6Lscss5@{AnF@reV;M%o+NWn5e&%X~8H;APUg=Gkt|ncEUj{5*BgHlIpz~(_ z#^BZWBmI&~?YwknUo<~C;BTTCJ@i?3igF`IJ1a5V8qJM>fFSU;VKd%i0o5X5h3FeDLhoiaZeuuXETvqiB2@yDNC6 z(jQxvT#GD}@aN09MvfKg*D7xa`JOqy9??rgOkt#H5*Jz^JE?%J{TXk_qi{(*?=MbX zn&>u?{vN3-Kf0`4^#wal$QPtrOSH&ZpQ zTk*4keNkj@u~r@rW*Oyb!KhWz0jTPsjFZ2ATD<1PVyv@d{RK;n{DsE6%pYT$ChX%L zYP^cI`}_BA=-L*#VyE+;7;%#@lUiyTnt4vi*A#vyqe|~{@{sZJT?4#$L`v;8r=79L zgKfeIcD-Y9s3bXiKzvCGJNaJRT=fYk6gUpys)jEycEDRX)rGI-r*v!hLdCTqDlaGy@CzA+h(wYDdp*v#Wc1oYIg$0W2{?E2 zaH?L8xi`-tRySTI&${!B^-IY{anz+BdZLSrSSiiu>eTD{@v!|(KW~zQ#9)TLzK=eJ zkgq)1SSd-k&+sgk5ff?L|Do-yqoQitwlCe?-Jx`Mr_v&V(v5U?NOy^pbSNO*-Kn%7 zEj5%h3^2eD-{#KeeQxjf{p(vx)|%OCoZ0)@SMBpW&fkHzoE;s?_FON0Wt^W?M&%db zhuoVHf1*IgoWp38-VbhGG4LC#?G~uw-uvjoHVIM7;f8@C_IIdIiWh^@CkHET#XAgt~(x{mni>Ci{3T)l)M$t&FGur=^ifFoSw{v6XEKrQ;;*%D@xT$Xq# z!i%phRA>3oQh8mV4&he>x0k<43e*Zf@9T;WXyn8WR6}QE=+{+g+MHN*LB`x&Sl}zt z;r1G!^$0kI!51TT4~{qA`PQ7wQe-*MHQUh>ecRbpbw4`Ld3ev^P43M-a3@IiG$nKC zEYTSaJEBIElt}+O;g1$x`Npx>Do3{t-P$a&p@yKhczhq2*6Rm5y3{1@Y{M->ttP5q zvfDt<--pNc0dlNl_?(_iqrv8s{#fLkU7ICForeBb{S+;l%DH|()`(r87f829 zK9r$TwDF+hW9Ty^d>|KWR=g{jMT;m0%MB8w^1sZ>I0BEenH7>i6M6xYuvoy3 zjl>Un;YQP_b2n!iFO-5F?N;G6lE5`PSV?yPxQ{rm_X3}Z)Hd+yKWEZDO6>91laQk( z9W)a;1`s-}_1|kh|0<;Ng0X5+KhJqo)zhLPXyA(mF|<>Nn8#n( zn{o!lsahjteBFqA9xS=!{^Lv2UNzQn?;u!x<(=nj1jgV^CT?H9>~n2LLKB*ho#e?1 z3v~Vs0=nLi@DS_3rcpBf4Wos&`V?Ogls#}Xmrr0Mfz3>T`=}YT?vtMhXI@LF-|NkA zXVCes7dGZDKWdgpgJc~8b&*Pz_{wrLmxmjrYI81}^>=i7xEw(z(s#I}c}h^m;#YJ8 zw_|&6$NjU=qMbsvV&MYjwl}tnSHlaO<|2Pg+%MNAq+y$eW9Ko~g?w+6I7aNq<{YPx zN4S9DwbNU{&0E>p+SHYrXYst7Tq~YrfDsB6cC36<8=4`FSHAnY=UkiWwkJ~kY}#tN zn1-@>Q+i{2vp15&otW2muF~^v8B3#53Dr3oHk?P7_@&lw;jve+syfV&_j9%doyEsU zy9P~j&s|VSktkdfMBt0G5!y*>MoWWpnu_`d23msc13Xpp-mX7bIqsjnn#c%5+63e( zPaaCUiSncDyqf#~XhLu{?efl=R9>hysl@#F@yiE!OZcvs_gNRPvxs=m)%Y#sK#}>q zV=mn^Vy?ym?o5}%adhmO`NxF!SRag~4ym(^OXOooH!+ufCU%_&UT_VP2lI>x_If?$ zbQtn+7aZ54v+dZgDLEo0Mo*X5;C0Ko3RGIW;2l20Jd-(Uu;hH2Zi6 zRHs6KnZyqRD$iAzRHo3U-yU-U^0nDu+CISDbZ=|*hP((!1{1&{4-ds}E{VwTqP}<_ z#{@G@&@(Zttw(dXXG`m`w1+XDIZ~Wa=M^D?UDU~Gg8^L(<@}Z(s9i$vu3=%0WbbYA zg_w)FO6qFH?W`q{$xe@Go42zVuQC>Wg+x>ECXW)3{8IK$pf?KlXsGc9ANLlbXU-MC zEUce|U_zs31dP=M|s(Rv@}brL8Mf47^EP7`mXe61`60!TlO2C#fK3p7s6 zCtZtV{n6QCKIR5ivXj9`XXwcr+aSuh86T7?iCLL=oKEpDz4d6=zDyyPE>6YF zR0BeEU9GF_#{pZ2KAE10Zw)B$>BmID%8kllP3zIdMZU$1FDFWa2i%<7@hNFMy4vfNuH*fg>^ zl)lteEFr~Fti-`)Z<;M^LiTm>utfVpGKA4{5_VsWJH%Xz^&Wnqnw5bDHJmrHw@^-q zIWN2rh4eYEqWnK4a#d{b`;}kxZLTg9Y@gTNPKx-!sfuo~pNVh!``%n}Y&}ioqUO$bKR6}*`tm^@jXp|J{PIZ(pKFxytcNU z84Wq%Jk?x8O)sMM2Bz$%QN4(I5_Em@=fUSt2wKi-&te9}81tJ09qsyI2wG7kPP1Aujh$keY$EbcmmFcP4WXLK>)A3TCW>| zmSauytc^Vu$YvU#26Zxp3J2(nx3Hz(le~Jl9EWsyhQ8W;7cX(Q6Y^+`jPJC759M$W)627<5ZrGR;41S)FXNh<<=@YyK>E~1)A?sk8ug&G;K7I=0$YGo$nmF;kGn}5Y;vbYrYA1iMsBuI-$#kS~q z`j(|L{>q_xIz$mFpqny)7;{B`w7-v;8uVoQyoD*>IOp-Xu8`-<+N6mthF3 zz)=>&MV7A`!(F(;}{BISPl z=m#>B6~6I6S-<_(T$+`r+xX9+w3;PY8_Tt?(Q)-M`I06RLf~@ro4U`-1+^@zlDxp&Y(xl8*fws+ciIKdR; zV$-40|EOi27UuLTpGJY|_6rfkD+zQmZgjc;3S%Lj<;H{;kB4|Fvk%1j=GXf^0|c14 zZDh`;ZWb1wpy}&aZH*|>S5kbv@F$Mh%XoF42TqaB(fMSLLI9zSDd2$34nLi;b+#PP z5lfY^jKO;+YDSV19Ket0kF@B}eKW!xr^0w9xvE3P4=?_{OPI7>Cygis z>FT5Vs@h3UW%zj;N5knVcZyGeb~q&8D4mFs`1(r*k&GZ>@0;p&!*Ht9D;}%;ZK2Ed z+P0dAe*ZGaGS7Z*W#ExaC2`!(Si>S1lUeAp^sxUI_s+F9^85{B zDg$f*UMOIq=f{t(&wtuBvt#!9`m}&=k{~voA5LQ%B?c!0u3Ovhu&DgGXU=0|W4lQP zA&ukpO&%LeIxV6Jg>?$Eq8kALsVHn{j*EOnO&vIluCgKENcpd%4Ts+K`7+R%v`(XU zRUno4^CK^wM?YiL@#I=eRYGSn;t{H3YeH*mSlGP^mM&k^jz(pP52=&x(lmz zeVD9X_Xfy?Q^)NyloIYMG-(c9r!^{Bk-3jc_3s0)wZ_A1lBsv zU8;=c_sTuOTKD>>PuL)KwUNX=jSKc@G6B~p=ZOCZwX+6H2V0A@hiZH%fCkjxK>4@_$)#yW9IAw zGJX_^hr}h7@BKXYnFKGKlDfH?4dS{x&teddGp>PE+d9p@qb4bFH>#>5U7PnMtuweXD)>r> zSIo)BT4YY`?tx(HR{EQ20h4wWv-+SbExd2!R#FXyatC^C_+bE25YYwa0^sh04^(0; z73sFDngAEzV?dzTA|#ZJv?zM^kUJpD-m(;A@^*4$Cp38M&FOW^`c8 zhpx~kbJ^^&%^wnyuW}c+AtjdN1y0CW#`0arLE09F$4fHnJnlgg1ds4!NlJB_0b*tQ zWx!9n$ro>s+y(s*aB<((@4|iw$aYCEg*LFa54wYW7q*N=(0UcCJ@bMMXedP7`+!0b zeKDvP778jN{!y|5H-Pr_DB5GvV`lqO9S1`9CIqK5r*}Cy75$#ZaE_T4d+-+4Z-$TpU4UzZ0`xtTzo{2yWryPDLj_e zWIsOE{M5uxg&A#}C;Y+4DQ{E^aGi$ebJ`NikshYf6z*&~1H~IH6B>YnUT22O#!w6IUTK+2o2J)@N{H_+g7i!zA<=L> z{^FMuYJ+w%J!}ZNy&sbd_nXR67<~71&uMDKKN#*}Zwph{!2f_n6cUC)@FvmoZw>h1k7gVNsv%Va)fY{lNGH zGKwGsyy!st6VI&y(}b+#u;Lw^_o)P?6``#yXF-ra_D@tKA;H27ps(O~=@({ABI;{_ z$ekhTiw@2GMR&7!RG__do5W=7HQbCNKZz8dtvF5Aj>{z!8_wO*;l$Yrgs)$}>a)6v z`z=m)FT~|`G1a*l)zK+_4MXNtc^f3}~jxiUprB&8Wp;r9<`j_?FpqLh0>6X2zGw$JFYc zxptIoQ~a~?4v+4R@CIWj#TFP4j=p`2wgCdSjA_%t^Y1iWZXN;&A&*cQdCJPC`*6JZ z997O<=PJO3!bd&9m$u44I>bk9qS|U-T|zIBYZcGL4-H4bV-!5GI9}~OG|I`QxP?L- zj&tdiygeM*QK@dr*p{6$s~qjlnAd!ldrdxi@s6a6EtZdO*B$!`&H|NTa8dC1ub6i^ zwNVD4(t#l{%?5F%Yml!Unb8t>OJuV{Z6r4<9)}swd!VQ&Gff&N0x(6H;uoqp_AyrE zih44_CS4s2H7#bpuaz;VlQAwXLeayQHeqWc>}MNOqls?jj}zWy-j?b+_vVLVvwigr zh0edz{u`Soh~F-nxxBc$3d*?l5he8UCdELoF2KFr2(x}XDMJf8Eh;EUV@5|}? z>FE|EIl>C&?rIMmozZ;jk!BmJItQF1X^oR3N`ikr>WhUxx~CXUy%*RYn|oD@O1dN& zds`|vX~GGnno?rq>8SHErk*s|J0OU3hvTUp3qieKr3Z0rPidOYINMa#Xuk`Mdp78e z#)ALy9?N)Pq>6soB}ErGM6T`&6WD9W2y##}Db7P$%8J6!$~3p6#K^ULVeRwk%h&oM zf&x9rrY7-jR9@)_8{K%D=oZ0(Wd#eSkYA0ubv9`V8Tm)Tj^cf$vEB|R#iQiNCmHNt z`aU?@4Cj2UuQ`@vsH%K<%L7MKQ)Z!+PT~x$-jhEeY_dR8`u-Rxy?U*f`$Xn`Mu6T& zF)_C~+&6In=&7`;y*Cq}z%M<)N}5Q6XCtf8NbBfw(=T$2$nIP~{&&lU0^g5Pcl~@c zX9pU{AoF_Atp(D>H@=-8O$EMJmU!QVnZMaZ{sAfC?D3_-CuIJ7xxRQSR@v>=os+R! zTXGS5y`Z9LvsfH|+def-6{>5pvN->t!ymm8{pu-<$=5Qz9iJ()v$43ofQp?+$^$r> z>d$O#)6Ec-0hd{}?i+^iJiBOk0bea+1bZ)MQu(%w_*DD-AkBEIMx54Fp|e;25Su-K zG89lr?^xEP{60`D@Kk$*M&c#n7#G*U-N52;1l>zH-21!JN{2DvS?&ZmgR^zazrs!_ zUG}0Zn04+%U>Bw}%EyY~DAJ)H)XnU3%_43W=zF`;FD9L$Xarj6+B_>&H?f94Fkc`? z&yfr)?1E3A-y@ZMxZ(rc3?M`Z_&|)M})?$xW*|lMIadZ z+SIRUdG*#&B^GT_n?O5IktLgPThPn-jF1baDp2nshHd z4r_Pv(|3#A=#u)*`N+Zm8%1|DebSRcH0A>7TQ|~TM|Soj*4U$VikVt~RRr48q0$=^ z@1R=L;h#O61v%@_i|V04kruIZP(}cx&iBfq$F2KHM`759B^^|IbF)WNE3GG)@wa=k zqx^)Hbj5Lg88zKbpn+I88KDy&lP0DkF(T2N(%O#LcCZTE_PvW_UAR< zYcePz(FIC&4y0mEu+cDM{9b5H4rz99!S362MWF0%>^%HDd|-WtE7x;VezSg{aU6iL{-te>rF_nEk)cwo}TnLbvc ztzyA7!bJ5pxP<>M8nOk$jg^`X7YG?i?Ix0_6~FBA(O#Nbu3@~_=iDFwCuMppdnwi& zCwKTetQ{*sH&%G>9LgXH9NpDwBMoiDo0mUw>B(oA5Sr&v7&k&qw4_i5?p{PB8dkc& zQHq!HjR{rInH0cMwp=#@Rsh!j&1Mh$k3o2-#b1niay?H`mS?Tz~D08sQzIq zJrEx)XI=<56;pAbB)qW)rky7X#mfRdY5E^(;yExfZ+I?FL~Gr4_0%7vUfJAvWM0pW z`A9ZAVJQ80EYWHxx>(+F+DL`d_))mr%J`>FtfM@smlV1T0KZU-^9*9H(^5mMMUc$? zRZB-!@SCP-qUD#)UXtjksJ<8#97-v zvL%6`h;blcI$vlfArI$gN}#6Uw5%r}JNuLI3`NuKz!$*#qZOv53}K!4@s) zf_HqMb!Fo+UU~J;N2|Y&7(p1Y;}WoS+sXOdS1Q$6R_u^NLgt@etp~U~`5(>o=OzAi zP*;Y!t>H6#@nZ#jJAy8x!P{4*_vAO6UXB?#3{Cc$Il)lx)-rH|kiE@qkT?;%GXuW(=FABIWAUCPMKo_WwOnc$eLs^gXk1f-G6eZ3*g=a;K zGr2mVZ{TKujYYn^pSh{msQo{#l>ez@wF7^oJjP!)_1|8!qCRx&uL5~)fA`^k87;Q& z0DZqoz6N06@h_kF<9JtvG%!>vPF1WZ{_SV~I-Wrf%r{9^)Bos&e|_SQ7geR$$SYce z`G1~g*P(1~Ma3AK06l%7@qFywjhM(W5A^!3zP=sZq3VC4Tt~!mETWe0v(tac;U=cXnq%`TjH!bbxQ0*L<*3x z^=2=Z8?Ua7aC#PxXVs(VrKBS(u{yj(mMa`Uejwq7tE(Hq8NTn0CGA($adZ50L`FuYiTUtHMMsmhad2~U zPt=$XBYrYa)TuE;nVI(mL}fNQG5}x=kcm9-CJ>Da+S-;z2kSo-6$zZ_w|R0!MMX_~ zpCKS5oKcqmm?4ew72)Op9Aed=$v%o&OH0dYv4IUJa3CGvWmTp6ukQZGTO%f%IqU)x zCKc-1KX0M}PBGTt84%g}O@N>aUMm=gBME+}!KS#@e))18$aMCKL6(9E7+qt?g>VJ* z0>qvvDPk8f)dbCIEu2e-G@i)&LZL&u{4?iQDN> zIj&$1!nlLw_EtRdHthj`;o%6lPw*l$>)%YC@sF~B_+TI`l* zQq7F$yghJ86ZaE-FjWZ&fuB`&NC3=Cs($#qoe(ht$O%mka0#lg43iw7=|})2hEdfk zFH2il8G&Dp0g;Tiq7NlH^S|%3=IBr+T)KTME-_BC*V1&D**OdGWHhS*6hVQ@_Sl$D za1cCHW5Z}okmLDrcE0?vSg#gW-TR}X#T^x?qu68?Z4-dTL%!Sg#A*@pSf>{8!4F1N z9;Kff3>i!O^SSc3Npf!f zp~$@!Ww_(FAqLn)bZzHb{b=Dk?7a6U2Wssf%Ir_VC2nAV{Jx~8XA>UTPRR+7`=%8U zA$wqkbR$v41Y-&h)lW_Z3-Ql|H}lHJ5UC-@9#GnxTey6VtuN9NI0SvzAH8mfk$(ORp3p}(5@%>6nk(8(ry(C!$0ASS9c|)w z_5lh+HeGu_`6y0L{zl9cFtnP@0eIW<(X6~k&|omhK&F+eHD2`hk8M~HI5>>CtXXev zpAq}hd;Z89`iU3aw8L5UxD*j<{&54VuF1uzop^2P937$)qX{bqmrq?cE^b(iL3MG1 z9K0?X?_t)F>KWIhmE9|b{2t}iD0G>RFx-V?1pW^A+@Ec%HFRDD*Uc&JX+vxM$uPdf zK8yffsjg@ObQ-yFR6@qn3t9xQ%5g2nSzIo0w02(U3@*~yCm^ccN>x#p# z)i$ik_(@?{#GGATdg`vZ#B1AI$)Q1T?Yue{~_M#sqkP!0ehfLK4s526PdXDKMdP(~(H{MUO1 zz`ko@KL|049`%Ve*IE9d`zo%(N!##oU&hoVHy!L%vYX)E66KJhQdduZ>B+7DS^^i#o{XnXfx#cI@eYKOXgm+`?8AT5 zZf%?HmigZN{CpP?7uoY!E7Wr>y4n2+UQl;b;Uq=5XL3B=bkbEPH=diD5b5|z`~l+o@7;otC9ajbRPb8%OEtCuF~gkbn6noi zLpfPj_;*KJ@qx?pmx~ydd3L4BF;I6>1HTLY7anRdIdSQp#}&~ymb9{<8nY|XHt|ab zE90KQAB|bznPnRT`lN>v*S55AE$bv1HpJhauX8%wJ#XVYeVwa|df2gP>$IpjIY}-*FPzY$-AqvXZvj-_}<>anhhujbJ#1fOD8x>D7#S1pwA}0BcEPfPwG$EVw zIX>GC7Z9Y*`|Qi@62-U_EsGKU^w-<)QOVqS1dL85s?R~&(w3RtA(7~Y$Av2tF;(#n z+iRU7BA+3OM(u7c2WKtCb;mFEMnMI%U!^sHjYkqtdn=Q9oLgj2Ix0fDB+CBv39Iiw zJX)3KPgFeK1-cnb&lVha2cl+JEGMOBJ!~3`SP*1RUtaXHOXi!j*Bwe`MMB$(e&s|Y5q&%n$IL}`xFCC9@9?!Y<3eAe%WB@!(uiIvB)lGA_sD*(weCw3gj`^TD` ze%G$#o~wcEKLunIH1Hp`&ZA0B?cljwl_*mDCp&IVKqgt(xVLhrZc4RwV@;?+qF3rl zy1K{5_ZWiF&YI>?kek~zDis(k#Gx}c=9N|p3MDdd3__XR2vE})ISyTBdZ7I5jbA*7 zkt0dX&CN0L>NAVik#>G*977E|DrwaF*qcXrW*T3V`R7s|!J+@EmEQd}Wxatl*GrBB^ z)U*H{K{_Hb&vZs~dy3Rc)n8D7MHz%(f}!Sgyxgdh8aRj4kH?%@&UW6%YTyx3GYE~1 z3i(}dx4tYGlwxz*jSO!Uj$%|3hhk>?{v^wU7Qu1u-%}i6PKT3wARo@%qv3zw9z8~q z1B4GwKg#Jes#>s94ID)=MT)7Ovu(_*A-VTb2&AXs;a?W_tfv_IK9#`pj&pWl)75wj z7j{dAqyK#W_(#u6lK6r5dV}rz1 z__d8mN0r36xLtSMuEmh|6>PYSXy4DdCnP^hPEvyDFjtg0szj0NBkHr-uD?i&@~!){ z@ww-1env*79uYaYh+bJcxxWediI&#JhpCi;^sY(%73|OVKz*mvL>L^6>W>C}8`pz0 zQz2p4>QT$F4-WO_#B;5&k=Q8vL0=f(1G0evoisz+*|kgWeXI&`E-p$L%p+ib#wOLw zAPNbE`mVQ=c!K&Dxvm$}J4hdwpeOd%hc(@s$|$?i+qaoSM6xF8IPBeb`s5&eDwoE`h+K*FQb0pNNMNSD_EWNazo*gqSk_o?Pf^2~`%!FoARSmZLVwU~95&B{# zL5}wrjE-&*E>}Du8V|wQhUB+I*lk6ySP(L*)5^m zxA!5lwgWC8z!A|Q=OW0iMpx6Fta2*e((qrNeXQ7FJ3oYnNz|o_+i(eQZRXqUkuP0M z;4rg4#@KI9tiX94VD+s-U#SgzdVt~k?*0P*u>HWLPsjP7fz7l^oDJP>%EXfvbj^v5 zlMr&K4^lzrETB2Lw7y}@JR9uACc)Wi6H2QMHb1F>t$ULmxcp;rebw(JuV*wq0pTkf zq3d(oaCJ+pwBsMFrNgp|BRU&&2vh!7Wp{yr}oa*2Bf- zM%#&60g+L4=Jfd-FJxrXC9B|bKePV7jL`~7GvF&F>%7~)%L1HTdw8w7|2nrk}X5J zkZGIWrZn(_vc&j$leTR4>tZdS;2-Jtv}W$VXuIxuti$ET0Mu`hx@)ay5z%!OB@V0P zT1`qI^ahLa3A^p?^eBIy_u!ckS3PCTdY`%1AY8a#RO59s~V7RKNyIeonnfX|5!=ckoNOkYPo{$muRD#tC+5CIG zri~4UoV9eJerZA$nT|mJdn!u~0WG?jU4gQao_DF^A6loSX|+UJqf<6Og`jY{M!!N7 zPBN;BsZz5N6tSyP*u3hvyMsHywX(Z3ahLYZkjw8K+>M9&=dsd62<0h&G&bP!bXbG& zIoFsk@c`!tz8KFnf+)|;__vA{qH?8w z0m_WYCsQPXLP;3-pPZ{rr>3T^MGM#u;@w}vj$_^tw$REZ?Eu-Hx{&E*C(=FeTY%ucHm>;TUKoe0g z)XY@NpCO+$00B+q_@{45ms>E!RuiWej1B6`R`=O?@p7Gxo@L9|$Yp#L%V_kqTa|TC ztHKPXfi$cwP;@0z>qTFPPTS3;R@&*UVjJhW^>OxQMz!Y2_Y9tdEOI*!WyHN%jp#8r z+T$U-BlwFqCkWDmtFfE-0C>U?e zZ7N1%K!)AzycZ?Ae5}nND!npTopC;gJ|wUFMtFKN&FL*{R}0mC8DRS}BHD@rJp1%2 zp;cF>G%Y@R@d7^~+m@2Zl*_&+9vKbIlci5q6WeI$lW>SG@~6ut$@`E?LD$1+LW+e( zc>@C$pkM-~`Bh}fi&k75V~Zct=wY@|rHxb+b{C_X`Ql70pY=S8n9+W|usWACqX+9h zDn*@{_lfjUA8~=Ktp)o2#48$UfHLDJ&rnzEqQU6QSYbOX&0Y z_Wo7Vt^)@gMbfT|%{29lvt^Z|gcn=V$w+~rNl%|-Mchj^?vrJOZIcW5uoP6e7_ZVQ z&O!O-qO%yH7O=<&A2>C2Dq1ehT*aE$Qp!tZ=^>xL(Zf7byvcTP2JI8D;?^~Slo}RutIkc3>HHDJbk{I(_pKKxF*>xNe^K6OK*`=eV zo-JOhrCZbFFZ4B$Q&ejh7eA@&$_?}|?n{#C5+z;Jm@Fx!9=cDkFGRUV0uMFVd! zKQXO{q4yQC&PU;3t1{Vq%{t1_@Y!_Usvu=){ zQ&odpP9psC?#s?1*v{2HXYXAu)tm^@y}vKtbML;q&l=CqEN)0Wu1fYxWkQwaI)*v4 z(0)%T{CX$?x^39s#AzGz4z&c{WU}cV-BFK+?Nmip6+VZ;KdUDvrqSsbIMrm;GjY>< zj{5pl&^E5fj-lq8wGs=?Z3HoXQX&`bpA)dJQpkdZa7ToCQOG_@O!_ysKi5}9STH7j zJVs>?u6)wKspc{l&RemVH-~iw{Mc8?8#wxC8y%?gE^CgI z(_mx_`e{lO8oZK2sJWitM>B5eQrPJ{wF%6enPk@v!HOoZnt)cQQulX2oAG9hrnOV) zD)rIcCnxK|Uc(m_7V21W-?8pr;@V|P-3UGZrN!$XP51!RQ-$?C7y%Di5b+|%#39V1 ztVPuM5;73f>uA0{fPaLFP~wEcPT~2+9OI>K8k79Xbrw2t*gVxlWNG{Jvl$&Ny#s0V zD;He@o>8E95;L{tdurJFTOKqvEOLK#43Tr%Qc(5!pokZ z#1xDQ;&INU21R3dRJxWHVL0pB?6H;;S>EWi98x}Hy(w3cUOxVZRQ#{#YSlUX2bl@0 ziv-O+NqyQCS!6@cDm9qgCCt-Qf^>+qUtU6Y53?4_43E>rC#9-Y$v_hZG0xrSM$plG zDkTxijNMLegQKoadg$ZVq;-fhNQ+xy034?&b>TEeLSKz52wYJ?#Ii)~V;bQ7juR`4 zqs;V`jq&H155bC##z{HDv(URcgMXHd{pW%L^x`QLUZ3a3Vi?Trj4%E>sPvx;^>44; z*?~-_NNQ!-KlxDqwc*=;1l##+sDz@*KK;+TZy5W?RC+9(2MGvji11B*uWXP~`47#& z;!|o?R&=1=T*lk`+Wqx6$G=m$bCN(V6p`@ycj75EYzBk6<4O#dqh{O9TaVll7sgYSXs-D?<+l7EQ4!NAPjds*-41AtfKZi6Ji+2%@nuAQ|Y4zgS zt%2vz!i!6Ceut-ijyY^sdJ=ZMA5Vz+FM|zN23V9p_171YHmiP0dpBuDiyd*ZD&L{} zodW}=L3_8agY2$^CB{~uHm1<-^Vq>lsER)DBtNhZC{*^gw!n7X+HsJE_VQJq@q&}r zzJ`WJ&?|3)ThZIe^&ANbZFQFY(jt-0cV$w~4nyghOFdwO-}hM}_f+}1|^ zP!~Yydn`3QJ>B#Qg5OzgG#LcdtF^%JG|wAaBWXhY6^INFj{IZZYDkrIcIF0v;AEJW zdm}c%;kPO%WEI(V=amC4K8an8NQMD3vrl|m9l(&g-M!oIvrv}3tMpA!yu<+D->@w{ z1rEuV)6ya^@V_#DAmEYpt8`fmg=fExFfTsql2$GGYv$o)f}y;d z66xLkhKzB01xCQf(A)WxTcx5>J_%?&K3!ME;r6A31fGIhxXx`>w^!GK@V9gG&(S#QO$BNbdRk>impQX&8om5Uv?jg?&7?+U|N;*UUDR$c7 z-LKD-xPTCE;m41_-krdlaQCqQlO>%G0EZbGANO$S@9S%702UTjy{3uKu>4zqS!}t2 zMh2LE4+0#)E`PvpL^^K13=kw38|RTx;oz*z4Az^On+wIqJWXQEGwk#q!xZCd8WdRZ z#be5H#a&-tU+TEMBjqo^`t_w9qrv;%DeiEftN+#Wfk_=mEukU+EtjK&-Z3r?6?6y_ zSoZ8C*)o^n7daQJEKzIlNbaX@ag#+)XzC|-b#|?1l$R| zZmIH`C_F;0MbVP;{m9*iY!r2`il_fVr> zZ|L#E&B~ac%4eU8DYNS6AuJAn)tG`fyM<zA_s7BNu*O~fx9E&u~>cJRYdlXe2&Z}W<^)xbJb|Qs-9ZSKXH#k?z@JEQjP@LA(8d?0$dJ(hI>xS5Cq zegrDB)A(#s7U5iit2|*sdoUo~w*a-ayc;UBOoExT{L76~!@IZAs&)*VZhr`E0}p7r zdVLv(#83M0^ea%p{7sB0qkt;)IWG%(1_l}i287;ltQD)zfQbUW$BJJg8caX77VBBf zZ^DVYtLp>56QLc#)m_zttFK3by=t1uCK>>V(X1f=Q!oZP*Tk3pNGz{S?~ zxsM61Q$}gdd!AD1tQ}UH&)F>vV-7qN9J*LoFkbxd zFoZN$i8~32iKYPF3Zmjoh^U14JP$&pB>aB4$Ga@|E)_0>k$8NwM)Maa=TSefjotAj0H za~sF9L|Wn3&~IhcVlp!`!)AaPf(n=oiNerH*55@iptr=M;bEmk`1+Zxx>(Ggl)%2a%T7j%&#kzu+$E=+M8lW{ z*%FDxT&sT7rI0J5THF3YFs^@G*9K4P#cZ?p>!>Zgi@b%c&s9XDP+@_p$i`=T+FG6L z3lTMZvSA4OFFx2cXfKM|F6vbuP&3j_A4K}FZF=7Z2?%f&$Nr$RolLZdF=eriQA&uF z6oG~S{NC3kHR4Nq^0#Glc9-J&L^{z2{gN)7v1p?wt?Ox|HXbXf^S55(19Tos?f0+N zAuY{4M`;pTe4P^xF8tHKF(wvebMHJTdqn^@xslCAJN*E7GY5gBqw;YEuhn5k>g=qxo% zomHp3FPojMsz{w~KP-IM9fs;zESi+O0s712g;Bjd8K4@Qo*UGoGQAooTJ^aKRmX~h z3~!1L5hamk$k4(M9K7#@v-Ii(^1B5Ex+P2x&$)K@=5t-Uu$tb4M1oCBu$N+|y_Ar; zE!=AbY90n$0z&%&tHg7C2S1Y6qG`BTA3vATa^RAnthozf77m0MkF9eXrMJ(`uP1H0 zN6R%)oDykGQ2 z0AyndhtwM-D3_V!Let-BMkn*w`fWCF^e9D2XIxY>K7aJ_Vu0Z}{ue__+wK3y+FM6u zwXN;L($do1E#2J;C@J0DAR*liN_QhEA>G~G%|oYjOZT_f;XY^Yectb%-xv(Wk|$j2 zo^#&wj%!|562ZnE!U=DM2YGGSoA;KGGqKZkH`Fs@R@Ja%gDte|DODD@YK7GHLt*Xn zKXH3|#Ez>vhzQ_lDB{?`gnRul;9g6wI)bZfX%(ZjuV;5~&RLt#b5QUg*ijX==a(|n z=A~%Nu4&i1fhWTRJ<{wG@H^StSMf?MjyoF3LS8C^qeD<*GBHB&Q#+R(@7DyqGr()V zLWk<;>|COC4qp{0vUVq0tSuM~yGw|T1z1B1TBLNfwX_aV-71?4%7+Hq9iGtvfBsNC z$2liHUjQ&_`oqKO-CsTVvh7!^H()yL$TmkiOYC?35Z$2{j?Mx-@#4dF%h~}FEluowEaMqtT|S+6+Zmh3hsdO8;62h>p=yN# zC%r41Bs9!32Xnjb9&4G7N)Cu(((343CmhFLNO~P@9+WZAH#(2wiI)$k-Ue;VDl{fT zqxcxn!kwNX(?;SE`K%!B`kNMRWY!B3Z<}*lJJ%7&Fsg@@nqU$0@}S%GV6FC%02oXG zEudx5lFk9c_+S$L1!mK9P7tR%x5wtVWsI^rbkjnbD<8HyjuYQonZA-u}(G{y15v;&gs2A3d5zB-_X=xA$kBHqt1EV?&`dc2u;*jZ*rHriC; zy6b3UvKZpyyPk6~oqRp$0)KrXn(6piOI_K=K(+-+$Fm$9hHx1-;(kNk(1bEGrwSZP z=Yf%rQy&#cpqTaXVGQGQL`YBm_{zq{%3|n|*FktMQ5}B&JKGD{(S;~J+08?`yqT{* zk!$4$N}IUhbe?P>m8-%#Ev&UZ8+dZ|!4i|}SOi^jYBj$?4s(Ved%7Z1HxyT=E=Gid z>g2&lQ4t<2a&EdfmIP3LL4k}1J1gNGksE>88=kw=8F>SYu~V})Mad0t+e@FoA6Lx8 zlOV*{ts1vo7VY+_Oj)T3SdB3^$_(b$$9YGE^N@;D^T4GiD)lGgSeXuzshB>l@tj+s zS+5LRYCg1?VY?rYP#Pvu1;TRjC9ZoO>C{6<2tF`89_NmZn>~>aC2RPi1y*&Y)g~^5 z7vsKxuXwmVJFVLq0hJqjxDcaH2NpkdSk*Ko>GjSmXgAi3c6DPt2r4Ez+YeVqWU^G? zMxkttD=?bN4cTOkMtrNgh?vIrP_5j(W`^vJQ zpwsi}E{fC_7R*!1K)4MAvjF$F7^}51qGLfKybg5-y>MpGEHv(H<#jyo_bl_WJq}bn zek-*8?YASOPa9bxZ^s1Y{E6(|9YhS%8F9u5p!X6GM107qe{zkcK&M^H|6u*iK${`U zjY{847>DI*v@&tk=X%|4o3WeLnh{*RhXcROAGDAytupj=uu!)_D96PBkh7y;;DEzI z;_nf_AnVw3_wq8DB~6d;u`uEfK-KOXBJ&BbfZ5tu!;9#E?IlGt5Ltwne;lSXKXGG!h0Yr9r4!mJz=jUM zg5D@QVqPxPBAXk>A5>Kz?lcd41&S@plHRRY562^E@AT|Q{WU~au_R+8O-tB0xa(j4 z>ngv*1s^`jY$vnBbmcCi10YKp$Ib&t7T_cdi^6V*b}F%nCOuMUkeAgqc+7P_{afnp zW)f`gmb^ra#hNpiok-|tpd%MDmqR+_C5AKtTc0)rKYu~^Ovb0HQ!)Qj%rLN?>hwV& zH`Oiuaf#_XT9n!C>s-Dd16-B3?#*{pO|QY1yy(cnh01evKkm_n{m2I11gX{S`#h1B zO5^5^YU^#NXFHXSux*R*BYLeP^4+yE*F8=cjO98YJvz5nO6)}WI$#KAmK9kOPbs6T z*9_PicwL(blr(O$yI=kclS%B83K3P*#I8UkD{m_6aB-U2$0+equ=ccm44!2|ch~W! zT4~0vpEI-SLx*Q|cp@t$rKcF_`XZRA6k>nAGmmt|XU|sBxFx!ew){%I%O2;f&lw&h zzyg<@-uTk*@^H7qA!bmqFXV@EcvCj|q z{(Ajgq>I<9b32?F>1Sm)oW}TsgpWuSx8xcEVLF%Ib6MT_F~7F^f4gXtp#bpKJ%M>r z5a~Nm<*pwK(AY>xJ5Da9FfJAOuQWr6zs4zqISAI29k1+-f3HdDaK5GpwjE(%w(RQKJ_VlYQwrQuPdPcP9+ z+K*-Nyy1hOAYom@ntmR*#mhC)qS{Hb+v0TAYo7&`1t1@+U~Eth_t1zEmLh}n7%gb!AUW;aXH9e{k9(_$e z@}YtZM62;fiutSl{ofZ0V#Ke!%ocjAK-5oNp73Y*&KLC>Co3ig7w7bK7tx*N_MZfJ zjuXDDmCC+-+!NuHm4SCax7KZgj4P)Pb5(?5KgeLz5ejD9GLL6 zq5EJt*h|}uZbnuk%u@36un6+2__vbqn&F?p9var0NbWMKrkc2L-xGgv$SZ-x9C?js z*3S&kkNPkv)bFft;&6ZYz>$TdHf}DDxDqN>v8{!DxX<#0K0zO zcj%2H;xxSy>?8A=3U^Rsbn1&1<6*VXNk%XH-n0oYG|J@w*p)|!PMO^R47@H=qixtc&iR#}47#1HoMl2i)&SX#CmVnwXdq7 zdf;l~xm8%VZH=`$9+j6UR7nWySavM=Ns-#~Huu%Gjwe@O+5?QvVM%tcbSHB9N(ieg z??e;7tM01zS84|4@pf8f;pX~p=nH~&Z_AvI(VRAx&BwU;p))TmccKA2G2IUZEqJ~6%dIV?ZuIP74Rw%Pr)G<*MHLa0}AUQZ&kL7M7xbqcKR!(F}B zIO|4 zGwdjX>4s3wxI=GyI^L^&34Pg1AsX>U#67j~$@Z47F?n@WlmCF`!PV~!YjY#Zg0LII z`1m_s(1u`XxS0DDT?pSBH|4$2@=lTclSRq&y6+|NQ#{<`eEh04>j!BH@oL!lMa$)N zKeN2Drb$2>1=}l~>)&p4*AHzJKFDydy&@JESej(xX9vF(r5}0l>Lul-LF(@QhWM3Q zbd(F%+KUMj26?Z0TKD|g%=zX^!Oj>xA;twzJv5^gU_rO*#nROVYo-IYD+}w^`x4$U zG>VzsF!3o`wSPoPtT)tVn_0w%oP`IsocWxkJYQ>nH*+2N214cQ4YJoyM11}q(0s=A z!ijZ!Wc0AxsSb3y>2@*MCzA=10GqtZdeh*s)7RJgA%mI%F^o6oDZ=J~qtp6J|{gXLN%mUP?D3t5Y@jKGjufY*GFRrjCNgHKxXHxRO zLM|MQmKsn)>z)L!L4w*nqE5};S2{PCBitQM`NvpzXkl(ok91q~nny^@XzK==k~XIK zwLp8%w-pUO|efZZo^4OFvQ1W>ksFO!`@(>Ft#YT*x0$Kjo&d)9g0eSLD#Xxr~nnmIJn2~;b0QvHux)j#td3u4L&c(H%=w5LPj&nFTS zI1ArIg}qnp?ust>g2Oh2&>e!y?GZzl*8@*!=pJ%R>1Fa2XEf17EQ^mdd5w1r{r_@P zz+v};d+}|&;zgp5ZV|yl=j2j%r!?Do?S3|tB`w=gwI>5^tm7Nr^P z%u~$A&qnQ8anxVy@aOJ~>d2={Djvt0KJH81Gab5YXSThPii?+B3*^H#MH0aMHbyG1GkUuaWAyyc#=6rEj zNe?*#B+H$eBtGo^NkP4nSLuFvI`{Yyd@O|YjXjxF-!Wk^!Ob6Q1FkKn*trfe9#ysE z{Le=Dz?T|+)W3Z`ZoI^7H_k!I_0^{y8gX5r6{OGM!Ed(1KWaT? zIec@r`4MC~%}5wRg;js-izv0}afAarXH_c-Wvki1*bDEkBdF^Fo=4Z8Yzd&O;^}CRD^+575lO75YnTFv#-P1hka1w6aZ3R2l-`Q;y4=rxBE*(L%!$?qGJ7GWq$pUA8hS&)7{nX)K~%$H6{P!Im{&u4!Y({F{9Xb7fbs-3o{3IsDUXn zJKK@rp};o~A_Zd$(B6M_&v~1{&PA6uF9365Xtf9zK|A??(2s=HK`;Z_pdc zSGdrsVpG`_m0?S?CE6;Cz2+R2=e4vf{oVjQs9XfbVlYIO)wPx8kI8M8$mQayoAK7e z)&9o<*G7)t@FX#M41HG|RE$Vcac6P05v1}Dto42C7WQ&du$b13I&$;`5b;2?UJ+BK zp3sVh%{D~u#_B^O2@l7J8PhPDpF>J=&s&ZkWX zrtL0RKbWLQ)R6gP^RrLX9cWKC__TC}Q%c)Jhu9t`bLbgby3!nrmeX~RvtB&Awsh`H zM6LWe>0cP4da?#)1ktwB9+F+C$s0Wp6sE0oUk9dLAQ&gZ^x29Z`pwok8)%1+YHXaC z2zD0AdIdFku#{EgPZZ_p?p1M1)JCcpZTh@Yu*cSuiKG!^=)$=2L_k-`I(tm2JDXuV zSr=!ud2;X@|Na)5G)o`ws%z23>9)K3pQSmOGe?@VJ)uTl%BbPQ+W9a3k5<{AbqGhD z-%0ppxcK-?!z-xnRHW35rv?t*mWHfS%w}H{+eOCV27Ka;4Cjc_+q`}4trzZYHiVt#bi_ybOgr83`GgnH4Fw9+g6^KkCzM#;}S>XW%-# zXQ?^B!M~$Km;-KD6oxBsC?qFLZLdut0jRB{4|@7go$r`Ao$Z}pPHZHt&Id1H#VTeN zwJa&AX&Jk7kq#SV(YO@`7cR9on2||v>XA(x#LfELo5Kg7o5%Dfkw)X=9mTi1y2`0y zImODlp|ekHbS$~fwG7N)>4j8MsyJ(=ng&hVQbNb7C@x?blfolENRv~uW5L#NDw*<* z_mIB{%BotloWJcn?6zf}Nkr{TC9IaQJh6jX&?7+Z&$cYztSP~+?AAn)4sh0Afm|@r zx36<4pKba0kZMsptfbs3R#FKN`YEPxj(1_^zB~>11lO!QlM>l^+=cZ?g4q$>^8LqS zO^9ZZ8Eg&D(bss-c(pc>^f$Ur6su3R0=DaK04M;GnVDJRzH*h-UjV7uT}WzE`-9nW z{;tiZ@cG>sbMQZ3^w9|uvU`a;O!z67jYhF32h?c{_ZIAHP^I;^vI-2tAEU zIR(Nt?p$e!CquZ29>G=3NEb>=O^)0PlF2~!MAeD{cFQmP7thc+pCun5OUt57<^gei z{Vcb68zgI1h2;y&(d*%2Tf2`7tR@@f!|bvfE?&cTTZ?B~ zTl1O=rTuyJ+skVBj`2$b-6k5K+Tsn4>Z9BPcBR^a(?*Mj((+ya&Op75?)RjU3NxL6=wA(|_Sm ze`R3*eSr>0z4phq{y+Vywl!-#2S00SJPDs3M%@SQN^Lx^ zh*^mLeUldy#9(2>H?5TBefz@JJWO|PBfLBC1X*a8pydYwLG)p+L<@YWInnVNh~f?1 z1&6Xtad_|FURuiFYR^ zs>)>J3M-V+pLYR_Itt-exq1Au9=kJ#`OyG}i+mW{r;=jFs z^33y&gEDG<95Hvs{d4~O{R2Wi#?)=~*2!zqCJ8kEm~H>@?l5U!;`KQ8&T9YfF$E0I zimoe^jk9d;nDn2s_CLPox683vNbE6FOQ+C7;5Mm+e8i~aw6vCX=7*pa{%f}X{yCpO z$~JE?(MB>$p=nC6)^CBzilCaVrkWs zfxt-c4^st+2>HQtq#!;60his5`Jdq`04jBSe7tTwn0BmKqjp}X2jFee9WZlA7z_ZY zru6_)-dr3&E3v2R3)(3Fxg%p>u&T}I12}#yN8pi>IgRW-eX;=vF4nU`7;dF?f5@GG zarkc+@P|=)hykM*)Ja;B{r+%VJ_iSu0KpLiNYf6nNt@AlI62^1^x#>CPDbV#A7i{M zo5TVIY}$!}UIGBx#uo}6GXua=o;dR*a#}AFRE6(#>>DPR#8YgL(!cNnUeG6JLz?oufnh`6CPkT4rH3s#)~+z=r7 z!t%4}fTdb}*82jK!}P&%9UBSC%gZTKvP+_&>Dj2G6BAXhUJ3r|G%%)ZV`ob~EH>%= z&HSqoa67ly4Q9~Y(G-$QZkLv5{*#KH^tFBr(@JOS%~>;I>bPX0@;MCEJ2X|2O|9#EQzL4~Ce>F9B`zW?rzzK-<2lcfYX4F=-*= zZI$I}Zr(vbk&>7suO-Vh+4kIKb;myIg))?;WFbPl!QK4pO~{8MfLKVDeOqJO^p0O? z!q`koSNc}F^!Sqm(Wckdt-M%+aZQ6q?H+6Cr;6HKnKa~~*R)zp9xv%Ew0dbk<@+yG z(>&FUs#BF@zjC2GV!v;>d2>2XK?51c`%=;oqSG#bPX&OZ{NvFte1tOgg$+yE7=oKEknH7R5G>^Z0!j8?mk+=Bi+p|%R|kO@HzTcOJ2vP zsyBw#sJI5Oi*$Lr1VVMn*6szn&8tk%Xw-fwXkU_OA&``R9IE!bK+_kAw@MI0zRt=+ zy%b%%2;z(W2a=1FmO+=#6=WofdcLHtX{*WmMWxN*t?30cBroF(0wx4qm<@5;`l_2Y z{Gv!N@~z__Uwm?s;N4VgHulw=gljNW!0j|vQw%N)huW)6W9P~iM0VgrbTw5RAVVQj zDQmvm&b8w9eBK&ufPt_dqeEQcaXJcp_H}zxz?u1R@!@%#+7;U%HSGImFub}tE+(C- zH!Yyj0Pp~@ZtpbUCVz=aPq(gMv=2i-tn2COO56ZWI2pgDrOyMw#Rv2Ngk3|-0{`xh zspXEr5DDZN62P{j9u!^?V*0*L64Znl=f~LWW3)GD2QB!>L`j6%WaVeUm1iO$5Jizf zaI?%ZKwYfGRjA!q(Iqt_jxdyeww=qIL&tT`k45(nb-CFQWI^;}+zF|hrKZ!YKuUjd zVwGEcdj}RZE4L#I%CX703SsqoV|>tP{6Z6^&q3(jLbbgX2TKU;I1Mub!Cg1VvruPn zC)aO#p#rYVR1!s>{4)wrF~P5eQ45h|I>})}Xs#mD9MK(0YJ?MXFUK&Y@xg$#48>0j zER)OI9V{sjy`O{P*=cbR2aEs6DE2Aw{K53F5rWq)xZFlTfRsEjyB#3{VYO68(_5J8 zoBrzC2g4V1;P6<*on8WQp6|yfo4v??{!oHB$?h`gD>L*TWM{3x)a?&+ulgKq{V>28 zD2|`Nu!3KsH+kt^G(smRr9j^Z=nd1st{=Gr6$A%z-Y%I90suJVE`)Q?VAZ&fx7)= z*u0KKN&4?ud4AK60~o$EMOhhvYCP0NfQmj&&&i3^ zOW=er7D0IU)J)-(XZyU!{}G!EOA}L6ikQJ6a%-isGX-eNPtzFOLR`VP-^L@pM9) z|CZN&$RKWLNDiF5al%9AkNwHB+CXdSNNr)Pg`VLZmxjVc5erRWKWPbPF$vWs`TqS- z`bFCV&xMwfbYPH@vQBmAOmVA8kzng>ZS12KrgL$d-SyXh25w+*_-0@PSmQzLUv0VO zHzH8+!;`P%^8{^(=g?(>U`e?s{DeUcV_>QIeN3tD9WR=AH+fZ-EZ;9FH+mgDrA5O#EH?4UY zEmnxa!LpMy4V%3$e&i6MC5(6HUbswSFM7*Gc}Poh?N<>wN`>BlklK60rSMZ>9<=s!IWBsaA-%w{Lw7bM$hN6&Xk8#w z4Oz|>^Y%E?_{dU2)t{G2D?s4kHp{3mp6fBuhgwI8RKb7xj>t!BV=xd%Sne>#Rz;Rx z_&qXI`eNed%t&Lslzso6<*wMHK?ed|`KC>UKlfsfbalovZ%X4zT|FG{U)DIfsSNmq zz-H`)mUG&~yV{{50IC-OdWCY-6$bhWD3~aW<*C4=dc3!A|2e~=&_nAtDFbVK-(ZPI zc(d(>y;w`Y(0FZ_L*ARMk$_~%okNf5856K?P2gi7cFw;2Lo%>8q}0Xok`>z}RM9+a5(6+==9-+Ur4ELhhL=21h8O$>gXZD-1k%H)1M4oEg+VLf-|?=;b?f$Q^C_DOa;KTdES{%V}*|$fQgvf*Q7H z2Re^Z!W3CxLl~dq3)k?2fOXW`15hYBtkD!gliF1;y5*`YMg^&xe51tuU8l8C850Tw z45Co&(Nfn{Y2DK$R@9V$RPI=Xq?jDxQKB;xG$NrXI6nLqeVan5r%#UdI5kLV={Y?{ z=RyFVsYNMj)w!swK4#JkR|1CrA0f35B_ps^hc)25;HjRR>W7EA)lgHcCJ@t#0#7dr zii$+1y0afY(lSBE!r1Tw_4Gq3EjC4KEdKeO%g{ZXHf=p#J)>ZSs=@ zY8N8whXbI7Vm&<6iKkO6e0+{ww5;uWh&MocKaNgPf}(Ve7>|cPu}gR`lKExGlG(ML_GdwLR{X2#37`tvXFm1d?z~$*fAZO67f<#_Ad%ZqkbmYc7u8Yk1oLvCL3*petg29G=$?l(57vb1! zz&?=#s30#ebsYDhaVcz4_8bQCe4!|Xb62QIs+Rn?I8jVr-`CKzDPSYeL3CT@P)aG_ zErqk)5r+#=u>b7K%28T7ReFsU?X3gT08jDW^#ry0*(!%TSks*;>4*6qy}2z-dtL@@ z7T(vxin3d8T!O>l;&44O5h8j=Hprc!ipdS~DV7n{kBr)XqpttvQzuoAu@|KOXOU<13KqZI zI}jsBLMFJI^LPb0k0C>AGKgD{v3n@uUKkD$8WQTJlDxs*3f@CCg@N>O+@$hDbB>T> ze*-%E91f2Sz^_gko2=`HXk_&G!e=H9oyaE`L(N9T+`CPIcPaYo#v~N~>e~WHBE#K) z6-%DsSONQx@N*bUxPc}KK@LKsqzz5Zv;mr4tuS2`_V@hS{sZdB^E$(nV<6Om#_whC zePLbcpK^PL90rYAJ!cKJ{Ns_)XcM^m$sD!^lksgJGxMdevf`jhj9eL(4)imTwXj4v zwd}P(6RJ&Y5Ga{Q{Z{YVZcK>i+Y{fj$~MwWy-YvHOnTegO+#V|CoIPjwFX{1bfndfQ{IlICtYg}@bo*aX+o@zpRM2lR zsQTDhdqU7OiyEEoMl)Eht!_YS zA?MJHUO|+b;%B+lC8jg&1IRiG-S6C#_x+M~)ff0~MtJg7`90AEPn5m{@9yK3bH~l> zs=7W!6bzI=q~aVM4cbUz99&cu;X0}IZz<~N3FeWVL&G8)r|C{vTA*zC3z_ zvlyjx#6Yuur%;bwlw@H)ip%IBYkQ%l&`Gi%k*M1(GPbm-WVZt#m1GffTYH|etGw`Gt5a!c<&4H`XcMsH$|cSLG|`0%Q_NRB{Lt# zqm{TWC*Gk>oxngrUmORH2`oh=ZO3P$sFu4H(&WiveRXT%YdqcX^6FVa6C-73AIBf; zrxh6qW_iPd+_FEvg;N>7{-rgd$L~t2GbW;i4qj%Pj9hg;x(J~_gHy}>{*gIPe>#X+jgsH#jHfH{m_EWMpm#Mc=)+&;&l{5(#b*gTI z9j#Bb#-5up^dBN%BdPPX?YCcJ^?L!Wu z?TlHcp`0KR#4TaKCM}V;wss~R>QcW{ZO)SW@rJbz9A`)B8Vfh)3Yx4N}?S+IWG8xWq(+U}RMAG@IaD^pmXVEn5uUCDT|aAmBFbGGq%V)Naj9* zf*cT3>0FRsPRFr1>R{sO+`xgh>dsttrBu)+@|63}^kW}id9#GUsgF+Br_CT^X?4o-kLW$VY>qu^tbtK|039v9I$^Z&Jky}{YR zVbXI?V0Q0Cx29hfstNJ((V^2k7>vJ%zVYDT{q&w9kp$xAF%(SggnQO1H*#X|7OV6q zJTKRi!3X^5Tn6haCu)KgFG8VXECg-H>A}>M(f4nk(D-&-6?qYe=3$sWJXSLpd$JN4 zSU-XvLU)mk7Q@?H6e}4&AbI4al(1$haPBgD;kkd(dYZ>>aP0YzHm@u6M7Xn3ir3(@ z|Jl={@%;@c!VW>036)0#XPy^LJD9}DTTG(kNhopT{Saz33(@_(G4%?nCc-4>cdrv< z_Ut9}CcKt3S)`Q@9UR2#Az8?|GD9Nvx1!m_s*3eUDr9=H`1k~j8kol%SX>?Vy)*sX z$g+$Z3QCQKJDY30zF2jldGpnOtT!?Lc@$>6cPyzJ z9N$Ca(#L?^tMait;?ZfR1uQ%r+xP2!j%v5$eLb)g{jMB=Gr|+Ozj;H>5ECNd4PvP# z#F}cOW6V^s0=tsIzGG5u|C-1*%@wQM2Dt9Pu!VB{O&&$XKJfKq4ibnh9#cZx^^UA= z+RdhyhFET6jzXj*P3J~*+BGOx>8$u`C@8re-EX^{Zw{?b)_-uDP7012W)M&KXeMQu zFW>8+;Gr>GlqOV=KQundFnv&zIuvqd6-GJQj2$Zw4{?I%e=#zon`WmP_p8!8aPBSUyt73JmHm zL`#EJMLwWV=bvQ0t%KNSHIfdaSE~?|CrnPu1O*E<7#gEXGG)J)8Gi%WET(Gns)ms+ zeuMjGWLmL&RP?I<0wk|_IAcsf%4?^A(*{`jhIny&^~*5fa&d5$TbilxP?v`Iz?nQ@ z)TJLDs-zQ*Z;4A6X%*z5+DRfTYtKK1)kteo^hM5a;(n8lKzTyYwh(on6`5;)vz zBn0Wc7fC#~3%NEj7j$Spd*x!PHEzl&=gWbibRr=*%~U9{r(=ynN&HPVw!rHQyfE!k zaus{EEaX7W3YaYgf7{_J*qvJ%oU@-NhfbWCTCTW@N1xt^iIH+})Hv>yi`o~=n~R8u z;CjqRe>Axc{AYggyE8W}11F4tx#dc&d@=1NBwEcL8A*o&*Uq9P9{@7RO`4z78vBHM zFY(zXkyt1RVPac?Lp<`Hkwhx@Lo_=(8>&3<-FtF5lDE>%WK$^qiE>HF??|x-3j`H& z%{~pDcKrO#4=t0W$?@hL&eA7xh%7!`Ih}=9B=DbmC0=A6@nGR5Qb%#@o-2LopKqvk z&q2^uQM|=s1XT+L=SUTZVViRl%s{fEezSJ`LX0lC0QW-=3XSa~+aiogY#{dCCxSya zt2T%8d#0r5*XrQZ+PirIB1R+^EW0KjgaoSz^V+&V*zr@E;whOB2WvmaiscYOD2NTU zRt|nll3?fxZA5;mV1dK8^=W8D3wQRLeeuiljB4$x5X>Twy(RA5xYtf; z)^WAk7kr3;v5Ly-jgELzpbaC`&(ySw4ox+>!+3}}JJrR*Ix8is-lgs;#GS9n^7>eC zA05^2&edmfKZ3}kmhM{idDN%q+i{)BpLlS8&_vRd?^&rw;2ynXswz>S7DfeP-RhOw zA(h{rV7BqalTV8i9%1IcRRM!US3g6tA&HA$D&`qe-E&YG$WkDW3Jf3RM+|$Vxi+YJ z*nb8+x6j93LIRB-IaO6>)k$hRFo+b1U)#SYSw>P(dwaWtloVTOD@_~0uhqNi1jn}P z-E+*gX7>BveE8vZ1V`9e$!KZead2=xhlej4a3AITy0c-mHsi|7q9TO3xw)O?X5ItG zC&k|3Ie(A5Ki;xB_L3>X))Ze2{`amOna{_C8WMhf0^o4=a6ahZ^{*-j7ne`E=PfJ+ z1;v&m@eGzIQVDeYU;e!jo(Yf_gAx3rDt<|;oBP3rN18i*wAU=}lD6m90%79>8#bRF zLU=f){Ht-iDhCVPMLxOO9}QJnA9Gm4o!lqYZSb|>y_w3&Ul#bwf8mpWEub_oFiQTh z(x$)>^x5_ndVBt*S2y+&=nR_rbB6sNI}#A90`~^Zmb^e<;`J}d>0g+KFgc+0Eap5p z`#6rt?EMJPb*>MUn+qMjJButR-k9J3BxHauqBp@61Mp9 z#dQDPJQadGr^QCVA^MCy_)jb0=6Pq&YXt9^n7n5eki0OPxam+6^p0LN0+ia zdPc@xpobbR;BmVO(ANk~CFptlGT&eS8QTl6r>s@P3=&IlsN@#!3u(*!>fNhEeqgbC z(#k0*1pxX5;Nk=h=nVDe@}^v!xUo+kceUJ*l#;aAj~=abF7}z1idOv{i~-(_DnWXU znkvU6mgmMl&~}%TlhXy~A3-c<DWFd_Z*bwpYaJ0# zAM^Wc9)pB5_w*P%-tutjxsL}?(bT4A-IaV6|7Ch1{+F|RYq(92e|iuwaC-GBAwxsM z!(<-ij5auY)we)_PIeH|vqVEI0&2!!Ks~@*d;e=`xAv|Bn_U_J#i}9UjWUP^#mPH=H6nv;UFGa<+~ylWv{A4hoN~B z52PafzDSanh=_=b7b#~|t;!FL7(ZZs--v|J(8h#%JpFKf(rWo(=USre3pW0_9SKc7 z!$!=ORW5^nds^Hw@7@h&zdgVy=f-{e_HAn#x2QI`B>f_@-M6BG<;>IC$i8*qFc7p~ zOs{gK5wGl_-ioxhZ<4~tHYk#33CBk`3th~5a;q*=`+hfQaFgZuBgd1M{5_LNcz0NxL@AT1s?mEeS8029So_S` zh49m&O?XfW0DIIfIpP6wF$TWq=^mFjEjL?urgGl^$Vc7vdPr1swEiCGLg4&*D2erh zyuALrBcS(1{Q5N`epmW1Z!_gdqwt+}*LYd$8`+ zds95mD2kzU!IfD4qq>b}uuJlla#GvbpN*TbL?|}wOyS@nKID?53XB!2lCGf?4{UDg z5qn8Hx(f?f+@Z4m6gXM-uP#o!N3LW3tVTo@Let1aa~L)j2CNg`UeCgeS*hgSS-G3a14Tm0|+sO9|1(j z6r||w)o}y>)mVX~v1&RML&v~K+Ia460x}h*Gp%(86@wXSSU{^hV5piX870#vT6fpa z*nCIeOpQhMyg82kgR1JwG-F!eOzc!)FF27h8T;zbHV9-P2r_fFPd_Bk_|yOaJ$nMA z!w6_*fP_j~ZoQu>olOmE=EkQqIiOs0zVL8HcK}eR3)AZiO!6KJVQeuCo*6!Q$(Mfo zhrbAu7kVNSsY#T8VZbGBbrSWp`5D4{a?#|Dme16V@Wt4aDSJXAjA~^uxX~Ch^X`KT z0^h*X3JmV~>R*sG-0v?~*>LyL^csIJ%DsczA#M<0!_L5g(Q0&Le~<#=seKvKNMVrA zm{cGJt#V?~OGMY5neR%ptR9m9b5kfl1(f1yA1MInpaGmqr~tj3#lUWr6fVM85#+Gp z7_lfIxc*F4Q)%ibA_5L{kQ?VS!~qoQkV^yfTY#^xr16O7S>wzYF%%Ft8#Qz_O#v7M z-o#92su2Mrg)A0^Ys4aXgv&dVS}Kd1KUQ)39W;&b#cVQH4?s#$j%69yKYgu9OvE-l zc~<@NGxpud6~<#9ZwofI0F*o%7<#`|tnW^F zjlO+RqG>DJWVgBXs*AhqAE8kZ_-%@uCsZc}zW-Ar@mo($@NB`s-XzqiYt3*1K_+ES zWSAt~o*%U}Ye%IaW8CxMSf!HPt4AMl7aE^9J{F-`a{ijbO1h(DaMIPa2Ux%VDFf`~ z&bc&Cen~q5fgGsVXfS@OI-^YxRBbG5zdO;5@VL(=c}A{a5GHk+lLF0TKM8hEWQbyg zh`A%wjI?Lrk(`+s@+l%t$WO^}b0F^e@$Q(}z3Zjj5unVN;S)MLt7^GWgQ!tsDQsj! zAtP{Oh~_^g!gDbRu&jLs*zK02LcUE(m@xrG-UyLWBO^W4hnEP(ru86gw_&NIZ(xv1 z=eVxT0PJl>z+M_M2+SR%SP0PiZCt+MW?Qw)Rk!r%|Gz)#I)nuj9Et0H{KCuXF>&?CJ&&{s7BS zgISE}z|0dMJ(cQPyb{LhoRJ%UYj(*=V@#Pw%D%^Az+xV_lV1PhKK-|wK>HhuAyRg_J19T4kd-rg%T_S~s< zn5<@Dh!J|%04Zxfxj(g3(>> zh^}`h?g^;7+rj4;z(CGHvQSDV8_wj7Wds;=@9?1Cj?qCQ0>_y#aTm*G8pc4aj~Ats z2I~OWkqzE+6ksy8ZaNKAOVrqhjTZ8?Qf|CxzI=;?3+k?=lKpCl2~``uuc-d?uva+m zgVtPv5jpN?s#8>(h|f>`L>5HG{z(+{{5LI`(yU9vh~@jG#fH78-rob)VionkJr&YrNK4 z$F2eCeqTTfMic^K#$E*gBNBTX+wS--t0$B+X9~vyyj+P_>Jg2((mp<5!*tw_*BisG z1O`2!1&HLP4-JG80M*-)WYVZ|Tf6IYnz9<=5CV}v6G!&jJb;YZ0@JE)4chM68kc*y z3X;1btI0`O7QU)?!|lmb4scD%tIREbWSeYIK5ZF+z|O*P8v{N>K-(T_H8f-jca`Yz zM{@PI1Z?#){0{N$GW10818(FPRJGUbE>&fGAT$>cVVHu$I`HbIDL6f~`9Rn{)!7;k zP^5i*=Z8I$+YY9cEsBNR3Q&(q%ZaiQdfseer2^_(lrR22=H4=@j%Di_PH=a3LU4C? z4IVVOOM<%u+h~yB?vmgZ+%>odg1fuBe$B}_=bn3S?(_cr#(*(;K=1C})vKyzt(tQ# z+qKV}s}sr!U87l|L+_K>vs2`+zNE+_UjXw}q7w~Uj0iTy6djrlqv*(ZKqEGzZdgbN z96Hz7vKZhW)(7=%))Er4cmxDO<l5fN=$w35NG{pk4E zwsJABG$Cv6UDLzm92Gr1`?%tJly!jW+tC|EW(QdCtpHS6R%I2Z!}%HQU0-+xpSc4L z;EodA0DblZ7_G@DDSM1#w1Gmc6tv&r_?mz=-{eEoG{?U{#c3P$I(Tmt3eVVt0%)7q zK`~y@CfR#_=54D0Zf6D#zVgJyGf)V4-Mf06bYXAuXj*R$CRJ2ppX{ELN^2@4O=b~PGhIf zTt5;)B^JVBlj?D%pme<;U4-3ptW(AVp$|_qC~skJYsd{2H&9ii(Qq z?!sZ&v|7dw9l%k=a7DWGW`tAwutOh4iy_n8rKjp(=BFNLR&#!lUHH*DfI{-ld$7Qk?SEBt(WGK<6SreCF8($+aX=^Z#gvC>s&P$s%OC+weIogqtqw zaf}(4#Z90yRMXh-%PX9Xy&ztA(Ab;06j5S#hBT^V|Hfb3HRcBPCf<8FM_gAwSW?@efPUdEOeUFqt$jks6*n3x}$ZP*~Xl5Xk9SqzNMtly3 zvdLc8?kh6h{<7K=7|-S=ibux0BjShcHnq6@`SF9%HJ+YisO)`b@5li-{|WB_H*tXN z%?-a#t*`HexU;0k?#;YYq&i$txIo!c+`1*3Wv zL+t+MZJW|dNckYb*-M1_Z zvf&Rw!s^wMlfxcOHwT&+LqihgQ#k(8x#CaGC$><$gF-><&N<-HJXl5vgW>lbEXM1x zuQW=Vb#viFuHJ>=v7$twl3rUMYSLYQNQtc>E3x5{CWpzHA)DV-7pkKE`KKKVU@v|uc60z#@$$h z6U}7tT!_GyV2CU3r^RH8&b^~0pVBl6|6du^YTLp)#JZtYM0 zb>`D1MAjLDl{!@5<7LhK;4b$E4psO#p)7s3)4J>8Wi?T*hiNHewPupaHJVVB8FUPt zW~^Rq%`vTQmvUdPFG9798IE0Dn}}|5bLkXmuEfCcG;Le~cZ%Cv=v4dF*OqX>UE?De zK?4i|8k>(uF%{8YNZ}J68|c8FJTA$at_~!=bVCiWskn30-lB~bv^wH_Ti>>w=h&2v z9k@SREA=MoYt9**H|B{;T2!XTUWMtH)H}7?GV+!v9Bh0XJS%&XtB@rtNg*hCcf(=l zc@Z@A+wAxs^Oq2)8Gajq@1h~eCR2KvgijWWHlB9RA`yXu*J+p{35tr3?RyoatRDHw zE1vJ9?O}ZLY|cy+a&<`b%IF(&*YY=5Epk}kh!RSM2-dATO7{-^8qwlW^nP;{k-H6m z$iopa5NTgcON|v+i)J5h4aj%lcJlW3XOn*lnG8@T{1K?=AJAYNrJCSY?TfWyVDKaU zbKOx3DvrYgUC-xTTI1lkkm8FhO1uv4l_hLDUxs4UKrf_FFkY z^XidinN=D8y!Kv1`e@|#@59l*5}?oRbS?VK5X&Dj%y-KHAIuAv3M9*ibKBrxT`uNl z3EIHF#`mW0j`Q9LbKWV)&DrC$?ku1bcy&Z}!e+YMNW98qXs+;@;=BHS2GnY))}pXS z21oS%ih{tHY)F10RaoO1D<9jT6*r?CRKuD{VfF`E98N6(?936nG$4*fVy`ONT)1#i>S*A$L9dsYAbOX0oz#BM~>-2`+wo5cYUjbWKvCX~+1s12;Fz za|7imqU2%aW7xGjR52can)kKWf=jKW?>Hf0wM5xyy&0S}udMU-^W4y5-89U+%iTtA ze|n(HVk@*cHiHGF3dy_Nx?R54NZYA_;U()$$>#n#V06*;Ag`WUp#(L&(fsbIrANcv z9~^1Ut0cv#7dV@brv#7ooha+{^b)bMoLvTXqAEuU4e|gF-zyIbJQ@~q|!lJVXFVL3AiA_-wAcDetR?N zCx2ngzh4n9agZyFs6t9X(cr8u{5GknHYb0AtV}o`={nG}yf}sw6{Tv2QdtF{CSuWF zK=YojfAKfh_IQMS$Gk>Fu)vGUE6w*P-sI4nkC5n`KkHAyM`icXBnc52*qiREYkUf- zU3wyxUWn6-D_&S>Yip5Qsi(B=27ls$#NWgw*>ouXSwY(UdP=%>AJKviB(VUH?!L5~(9&yR~?A1Rys17s#MvpdMf;crI2D$~xh z?7r5w!pauCBJ_`Bw!x)K7>v+0xE>EcUfmy{T<#oc5!LnjmQ>hF4j-~NOLRa!ZshO{ zx}D;MQF2?JJH}{vg6W6EOo=;NzEdATB1LxMN6>X9=FdXu?suc5U;orVLyA~sp`?a5@@cX8fYT|AGJL8RdyGIR5Qh`Ip-adM!Eiv)H zY^^;hl?}Ic*KQ)*J(E1liP5UiFB~in6D4I*Bu2PuWtstYF3tYeSn|^*LkLi;yBp5O zuYS#VzZ=q!>V=va;dEmXzs;$M%BUEgUPc;u831hUtZKK`YPW z^#W}*`?z(=y6Y6vEL*vUL&|2MBeVEj^~(|2TUc{q-+|$sP}nQ&&)fM-VSKra&WD2% zMQRP3f!?&eQ*6s=CT+u z<+L2Zlu7>X8~*WXmq%fr;Ush#`@^f(--%IR?CK*oN=33u;Cd4t81kqMbWPOl+%P*D zyY z-$}h*!cAldHAAyGmVk>v7~S+cNy*5GbO*{-5{XxP6cITSs}&FWjA()4ia7d<6$Mn( zAV~%2aB%VA$KJ1ZUauz&*u@t68C3F;r*!yzPN}pV_d7EDS<5??+bT=`vm-$AFaw~N z+jKW`rhSgD*bF3_4%nq(D7z>-PqMBx4t>86QM0f_^A+;K*rPf8Aw|lhZ z+^YTqf!Lz>tteiY$$ml@u%H6@VT@YxFW^jbNpWHQMHa}^fPpx{HyK(L_)`oj1iHn7 z3E6Y=Z1(C>flf+m;OJUS`=MUT*MYLcDs2DCxHXK&jJ_|*^GX}AQrh{t+IXJlHj5>FgX?v1j_LmKdJFE;9^IA2F2a# z?j9t5n&{P!ebaTjq>kjau5zu|Od}KiaUB&@)pA1hbIrY`id_ooy9?cLj6y?8kgU$v zP?Kwx4bLhF*6c_<23$hTM106gQWShc$?hgTkGlmq^MrJ$T8M#H0;cZ=HWplKwR?2k)X3?%K|w zY#5s7l9yt3)j%LieLN8?{QFR)dif|>nJ)dnLWk#M%BTkfu$vbNq^X+t5S{<{)1L%h z7>x2a6c!p9xE&oGD5#{Y)VUc8Lsx%|Z){c|cVu&_JSg@G8Fr@fyWBhWIs>cI zZ_eXa4M-y4sd-0EpZzP8`g(o57eX3L*v(fMqRbWLX0(SE^A{a7g3Cx``ZZIasMr#- zQqs$M9`=mPk43HkKDVp78-34nUwrY$t~c@t7QJR0xrGoW7S<{8mfTZKWzb*GK=8%K zwEBY@AP&eb)_5=2Td9%y$LF5Mp?#faUe4uWDmZU;J5(iB&r1x!#0qG|GziXO#U4e$?b{}KviFz;P3Y5V?GBlPRbn`eWC z+a`1a!{^~iRzD?piLhK%-aZy&c$HA3j%FtEBPs8XTHmR^2Fd?=JiH7aqk;R9AG<7> zatNzr+WCd2c1_hM3z9W;s4yyX?5>Hl$TzHi%s2mPo_}v+B6<&;BSF$1bg#Df4`}gZSzlRApE_vQ9Gzi`_ zDjJ&ajFkWno*U`wM*1w62S9%yzypz_n40_d`?ZIOlJ#r;=FRMa11@Di z%x81PF958e|J4*g@R2|B#(pdq>;3O@fUE!i`qA3P#O}JR|3o@SMM6$a)f2?8k$+x)_)8-Ms3Bl{Es#RfnzPVt{^3bKvIuB zI|7;ea0bZz(XgOE`eox(3~OZM&z2n_77u8r{r!!Y*; zljO_HVt$)K50lmI>{{rNe}3zn($vXQS|gx$VckP#lX7qJS~bx~RjGUSvSx|raCDF7 zqTy%$vC4s(%W}P%+n(y&mxPm~9f=o>x=q8mAJ1)7<#746Yg3;Ts|$*TckW$bAO>|- z^w&rujWGfFFL`D=`~&hbu!)lilt87pqTj?%A>x4u<3a{F4#;hL3k@h(xDb$%oA z(Ovdf^vdCG4GiyfWoRoBo7zB;y4Ujgmfy4f6(smd{{cx_LE(w)=dt#Gz7g{mV3VyU zctXn45CA3b1!TCCkSI8$-Aem(L*ct01kSI%S(s^itua~7R)Dh#Ceg}Wy~+~BqhZq` z>kX`{GiSxoQQHkv$j%3mjq~UqR6~-Hl41zJ>c+;$CYSDcsegiJ&SEgP?V#Vvo;70; zwjbc@)u<)f@%*^D4jmcU4eo61$mV%JsNx9S$+`^2_b~VaM zEmsDkAyhm>G*lFQcx=nP>LG2XNQ2AG%v2I2c|`{!Z&rR!aP6n-o#~$bi+S~rvF9I$ zl2VzA;CL+I5p_(kro~iZ5e8Prr{l__^+vp01b8UNq?DA%%1`Ws%DO4R(VypzkH6#F z?#Zd=RQ&LJ6QkEnwx%aBP+Fq6ekonv0UDyMS>Bsr)Sr&@V)4=WG(JM?XT=*@ZZw28 znEa5we-x401g7o1#^rI@E)7u|(^+N0)@D3QmVC4ByuuN?t10Ncw7r>lj*5R zC*Zk943rnqDBX)I$bOwoDh!Jlef=8$`-Y=I_u!FTJ9iG#z5mMf@b=Vb?YS4(^&RMh`M6%=r;BB%cQENTKI^Whu>bA8gXrD3LL=*D6u-|XF6M76D3u%F z<{#KA5Mj(GPaz>pOM_Dnw}_tpGm*6dGGK&^ z%XLOjNj$}Q$K{Ii*WT(YK^Y0l3pSHL#U9|h8l09(?>`nEfK?br&WG1co1^2*G($ut z;xINE!_n%&h`ySVXZ4f-*bEVV~x=!BEtxw*pY@4zW&G9@{cKs;Qce;_J`QJ zm*YKNCsUY$V)fFgfg|?5#Z*#`W;(q_O}zavN2QTLyU}4$LBy6K%tCA|O&vq?HYe9_ z8f`MN%Y%(ogrYE7y>h_Aelor2P7daaCFOAAH5UTW#9Zuj4L`hc`dw`m8If747aA6W zZlH_a6B@M+&vHGsuCS5_+kQJ~-FgQr3iZn{RSS+~LA^mSV`H3_r}qUDWB8cS?JFm) zvO-2{xh1YooljE7UGK3%7LsC+#Uyj`r;RF@Y4~-~N8gp(atqwA%Q)_i;+dt{MZ|$)RC@P3aL&^v z;V}u2`C38}#jBRm>lcx`}c-|mins?W>Hh$j+-)Nd2d_{7HT zCZKL@a&Mzk$ZcDb=%>_9YWScNW-htf8HddRNRo2iTK?AH&=7%;N?Gqsie_zacTMUm zv4CEy8O$W+jq55iP@BF)sj=z&#kJ@*r+)rTy$0KpBtY$3okmv>MedtSJWhpRmsp}# z{@9f?h$k;tZKizqo9(_o;p!*~*Eu45 zm)`vP1qxXNz@9O`vJ7jkJ#LeDbZUk}Ol*2Q1uGtyHZBYW#*=hUd!X#_P^=U0V!Wqn z?{I-o;&?bxGf(El+q~83am`n~vQ`1fJK0cKBg^gP3-nl2;{~CwEnmSW>`mrqnVBn+ zF?#13n+$di)#~^U{?sTZ?{`#h_$kJwSu5L3cqQKvL_8uiV*S__4kf`+)-M@5V2wq5 zmC?5K0J{xrnGe4?Y#@wII>|IPm0}V-oa(p{Tpr`?UYQB%UTf5NirXh-Y`EH`99MWC zbi;L{qAg<@k2xswBKn^r4loG@k}8(HL>wA@F`1<*?Mt_~KN6jPs@5yO88ScFK{KmM~cVTjph8}n7aRcqX z;au;NER3Xc0klb+>U63Ol1o6b*;{nhG4Xcxv+T?-VAk9W4x$>WiBtL-j+#?@qmssA z@8m{v!5D&CQ{$|DwAygWdR#lFe!QeMjk4u;YoL<~58)H*6WL0&{=hle?8Vgnx}KYS zL$&Vi1eStT)Y}Vd=m{2i$sRhIr&?sdnxAO{ba(QwU-mksTkv7R4=@0CiMzk=dv!j^ zKeH$9`d_PwB9JfvB_}O25S6BGn2o1% zN4c`Lvy?4*2THsiDR|dnpO(ZS?T5+V;L;5r5$0<%hnHv>oxFE=)J44gg2QStyoa*Y z-%QVya74}%JT##cJX*6&XL+FnuGSbjcAztZeHGdCekort%i)Nlmwusn-C8ztq7GOoa&I2=1l9SWn&a9YF|A zosZKGw#KF=YvEcFN$$mYblQmzaznq_|+X|ZjTIT;=Dj?tHZmZAeEN-~vo zFzTmiLYv@%Q~2CU$S{TRrN+`vLEsUCv_vyY8|%Z?zeu|Oh@(Fb5%n^4!SNaxBpTPH z2A|wHu;_IIbAph?Jha(No30cF^=u)5j1IHmwSKW~B@(~FJ-(u%$71ujGDHAbq3 zCGj*N64(04LaLkV@#9>%sa#-GO`kiKck`}?qm)7vLYQNfb=-$bo4FPp-2{?WV-oq+ zWe8a}4>~loRol(?JtPhvDQ2r>5^jrz991nQt)eUBld}7Xzl?6pUa{ZFe!Vp|jV>(J zIm@=%V;j6awHW7UEI!CgB17gsp*WqiiJU2Lq$y73+0ZO&VR|{G?S>VIxDyn9FsDBo znlo6Xp5~tMc@RuPY+g`C_2BFEW-XYmons%H9U_dq>F-dN5WI4y;l5Z@Z|8#S~xYMmT@siXb@0{(HxMh9|7G&i4evV!B%TI>F% zu+lm81VTE{OltGP^cZD%^ClQ)tdhKCB$ZZ!;F02kPp0=0k-Z9Wqs^1+wP&i=t{sj* zm0+~i>*ZDWkwW@N+8JE$r>)i%*}c&HFam`7OFA)0@uYX(mqDIiQU&hkhWEAkSPyha zD$t5+mi)pROT zk8|{8R$air!U}f>$|+G061r0ueJ`IoUZK3o)XO-8so3!y`Pn-()CnFT-^ux;5xzeB zt(^F$NN!GY2(Z7lJe*P{68Z(A?=q4kfDWnIU=54!_^KfBZiuI9KU%+K3^`9Hmzk!g zA?!j*=FN8rYHSX#wWgS>|gW(@ibVa(gju|b6TXuf&- zgODMa^~4(GHPIhftY1n4zeRT8qfmO%M+OXg&L#A<$hw_NdIX7SFm?tK7IF5QY4B;Y z2>P@udB;v1#ZVWnlhr<%9q}y}6;?2`{J@8W5V6UUI(kJC0=eIOwb*DxNO-qBO-Oj* zuy1sP_z_#BQ2MC3_600?^YMX`u6z?8It&U)q?1>Z%Mt&>{n5ZmbMw*S$J$qlBLW{C zD2F;mvqn-C1<24Y}iRWj$5}1t{Q#8ZMYZLvY;O4PJCC-IbzrILhC zrT_ce{@Y*5nL%mUqstiAqpmHGup{rV?MpC3vE{hCu(%p~haxvrkk*r{MM@$2N~zu^ zwX2x-`w8BKbSF2V#jXd=#;Ms|yi2hzyEgQ4jMWH%=tis!%S?H-#K?PX!Ch`>)pcfs zAKuwE(eBX?ZycYR%+!em?#C+=hP(k6*>(%5`8> z^!oapb!vF9e>tmztSE&bA@wjmDQ^>|H<2t(!84Z0PkDu7mY7J6#DQHr`#rLnzB%qFO?4z#WxK^;-h2Zn=%4mV8<(S?#DxX8J(uLDZ(QEqiM>UVUOS3E_vM{>UmyZZVshfVP`dB`!i}* z!RUG2kaq)cv$c?~=8BGUp%q(yYkgfGkS{)qV!qjL^tTq9j5k|m{MbtWhy>`Fx}F6~ z#l^)mDKV{#MQSA-wM$-q)Pq0U8ey#U)^h=LcBMqPC3>xk^sXQ02j=IsIfsoR_@UBS zZ$7RwIFI7TQ9oUIKS7sjz4=X;7oxwUW-AIpr zoz5;znvS;sa(7phBvJ6w!;v>t@5qx3f5NwzrDqpnaAZCdz;2o;-^bweY)L~o$1vdi za4L82z(7&O`)emago+SE2gv)t+r5@;dLt2gWt|Zl)v%K%2W2g{9~}24 z5z}~_L~bj9{#(&QSuy*YQ7&B{|HA#BUr^_**(xH zL@SZBc!$BJ`fHd=lXii3oY5jjO6@MLNv%kiJ9h@@3IZc~cSbE55ye`yrLu_!XS6)G zUcR1p4K0IuXWM7KXYldRBO0~3&F)+){^;5qU3*@s`gi&nD+itr`d-JWR)CaZ|Bz5e zK`xQ$#eJ&In8Ll7AKR_(utad>XJZ{QY2Iz?<}e=n4$k zySUI`A_5q-f*TRQbUtk#$z(vG<_AN}KUISQl&J=Nro8V@ptnzh6&A-savZjXhYupl zb_04HJ;kzk1CBjHTAdGO8M?*%pb$l#BDBCS>j9%~^)k=9OY_AhZk%N|K(kOP4~PJ@ z(~rg@zOhS&i`Ob5%*%lUw-m@BN@${EVisI=j|mBE`eUgR0lf@h`yd1qSWUX{&?(KZ z$T5L1;!^Ab%3vo{JZ__fdYXypLC)MRhfb1l@O zP&qH$L0KXIXYWjRY?ks19<0-dfNKUI_zq#Wn5-KFlv0^COD!gzEaM}fdyl7kz9>jR z{8Zk%(+gj#I;@74WC2ee9La#I9G9a-DJd!2wSZPTghdCcGByoE`%6KSX&?>hVK}%hoUj2+M5L6^X$GaAUiR< z08!(`H6SPB)I7@+_OqR{s;zfL#Q33XE=vyS$M$rV@id=*$lOEV*Lc`40K2Y{@W5gU zZDh~{s50dhha56+_WY#7Gi~P*taj|Kk5`_ZpHEJDh;1g$pY<(HQ#xDP;&ucS!=Z4v z-Gy~4f%yGkx|LxfE;LM=$s(D(5cev_itR#KH+DUQLA*yz4rBpP3!-VRtHIZRB0cpK z^o`#4ee1(zC!lZf0YXIeLz*M+&unq{dGDu(XH9Kndir53SyfnOv~PQ#P4k6XZ6KgY ztCuAosy+&OrDp%I9wZc6 zdICX(W4ao11<3LIKi+2eo_8V(5`;xCcR{RTST1>-4V4l>ZaO-MeeJ3o5!mzZ>mm9` zw_W-P%p0zky7&>OKd^xQX!)H3P}p<omio%a*RnKN#0hRoGMr!sgB>ILqxb-}p2y4KjjD0^&W!{;q?)f?u-vwv zz3=9hv}Iv6Ntte@$rjmTGH9lH>hKs_?<E29#59>c^mA^T1t@V}9GFATI27m;r@M6c!`T11qU%>v}0XfS_sbN)AcS zn6Hv;p9`pV!cj{!AAw1c7apnMX4T!}6{DsffeC0GnP;1%{3$jTLow-e!DVR$kotB4 zzk9_zP1zuaHM37xqzVEzuiekLSLhh7>q+!zMsQ-&vDo%A5CG|q1vaP22>+GimF-dU zm9sQu!&0`PzZ%)&S^sTSul^DpFr4)Wcj1>YIQ?Em`*-z+4fU9|LB_x9!?s-r>6%UEF1QdrzZMz3dAKLM4 znpS{{W?@2_SJXRAOr^HqS zkGdAF=GW%gzolBI=l5Q3=R_O>$jh%EY%*)rx1WUOW&Z1|Rno*d)6-|+ZRKLtSNH`p zW~YPMpVSmy4;SiY+#&(sTgxRbT#_`H!`xe`6%5?SG|BmcnbAR41GOt=A#kM*Bm7g; z$xRiePY+iv(zrAk9th~pbpWuu;^w{G&LKpnkxa}-$qIndV7`n4%soeWm7*MFGf;gvZx3sCaugU zfrL!J(a$`x@tHO46iG#~jb&Tp0NB-!K_K{eodbV6Y)_-oa=S|l=zqHq;&5B!fWlc0 zCtdEwd+36dRRjin$is|WW@UrNPELQq!hawA#7W#e7c)*+d^_R~*fVc2LCG>C>xb;7w_m_?yf*Fc_p^WafybO)n3ONz zWU;~Luz-t_#?=?>s-k-%)wQeVB{adZG;WR`7QPAw6LVw(ag3z5pvI=J{~R+J8F=gc zX=o(}Dg0!Xp-(^BDdVvm65Q&ok*snJ<#`()B|nS!H6mslt=O>|;!=and@ ziZ*hI!1Ya*NzhZ1;?3HTK{=w%8%XrKY;mL#?)-N-7x03uD&>k%9Ulk2ySLv!-Hq1& zwQ&3!ll#D^K)&R;Q2%@8nHB-Vn7`#hUKGh}V(>^(=eXvtGrfVIgg57V2< zPZRtBEh;wX@-MnelfT(B%}fIOAMHzb_1Wxh~v;Vi)tI)FU64zce@Di#VR2pujN z09%6Du|OKfnNZk!W|8?5q%HuFCOY<^1AeezUOzh;1V_*?)iAW`b1ao0+Zl-Ha9H#R ztcDw3Oc&!Dh@A9)A@S4q*8CvM3*-Kc>zY1fU$v(E& zwrr4>F;UZ2-u3X&4C54;EVqG%#c{w#T5mQ}7;t!v0u)Kjh=(god7jJS{l^ce@HuUw zimvMYt2G<>U&6Ui!zk3|T_xc>tz54~xOCXSeM_RUH06(HW*&YKFw{=DtsxIO2W=jj z>p6^zkv@?{@c9B$RgVVSD2v|#4o>P;80y=%hI+DdH1d}Z$J%LNy`XGZ{eTp9>$|93I-BDNmSDlTWhE15n`t!7_rh|zq7=|jPc0^4}mTQvjEg$ z0?I57*Q?_(8~SSzVd6f%@n~Rl-z%w2EuU*tIrY_TfIHp7c&#inOu2+Ru79%H#o)tw zcVZo?3kbYF`)I@jX)Wop7t3K|vprBC&Xw%zN zLx7P6CVy~Q6Q?$(Y5fR(z%{~?0HMP4G-7m(MoKwD%Ehz-DX`(9X4gL3FhH5b_#>7fSJr_Yw?rk_-9N%VOF0D zz#!}ZOyG|DB+)vaOrRwIzY5I;qlOg;oSqQ}C8fjq@<@+NGTc$Pm-|SDv3UaaVaIw0 zfZ7hAf%l-3+r2Ws!wMXj6g$l8DgpEb!xD|k?lfd#fq7~^*LFTlN81C8#E%Bt10Hk; zjEQDMF=goK#59nrgP0ifQXc#nPYF2-&=gC^kmpqd+}oLc<+dv#W~6Ha?;_OT-+X0A zaUHs0(dRNUV98L>;&RDfRJ@OXuZoI(cd~ETuVCddX4n^o4rVJPT zs^)fvyl6DJMUa2o04$qQ4Of9z_k;sc8d z2v-|ik8~{j{k6>$8|O*vc5aWW1()HDwS(FvXOH~IEU-1eWyn@J@9U>pEZCZ}y&rFX zm*n#G(Uw<*6u%(-HLk9~C??91ztEwg2SBdId&OYkm#mO0k9{uPn01E@d#S!45eK-h zxX^xw?o{Vn%8!mtVvvw=;Iar9!-@FC3hK%u0(W%4i44!6yROrhnb^AnR-Rf25=QxG zu{kZ2GF^Fw*=CWCa&*(l1g!^`&$Ya$pYnlLTBSwtjj#b%9IhjF*YZr2X(maAGz#3> zlINAqz7a!J;xuO*ErR3lJ&;U8LFPO5QQj_3{U*2#7Ceum{6!{#VNgO>H=P%-c5BG- zKD)ta3!)5XZ>K}T0f>M?qlV<$Ed90A9OUrmYcfo3ChcP!&``8j9d7X_x|Irm`RZ*o zUrm-cbR3I{dGqD0ML1#{n#yaYF+3As`Dwr9fzPDx!~49dNUJcKohaZMk43CfTe&Y7 z1vG8MdBs_2uJAD4>rMH5nLk3Uo?WW)uCu{D+o@mUzE;reVG4vo@6jKO12Fw0H4~Rv z3r3hE)0)^eCW zo_|1ixWTKk{yCAJIOI?&*)(eZF*0jZTA|+>xnfrtLIk4eIKJb8R1~NY6iuG^0qn!3 zvXFvBAkbZxx(JS)_<0551c4=#tUq}QH5lf^0t{Jz|;Q-zbW_U%QYt3k<(z| zW@@%AkNm{tWa{n1#pcGM)w?aoJ*7nNjW~6luh2^j)8< z>EM@sRzq)L3}>IG)1FQINVVx;-4eD3AZ<^UT-Xv)kED#s*4zO`8bX{G_oJf77hU*- zo;sJ30#%NghJo!f8r|zN$GXw{^xD0ta1bM%U2*z4Q()IXA-uFuDjLFf32VwpqF=G% zkhOb&1jyEE9z;(TJZr@($?hwd_%XOZheQ^W5Tl-HS`|K~cvxJh7(Hw^Won6{SN3KE zDE2u@A(Ij+xpvJ0C#>55*2c)5qq+2420=fXAS_AzciIk=Eb4=7dy2`Ro-Yl?E0-nA zEaYem@gq7rDnZY`YcDiXmoh&3W$_>qKf6X8e2J`{Zc>zmbl8619%%E#JQiy&7LP-f z9CexkOv&65U4R`6^g*T#KIj30I@a{`8=5PTE~TqnC>&h>jbmd3`AQs)(LX0fLOPED>@pZ_YBM0N8*fl@@L5(x}6`saEZI*wU0>&#?v{EchqD(IquA z*LfbW?tSoYPNh221ZCh=#@2yy`Y3KL;n!8 zB^y`nIjFPz;(qqf#91ulFWHP~?YC9Y2KeE^;_-iPs{8{RN4AA4n(t#ID0fjbNritF zaTaFIMM^P~DM56U>k5k+*q_J1Z^kFTBLQ?OIR z3ZuZ2vlBMg`hp9)Cd99MpUSJ%&oN*1W_aE@ouu}S2s&cAUwOR4qPyBk4`_YzN?*B{ z(F$H5Ta;`yokYCs43`CR-3Z_Kw?-y|iKP1)u9x7mnb24tK|9LTb6yZ#1>jx4}a6dw#7K zQ`lIIWOF^q6#jcORh;2n>cvCNqnkq>;T>+VoVl3PzlOK{RS2rV3osoN4*o8#hl&nzE)A4%G~kmd_aq;%9lGfUtRRcdnK(nbYQQvhB7? z{s!<~JOSpxRUmIAVc|gD_5JB-V9!Q*&*2GLT`9P-z-u0&WAR?Ivtoe3(=Afm?aU0k zEE9I}g7C)c-s!lmp8Kw*MbDdXpUm~}?(E`$eJ_^NMelG=U4@Y=a9I4mYY%7A!0y`y zNEpgOV3}fNo)HyXpZ~-kYcMgQ(~Jh)oNeZ)?#0(_&*zL4i1J+ST)`w?6b5uzoV^uf z?(VA2d<1obn8)hxG`#xqU6S{!WST$Kpmq}cAPeVPs}EULntUvBFWX}+)JWc<;*TN$n^4yxXIiEEa>4A(r~gZY8?e;9-Fpuv3X;G&oIt?YYm^7%tpGEoA<- zBTr!Kp?mpDl=(Pi7qx^v9H+jjbC$BBSQ1*NBX#)>32WX9CrdBKeBprz5!&8CT9Gt= z^TwJyI(ofddtv%lWQyq|uF6I8KW_IeEM_e~AkAsk)ZZ{IJ|`5zfmAKrGs-#vx&Wv& z3P;QDWgjl@`ZI3%CPu}yq@6Lxvvu&(OJ=i~+^N>MMQirxvb2@lL0#v+Rw_WXXABS_ zk!Vh-hVUTElv0MLCt)~Gl^~QiH$o5L&6lAlfu0nDegBjpnr!;Vhx5j|m0g8zM<*Zk z_vhoL6%{}E<5ZS2$q7l=vAMIAeO4`+d_`IIqKLvY`>wsj?pmPa_~Mfr&eagX#8LW( zkWXk63MB!Es8NY$1C|61lj-K#jHpZD8et;J(sY%V1-bfJ_68>Ms62U7l1k@LVMNXI z7{n8fVmV$@=C6|s4Hs=J_Ia?>7behW>LIU}#*IK~cIMG|p*&6&WId;9W=RSTDQgu3 znKJG=vYP)NWp4o#SF^Qk2ZC#G*CY_!U4uq&cMb0D?!iKW26uONXA+#??hNj3^KYI! zc21qI-ukOj)MhF*%yjSFy?U+dzE^CJ``GIrEEZ94PG9BPtr_Umn+Ru}Nf>g2PfFC^ zv#^^_JxrMccNeht+WWt42Xg6}cS6qWeGvrf&;@L!9M3ZpSM55W;2TzU*8W!_jIzQ4 za|VNj-MK&{fDzYJyVBAMdrHO?1~y3iU9krPp>w48K=6C~hX+<-I}LcxlgrdZW;f15 z*A3B@(~ZgNt-BcRhaT{l7bF;8$dSVP{)W@>a0WL(rV;O(cYgXy9&uQtk?)C! z=gDL__hxXtZzBQz=|Jkc4`q=-$O&uL*_XWrif^P+_>sNbUHX#ngnY|=w2>x0cVKwb z_Rz^bpbDWp04LT{96>LzjwfUtf!OEVM=DyLXnJv!2;d2M2`9?;1f{FD=zD%92t*cs zoYH%YM%O#}y188qZdFl7QEh&O!3(wVmvx4FK5XdBIQ2xmt?ly`oPX9 zj=&%kYmNq#Yt4sSr#=7!%1N~nM>uFCYzEhD>5HV4Y+I>_U-q+ zvl)X)mr)Bw?S>9Wxlw)vEK ziid*FM0rBQSi7u*Mk>t2hX^O9K~fk{`Wzhkf>+@X584R-6{C*36Z2fXRaAXi6SlY_ z9=XL3VfgbZcr?D+)W}A-#ip9dOO}~@S;=PL^zg82xZK*DRQ8f=y)88eEpiQl+1^b{ z`)9b>Vg25hF%k&oFatVs_9lzrn^K+%XNtzmy8RK0r&AX4|Ksc@mkZ-uA<`YJnGkM$ zKtwqQ{gGa)r3=>(8V$>ydDOWC_OfmpOU)3;KXwD=yrdOs`SF_1Jpa||7T;ChFGM!G z<)qe_So~8@RG!H)BV*$NUyLzuG=~2 zJ8figt)=m`^-9b79MSW@p>10`4GJk8xX)CpN`5NQ2mq1cTd((Ow(XEfbwJbi-I$<= zq^SF(*#BH_*!mn|w*#B7K6K?H26Em}(IIMo5JUu25!>UrETatszce{Tr{;)rCd)(G zB%R-e;MSIoh(#oGgLd)eK8yPG9*F!!44A_}(Q@GDeE%FCsZ-~^Wn)GhLI@=t{y(y* zz&aP&fu?MV9XHymfD&)StOK|IbMTV~I^vb|us#w@JT(O);}Yx`5)p(>Pm$23+~bS!Vp z?VOQOWymCy%~q1(Ye!Z5$P3$YwnD50Ayo$LF%urlv@YZBYRjCj+D49+E22z9a~ zW>ifBX~5 zADJ9tl2t>84>ad;!oT3-1JB5DHl<;XPI9=JvX2J zEIHLLQZkayN&&sB*|_u4_J{8!_vTD11G=lvF|&uwb}S8J_BjJpAEl4wo6aVuy+!U> z&b}`3f#XG4A96N2BC-}%Qj00$Y?J9J?vo#aKe&Q=C!>8LTNbn&wZW3T%sUIP+#e$p7BOG^663G!c--|*m8hcNV5A{Q;@Dq1XfPG`Da zr@5pSQ_*Ytf^^z3e!f4!KRr9@NSWjeLO>znKr~77e4*pE^9gLddzUVu>CGw;nQQ_= zRcquOFi=9NaA-=(N-j{(1)B}0#%aekMOTI5y7ijQRg)gN+u}buZ~o*y&uGOUBMLI2 zV?F62`c`WE)gh>pGcz_o-60`MygImHCWL_*Zg zj8D3CqQG=NRWvBY?ITNu4;MBrwtinkBfiilQr`VVY=T+wDT%+HlK)A7J-wJw1>6Wh zaMa3D0f=JjLL;c0ubutK?z~{Xty2~NR!@R`BtkbJU=;#2Vhn?|0OI-#9 zberOt;?+-HqS1=Wy*#G=nhgPi=t@v3?#B%1oizC z{uIa%^H`E7(nmN<38S8L?bHKvzB`3`aQP1OV+_kznIKOsjIMNBV@C6N%h{@&%gKO8 z{mPss#4jLfFPLk|3#Po)3vY|5Dr0TxB$4HtJQD{jLo!!aiMnx^z9Bjp2S()X^xLyM zvqa7Sx4a(}jodJ0lK<-*|F2U{C{P3l7IrkERW?fHFs6uUQ?+WjKIeY*S|DWJh8q=} z;*2oNJDFvcxTN2g(%bORHqGp``=Q1(EtJZ5>|{rCoaN${XS>>jTIxG1kz&Z8SOOG7 zWr-*vTI0D7{-t}1y2N=$+e4G4RyYE9cz4{!yV^|A?FEwH?v&mGn|w@{8qg?<9WI1y z0sVEaxa)%`GV4-Z8-WXUV2hG^Bqknq3zUwl13&NK2`M8FM2+7LVg#tBB^({GgVBi4 zKjFNr(zfzE8<5hnl|eI4a{9S)B+cU=1%w%CK+Rgc<*rdZh5bYKcYbw~FKz$pkOzGI zzyDbx`h_+vGS08UJ+JxoE;78QWqCT|B(0F!cv*jGCouOAcO zUwKfiY4)skL^;XV%TNYKXn%IXou}-Zv+JBOZX?X~x*c}*EnI8U}}=Dc5|ogBpZsYMB%!rAdsvBQ^99@4_f7`3m;f$O&% zA_gKTOShA^A3}RHkj=ftuHd$Q(PAE<>PBGm(N1bX7oNx`3b=CwKYF5S)SG`^Sm1v} zO%3#muT^qWxv&JVEbZgPq9b*I|5393d&oM{VUUoKJ5>X(63gWybLb0u1V z^MtOZ@0yugyy!p>KKpiBXkg;IPwK~qZ!{Z%7HvNx({eC_7WitTMZK_4s#}@D*$#T{ zg{p?muoi`qeq|{0f6wd7>Yz1JBe$hz5;=%E2}N_b61(W?zu~Oou$U3-oHWTe-p@k) zk(z>B6Y*AbkGj492K8GO3){=(ExntFaSWpHJZ;Pv8$-29#TI&PORv&d-{90WeRmA8 zN=Q(=iK4|Q3zRscaHA3JDTmq%>=`U93jfzru?YY--wP?2}qAXzl6wo+96Lq%mn6fseF*n@+P<>f{9l_45ONv*Z*H1G_>rq%tXjPtH=HuXd><}9&{uu z&=o#^g4DS%oPh^1gHwWq(f|6Np&f_9z=vO4Wi&_qGnkcojt~grNb~~ZL>09UjxOh( z@vG4v?;$6O7#nK|H0=mH_~rvEXKP3{(hJ3>2Dw6uj#Ji8bX%7tX9W{MbZ1gHd|Q%R8$==gwy;8rwz! zO%rE;zw_#wRVWLKd}tVw8Bf8?ms74l2>`wSCzxs`1xD4m)gzDK&j}EYq*w`Z#Qwpc z6%!DH6~Eq$n3PzdC!k`CN5KK4WWQ6bu9OwdGuq)bkFg@+oxj7px5EhufZ8RQsOr2%q8aPtqbTYEtRV zHi@)wQX!XnwLqJ4k=dk9B2lABNm{Z`uaQcX?b{YnLvuuR!VDN~GclEQPziYZvWqWZM44jzeL9pg$^S%IpxgYX=T zHXCaEC>nx8u3Gw9sjT`R|Euq-r|9um#!WE zA~~?r8-X*GV}g<&liyM>dd@n%Ven%H$94NGhGM3`$Rv~@N-@-o=U5=Lqo(Is~gS`;4 z?g6i6U^PoC*VWOf7>ZB3d&D)_SSiY+Vc#bcSf2TNopttt z2B2;cEiGd6$)e4)uG|$`CLyxDsWJvlElW#!^Xak!7$7Tf57$l$} zhBtkwkYs;Z3FFn6j~Z)ThL^(T_7QtlUNRJ)t(K$M0r%f&yYffv-PNjuywwI!p8lsr znDYY#vL&NQT$#6%|FN^`<-eFwQvi^}jJJQjt~X7&0;`X$K;k^3|HM}xU$sLP5fd7| znknXInT(`005!W*e}l0saqf$naof}P)I6C}d*$s>(So-K0`BLw{w%a8kC*lBnX>7E zGTYAkm7q0%g0UB2iR#4*asfK=Bq{1>0ko}k2@V0l^_5|?KA;_VXETdw2#V~M=2{HQ zj^gc)Q~YrS)RFW70O6^M%GC5^Gt?fcF2IEypg_0uVBM3CuAwA!l9$Bmb?kYbk$w`_xDx zF6XoSmC~x8XQ|9|SSD@!@U89>T>%M>ZwDmamxarJW3!zdU}Lua!1;pA&jLPgHJ*)^ zywwi#uq^`_*Ceu+qkI^YMzWjqHTw}aT6fm1C=Fr%GtB)m9u7RfZ1+C3i~@8UWtNa9 zZeigjH$i7N7EOs&>N`8)2>%&Mn%AXzJ-Qd*M{kX;u6OrVQPG~Y)@&ROat#tX1~A4f zkE_EKK?x(Mt1`gre&oPv;Uj>jEqeca8Hb;Oj)U+7zAB2@%5KgiXxWGDCEU%B)Gr@i z#saNn){BbE?C)^FK*iHNCqLita(~wP$b2jdHj3|%y9M%iXMLm(57hf#lDe)y0h2^3 zQ~5R*8r<0$ZFaix(|Y8IqFhKREO}y;7qqzHuE6ni?Erbc7%5+_BS3810fouQiuiq_lQWswid zF73%73GIU&2PT}o%Q8oSrsHQDnhO!Y%i#{oiNH|@>LTl;eOzO$q+7$VJq`6_?RCry zSa;p7n~v>x?108pj(YeoU>sf5-BI~J(m#Hw;y`eaDz5XVEMvb4rbXDcD-dx8fH^7w zfFR#=MX^x|drh8Pl7cY(I+eu`ZkT%=1*p~b(?DP6oN-KRtp}Ra#6Rg^QYqJf8yl{a z=M6I*lYGT?IzrxS(DSQhbU}=VJ|de2YM(;S4Brvz;c!`zdaHR{`fSX#mb2k*aXjrn z%j9l1aN1tq!9<~Al5pg@WW1o0ls;)tDxQ|D9~PCpC9NpXM+K%V6&^Ds5N| z%mFj*7CsYyF_+fN8Hs=00eAosB{ zZNB{TO;<}vBhzx7*5lIwCc_S2sO_1`?0G{|hAx3OO*O5TS9ImJn+-~0w&`a(Kns`-;Ep-EtPw>Qm^dh07~xI_p6m%4erteR&rj4N zb-QVe_NuU<^_7gkUyYquSWb7}rC9)GQ09HN*SD#cz_U=k?)8Hr1n{87gOtyF$5s#& z(-Su)J$$goAawfB^BPp=)Q@s`wOG*0ebq+b3NE|co)bhRgRql>0QJun4MEG0qvv-ViWBY^mvCt?@u|fh0FMUqC&9hOFYBcquTbTblrf1py z{_%JKXD*(E>DSc`5D){1&t@M1Hm^I)<8Y_sT`&s4T2s!8$gsoA>y2vNa3T8$)Laox zq{o7s%v#ua`E71Mj`2i%$RKum&qzLz9(Onzan~E*opL&loQx>5lYTA!vpaL*Stf#q z<`9_DL5T5$f}%b4dKT7P_Q$U#?Mgw)M%z>^>rS{NZkN7cr@Cu&9@A z31+}4*gn2c4`ED>;@QG)0k#QiAAQY3wzk$~)qt~O!?W@fuRC5P?@@(_DH|!`Z%Amg z5{U)EH^8lsItd@gi;PQed(+l>l#nDC4eoSO%)AFkOdcaR@H;$zCBknsrMxrnaV#}R zZUB0ddK&8W<(n>6dHpN-n-2D+l9{cDfxbm}4v{msFh-{O3>X)ms%QCRNx|RHts-T1 zXt<9GIE6nd_6_Q;?;an7K1%x;k6pX((>iDx#k1G7TO`wWXJh?!oOsF>no9xeh`FZ& zJI5cd`t3?~AZJ{^V#Yzj-ck$`-VDaOwwr1r9od=NX+CNb1)A5n?R;g*&j^R+XyRDK z`gQI?ik)(EyDF*SwGYrpGBF)(Zx+_a(j6FU@-oPy9f$=z8>**{AFfZjquWv>Zv*m` z;e#Sez|+$J&c_P%2DCXo4*p&XYKJ`7Tm`bnvDfg=e%}Ch3oj4kZRrbxc;CIRp-%ev z+UoAg)OT#;LSns%qMl*TKAhP_S*XK8cBww$N_6^FpdkhneYqVH#2B7j2{m;i7^U0N@LZu3)L=fL@5O_T zM1orlf2J}5&PcXoG-L6=H@)RmFoqJ6X8Hp4e7()lTqTO5hoqU!y*`Yl`h6ad8$ys%6Yrb@xRz9iBU}<7 zS`?IsB39qxOZ6{X_JNR`C&-BRswsNPt;lv#Uya+xDU%6%U8bd8|Q9>)7O z85WT*wm~UXKmx5{bt|#T`Mx*m+4@Qr@By*T#2^iC+s+$%)Q+B3 zmz6#qKFH6pSbX)B_AguOw#ThLpaWx00lu1YpgV3&{Lf|)vuR3f>=d^fT>Sf=_4#XO z@_Wo1tE5}jL3`i_KSZh0#am45t&eh&Djxv3gf}EBqdlp&ZKX|EoEGQBKsvZVU3fCL+^Akk+P-)()dC5t^1#A=@$T>aQ$kAV z)vdD(-JWx1;||}+N!X0XGWLEmI6Z~uD86Plv1NGbNLx3~S!+sz`rpTIO1TGwefA)@ zq3_a%zLZ$}FjzjOV(M=zH*K^qOKP3K_o+BWOyS@1|C)VGabFQ~r0Q!i7DJ^4Km{Z- zD$$7(;J}7Yug#g8pU*c)Gh(RQ1gvK#t$AqJJEo%N z2;)8rqmzl~Y0Zs#1~I<58fw)$Txe`K=_5+5NG(ffFkzPCj7wBfV7)n7Q<54j>Y}54 z5Zt5PlW+flI1jKp{xTKDvn^#GZ zgi5xZP5U8tMbUIh1|MMOYZlsid6=)c3@vSi@_LEl&^AQGkqtjt(%!Tt-tmF56oSLN zQhL*fbZSz99OqC6tAC0ZES}*hd&TqaGBKCaatiJrexD;0DYGiHqWlz{Z>>f8vQftHx_ z7+w;G+FMySNjqYb4*4KN633Ayrh?r%PR_CiypxNO@x3D^~XXLlH;t=S;isL6+bwOeN6;;#`2Xn|BGig&V zYMNGD1$}@qDr0K0Zv?thr5;DL%|Fqm#Vo+vmITqmuTA7tCAGZapE4k|MrWUAqY-QP@#FrrmtP+=u zcuyPStrda5D=}=}<7n9Y*Gsj`4)%X;HUu{4DW--CN4{RUB5z903g?F_H5RrxLaw}Cl0$#M-#5R!syl=SJC zc6%iCCj%!FPiGktFV|Clq@*8_O8ip2EQKjDetI{#Zo~*ej{ci>`H-6$_lpIdAdLu$ zxAhFz<_vFe;xTZ79Ny~lrBCFp`Gxs&Ax(So?6G!S!(fsBdW}=coF|pEwtGrrR%ev; z8ljX4-!tBO3p|(<0E{Ln5S_BWk;?9{2dEJThisv%Hm|#^4$uB^99%5WD%PCN$~Nmk zq^!9EabB8$$6BnSOy|=%!gGx61y4{qaq{Cz(hPdK*yUNi@M!ajJx<;65aejNLWdI0 z`F)p#VReP26RA;XX8xDw5Vv;kj~2vB5LrCh;4#F#sV(f)>dNA=A_rm(b|kOqG{UxNmL+&UC5B{2FRwA69{0tG3qJZ2;q9;@0~R_!Gil zx14pH?(qz(JgK2x-fUoF@7J2Y2Qt>)Neb{3M67s=2O%zQ%OA_Cy0w+pS_eZ$02WC6KDf%kN^_+`oVU&TjOGmVmIMfzxY z;*+{saIbwtiuwK`;I$#^$&p7?7ah7zU@e`h-j9&#FH}QQm@RZXTHUc;o^^$ zeQXaOPlmxPm7NN!ui{k+*>t4CG29~kX(#_qx{8_~Tv|z8E>lTHSJ(g%(j|Y*(YGs^ znZbRYn}+hWB>l}@maeHAcFHmTPw0qW^RO0UTrLJu*ybR~WfD$+@m zoAfFc!YC+&x#kjoOB_dlT_<1A%l{}a4W!o)k#>kuAzT$v9{w}^0rr$lK3OBKNWW&L zAgQPCpa``E#vu1FipEl$`efR*a>sqt;WuJTUTthFGgj{IynkrgCRT{A#n=`b<#vSW zkdP1>LAFBgD%1%1O`d5ECyPjGgf?&!q=a1jY&?0tAJ3rKREP3rxUS|fgEGWW$LnI! zIx6M4VW91j7Tw*J;uvwV2N8Ji8-V)=!9$scNF)!V$G?~ zUPjAX6RppZOeXZq;J<^g~eHb2Gb_X-%UlAr8q3`c9l3(}P znJzY6NZjG!qtl!H~bhLI*Uv@tPc;G-T8^J6r>A02)q7Cgg?e7oY zFPJp=$c2tPg!H4(S{63sX2qQWT0L4J(^&~TRjK`ub>`^J{cm@KaRfDJ>!f{lNuQ<7 zI@)%AMOOxF7wu*QpxN7f#?MLlpiawEFt~aHsK-+%0~`#}OJw&D4=ahkoo&sTVt@f} zz)DgAd!IAIQ6s!71KLjo>`!87t@ISUMNK=|*LYM-my~;ea)&}dU~%(?qal2#?F51v zj-y0}YX~ms3itW6c70fAs>IYeB`Zil3Pk$y)UTHgzS&(8QS~sMYl4N2Mmmm9_Ux{B ztVZk9?F&fwl>gScJynDJtjIf#);+CeujUzun6s?w2i`c$>W58XI!Ku4_1A=q7$f!6JRRW}iNY8; z3tZ#YRa|)s4Ug$)E3OJXyTjh3`XwC&PU&}QO1gT4q;LC#p7xfd55jJXhz`3o&3&YM ztXQtdxN{kXm`IVnJCqd$DHpmzgs90>i6Wn#S4q=-6`DMqqsnqxqkg1p9}!u6x-*uDRqym;p@-FRIV@8I$TerJ{RMJ0z(+tHCF6aPjTBDc{I z?{&My&~9IBOWXcuYS4Q7v`lY<8GNS49iW|Dy0Ipe@v-D()*#(D2z8D&y27&DZx(a_V@_2OiRi3b_myUGqkLr3)~$0yS}eMJi}H>?u%ktZUfIodoRsE^`N~Y zov`t!UEjv`QVHYt*rc9ot}PFpx!WESI)?+TXGF~{2qtO2P!qHk(dne%F%5s^;55E& zxFvgT+lQE1$zR-oC^5$&)9K`b)&rGFl!$2zyF2w6w0uMm=pxhvcO__#9dA4{UTPdv znb(m9W1&K%dhex6PplZ~Fnnm>(24}wAIGegG$Mc9h?`dB)_$`T$KAQbc~no|lRO#x zn~CTsd+*yt+191V%;c)0BM`58E(A5urMuq^K1s3Sc@9TzlEvDcaFGhvp z{W+~KxOXxqr2WeP)mMBkq);9dQ zACJqZwTJVCq`}byzHI0gJFM>}8k1j)~V`q29h2xVdV*`L|q-os@tP`wxJpIAab4_36LujD#p`E*z9as(J(awu=abGTviaFoa~%qSl~>#=1>tmX^T}Q(~L$>mn+ZLWf^ zcjJe-?~``8cXhwn^-|E;ia>%*XI~MN{@QCiLT!(zeAaxq(%G%cl<)Jd#lZodxY34< zOGA0k2sy8`h*ItVQR$ouWAF-mPqfk4j;5>Sb`$rLt~~09uZ8};Da##aGo=1%`f+>y z#*kUa^>Dxq1*rL2lP!teZLHXY_0c7$k={~kfF{~2Lduu7hZXoSC)~qVo0{QvE^WgxY<$hT`k^cfMf6#S3wfwlxqkW~} z3Rwf zpT!#+9`Du9ek<+Gn{mZgG+_WCM{5VO$?(81`7lF^f*3{CC%8A}(OzR=X=@=v!-fkYTo~ zrpqD|YlIsM8&vIsxngDdLjZjL7;O_CTzDd5MsL(N3|TOR`}b67O6XD3un)$;JfE6P zIBxaO)Ke6^F*@5))T_i@`KCK)c8ij;wtCgKw(5~yaYo9$m&K@7KfI=>X1va6AM;Url^@S=r%*qLKJ{T8Je4 zY?znu;nvY<`e3fM=c`dM{jn~kyOWifo7whECdby0E)2a!#NOi#PQ^VMxwTJ=JR#Lch(CiH?vfse~OQQpyy4(vvxAacjb}!Wd~t&(L-G}Pi$NqZT)n<-a1h13t-akR zmC89HLpW0mvTX)FMSPzHEUnfOTym&a5@e0_;u=4je_O&gogRXY zt?%$izYi5f+YTdL#t)OXBOazmz501yxK~$hCQL`hd7Cfj1#RDoF zkEzz>woqEQyKkyK*Fi>zI6u`flrNe#GF2X=CCBoPo8e!m6fIRpe$P#YsFu{LrX%_3 zR@$N=clWghO5T3J25*K0kvkU-&-hMvF;ZGqB$opeHH7 zt00MI1Elh%briTX@5d6W=+Sb^I;3xe-}=Dmc$`S2*qu@gFOCb5xuYIMYDK8)E?OaB zz+phft0<;=>kza(ED+(Ca#`w z4a1ct%J2ZIaD#KaO3;qGzf@l8gxaL@!^4w}61Ag{g*21h!3fOfu8qN$G1R_VEr(Vm zVW~8pL#5TmA8YL}iXs%~r00#Nq@f`d$3NUjFm6nOAC93R?jrEFJu5vTz^5-Jt&?4xcEc~bnF`RCV$3n;{1*MbGA^_;BLQ?qgy1bvRGK2}bxV&Vs{Cv~R1a%!UaKAGd>q=koxC9TCZ;X_V@cClbxi_W<9oix!TL z`~htP!h>#tqu`qc&)zF!Ll*Tdis+NW5?Qx|bWdo^`e>^rd``V-#PbnX{G`3pFQ$iV z*jNjh6UqOoF8)RD{hJR|dPknaQBcvai!2h4J@(^=TIA+{>UWbo1MnX21{E!##Z(z+ zcd$0kwW5KrAR7Mh3xuRqrq1rwOZw}FO}w;Dp_?xsX7~4 zRBZl~;&gC^FJwpM?zs3IFWrS*rF^(+TPTl`&LVP$?6W*I4emhj2q=O8kP0M6(~O(; z#`wGFe)r#DGk)qMvQ9lz)dHh8<=*mmIW0qA-_+Pn*Z@hpz6=>tYjJ!%m7Mk5G_)4C zgrBLsV~8wN+&h!TbWY5$!%f!Ux&U!|vXrE(<(z$gzPz~9ga_9Mhk}Rm7m>3X*?*>n z|IQKr>jDSk4@I_u$$K2-!Z}YgTq|Qjf{$l@ajy&Jd$Hl-x6NqhCg=$E81i~7!v)3! z(WL3i`o8A6b?m-7=gFawK-Y#Y8LF9^$}^)UoQUrJ%&({?Rn2R)KANF|?S%8L3ZLL( z6_Ov1#h9ls@^CT#>nt6J1?geSNl%Kin3%|s9C zbU=0r^?I#@e&m$FxlR#jG)iCJSdcKUhjKQsD`+d5rv1Fv79TMWan&i)41WI?4I-v= zVds6X1+tcop~b-CS?MyrqAl}@z*sDyJSWxNCw}BhJQKIEcvYqEAa^Irg%*DUdQDZ| zygcmp%to;D6--f9387bK6&CJ2rystFDQcArF^EVwQxH;vD3|exixPYRC)9+k13RStl(^IjrP zOmB8qi>>pG^kBXMIf$LCWW<81E<1~&ya1o(B=fcL*w>fzm4b2j7*li6pqKNv{IpbR zHl;M=Gd9_Cc4hq2x0hizS~9czGaZ;`scXGk#AYNMAHqcaq`BKu8Ur={T>14>DJuN( zg8sORD@tMZ`3!bcK~@KazYc1BlnLW#0l3`KU%2<}HrLoY@xQ+VWkn>6^Jfu52M07f zjFYAg8sUB$0v#c>@DE>xa!<`;6F_Bk&ZNl}&5b-ozYT&j+%ZO@uXmk<*olBJr5|hb zQQ1wU=yz%J?@^L#MNs+}Byk~QHxVY|I*BASJBxQNGDz(XdqF z&)u$w?Fl$OTO3>Xeb*J?KtTe$I{jhQ+B|u_b6RPXZsq?3KEggBj47g&OHN~BBcSC+ zYH~S>3JeU4)GAQ9_;CNo;q&NOqSc5g{hhsVmtrCvprcXAJ6d0{vSRYg&L;A^u>$sV ze%II6gcbG-e|u|WG(8d2O$&%aqJMlEy#$JoHaGwj zj1#^43#RTo&&tzSYFE-)4h^*Cb-p-sJ>M_v-iI&Tp9)$szV7+3*>()=jd^5qp zbM19dY2W|;6u;+j9~+wQF{UE5N(z-0deXP!@`79*6}NyURCtJTCEd16z4#B6>)(Iw zp9?)OMrhRBu(!L*(l7QZ1SKYDiq-rY!pXfdQ_g>G$G`8Ozb=}Q{I(-5V4M53IAk#Z z=tayrwCV=j4STQ?cdw`GjwtFE55Lyyzdt{wCFR~wX}R}S$0cUE|E_rA z_NXRLdd3y!e?V8Hn#60VIqMD(w^q%^-u=zS%Tmef1VzeDN8ZxFe;e`krUw&umv_&@ z-W6|qDq8s)dD9D7f03`yIbMj~iSA_eA>E$_2a$Q*RWzL+#p}Z4OhgLXRi^&nrA31|tkv8sZdrCwU#>kJ zPdBr;Xb@iTroi#dV&&qUPdb}NynZUGfyo{Z@0pjw?H#Iv^A`W|T^q^#YjRF^?Nn)k zzn%zSmN?0v5DAl`bKUQy}X<8b%KWE99kW3umlYwN@P z<5$y5mmnUb$~W zry>&ZV{@Vz!Azzu;a4n5qtvn3M~rQa-|GV*gP%JdJ9IJw(1)t&mO{x*G=Hp3^LxV| z&z*2K^$J&uN{O1^lg-po%Nbf_z~F+kgY>qxPo<08_x%oeQ4* zCMy`msNJMwnYX#QslV|fW(Gi=?B>k!^`E#@z#6y`crRXxEWp{k>;c%IF|e@K`)=04 znJNIkTJ9(|li?nKk&6au&TH|S)^=GE$IFBDOYX?g0+)$cngBL$6_q+#s>wAC(eXG4 znwe3X9+4#T3VgDSuxY!=!xFgX;JoGfh&O790Mizx;?-zvIN)4&U%C5Ty1M6#hLN#v zrqZx;gr5QHO_uZF0s-Lh3_4!zt^}yh=E%nt4sus%MDhG4d)+=~qBFkEk>=Bewq-dl zxOzn8Xg9%h6e{U7%g2jwI`qtG$`_o_oe*PD)ZJ+Y9~9+$o=rD2i>i;d7u6q+XOC`# zKhqGtex*($KbA35S%rc{wjb>0rSNDs{yrDi{qG3o1swDRwP}M5W=v=uD(=#14`1qs zkDnJ=CZfZ?(BId`taN>F+oz-D)g$bGd_e!QLiM-xzU_;ZW%5UDz>_!$i&Dq?#)u!V zIc+#HwB$lvFD6y(k`__-Au^{n;1;*DFR7^q zc_Y%j0QJy;h+4wter|fj1<;J;u%*m^w{ei5*R7J)ZqEB}TJ2B1`osKZQe4jBnW?r~ zNKDvtiQGrPpu@ov+dc!VB?A6{(nq9+(1q?R4BE=7(!??_Fjmr(bIZTrRA1d#kAJ~Q zHlz*d;IJI}&LPA6FyiyLE(pPvY7~^;@C5v6ZF?{z8)`es6(8SOdZVHUqAMTn!6v}g zuew<&9MAU(Cd~kuGH?22UE5^2i$rkglxL&Rt)L^epDpI-ZLM8R_Tc0MF2UH3ebPvF zQ;v~v&4v~+Alqo*6E`)y&Y^Ba}v^fKf(`U`uZ%wwO9hwo;CWg;xr z_x9fWNqDSMu5h(3x!+mI4Q6mN1Gf5H{H3fC@8AP0c=jyX_2(dlFT)j;+;F0<~gv41bD7g>-GgKzV!VuUUp*Qc&uC3fT z+a4LPz@%fOmAEKV;JcfI_&JxU9|$EsY5NRq7&`4dOfoF)u6-uXE|T}hrU(D zfmR4J002c7=of&-`=hbiiJTB{O5EbO;Ct z_Tn7)o9fpBjI{P299VZ>Oh+>T7L+h6=^QaSuB^h4K~l z@_)>YwvtXS1tH@#?X6v0Ut&8}Nr@d=T9o2(asw7lf#JTo`QDr^?}cQ3_VGZ>@j{>L z6P<37A#&lgdzWe{296!k_rAZYlgO{L8?t#Q+Kpt<5jT#DF@#e=bk5N%><^=p3fsX) zmO4PlbWtiRoBn4PqI_{Vk8&O0XCPrMgR4XA#U_!)330!82E~X?`+@m!pq|C!w*1ntD-iW=tP>Qi+a8|e^GN)IEi=H)M#u$S=VsaI z@_BB@#lf!1Zh~P+!oJaQLnpWd?|v@5d|HR;&*+Ri^ch5j*^X`ha^AWdSR7R;u(|2P z?>u_(3GuWYF_wHFUiX*tHaMj_hh`uWmmWY;fn<%(S_as`ZVW-S*+g}q@<5t;7BVUp zfU+IP;JbiF9Z^U;Ikw{V?PrIxWRJ410(RXo$^MoD4%TRB1gt$c4mwiw*}^~o7U}2v z;NX<(T`ZNu?K5bgb^hxE|M@9peP!>Ea6H@1P)F@n{6>5}?n+UD%Ial8yqcbxhK72p zaINu;6y8T~k~JP!UqCklRdcyUC|Vr(WEJ1okxfVYx#DWUA-bCCtgGXb!=xj?);9Cj z9CDxZV6oV(iPgi|31nIn14owCj_2r(7Oxp*5+S*b%fOL^2(a0d?|gg$-m(zI4bt(G zL%Wz6mJ%DHTCEIDAMXypfuE3SQ@%yZ_wXVQDMVR$Ki-(!1Bx9d|3fTU{!U7~y=^~u zQrkUUJcnv00NvBSzdqR>%a-i@zDK+J9%lE8uWoyMIZfXVL^A?|sjMTroIxV&!3@3D zyL6boPg<6-W0I+?OQ4+5F8OlE>!GN-Uh*lC`8(1T#y1| zOoh7tu#M2;%uMZg8fD<4y^Hji!TJ_Ru$#mX^pb*Q7Lj5O|89fR|Yxwc6U_FBhdVxO~=-Dyfv%bt`{dltNOj&Y*t7*B~lraf|#Rai< zum_lZ_!P%q?-`?lmuuFSdsce*5MP$IEi+of#V#z`fWD zOh>|7i{q9f&FM)`oLCgfr{N8d9n<+MXw($n2Sw*BarY1Ghx~1*b=xlj1_U+M#Uj_f zc1TwC98TpLNyNj3x0>yHE}R*nT`o|OzC^QX9sZi_r)#1uJ>r9xAKEFl2VW+pg$EqW zHw_Nu^jrZeKHDLV>Hm+gw}6VO-Tub~>F#c%q`ONArIGF!2^~_p28V7$lx`H1?w0OG zVvr8$?uP&2ebxKk`@O&aT}#(GYYfhu^E`X+XYWreusG?t0fMf}0NKJV&eY~<>Eis| zZ8iQAWA{ZwonUQa0I^?|$XEKz}hHxD(+IS5d}}M8M=&X*@@MC&kh%7 z4C^6C*{F*?2xI0cZvdYQWsuY-~GON7=&bo*&G*E4GtOXV9v zWC}+j*U6O!^`N{5?4$b25sG_d)Y0l3aY%yl?u`q@8wr=wZbAI;K3CYXxw{5*l7i$K zY{eRD97Wf`ob&vpbf|%d9?v$0ABGe6cb=Iv1QNX>UH_3QNu*DGojL9kZo7kS__gwe zs&L9#DkYREmMNkvSy&iE&LQDW~LhcHU_Mu zm9UGmcLzDl0sDhyb{O|is9g>-JLbcpWql^J(+O&J*7d`%Qc)qQSWTe|lIkZuQyS3M4sdPvu-ESY2WP~Uu8uem3ic=S&hjN&p`jT*wPulaQzGE! z<>kHis93(c-MdR!=xw-{pA>Ic7w(oB1=Q6S0q>ozrrQ%v2X-@4$1uvdCGc>DG+}G= zdj>^{Q9$f@TCYgGJ)=bGAYeCr61h05=l!f)!DjgbCM5(56WWuKuB<*vkDKX_TP- z1&`Dc+4umX6;Rqs4*HuEZ~Y0OhPI2q22uUu1@nt@Pqt;GEMktDL^(I+nGjIcxQvTW zt-^|GBh{V_1O~jpRP@O z26^6Lw0w4L=YY66WOrjX^uVQtW4LUo)+*g)sU6@LZUtq(lWzGKhtK)#7;l#s&UvNx zql(Q!($h0f0Pf7LB7UMt++}rnyfLUhP|QF;o4^6!OPRb&3Qh1eUc4wtuZ{EsxRR-& z;`&|=7-(c-xuW7lF87-9;ll0wZ#u#VRmMy)Xrk0##*rvF$9C}9PVu=NgUQ&1nJpcP zlq!x*0VVcWljvAm4h1?jY6N$5N?7r1wPMGSRi!-a=QG3eiPsT6VD7ldOP9V_%ftgG zj?-O&@Gg$^?RJbplM`UJuJk*ALF;I6faPS20MgyA+8ap_USTqHTcn-zX##DZaIcTM zy{ipVaD{7#=ST?`Es{XhnLWtTbEH^@cpsBDh<47ok=l8mgRa*Oh8(WvC>;+&rc1W= zv-?G|n*&0g#0y@RjFzrJnX+gkVK+hwL+c?wKg1DUPq}PH9;AN@*rLA0_*VUc3jHoE z6t@erbDK1Fq*6z+Uobo>zH<(mOyy2|<2KnAN`yW%`;aNJ2LGv>Wpw)LKH+Ios3UTP$Jwdn_(3Xm0G4>Xsr7fPNtgrkm9(TPVCfz(fEP|`3Ob?`l4qk zW@+9TH;cD3@70W{wOrpjAPR!))+g3n0D9e`BYt3wWC2^i@y3`N$7ppHpw`;$kN_g# z<*n<0pe;VdWSV)GMZD3f;WofIMkF>|FjP}-Hf=sAteWcW!1V+U4k@4I?>&^1u=F7g zVs)#_Bm`vZkT7@zcE%cG4nU1LlTAVuRRVvTQV*|Aw(RR5)-~*So-62cofiZO z3Gg1LN%Q+`A3qS)D-)+VU9TwKwsNn{2eDUC*+5&-2U0b?_N_1r1$EPsQ+x8BfyX=3 z=(n*7HEsBMSpCarg~0A=acX@2`b`)i+?vOwQ#L}lrfNm zBtx5k5?9sAtOZjd9pDW7RM`km17PP)oW1Gm4mNe%Rw zdccj7Q`7~4I^&elfT;ek{b`SfV+;}@!P4x} zTY74P0Foo|`3ZBI2vS;Z`tzN`EqKAxjWQM?M|Lw|Sv@3#+-FiPEyWhjS2BX?#*Z=# znDtXeUN24CH+7(bl;*O2YXLB&EVQF_R+viOT%C0RJ~5#{!lRBTZ}9{;VB$f5Q&nO0 z+!Utr!O}fp73jQdusx0vT0*hXT9wt&kLPxz1Z90j3M4GDO+j8$($S4jZT|_IUf@r> zATVXl7&fBT2(4VJdmirtgc=is^Ze0pvU*H~4&pWVVDUzhm@0^iYIx^5ZYqQ2*#7Ez zycgzM&ENWMm>;AjjZ}oOPU%0l%U~kyQi<6}8hS3r*4#D<4s?t$t#)ou{aksO2+ox) z+N2ju531*ooTLg~=#VdNkt86wqHVqN!}tAA6kiP~eNF(?G&e^8$4#Xcehn2{p3Pyf zp07m~ao!F{Z@OzA043GdUT)7?4J3h(nbNqTIwP?g{O3D;{bHCfTGs11TKt-yLhTVq|r(<*2_kn z5!o%G$)x8K$Xl+2erbl&hpxAK_{(Nuz;Y2(C+2npC631r#L1G;g0 zzlE!jhDxjiF>>JC4z=5|rgZEiwbBeBB~aSfjIv9qF&@tA@9N1YaU7OLuH}!^Hm?X; z@V8|q8%EpFbW-k+QHO@YBZ4l}XfnH7DkDOR@<{fRvC(KFu-iWXQ>D3YUjEP);i_u) zY(;TKG&&{Y?k~FY=h$s8s_kZHM$iXKEekt;K8(FO8MgpVR;|z8-aWY(`mjYjtNQKL z?vZ&cXuk;^R+|_WfpIO0wOm2gp=KA}JUkYpqGbD+KP-RZ(Jujpj8{14!4hg~@fPzG zbUR(Bfe(>xaMm+_j&bHri{*lzP~| z8;YjM>ryL0sqvupWKe2-Xk>O6^`oLV-!jiz@(7DktR{rIw<`jiNN00rAx8(FI)0p= z`ZICc?^2|k?OhphIn>21iCrdN$kq)7Lr`)r| zoMWfake-_xx8g&F79z;PVK&3-aa|E;-!_cEIbU9}(u(~Q3DyMApz164!&r+Pz|@5u zHMd;a*d6Gd*JfGFEi&P#NC;~}K+C5PN2R9dz_GYvy9OZH%o;MksIZ@>0yx{ESd!Nc zS4C+7;x;#*gr1>yv!;#_0E>vAf-#dJl=z7u zlP4bZl1A+|+65b(;PxP{=b7@vrPk=QRBBfvJncvo{(GfTW@9ThgOLxqAf-0;$2syL zVK%>VG?1ysc%da|WHVf$Wtg%3av3Z%#N_i{CH9B*V3qo) z+0>F4Uj+`khI1|_t&bL^BHvRp7Q6mZFFIZppnj?T;2O!0ZqGlWnnbR+*H@3 zdio#Xq&AB4I~W0d?GiN}ME2yKR;ey=(;to?i$;MNOm=l63}N%1mgRo72BN2mlK8Ii z2oehu;m_7k_#BT6B8!`h+M)|mf)yItlC&8h6u079c+gIz=5W%>TneBXvPxb;fMFr^ zoqDv$7cm&-7uBwkRsZn4xAX|-q1rIt0~9Nq=QCdc#s~|QFyl6#v)MHyGfFg4ise>x zPG@Lias93cQ{Id2j4zAJFCCOtDS?y8} zoiv`MhBhAOi@~Y+toCMdV8PBY6k~f2WrA1M;$0-qfJz%@qCn5TT*=1cYNtANJh=MY z;o*H#1}e30V#H}_eS>x*_YS@3^bwPc`Fb2G$<@aWKJcqN*-#6laH4?TU|Z1?0X|%# zsg8Oqf4yKT>DqtQ6UsD8D~^i(cQ&vM_iJe>Y0ysIiu+PyANOtkS!pcdk6lqyk{sn* zs(GJ-Ds=e9S(Pi1#idcVv@*}T&XHUr=;kjbXxv!P|kvcW&r&o{Q2cu8W0nFhu{L?MQ=SAqrcOla?Rxvw^MGVkuk=yiwv? zWE76y83qG2&CIA&DCQ8Yj64OgBv!==bn6}n8oNKJnzF+_p0*P_iZw8o45EvPOEZJ^ zRkwB7N44md&k0=}di8h6;ZfDB7;d{wz7pEFPVS$2#DUZIbe?AGc>V=wJH-SNu0UU+ zd}Uan#4MdTi}b;~i1?hop=ft|8Qb~pX(Kn!Xo_q`Cho#KHzWsADumy_nqa5oJr~r^ zUxSJG<}xm~yJcZUsb%GsKe{E|>h@r@uC-37#e$PJ+Q@zdM(Ryvn0x49 zds#RmS1g-Bi@(s7SJc#d;M6v>(;kYS`mDKmN}`x!^1RXaa)XH}i6^9bX!_m3WDv_( z68;P}dmqppUV#s~(#8KceR&d1M4TC*r|-AlQaKTh)dJGwzw5nd0lIO+hY{eSYHxH7 zY>6`v;@U`3R91Mpy+dMgaW={r6V#!|B>B?kOQVn@;j-J82EP~A0_>x9aTW7drzWN@ z27;^;nTvAsX)M;UR3NbB?47NyVBO%a_G*SB-on|=0(r*f(&N_GnJv^346AW|6IjH? zM zVS6hl-?g*=lFv}S0whX(=w9gO2=_(aL<>#(dAl_Ojg-0ul&hA~yBJ~`Mvj(H;%m4! zKa@c+Q+cUg1JhjAg^nY4;=6ucyz7D;;kyQQRz$>L?(u-Q>jny?WFf30;{L8CdoL_6 zN@u*OB$v*DulO;Qt_&PChtg|2tQ-;}sOp#<~EjR?5aGrNV3N_*h|MN{U-=XmRWSN+rd41%uCL))QB$04k zw>mvhaeHwYA>1zSK)ow_5xd7J6utKG=mvbM$cjPDHZ#-~AN+dj=sLx_*>UwsER8i( z>M3o;N&c!^t8cJE2rp~o?N@%m5;8pA*MZMJ{%&!zApnnlu{gx!mW8doy*;y{V5~NJ zN!VA3Ku;;sg{75^%G_w!4hc7!Uq`(mHry_EQr?;G;o(O;u}C^AQfY3w?}l;*=n6Cf z$g^sM^%Cy&h|4Bw%$cl@qxwXkFFjnXNIHlHx5Ba0y5gzDg12^kyJ|DcQ)g0?%8!N6 zZjkn%*(#JPovSJ9evcW0j+Q(QBq35Cu`~QpYJB6%hKD5)aII$$u2;ub8|h}vBHOuyEL1RJx6#$aaXErz2i^3* zl>@PfleM7H*kAqZv9PzdkTJB_dwFg?I{4(a*8HN0PUQ_ZjVXbwYE=zFsHODVElq@M zf7V|D-oW2`3~%Nltd)o{E6sf&D)u=3sIvHJDJ{*jg8UUa=?`|c_Rlcj22@s<$D=nRC&hY6}4krN4IX^dB7v$Gy1_O0zu}Ye+L==ZsPwF zKoSrwHHyP0z+3zxJkX4h`q_|mw6?8?NMGcO@Y$9G@;UgIyx3oVk@0`6<$nT3eqZO& z0yqW&3VhxK?+C0kjqY~|(s=)R-G7qQzg^%!05nFP83$Z8nb70^fPLJv7ytPS%7up_ zvx2q!mQ3Nl;Z}a%#@}}+@Zl{&Bql@7<-h&7-{_`)erU#oW2CK3bSllCfx@Q% z?tT`SsV^^M1w3PNMVeMN!8Ud->zk=bH^m*`)%c;-oUjT#{f7Y|(BK^@RiAo?`p^oK z7Jel;{kow??CBkaOhzewNNA`u@NNJvR14#u$BA~^d%3He@F&u5!Sc_aqXH(^IFytU z#3dBrTRMP|w?6+y%o&gwvu*%-t-5~!Q=FMN_7z%b z-?{G1%@&#f+zGpeeLi5nLOmH4V4QQ4F-b|At9Vf-`Nn_?@1KDg9|`=(+>oz-Vhkf4 zyzmBb_6f1>b^dn;&@h#@}|vX*NYkT?WPZ1W*xLDVS9S_9!XdSA!$`+Y$pX>la3P9qajr+%$9sfP?W z<;x7ckfuB#w z;_T@HL-DzBEixq3brmv~(iITPmkQ_J_tUZ6*wPL+5>8cy%R?OXBA+}P^&od)`1@-t zGA`UfZjLxd8T$i#$Ij&)p1u&8vgs$uEXvG-2=Ga2&i5bPoP)GZ{I-x_UzO_@E?d_4 zD+|Ah)fB6(Roa?NduDb{4mU)ShTA6OUAh??r8o9dKNLqyd%A)cf8}H{MD24ML?u}H zEwW3sF1r+$`U*4EZsl^k$Yk8ez{R4)UsH!uQ}0W|!tIk(d!=Nx8_kczySIYQ4SAba z-o6E!m);QDt&7WK&w>H>%*}w_=?}F5o1B`)`y~7ykztq;#d$_r7cI7eXC7a;fgc%7 zo;6cwV%WU>@?ldY!dbyTveYCqycsdyxexOMLP(SEU>WM#basXJ(y7)LC-mdpgb)ki zj-tUb{=jCmzUWXXO>!D`iB0MW;)-tFvOJtCEIa>YJ28YkhzM$NbaJ>WsUtI+qa?l{ zS_IjzC@STG&_L3p=i9AQ*<)}clgEl_RF%1GV|_E6t^F<3Nb62(1vchuI~ACR|F}4> z^zj;OtSaZQ_eKk1G<{lGs8f3v>G6#lG(MICpqW;qTb!^y5*zLg`H1@-UyyswjGeA5 z*PejNIp`|u-@S$vZx$8Sl`UlE=M(3#X&O8DW)2GGD1U+E^`g7ok&75>o%l3LYPCdYA=rstSK2q6rt*~}3i28fU{vT}5soW%Ls1+8n#YYXMD+$`9;s)YKS z2R`+=%h~oE)ye0*!xlqZ>Nbs^vIZ&%e9vU31i#>E$m5w0fF~LZk0omX!s3Ccj7-PU zi4{aTTqU*SdjKwEFdG*O+Ib3(QC`d5troRd)MGh#?U@a@+W6mIQQ>(yFRw-$}#G(R(hJ@n!Pyet$c5 z<0_KIYwt%2{R*JS6MS;Q4P(y1$4_FkHeTD7qxYqR=f8<;tAzco?lfha>t9^}f{yPJ z6GH(skO(YPYh33qRa{NkxAR&@hlgCCPK-y79?`I{yb{+b(5jrC^lpNvM+^)OPFz-& z(u}Jh$UMMhwEVgH(-hDwy$6%w;p1;P-{8IZ9eCkzQ5b=D>Cch7&YeIf>#8F0qKYiA z(35GgX3=2znispY%dC5VwV9B9UM3h0cYW-@JD)oAQ{jskJmN^w<@o6>1m`(gL?^}q z2cOaz2!0*Gx+IPl=rVvmjyUqv|9woNNq|LDAyQR|=t953%S-I=0ya|00yY={Rwx-9 z#Pjdq`X%uJJ8#{``H#9zw&VAa7Mh33W>hzZib24)$@kG$Z}qaWgUEn}oxM;cSmQng zD<@0U71Aokv6Ye&IHhKcR1|CL{E_GxeuolZH0*Gj74g7GPFRrgYm9EAJL0YVnBqBP z$)@4rm|tk+{Knx3-^6Uh{>SS?>bVA9RAv1`R=t^#+k3z>rri&`?dBK!J1n-Uc*qbwJd1Ah?+bc^u~s< z&;-jr4fs<5zqcm!KVQG5U-%mOtKbi5X2eBl%24;HyG9})45Ym}ZVBd8Zcf)3?990J zDLINGJ$E=X->zuWRvB(K>2tiOyjp&L6y17An166ZJzDBApEhY#oLbklswE$@*K%Gq8zqo68;oxl62JaI*=b+&SfoS!p23r5{ld?|77iy(V75R zWEiNo{;I}p9st}4$it*BdG{Y^;_NRyk(HxNM~Kcn$&WJ2_rkT#LhysdFnWg8vHf0f zP$Z>{TSUxy=?J(w9V=qe92qv$3@KM6wuq7b2RRfZP@Ue)SC%dzh@zG*btU84q z5?*2_dm`r~hj*uMnUoi`WNA#E!O9`T585?p0jftWt|9k2GEByTP zLz+iWPkg?|G8pK?sTAa)6_3cm39`Ul6Lwimsuz7xsJjgjX}ty&&6Dzsi+SYofHl$u zqqa?Se5YZc&V1HqIf1Oyd=}I6S!6mEZY&m-o|?-XK$)mIf7r`fMCIF|i`YSbJr@Wk z1@av(;rPI|<~Ye{AEv5jO-j7dIr2La{i&D!^Fpfak(#}qTyt9M7@`wDzb}LXa&7bY z*5xp2vUkaxtzwcWUD?2*-5jRg$9JZ-5#zpaT3vDyr?a>pYu_?|M6}rb)UAs<*DiP* z{KjxrYxT`$zaq691k%_sE%bfr5j`(#MsJ|6T`%{Vc<0hE%3#l9P!Aj-wx$Nw2ugTX zj&Tq~vn4l{_Y-C~8ZXqN56wg$j{4%kr-_ zk{^s6QPXB?DFP7%RhMVOzTx~*xaOir7F>!a*4ld z#~1Q7(|F42meSxoXt}B|4(f!J!W%kc0LKoF4P8$VU}hnfflpo6`yX|yOa?9PeV#OU zNkp9bet(KQeKFdY(hz4KNXysX9Qdicz<9T}mQ;0^gU#RB%4@8X1FTF+9?TJ))0y`P zO#1H0JC5%7n%2X8!%a^JkVWrWUX_<{61z*NLgD;&N!xgBhc=uUNa`lDP_~uUPTI8_ z(Y-|BJav1=wOVyEQtKpd6?xFt{takC`eKgc zc^8g!s4>OQZd#NOmEQv_D9-)Y&@l{hEd)6(^H)T5mmxRt56^1AzrV}@su%&?M4jyh zC6{%aaP&c+j^4KuHwN@}YGOt8hxA@2LEH1<));D45HYSYB?lST%=3reK`z;$6JJbpN`l_#`)|F1!FIVGf z>W9tEb*U-8gJLFLp}*8Kwo@mqn)%AwOA^~AQoUHx%f<`OQ9{;c@LBjjNM3(GaGS@_ zw?ZvO{!*7tmPBZ^(Q1%8`f31Sm=^ItX4?B-L>6nc`y1ciWT7kIkAX(_zM9nulln%u zvYT-|i<95)nf)fLtd$O5DDEX;Egk0zH6gte(r!>o*|y8t$E#uRe@5;v?9j4i_PN1S zTifX`X~1r8qv3LVO*FCkfrgd?uY7fF@e{Gmlu@iJj9&23c{fWNCUQmd96+=A^vFTu z=i9j+>lgy9MNpQRVW{QbPS-T>1JP37P{;ZGs8hFLw9^ZxVgK6`gdZj$rd? ze7W`h(Ros!4OAlop>HSNx$IR<47#v6Gr~s(K5%&kVW>MulyzSlG&KE^_)V(%GTkGV3V* z`pVEC)r>mjl3t6CCia`-TI1={pDIkw;eLQLEYam(>L`mbk=A_$z57a5BV{?1TI;!p;~xP&V|Jm>Y{Me1Zy%zGs#*+}5o6`&7Ih@1=wRL!&RuPW}s!4+anV4n@2Noa|}ZxJ|N!h|aPfdWih_!?957<0i6g#aX zi#n6~&9ATsaKHO~9L;c5e`($+A(~U`PIj!YVq^$g?2B@hG15n&*=wu~Whe@G7 zO?9N<-6pkIlWoAL$?fQ$KOWzijPk37_y=mM>)Bj2W*A{iK{Mux@q&V0(180o7(Z4IX!iji~^S+jscxO(e&9GIhQ)C67b1;pyEH?4S)cwmIDAmxdbRdN|~F}9UdJSQBrk9Q%9elyT7!s zSgaUTX-l+93k5V!hfbdAFjI=UIxmlifB+Ci&Go(MXqorLn+S42tb4HV09prTj-=*09CNEFHI74-U+)bf0KNSt3jex z?rtAva+7(J<->R({1Z#t<1zm{UCgS%%Y!iZnbR5b6ekC;%`ER%g@d+gaGrcqPx|}! ze)mwcLOsAD0uhE2_W9nz0!#c@4e)6YUG$%@bXJ$G)^yI6uhujsF*S*hmKJm;08Lkd ztJ6BjV0oVv4w62Yki=9li=ZRnJcqeyrp z!Rhk_B5UQcb{KDBOHG;}x+SJie3{9pW#&*RjYkW2 zuZnvgcQ%nP0VmWub5@B!_RsIG@|Hty?j z6d`Vvsxl(&<Pan`lIY8|y|TOZK!=4zWoYORt}R7t7!c4<^0Eu04pT zyxeN1E9{H5bSN9yH~&m4X!;ReX5v~dkO7U0zfM6l9-Wx=0K9FsHfm{=2EPMiF0x;0 z)>zHdsOiUQ&joD!BO>hNB(V$Pvv2I{#w&=r!4t$S(9n%H(|Pte8-))M-i~+3b#+uD zi~BMKOK5E*ux2ON<|<{NYdqIQ?)~hBp}E`EcgmhJtw55Kb#F0IacX%?g?4_X3OBpl zny=lEB!eXeC(K2C*=jc>LwV+i@8VR2UDlY~K;wmp#=)qEt1pa2BS)wYGmIEW0S)-% z_W)pGr)9ua!|BEcK$sJ5VH>ThKezw@Bq`@yZo7s|aSl6EuG6-&H4&IHxB)%k8k;f9*8fNd+EMwiXXgWmq8XoF@9>&|v45osgW(k$5K{-FkcX`d1CR+S(u;OGZ z$G%KS)9|gj<<}UTYP%c#hAf=;AD1VG1@I{)Ai5%Co`!urc(_)?ctTH6B_{9@eTQ{eZxTzKEvs(n+XM= z08MD`CRjc!wDLh?|D&FS+HeK~X`%2Sw=@(U9=`)Aef@D*s9mA;^Omkb(Y@_MW}16i zO~!*afQ`1$nJR(Gy+m;UW}Sc5$ByHT(NtEpQ|UhRQ(r(mTqM(vk{b&JCaxcx0vM3& zc6;l`8#;Cn)^y86>f*~KFqf$Hy94}3!v@+_^#?ic+D@hnCb^vxuIv4o`icJuDX zrAsG0x?@|sUX=i%3}#)reoe!J`1o+ruG6Tl)HP?h;D||p12BGam38~;WFm#oDlf&6 z+uEg1|9dV+FLm}P!b2Jz^U`{4w%>a{rEwi-1a1Zp2-17#@C#CtIyAWL&5`iD45^R3 zyk|s6Tx~CReoRnjIv;=akiJ=2x-LHJ%R$qTsG0S&^~|LU(XOus`}+X;whA(6#H zsLYCDw<)6>pLabdY$wSn%LNJcSYQ5_N8Pl3r>YIujFI$@9sUb~L) za7I%jMpV=h&mH%^or%ycZsr$v0x;B?xjK2pD4+tZyS=v#f;CyGpjU~XO+Lhs8f;|l zeG&0l7IBE>*>INM1X)`!w%ZchvL)96f2>=57kI&Qja|&$R>=^Z#Zs-rGzkRHHTo9r z;mpXI7g#X@KWFq`3oo*5VU{3LgR0OLL(KWABTx^Yl=%t{<2Dt5}u0+CJFi#=Q^^D;`1W zjn>a#SdlfXD0Q0Vc-h{)_%$tjWHR8LC^a^jV}+OYtpxuCezuymd1n+Q=-{26<+Fa> zv)ebAKm!1Pulc7Ru++qv}J>hq?oHQ4W`{a)n{4Y#3HZj{sn# z(~9AI!uRjrXJU$O6Rr(ky=u86kn%gKl|W`1{HV=hY;(G^Y!33z-!MdIRcQ_{d+5ou zjA+obw4C)W-NaA4KFK06v=omw&sQ<2w#DegLG!l?5M}}}&PTdlwSN6~=v0n;zgroVagKC*539)g#6~_mZ1erz=)5$AVZ4EJ1iYg!u|-!>$#$y3(dS?W9S<1U-q+Cw09ita zHNvcZyVab@#v33vaZGeD#2YRdUU8@UY@54~kchzw*c&V;jCQy^pzXzxxBAyH2RsSu zWZ|J(f*oc^qPbk#)3r|kazO$xG4H#L8hrsOKt!XS;DtMT(O;4PY{fhXZ;H(;MK9a9 zGlW}IU6ZwfI?%^IO`9=cxZ0FFr*^X6eNgx&|C=~wpT5x{uP@46NNpA#U)m#u=9DS& zQj7Qct}U~N+~lYCGdMk(o_A$SniQuQhe+{00wUe6&PlasYI*}5sw4O56wf0|5(2*x zU}}kuh^{YpUfEVm5825y$h{|B3(h*nPM=#zESnC?68gJq;AF}o8HvvVv=NRd>_X~w z{V*aj=dJb!a}nT{b28_)Ga$g#`NrH_o;Y>smPg#HA{a^8b<%HIX2IhZidgd>0ttT! z$ii>TQ4!-O3JsR)wu+oexvk-z^x=dt0d{b2oYv=?n!tPM-VvC0NxKEi@3=@V@SL(` z<8>oQ4?oqmWa%!k3wZlb%F+PXCS}y=6Xh2CZ|{LUxP%!}W+3_s$5K)J{q|R(0Ayi^ zeRx%F!;>Ot<+`TUpDuFi6Ci2pa)ELFZXc|wG~Z#Zu2KuG)Vw0@#RF_rHjo%T1R~C6 zU9U;reYIW_B=G-HHspJEHE*ZEDU`leFjW5mTz2?nMg@Ql*5WLzja3ys7I?TrpZyZy zTEETZ6#yI_JZvpi2TI;BVrR(?{*Vb0z$Gri>A7sFd3y-%ARz%_AaJOoqV_q)1@aIt z6SEqE2f!wF?C0*PDZO{kVtT8_m*L*Ze*~}{T;#)$0WxR^33|u`;oN6}4#Y4l49&)1 zL@cYrqiH~c!pJZRAb;Wm<~*((B|zcJ|5wTZ$-6LGsI#L6KkAS-pO>~AKLl~$*uM_aTfS*S^Is+Rupr?V zL!1?5tQkCU%e3sw``rw3QKq5unRdGPCJ%5YGonyGb;z5Y256mcZpEyRq(VL&0g@$4 zkO0_ts?Ragj&C>YVC5??`yfK7DeV{<16l=3GZ8OezO2C%^BGL)m@W>tkwZ9NsVS<& zXiDRfLnuV6A>BLLgs=U5+M(_cymR@`~(^QGn!M)~SW zR?U~qa|;5=I#&gmunwK2T7<~UbDcu&zau#rKHOYJPuJ@8t?|XBMW%XUi19B3h!iD0 zaPrakBDqg)KqR&@Fe2~ZdmoJCGKxAPI{VA(P6A;aE|D(Q?5pfd-pLDZx|%m%6qp{m zczdm?ukTM_vtb+a`uI%bD@$x$NIZ$`dBZqw?~vZ&l!b$5cyF3zlnDq-PyxM#Jn`%G zFDN!Y#J_4|KC=9UY5S{puL~eNP|z(g{~X3pG^;NvOD(oMiy=;hb+`Uk!+%*92hgb{ z^S8qe;Qf4euFxSyAKy0Dsa&!X^Y~4d065B3Zmo~hhT7u|ep~xuLKl#y1A;Ya_t~F} z7|t_^hoOAx`Oy()U&L$NufOpctqK`)h?sSsBg*WN8BQ5meG*Z&R=`}pYlvStua0Ix z_U-0nQBhR=XakljZa#T=U2U0-#?-`EbuaM$x2nFq!hONoI%GuutFhOdgA+3Ip)>h= zoS^h4CYfvt!$%WD1oY!kw(mg#pZh#tk%KMsMo!tk1SN}XTG)~OHSPYhtN-ss5*pOA z6rj3V^_@TKzjRDY{XtNr&{2TOy4~=Abh_&4tJGk#88Lfm`&HdCx_OFgxy;H??v9+- zHKIKNs%M+Bzs_3!_doviNk+;&ZQky@D_N_}MABj@4Vafmc-ci)+47cnUnpE(_oWM+ z1f2~Jn(q)MZ%X$Cn;$)0F;XwJji|IRe*U#KkLL=nF8^@_%rTCdnlkiy=>@@u0(_zd zMb?SM8b@uW;xA*Bs6^DvW#V zi=#SoS=H~M>K|~_|G7wxzh@*>Mntz}nZOnfn4()h(?`h_Q>B*JY`plCo3ro%AFI74 zcd+gKeA&C#Cur#5>MuA{hCgdW3{-`9Xc8$!hM0e8eveo2JWQ^rw(xQ*m!FxLjax^% zs2vf{+3GV?B>ynXFqH=!Yk7lQAKLpYLe5M*l+~mn=RslNP{JEr`YD-MzbtbZ{bZ&d0ixTaKx13HgKdA}#hgB@r@J47tu2=Cp__j)-+#TZf!t$MQb^2Tj+4^$G)#in z20K24m$p8_A{9*}(GecVk`d`PpJUEh>k&r2vg49dZxzt?Gs7#s>DSK=Tk(Ftr;Yzx)9UyFraepTZKCRqN|V z9S8*+@0D2+g&{23vJ49|ib$Y(_9@+)&$?+k5eTdXi6#;4z=&L*B9v5nF91~C>e?on zLBoSZy3r$$uAA&6q8%?oOQ*~5<9aXN^WRJ_|9$5A*Sz2IIn6dlE<9FZ|A~eYUvK~S zrD9=WBnhmYD&=-Wt48x&3=HF-BkU<6Ks22$&HMJDq07OQZaChI>kvffte@WT?&;8S zC%AJVP{*$Q9qmId+e<@?X-9*KXV05`wXGI^#7nd=2zozYGYA))BZUI35j>~W{;ol7 zCSevV!9v||?jgwAe9jvjAGg<`U+0Tk%Q@4c0P9S|SM0Ix;!OhVyHs0+btJ=^bbe;= z(NU#~d2m|}UZd~m=bCjfX1fSP(*;kb^96x#%TQ4z@MEul6 zDM^g-YFmOtb=F5j!OAlG=@vqTiaoRqYX%XJU22`)i)yRi^MTGYsg`mp3F_5Rn0Vht zeoI5D2zyHV(EEQQcx?pkS2qv3;)v&T+&Qd8b-D1YN!n9iF~#dL^ZX{(;NY4y7j(E} zUSGpQV32~IgU5uWhEQs{(7{$NMqDB~(J{KPm2BsWcZJ=A*G26lzU%YT#q!4%?P? z@U`dO*P>_t_MMW@?_UaqL+vZ1Y`e*ayyf&ImuH{X#XnI(?zZWb5+#Q8UOI!5Bl=U| z&!|4>HlS9kSMpR2)_iVCt3eKyx!Vx(w-$Zk^_Q1Xc$*0wfA(EwwyWyrmMhsPLKblMO z2|Jrgxy^~-fd^jruZep(x9n%1JuU5+Oc1lG%_sWOBk-#f0LDDt-eLe-DG>M#&8m77 znRAM@YyYC5~$OEUzSUg;mIHH8C$xi13rBN21LM_Ls765*{3<(IW;1GoS>w+|gm)ipV)e z+R~Q+h8Z<*&%#3;9yLK2n3(EM^51R6Bm8j4fRWG*|3uk`*}Wuz6*D5M5&sq^peq4* z*pIFE1vO$W{9juXqTIf? zVPfbODM>{Uq`QWaQfW~-Mv;>4juFM6M363N>FyL^kj?>Vq-P|D&gaHD|D(R=y{_le z!w0TmhI{tftM=M^|JGVMQTo9Zw<$prh$*Me5$IO7yEARVQ^hlJJVWftqPkIhx4RkF zTD8_?J3H;+^b*h0tI;M|M0FDVtJGf``;Qm+4I)PAuddEDi?KUl%G8fnzL5DkLk&JP zCgijJo>%N`y%qP6*3Y8#Wj|vWvVZ*^99#x4%n?)AYWK= zw$)1NfMO07+&}Wh;yBRtR<8kd$iKnZEv!M*vdot+lg1YmcN?O992x!$1udq|XHF`( zcfiJ*$J3u`GVcD-`GuR?^ZtB0Uk5g=cz;qWjGOOXV(=~o=8W%Js?tHgA=dnx%a0!_ zbbhqx;zz$M#5H3l0u+bzv<|Uxy$!pBbaOGn6DxxhwLGjva;jgKoI+eJSC~d;YcF$z zv+e0Dj`uI4vXh%_`iq62Q1Ov;I5iTKJq+WMhS0MynIw`g!*iZ+fh$8Pb!_qaI6v)Ls8Ih1K=e=>uL8-w3tn9$Bfja3xd)T^6VTTHbD z-`9*kM=b!&1cd>Nd$rWw&T;>e-Liw`xZSe*Y@YFC`F(9lFbsisc7xz8)r4_2D#VgJ zCpl;9!{k|DsP=uIvPI)lUFUGo)p1&D1n6-7n9* zAyu`yC`lq{24o^QSlu8g$o-$p0bCsDi*1`;Lq9hux-{D_)Yn|yjDtp5yUwl+dh+q6 zc~O-qh42!P1led5u;%5Kwjp%7LZ%G+;2h$tW*`EJco7OgDySa$Eq-mQ((8F2w2i~$ z!q(`+&5L#K-o5)eDoSfsG9R8eT=KB4x}`ttwcdxj-}`$1w3lC()!%kcAuQ_m?tvr` zj#4E?oB~2ogp<;ylu;s78fP}16kiK!+XWuic|$VhQS9u>^3ST7zpe^@dl3>O{b(+( z!0dZ|&L8{qfBw6V?mS;6biJnF{||R~i3Wg7gnT6$bpCY{@%MS8h+Zh7z7#C<4sfvQ zd!S?UZbqc9T*!YA0RYRNNef(|rb?jABO5l*yuUyjXWUJSq&Og&Ml6;Y1(0&z?B)dY zZty%;!uIPyU~ute7c2hc?&0acU2+4d{kpA$T=rap%AApRhuH~B#z!XieQXwwaWBbj zqAJ+V3WQM!Ly7-b;4f4B^+MnRM*lSrCE8E-1Xa${TJI#fSG9AcmN^)|F#FG>qaA4` zF%0}P4U!4LN2OPrdiCy=);SuM{nuCkVsE+qDIBh-Qs9L}L0iuTZ+J&0=Zw;dmc_55h^-PI7WFeT&rE5xd9u7IkdpmzPu69nVO! zghll7MSOJL&o(N5Q&<4rm8zjNL06zT0sv>!Qi9<$$8{RFzPCZrC+dg}c%jn2j&=9% z>KTWJ9;BXU7-_a;gycge`k&<;nAEG?A_puX+Z^KpAL4+hZ{r`m`X6bByzsmW^32*ML458Q)oHfb|ejdGPbEZAi4F*IvBc@A+1ZrFfd zcA_$!ONsaWkarNf;BwVkg!K1v=HH!8w%d6f@)OR!AL5B)hw{Ze%FDVjl=RfD0^-4? zXUxmM%$O7mE^UBpnV!}S@zp_iy&e#cvb@H4h}BsyqhKq;+i>c@iH{%DiCq340sXJk zhys9%v%pPB3(D{Bszjnad+HAQn2KgAfF(G|QPF16 z+C-7X#D*I3W12we?PF#(E_?@GjfEWylM--F9r!ZWB@42Jv3vIhe&tmBTnxWUMF0(> z!s^^%d?!m?8+1jPBfBHw-bFihRxVisYiY;}M;gK2r;b=yuPzWg+uW*D+U#l+o2`9s zwQ-=P)L*OADa}fp`>|NNfU_yxva|T1&E<+|F-$DN&l+5T_E+Jt^-SaunT2wmcQ`sy zP5eK$;|GPrnWtU)-^u^;nv?)WjVq9d1;^>vJHPUFh`GH1tgYARdbH2PD zK`M{g4>(@?c5xACBcvoTxW}jVE57-9d4|erZ*I#?hkGPRe>8wE|8~g&L|St zBbw`9lhX1u9edp-4xHp8BYAm+{Mummhj-$jC3)HThq8aV0yZ z6hfSh`vhGMD>8+t8Q1d>2n6D)L*-7I7giLSXPVoYyYB6d_?p$$*4*_o-+hFPMLbp+ z*YO*CJHRZIn3&`_vA{MtfNxtQy?<*K-keIalEV!avuWth$=9Xm%A`)c?A+5T zoK7UH$15>^YLza^-gB$beQ99@aZs7TxmIb`<9E!aoh}+gm60n|r#a?6DZ0WbJoR?h zx7TCEZ?7WLvaG_-$W74`@%$_yRLnljxxIlzrBr{lac06VV>0Y`@Rq(T=b_y;}E8BLd357X`JQcuvrFH@z(CfiIL-pgrzT%M!nag z?}lZ*DF5J1cwPI*es$H3Kn{&%28ANGn|xgWOoc3B|MR-tEu8ftRb;c%oF`w=CTQah7cHSyTKm{`ITr5gAVkukIAy(Yy0ciYD+#Ux{pp>ZfBS*UPWg zpER2Es4SXb8IX+;Cr?3wrV z!fIcwuE9+T$G8s3Vkg*tC(<8-oT=y$$OPk3k8*?3rBKtSQCBa}rCVUJnsw84 z@RU+mLJWgg#;g5mc)Rs&o^4kLFeGR_=46q~-Kom9T09e$*gNr4^*dO~x6B{bR(-!i zorAQi_NuCRXRez+`k0;NRhM2d1y;4COwufw&Ek$&J0kNpegS_f^FjN2i(?)omWqF8 zB}Xqt-ZS1ZA}n47Rq+V{bP@(?m$rkEET2kUc8OUsWJY3TTs*_UjLSfQxJ~2$OX=W2 zJ_J@{)va1Q9>$uja zsCRslAv|A}34I&>gy9cK`eXP|+6O8mL-UQkX(!YcqkHjD*%!k67dZTq1^Y*S16d@E zT!|;o6&G`;Dj1ksO;WYu`Pu82c&)O0!j6>2kZkc8dtV#n$7bJBr5{u###f9xzunGY zU%NG8NQ(Hs+^y>#Hl9p7m30JQ-A3iK?Z{m^K)R>vtGWb>6F&^>9@1 z`wMvy(oOztSKZ0KJ~>yxrtOREVQG3VVakHFXIec84wHgrS;oh<<7M3e@J~w|7Z_*> zyrmrCT?Qrc%XupNHYS{hqh0ST8G4!!2OgUhat_eDao*zPe|opjdo5rfjj_VDW;sWH zu+2$Z6$W;$-!43;%v2GkGxAEaTuQ}Uqxj8e0YjC1uBEFo6G8oz5eSeUEG;M~v z2>)e}D_FoF=?ymD>_}N3{UKK2s8y%4I(sUQ)3eKg$GADzS3sA>ILA~xX#4Q`0O#x$6DPw|;w)y2zhj0GrU(>0hqv z-yI8X)y?tEjs59BzCDRa3CsjyAFugu|NXzzL8k!bZQGjR{mIu~4*}$$hoA?H zC;Z!${)?V%E{JrR8^6+D`1x}ZP5!`8gwzv@e`Dm2_fp(|rKLVz_vW8W`9(E02SnPW zX+|jPUj+Pn;Ccc;)hD|y{2co)|H=aahQh?I6#b8xFB>-bE3BJE{TlmE(xK?UP}DqC z-E99KLI6mRUdO-|@Dwh6`X3A=)B(JOf3uD1f6UyIiYKMNHul<|RQ!4f(gY0k|B~u7 z<)K%a|4?LQ4`^!;mD+*V2KLV>KxJ^dVt7qRyGH|mFh`jjkjvnF$ga>g7uKfMB-I7$ z9fhdH|Ue59vh1RVe_=d?Z#6%VzP-UykF`JN-TVX#TOe}})>Nzg-3*|_7-W4Los9Ia z>(YNbCvmQ0v0dNE9yEjm5DCSsfum<85o$CY^dEEGJz? zRM-ou9ttW=_ssE+uQ>2v zW#q#LZon;6R%QLFjb*f<>uqgAm-|95t*((;)=_~Zfkd|D?Jjdk{?)3vP)nc;rP}pm z?=AIe`Jj#~J`x4kM5(=QC+mJIJB5BeVwO;o&{sH}|tte-p^-DU? zF@cLJ7n`PLSP+f`K=gLRd3iO69Kf+E`1w6@Kr&P%)7sTJ%scVJT|GJXTDg6B4ZS~I zTS7_{F~clbzV2^76HM8x5BV^P~1TV}j< zv}eVh7Jqx%!g%_cyKx4K`xe0N>Fhj}j<6ejbGJcQJpiP0JJhX?nck_3*Ai%3L`_lk z;^;D|(DEu!_FN6fhw=72o${)*iGy?kK-y@yj!B4|bj$a!?iZ!f&`(!*4%q`DGPHMP zzr72+eS;Lq&BI!Mh{#+q;E?I#Md6ivJ0GhgnvKXT@HIpC*=M4R z0h60KQw%FE`i7o{^dWUksz82&kmu=3jmtIYt_?n#HK4q;P@+0-q#o0bDHYS39^rt%U#E)iya_1iUo;w9C5g z`E(%$SP#(>?I!q{Iw2fZmf%I+uurQ7uAfakB%X;nEv5rFUkM!19*Mn)x40OaF7?<8PY^;0s@)%I7UpUQ`!L3(lLr}*ex$ypt)|4CZ)lR z1bffXPT$%-@#yt3dS$oD|GL{J5y<9Q0l?fnE;eQT3I|EmVfHhDB4!mBg;b6Et<}@U zr(4m3#%G5k6Ka(0nMPYDfOk>LgEl9{Qj+*A%aZTiIXa^!z7Tx#aiW~DF3+*K)RAt@ zVw}~0(71I(11fPn{rnp&r|LXj#_|pITAdyrtOWD-v}fiO7EgI17X*bE_NrjlFi3}B z3PuwRr#ppN0B`suHlQS3$V}4oLu0E4>$1e6S8W;&7Sp!Q4to7m>nf=X>}8CCkDp5& zKS-)?IqJb!2=kt29!#B;#@<1VvQ5naElsX*llCk@UC%qgX{^gmFeeuS4a}}rHv90* zeYs5Hamy2-nJjk0uL0_CEMt6d>|iGad^yTIy20tJs+d8sIy|`-tEbE0OZ_9m*z4d~ zJm%*5sw|-TIk%tdf%QfVyJS!mzvg^V3vp+1c)NbZZ1Hb;v3fnwhz&0vWruhwX}zZ#o+Q|Ey-`H9qcW@1N3wG}@~HlOE6D zNQscx`)#K&b|1J-`|KsCHqCq_SL+JcCzK}ddja*rem}htC_H}*gsm>bF+BFF$tx|n zMmss%aU*pss}gl<7z?7gVQM8Xq1I3gk}i8J`F2ugwolY(rH-90hkjc=wW-nzaRKqCf6F;iEcc;R*y7X?HRD1~j;7C4alSVCi6Jl<+xN&1oH8Abo1c8)_WIoE7}D#zTk15koShch zv9$R)go$STTG&HopWOV4JIK6JRk7Vs6DK3O&T>&)xs&O$6JuidCPUzeS-efu)OMzD z$Q-@S+ux*r57<@l4o}M*Nnv3{`98YSh@G6t6l$mi86fz$&VDpX{nRW-LWW3)lF0nE zUngf}SHO|?Q4*vLCUv@5*2z^nuZwrypy6Rkj4oR8;ZW9A7e>Q@s`RY>^r0598P9%i z3K$*x3DvymB37yNXv{%VM5ApZeXuTBMhLE^g+P)!2D zPueZZ3IVCOA5#Wp`R(Kyck`10E~zZsZl^a%LfkL@EG2-rgP(U_OA>PlWEkVI1&Ca% zQ01m{s%_8_lX?r<*>%aym(CyXuFuDEg52E;Sv$Lkp75Vr@&(LIP*Bx%vsXpxyIxbM zByi_!h@JP`3!-%ztuDC-OsA;iGjHNh_3Qk^qp3cj$@?F#FU}CH^J1dvDz}4DC}qfm zwoq4ax8Fq9UByw>sY__!Nqc%aC`Bo_)Ni^OYm!OWOlXbELPlB**sW9FJkUi)WxUL+ zjr0-QQ3P$?0a@~Nc07jz-AF3+&I397S%7lSHB2ZsT%h>~sCLfi!F@2XK?8UIE&Sxp z>t$x9_J#AU`)>=qO;eBS;9WFAlC&i5O_*CtiJ!Se$U*N4%HewKOO}pmqG?jUMbc4L zKxEvxGmh+5kwkCs5Eroc=&w?)2|k<(6N9B0!{0UCqnz1$)jYhl7d?YX+l+_Z zWC!FNZV$ql*4-M9CKGy^q-evizi8Yce5%;wzYQ$W!w0;QAn5|2a~;^f$0TikQLOah zzLr#&{6mvnG-wcOuZjP@kQD6>xd6n3DF!-6jcb-{j#E}Z)&xXmN#V$<(?cYxiwtpn zV-p3Rds?MX^i-0R<1*`ofL?IwN_vE9$8@IOEP-*%u3b0bix6~I>}p-6L@?O%6%CEG zyG9fZRr2xtbyRQ!E9?|dA=gxHkj%cB)ZQnB#ID!w2c5=Aov=o$;@((2sTi6j3DgzR z4{gH7Edfp#a$Rp8P+~8Jhs_XRuGI;HE++wB6vn~Iz%K}78+L40_xfdm5dvV_B!&gcUqY2NU3jv4LpjXSxP@=q1_re1Xo|b?aG)WNJxjh7v zVxTh~jlvvXo6Us?GsHf4!%uYfqSq$2&B23Y*vdh5=IP9Zu$C)T6yp&|+v#+DsbNDc zK`E0N(MEBY&LGo=Y3o3;jZ=dn{O!x$i?(%Du--}x1+XN0MK*uWvo zs*CMIGZ6{_l?ulQN~8HhEg9J4qys{93AzX&Lg<*AdEL@p){)m=d=M=aqOLNZ@iu97~Q6$>~5fvOZ5KPk>YQ}Czvxlht; zche)HnUqDy3|@vbyRSX70jhDdyF}So0|gmGUpJrd23!+Dn7U#F)o6x>IF`wHEhrcb zcsvu8;i|A;--?JnGHb(ZCM-aGaUWC>?*9Cxvn%VMn$e(dan$T$yo2-SbLxIZPgGUGW4up21leXXnx zox!*4%9}}OUOBhp1IbUY`jP@{bW-er&sU^dc=^d=@NPoV4L*u*@dQ2njJGW@Kn*hI zN%~Bp;0iXFqIwQod9c-mf~mBDGdM&aQ+&}Ix~!nb>@UApF{06^ypf> zrW7kV_t*=u4&U$|g^epyEn8wTH-s?w`bij0_--wi>4YI)nu*pPE`&uX&#Q>2-+312G zm>Iq^qhXMTep5YkhG5SXX!vPbY7rrTK@BR$^_KS>Qq!$?>}lXu`19Q0`>{`-6m*}~ zrQ}S-Oghak8QtpJsaFgOhPP9rSW~Yb?me6)8Z?%e<}u3i$cvbZ^q7g%v4bsZ-Yzft zQKY#R7E+Oi`N_;DAXfiehs%#tDVh;jh6`Nn{M$O;PL^ngo{D&C&v45wD=A7(p3PGw;{j*!j0j=6q?sn)4h3A$PYW z*=YOq7AeGO%L^C{S=Wq)4oH49yRb{2fkbF9!s_g>%4h4hqe=)5#_^<3l#ROI zcs@^5kFLUx&1w|xN8!%T{2R0Zjf5L0MvSj^^*iwG+UXN6Dls6u8U)UICAMN1PSuk6 z7S0Zw=%!zrqs72vi&@b5x!tWVOuq?}S#g&mJD~0}n@Ngg1YOULR^)7@L|XJQGH8Aj zH1jMf*22PsgYko8Y~Tq@qhwajub(S&^J@CjP=IhoJG{*D@hQt#mYX@=s_IX02kZd& zLlJf(>#9~iIS0S8VvbT~U?oj!awmCcLxKqN^r+JWSG+t5X>dXKWm#z?9e#c|{a!iC z3RE$wqlCq6HXg|>$ymn?qWqAb$35Wge32H)@kA|Vpwb@VicjEL+Pd_lWIm|Qo6E!c zB2?7Oe8WgH5U5aW($)(U)HtC_Va}GHUS@}0+1=)6n0`yU^_#yvKTGrT+C`ZG7e1OJ zHQ!D=GoMKl8kL4=@3h5I*rU3Ims-o_;#bIuAd6Vd=-4}RXUFfCCY*Y^g8~&L5sx5V z0E0V+VN9avxY#wF-XNjuFiuLmr5G7g1{Zl62DMh<1-ycy%~em}Wr~!Q7mqA(yWc;( zOode2ut)!*L!!9WXf6e77PLu|GO32uf8cA~Mvv`XEJ#igsid z-!)V*Qi_{Ht+89u7zKyRftK?ho5X^W23f*3uh`$MIB)a5P=2|}*sOp9Ng3vQBS5wD z!s~Zxc|kX&cnfUv6q|$cdsqjmq^4Sc;u2DYPB(oMJT^TJd)yi#fYK*lK3-V?U%F9% z`g{%dW?yO@$rGvKCgJ2#_ic&=AP82Q@4a1+zAt0gpM2%B?C)okHTe7O?+fJ2ejJln zqGPwwkeajOxaW#~g0)o+;VJOgcRl>%wOX!iq&AaR+HbudBL)$!6AmzMKf*^&+=N3Yi+4beVqP49n9oH^ zNnVT2W+CV23GJog?&*{=6k{BiIJ0D{4vqVpxAI#^q`=$?K&+!U54%Bj>p=z1UTf;g zDgzdUORMA)yx|J>U`m2qa?%-}QNObj)QlwZ#m0Vt5?A$A9C6${`^Nc~dqPu4i?qq4 zqj!@!kOVXkzbtMx;I;fQJlwf&=kA7kT1JOdRI8>yXNE36r+dSQ}p~O2kt3wh-dO> zvA2!ZE&-DU#2K?HOJCfBpNwN>0o2s)m{Xn!*wMBKNcnX8dZIuHn)f#6n)r0x?-71x zx^%GB(20#0gYYA16Y#}+S!f%^PH##OdQsbOj0QRui!zh1@#ksxxsYl`!Xei&>LJsj z=w}g)JNb4QXU&d;ot!8p&tn*9x~}nSN#|Bj>qie|bwY#QcwugmEf|MCk9mOjZ?<9ahW zZh*e))J%w25u zDfs|USSOh=ye%)@F>P(jjb$yOmrwt|C~5)x+ci*(7{d4| zUIZGB$kLb=v^WS=I6)Rem~mS*MnhyY5F4es_qgKrNyGu)zRe^pqF7R)l4QB9SDj2xZO2K zJlyFh7Vz5SJPOpB?79LW|Au&==@H zy?A6w=42*f?-!XRbzJ+oCvKNj2`wW|`0xXon8&hsQ0|~z-9w+3+3~0LiITjf*Wi&U zy@?eT&-Y@m;IS(=?O(zkZ~8hFf+8)Ri^ZBkwIfYJNEm8c(^~fp!h3>F)uUjh8rRp! zSB9V-9LK79wINn@ZeRvNb4fSvwp(+j?{S33@Np1d5|}{A@>6yUw5?0NMo3h9FvA-6 zmLjc)rm%0_*}SP4&dJpm3ciR1qh*|AuL60A)wO2le|tjc8piR7NPEcczJA0eRMp0< z2A-Vs2 zKB<*{h81g5$)3)dxfo}mp3bstt7Syrj0>HnUm2O@*AF`X;9~E9T zE@RyLvZ&%Z>RHC^>0^~gbFFg@wObCcN=a? z2h>Do8ivQYD{Y(xFm0jjSCc zc1`OI!kSmBRQ0U;xt0tH7hl4}9O~(;7r_-Jz2Vof%jdvbcl~xwzvxwmd8a!VOvy%U zX=raJi@CqmFVkqSo7GqqUPv_zMHWODRmzJS%!y3ZE~{-97xY*})P+lp&43FOKO@Yq zY*mbF;+SS%AA50HoHg~hHvRrMQDt_Tby!(Di=`G@;|l{MGJk*lUi3? z9IE1)Q<~?;0b6}?Ns}q1m{8+|DXSRee7`A6*=E1tru7@#jacfF>KQ72qm$)FQfC?~ zihlWj0cD#+;gNVG$xJyq+_ap8PhwFr7Rb zpER!R@+}QckCHn4eHuNZY29_DzmW>vyvT|PT5u@S$mCRFx`HqWbKnqISl%o@nX8Xw(}tN0e%q&mx&kr(`VDfI46Fk`jJL)zMzGxz0#sqRNN#>@@e zOGeNYHt2@^`#|C6)+^W$!U28aYVx{%h{8pHQ}#5|qt;USiN68~G?1Z$IR8FRn#-94 z1!8f*<$px-8>K7;Oc(_H;g9=LqNQ)@Qtb$^>2s2wWRELOJ!V91x!#Efpbo(IDd-~k zhBI8gY{X#%km~U92N=)0@1!m4KTlt3nG+tD5`#NdkME&WPT}S4A1#Z)>+@Es#g^c)S zw-K@_EBi*3P13cVh>rUx*UrHe&&l$;Q57`{FOMxeE zs*Jg?zML#k2^dXS9M6#+ZPp$54v8k6lyd!Dt~wdWCf08zEBAosrWvK&glgGeax6G4BqA5kVu$=iHoH}{$SO^$RMA{2gce3T_eB* z1rh1TIt4E5oh}uU*aDXbPu#i&HovH^)1G}b(%z$^zhqT^+ZNxMqCGjH?n&ESq=kCL zF)Y=}kcfXfttU_={`V{DA9mKJo5H2j_lU6QcJ9v3&#a)Ec}9xBpB zyHNhKt4Gu|2}Hs8U|7joJNZd&Eu5=Y1Y1BXkVx-x!t%|@hPl=njJIWnTRZRbQ?Yq( z_$CqG^?$BPWFrd=@vWpkiJEl#!iR*=}2gd~Mau*reW zI@41=2?Xol*uu5M5WeAKS?#FfVDKDupU&NbyNbNNFdZx%Q!T+YJOD4E@*D4*So}Rb+cm4deUjB5o3O!O}kxB2uQsUH~)vAA0LI-92TT4cQ)n4^^%s?KKHHLM|v zPOCSaRP=GqyLYQQd*;ZnQ9;PlE`t;Gk1Gih8Ir2cs)e8dc`wZ1$SNa%ahs%x$0M3wp?-o ztcH=pC|8%Ex12}!;hNEvbqous`K}{hJY%H>QkB1U0%JQ!6x`p_uoE;+h(4+*S=h0K zO?JR`Y5zwqr2u$!rMFsLkq zYe#1(=XM%rmWyYG)>xV9m7No55GFBFyTFEWG>CeMYS&y#o8rr#bf~Qxu1NLHI-Y-D zYZOhnbAK;9+J{aXWk0o|C+IgH;#2M&Ys9JVHf43H>JsVl7S<2r%LjO)TdyTixEO}2 z@m3jV*NiUAtDM|%U5N~_`Z`yY#~)g|nr7_+c)Ix@q=Qi#yHB-%>Yj3FwbZXW01}Jyh-w?NL@z(gPfgr%Qnc%Qo6c`lqDfg zWO#K}SW`!-S^8Pmnemug)BF!x zf46iVbd2H81S&Os@0Dw>GjvkA4Xb*}dYLCw) z*aV}82nD3uyzwG13odvxq^lNCns$eH(&qThD9SSE7@+!K94*wIBSZ5Gi$c8$Ws*AssdSc;J;<%#%PIZH8DH)L#U*+d zI4dr1j+_Y5YIOPW@)%eX5{w~xmR)v}MhT1mOSE?|#Rw%*F|nK|Hv8{!%^a6H{Wjws zb|8Bm_EoXO=#eYZGE#H$Z%sYvB8=*jZI_1m&KS*#KQJC(Jjq1^es0UF$`#y%Jo!Io CLegmf literal 72431 zcma&MbxPWRJKn7r&yBzSyy2nYxyDM>L!2#9Z@5D-vbV4*)-MD3T1At2D8J(Sg) z6!l$6?Hp`P%&m+`o!sq=NsZmiO&}oLRx8si9SGZ^13w(GfM4TD;T*WVB-V+>Zw{f# zfBdTZcZy+3x3*DMwF@O_M7eD9ap@cVLh?>QKk%=jpCSMe9U%ck2tf9vozh3?aP0~@e?O(GbvTpS|w3#-XlVJB4|55A1O9c}IuG2eVL)|Len~ca%@WCG#`p>$0xvaEe-TQ$>=X zFX~Oo>5YKzD_3oMHI>hM?mVi`+T`gLcX>82XX9Qu)Xe?+2{dmzQeV`b$RsiCE7b?f z<^4OGPuZnU{LOwb$?N#E2*vYI5T@$UPN280X(}D_08!EsEEQiZLNYv3mF43Jqe zYb5o+xUNjlwz&hyIeR zp9cJBi3kk59o6iF^m%drx8)0wjAggWf#7Hyun}V#8rR2YH8KIL6*{=5&f9kCfF>0i z&r<)5%Ch|{mQeZ%zaUtMkQVNwf=@a;05LFg&mfLDEb)kK(aiBSGT760yjH9Ykuxro zPCRj)?U;i0AEFPN3e!9gl<3Hy?&#dIuTxkU|BG5964V+_IiaE4U@@ojuPEKJRKwD` zp=tRm@^~eibwk6Vr&CrKkD>F$r52w3uq1g5i_`p!L{7J9QJQHQu#y4gq=qnu9L zxnaT8j-Y7S{7}2`qIK;8rhs9}Yf z`n4dziksY!`5Q}BG9&~My=iA~V!rXD1N0G9IH9^x<9ag|onaxwZTx9lcCBkQ!uiHzD}c={rpl1}opr%dEMyhlKjkJH1|2qJEv5d~+SzAKJ^g;*S6afd4OEih*lU5pDrYz;+cj@-fAKNn5*A)6tf4Z2MWmQ~yT_U6Xhe?uhM zty)zN%$Z%fC3Vl2EgNW<0ZBl82*BCs2Q-p}-1L;h(-(iDV*!+?IOC8H=tpb1_7$9I`@X z|Jh$qG22+kosrTZF>e{#YN|- z0@?moG#twv*%a;Ds{$!xKde5bRry`qA^s5R*%|}uP*ym4V6O!-g^R_^`JXi6b^)e; zoiSXRa4#e~zf85u*LMONc_od>$ej-6dTjK#;yjayUp6@lZ4H_C4rf*^@mLhl z&NF)XUo!3g2+?D$tH@WAsy<{@6-r4Z8X>k~demQqZu>dIHh&?Gc4#7+{ zZ&C!Fc$l0U&1}}b^=Nx6PB`wQbS8`gB+4*t4E&R(lP)$v$!$(hZ(P;vs zIo`OA5)_!6IoFAj^iNxuzoQ`N!xPAqQ?gE9Q&dStSFFq~7f|mL423~;^2O?#6xPhz zyG{4RON-_z#c~92xAeq|ymOBuaSeX^=62x!U6(`)7GFxlLFDW*odV~Gs7b#x>e+gR zCGG`#>5bN9D>s4y`bWJYGRO823oCR~zoKC`!X30en;VKI)C3$2Krrtap@rM9$<<-Z zo`&^)1-!gg;>zKtomW_=nX9mIB_ePnfK~rTuEt4)(8M7YX*q*?9GP@cxI__Cl^PTE zUew%^wzTPu0QF%17NudLb%PTD20ifb@8Ss@|8boJq-$(+TExG~T=8Li{ND>fphYn& zy#cZ6VU^$l2-J}9F=1l8+Ts4;xcX$HnO>n-$~i8!@t>Q^0sq|J`hOy=!kExr<>io; zr)XV^ZCe9rOyaRou@1~v@{*Rbhp)D^9hNXDp(6*Z>e+0>SPi^yc(`MCm@NhxP|y`@ zsYBr8qVuEq7!@;(&p`*lqJvq@slx!>5@Vnj0k6dY3N^anV>LJjSXHJ=}3C+Ss2* zEz`HBmSX)XO@*7)FN+>6EQmS~u|tEhwNruVlO1-bY|i@@{Ww3$@50757>AB4+Ne;D zWLzJ~PXi~#nG^0C;_d4H2+gd4zY0|^WW1X(SwXw2|P;sJfV!EOUDYRxBl zEZya`piz=H`Dth2?I}Q+sZAkoElm6ttH9QW)|?)tELz9j3ysrO>c31JihF zb>WOWbb9l>RyC^XBHQKIxLbCCI~tFXfxJae!Z?A+dCSwGjq8PfY{proekuiLh5Yrh zuM)R=Sx`r@kuPTH92#z0JSjwm$^XU4^Y2>h;D{8W+QLzd!NLF)=}K6@N7-8Zkp=yb$6149$V zTek}v!ti`J?ATTQ@Hr~O$&jABIO|pd(!f-Y3jSRlZ0Udk!g2*1Cl8l(wwJqDb;ZQN zX95T`a|5`}juB)VIP0Z!d49PWC_HQ_1Yz@x@L8%JC;{~tkmvqQuKe#=z0~ZAW7E=| zTrxa4p@8JlplE{x00%2w`)^-pA~b0trMaDBJDa51FO8=|UsWJg3zS?y3#*TT0lh?j z)(G`FF}FWP$~`wh(&c_W%IYRGf|5tsS>et%^IN{(f z`_5F47?RRg#8ubft&4)T)>)yaSLFB08e4%xJ}WMBOy36IXj+Px$U9EFTG=mRWQ{B! zk3XxcW080j0hJMon3;J;b#of75?<4KMT5|=2pmL$!W*cOLX^Br0q~OGFah!wb0y09 zvjfPtQD>f0r7&9Ai4v3w)0YLS`O|=|_MW$w@7V7bNLPnt|7glPJ}F z{3?1|3`d5cwm%YYi&$N1T&kRX3kNmI+mFkuoYo+}ob$h4+O2EwGxyixvegcE&rYU@ z@*;qC?=5Fd8e{^PfS~2RFpa4%mvMxI zJX$hhd=QgHr=C$cim26RZBuDn7D{i@YQq$vwS7mTT3+GKPUN`a6Dah(`u?^NGM-f3 zmKWrPm&xgSJFsz9N3J#GKT+#+vW@dY6!&Y?o8$&V#-GBnSn_44>XNUQg%YsIJ#5of z3ZXh=I>*^?3Z36*gTG3&-iHBt5nNmwRfdLDE&jTpD5cI4imBiVgoulqg(ysh|tGfVr~48E$)sGWv>hoe*`K1sfgf zVk^EyRKjj{sxGHJiUD^;fvLEoZ60S!ISOflPO$9Q49_hxFx)rb_k4DlD@46Kc5R_@ ztNmn%eg+e=0x^!=T{%Sjs+-+S>(xED`s)LcZ%Ym=oxs%GCIX!UZG#8itz7*6`#)XF z7+r)D(t;b9E)H~=3KZHO*V@m{1xZHMLb5jqczqxjd}kiEWpKWR^VjTXVVn*Mt<-c=|3?9cT;wEVa^p{<5XZ+Q$HE& z=BgwdTm5uiiB@XOBAb~t7&yO^!2TOhJb#&uYt@Tt*63X=ouPuDcJ4U+tnmwIU3tZ^ z^li%MkS!b9XKhE$sMqH3Rte+ZQmSG#JT&h^?IlTXAIHreqa4R8XZ6KWkj0it_X%xK zZvVO>@7Na0Y}X~}+7R-MT%;IquEY6drPG++sM+?#p*AqS57C19`xUG$kWp@3XAtb- z5FTEIDXyVp2~kihc!84CXe%E_4b&jd*x~31T=iRFHC2y=W&)uO~s3X5XvZs711nvIk-i zV3`&Q9lmw`vO5pY&m%#|EAE|!;n2xv7xlt8+2!NrOfuD%J=?5l0xWO&fI>#OV?tv- zDuRH33^ErLm6s9~{U0LoiGyT%$MH(`$zuk8G)gxpi=C0~#^&*;qv~>5V1>(npNFsH zNnOADCW9Z3iSx6kVSC%K_GEjvqCB9S6jneelQ0WYyRhFeK_mi~k?>;3_wvK>qUCRw zj!D@HGjvJ1aUr_>_XKS$|7zI0j4~7@bk=)rh2Oo7-Wz>G;5OW;n@;fIiGl+ql-HMQ zx;dl>w*WVfxmoT@jGt->ilI|sKanhAALCBz#%wbYkYC(Q`X!4G!pQqY@}l-ROvFWh zWK%@blu10jM~=ZC(ob5njHY|v-~>CajJy1{iNeI7=85rq zgdysZWJuHc!Q?=f`voh059T9a3&?(pv1g;mh3wS1{4m6Xy8aEHeeGSK@q7>t)eX9; zKDkQ~#za7>x}p9EZMqFfORI^^9i@345<1&yB$kj`Xn*R^56t>pL^th=5V*MB!aimH z3{hyVIzq6$jfzVx|Ij+r*0y^1aG7=R>I0!K;gIWtEwb>5R6!g3`Y8tSyY1dRVqNQm@I+|NcBCn*_m7!W)hJQg#N_|PE)1Sy1+n6R?j>e+_3mNM86 z{Du-kjz}sh{u?q1l^nI7ikVZ5;D|ngfFNNxY0y(9L#%qaHOigMa+QsP86~5>k&;~- zc(6#hHoGv2l`tJEt%rk%#(<@)+>gt2GyPXB7!i?6&hCvll&^e73~>v5x_=tvIGM(M z=<37jIO*zxLi|2Qe6(w6qyRCy~(bqT>I%!$?^kmxkAC~_7E0nR2x6`%NV*U@2~#doYsD&P2vR9R_Fc7j_1ylY*!9?lx_ zTi$V_N1iWCdjRnDgBQ77Jj}&7RM@w!X5r3>)2L2U{G)pvvQ>0FLg zE&H1O{Wx#wsl8G(=*O&_m>Md|$1NSptN{#Nd5OTpnD1)e4~0|JB7b1cG%@I%LM7m$WHM)PD<1dj!5f&2MveatGc})-y?EMyOYfWq z8XL?A0^;pQB@Px)gCDBM&1=O>v6R@4W|^>m5Zs$!ewtTYx#A^h{i6^&%KMs&1;IFk zm@L!v;H@CVd#mlu=L+T28+WPrh26!+VUM-`JV0*YW3V91@Da#4cmJv0Rtg)jP$cpa z{zC=rPV%;gT3`lU#-vL5+blM05r+zh=&fA?kL&?Xq(HN)Zln427#toE%$&cg5C(C6T|tjk*) zsg40|>_(s{1F!OAyptC=Qq$Vbux>a7#a^AaKvkL;D-WHm;}PB6r1kI0v{P2kvE&QI zVTcHz9pBED;x#V`4LV^7sIi5?vOQ|^n;Q1{NA{k>y{bd>Z;QeOI1`GZX7g`X8Jo8_D zI*Y$rEH=l{O=3NDWWygNwNlqlC%tP`{QR0+n+T3cNtJDyBLWBgOf!-esY6Fpf9>BZ ze&3VVgAF+VDpAM+FZk@?=1t3`7Ip?LbiySl2!}8%eSa=Vo-DjSNKC(6ZFgCdIAl57 zf*|cqGYORS$eIHhH-*j07bd|u%RQg|Og0FkxVAj1Eq9g+lJ}nCxs5MyGa8v7zj52m zAFMqe2%`8jZ|?72-8Yf-YVk!uz|W?ycu0Q+xSt1ZxV`=5|FVOp@a`Usa>rI#=KK0Y ze|T{J&Yqmb%{@2FURM?ri;(3F>Y^>;`Vj>R6ibe*HZuHB4U=Gm-=g~Q!=2+7NwXh) z4AB$1XpbKDkF;*Dzr8#-&O!IFZhqSPFpi_YlZb3!VJW^W3-((;>oS5#M0M2KCX;fe zFgIUs`4meJ-Y*YhdG*eIG0yt>2v!5y9`_2WZ$ZppLp0W6pobKl6q;Wo*jYhSTkhdu zqg`v(Kgw>=T2oR&2&^PFaZms@am~puSZ}ELW2uoNGCTOCN$Z9S4KURd-5MPKNM3O$NEnp*YOf)bb_cC>o?)dl+^(WDn<4K$(Wi)d+5Tut0l5Gm;ez)Z8tq zDWd-tOPwTMf?TxW5fRV%j{K_Ls!!d;M6|uw7EzMU7qkGLFzQjp*z>mbaya5*?9dRq zAg#0vAfv21igv@M2u8DJVZo-&A>`>9%MhSwvx(Iw^Y2sl*qWe|Lfqq-(<5<|;R6~G zXlgLrtl#kl7p1Yfzjq!Nis99^cVwwobDrG(5|e;L$tU-+^I4Sk@ph#hUqn8^=e=s3 zt%T{yUjpA;_hC;={$hE%z1x>lx^5dZ{hh>c!<;6WG>{>=IEp&^w%N~T28}=$UB3~JQU}1V*oM; z#m((HNsoLCnz$N171_R8&!Z(&^_yKjchycQqGG#ZWPVe>Xs8J<(pNn)$7k8%F=Pq> z&%o<9AELQ0?M(ATpntt$@2tigmhsWRy8iclOldJX1hO6FNqZUyX~CjMyLiUJ@>jjc z_rT{bh+8tH$MXCQze!o#IZos8)m2!2AMy>^H)0mYr=t(V8|gZ@-ZMtnVN-Ynw8U2! zUi=iXO_Q5Ek&3itRgzbfVtDVD$r$e9V~;yZX{_-|30eoE=;4t}#wW@T*0a~}SNC*G zMSGlhL<2I}85#7{6FGDK#HcGNzvk;8_a5Y|NO-zO!CRe; zL!8x{Z~pB}(8(rE-vsd(Qr6-2M1Mq2^95*jj_#9X2GJHoo^+3?`hvmf{AI*^OMl2M z^>$A9i<=Uvl;a`rc!o-3LPmK$?K@54DY`A_9%pCqpZEoj)5F+lw78YFfM4zMV_}_N z{K=e{SdgQr0blj|D#_X`p@U~g)nJ&F+r5hoCpWjE)=1Y9H(VA+P$Vw-=MucVd89J| zx1I;4kR+gHSP$!tSJXt-=;jwRmVJ-vI~Bx?5|4Th{xS>eq}t75FgiUoHxf~ZjHS( zc|YEL&zAZ4TLcz!K(-zjyDQ?3z%2g&qRjdnBp@GbmKlY2ILh~IZML_{EY}z==<`+* zwn-tydcPw*#`Y|RlatjI_X<}|Ulb9*Gc)ICh`cX~6*i0~O4A&lJ>6Es6@*u_=R2<5 zio0cTQi7ik-CHB)%SUn2+lFJ!W6Z6gP9`H7WH#4YmNJnoplfqMN%p_J_4X1w98dJb zh{PZI3VG1ZtKf@#v_@inXGvgFR zLapQ-b{8_0aRwqj%dPSx`UvxTH4BgU7_}|-^BDpi3fS&rou64=;5UwszD4qJ^}+nT zHm6iPE6EfF6>K#?kJ=ysN9D0Ncdl0)e^Di7*8Kf(1G>_>pQE*{;f{-?%n=KUp;D3bovUCaG5 zBC;T=7q{Nw9pX+xmpiFc_Q)`F#UNcAx0nXTrWhO(3k6?RtFF+KStm;p^|# z0kH)JUOfOfip&8pr7tW^Whkq1dvj`hC?XF|VniTLo2sM28OW$>KFpfqf5GN)WZ=z? z+JD+t8uqzx+wzu`1T94^@I43-s{_vd;;>%mj<5ZvlLZ_?MO6FU5q2Y?BV&Y4ee6f4 z?Uz#5LDaY{Uy|VM)2M$Ss)52Y@YMsSc3#rk-7=}$G!iVqis*IfmK%Xbsn@|(6XGB=hCoi~Qt-d%MbV5o}m z;bIk)psfC;LgVW@?2G1PR=_tw-QMyM1wJoJrWStIK=-(t1ow3t2gOF}grLjJj=dGp zhc1R4L8afgqpnwqX{WQ3QMQ;<)erd=FUZ)P3LpC@eWYY8vkXqkB+WoDEmWG zHDFf2d55O4hX*U?2#MimH1(prjJR1y2_0(#jD1tb(dp@2-`*=MtPh|Jftj&~9RJ03 zwZgF0V#mH5v&KJFbK8fVp1Iu{aB7PPY7>@)e`r!%fW#p4I^IaL9%p|a{}<59PsAnT zvrc?Dl{qCqQs`U&NTBqX(^-;T+>wQ4ViIKjJD-g0(8_4+Pl>Nicsf#Bpz6ZmY=Fqj z)I%Vf+;k1LUZ8Q^>|D77-REI_Ey_K$x*F2%7=*nC|9DDAhL_eH0pA#y_{T-|%f1&Zb386r`}r7+UlovUwf;jJ!V}EEx@2no+SnIk4D48W5v)vH>t28>Duoa5Q(Gxdx@_~IoQ4ffx`C-dq{QBQVp za8a*y^I*1?fWQmOvSM)ga3jG#7;_uLb12tRyE2e9fM}9)t~(AL^OiOF%2W=_s(f-8 z-`p%*Clu=6S{r_w4?;qzl0(!>E)U4mMjm%V#~~4t@B(CvWi@x&X~&Z1cnu;gX^Vos z&c#-Wnoij@)!j=HcYn*>*xkz?j0$D?3_G^(bEZC1 zy6t_zb36`MA{x{V=lw-A&+Nu_$Jl?aze%nhANZo|UB9!L9EI+Q%Y(((v2UL=?K#pm zQpyaWiI#c^0#K$M0s&jhqH27W7qGAB4=rsGUTacyC|L-j4hR_#u792o1Rk#S3mMa= z7Wa1|dqznwMtzeY+|GwwnxHJJ*E=Ag@MIa~zVl9HY}M)UW- z5+GFm<+N9LYvi_%O*-|2$-8ij?RcXq{!4%kxNAit{^^kqSz`K2wz`O9c|({BdRD2I zv9E<$k1KhKKZP0=Nm|w* zLm)6vw5cCsyphk32d%+@(p_k6b%*0vpD=@{81ajnGSxZl9J_M@4|@-HJ=M*@=^ zj^{6Hw&jf;ydIzNtZ8xe(xkUyZM6yp0fEJyp$>+W%!zX=#wMkuRVmR4yyyl`bbP+$ zeH%c?Xa(j1uX&xz?EG@iW|t?ExfU4N?Ku|xRP?r%!@)8$m6xZq{8Df0xaz`oBfA(c zzqkcvZg)t}%FOg zT@0Ey-V71NVc}K^C85sC!c1vYnwbr2Ss3Y@brHBG5FPE;p+~poa2U19v^wu!-zDu@ z4_+`Lo#g6g9*F93X^+YulwHlzP9<2QwzxS6W{N>zWc>qv`Wm@Oyltm{eAK(Rv%6aCno4xKwEUi6-?Q6T0s;Us=P?CG11+rBS8z(B(m8_DxnG`(oqJ}1`92^p{;XRufg zC4$1z^FAPU_%#_cS*gnV2?)$Y-rz_Y21koU>j*Cd1*#uvUi=Lo|I%1YPqDYDj14kQ zur_|w?PQd2#d%^48lpZUN=za=On4CqxunT?)t2q#9Nx2*vC_@yO9;`zarh;n;sc4` zdagf6Kc{YQ%6FUJ+w$TUw4S9fF_pa`br1%eniDyAk$D}vfuoa#qW~7~>v18(BkBpP z&WRJBjI0{(Kh#q#cBGCbWIV)rpY0wFM@c%Mc*?78{{6n(b!mhkl&}Jy8j28bwmHFl zj(zg13u>%)4Y56+zrSofH!Chhf@MNA`pQtkUWEg#1v;3wqht;jm(syZZf2qwir=;T z!CFlW+u1#m=M|PpD6M5gI>UUiwi7x`n%nS^^;IPbD&W0Usq$|eX@Y*Nwa2%Ry&*Il zV=|timp=ha7-&K&;GgL|OWFXdx6`JMHp`+ZV=m6+;ZZ)HG?R3qp*~mK;rG%u;;OaA zznH)@$`DRy5`vee!&(~O)V8c`yX1bg5xOUbi%|cf!B_)9*$23#AN+Fb9fC&MYsd$t z4LUahcqT2reiND}=B>ua$L=iwk(zW4N#F8S zVP25AmjCRCuVeZu?)s~Np19K%va~Qxg20@SuVd{#1nNY%kO+FPclxVhx>fA%XrOby zI0i)MNu>27r2BCGJNYFue>?-EasFw{9I?FXh93g1~WJdq&yX zEq{mMDBm@ddoJDI z;aKpwJa*YKwgk8grUC{m_TEh$LtzofSm@;U0(B^&@OW-)=*=t~zibAkzxwU;HBB>t zW*aG4IJM-=Oj+f zD;sy$LXe%lIjfR=MO^Rhb7v(IV_-qn+tUEu)RyQ`J+3Naj*NsdCSyBSOp(r>^x{5I zp_@DYNR#aM-K>iX&|Fb0F+_vyp9lQvS@E56Ubjg13aN<0g{^*I)is6s+Sqcc>(0v| z@>l%>Z&%=DXWn9+0h5l}Tg?g{*;iK=n7=w&XRoe}@qM61@i?C`F~lG2g{7<)+k9qt$W3(x2fxy(N9UlR5D8FyJeq@u0yb>F`TDT{d51Ud zxjl!Tm{?b129yTn+uM@751%&96i-mHg^JsVwQmBYHao_~0!~e#nHjsK6vTPeC$dn{ zy1Rb!d-g4eFWB>*bH?cR(EH%>@S0oH$2(Unfy*Zb7eDc4k1VeN=N-!fPZaCj%0I~% zpM?$gy(bc6wf9eZf47$T?^GUFUG{)3!8xd0%0`?m0BC^gSHDj977|IW+MOXuWfO#E}B z5yBdh6&?M$TNDfl2v(m>`tCKac zQGTd&pMf$`<$@OH#(H{z6tt?ih^&ZG4QdJdg;vf_<|Kq{fORi#i=|D_Ef~-VCf2EIB-TH7<5R| zl171{S9Q35{|#yrCT}iYy@@XzmZV1cQh%Q@2CPoZ=FBs-#T#@40VT{<^HA#Lph?O` zoh0+CcfHaL1q=8ZH)~vzjW=20RWWhR|j`VTtAM`;(ouApVRrAk&b8fwf?|>?M2YsAXcpu6efBbJz5C} z6yQk_m%y2xOupwfT6hAQ-HVWMyk$2Gkvf^MoL=yHR=~J9yrIz$qPkw9FFq0$jbh4* zrH18>e!=e2i9xC%ECV>XbAfWEak8}#p_O@keb6do*4!^~pp$1Pc~uMS80UCIq1~XTsD23s zyi`E-QJMk}Z`QPPE~BT$W7#GNs_}oyA&4ubavBm&>Qss_MpiELygR_CBnu^r|Z4mX8;i?=A-j<^U&3;!HfSW!%_f@)cb&FPGc+zq=Y(G!zXByYi=%V-q z@6Zz;ce1#7kCj(n#&TOX&?#V1MGCZDm_0Ztza(l4x+3B?Y*XYFQ`c_mS*UgJQDdY52xt+fs zX9MJhEuKXQ#P^R^RL4*q-5tWOMH>#+Ev_vwf&sTPhb2h;Oxt=Z5Sg`O#O4kvY~@sc zO(nWArKTiX9^s#A(97Tp$^U5>O2y6jq&ipSmHj(%cjty^W5xjw1I*u~$Z zC;{j~i6zKzN?Zg18?E1W0%s)Hnl!G&AQIqh?k}*ntTBFX<#j=dr5dW{gu~0yJM7(& zB@UJ?1hssC{_()Fc<4i#tjGNp+vLFR+=}iw;?;9Q=^Iz(_}AbtHL68J*ck{7ZvDq^ zpJ>%N=6`K$#kX%WU^B8NtYn91Yf=w<2%{?_WZHZGQ6XEVZ!FctA}xKG+7TJZK2(e6 z4R`XPPDt1n{BtPoUWihi{sL~2gI;hVbBq0pPJd7X@TP?@J#{$PniIG!$Gc7sl1nQ^ zU`6W*qn&S60ii!V$Fqz{N{-=$mz}Y#bZ;5yO^V4`e5Pbu_bTUv8Mwo*VmQ3Cglx_t zW+jk&Tf8MTt!>qENY_1goG_1`38=imsI8OU$%sWyh~fvNv*=M?I-5OG8J_5Au)kn} ziW#fgYQ|>5=7u7UQ03Bj`CyR<`ZDE@q>^}uFjZQbc^8MZ{_q-4Xyf`LQ9Q1URJ+V< zE{4r*i)w{Nbr5Mc^Rzw(1!yOd@b^Woysr}*|2qQ8wu=lhU24c#ptx#Xd2TsX#wiD6 z<>Lb;kWt7)`11Pp84L>;5zxJuNh}LWKE5hU5MjQ1(>Y?i)h6cT_3V$_Grwz@068zYv#0s$Oz^LZeOdGXOS<+hy{N`6Y$x7MOAlCBDWbIefKLFupmHP7N`T6d~z>J z1s8bo1E$?a6!+lWLU4GHc zaC0d3=DK+9yK9JiiFSe4WhCRZ{*I@x(?3HL%(GexvFD8lg%40pQdRQQC5xkGm2xLf z>5b!&-LCC3%Ggs+=7TE}NJ>l$ z&;535ax=y~j`!U__mV>WN5>W|2MvwpI@r5-oGncRE-24-HULmdni~ulTc6UWrX}ap zW1#LaR(3W$3=X6orxNVmRl2ybvA_d$xQ|9!n;_E01JH4d_1f8C%?hvHR5>bShw9!) z)!WuF(o{ey25v?Ir(VBS;$028J=v&@(N%1_q=KNjWtDOeyMo5~02wxLMuymhfYo7- zG9uAl%=lhJzAl&9mASu_lfHU*L}e5&o3^(Ha$YNsvOj{}1SHmnJW>bfAwD5NN8GUp zrO>{POXnn{jh65r9!GGXl`7ar)=gZ3v@MIMjVF!S84+mm6@7OlsG(1MVMjIm zq%DD{I3Y7%^`QTS)izo%SQbad*R=+GxskWp<3eWgXb*OMt!-SUrnWs?ChWRxw*!Q{ z%h6V;`NpFU#UHFlIVNQJPjz-ThZXY<*H*w-}mZrmiz5uLqe#LfSAH_;hg{f+%ccf-Gc${$(dQJ=hqK@8KjfS56Q zoQAGs2?Ci~AG-Dd0`e?2&NcV)^^5m)4mUJ>P+tQ^2epL@GUCgF2I*oBH*k<;5tyJP zW4heACLww$k@0zTdS$$6MIv%k*jN=MNKDlFudw`cmNs6gD6px0J=0(;1FXXAJm*wt`-QG7wfhTG1R)-DBWU0l|w zOZ5_TBG+ZPJ5_OZb!&Q&2yqd0zN>5q(}G33PQ*VhaV{(1{K&#N5`sDig94)0IEq^pW-Q9w(Q|v$l(qX zE_8R9v9fmnHA5ZYcQOp`x8zu#%9GN}TASLlfBqCctKP@Ak8vlIpXDl;7`hhsZjjU` z&h<%%u@fVf6TdgV@V(&oseN4wh^}k_Hm850X75YD%c$imCNes+>AxP$+w*-;7rq>a zv>Q`pu0HGF_jcVT@GotF-Cp@9JP-9tmSK&L?-B2NycER~rnLW;w8XhPXixVv;-w?W zA+#o?+Tp>B(Xu?USs#KmvSo5y`JU38W7;;%8K=oNpZf0}l*}_CiC0wqinGre=WIFG z%-=og-xQ(N){!2b-946aq^YQ$VgivqyuQ$tv}exZI8N7Q3y-q3HtDSP2!7WFdH1RT zJ$O{s5T1i8(e(SjL70rSOqbLq@ocva+r|4I!v#Zd*~g~*+Iw=|y|2*J?%PeFc?k^3 zQvzVjxaDtI9slVyue+h68-vZKJA8C+(_&BCIOH=}{p!E@$s-HMQz{19JnbKo#k3?* z>Uk-1v3k)^b8Fh{0qRWd&wR8&knO2)sPGR7^JzT?W_X=i)sHRq`i^e;SI3^jIzDs} zzvrABbUxFV){evFvQ3}rwjdM6IH=9tl&kF4*rd$9EjR6M$MI%gv?-Xx+7s}TRb=|u zxohiCOQoAB%q>EnS}b!05gwq$)EG}?C_Jr;B{s1>L%nH7qLHsQphBRU(@Id~#6_i7 zNg?pU|C{yCwt!!`b%_H|!AGgn;I2h{)i*cZ8L8Te_a}7hj@pF{kNPd{cD>HSlTyeO z8@>IqgA=>DOCf@nV9nA#sTmyv@c`>B3v$mtTdu+bla3_a=Q)ja?{hsj^5Dlpte?;f z&u-N%x}+-u?_{hmFdHolWzqGD3y1C+>RYy1!E%*G$c-dUH`jiPoz+XLJ;^>`g7XQe zm|KH~m4%(ck_zW(mco{R@d6z>W@NM~7Z*kQLhI?e6U{-6O~R_51?87wV(0W64&lK( zMLVO2=@u33>T@q>1l$7wf`8_UD?!1LKUQ6??@enQdyZW=KzAEUtrnJDhD|`9KGodKwS@2TX{GX}rFk z>Dh7=UHsN!^$e45%l83=El@A@jl5u%^^*!iMi*OXzQZ3gARcA-$sY+jTB506usD2? zYiG~b{EnTnbQ^!<*z%|vIimucz zv1DvZWmBPVH3b>9<5?p9GIQv~m4|y_SvmpBe-H*8ZyQN%7Ki-DK!co{2OgT}BK0^pe^5o3(|EPjk)h<}J zmmoh4-!kddskW-hJ0{*WN@hlPVsQcks)|N5Of*?V#6*y?60e|zkgV&BMs$LD_BaAu zZTpiaM$o>A737|zG@Cv(pRBzY-N>7N>Dxb~OsUNN)~&69{83dp7t<-0h?U<1dH8j| zWL=l~XuWN;=Gi^sh@tO%Q5J}dR5sB$=uu2Bo;@CL{$MnLOmiN>qpMR1< zBS)rL{X~@vdC{0DhixgGOc13y4zM;I=(_*G7rwz|A)G_434V{vINDl$YN%;n3=(1W@ct)mMpNC zS+Xp)n3qQ^=a zd}5}5*Z8vZY%FEB@83}Yxe(Rjpj{>XTV(Gc`}bNbb9c-Fsnzjz7{;ex2(-lCa)?t-)iA@8=PL;e$t2^t!(d=Yk+AFL$cJdSNBdo6yndezJ-^=& za?P}lwEEGt`vNT7S3!a2M7pi4HqGFt*c(#wFk2!j!~0oJ-b6?2*9Ivg5n%$hpsxKr zmG3YmL#sLxcVUES!*w|1H8(4xQa5|?t+}B6snrRn^!M%C)Wa#`KYQ)aM%n25ZFb{_ zbGfez4eD`@a50((wL3BgaLh|(()cY55bB)C8B+Wd0R_mqXcGnYcax+7X(N%XTN17KZO^bfOp% zT&WYrU01d>%nT6;M#$20oQ% zw~(eS$4=;26p!1>$Z2UJ;o#l$dtgRZz*~)Oi8xI-E%D&G_3lBgZB!CL+B}ZcTu>{y zEojr&F==fBbe_(sX zm=nQQWJ!LZg-ua^Rv?}%`RYA_nDpM?8;qWOL4O97kOaSxH)w6ao^U2VB8S*S@&;^| zR~s4K5rNlAXxQkP1WcD6A23Zdg}M@mfDg$*>N2XEjBIjB@ce_1;!=Q`&99^erm!qN zo~6vH%)VGKnjQe?6n!lcYq(AITzt6`h(qr7?9VGL;*IRHFg3Nm4aFwDIrhQ>)Cc7S zw92XA-p%2iyfZ(WY3S*K)Sem1OA1`fdd1C=tCNz+8K6|tu_0UMahU+7htFs_elv4WAdv``83RA zuW-^=!67b4$oe#w#bPd(pwi*9_vi#2h$!S%FY528iq3B$Bm3h-pJR#%{;EHAz5{d} zxk?0*c$s5TR;}sk#c1xjO!nfx1IK4^nORapkb$r5t3O9(yq38cLRv;2wnjqJh^oDf zARiIY{Sj3-d@~!J)S<0=Zf@>~_7V_xT#PXx`a+qJs=uBAAhnW@Fw4v=d@HL>7^_28 zPjcXT)W?t4){_0U^o_7nSjzwf21z=CW#PJ}kQc|mMgJdB$s?yCpj#3*wO``m0y|fG z$2ZWcC`)u|TzxXxvD8}}s7SAEjl`FQ(To5D{?2qJYn7uTEIbt7`FrCFACHMy4aW`s(iHwr%)HMF1d(Aa&ZM0-TKTySvBp@8i?L0ECbJOxeGd z5PdxR(_*Li|1Ua}|E}bvwIu^rw*=IJ)1Kc{nKoSe{c$_1qo=!@_W2s0((wAc5 zV-z2a5hgv~I{)1%`Y+D^uWaXI zO#hWC=>OHJHrJM11oH`U157)OPy+^0ucodKxfMV0zF%;k^Y9-3thXM~?z^8)6j~Hy zebWb@qLle-3XcJ;s0d;I&I>^KVRHrOo}4Ca$@Gl4tSq_rl8Hrmfw%4AaP^JT^o;MQ zCR|U2MvZ&wGe$P;BnSGRqhNx640{mBx&`J7q?ec9G6-*x{6D&`!8zFvTf3R*KU?+Y zhvidqL7p76$^U2CuUP&lr=BcUq-{755+yvItPK2?1K??kOIqo8lPk+A#{cP@7TP)` z2qNNPnYAnJF2B){bPyO=4A!kCWeLo~4-?E0vd=A~qVLtId@sgqqLyQe-aVmNS;N|r zzWn@tbekA=&e(VDefafl1MRtRDVL;-F031EmY5Bg6>=EME(sl0Z=k+Qt_jN`R2V&> z5p7{N-(2liN}&QpYsVnnEw5SES6mYyYH)QGNvdjQnvNki3_Zl&6JZ|IpI}a4Gza_n zN(q`D2{hSkj0V~*M(4$n-rm#Ix#e%=|5)FSgTjZHt*y3-s{to;2j8S6zH#YLzV%Ln z1sq?g%1ZraGkvLKvDpGuld_Bl{*fK4c(pCk^bQSyMgL)eyU`w%Z$;jlmiE?OPV)`8 zA`v}?>!n{eYA@Q&qmSu?6%>Qvs|eEMJpXO`y_@xhXRGT1m6ZtJ8FNhO-H?ZF#O{4j~n}PqcGRc`AiaU;d zEA+>~B`4Dysgz8K9m}~{G!(fS>NI~$ZEXD4BhxhjGmIL1t-ip+8K}#;8Lhfx+;m;T z72e|*x}}(BCjc2sXqJz=cY8!3?DJ*cx>rJgb^20k?h6%NaIWtIP0QFOa$St`u?hnA zrcdkRf$n0XJ=;vj3frgrJ(?_^Cmsu25u~#x%n~2^aHw{_w$!%t?7c(W-Iu#>WhAZj zZhDXEmP_?ExX;T54`;uOsaq+KHdsH&TYu=G5D}YBm`y%RkA3AUua(PD)1-|c@q0z#e1_T(|_5qqElU$->p z=tAVvb&QP|wQ1GT?L_vBA)Vtmv-{I}Q^2DrSO?+QBro_2A zaM4|TMlkD*FsgCMOl9DAgnPoyxp=HxeXfTgsryI+pD$}_AodBFEZ8hh=}(7a>HI$? z{cv7H0B|*kSXi8&T_C5N$_P<>!AJ?l^P)-?@f&P$qj#>ni)hD)4D9&neea2)0gR+p zZ-53ISh|g8dsDI_$$uZEbjl;P@~C1vzFX!Ub!M2tD;A0=XyE~(SKxKOIV1ON?XIyO z@*|8OB0fU@Q7AUluc0)ie~VkTHJc2pF-3-8M%$TY`&_24d(WN|TVv{2mFtPe<2Ly! zDn(fXDk32O7V{K%js%gy=N4yNqVt;$(y|i1s=lo?!@{p9T?}IjkUjKV09#;NK&o39 zxLDL4W&iOqgR1<70B7+D$Qt~N_KJD`s-^|{iKebaQ;hYj;D-hH6wgv->YmBHu6`^! z-&E|<$!7)c?(Rk}^&&c6^fu`Y8m~gWUaASmi|%IYgyfC^TYTT>$N~(7^Cw`w#xhkl zw;xSSU9Y0IXg(?J#t+F=oJvz4m*CF#$AjGH6kcymVBkPq?}oZM`-@G1_S-Vs;>sOM z$qyF=WQ~I2B&)Pjc^8xb%AydX*;kF)rootH&G`7$qegeQUI^;b!Dp4Sl(L=*3u{N$ z05&X09oXPejs9jn>Yal#t+%WB&(M@qkDGPmk9sT8rB8D(dH3h@Hs1|gPA>c_C!}M4 zS$sq^hp%2YQ=;70qfEl0qSqz5uNM3S%k%U7+tGY7%F3u64}Ekfp~DMFAGY9M4{}d^ z#Y@5cz8*h{WtACNQF(EBZ?>8#Irkm4aeER7M9@#f%*HG@{N#%*^CQq_oJf-stpX|L z&Y)*=Y{1T5=tgeBqOg9RM_xfJ)ciu(ODI%YTgvLlKt*Y{`tqCV!w(nyscc?ta57y@EG9r~ zH>3UX1L-35mL|CMB$YQk-OI0kZRZXa2TR<_)o|eG>grmSbyoIf>FrM0lf!K(iY)1H zP1f@ww@oPOHq4t1Tol+VRGj_&=IkmIk{)k=U zF)yNH%O`7mUu}rARu1HO)p1e%{1!n7FmfopwK4oRMzy%b&i1uBbkEd*6Q_G#b(n^W zkY8O}a$JyXHYVAi^aXM2big*2MVQ=+aP<0|_b4{V3QaT+KhpSGRrqTt6Fv+(LM{;e zC$yk(tvtZfGcL2B$7n(!2pviHXg@qS%viY&E;c@|mR^P1soQIN0p!<%3OLnd~3XB56W#6WQQW$xvITY1@m)19#D-VlDvqchZ3 zyXP(Qh|z7UOa2vXvb>yD%<+V%nXS_`v2F?G6{>y*TJU^F6OFF&LYKtz0No^-?pa4| zy|>vvcH#om4`y?O$;uM6f~O$?pg+BW!1KJKL1KG=S6aiJcps~Bw>@r7mMQ(wVYd^# zg35#EFfM!b83yZ&P`$rmyjyDB`&yr7aNj{K`Ht(4wcIW~RlJpXK0fyg_X9=xzOl$p zcrV+2<8N=f$yQ5e_!eZxH$J;?*>!iXUbFj!WzBN+ZW^L~4SGPwCn;4^#+mON>8+h1 zjNulae*3-J_1<1tTj^5X`c> z1}C*C&AaegoyQd^@?Occ%@p=B_8UpTZi5nWQYzGr`u37TUc$+4&7jWuffG6M#5inc zQf-*ff}>)v-WJ-}&R{8tgDk#-l~?W6m7vT+gcJecQMEGfsSHMB`OevjV4O3|Pg`~Ec&e8ulb`Ai`}s&iKD|hVeyvatFSJ=WKHE=vnnq^k$mjs|Koej^!#fMuHjhM3Z#pn0Ons0Zwr5<$Ipct6kPf+z^ z#~TM;kU!DhqpTf$#`S!^thpvAa5A~wnANxOr524RD=*CSHWCySaYKSJ+KYv#EYob) zjQQ@rMt@c~I-6)*EVfENU1(h_&V8{KMa;kTUg6G!;<+UXAB_A(v=WQFUT__CSc4kTXH3Kf2V>PcMNf?|_VX`6#Zt#f9Wn}!2%QL+`L1PcyZ znT&Ff3Qs{y4r4@L9VSVCKqLUCI0K-t0=-rRb4EIJ@+*D{$*8?TW(4p9^Da3V%LN%u5E4m$tz2n)MAqh${ycC351OM*ZDlyEg4w2)G>&yCTl-g@UM1;bdy zJsy1{f?3)w1fdV|R?ivE%S431y&N}@ImtTwr*7U+!5fNDTt51)zH+>cy8G~Y_)MjO;{nHYM#a-gJxs(QPt!= z4qAW=tJTqV=tKKv=e*K`>=?y0?ze9LXbVbKIS21FW1T9@p>@T>UV~wm+S~Ku=e-V!Y3%J zA4=kf6a%VwXUPRIe4L}go)(YXC812}o=fmIcvt+nKQ_Md>NI8?;^XH3%#9r&lQCMU zCq2=={IZ7@5P8~y!2o{i;Ps+iK8a5*l}qeqza<2F2O*;@nCmllf7x9%Oy)nFrkJP& zbg&8>{S=sqZ)AXA_khhyM@7U`i1x`y%S}>oS2}OEL)_VhC_S4|P@1BG=K0sH-j}HM zOklfnzruCn5Ay^x^RGHOIwk3(+IR?KBO^0@L%7G}2(Y(5ZPyCEnhqZ^(ljO83u*aj zMD%14JTCEz7UVUA*+(vb)+NVqnljwOxPtqGIvCUpbew7!#an-V=uB|{#eRQsaA3If7z||6 zpbxJHLhp9H+G>_{Qw~mGpk!BbE1wfF!8=UeF@3pa3KQ>cKG8=NmEo8?9z{K;xL-Wk zLj7SufL8Vi9^SMo0KDuBOBWwOyHXGM_wV0W^i76qS9)d*`nh4ZknE!y4o3mLIf0=^ zpEkQYf5S5z7W>>l{9uy(d~MS>-*irpg*^>7{C%vRWg&&*o1s=+hxh4s9ubIVK%lTBmCcB?BN8P6@kwv;nw zF+AK>l}Wl;!bJ#&EtAKP04-49wg~eI{V%9MHpgu#Af=tX8qxFWKdCkV4H9s3dq7

cjqqo7hHRJY<(aG0wNL0SGVgEnV;N?e&36P}txE!LDgT?(PfBNf z%_rDxLw{EZ@Th7Z&M1_5V9AO&)c~6ReC2L=cV0z;J_k;JZoGwn{v&SmG#SL7Mg7qW zV)GRU`4z|$xMq94#$rHT;<9_^d_2|L6Yv#FVOy@}*C*B+LbAv=4tQ6NiScMwCQjDK zjNohZN3Wrh7q9ityj-yTKXfoC0O7{+@$ta*p|SkXMhKDtfq*=F;`DYQYtuHHeXmOt zmZ>hTO+#=i&WjRgCW74U>grau;aa<%@o zu|(037s0?kaqDBiY|314{x5Q?Yg+DaO}fALj=KB8>%1~mCrcwrBq2@IeWl?IRZm!4 zct_qOK*|4|BDTIU_~7PeL@rO1iEqzSD{V>BJsXvT#!TV790G7YB(}b>;@TSRH#OgC zFyG$Xfihn4HWM)sBVbqzoXYcW>=4f@-eQvL6Xj+O(u(?Q<5*v&@6iEUSysO6g}C?O z=2!G|R)jZLV+#0byf5D8ENAX@Me6uRE8f(zV&_qE%@uxiaIdya(jkteY4niCSzPzV zoSMqgXXU>1P8S3|*Q1IrzdP2AWxKk9uW^^TuD5Knb3&@9$Ym32xO_W&K)(8byxnK+ ziw~87dxDcsG)>OU0D)bdhnEGlbcdAzZllnmo!aDqW)ira_I#QQW|x%0lE%*TSmqYtp0$JDg(LVWYY4_e z^7o;|ieW_$$c}Hd=#`50S2(oVJPfmI$2-6t`+p<7O=3C@?Cn-l!XY&Shk!A~W5~*H zY@@Bd$Nj-kje-3;375T-{06Scd~`$s+?;}l2?vQ{;H&64&R>VmU+U{}88~{rDdz7a zJO7ZC<13(nGP0Xkc<4KdGoevE86o%frNMji5$3)@csIWLX`%*=JxrVE?vxOAmy!Fk zAAEaKAVrsks8AjkFURoF=oaA#L@6n@177sqH2JcjG;(J#z1}@%haelj{FrHf$^CBJ zH|MQdp9)9_65ilw=aTw;n-lsDzk5kQ^|)9`Mfs3iZijrc-E@7k z%+zmSR@beE& zs#N!Mp469VdM=)P_>q+iBrcefXZx@Nr->)`vV@aLrg3wsvXT^c662_govWJFtadDwrd zSo1ASU2^yQKc!?kM)8OJ#(h#U!;e71c1zRayE**`C*(3;2ymXUj%VRHCgsE- z$wpnk1gTs~chMC)@-Plqtr%O9Qs~Fr0-(<6OxKM42d1slAEW2Ufn2jZ>0(I*UPBqX zf@O~+!I<@1(Qd!Zh6E>3_rw+1RR;~V^i8oFN_32iA% z5J&=QRblCFy+<_O1SHiKPBxT}^-jUBXNJmhJ zOM7@$YQbSThCEhRf4Zlr}}CE)rI$ zI42A69cbg3mB!T#t^{oy0^(sX{{C6g5>7C%y~PJ=Bi1bt(-p@I?QKUL(xfLe4i+j6 zf3vd}r?nmPE7De!=Z#w3E3kN}*A(Us{P;c9tEJ^S?zTL6ze|Z#?F*nEaa&}*MJftf z5N?5@$)7tP$&zm~8M2QsliTUmE(v|jYJK)|d!%X$lSWI4#!ogzd_WGlc7UJgC=B5F zlcO+zWfH45J~xJ3i_g13%xhF37o3&F`5nTB+VI>-?UtyC(gfpH=J|w-Z6i>PWG2udAQHW%s_Q?yVs+cqa}88 zp87PKqUVK&*j=XsL^Y4_bvf4aBNU#jF_j6ua>Kv_Pham&@E46%q`Yw>}1Pd6W6E|HIK0 zQjK5rjjurmeP6Ya*@=6#rrB5Fmc`>|$tX>ygugwuf~e+l$2z}*?Og%Fc)4&~8eJ#C z*fj0rzJ(i|W7WxWj9AA1DXUGKvZO&!ca~8?6T7tJ1LgvH5S)F_vCgLk?32vME{ntc zXxVx4CRUneGm1YuT%w?L=)zoxM~w=lSWc z*gDyltThd)M>Ngft?_-wQ-^D?KEi6}ZZYHX zG?ZLJP+K1Lj?dCPI{qd^4yQIYCT8zdUNPK3>)c&&ak(ffT+;*iMI6J+rB{}vJKWf4 zHYBW0M1lBN6XmBu>~{5_-Y;C!SsE_1&s5kNku9i~3RgO0w1)$mOR^US^{x+#pW&lf ze?y(fY7c@AMM=r*>&H{*8vMMkgglJb>IQDte|xD274wN{msO|0Q0e#odF^&{3fl>u zW?FZL$Cd>zNVhBKUH@dQGtpMmAR}x88bxuXACCcb>sE{CdOh>=iBXQo)7C3CBHn{_IEaI#bmfP z2NW{~Y}e}pOW5=kw<~AK13b8ne7GLn*DH$phL~ay<4YTlz(0_8K$%Fqc_jb6_S{Ct z5%++PI-m&|fAe>Hrn#=-P{Z@}V^1}__?qL=?A_Da4!o(u`$2!p(88e?9o?vJfO$?E zy-R7nlP<7j{KB<@%8Z!V>=v6i_PX1!2b<0kcQ*R#@0Gfa!i~7kFZLmcVYpQ8mfc!= zvlYZgyrI$6WQ-|{I@pPwl3$~G@O$%y--iPc<+jA^?t9mA#zd7U4>HxjvBgS&gui3m| zf&DY0Sk}@tZJD#x-JSAS1Vlt2iDA0xahKqXK=egj1IQm04Zx0io@8p=7k3tg0)N*( zb|voZCn{nrMEJW#{~p~05^o*znHd-xDu&kd=rR_)9vrOS&Rh8Bq~zk3 zfa+(hy(J$A^2w<&s#)ghz;Q;=ls5Cga0mr=VWJEJcPDur{Q!bvWX+x_q7!9_uNx)ejxKH{s zHa$>+t&xNgGlg^<^{EkQgwg0UH_-64LRdw%0Y0{@gbKsmRiso{grOqE4w(odCgxB= z3l!#UnP1*`NT86sgIMLN)Y^I&+`A=}kiZVw9?UN>#!kgCPb7S=!-{ zYF59lF)Zn*9x?IxbX7)#RYa_Q;~Xokx{QEFb?$p?)EkK@uB{VqbUv}6*bs%@_D>4o zXw0F(0}ASDR9#VAm+#ijBM8DGNJ|%(LJr#R!;VJ-cCZYJNNU-xhiAiVGUSZBfd(rI z1;WuJ4ki%FPg4L}2VBftF(1H)_39gvXOD?^J*a89Mti%I93AMhWVx|Nvw!7*nO9UP z87)~>LIn3Pw(cJ4(BdQ|<23eH)KdU_YMl-OEFIve?>M77|&g9yu6@imFm6 z-A-cugHZraW=5T(o@?=1EGtpad{HWHAul$K9*wmx-pCSVy=07pB0piUm&Hn4lBkq5 zKHuJ+1U(2byE#l+JXXn8oP?#~wxxQ-JG8b@-!e3KPk)3J3G?V{ygbDOIS+Ps0ZGi& zKFStN%%uYorZiDmzLkVTA6}x^a)ET7F!qDfp#)a5R~;v`LTX})Z08M1y-{9`g$YKc zR!>T5hLjY&T*`21FE>b*_HbIQ5?Y@V5fiWfP5P_xn^u?6` z96-Fv6PPMa!_OPth3%@!%cCYBE6Rw-Q;&$wn*O3W12>XFUzs1Ti2k*9e{s;h1_2Xf zQC3mM@<)6$yp4^{Lt1WV-NU@H7Tef5P;hfaI2-K%8iHu|B7#@j>IHPAUgT#2F5(9H znCXy%)T)#=*iAGkrIc8W2Kv%@yJqZ81p2s7*%r_+Q4JolbuL`Mh-jwG5F$;qLoJ#$Yz=s3y|xV`JC85Lv<#aO!h+=|HsK zAA$c(EBNF;+*w}a@ytwTdv4T;TnFvADh)uw3vhtm9y*@0qnp^{4^6)#qNxZBrJ*UP z%0_D|40hdgVfU)kR!yo&TcB(DB5^cXZIVa^G%{>r)e09wqoYl}bHo+J8iq2v7VajU zo{aDaTE8juM`$hIB*3fdJE!8qiPm{zYvRfL|w+5Sw2?h!R;H;#5^7y|yoAAg4J5TXdU8FgMH6J))q zrw(yI zb@=*ypjnnGmA!WxOQTXTMb&O8m>Z&Xb+vP-ju3yEjq&@B09Y$l>?R#FB9OFsgA3?> zMju+L_+S|| zm`<){Zo^Sb$hTSMjEYPc1=kJS-d`DqTI8+OkkW>&Uq55G8*r?S*DK}A2(Z*O>1KNk^ivHnP=%A)1V zy?S|P@I4(0wk!`*B?&9eTuez5(J6T?ayT2WkesrG$TTo5!%fNi;Jo(b(zZ z=3{V$=gq-o;6jJx0&RD9H}m9Ge6_)``TX)Y0#k21;j8wiqutPiQ!b1Bc%FSVORkFQQ9~mtQz1&#zXDs2ZePrXR#ZzU40`_4U()6*z;XD< zwRyM{nnv54yij(`F`n6!!!o8nJjBy6>k;t@tm~YnTnq1~FB&kI*N|0J9pQCvAyoyN zt>4`;=%r@y|Gr`wjTFxr*6Y->>mmOVa5!WLrE2uIOq%B(kaHDi@9MbwxbO2Vy#xlG zZvM(yN1%qWoEA65_?3kxLc3~V@sawO3NAw|eP7bb0-_2l6no#8jJ2zj;UA9R_w?5E zR{WurjXchG>AUeeOLmK_w$P&&w#Tg{$wMl<+yceScaxflg$Wl&l!M1Fx(#;gm#G6V zKKvV1IK#g2M;TDuTB_$g*Jyyxk!4HSx#HNt@5jHe6|dVD!}`^1fKuzQh6r)#6ap%o zA?x9Wm-Mz7noalv`_2;e6Wh7p(D2Sy6`J};?kie4E=+N}R~P^k2iN`)r}|%MtJ?3I zXD#PjY!lpH-#0tS_PJ_Z_xd(?g#a}GnJh|E*EgS6uz!DsU{KoezRaTWzPGCPt~EFy zG2g4+kS^6ZdjdY&6{mt?_qy4TOY*3GoQ6Y7^^5>=9ON0k8o6^xEPEViEtI|mh@PH% z5^?e7_$$8OPa-TbmwRuNmFEtVQS&#o_-b@-?; z0EznMlCa@g>LI>SsRv z7XXPs%cs=}e!e)$a_kyS-*Y(Zdnvs8%bYD1Dy9CuzT4(rhjywws#!K?IMc5T^-qV2O#k^Ab>YzpGPe@4I><)<84m50U!4Z8^B6%YZsx{jXL zcBG3=1*TL_-})2;lCfBySVs!v&yBXM;fZ^JYtaoRrzn27v_}dto;BP}KtA{$bKp#V zwuoNBD>S>g6U0Ir4Ah|m0I>eu+tN5&2IP$6E;ez-zdG$TE&Z}s$8y?)yP0l$y3xEVlM8UVzQY7tj zo>}5)c4{m|MJFrsj6ZrIMWJ>_Al_eWvZBIgJC1}Y_|Xh5C>@Bg?`qnibqg%d@Bbg= zyQbacwO=MH+0_i5*9ff9QF_x=SM%X}rG=(eFf%(9Q(@kuN}JJDg=~SDTs^;Wy|Ns$ zGPy&CjJkK=OV_it%a-V!0p>2*I-r4o(uafRVL1$nm!ur+W?@?f0k}1srl>7PNvzp zS8Q~$_n;{Y022$nk6xI4YmSUKE8@ zdY0bfmwT=^(v`c*{fuJnl5Ez$@d8j6&Od9es;s$|Wcx10~nJ&ryf4Gr@1#SXq%L@3QC4O1(O0nz;Em>E>dcJ%FB$ zUUfX@2hrj2@x8a_^LP7|2gdv6;f=C^e~$s!@B!k* zf4urH>*2LC2!Q0okcEXMAt6CdUS7c2nFXl4yxh&hW9#%ZvZkg60&r7?k7gqY4cN|$ z1qy|HwtsoBZ*6NkTxp;Kd=yAYRn_rmmZJUXh^pE3e4OWgHMi?uBqi#jBOsj%y1tR= z>DZu zK;Ag?O&1c+`yPksni>oJ$)#2)YCp*Ca%yU!fMa?;-I$|&vM$wbak)913AsC6QB+fN zgJVID$3hl&k`TC7beF_L=E%IE{Ibseje$XOsA?naxuH*#(Lr8j&2`tPR}>jHTi`Kz zN=*Qo!O%dArcrS{%YjuUnuPQ5t~F@=xp4sa(;^ySsJ+5wjX!?&Z3km6h~d(P$(D1iQX!;3PRt>tVusH|qAa z<7l+_OwTk=qV5ZjPnZ*>F(nO}I?6uCBE9?xs2FC@nICwq!c{a zG7P>ZyANlor!Y$Ll?-8dEF3rk;hU8V56tT8o9xFGPf|dh4D|_#rB%E3y*8cXB;P5R zXm&KV7iUj6L=_T~C8n=nucB1-5AXzcY7_ymKwA#KmApFKkTYh^t^eej=r7ExC%%h= z$xboQYp+#WzTDb5JLfky zKVeA(>p?k_@^K|UrKl_0u!cnFt9ez}sS7~U6+Y`pF9~~W2^Ul>4LTvSf2Ahk;mN$| z9khfEN{nwI#jhBoAjyA8-JKDlkd`DMJ;p-+;v7z5zb(RQvpU!xS(0Mo&^?4A=|sN> z!#G&Ph!`IuWg$Ui*m@=I;ZEG=*A*w`afQKY53l%^A>-kJWluvowd|j-zxFK^PQ5uWNloErJm7? z!&^q27=WvW%&#Rmv6R(!_27gNFQ&1K6;~LfU$h&{q>%hN!2pEx)9b>?{>Arq&<(Mh zli9$TJLn>hEl@O=3yJ$PAMk~f`9ch$hHu%Q)Mhp7g9sFhuN_K?QrJp;1ymWIuFYR4 z2cw#yXZtUd0}Elsjk}`Pm{q*A7A7G^z&b7EXX^^mZ5YNn7nkmNiv5PMx@za>7`Qr@ zhX!UC84bXXQntl1SduREbeo?cm{|J-3yGI@ROhr8J&-l)zQ4*{E}0P=v5_Ppv`IWj#1FU=e7njn4KCB&~b&$JvbxJ zo0r++9@v?5oVVTQ+81loGCe{)4bT~=7{PQmfJP0sF_7&t>4XI+hn8$2#KrVVIsDGr zyVMK}tK90YGENIADp)3m1|Zqk=)A_{dvzG%e2la4ih5`AadU-ecOjmgUG`FSoqA?= zO@Ux9%%Au^iG;X)_#at#r6Wj^ z&4E%?Pn@Cg18TrOAUhox8d?mX;fnSal~GjwJZGc4pem{i__1UTu-<&6B{Lsy<($YH zC+sG0(Jy7R`^-8@i9L0v?M9}$U5KB2>JPE<33I?|zO8yIN8FcH6qy4pj%`BRQU3Zd zclZ|Rj_Gapi?Xl_Jn;a6`I%2oYEDbEis9k*)Q;MA9ltds<>L%Gc^a4`Fa#iCx@<=HnE$nv z{{O5zvV0?$M~a7sm%-_L)Et;oD3jUU)3dz{jon=tL{?4Kf&kmJvm%s=c_no7hQ%mc zawQfOPR>RUOi=};2HH9(9=c;}QIVTTSsfQQ2o*}qa-={|$?i=yKTAn%Z!Z-M*VW*~ zQx#t?=Us!B^~yP{qN=-q+rYaHq1AED)Ku;eRSj$yuVzR=z-35$jO^Q&HtD9Yr%_IU z;Kg&a@Y(xmU)y~@mH+jRo)@vHr?zEeC=)=HV?hE0Fr^BewuhsVigHWQ$8dBy1qFrf zKzLra$TEyiFif$v6d}|?w90#?O>b&uYf6sK#sjNAe*Kb9Dtu+0iyU5boLN*b);I5V zvtfSINKBMch=R8-mXBnlfLV%5ik2PqD7`KInQR-dP7jcKctU?-6%?Tb57HeT939CA zB97%#v!LGD+3}Y}5ePn?LeB>B5+m*q@3(-NIHa7&^RxphpKCHg7qsiFa-M6v@C;E=ha7po=nXBR1jD@gG>qb)OA0uS} zAkoY0rlzJ5BE*0Msd>-xGY-4olv(f6wL%(!osFvNvmW zRI=lX8|tPAzhCUp2{Y>+s-l)2vHKntG?ucB-D!}bGvSi^00p0p*mysf%06@1!Vpm1 zTz)OPeLH2y&Th5=4t7Xi+4o6|9DU%d3USly)3rSq9D?;_F<09I z6t;w&=mYSS_v{bm-gSUr^UsVL87GyIXC;WEv?V66d13*(9)98@%ZniBHURq zt7@2E|MF%CVj%m8+*AxzIzf@w*a)O4L1g}%#(h}Ai!5jm3P(v!qmOv#-iJd+OJMn` zpy2RJ!)yF5KxqqkK3>r+gVGcAoBOu}F6RNaz$ybte0a)nZ6ik=_iIOQ9yPSnjY=1@ zwlVizrHqzEE04Cn=CAXIGYJ9$QuOQBR;NFxR;ydpoC7+Yj?wvgn^vQpzn|Z6#H{-_ z{iec%X$55c9JcypL%SO(5tm_73RVcqxX5jOyHTlrFdThpFPwQ2Y~J~=z07B697zwc z5LTB}y=ury2KYmxDFy1jU5L?oqna{tW$v|CZXT4`S;0c)D*aKoD;HrC^aw1L&|RVm zq2XcrHWqlmOr}?%in-xy)VwnT(ys=cIE-8Sm{ zSF3jIvN}bIcAxq)l=d{XLv3kg);BeF!Cl<85)F6*b$T2bN*&) zG&_j!J%(a;iQ#i>KSUDvYK;C+Us^C!#kBw47jgfYtx4s>JcVs!U7K#Q?=pi0@|t+? zxi>DHqw#orE}mRZm%pm1r9f!-C_ux&P|(oit5)bZovk+M<4zqNSqu*kvwPfp5y`!L z0?lzyT-qX5Up7b7G)ZN_C`$pz6D6Z zjmIsp`5c}!tJ=S)v1(Fg=Gg1~#`|{trSX|_;=+fVdzt{Cn$v0Xu)jHwzdPTMb*?ki z{;H~~%FWH~cF_e{VQJC2D(C{&l?cdV@K^kqTCe+{tRTlLPbho=!9w5j^4;=F4M#-= z<~eQwU?(y~_%EKY3j8PLl4WFOCO%SFcLM@6z%k;>E`=0IasgbfkhIAjtKpf)m zr@twYxnq1d_zy+8 zN%1dz**Z8VdcUH+*waqcl%N6oISk_^M;>j2B!*zE9^KO!p zx5I*me?s)pW5w*rA?kj6?@DU^YiBt6x@YP)fy7o8eOkvmX4-`~g}rEx#D#DmS3U#N z=&dF@K980K1&;@8);uedDRS~6^IIRTi^^NfI8n&y6*=*@`VOSmBRm==r^d-CPov=pL*Ri-AySby_e~|aqL2*6bz9<9& z1PMWc1a}DT4#62TKyY^m!FAB!?yeIY5(q(pyUXANhr!+5xdZv0-#NGHynFAt@BH)X zRY4Uq?A?2JukO|B^I4zn9#T_{Z`dV4B@zgM2%KX3T$Q=bsMsBiS^Tf5%YlMEpY)bw zFwr^4KjFm@+M#(SoQ|7Yk*nB@&GO|KL2b-ACrDpKFtU!t<^t)aW->NSS*WbR+Z<53 zs?oO`bn`mn<Q8w=?{d0?)$Yo;b53_nNetBuIf;+5^XT zv4uKkGlY zU!e!$(j6K^-|Am--&tF4K>zo1aWoQ9TYcOANS4(}q-HaTgF{#S-$o77FB36k_FspA z-;oxPpf|ulcd|tIj65CVd_22bT=Qr(nZ(2D)=f*yHKFVX*CsebOw|Ja4Y#Z$k^-p1 z;%&a2AlRKotH-w!J?u zhL#3b25s#atllfuL#C&Jwfug~xKXMO1*6B%RacL~9+$*=s$QEOcZ|Vb5&~pbsODct zhf$ufN>i1hy{%EGRD7EtYGpX_$*M|>fy(&h9(?Jss5`tnK#QP&bNcn?Y%OepSS-Rw z3$`(F)@2Jnh1!=K81l80k2 z9Zc?PHp999c_+x8{If7NE23^g5}8TpDjvbzYnP9BSRh@+N`EDGmdo8|^$6xO18skNJ|&?3eq2IPq{xiil@0uQpdD-F&>`5Q2;CMq5RJ|ADgZ1*B0}D6q`T zypnbut$Musc0|ab(j3`GE3FG$A{xf!Y0^WiZ+J{0OxiT(0sg)KD9dx3)hGV59jHK% z6nnpu!y8M$q~#tCgRbwbX7v#C-xpV3RKHUXzs@Lov(3O6r4#OHJ>y zk*>^YVR5T0qcJdgiu0g)t=%-t&`gtmH%ewqB9@ zha4w&`owFvvaT^~Led*sx*WuNFD3Z(H-a_ovB8ZgXO*jNJ_BYC7rjOgroc^@OSmkM zBRcMccQ9|N{_+n1oIy`RiOvl+`(}yJpvdr}(^$(T+P6q+ZLZTAMD?Re{W8n4U5pIS zU-LNS@aEsvsR&o0j`>asc%Lo~d);0L?(9RMSz1*U2MXh}5fx(eqt|Y`ciTBy9$EZ<2n5T?y{2xlDmwstwnLy zy-7jFbY69|{4WQT2N0eR_69L+ZGxs?eTm7);*3ArMRRK475${w>sb)Jo{xKmnc^OKV4@tWhUvPgK9GsuJ9R|Ie$B3}8 zy-1A#pN}-GHiAE%20Y;%O0c;pJ>`PUGLLzkEt#DZAIm4_dRZzA9x8|3E#)K z4Yo#_26Ej4fu(0?wLiTyYjYAJ7d-6E3cVT%O;6_tTJ~NhJT>_KItLIiM)TRD3@wBl znbZzv^W`PU^7XbmyQjEqwavWyQd-r(R?CWazZQBm1fl=ENBqrjafYT+KS8ECnox9G z*}f)D!AjjFo_AJ<(6iw{V5qImLT{p5e3i6MQu)|H!2?{sd~E7sgKabWCyG^4`OaNG z1)z_8*!@xBW<%f~3<9=(zr^cTU7d3~TUO^anBtb9J51~cnrBMwq`pmXkrA7l4T|Tw zLsM%+u1Y%9(SB#SFn3|wAkdPluabIxg}|KmRF^)>leoMnB@m(}F-*t<+tlC#BgFf6 zMo;pP@lI}Tiua+jCoc>L8i@QBUH}iG*Hk!LaTHH&*mVGX;kX4(^KJi43xF(kxP+F+ zG+;iHM99Z|Dyhlaq;inm1 z?YRX=)_Xq!!rVrk4x1yb=Pnpu*)AY7@k_Zt-yg02xMHGrb00RAfY9a(uw;37-_m=Z|Ik1F0a@(0 z`tZ{9Hb49R+H8mGd(IdO%JWzy3j0Y?y}#Dsv*(Ati1=VA;WV~`=~yihBh)?eqafb< z@v!zhcn%3M2mfa9iwz$$3_Kn&VgJ2PZK71QDCtP1NbcS1(cqe}H-=;L!lSYKo53vtKL+Tbgx|cvJkPkH$Foh|*U>Vj@ykM%>IW|N zohMCs?V-R?79=2B*Hh62GxAu)b`G$wa#e(bw})xjBr25wKsH> zF!=Pf|8?5iDLmbVTLvefxscgwS&}tf6Xqxr1=Hwt%=cf0k z;`Gln0G-dbT9v+`x+}JW{u^Z6EHA#o1MlP(;RZj@AYHBzO+>>bH$(H8vBG021na)c zijQbl@Sx>Jtzv3#F7GKinQAF;TrYB4Y4|-_>kSvp;N}Nngj7I)hk^?G%5g219O*EU z`chWQ*B_5b?NfhTl#R3gs?1O9|MM6v+wB=iqIW5HaC_fTQVcXcyct*6?`9Nkj%q$O zuQ<=7)+^TAwf**YI*UqOV@!Yi9b#X8ATtIme zsSw=&%pJt;JOtx@*y!&=cl*uDYe{zPkt&_w2Cl*NSG5c7u{mfkF;$pS&=f;|n86r} z|H`Aw3UvoIgZ1lKUe>Q$T{Sh13W6f<)7H(sFD2fQQ?7g07YLD={mwQQj0tA1mp&QH zK?mX<#&F89x}B^hXMFNuUU_}mTfutKnsI=9WFhxqd9@K-x(nP}e&ITXj%L8N|0-Y` z;jm>~`n;jgUSZ6xUAA&}L8`*!TQwo|v40$urLU(B?anD)3RF2hHrvdNW)4)Akvd-+y>BMvR^9yI zw1ZxjbMzn)(QsyjfVOA*=Kb>^$y?pHYWYyCtv(Aeyn{XTnWrv9-{?=sg5hYcGwd0F+#$~`RV;T+u)wO#FGnf(jh90%~sM$aprtnWR>^E`*qAb zT(Odyp0BVmxYKvrYzvH?BbmVx&5|r5-UnU0%p-z8W?(ZE4%rp>a~9EL@Q1!d4L4;i zsOMaH@B8lEYNxr~S|sO5xw^yt@|U-D(JS|tKYVeYJ%wKgn_1X#8BkH^b`-ZfA9GIB zy>P=c=d02}%B{YMk%N%^k4svN(#GT4Bfw>e;HtLP9HwaT4g)b>9l}YeH&9x6ZKq(kFBZqqz)7np@MJ(y97VbC zY2G&fecBXtvBFbGa&(aORuF%>Z1oC}7o^f)P3kdZrNd4Ws0ne8l_Y&Yaq_>v9yR5# zo$K?tzkmBLps3zsnRe$f?{gj~yG5^HdvwCr#Hl!SqEtrVZqD^15T}7(CL%en?RFI7x+l2tO&*g`QmS%js@gN5ze1dzWBl zk|N8{T)h0aO`pYcC}`L&tHJH;fXXDQK8O{P0^Tua+BnZFv>%!gI+j`MTj>#g}Kl`*m=X2^92mTOP8a| ze4q8%b~Mnv07SPdfAzHPK6UU5dS|@Dh2@vB(hXcqSG!{|kSA9IJ!t77jO)3jx9f&J0b9lf)r$Q8GR|YLtZN64RMW6*FVP9jyNHY->|U!yuaPr zWAMvor!7UrLPcn7ejdef_9jq#5%LmAMO`4BF20-1r5(DrR`O_-t943H7!^1(t1qCO ztXrXOy-MZ9PO>JSuL_{cW+1i-=x?+S5GCqC+CveY>%zo^ zlV^O5_7p$n#m7FOo{-;3I5GP1GHQBcr)=?-o({wk&;X80>Dq+TKAR3qe}|4-5Se3d z&lG67tiZ^jl%XztKRqL$)))g`Gl4GS*Hl+tq~oy8Zp=#xFyMpp4Wi;*6TmlcSQN!! z2Aj(mp`DsB=DjA>krYv2mFqP=UI|=N1W=41_|}GknVbE?zbcJ~(=jobJ#-2oW_=W+ zdK+0zNz((ge{z7^lqPuZ0enVE6cs7l6rCZRzLe1eq<9b+-N7uSOfS6TbXhan#7%3b!Y-So&A#ZQrhJ2i)zdm*Z-RhWU4yliazXs{!;*o@l6DcT zmYkOiLOfjAU#yoQZMe;>PV8%E=&1%?Np_>j ztMHTN;dUin>>*@tqgMQ%+Tsd&Z(q#>$`Nl-JwepG_{1kz1q!!YH_H0*icQd!Ef;I` z^={WGoc}iEM3VB%M}Xz}8_~9?HFnn2&LZc8ByQ-ANxrove(~n4K8x@#h8cHK97sst zbT$8Si4IbBaKstG>8IkJ=s}@*8s#*sP?}+Ell%>l>?bRtbyYMD&IzS*%H)@CgKHf@ zZ;XRKsVIMx`xEoV(u1x4Lku0hSP^5Igf{>UO)6N#F)=taG)6>|f+ut4 zVS%2j89gEA(7mA?6A!SLBT=T2Gjx}PoB17F;Yir_5%Y9YJ)R@~6dQ=g10)WF?!v|l zCSxZtx}jwI{Dm6Ex6?`V_%sz%8FP~aQ~rvgw>Q0e3+4k!x2e9OsUDdoeoPE&bV;{> z8pn&s)4qp?I|V7gYE+z#q{rLm-k#?mbLm*q_Xae0{?0Zbi={LD^T3TA- zX!Yn?TQdRx036Xc%EuttIHymE$;s>vYrVslct75wTX_`r$v#yDinx%VsO0&tIC9Y` z^f{N4Wnq^##6!5pU*vq2&Y`?>)wTIkcQV}L;@R{7^%Kd6Uz z`Z6vr!~i-HpTDLf#|!#PX#P)+`7d57aYNz<)sZ!M)IdG=HhhK0rA@ud{%0~MqLOluomt)%uF7?LJ;Zn!eU)uTiHF~^b7Cx0-EUitRdT^j{)%C<#XjFXNHX^Z2O|sh= zS-c^#K5gvIiAiy%?|_@TGM1-1qQ#FF!iVzp5;ET!hO1}Xg<|{5-t|@}@^{&9X8Jj> z3(U%u(lQ5fH@d0Dx6rAn!yI*GZQ%PLbjhmSbqA-oxP@iji|?ON%~pJtZEKoFx_WU*rq zbfHl@@0TU#rSty5#>b}RGh!dKVFWdMh(GgD4X5_B7mx02UFr@aOTi<|fjdjMBTj+? zKjOP|`|f(2^PoK$Vw5$qY2m=tEUc7vqamSD7E7?&F(_`FdNCz!wyl#tPcT|nA4<9B5AMZFnKEIOL9MS7;dQ(77fLy2cwpVs z4%lgp3(T^^ncAM@-bS|6%655y$4)G{X6v_eFgIPSM=Vjmp)Qf1#4hjVrhYCr&!MF1 z!oKPH<}KKyBILiWC_H>Y#LpP;PsjSfd=S%vHqh&RM9ITi=qvBF# za*A4QzHtY!j@O|)wkJiHUn;*wgEwN~1;Gc~Z}!*kvxFyBGx=kCPMOW#Kn#bH2EQO* zu4PpEE~Ep0mGtrd!=v-HIbW~1?EmHR=PzE+Plc38>PIy@uFDay8IF8kO5A)cr*Q2c z2)5XNJ@B@;>;HP9c=k-03H<{D_`D8ln;sFWiNU|UaGAsZwnU!vf3q|HXNnm4gDI_? z=aC4xU)pROYanULowcyq9U3|2wpaP(bGuVRe`UN3aJAs!@kAI9(3BwW|$y^R=@_uL>vQVQ|Ol>5x? zJRP^TrUN(@j6Mk`*^hFQTMzpbA|5`eyEcs>jL||v2=veBo3D+B?NeM8XcCQ{c7Y}3 ztnJ9G3+KiQXyNguzGsA)-DwxOI>7LVhVyI#*<0v$J{lOARrQ@2));xw zf1rTRdLRY3Nq|Z`FBDV>S!(-I>gIoBYQ9fEbkqH#(SF0=C$a|TQ`?V)<`3Klw&ZPH z24XY1wSIIL_3?>Wgh>L$+}~jxSvm7!UDy zXEUqzcy<2V=MH2;u0XNz-bpW$p2{QQE)+Q1YaltejsxCbmWC*31@p^$53X__-p|+A zLUI>De|g`Hz}V;RNRxkZ;sD(`;rg1`Sasdd8qzgD{@kG`=`jC0B8}&={!8)8b3Zom zIBdFf+bylJu(GPegGSt>IXO97&;=GyX150VnF!&TxBX^@A?u4_zm>oV-6v!zX`4MXG<5li+}+Oa zhpP^aidvoo4dvKEy!X6?!TpKDT0Gl4R#sN*NQuy3X}naM+uT_?&IvwhfL+Xk>JjTD zI`*&Tf)rg&6yUmVdaarCMDsc1nzv!{ZRPH=?6!03qLR+(2Uh53l^g%XTb$@?^|UGI z|H{xb5W~oNoIcAnS?F`_@EPD!q(iLSvxlyGAzC zR$s5hQ7mp~76 znZ+uV*Uk^jMZkU4dT0K`?QtI&(e@A3&G%eDplhXZkIR4HOjlW)@LX}L5VNgFZn&B1 zz0l@-_4>@mf0*5mqKU#XUGZgSwQl6!9kTj-d$*M+5OG!io8v{}Dfzhf5jl*0lLy(b zE|I~UqmR+uzSZl+MJvUlCK7=z8c8?j$G&LuYTI}RZHym3qiF=akV1}fTVJ;D zzHL<|)+|q&imqx}RzD&^*BN;L_Jo6by?2b~c3iqyeiGuNd{|A|2x%ziL(JiRG(q_K zuc+XMF+orkzAVYtQrk*a&%DU0{mg=g;@uj7wyRyE>*%^we8)EKR1fHHgE5g$?LKkA z+ZL~@;Dgi;TR8z2!~AKfJtFNRDBj{7^(6bZ&DWMTK3n@Cjdjom{Rd9b8n6C$p!J7C z_Fz>G5wB;k8mI8BH_W@Wy|H0P z0i1`A*F9!-2pol9j&6QeOr>u5og}3ffWoYpgML1LG2CL-HPqYGM(k}tHaIw#klbYO zuLj(iZic2KaW%`~mUh8utX~ebCp|J1FVmUA?+udo;fU+?>iv;UoR0-?k#D5|5P>K?8aYgf1K7tZiTWBFzIQJ4FoFuN}XT+Z~>3Y$&FG3XXC8%7Wd&vEoRfX}-* z4ew9$UFPGO2bRyD(e5{r8LtS`%S^h39|>A?Yiuv4Na}+R2Q~Zp>($(8UK_lLEb%iA7n;^<+eKj#vK%FL3LCeIE+1@Td z9fc>Y0}_`sH^)Hzq^z~QB@Y-Iz1*cqt8HFld{8?+ek_}dPh>QC*kJU&7mt4#Pr2=A z_Vf-6zIJf=*Woilnz@NAn+N36qFSSxsHm zx>PKr$V_>O4stfJc-oR(&jgnII^$H9n(wO~D4eop?51NH-BFq1W(s7cXv+$_4ykO# z#1S4l)UjKO#tLvV*If7CR?xv>CDhTXVdsEp9o^%I5W6E;BirvY7vD#nJ`S0ABToWf#Dx~?MZQZ-^t#ohWO++=d&n9NXn>8&+AAWldaN>}7{ zC@ZB%X^JA$1y*3q9Q>OW;3XLEPq(Av=3%$SLOJ57q3`dq}kW0GRH>jUgp1r^~=V9;b7<^+w6mhKgFqVGZa$6v`XQ& zB6EgOl~}{(!~ioI9j}NM1REg*!@>Iju{RY-lgQ+*qm9DCapM!+Vz53J<>jy@t*zB6 z3LQsZH00{Dhwb3Uaz@`CGLu?~Q%HInRqe~3f z;1H=X!TnV9Ub|#GkBluSnF9!>pr$SuPv%*%$j)~>nffFK z0I;sF|DNEi85u(e9b|0Hx?ce(Obe$*pae;$wWN&IGhkw6=Tv2+ED6NH?)(QHmAQ@D z;OtyZF|%tPfS(Wm9mJv`qj1Krw{_C`H(1LAk|Nses~)kOu)+&WE8P@p(^*3A{Q6J@ zG+K6{rN|nH%53^HcE`pgeV8*&oM5eQdd%eb^k-yVOL1s=>|VVp@*6KW^jts?W(7*BfStZJ)@c~h{yjSJz+@Ji{k zpRsu%?9N{pplQZ6#o;fAvr+-?psa?|7hb5g!fE0Yqh@bC#>|@OOTd96PY0nfQjl^{ z8LhdBDqSM0h^R-!NT&&A?xd2|;!z(XWt7=U#EX+c%8zwdeHV0>z8`K42BOW%(el?e z7``zk$l6#u25`Q?lG4%XQ3TUuh#9ekDxH5)bM+?ujP+;zT%0PJ$6|VhhBtO=C zhYFt>#~Lu3qq&wmF)}hCE+$VWA^Y9}mW#2Es<0@bJ_x6sF}>)gO~4iWO7^PHE**1e zZgIA60n3PT+dMvSf>O@#CeKal-8%~li*MQ4>qmM<6QgB2IXYqeI~Ep2xg{*gIkRLY zIwU5;Dzb2}7M7naY(2!OHRgqNGl7Yt$ai+or_CV|(^a>ik&El)+~Fn^1WFR|yO$Qh=hb z_V)GxC&;g`-+oL;mk{9z3h2ZhV;^e-K0`n}p7jC`;zv7cYcwdz0B$b0mOycvuF;TC z$^O*8>k02QpQ&gBQoe`JR#pHRnTj`Yl-Jxkae2Al|7Y=z>;39VA-~YYH(5FKvS;-D zlt_ph{(q@5{J#_OJZTrn@6kn)nf4Z&=0u^u>BUTw2RC$m`fup_6H!09%P0pBK!Mu% z`MD=_Q3{S2G#ju^;-K3wR;DkELIxOHen9&TABR2jw3+fU=@8N@38 z=c>s6`}h6-VX4m&QUy=j=Ia897-;>SKAr7O4U=!>nL@*zhn}$KCc$`8Qa%3_NB&<- zIs65m|J{j<|ISj0iVKt-Xd+G96UaZe)6DfH#u6Z7q*a6j8N1KRW19vc1)8ZW6H!AS zxlQ{ND2Q}m76ga}ILHdS#E;Ft$sTc%Y5tI{FaJO(n*C~Rim>oC$CT34yyR~EyPaq= z-F`DHVuoD{y#5FP00-LC zt+hlf@6ClpHElvZN|HJ2$v@Q?<0*{J8zyDqO~il`j;7>Dx^gRGpxZR?7u!`!7{?<} zyff#ML>Xk%6MMQmw&Uq?<~%%QNh;_@^N6osFNj-O3YO<8OJB+LSY)rObd13G8oMvT z<}_VBnOBJBZ-zaPfL5Ca6qwIB!Cn0(MA1N;#h(s8ce_&G564W!5(5P%QMvpleh!xO zYXwqzN5W7KMAhn#0$$`Vm+2VZBPpwML|=EQP6Usq+^;bxM~mvyCt;T#^~vnK(Z$s%4M}ok(AMqUxHZQ(YEA3GE`jC$-uXpBdKWgfA5d~J_5!DlFJp-o(DF#O}GunyPx}9nrEklv0mDj z7l|DAQ6B)@N8VV#mr_=uhP$-EEBMY5p3$>ICKR|-OUlZ~nUE|bsV70j3A{Fm+``rl z+STB&eupyo8zVEOd)k13@tjkD%lI9m5(zKJ4()n|-K~Pox5_?ME(Bdgzs_$3SQqmv zro_aD;E7o#diRv@j(w$mvgSC*@;=xPzwN@#ZeE;?jld#wj{AgFM^*X&+Vo=%wEbvf zS+ELb%CleC4s<>k_VH{u6Q655;sr8Vu3@iZ?m0TuYpEyn+|8-v=;@u zo;l5;u+@7Xe#+P0T^|Vc_?fyDZl=OQ|NFktk4&=DB<{Bx(Q=+<00MElNcsSsb(;7? zJGhBLk()8C9S!9(9h0@ee5WFR9asA@&zL9x8$03SBIb@8K!u1ZKeDqRQ2Mp{s7tH2 zJoe^beg&^DATZE`+NHBP)tD~L0WKUREQN|@;-qO@V(lT{rl1-+Af7fQxo@#UL8tS~62*;y8>AF&9y?p! zXk$qH)Yj=qV%|0}f`5>>6JpXiqx*+w)KugE6qNF*vIm_}*uWJNws&bEyQ^h6P#2OX zw#g3FNPy&3;LtLg5ms(~*^U^eob+VnrBbpLzT6hTV>3H55k3B>dozj)FA>XUeWQdT znjJ>fb^gW+L#0A;k2~Mpz~UXg*tb%LS5!na7(dGj@ud}r${}0Bol11#DvIazM`Xm@ zfhCRL`+_BOIaw-VixMk=D(dz(xTvw>oK*Ql=$9q*`Hiee2DJDVf=BBz>BPqN)&%9- z;}|Ua%kfl~l|RPy&3U@k!VxYWq<>;Y5fGD=s`jkyx%=u^q(|_%H5?{ zw=etauXns%0m7#TSx%;1UHZK~Wi^k+CqQ%gZfxceYAN#R5|m z{4SJ@%B6AvGY(0m1pYKUqmWR97r3$U>`~MH#Qa$S!>98(#-Xrj>FMb2&bTGhNEBW_e6IxY*fkP57Z@huk0?PY*AX)1G_9I+85#W*bZ_|; z2AGdN{;?(uc-XZanxrI3aa`?%ioS;NqveJopQ$lAuHX7oqQU}o;ZbH3de89b5le@i z=|_IHPL?-Fk;2qMER!8PUWtxuzjfMwaFrXYGWstNRqa`97_1@$DKl~;@!LP8x%0MJ zp@9x;S3vmkM@7F*1I|;X;UonrX;cwXE%!gfp_->j74EXEngnW<6y?$4pXp57M8m=j zeWs9#+gX-NE|*FN>CFt05Zw8<6%yqITFt>Mx(jfvFJy4_G)81klkQDaImXp8JC+Q> z47`Qv4oxpRO7KkcBRQ(~{TpQ#$x*0&)W#6Hig*da}P6{SmiqF~|#IeA`nDnUZ#PMjRb8LOxYSB|4-U)gE-g#rZ@eV+LXrGY{DXnYeEm1V&SvFjSt8xv zk$XQ%bNJs;>3sZP=k)a<3x0w3PZbw*YZKQPMJFI^gNyU%r<%k?X{OwSk19=P^>Ez= zwJvu8HRaj$$r@EDC8OS$$4o(EFH61@Wlqp@OMlTt+h5?{n$Z+fC6}=@#L2zarMwZt z#CG_xS9jg1QIFt%}GAGBVxVM@3cOaUPJg5^0KK zT&E&=vijc1`OGfl4e!gY&d*jw49%mf3XbIEAS@LIO^uHdKzk;56wQTcx#dM1txMO@ z#iW3Y7h&o)-|Fi#R%FMjSQ6i&@J#cGyY>iSCJXPTf6op ziLLYL#$>cT{GulP^Za<%aK$xiey6Eogp!S*)&TMfGBP7ou84Bv;3!|xB4eA z&#Ct@#9(Lu0Z8s+WXd&@#pg_Vme&b=%3%MuMWB-ZS_GQ#@9RZP|DEo@|M&Hx|DU?F zwhL1_4rYk(ip{6qgW9=)3*%K(lgq(U?p@!3dY$72s&t{@5Zk1X1BnC~Ph)AExwkhB zF3upTa4Q8mBUyU6X{ZK4I(hu1T|{AXReBChO@#owUC@wuI`Ciw5EqbwbuaK6oyk|?q1s*VfQvkF`HBweH z^(FD=kw<^_(OIfzwHG|%Ithpic$F8M3td`!#od05AO5*nyyw%yeUQ#MT5o$&pm?xRUJyU8-I958O%~^cXz|4@V*(4Z z)O-xH?otoDXvMGonX+D-Y_s%W#~DsCI~0!&D+O9XVP;rKPdC=l*q!INz|*^3Gu1DM zcm)<1Zrfa8_z{hf)@U&4umjS0ow3@Q+yYMht$6Zr^r~V!bG+q}lAqU!Nrofy8+%=b zhQ-Zg7CYiQ3!cW5n|(+DZNiFEV`vYFN1D9iPol>aQ-tj1g*~CVfZz=e)hAb?PWTl6 zvg=YJIr>zE(>P;~7d6PjjAT0k=3Kc?zq97-W@p-2s z033-h+AyZ&wQTtDC^3~n;Pss+&S2)9QAkcsYUBLnNoOXF;HEcmiu2b^zI$p(DBISw z!U6>JyO~Q<4k^{U2*I0Urb0aXK2D#8nsGJbeQibEU;xIHExI1--!co;#6~udu^3Z_ zA3(tV665oxq|!^)#KUu3z+k%GC}@W8TPR%NE;Er0y`>hYr&U$R!q@VPvt(5g4Ks+{ zY3$?&T09LJShvkzLVr2+3VBk_%67c*xz9MTUNRws{MxcY><;dS7`&o^)Q^dfqc1{b zF0SLadZMIDZ=vMinGKX1%#(912=5#o&+z$7u>b?$N#CM=i!V!VCT=y|?6TA2!QgIp zJK@{)8%$?3)Q7uG;j->8wz%T zT(46|df9}Ux7#EfAPPH_Dcc@chBegKU$=*U*q)3cf+KtcgGf9#7#`co?{Wxmp>|{S zigP63J|T=*GaYiJ4wa`tLmS#0!iQ{jjsG%Pxy_;U@-nYG&Eie29@HE1k7YhRgA8?P zhC}D9MX%TK$$6-*)EJw46?|Fq&6)diHI?@S=nm%Qez-pAt3Bq7tBVp+z>V#pKAXtlsMY^ z;Fwu1lw=+D0{t_Xp{D@2uC|I|5bJoS!G$^=jx^B#HnVZ*N`Vf0UN(@Sc+UJYqMeM5 zkJr~FSM&RXo}anB0AH~SR~5@|_XIs{wi6-M;Fd>`4Pt>0AVFwY9=GUx0X0Pg7Z*Gi zU)`p;OTcvIuzeA~vskv|<@m1V?ZIzadm+vhmHQ2ONyu+9lg zv^eC(ElJM2=4ROki&^sM4+~ttrJ-ePMbYve^ zr<-{?+@~jr)Q>4~$NMC_4gnS%pZ{6b9jr_q`?Qbz!~VhQVZ+ZKO&xz% z%*RpwpM?v*i`vZpO$+c8#Qq=LW>XFiy|RA3OvU!7{|8-MG51v=_hJ?4*ezJN;hI{s zEggF7a!fS*{wg_O%;`y`Nx6;r$a6yQ@IShH%c!{8u1%02fgphdf@^Shw-5*#Ac5e) z-Q6w0-JPNc9^BpCDV*T$S`=336nVaW-~Q3v)8AU%Gd;sv{Gd4N)Y76=-tEm=Wz+^W0|5L#`2br0vUDqlXsuVvGlHv=0{T4H?_d(JS|dj3UjjhdO}Wn!daBe2DRW?DYP1{fb9% zZ_J9R4KC3aO6R_GFbJ}d}W zG8ABF_ETlp4GujQEbo%rc~BBp%27qaTI;A~Jc#~E@ISY<|9dWd-m%=nSlDO@<6Y8? zWrPT<++TdMpPVtado}QgB3-`J7sdWeTr?}izy_#~D>|xRJS2LWGm#mCnpmq2dsJK! zl*JsiFaqJws|s$zj|iFad~^{i$<>aEce^qX__yBSU~6fAQYo>sTFoJuF12`XyXeCc zC>%T~Xd8n-)Hbk^%5LENAnN^)IU+Kk@800v%Q=6{=mYAuRYZEQKqs$7q%k^ zJXA1)t`!Rqb+?NE{Orm*HeotA04eRTD8*WaR!@mAEc!?j}rwo5=Ccb&oySw zGDGtE9;d4z;apK=o1!zH`gKrD*ee(G0RH}H>M@J*Eu5dM_O}ORe{5?l{-zI_-ottb zRdwOA`jx3lXwV4c+t)s?Oq92J?1q<)JeSo4wEDt20SfBRFE8J~CrLfOiU=@ENhLo; z?pc3q@XHNBUQ4x_>$|i*tmG>%_@~M$!9Q};Nq5qRVmMny)1{A~bqLqL5xTV-{SW~E zfh~LqEV{d^gRY8it*x_OR34f^%HkvFw6jUjhU+Q#+lZ0yA zsgmke$8lemZ-EZ#9M19vcT3W>Gs`!S`sq>2Nm_3oC{~1JAIj9+uG(*4uo;Z=%iTgT#wDh>4(aQ- zxn>HtVR<;AtBmUV-6vl6-t(v&nFB45&Lj^uKvcM=d~o}USh{_>bKU4w`_ncJ(9CKL zctqOvaM{|zaV19E_Fx}nq*e({ZS{JXV8H#phX54#e29JgWu=2R(>(7{?khY!N=&bI zEYkh0`rHlAS|Zss*1*_?sT8Xh0_RYup*W6&Y|4xK>{g)lD#=<|FdLZ9%QcWtbF(Sd}>PPIt|!yW9oB< zId<;*ur+ko?fYOJ_jGw@%4!Znj`?^#8>A_YG(seYfx$EjbC$-&7{bPq;%|hg#j}-G zzY;TorGCQA-4Q3QO8~I5-pZo=EK$lvn{sk?((w+QW_tiz9|<82GZ+}TA6&4w8GU+< z`8d4j4w1j92W70EdWj2=d`}h*)lfH{e&v% zME}JI3|Mb<^C%{Y3jv*2a|Kb#UL)F}kP+c;jLH=|mU=o27h$`7X09L-baS zANO-y?kWcvj7K9asYDN#t!sepdn(&&P9gLxz&N$OG%7iMR#Vdt))T8d-+kFKRtysA z346c?b817(pP)Rh?;T@TuLVM}Qq7Wg^d4k4J_uyc#8k&rHAz||5e6PCLw2ttFU5UM zPIb0Ht{9bu)*`&i=)SWYJw%&gdx1P;s_kAr2!~hQhx=TJ=MBYHoav9h-3)w2Ax~Je zD{pqySK->FUqcuj_cgNQdxqeZd9^yee1&`4TB^remlR3 z<48(Wskk_~ue5%&Nl+Xy+brl;>Jgdgjh&)qVr5*%TeaRS7@X-*Z~<2n{ zB%WzF7Cmnx4B zKnl{koXWfIxkrzq$1mXhMcM^v*g~>Am;5f6zDvKbcDl}|3;&*E20U1I6ZjyrKCLh2 z?1{8E*Mr6nckf(>jF62RJmt2dvITZH$KjWKztGjrC zv@+S7lcD1N{Dz8IyeKoC)(=CK6p+HX)vZh3K92yCk65T5=#m9ZwsqeGd=w47x@5b< zB>H^|dmXZhOUOoc%Rti=ZT4ucnTw#Glz3znI0N)M%I~79pkP$^;On!^*vh$6&9^&q z{T@(ov}XgQQWttvgQbj!sNm#as38K=&9^~y)=6kD4&qBb7prpwiy-KHG;%{8-AGGu zYe8e+7H0=YhwyS^7NfYy)M~)X11334B`{$09v;chj%Pl7rb_Lh`gnW1yw;UnLv!Zi zio!zu%!D0Xc%t(RPJ8cyhTPoO#Q3#w@cxNyf`k<2PcwdFNT1(WWhP3ofjbqpy@xQp zN9HZ$&cXdmcLn%He6xTyI680*ufNI7*eFF!C#-sRur6G;gi6rd+kgmcSm4H;x-k_&kmdG*bgCpk=U^<>-)79dgnbzF4U8CAl zH3a9YYZE(-Vxq?|Aa~K$YbR*DK>#HYc2inWq1Fh|n`Q8vh|ZAq?$rc=y3F=cxDhZ}r1_m~-dejy9`dEry*BI3yGr`zh|CEl!%M;qIi_xB2eouI$j_ z6qcy?8z#y8pOJ#)cwqHQx831wheig*tOP%YE_TDr^)04%?Z>$IH;KDq9-@Rd@4Sw9 z4#{OEFPmWbS0$6Xj8!Pf%a9r5&qcr?4vmNV;3=%ofC+KMVPBh-rLbfzn z45$ni9r6S}H_MxMEdj_OqAtaOF(My!e2-Ljb`YN%MyK>Z&jkQ&gU5aw>eEbQx9u66 z!D*uVDI!}<%a@8(NNKGXlwUyqp!y2s3HE4zXv44xw~nYjPuiS^|ekQb!DYbRaw8TJS( zhF-U7F~Mj|Bztsk6aOOasJzKx_1w6N$7B(4tL3kc=gGE?jeno!Ifu6*x37CPfBC8^ zlb{m!?+sQ3o=-K;o!T5zKGIbAqd!=ng4Nsd;>Cv8?Z1Na!{pl+h6s52By3f0qlIRi zZp|Co+O5W+txmLdg2^r+U+&nXK**sF;RNV9RVTGc&y|fhC+g?>)-2Vqfa{Xl8#x6o zQFnvgugDys*Jw5-GW*9RcN1*o%#HqwKjw4}Xs?!aA)>Yo={NnhJ2%mfM41^i2mf&= zu6NJdoNuf#Aw}!6@@lsUWGCc>{3L9*7?;|kBf!c2?GgI4$yEP<8F;@KjocWirLL4V z+$D3UWBHHZ%9St?%r!TBA4BPN+s7#{iiAHDT)+gnU<^`#4L%iZI-5*yak+3-mD1VW zV%1-di~erBJXcov{qG7DJ{$QJ@A5}KQ)H1o%*+-`1Gl?IvWJ9wG7Lu+2V^K&V0?Mo z%aIg=?KSRygy-UZe&iKre{EB~Td-h=ya%)A1PPNn7#)<^o(htz;)l8xxVfBRNVoU- z+K_GS<+eAic>KrHKBEF#hn0??6Lgi-^^S{M$+n=KwiaEm)%D@lQ~2s8)EE5n>G9AP zWUVVQR!x@P>m8rZ&^qhzSLN~@{>SAmksIIGmeS{fGfX%#mrx16=F_&j`le_t+t%Zz z1;OXt4zkkG(0lV&X~>>gC~}FawtYGNQEG3te=Gcd{TlqM&2#AAycPeQH|GDNm)5Jp z%M4kuw%=F5UbKA#3z#j#6@#Tbo^8P+<NS^I39LML7ysB%wo zLTf*~c8BugFok?kRrzG4GcM37u5j#~5cGhs#q?dl9bFO)bzmsw8hG%#prtZAkm#x+ zN60k{ciF1=Pfuq{FjP@cE&g=-xw^Se@=Pk!L|2|Llt>*Q9Y;)PVzA~O=2n({l04r#w0 z`a>{o;(M-A7kZ`B-u1|+xRxEkivGV?6-?e|{6iVNs3;!3i5p@Has+V0&XeeFQ~Tm| zL=081pIt;S?{vjQl?-XH=c2j!yp-A#Ib|UeYM5N%@3Q+~B zBKxVEBF>zAzAvqc=T|^eYh;4)i(I~Uw#IWSe9I5wxv(ayfSeMahTAZHEyd$PlhHDv z13^M{eB10wY6Z}jn6Mol3sly5daWT+TK30+vLJ^cLb&taRyKJa4vXY$e`JQoa0LXA z@hQTQCS)ymOS5mz0T*!|`6xT@lr+D}j~1~twnv8BHKYrVG9R7po4O%Zp?38!6c;iu zwm7xTtlM1G)l{)g41Zye))7llzZ58vt$=Oy z1;ul_T~q#gr#^Xj33<$ciPj~}>!_UyHYHgNzVL~kEF0Q)ZeGeUJF^xv=e5e7zsr|< zKj_eJCo(7Yw~%YcxY;?O(GagJE+kRQKim;?le*9i>w`L!^0~&G&zjjvV|3zuj%5{(Ip*V zHa1y#op@jh3c@%Nh^6_4Wo*n8Jn>WK%6g&0Y>hCGDdelhJE|3r80^ls3=DFE6q$jW zVrH(67?*c1$;$)O-6KjSbl!r!xtv@k7RM#zZ_fF(r1ouBWg$_B8q{yHk&qH_zqd_& zWH{jY5@%*;Y7pYL4db_ZIgou*#SVGSg8r6aQ^LyPQtg1@ses~w^KY4Qt(9Rj9+i(u zi;Z`|I7oTMC7P+dW{Q4M8R>i?6_H5`XM#j@`5xv)%RP_l@Y&ll-?sAb_XMOpH*x9A z@^-qGF#PQgEQ$6A2Xli7HTVa{|V^TT+Fd6vr z8QU@$41ggP_Dk~w_V;AOf2G8IB*!olv)46g|h3vVp&HKU- z6$3dAa@`V`J?Aqj=&z9nF%{}Zg|eynYmQH|ipjb*t%I5c2^d1&Qj4jnu}bt|S5A;N zy(F(`NpRnknp>Uh7~7OVRkg_}q4(lyrvuXl)~N##t}TT$>7 zD(m}NA~Y-H4u;N7)ll<*Yar8l$XXVJ`?Bqyn0J|tFI$*iF1&VlU!0Oytfok8Sk#;v z)?+91&MxQ0KUEg@-esEdGiz*(*mnf^t@=dTlt|xrc$I~} zb|lu6Xo)XI;i@v1wi=BsZgpnVB$jNZk`$+yqCPW386!SpTjc4v1x5-_UZ))p@^Qf< zsiqqB1c zd5s-4BVG{x{)ve}d27Z6{Q6$tpW^h^V+1VCzNWM@>YuwURT$rxe>lzQ2)tx1|L|iV zs7cOmJk%OVwy>}nKL8#Pf~Ry-*B@LUU2|+2Ks;p0@-mB!$JEl=ofL+3V-@k}P2)Fq zwtW+zlL<(?6cT`9ASb^?L)fPX3Syo8Rq73Qu3EYN5zAwp@OP2k3xpnQ|I@QxzN^gG zpILG{TBDcHgp+R^`+W(M^8ozyZWj_fEs;j6< zYG@E_7pqpAC{x}=BWtKvCR!Jxv04fZ6`t^UYNh?rZ2##TX&=`yK0YoH-ZkVb;TT8k z(GupYb&UlptZ%G~O___k(d)0dsex{)Dzm`7v25#h?0~&&Qz6=;Qvrpvz=WWF>@ww6 zVnz7gH(hE0I?x13)F0SRB<)`1y%Lnk7;V;H^u~6{MVmV%d{ZFhk6`1no0mn)PT0AvOoo|Kj?iZZ z2X8x3=?x>5bjTMvX;q&W)Ol}qv~lk-T8Tm+5M7Q8Ijidd z<-i~%9=Y#z^acN2M@K(pgKO@2D*-P-^#1UWQFrKD*gNtxbG^*NTL&v}joScbxtRJ0#w z#dLkqYNPAeHP6}PM~hS4=)9h=nC*Hjxe^| z-ZnOF!HFY?ci>=$zs6)wAVn)@6{c(qP_s7ZFVcM3ENC`AG1A$Gfu5&n#stdb^QBH) z{LPTzF~|RpF;>cVgr)TMn=V|O*d$1|x#RuD_fd9~zN-#LF*6RzU{;jpTsJL@gePGc z`~YHiUt9plzbFZi65=XpzSYYuNbYlzzqt03&55A^p@X8-p?>_EPl+S3}-rD-yjKRjEXbN}R zd_A#@u|Ck-$k8p=%{Kl?Xkr+pSAr!~l5?c!9A28#z|`@Y+EMk4nzJ@U!tlu?MT-Q!5lG%sKC`uBKW_#?_gTZ;@@z zP(v!3gfJCqCMyb_!mPLxnW9t^2$}Ho%%D0@Kb5XFrF+5=P${s1cZi1l~@I|sT z!u>5qRx9ImE_!0`sFRI@{)gz=DF6c3e;?kuE$)$zRS@?tEC4-znXz22{lRtZe*-B? z?Le8+Ldl-}Q=IC`I*a99PKy5d5bswVw}*6wl9jIRR_CJyZdHFX@W*pM@r!Gh*ge+V zHfuDw)eZG-?n|aq_XPhti$YnZ+y*vGNyYI+&JS_K*_Y1WN4%At>Ub(4U&6ep*|YNk zx7Z~>oi1bhuOtAo;yE$#iuw(4 zqnK)yT&_*lG*_R^VHQt7!NH(y+`du2%ia76Lrj6sYx{FT=SNju9_5Dmz}W}wntE!A zf*%`+M8C^e3T`|EzV9SYjVFi?m#J=K1pir0@Ai`>(9$d#v?$KKdg&1yWSM5pVxvC? zL=3$mRqBKRYu`<)sC`v+^EYhqmn*dVE@f!Qm~nSKnW^@HNdVQ>v>Engr)F7hEkKgH zx48)XI5^2lt$7CvS=Oon@}r{xT3Ub7_woeoi9hmhB{LNi6{V>p1D42RrL@e$Y9?oa zzj7|g3-w+@k-pAamzt;;O@eomC&raNc*-#zhaw(_QU#pTF_q@F6^;C*iF?aCdb4%Q z%k1xzmnqb4zi~ZZZQ8!MDC)3fIO;>39Xx>b*zq5^VujEAXJH##18VI_+av?I*lhnMv0*dZ5U7 zeRQ21zP5&#lCCy$Mf6k4hUoK8#jUENG4o1soa)HaEZ zmaeD;l6}*qlHg1Q-0PtHuQNV>)T4<733sJ!LteoHRz`2jdtQwU(tJqca#U}tOM3T7 zY>dP!L3I9s!Lf=W<275sh0m-|MS_MH=s5~u50k>wHH6f1P~oKrSO ziRh2cg35)y3)DOD+M!i+;6MU+#v0CFSH2$~4VNKzj*G|nisJ`E&QXPlD2`4x8fp#b zjEGw%+uz=S938^MF>smJ$Er>FJ$wBcnY1D9JH#G8#}W$uBDcBvo_jw-w+8)tK)97R z3cpwtHm*?QOqeM+s^f10xtzeeZn=AO(WSl#1;EoSCx`n>y#VA{BS@Fp?9|!Cizr}Y zj0A3_*`v&@&+9@%DSdl-xv;T`hxca?5rq|v9&hKad z+bCY1@9hTEjm$DU?1$C+XfJ=;4E~|jwliT;-6x@I>y@D}C>w8GC@p2dzR-=gi|@Ak zYoQy&%SsKLWdbUD4jJ9mVI1bn-g|z?{^1cN|I@A!UTMl>7EoAvvrl1iSBHjsR~I6^ z>^%ZMrai$n>aN#vSM;oIGB6uW)nGI>{)(0#;Feh3SzZaoiV^8AF68sJ3~x9i&RZ*? zE>Uj|*jtj)c1;+(Zk2-o1C2F zaC#>}hfQ9vnLc`rxm?9+vL70pj<*?Ji~>LicsePKfTm+itT5iQ?H;c%6YXtg>bpHb z!C+4l50}3=SHovGSDL0SK;ldX=;k1}SZ`~37q4h%TM^&wZAbISrP!~?!6P6tI2d2f z;kfge&;3UM@OZ0h@5Zq}F+Pny+3JTNUi`A_kFmAvN;dAA5H|bQ0Ki~2)bLn*!ZY;Z zfqOI<6a({8Hs=d4*`FTYjkF$}RT)|~ur{U_VA7TgT}_*BD0*jBUwcL6i3mThR2+bHJL(ku^d>2vnfcUufgNH*&|-J^lepl#EAY&I!>&$*PuYXA1I)++DpZF z^p~$j(W~oRpz8Pn#Ua_%Ah3bc8$*&Wrj>3J{EtnRZy>A}(iki|Fy!IZ{7$>x1E*U$giN54#cIqG|mJH*6ME{k+kabY$=%Zu!RVGy?Gz>bGCNexS|;I8g7O zxSUSbGz#ZwQ7>C2=_KU_%7sXaAv2;RRals|*5=`-rVXu7mMEJT3wC?FCrUL`SDfwZ zzIv>%M}L8fVwIk8eaPy$;|_#AwE2*l+>D%Fdp@h&KJO%Wv%m27?}mc%*i?d+(&G%+ zUOz7|6+QF6vfi?2&zkx7)QlxqH6Qu<$cDOLqyT*8XT3=Y7(Sy1=|{aUCZxPz^NOfBIj3NsUhuPvh>U@Jt~h-T>wSj*UI&25 zV{T($&;>itb3v&iBcK107$W}*QkaJXiMs8dNu%BUfyBZyvULI#xS%P2!*g{geDk)SW9~e1ruVwWJ4gHO z<}p-=$GV*In&omFUf>yx|;LY`7a%EMJ_!P6`WR#+d; zTM`1wFD-G$ma^ry9eg~0J~?F zgCX>A=R3&1!w@wt1$FZo-qye^beIjHB zSzYnc@=DthplwM@?-^r^ynSv6Z2IWjpCNKcco-e($Zk?jsBgT>ilDc3|IH6s;2%wy z*^;#xGTi>Lai=}RX9|YU6ojTj9bt3f#P49jB1+LUQ+5YN7ACpM0>S0a;LW z!B#K?)UoFr$+EpOF||(Iy1bIN4M0LPgaV)A zZOv8Nrg-EP*Rdq@epu~`Zj#``%RioH)C#!5omJt-T+W%)HTU#9YQY4NwG&Py!Rj0z z8h#~IrDNPSYv}%fqYr9|o}7MTO!bbc-U%ssCfSPvNswE@)e$XYf)7s_#z#h$Yx0SC ziZl8y>U=WGUe(*1MD09(;>6ploidoVZf=#Wp}AB9E~ipKbFn$JkM$#6^rSK4@A8zA z8RnVFwO2{@ezWo3RAmRdSf%C1+gqQ0h__sbe3|Z;n@fy4I^1a;Bh7vm8$$0^5f!ws zQMk~*nXNr-}wRu-8SI#T^5PvK#EDt!6W5XTE z%U)+sGQOHza`4-e4*Md;U#vQ#@YGtC(dG?l(=Nymw{EFpm09i+@<c=Ote0!XpX@XujO5H((68r%;;Ld=v4Z0E->=w1~A}c-sV&f zjG!-_E$mraZ?~@C?<&98!Anq{I~#2DhFDU5+E4?M^`n$($y7k_PcL=zH$so65kYq6 zt=LZ2n6aOcUg4=tj(-t;xDI3wYSJ>_kKk}(|HM|1Ld{6{yW!8klhqOOn$)UqDA}8A zhbfl>2UEaPC{ET9Ji~$N7vWmc0kb)gwpHD+q$>Q#(pGd(xoLk83Z=F@jPmUEnzcM( z+nGCITi~65_IKDD2(4}x48FZ=nXJOc8&X$KcI<$2i7 zSM<|1kV;ltl87buP9q)NIW&JYgHa|Li~qE=hG>N1pI#UIAV-QN?o+&C|Jb?3p3M%Q zMQ#e?isU21r}Vjc&wq{gS;3yv!gw`HIlz+e5B?y{yVKl4cd}pv#Yz|w=C@gHPHa?_ z2aj-dlpZ;pf9a8+U0BJG$87R$hjcQ2#0SL52&q~l=S`rPQQH9j;*za{1^H@Ki4`VEukl|ZwcKjsuOS~+x9iU9%69-9K z95iYb8{W4{dK}DF*x#iM&iUTSiaHj0dLTotPjk6jl1vZK$g#*j__YD61BMRNbu%$2 zY0ZTkQf6CB)wLaRqUrGX=cg4$i)yWONsMw2;XI>im6U3tCf6eCPS@7^d8)kS!#Jg` z9k$L$Z_kGcb(=G0eu_l3x`*_QDFMq`?Q3n9>WPl*px$Yk;w74QIK>F!T&JGthPHhJ z%%z4ivxxzc3vQ0Kl*f6Xr5pU~wT93#abh#YuZ<4E#SZZ3*EM!W9Tb0>VvFozs+<*L z1#Y|rK1PGTdB6B+X_=-a@9mx-ez!vPRatz$_qB)L@Vwh{B4*#7^^(-)mlv4j3(_)t zpYFdUfVJ9q;_B^usP@gRsH+S}Kqarl;erHKP3h|65@*~4LZ@6BVkwWq|LBN9YsIqf z*zPpr6J|@ZviKx8X2rc=>8rz=+a*;UU%|e(0v@F|s8}eUwTU84L%B){vdH7X>d3*E zq!f;ak_=@`ZA7-)c13AH@}L+ZbgU3=$KS#-m$%Fbrn)A43!~CQ(aZZ=$G)G>SpAdp zeRW&6hDRPI@ieX{*Dlhsv?tgf*V|bQF5_4s*T;A-Zjw9YS{|Zb)p2|EzK7{?^i5(8 z)qa^YwSg1qp&nmB+K?9t&RnI9@jGhhd_Xm%P*YWQ&nw~2bd~uampgZlE2Y*(id%EekG>wUwW3pO+zCUUNTo!ZzizN2ewsH!6~udZc?D4 zc6HD#Gt)1T}qazMm`YPZGSa&lxA5=#!eb7%T*O1Zh;hAsj z{Z_6aMr9v2dom$-Z@FYcCb@4TEN0+w$2JcmG*a2dZio~Z3!ARYf1gZ_HNr~E;VUU> z`gsx)vFx5W-(Zx`VmNGY^mV+JlK5rz@Wpk%?LzY>Zp(bhZKMGf&zP3s?j9+dx`(2| z*l3M}Z#e9S4k_m#Ngf{~T&LF%rDG#Nqbuz8 zZPyEwE!8hqjM4VV(dh!>mY#r5&X$FYqqKs8$&*d}zlS5IRy=}muuu?b+oe$uU%m97 zgU$+lXdn-UQsAHPb;ZWZge4_9E-y;Lrv4I-<~ocL7KJ&!GTl|y)EbVszu%ebW6mH; z9%Rz)9Ev;IhcL8zU=$b2(GOr?k1^-fy$?2G8X8C>tN~Itn~*Rvq7J+^RsNzX0H&jFUE|QvnHr~z!w^J|0 zuS%2Q97B9+Z~|NH>F!~h)jPHmQn*i!Lo){(B%ir4JQo4q>Y@vC4_u*zpW~c%H$*os zPwTq3hrgTPLMRhUv{i&jA|fy@C#R<5biD$4)Z?fq`rj$_gDNKBS)j#H2Sd!bPwC;VWrP&&HRp4z#{0i$~pTmDMrL8y<_R9wgaj9(XELI;bcw%Czwx$CRQE_p5-4`L*&V5Qxut>yev+p~u)9EU>oea`xs4ta{v6R@ z6Y0ZW+ByN(FV@3Vn>@<+i|NfqKogYwe47rCi2p zSz`>gKJyBF-n`(c$$7`qZ_4yH~`|51sDIHxx?wn&F zlPm=YI|QEh`Sc&BdhG3Nc_F!_Dos{Nu~clN)2XTM9bcZa^Yk1}OvaJXHP9yCFrwtG zs?D;3vo0&G;NqWa9ok`Dn_L0j4;82ZU< zkbwaP=kQ2W=Y;#wd31bahC;Ud?wR0H_Kq3wWVus(+tT>4-<>}ooq z6@JNvN#ono$aGU}JwtjGw<0!yKzCt0t=asP7$_{Vql*Hve5^uE!}_N-@|11KzXBObYKj(`06#1)v0X2UM43 zC~y6VB#6|D z6w$wNA!!&uiG=t{=?gK%!^dVN!avVO z9KLD-cwSWrKGxt7Futj&0z{#AFIrru^IJ(e0eJ_XA?M2;YHVI2VfSY5loFMe_RLBm zA=f+S`{_Xres8xvF{Ou$i-_bZ2YGJlMh){v4C@~sXdjm{f-01=%fGCq2Ji#5WWWw9;EBLa~3HN_OC%V?=Iq@Z?>cMW=i zvic03K8;Gy7b+ zk`=#00FYLvJ{HAN4AlAXuvZruzi34=2QF~dm>5XQs5lq8*2hGG-cq6iEq2ZZ5yKBG z8o`d$yQNo(8B?d3L6z>Y_rAv{;^1J$uv<-h z*&h_xDsYjxMc*uc4nfG9W(KWW-Mv{DLywh4x6i*sIR+Ca%_XdS{DV)W)E1HzCE4^+ zO6En5zjh6fu;6#e-qal)QvnU@U}(40epF-(KEc-KU3Seo8)ns(90+B~rw}UeqnM-= zHagHer;^e~1;Zj~gm%Tl4OSbDVEeN;R5{i;$i@qhPkEEwDjq@eHX@$}^z`Lvx;7Ty z*omsIu+HYC?b~nBP9!^eTCC!s>`I1|J!3TFLQi$L663q8ORxiq*M-Sw=JfNU7O7~a zLH`Mb&t3E_^jP%3wErY)pIbuhK>^qH=!^wMmH8mR_e}PSY4mmCjLnot%N=rwlhwkJ z#LH#(lwLXo!c)PSm|GcW+X>HwcN}EY?oAuALJ6Q_jhtM7`P2}4ct=NiO)$Q*8ZPGu zL%!K#G#|TzD7RWr)iz!wY|tGh{I-ygA+7~^dsO71cnueCWx247-pPAeHJkjhi=D$t zBssN8mI(>RezB{XmxG~9Iawz@S1vm3<-AA3%(F#S%O+;l`N!j=EcqX|aMY@N47sP| z*j^6*w%yCAY>o9U__&oc1O79FGjd$%U>q zY?+jFDf3IzkC|Cfb3?k5AKY6)^Osu@&kz1IWCf{gOPTREq@U=eYzzqyCegYH0)cAs zk5!4^UX$c(uBAH#glp7UkPK_;7F@HmNxEEm00*q{^Nh)80jwK7^WRO*5v z3sp6Vt|x)Sl=wqAzil2c#sb;i9rC_4vtDoQ_B4|*=4fvKG)-y&a&^3rccp&82#Gnl z>d4kEul=?plg{kKk-8%@5i1&NiQL~$f0ny@aRno{!Pb}2b~_ihuSjTm_myz*8th>j zQl+9*$XaH&iS1}^ZLYr>$`0>XJ;+#=OXIw$IJq9@;Gnt4U(*~*MF9&)t6Yqc$_UeZ zc6|23TW)&bfi^d&Qi8q;bBg^T(Vt)ba-V-u)P`&1e3?9$dFne1W0rA-b^)r6pDYM+ z6$LNHs;5L*zq?%*Fgd%rRy*x0p_TeNl6jrIxL);bhjlm)UfapUofK{MM=O8*YOknE z$qBh?b8sPrB?F(6b-DkWvXB4CoaX+c$o$rZmD~h1``@4+C$&J z$I1PlA>;q^+2D^IFg_f;l(N=xu<9M0%*QQeZBP_#a%}tyJcU!!IUr9q9!-j{Eqk=h zqNF$#Vn?9sQ-UZT;d-wc@0(PwfP7JJAexr0u{ed({mdy?)9D3-BVWfv4B_cG@a4a_?r{eNGc_(S5(92oO%u6%YxTa&L{8z@2744bOjYf&o_J1LLTqmk&eD#X&KIZ?6fC! z-$b$y?^qKQYUcFE)3MQ8$rD?+Ix_H81B6;0yegQlg|hCaCil0@V}JfULQ&;Gv}gg* zw-a4#zhtS#8!>#(-w7B{;;~x(02TV%Ng~tqH&1@7oozY?IWlGReXLj6GNKBtyF6p> z=8U>Z~_*VGWvLuRe^`H(qQEe;>r@zE5eD@4o*eKIR<#I1U@$WHK`ojq$yD zbc0qGxUd6%>T{=Y<#f*sVjhO;3fMe$qPC<(ka}r;D*JY z)%b&=Ai}m>>eNc_Sq}7bT}^Pk3o!p6PtBcz)_>~*vNo9_9Cl4>T{s#>`I+nRPG{g$ zO~zx29L{-%_pVvEjvoM>CF6wRjYxmAFMO?vojJKI$7F|4oRinJdlbu`&SBdm&eKUY zZO_NJUESfe84{JxJP#tXZ8vyAZ{g##kGx!H<+oX<;40SMw1J0So;UVNe1CH{`31&zQMo)b=jd(PkE0Y3&w#)aM!{y z{lOD_a1&&@diy!y1bdeF7ByFkr)o4FHigRaP#dTFt<(~FQ+h0Pr{-xc z=&j$R%O8s}h>6_LLoP}>pl?4a&J+SPNufd@`>0QautD~g^}E*$jt|Fsrm=8NHkeGW zTjUl$P{ei}IvbJ3cJZV4vAO}DOWc#%YcQ`~*0q|mEaGQNq_@ev*Q=cwo2(*pT6uqv zeEG3^Z?r3{P^lF!swVQlkWPAYvF>v|TP@TqI@st)Q3UjgV%}DZn}0>~Q4WfkWeeL9 z_U;LkS|YtCj5Ka{;uK-i7vW^SKWQmp`YRkLHy`&Rt0=gk;xNAO99p}&HB%(kwDR!c zuKn`0ykgscV=ccC4$oQFDl_%MM30~+a2q*r(am1!9ppsov(Xu?mz%!dR93K?5RU#2 zvh(;Y1PNryZ`%mF4fetXnKv=wAW=ExbK+eNT*wWt%V!n@yNnUYTkNGrvcDr!orGZ( zxP+7Q_gfhE0d%$BGn99}q&%?)XXE`ki=W8qo}k-eH74b(ZuTPe<>;zkImq5gdy_w> z#d)C_p6L~6tNQ8L+Tg6X%Ku6eo1uisp%dxN?8sw=1f&(UiRv%v*A5)n*FvD;7f);;%(Cwd;HourCw_o(G9 z=g+?ktG^pNz?(39Tu)i6_V-j0Uwyk6IkfXs*L|)#*VgX+!rbf0K)zk|4nKD|C=-iN zf3tiyjtE<7zT^-ZftkhTX4k+LVHQ7kNz?`GIYHmly?&Mn+uApIF>}wEq(L#Pib?c9 zT60d+nP)ruE_A1tA!hMg5X*aL}xmz z+dgna+)x@Q*Nr-l@16%KjA_jN~nu|Vw1$8{a;9WY;*rB?~c<=Yf Date: Thu, 17 Aug 2023 20:10:13 +0530 Subject: [PATCH 31/40] feat: enable cluster dashboard for FSx (#2303) Fixes #2299 --- grafana/dashboards/cmode/aggregate.json | 4 ++-- grafana/dashboards/cmode/cdot.json | 8 ++++---- grafana/dashboards/cmode/cluster.json | 11 ++++++----- grafana/dashboards/cmode/disk.json | 4 ++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/grafana/dashboards/cmode/aggregate.json b/grafana/dashboards/cmode/aggregate.json index 8b37232b8..205142dba 100644 --- a/grafana/dashboards/cmode/aggregate.json +++ b/grafana/dashboards/cmode/aggregate.json @@ -4890,7 +4890,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(node_labels{system_type!=\"7mode\"}, datacenter)", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\"}, datacenter)", "description": null, "error": null, "hide": 0, @@ -4900,7 +4900,7 @@ "name": "Datacenter", "options": [], "query": { - "query": "label_values(node_labels{system_type!=\"7mode\"}, datacenter)", + "query": "label_values(cluster_new_status{system_type!=\"7mode\"}, datacenter)", "refId": "StandardVariableQuery" }, "refresh": 2, diff --git a/grafana/dashboards/cmode/cdot.json b/grafana/dashboards/cmode/cdot.json index 1b2f54e59..0277b30c1 100644 --- a/grafana/dashboards/cmode/cdot.json +++ b/grafana/dashboards/cmode/cdot.json @@ -1922,7 +1922,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(node_labels{system_type!=\"7mode\"},datacenter)", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\"},datacenter)", "description": null, "error": null, "hide": 0, @@ -1932,7 +1932,7 @@ "name": "Datacenter", "options": [], "query": { - "query": "label_values(node_labels{system_type!=\"7mode\"},datacenter)", + "query": "label_values(cluster_new_status{system_type!=\"7mode\"},datacenter)", "refId": "Prometheus-Datacenter-Variable-Query" }, "refresh": 2, @@ -1948,7 +1948,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(node_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", "description": null, "error": null, "hide": 0, @@ -1958,7 +1958,7 @@ "name": "Cluster", "options": [], "query": { - "query": "label_values(node_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", + "query": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", "refId": "StandardVariableQuery" }, "refresh": 2, diff --git a/grafana/dashboards/cmode/cluster.json b/grafana/dashboards/cmode/cluster.json index 0b98961c1..8ad681865 100644 --- a/grafana/dashboards/cmode/cluster.json +++ b/grafana/dashboards/cmode/cluster.json @@ -3977,7 +3977,8 @@ "tags": [ "harvest", "ontap", - "cdot" + "cdot", + "fsx" ], "templating": { "list": [ @@ -4005,7 +4006,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(node_labels{system_type!=\"7mode\"},datacenter)", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\"},datacenter)", "description": null, "error": null, "hide": 0, @@ -4015,7 +4016,7 @@ "name": "Datacenter", "options": [], "query": { - "query": "label_values(node_labels{system_type!=\"7mode\"},datacenter)", + "query": "label_values(cluster_new_status{system_type!=\"7mode\"},datacenter)", "refId": "Prometheus-Datacenter-Variable-Query" }, "refresh": 2, @@ -4031,7 +4032,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(node_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", "description": null, "error": null, "hide": 0, @@ -4041,7 +4042,7 @@ "name": "Cluster", "options": [], "query": { - "query": "label_values(node_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", + "query": "label_values(cluster_new_status{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", "refId": "StandardVariableQuery" }, "refresh": 2, diff --git a/grafana/dashboards/cmode/disk.json b/grafana/dashboards/cmode/disk.json index e2815844d..8cf4541fd 100644 --- a/grafana/dashboards/cmode/disk.json +++ b/grafana/dashboards/cmode/disk.json @@ -2470,7 +2470,7 @@ "allValue": null, "current": {}, "datasource": "${DS_PROMETHEUS}", - "definition": "label_values(node_labels{system_type!=\"7mode\"},datacenter)", + "definition": "label_values(cluster_new_status{system_type!=\"7mode\"},datacenter)", "description": null, "error": null, "hide": 0, @@ -2480,7 +2480,7 @@ "name": "Datacenter", "options": [], "query": { - "query": "label_values(node_labels{system_type!=\"7mode\"},datacenter)", + "query": "label_values(cluster_new_status{system_type!=\"7mode\"},datacenter)", "refId": "Prometheus-Datacenter-Variable-Query" }, "refresh": 2, From d13c51570e5b4ec3212639d66eb802c0a46a65de Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Fri, 18 Aug 2023 06:40:06 -0400 Subject: [PATCH 32/40] refactor: normalize schedules to minutes (#2305) * refactor: normalize schedules to minutes * refactor: normalize schedules to minutes --- .../testdata/conf/restperf/9.12.1/workload_volume.yaml | 6 +++--- conf/rest/9.12.0/security_certificate.yaml | 2 +- conf/restperf/9.12.0/workload.yaml | 6 +++--- conf/restperf/9.12.0/workload_detail.yaml | 6 +++--- conf/restperf/9.12.0/workload_detail_volume.yaml | 6 +++--- conf/restperf/9.12.0/workload_volume.yaml | 6 +++--- conf/zapi/cdot/9.8.0/security.yaml | 2 +- conf/zapi/cdot/9.8.0/security_certificate.yaml | 2 +- conf/zapi/default.yaml | 2 +- conf/zapiperf/cdot/9.8.0/workload.yaml | 6 +++--- conf/zapiperf/cdot/9.8.0/workload_detail.yaml | 6 +++--- conf/zapiperf/cdot/9.8.0/workload_detail_volume.yaml | 6 +++--- conf/zapiperf/cdot/9.8.0/workload_volume.yaml | 6 +++--- conf/zapiperf/default.yaml | 6 +++--- pkg/conf/conf/zapiperf/limited1.yaml | 6 +++--- pkg/conf/testdata/conf/zapiperf/limited1.yaml | 6 +++--- 16 files changed, 40 insertions(+), 40 deletions(-) diff --git a/cmd/collectors/restperf/testdata/conf/restperf/9.12.1/workload_volume.yaml b/cmd/collectors/restperf/testdata/conf/restperf/9.12.1/workload_volume.yaml index e06f54afa..594736479 100644 --- a/cmd/collectors/restperf/testdata/conf/restperf/9.12.1/workload_volume.yaml +++ b/cmd/collectors/restperf/testdata/conf/restperf/9.12.1/workload_volume.yaml @@ -9,9 +9,9 @@ object: qos_volume # recommended to use large interval, since workload objects are expensive client_timeout: 1m30s schedule: - - counter: 1200s - - instance: 600s - - data: 180s + - counter: 20m + - instance: 10m + - data: 3m counters: - ^^uuid diff --git a/conf/rest/9.12.0/security_certificate.yaml b/conf/rest/9.12.0/security_certificate.yaml index 476db647b..0c7d3a79e 100644 --- a/conf/rest/9.12.0/security_certificate.yaml +++ b/conf/rest/9.12.0/security_certificate.yaml @@ -19,7 +19,7 @@ counters: plugins: - Certificate: schedule: - - data: 180s # should be multiple of data poll duration + - data: 3m # should be multiple of data poll duration export_options: instance_keys: diff --git a/conf/restperf/9.12.0/workload.yaml b/conf/restperf/9.12.0/workload.yaml index 92587264a..98a0e763e 100644 --- a/conf/restperf/9.12.0/workload.yaml +++ b/conf/restperf/9.12.0/workload.yaml @@ -8,9 +8,9 @@ object: qos client_timeout: 1m30s schedule: - - counter: 1200s - - instance: 600s - - data: 180s + - counter: 20m + - instance: 10m + - data: 3m counters: - ^^uuid diff --git a/conf/restperf/9.12.0/workload_detail.yaml b/conf/restperf/9.12.0/workload_detail.yaml index bc1faeade..823e2a579 100644 --- a/conf/restperf/9.12.0/workload_detail.yaml +++ b/conf/restperf/9.12.0/workload_detail.yaml @@ -9,9 +9,9 @@ object: qos_detail client_timeout: 1m30s schedule: - - counter: 1200s - - instance: 600s - - data: 180s + - counter: 20m + - instance: 10m + - data: 3m counters: - ^^id diff --git a/conf/restperf/9.12.0/workload_detail_volume.yaml b/conf/restperf/9.12.0/workload_detail_volume.yaml index a5c57ce1f..0f542aada 100644 --- a/conf/restperf/9.12.0/workload_detail_volume.yaml +++ b/conf/restperf/9.12.0/workload_detail_volume.yaml @@ -8,9 +8,9 @@ object: qos_detail client_timeout: 1m30s schedule: - - counter: 1200s - - instance: 600s - - data: 180s + - counter: 20m + - instance: 10m + - data: 3m counters: - ^^id diff --git a/conf/restperf/9.12.0/workload_volume.yaml b/conf/restperf/9.12.0/workload_volume.yaml index f628bb359..ae9ed17a6 100644 --- a/conf/restperf/9.12.0/workload_volume.yaml +++ b/conf/restperf/9.12.0/workload_volume.yaml @@ -10,9 +10,9 @@ object: qos client_timeout: 1m30s schedule: - - counter: 1200s - - instance: 600s - - data: 180s + - counter: 20m + - instance: 10m + - data: 3m counters: - ^^uuid diff --git a/conf/zapi/cdot/9.8.0/security.yaml b/conf/zapi/cdot/9.8.0/security.yaml index a7acaf7a3..257f18020 100644 --- a/conf/zapi/cdot/9.8.0/security.yaml +++ b/conf/zapi/cdot/9.8.0/security.yaml @@ -12,7 +12,7 @@ no_max_records: true plugins: - Security: schedule: - - data: 180s # should be multiple of data poll duration + - data: 3m # should be multiple of data poll duration export_options: instance_keys: diff --git a/conf/zapi/cdot/9.8.0/security_certificate.yaml b/conf/zapi/cdot/9.8.0/security_certificate.yaml index 67108e507..ad22c3437 100644 --- a/conf/zapi/cdot/9.8.0/security_certificate.yaml +++ b/conf/zapi/cdot/9.8.0/security_certificate.yaml @@ -18,7 +18,7 @@ plugins: - type `server` - Certificate: schedule: - - data: 180s # should be multiple of data poll duration + - data: 3m # should be multiple of data poll duration export_options: instance_keys: diff --git a/conf/zapi/default.yaml b/conf/zapi/default.yaml index 0477410f1..abc8c29e7 100644 --- a/conf/zapi/default.yaml +++ b/conf/zapi/default.yaml @@ -3,7 +3,7 @@ collector: Zapi # Order here matters! schedule: - - data: 180s + - data: 3m objects: Aggregate: aggr.yaml diff --git a/conf/zapiperf/cdot/9.8.0/workload.yaml b/conf/zapiperf/cdot/9.8.0/workload.yaml index d561a725a..e5c969bab 100644 --- a/conf/zapiperf/cdot/9.8.0/workload.yaml +++ b/conf/zapiperf/cdot/9.8.0/workload.yaml @@ -10,9 +10,9 @@ instance_key: uuid # recommended to use large interval, since workload objects are expensive client_timeout: 1m30s schedule: - - counter: 1200s - - instance: 600s - - data: 180s + - counter: 20m + - instance: 10m + - data: 3m counters: - concurrency diff --git a/conf/zapiperf/cdot/9.8.0/workload_detail.yaml b/conf/zapiperf/cdot/9.8.0/workload_detail.yaml index f79d4188b..59cf7ccab 100644 --- a/conf/zapiperf/cdot/9.8.0/workload_detail.yaml +++ b/conf/zapiperf/cdot/9.8.0/workload_detail.yaml @@ -11,9 +11,9 @@ instance_key: name client_timeout: 1m30s schedule: - - counter: 1200s - - instance: 600s - - data: 180s + - counter: 20m + - instance: 10m + - data: 3m counters: - instance_name diff --git a/conf/zapiperf/cdot/9.8.0/workload_detail_volume.yaml b/conf/zapiperf/cdot/9.8.0/workload_detail_volume.yaml index 8482b3c27..f75c6d225 100644 --- a/conf/zapiperf/cdot/9.8.0/workload_detail_volume.yaml +++ b/conf/zapiperf/cdot/9.8.0/workload_detail_volume.yaml @@ -10,9 +10,9 @@ instance_key: name client_timeout: 1m30s schedule: - - counter: 1200s - - instance: 600s - - data: 180s + - counter: 20m + - instance: 10m + - data: 3m counters: - instance_name diff --git a/conf/zapiperf/cdot/9.8.0/workload_volume.yaml b/conf/zapiperf/cdot/9.8.0/workload_volume.yaml index e26f938d3..60d34b340 100644 --- a/conf/zapiperf/cdot/9.8.0/workload_volume.yaml +++ b/conf/zapiperf/cdot/9.8.0/workload_volume.yaml @@ -9,9 +9,9 @@ object: qos # recommended to use large interval, since workload objects are expensive client_timeout: 1m30s schedule: - - counter: 1200s - - instance: 600s - - data: 180s + - counter: 20m + - instance: 10m + - data: 3m instance_key: name diff --git a/conf/zapiperf/default.yaml b/conf/zapiperf/default.yaml index 8eaad696b..54e6835ef 100644 --- a/conf/zapiperf/default.yaml +++ b/conf/zapiperf/default.yaml @@ -3,9 +3,9 @@ collector: ZapiPerf # Order here matters! schedule: - - counter: 1200s - - instance: 600s - - data: 60s + - counter: 20m + - instance: 10m + - data: 1m objects: # Node-level metrics diff --git a/pkg/conf/conf/zapiperf/limited1.yaml b/pkg/conf/conf/zapiperf/limited1.yaml index 4d7b89a67..23e7e5731 100644 --- a/pkg/conf/conf/zapiperf/limited1.yaml +++ b/pkg/conf/conf/zapiperf/limited1.yaml @@ -3,9 +3,9 @@ collector: ZapiPerf # Order here matters! schedule: -- counter: 1200s -- instance: 600s -- data: 60s +- counter: 20m +- instance: 10m +- data: 1m objects: SystemNode: system_node.yaml diff --git a/pkg/conf/testdata/conf/zapiperf/limited1.yaml b/pkg/conf/testdata/conf/zapiperf/limited1.yaml index 4d7b89a67..23e7e5731 100644 --- a/pkg/conf/testdata/conf/zapiperf/limited1.yaml +++ b/pkg/conf/testdata/conf/zapiperf/limited1.yaml @@ -3,9 +3,9 @@ collector: ZapiPerf # Order here matters! schedule: -- counter: 1200s -- instance: 600s -- data: 60s +- counter: 20m +- instance: 10m +- data: 1m objects: SystemNode: system_node.yaml From 5d8fa410393ad4670429e273770e10a57a199913 Mon Sep 17 00:00:00 2001 From: Hardikl <83282894+Hardikl@users.noreply.github.com> Date: Fri, 18 Aug 2023 19:33:54 +0530 Subject: [PATCH 33/40] fix: handling UX feedback in security dashboard (#2304) * fix: handling UX feedback in security dashboard * fix: extra panel removal in security dashboard --- grafana/dashboards/cmode/security.json | 517 +++++++------------------ 1 file changed, 132 insertions(+), 385 deletions(-) diff --git a/grafana/dashboards/cmode/security.json b/grafana/dashboards/cmode/security.json index 4fed00338..b804755e3 100644 --- a/grafana/dashboards/cmode/security.json +++ b/grafana/dashboards/cmode/security.json @@ -206,7 +206,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "count(group by (datacenter, cluster)(support_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", asup_enabled=\"true\", asup_https_configured!=\"https\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", telnet_enabled=\"true\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", rsh_enabled=\"true\"} or group by (datacenter, cluster)(security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", hash_algorithm=\"md5\"}) > 0 or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"\"} or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"none\"} or group by (datacenter, cluster) (security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", locked=\"false\"}) > 0 or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"cluster\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$Cluster\"} or count by (datacenter, cluster) (ntpserver_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}) < 1 or security_audit_destination_status{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", protocol!=\"tcp_encrypted\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", fips_enabled=\"false\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", type=\"admin\", ciphers=~\".*_cbc.*\"} or group by (datacenter, cluster) ((svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", cifs_protocol_enabled=\"true\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", nfs_protocol_enabled=\"true\"} and svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", audit_protocol_enabled=\"false\"}) or security_ssh_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", ciphers=~\".*_cbc.*\"} or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"svm\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"}))) or vector(0)", + "expr": "count(group by (datacenter, cluster)(support_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", asup_enabled=\"true\", asup_https_configured!=\"https\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", telnet_enabled=\"true\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", rsh_enabled=\"true\"} or group by (datacenter, cluster)(security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", hash_algorithm=\"md5\"}) > 0 or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"\"} or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"none\"} or group by (datacenter, cluster) (security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", locked=\"false\"}) > 0 or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"cluster\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$Cluster\"} or count by (datacenter, cluster) (ntpserver_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}) < 1 or security_audit_destination_status{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", protocol!=\"tcp_encrypted\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", fips_enabled=\"false\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", type=\"admin\", ciphers=~\".*_cbc.*\",svm=~\"$SVM\"} or group by (datacenter, cluster) ((svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", cifs_protocol_enabled=\"true\",svm=~\"$SVM\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", nfs_protocol_enabled=\"true\",svm=~\"$SVM\"} and svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", audit_protocol_enabled=\"false\",svm=~\"$SVM\"}) or security_ssh_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", ciphers=~\".*_cbc.*\"} or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"svm\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"}))) or vector(0)", "instant": true, "interval": "", "legendFormat": "NonCompliantCluster", @@ -489,7 +489,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "(count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", isEncrypted=\"true\"}) or vector (0))\n+\n(count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", isHardwareEncrypted=\"true\"}) or vector (0) )", + "expr": "(count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", isEncrypted=\"true\",root_volume=\"No\"}) or vector (0))\n+\n(count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", isHardwareEncrypted=\"true\",root_volume=\"No\"}) or vector (0) )", "instant": true, "interval": "", "legendFormat": "encrypted", @@ -498,7 +498,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "(count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\"}) or vector (1) )", + "expr": "(count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\",root_volume=\"No\"}) or vector (1) )", "hide": false, "instant": true, "interval": "", @@ -592,7 +592,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "((count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", antiRansomwareState !=\"\" , antiRansomwareState=~\"enabled|dry_run\"}) or vector (0)) / (count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", antiRansomwareState !=\"\"}) or vector (1))) * 100", + "expr": "((count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", antiRansomwareState !=\"\" , antiRansomwareState=~\"enabled|dry_run\",root_volume=\"No\"}) or vector (0)) / (count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", antiRansomwareState !=\"\",root_volume=\"No\"}) or vector (1))) * 100", "instant": true, "interval": "", "legendFormat": "", @@ -685,7 +685,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "((count(svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", anti_ransomware_state!=\"\", anti_ransomware_state=~\"enabled|dry_run\"}) or vector (0))\n/\n(count(svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", anti_ransomware_state!=\"\"}) or vector (1))) * 100", + "expr": "((count(svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", anti_ransomware_state!=\"\", anti_ransomware_state=~\"enabled|dry_run\",svm=~\"$SVM\"}) or vector (0))\n/\n(count(svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", anti_ransomware_state!=\"\",svm=~\"$SVM\"}) or vector (1))) * 100", "instant": true, "interval": "", "legendFormat": "", @@ -872,7 +872,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "count(group by (datacenter, cluster)(support_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", asup_enabled=\"true\", asup_https_configured!=\"https\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", telnet_enabled=\"true\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", rsh_enabled=\"true\"} or group by (datacenter, cluster)(security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", hash_algorithm=\"md5\"}) > 0 or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"\"} or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"none\"} or group by (datacenter, cluster) (security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", locked=\"false\"}) > 0 or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"cluster\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$Cluster\"} or count by (datacenter, cluster) (ntpserver_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}) < 1 or security_audit_destination_status{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", protocol!=\"tcp_encrypted\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", fips_enabled=\"false\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", type=\"admin\", ciphers=~\".*_cbc.*\"} or group by (datacenter, cluster) ((svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", cifs_protocol_enabled=\"true\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", nfs_protocol_enabled=\"true\"} and svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", audit_protocol_enabled=\"false\"}) or security_ssh_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", ciphers=~\".*_cbc.*\"} or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"svm\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"}))) or vector(0)", + "expr": "count(group by (datacenter, cluster)(support_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", asup_enabled=\"true\", asup_https_configured!=\"https\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", telnet_enabled=\"true\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", rsh_enabled=\"true\"} or group by (datacenter, cluster)(security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", hash_algorithm=\"md5\"}) > 0 or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"\"} or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"none\"} or group by (datacenter, cluster) (security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", locked=\"false\"}) > 0 or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"cluster\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$Cluster\"} or count by (datacenter, cluster) (ntpserver_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}) < 1 or security_audit_destination_status{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", protocol!=\"tcp_encrypted\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", fips_enabled=\"false\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", type=\"admin\", ciphers=~\".*_cbc.*\",svm=~\"$SVM\"} or group by (datacenter, cluster) ((svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", cifs_protocol_enabled=\"true\",svm=~\"$SVM\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", nfs_protocol_enabled=\"true\",svm=~\"$SVM\"} and svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", audit_protocol_enabled=\"false\",svm=~\"$SVM\"}) or security_ssh_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", ciphers=~\".*_cbc.*\"} or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"svm\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"}))) or vector(0)", "hide": false, "instant": true, "interval": "", @@ -1138,7 +1138,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", isEncrypted=\"true\"}) or vector (0)", + "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", isEncrypted=\"true\",root_volume=\"No\"}) or vector (0)", "instant": true, "interval": "", "legendFormat": "Software", @@ -1147,7 +1147,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", isHardwareEncrypted=\"true\"}) or vector (0)", + "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", isHardwareEncrypted=\"true\",root_volume=\"No\"}) or vector (0)", "hide": false, "instant": true, "interval": "", @@ -1157,7 +1157,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", isEncrypted=\"true\", isHardwareEncrypted=\"true\"}) or vector (0)", + "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", isEncrypted=\"true\", isHardwareEncrypted=\"true\",root_volume=\"No\"}) or vector (0)", "hide": false, "instant": true, "interval": "", @@ -1167,7 +1167,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", isEncrypted=\"false\", isHardwareEncrypted=\"false\"}) or vector (0)", + "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", isEncrypted=\"false\", isHardwareEncrypted=\"false\",root_volume=\"No\"}) or vector (0)", "hide": false, "instant": true, "interval": "", @@ -1236,7 +1236,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", antiRansomwareState !=\"\" , antiRansomwareState=\"enabled\"}) or vector (0)", + "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", antiRansomwareState !=\"\" , antiRansomwareState=\"enabled\",root_volume=\"No\"}) or vector (0)", "instant": true, "interval": "", "legendFormat": "Enabled in active mode", @@ -1245,7 +1245,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", antiRansomwareState !=\"\" , antiRansomwareState=\"dry_run\"}) or vector (0)", + "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", antiRansomwareState !=\"\" , antiRansomwareState=\"dry_run\",root_volume=\"No\"}) or vector (0)", "hide": false, "instant": true, "interval": "", @@ -1255,7 +1255,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", antiRansomwareState !=\"\" , antiRansomwareState=~\".*paused.*\"}) or vector (0)", + "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", antiRansomwareState !=\"\" , antiRansomwareState=~\".*paused.*\",root_volume=\"No\"}) or vector (0)", "hide": false, "instant": true, "interval": "", @@ -1265,7 +1265,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", antiRansomwareState !=\"\" , antiRansomwareState=~\"disabled\"}) or vector (0)", + "expr": "count(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", svm=~\"$SVM\", antiRansomwareState !=\"\" , antiRansomwareState=~\"disabled\",root_volume=\"No\"}) or vector (0)", "hide": false, "instant": true, "interval": "", @@ -1334,7 +1334,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "count(svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", anti_ransomware_state!=\"\", anti_ransomware_state=~\"enabled|dry_run\"}) or vector (0)", + "expr": "count(svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", anti_ransomware_state!=\"\", anti_ransomware_state=~\"enabled|dry_run\",svm=~\"$SVM\"}) or vector (0)", "instant": true, "interval": "", "legendFormat": "Enabled", @@ -1343,7 +1343,7 @@ { "datasource": "${DS_PROMETHEUS}", "exemplar": false, - "expr": "count(svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", anti_ransomware_state!=\"\", anti_ransomware_state=~\"disabled\"}) or vector (0)", + "expr": "count(svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", anti_ransomware_state!=\"\", anti_ransomware_state=~\"disabled\",svm=~\"$SVM\"}) or vector (0)", "hide": false, "interval": "", "legendFormat": "Disabled", @@ -1983,339 +1983,7 @@ ] } ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 155, - "options": { - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "8.1.8", - "targets": [ - { - "exemplar": false, - "expr": "volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"}", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Volume Encryption", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "aggr", - "isEncrypted", - "isHardwareEncrypted", - "node", - "state", - "svm", - "volume" - ] - } - } - }, - { - "id": "calculateField", - "options": { - "alias": "Encryption Type", - "binary": { - "left": "isHardwareEncrypted", - "operator": "+", - "reducer": "sum", - "right": "isEncrypted" - }, - "mode": "binary", - "reduce": { - "include": [ - "isEncrypted", - "isHardwareEncrypted" - ], - "reducer": "sum" - }, - "replaceFields": false - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "isEncrypted": true, - "isHardwareEncrypted": true - }, - "indexByName": { - "Encryption Type": 4, - "aggr": 2, - "isEncrypted": 6, - "isHardwareEncrypted": 7, - "node": 3, - "state": 5, - "svm": 1, - "volume": 0 - }, - "renameByName": { - "aggr": "Aggregate", - "cluster": "Cluster", - "isEncrypted": "", - "isHardwareEncrypted": "", - "node": "Node", - "state": "State", - "svm": "SVM", - "volume": "Volume" - } - } - } - ], - "type": "table" - } - ], - "title": "Volume Encryption", - "type": "row" - }, - { - "collapsed": true, - "datasource": "${DS_PROMETHEUS}", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 16, - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "transparent", - "mode": "fixed" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "filterable": true }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Anti-ransomware Status" - }, - "properties": [ - { - "id": "custom.filterable", - "value": true - }, - { - "id": "mappings", - "value": [ - { - "options": { - "disabled": { - "index": 2, - "text": "Disabled" - }, - "dry_run": { - "index": 1, - "text": "Enabled (Learning mode)" - }, - "enabled": { - "index": 0, - "text": "Enabled (Active mode)" - } - }, - "type": "value" - }, - { - "options": { - "match": "empty", - "result": { - "index": 3, - "text": "Disabled" - } - }, - "type": "special" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Volume Count" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "match": "null", - "result": { - "index": 0, - "text": "0" - } - }, - "type": "special" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 160, - "options": { - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "8.1.8", - "targets": [ - { - "exemplar": false, - "expr": "svm_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"}", - "format": "table", - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "A" - }, - { - "exemplar": false, - "expr": "count by (svm)(volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "refId": "B" - } - ], - "title": "Storage VMs Anti-ransomware Status", - "transformations": [ - { - "id": "filterFieldsByName", - "options": { - "include": { - "names": [ - "anti_ransomware_state", - "cluster", - "state", - "svm", - "Value #B", - "type" - ] - } - } - }, - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "isEncrypted": false, - "isHardwareEncrypted": false - }, - "indexByName": { - "Value #B": 3, - "anti_ransomware_state": 2, - "cluster": 1, - "state": 5, - "svm": 0, - "type": 4 - }, - "renameByName": { - "Value #B": "Volume Count", - "antiRansomwareState": "Anti-ransomware Status", - "anti_ransomware_state": "Anti-ransomware Status", - "cluster": "Cluster", - "isEncrypted": "", - "isHardwareEncrypted": "", - "state": "State", - "svm": "SVM", - "type": "Type", - "volume": "Volume" - } - } - } - ], - "type": "table" - } - ], - "title": "Storage VM Anti-ransomware Status", - "type": "row" - }, - { - "collapsed": true, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 196, - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "description": "This panel requires a cluster with ONTAP 9.10+ and the Harvest REST collector", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "transparent", - "mode": "fixed" - }, - "custom": { - "align": "left", - "displayMode": "auto", - "filterable": true - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ { "matcher": { "id": "byName", @@ -2378,9 +2046,9 @@ "h": 7, "w": 24, "x": 0, - "y": 6 + "y": 10 }, - "id": 183, + "id": 155, "options": { "showHeader": true, "sortBy": [] @@ -2389,7 +2057,7 @@ "targets": [ { "exemplar": false, - "expr": "volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"}", + "expr": "volume_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\",root_volume=\"No\"}", "format": "table", "instant": true, "interval": "", @@ -2397,7 +2065,7 @@ "refId": "A" } ], - "title": "Volume Anti-ransomware Status", + "title": "Volume Encryption & Anti-ransomware Status", "transformations": [ { "id": "filterFieldsByName", @@ -2405,27 +2073,50 @@ "include": { "names": [ "aggr", - "antiRansomwareState", + "isEncrypted", + "isHardwareEncrypted", "node", "state", "svm", - "volume" + "volume", + "antiRansomwareState" ] } } }, + { + "id": "calculateField", + "options": { + "alias": "Encryption Type", + "binary": { + "left": "isHardwareEncrypted", + "operator": "+", + "reducer": "sum", + "right": "isEncrypted" + }, + "mode": "binary", + "reduce": { + "include": [ + "isEncrypted", + "isHardwareEncrypted" + ], + "reducer": "sum" + }, + "replaceFields": false + } + }, { "id": "organize", "options": { "excludeByName": { - "isEncrypted": false, - "isHardwareEncrypted": false + "isEncrypted": true, + "isHardwareEncrypted": true }, "indexByName": { + "Encryption Type": 4, "aggr": 2, - "antiRansomwareState": 4, + "antiRansomwareState": 5, "node": 3, - "state": 5, "svm": 1, "volume": 0 }, @@ -2446,7 +2137,7 @@ "type": "table" } ], - "title": "Volume Anti-ransomware Status", + "title": "Volume Encryption & Anti-ransomware Status", "type": "row" }, { @@ -2778,6 +2469,10 @@ "value": [ { "options": { + "false": { + "index": 1, + "text": "No" + }, "true": { "index": 0, "text": "❌ Yes" @@ -3634,7 +3329,7 @@ }, { "exemplar": false, - "expr": "count by (datacenter, cluster, insecured)(svm_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", insecured=\"true\"})", + "expr": "count by (datacenter, cluster, insecured)(svm_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", insecured=\"true\",svm=~\"$SVM\"})", "format": "table", "hide": false, "instant": true, @@ -3734,7 +3429,7 @@ }, { "exemplar": false, - "expr": "group by (datacenter, cluster)(support_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", asup_enabled=\"true\", asup_https_configured!=\"https\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", telnet_enabled=\"true\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", rsh_enabled=\"true\"} or group by (datacenter, cluster)(security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", hash_algorithm=\"md5\"}) > 0 or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"\"} or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"none\"} or group by (datacenter, cluster) (security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", locked=\"false\"}) > 0 or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"cluster\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$Cluster\"} or count by (datacenter, cluster) (ntpserver_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}) < 1 or security_audit_destination_status{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", protocol!=\"tcp_encrypted\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", fips_enabled=\"false\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", type=\"admin\", ciphers=~\".*_cbc.*\"} or group by (datacenter, cluster) ((svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", cifs_protocol_enabled=\"true\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", nfs_protocol_enabled=\"true\"} and svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", audit_protocol_enabled=\"false\"}) or security_ssh_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", ciphers=~\".*_cbc.*\"} or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"svm\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"} or sum by (datacenter, cluster)(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy=\"none\"}) > 0 or sum by (datacenter, cluster)(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_autodelete=\"true\"}) > 0 or sum by (datacenter, cluster)(svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",anti_ransomware_state=~\"|.*disabled\",type=\"data\"}) > 0 or volume_arw_status{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",ArwStatus!=\"Active Mode\"} * on (instance) group_left() metadata_collector_instances{datacenter=~\"$Datacenter\",collector=\"Rest\", object=\"Volume\"} or count by (datacenter, cluster)(ems_destination_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}) > 0 or support_auto_update_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",auto_update_enabled=\"false\"}))", + "expr": "group by (datacenter, cluster)(support_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", asup_enabled=\"true\", asup_https_configured!=\"https\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", telnet_enabled=\"true\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", rsh_enabled=\"true\"} or group by (datacenter, cluster)(security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", hash_algorithm=\"md5\"}) > 0 or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"\"} or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"none\"} or group by (datacenter, cluster) (security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", locked=\"false\"}) > 0 or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"cluster\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$Cluster\"} or count by (datacenter, cluster) (ntpserver_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}) < 1 or security_audit_destination_status{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", protocol!=\"tcp_encrypted\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", fips_enabled=\"false\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", type=\"admin\", ciphers=~\".*_cbc.*\",svm=~\"$SVM\"} or group by (datacenter, cluster) ((svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", cifs_protocol_enabled=\"true\",svm=~\"$SVM\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", nfs_protocol_enabled=\"true\",svm=~\"$SVM\"} and svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", audit_protocol_enabled=\"false\",svm=~\"$SVM\"}) or security_ssh_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", ciphers=~\".*_cbc.*\"} or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"svm\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"} or sum by (datacenter, cluster)(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy=\"none\",root_volume=\"No\"}) > 0 or sum by (datacenter, cluster)(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_autodelete=\"true\",root_volume=\"No\"}) > 0 or sum by (datacenter, cluster)(svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",anti_ransomware_state=~\"|.*disabled\",svm=~\"$SVM\"}) > 0 or volume_arw_status{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",ArwStatus!=\"Active Mode\"} * on (instance) group_left() metadata_collector_instances{datacenter=~\"$Datacenter\",collector=\"Rest\", object=\"Volume\"} or count by (datacenter, cluster)(ems_destination_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}) > 0 or support_auto_update_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",auto_update_enabled=\"false\"}))", "format": "table", "hide": false, "instant": true, @@ -3744,7 +3439,7 @@ }, { "exemplar": false, - "expr": "sum by (datacenter, cluster)(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy=\"none\"})", + "expr": "sum by (datacenter, cluster)(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy=\"none\",root_volume=\"No\"})", "format": "table", "hide": false, "instant": true, @@ -3754,7 +3449,7 @@ }, { "exemplar": false, - "expr": "sum by (datacenter, cluster)(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_autodelete=\"true\"})", + "expr": "sum by (datacenter, cluster)(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_autodelete=\"true\",root_volume=\"No\"})", "format": "table", "hide": false, "instant": true, @@ -3764,7 +3459,7 @@ }, { "exemplar": false, - "expr": "sum by (datacenter, cluster)(svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",anti_ransomware_state=~\"|.*disabled\",type=\"data\"})", + "expr": "sum by (datacenter, cluster)(svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",anti_ransomware_state=~\"|.*disabled\",svm=~\"$SVM\"})", "format": "table", "hide": false, "instant": true, @@ -3875,27 +3570,29 @@ }, "indexByName": { "ArwStatus": 5, - "Value #G": 13, + "Value #D": 10, + "Value #G": 15, "Value #H": 2, - "Value #I": 17, - "Value #J": 14, - "Value #K": 21, - "Value #L": 18, - "Value #M": 19, - "Value #N": 20, + "Value #I": 19, + "Value #J": 18, + "Value #K": 23, + "Value #L": 20, + "Value #M": 21, + "Value #N": 22, "Value #P": 0, "Value #Q": 3, "Value #R": 4, - "Value #T": 15, - "asup_enabled": 9, + "Value #T": 17, + "asup_enabled": 11, "auto_update_enabled": 16, - "banner": 12, + "banner": 14, "certificateExpiryStatus": 6, - "certificateIssuerType": 22, + "certificateIssuerType": 24, "cluster": 1, "fips_enabled": 7, - "locked": 10, - "rsh_enabled": 11, + "insecured": 9, + "locked": 12, + "rsh_enabled": 13, "telnet_enabled": 8 }, "renameByName": { @@ -4131,6 +3828,10 @@ "value": [ { "options": { + "false": { + "index": 1, + "text": "No" + }, "true": { "index": 0, "text": "❌ Yes" @@ -4528,6 +4229,46 @@ ] } ] + }, + { + "matcher": { + "id": "byName", + "options": "Anti-ransomware Status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "disabled": { + "index": 2, + "text": "Disabled" + }, + "dry_run": { + "index": 1, + "text": "Enabled (Learning Mode)" + }, + "enabled": { + "index": 0, + "text": "Enabled (Active Mode)" + } + }, + "type": "value" + }, + { + "options": { + "match": "empty", + "result": { + "index": 3, + "text": "Disabled" + } + }, + "type": "special" + } + ] + } + ] } ] }, @@ -4577,7 +4318,7 @@ }, { "exemplar": false, - "expr": "count by (datacenter, cluster, insecured)(svm_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", insecured=\"true\"})", + "expr": "count by (datacenter, cluster, insecured)(svm_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", insecured=\"true\",svm=~\"$SVM\"})", "format": "table", "hide": false, "instant": true, @@ -4623,6 +4364,7 @@ "options": { "include": { "names": [ + "anti_ransomware_state", "audit_protocol_enabled", "cifs_ntlm_enabled", "cluster", @@ -4636,7 +4378,10 @@ "insecured", "Value #G", "Value #I", - "Value #B" + "Value #B", + "fpolicy_enabled", + "smb_signing_required", + "smb_encryption_required" ] } } @@ -4685,16 +4430,17 @@ "Value #B": 0, "Value #G": 9, "Value #I": 8, - "audit_protocol_enabled": 4, + "anti_ransomware_state": 7, "banner": 3, - "cifs_ntlm_enabled": 7, + "cifs_ntlm_enabled": 6, "cluster": 1, - "insecured": 5, + "fpolicy_enabled": 12, + "insecured": 4, "iscsi_authentication_type": 10, "nfs_kerberos_protocol_enabled": 11, - "nis_authentication_enabled": 6, - "smb_encryption_required": 12, - "smb_signing_required": 13, + "nis_authentication_enabled": 5, + "smb_encryption_required": 13, + "smb_signing_required": 14, "svm": 2 }, "renameByName": { @@ -4705,6 +4451,7 @@ "Value #G": "LDAP Payload Signing", "Value #I": "LDAP Encryption", "activediruser": "Active Directory Users", + "anti_ransomware_state": "Anti-ransomware Status", "asupEnabled": "", "asupHttpsConfigured": "", "audit_protocol_enabled": "Audit Log", From 9f791b4614a1afc8666d60102c1553e45d39fe17 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Mon, 21 Aug 2023 15:00:16 +0530 Subject: [PATCH 34/40] feat: add junction paths in Volumes Dashboard --- conf/rest/9.10.0/volume.yaml | 3 +++ conf/zapi/cdot/9.8.0/volume.yaml | 2 ++ grafana/dashboards/cmode/volume.json | 8 +++++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/conf/rest/9.10.0/volume.yaml b/conf/rest/9.10.0/volume.yaml index c45c7ad8c..399be8bdb 100644 --- a/conf/rest/9.10.0/volume.yaml +++ b/conf/rest/9.10.0/volume.yaml @@ -13,6 +13,7 @@ counters: - ^anti_ransomware.state => antiRansomwareState - ^encryption.enabled => isEncrypted - ^is_svm_root => svm_root + - ^nas.path => junction_path - ^snaplock.type => snaplock_type - ^snapshot_policy.name => snapshot_policy - ^space.snapshot.autodelete_enabled => snapshot_autodelete @@ -50,6 +51,7 @@ counters: - autosize - encryption.enabled - is_svm_root + - nas.path - snaplock.type - space @@ -138,6 +140,7 @@ export_options: - isEncrypted - isHardwareEncrypted - is_sis_volume + - junction_path - root_volume - snaplock_type - snapshot_autodelete diff --git a/conf/zapi/cdot/9.8.0/volume.yaml b/conf/zapi/cdot/9.8.0/volume.yaml index 250620bd2..24f304883 100644 --- a/conf/zapi/cdot/9.8.0/volume.yaml +++ b/conf/zapi/cdot/9.8.0/volume.yaml @@ -14,6 +14,7 @@ counters: - volume-id-attributes: - ^^instance-uuid => instance_uuid - ^containing-aggregate-uuid => aggrUuid + - ^junction-path => junction_path - ^name => volume - ^owning-vserver-name => svm - ^style-extended => style @@ -117,6 +118,7 @@ export_options: - isEncrypted - isHardwareEncrypted - is_sis_volume + - junction_path - node_root - root_volume - snapshot_autodelete diff --git a/grafana/dashboards/cmode/volume.json b/grafana/dashboards/cmode/volume.json index ef144d091..dd12e165c 100644 --- a/grafana/dashboards/cmode/volume.json +++ b/grafana/dashboards/cmode/volume.json @@ -71,7 +71,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1692165657901, + "iteration": 1692609731870, "links": [ { "asDropdown": true, @@ -892,7 +892,8 @@ "Value #F", "Value #G", "Value #H", - "Value #I" + "Value #I", + "junction_path" ] } } @@ -936,7 +937,8 @@ "Value #F": "Compression Space Saved", "Value #G": "Total Space Saved", "Value #H": "Logical Space Used", - "Value #I": "Physical Space Used" + "Value #I": "Physical Space Used", + "junction_path": "Junction Path" } } } From ccfa3576a2fb72e418c3ab6a3b959db73dcc68c9 Mon Sep 17 00:00:00 2001 From: Hardikl <83282894+Hardikl@users.noreply.github.com> Date: Mon, 21 Aug 2023 17:42:58 +0530 Subject: [PATCH 35/40] fix: adding log forwarding column in compliance table in security dashboard (#2306) * fix: adding log forwarding column in compliance table in security dashboard * fix: minor correction in security dashboard * fix: minor correction in security dashboard * fix: add test case (#2307) * feat: address review comments --------- Co-authored-by: Rahul --- cmd/tools/grafana/dashboard_test.go | 21 +++++ grafana/dashboards/cmode/security.json | 109 +++++++++++++++---------- 2 files changed, 89 insertions(+), 41 deletions(-) diff --git a/cmd/tools/grafana/dashboard_test.go b/cmd/tools/grafana/dashboard_test.go index 6b7fbd87d..d7d151b39 100644 --- a/cmd/tools/grafana/dashboard_test.go +++ b/cmd/tools/grafana/dashboard_test.go @@ -164,7 +164,15 @@ func checkDashboardForDatasource(t *testing.T, path string, data []byte) { // Check that the variable DS_PROMETHEUS exist doesDsPromExist := false + // This is a list of names that are exempt from the check for a 'true' selected status. + excludedNames := map[string]bool{ + "TopResources": true, + "Interval": true, + "IncludeRoot": true, + } + gjson.GetBytes(data, "templating.list").ForEach(func(key, value gjson.Result) bool { + name := value.Get("name").String() if value.Get("name").String() == "DS_PROMETHEUS" { doesDsPromExist = true query := value.Get("query").String() @@ -176,6 +184,19 @@ func checkDashboardForDatasource(t *testing.T, path string, data []byte) { t.Errorf("dashboard=%s var=DS_PROMETHEUS type want=datasource got=%s", path, theType) } } + + if !excludedNames[name] { + if value.Get("current.selected").String() == "true" { + t.Errorf( + "dashboard=%s var=current.selected query want=false got=%s text=%s value=%s name= %s", + path, + "true", + value.Get("current.text"), + value.Get("current.value"), + name, + ) + } + } return true }) if !doesDsPromExist { diff --git a/grafana/dashboards/cmode/security.json b/grafana/dashboards/cmode/security.json index b804755e3..52339923b 100644 --- a/grafana/dashboards/cmode/security.json +++ b/grafana/dashboards/cmode/security.json @@ -2153,7 +2153,7 @@ "panels": [ { "datasource": "${DS_PROMETHEUS}", - "description": "❌ means this attribute is non-compliant. \n\n| Column | Compliant When | \n|---|---|\n| `Snapshot Policy` | All volumes have applied Snapshot policy | \n| `Snapshot Autodelete` | All volumes have enabled Snapshot autodelete |\n| `ARW Protection for SVMs` | All SVMs have enabled ARW protection |\n|`ARW Protection for Volumes`| All volumes have enabled ARW protection|\n| `Cluster Certificate Validity` | Cluster has active certificate(s) |\n| `Global FIPS`| Cluster has global FIPS enabled |\n| `Telnet` | Cluster has telnet disabled |\n| `Autosupport Https Transport` | Cluster uses HTTPS for autosupport |\n| `Default Admin User` | Default admin user is locked |\n| `Remote Shell` | Cluster's remote shell is disabled |\n| `MD5 in use` | Cluster does not use MD5 algorithm |\n| `Insecure SSH Settings` | Cluster has strong SSH server ciphers |\n| `Login Banner` | Cluster has enabled login banner |\n| `Network Time Protocol` | Cluster has configured three NTP servers |\n| `Cluster Peering` | Cluster peers use encryption |\n| `Notification Configured` | Cluster has configured destinations for notifications |\n| `Automatic Updates Configured`| Cluster has enabled automatic updates |", + "description": "❌ means this attribute is non-compliant. \n\n| Column | Compliant When | \n|---|---|\n| `Snapshot Policy` | All volumes have applied Snapshot policy | \n| `Snapshot Autodelete` | All volumes have enabled Snapshot autodelete |\n| `ARW Protection for SVMs` | All SVMs have enabled ARW protection |\n|`ARW Protection for Volumes`| All volumes have enabled ARW protection|\n| `Cluster Certificate Validity` | Cluster has active certificate(s) |\n| `Global FIPS`| Cluster has global FIPS enabled |\n| `Telnet` | Cluster has telnet disabled |\n| `Autosupport Https Transport` | Cluster uses HTTPS for autosupport |\n| `Default Admin User` | Default admin user is locked |\n| `Remote Shell` | Cluster's remote shell is disabled |\n| `MD5 in use` | Cluster does not use MD5 algorithm |\n| `Insecure SSH Settings` | Cluster has strong SSH server ciphers |\n| `Login Banner` | Cluster has enabled login banner |\n| `Log Forwarding Encrypted` | Cluster has encrypted protocol for log forwarding |\n| `Network Time Protocol` | Cluster has configured three NTP servers |\n| `Cluster Peering` | Cluster peers use encryption |\n| `Notification Configured` | Cluster has configured destinations for notifications |\n| `Automatic Updates Configured`| Cluster has enabled automatic updates |", "fieldConfig": { "defaults": { "color": { @@ -3265,6 +3265,46 @@ ] } ] + }, + { + "matcher": { + "id": "byName", + "options": "Log Forwarding Encrypted" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "tcp_encrypted": { + "index": 0, + "text": "Yes" + }, + "tcp_unencrypted": { + "index": 2, + "text": "❌ No" + }, + "udp_unencrypted": { + "index": 1, + "text": "❌ No" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "index": 3, + "text": "Not Configured" + } + }, + "type": "special" + } + ] + } + ] } ] }, @@ -3496,6 +3536,16 @@ "interval": "", "legendFormat": "", "refId": "U" + }, + { + "exemplar": false, + "expr": "security_audit_destination_status{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "V" } ], "title": "Cluster Compliance", @@ -3529,7 +3579,8 @@ "Value #G", "banner", "insecured", - "Value #D" + "Value #D", + "protocol" ] } } @@ -3571,27 +3622,28 @@ "indexByName": { "ArwStatus": 5, "Value #D": 10, - "Value #G": 15, + "Value #G": 16, "Value #H": 2, - "Value #I": 19, - "Value #J": 18, - "Value #K": 23, - "Value #L": 20, - "Value #M": 21, - "Value #N": 22, + "Value #I": 20, + "Value #J": 19, + "Value #K": 24, + "Value #L": 21, + "Value #M": 22, + "Value #N": 23, "Value #P": 0, "Value #Q": 3, "Value #R": 4, - "Value #T": 17, + "Value #T": 18, "asup_enabled": 11, - "auto_update_enabled": 16, + "auto_update_enabled": 17, "banner": 14, "certificateExpiryStatus": 6, - "certificateIssuerType": 24, + "certificateIssuerType": 25, "cluster": 1, "fips_enabled": 7, "insecured": 9, "locked": 12, + "protocol": 15, "rsh_enabled": 13, "telnet_enabled": 8 }, @@ -3630,6 +3682,7 @@ "localuser": "Local Users", "locked": "Default Admin User", "ntp": "Network Time Protocol", + "protocol": "Log Forwarding Encrypted", "rsh_enabled": "Remote Shell", "samluser": "Saml Users", "telnet_enabled": "Telnet" @@ -4519,17 +4572,7 @@ }, { "allValue": null, - "current": { - "selected": true, - "text": [ - "rest", - "zapi" - ], - "value": [ - "rest", - "zapi" - ] - }, + "current": {}, "datasource": "${DS_PROMETHEUS}", "definition": "label_values(svm_labels{system_type!=\"7mode\"},datacenter)", "description": null, @@ -4555,15 +4598,7 @@ }, { "allValue": null, - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, + "current": {}, "datasource": "${DS_PROMETHEUS}", "definition": "label_values(svm_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", "description": null, @@ -4589,15 +4624,7 @@ }, { "allValue": null, - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, + "current": {}, "datasource": "${DS_PROMETHEUS}", "definition": "label_values(svm_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",root_svm=\"No\"},svm)", "description": "Displaying only the data SVMs and omitting root SVMs", From 874a1f87004214bd9dbfacadb97e3ae8d94d552e Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Mon, 21 Aug 2023 08:32:41 -0400 Subject: [PATCH 36/40] doc: changelog for 23.08 (#2295) * doc: changelog for 23.08 --- CHANGELOG.md | 260 ++++++++++++++++++++++++++++++++++++++++++ README.md | 5 + pkg/changelog/main.go | 66 ++++++++++- 3 files changed, 329 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9b252c5e..707d2b2d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,266 @@ # Change Log ## [Releases](https://github.com/NetApp/harvest/releases) +## 23.08.0 / 2023-08-21 Release +:pushpin: Highlights of this major release include: +- Harvest Security dashboard highlights compliance using [NetApp's Security hardening guide for ONTAP](https://www.netapp.com/media/10674-tr4569.pdf) + +- Harvest's credential script supports ONTAP daily credential rotation. Thanks to @mamoep for raising. + +- :tophat: Harvest makes it easy to run with both the [ZAPI and REST collectors](https://netapp.github.io/harvest/latest/architecture/rest-strategy/) at the same time. Overlapping resources are deduplicated and only exported once. Harvest will automatically upgrade ZAPI conversations to REST when ZAPIs are suspended or disabled. + +- :gem: Updated workload dashboard now includes Service Center, Latency Breakdown, and 50 panels + +- :gem: Cluster dashboard updated to work with FSx. Some panels are blank because FSx does not have that data. + +- :mega: The Harvest team published a couple of screencasts about: + - [Why Harvest](https://youtu.be/04-66_9egJc) + - [Harvest Quick Start: Docker Compose](https://youtu.be/4cbDKzwjGHI) + +- :star: Several of the existing dashboards include new panels in this release: + - Aggregate dashboard includes busy volume panels + - SVM dashboard includes per NFS latency heatmaps. Thanks to @rbrownATnetapp for raising. + - Volume dashboard includes top resources by other IOPs panel and junction paths. Thanks to @tsohst for raising. + +- All Harvest dashboard tables include column filters +- Harvest dashboards use color to highlight latency and busy threshold breaches +- Harvest's Prometheus exporter supports TLS + +- :ear_of_rice: Harvest includes new templates to collect: + - Iwarp metrics + - FCVI metrics + - Per volume NFS metrics + - Volume clone metrics + - QoS workload policy metrics + - NVME/TCP and NVME/RoCE metrics + - Flashpool metrics are included in RestPerf. Thanks to @lobster1860 for raising + +- :closed_book: Documentation additions + - Move more documentation from GitHub to [Harvest documentation site](https://netapp.github.io/harvest/) + - Clarify how to tell Harvest to continue using the ZAPI protocol + - Clarify generic vs custom plugins. Thanks to GregS for raising + - Clarify which version of Go is required to build Harvest. Thanks to MikeK for raising + - Clarify how to prepare ONTAP cDOT clusters for Harvest data collection + - EMS documentation should point to Harvest documentation site. Thanks to @cwaltham for raising + - Clarify how to gather log files on all platforms + - Explain how to use the `--labels` option of `bin/harvest grafana`. Thanks to @slater0013 for raising + - Describe how to run docker compose generate command without required Harvest binaries + +- The Harvest `doctor` command validates collector names listed in your `harvest.yml` file + +- An earlier version of Harvest collected cloud store information via REST. This release adds the same for ZAPI + +- When ONTAP resources are missing, Harvest tries to collect them every hour. Earlier versions of Harvest waited 24 hours before retrying, which often caused metrics to be missing after a cluster upgrade. Thanks to @Falcon667 for raising + +- Earlier versions of Harvest created world writable auto-support files. These files are now only read/writeable by the current user. Thanks to Bunnygirl for raising + +- `bin/harvest import` should work with Grafana 10. Thanks to @wooyoungAhn for raising + +## Announcements + +:bangbang: **IMPORTANT** `23.08` fixes a REST collector bug that caused partial data collection when ONTAP paginated results. See #2109 for details. + +:bangbang: **IMPORTANT** Release `23.08` disables the `NetConnections` and `NFSClients` templates by default. You can enable them if needed. These templates were disabled because several customers reported that these templates created millions of metrics. None of these metrics are used in Harvest dashboards. + +:bulb: The Compliance dashboard was removed after its panels were moved to the Security dashboard. + +:eyes: Ambient temperature metric may experience an increase due to issue #2259 + +:bangbang: **IMPORTANT** NetApp moved their communities from Slack to [Discord](https://discord.gg/ZmmWPHTBHw), please join us [there](https://discordapp.com/channels/855068651522490400/1001963189124206732)! + +:bangbang: **IMPORTANT** If using Docker Compose and you want to keep your historical Prometheus data, please +read [how to migrate your Prometheus volume](https://github.com/NetApp/harvest/blob/main/docs/MigratePrometheusDocker.md) + +:bulb: **IMPORTANT** After upgrade, don't forget to re-import your dashboards, so you get all the new enhancements and fixes. You can import them via the `bin/harvest grafana import` CLI, from the Grafana UI, or from the `Maintenance > Reset Harvest Dashboards` button in NAbox. + +## Known Issues + +- Some AFF A250 systems do not report power metrics. See ONTAP bug [1511476](https://burtview.netapp.com/burt/burt-bin/start?burt-id=1511476) for more details. + +- ONTAP does not include REST metrics for `offbox_vscan_server` and `offbox_vscan` until ONTAP 9.13.1. See ONTAP bug + [1473892](https://burtview.netapp.com/burt/burt-bin/start?burt-id=1473892) for more details. + +**IMPORTANT** 7-mode filers that are not on the latest release of ONTAP may experience TLS connection issues with errors like `tls: server selected unsupported protocol version 301` This is caused by a change in Go 1.18. The [default for TLS client connections was changed to TLS 1.2](https://tip.golang.org/doc/go1.18#tls10) in Go 1.18. Please upgrade your 7-mode filers (recommended) or set `tls_min_version: tls10` in your `harvest.yml` [poller section](https://github.com/NetApp/harvest/tree/release/22.05.0#pollers). See [#1007](https://github.com/NetApp/harvest/issues/1007) for more details. + +## Thanks to all the awesome contributors + +:metal: Thanks to all the people who've opened issues, asked questions on Discord, and contributed code or dashboards +this release: + +@7840vz, @DAx-cGn, @Falcon667, @Hedius, @LukaszWasko, @MrObvious, @ReneMeier, @Sawall10, @T1r0l, @XDavidT, @amd-eulee, @aticatac, @chadpruden, @cwaltham, @cygio, @ddhti, @debert-ntap, @demalik, @electrocreative, @elsgaard, @ev1963, @faguayot, @iStep2Step, @jgasher, @jmg011, @lobster1860, @mamoep, @matejzero, @matthieu-sudo, @merdos, @pilot7777, @rbrownATnetapp, @rodenj1, @slater0013, @swordfish291, @tsohst, @wooyoungAhn, Alessandro.Nuzzo, Ed Wilts, GregS, Imthenightbird, KlausHub, MeghanaD, MikeK, Paul P2, Rusty Brown, Shubham Mer, Tudor Pascu, Watson9121, jf38800, jfong, lorenzoc, rcl23, roller, scrhobbs, troysmuller, twodot0h + +:seedling: This release includes 42 features, 40 bug fixes, 20 documentation, 2 performance, 4 testing, 1 styling, 9 refactoring, 20 miscellaneous, and 12 ci pull requests. + +### :rocket: Features +- Harvest Should Collect Iwarp Counters ([#2071](https://github.com/NetApp/harvest/pull/2071)) +- Update Visitpanels To Be Recursive ([#2085](https://github.com/NetApp/harvest/pull/2085)) +- Add Table Column Filter For Dashboards ([#2088](https://github.com/NetApp/harvest/pull/2088)) +- Update Lagtime Based On Lasttransfersize ([#2091](https://github.com/NetApp/harvest/pull/2091)) +- Harvest Should Add Grafana Import Rewrite Svm Filtering For Multi-Tenant Support ([#2092](https://github.com/NetApp/harvest/pull/2092)) +- Fetch Cloud_store Info In Zapi Via Plugin ([#2094](https://github.com/NetApp/harvest/pull/2094)) +- Collection Of Other Counters For Fcvi Perf Object ([#2096](https://github.com/NetApp/harvest/pull/2096)) +- Add Nfs Io Types At The Volume Level ([#2098](https://github.com/NetApp/harvest/pull/2098)) +- Add System Defined Workload Collection ([#2099](https://github.com/NetApp/harvest/pull/2099)) +- Add Workload Panels In Workload Dashboard ([#2100](https://github.com/NetApp/harvest/pull/2100)) +- Add Volume Clone Info In Rest ([#2102](https://github.com/NetApp/harvest/pull/2102)) +- Added Volume Panels In Aggr Dashboard ([#2104](https://github.com/NetApp/harvest/pull/2104)) +- Workload Policy Iops Metrics ([#2111](https://github.com/NetApp/harvest/pull/2111)) +- Autoresolve Ems Would Export Metric Value As 0 And Autoresolve=True Label ([#2120](https://github.com/NetApp/harvest/pull/2120)) +- Support Type Label For Volume For Backward Compatibility ([#2132](https://github.com/NetApp/harvest/pull/2132)) +- Volume Clone Info For Zapi ([#2140](https://github.com/NetApp/harvest/pull/2140)) +- Harvest Should Include Numpollers And Rss In Autosupport ([#2143](https://github.com/NetApp/harvest/pull/2143)) +- Colors In Grafana Dashboards To Highlight Warning, Critical Severity ([#2147](https://github.com/NetApp/harvest/pull/2147)) +- Security Hardening Guide ([#2150](https://github.com/NetApp/harvest/pull/2150)) +- Harvest Prometheus Exporter Should Support Tls ([#2153](https://github.com/NetApp/harvest/pull/2153)) +- Latency Units Should Be In Microseconds In Harvest Dashboard ([#2156](https://github.com/NetApp/harvest/pull/2156)) +- Simplify Rest Auto-Upgrade ([#2167](https://github.com/NetApp/harvest/pull/2167)) +- When Using A Credential Script, Re-Auth On 401S ([#2180](https://github.com/NetApp/harvest/pull/2180)) +- Upgrade Zapi Conversations To Rest When Zapis Are Suspended Or … ([#2200](https://github.com/NetApp/harvest/pull/2200)) +- When Using A Credential Script, Re-Auth On 401S ([#2203](https://github.com/NetApp/harvest/pull/2203)) +- Merge Compliance And Security Dashboard + Added Arw Fields ([#2207](https://github.com/NetApp/harvest/pull/2207)) +- Supporting Topk In S3 Dashboard ([#2208](https://github.com/NetApp/harvest/pull/2208)) +- Aff250 Power Calculation ([#2211](https://github.com/NetApp/harvest/pull/2211)) +- Use Single `Go Build` Command To Build Harvest And Poller Binaries ([#2221](https://github.com/NetApp/harvest/pull/2221)) +- Harvest Should Include A User Agent ([#2224](https://github.com/NetApp/harvest/pull/2224)) +- Add Collector Name Validation In Doctor ([#2229](https://github.com/NetApp/harvest/pull/2229)) +- Harvest Should Fetch Certificates Via A Script ([#2238](https://github.com/NetApp/harvest/pull/2238)) +- Include Lun Offline Ems Alert ([#2252](https://github.com/NetApp/harvest/pull/2252)) +- Add Panel For Other Iops On Volume Dashboard ([#2254](https://github.com/NetApp/harvest/pull/2254)) +- Update Ambient Temperature Calculation For Power Dashboard ([#2259](https://github.com/NetApp/harvest/pull/2259)) +- Nvme/Tcp And Nvme/Roce Counters ([#2264](https://github.com/NetApp/harvest/pull/2264)) +- Harvest Svm Dashboard Should Include Latency Heatmap Panels Nfs… ([#2268](https://github.com/NetApp/harvest/pull/2268)) +- Added Table Description For Cluster Compliance ([#2269](https://github.com/NetApp/harvest/pull/2269)) +- Update Ontap Metric Document ([#2270](https://github.com/NetApp/harvest/pull/2270)) +- Add Cpu_firmware_release To Cluster Dashboard ([#2274](https://github.com/NetApp/harvest/pull/2274)) +- Enable Cluster Dashboard For Fsx ([#2303](https://github.com/NetApp/harvest/pull/2303)) +- Add Junction Paths In Volumes Dashboard ([#2309](https://github.com/NetApp/harvest/pull/2309)) + +### :bug: Bug Fixes +- Disk Dashboard Power On Time Should Use `Seconds` Unit ([#2039](https://github.com/NetApp/harvest/pull/2039)) +- Update Metadata Cpu Times: Breakdown To Seconds ([#2055](https://github.com/NetApp/harvest/pull/2055)) +- Workload Missing Label Value ([#2072](https://github.com/NetApp/harvest/pull/2072)) +- Fcvi Restperf Template ([#2080](https://github.com/NetApp/harvest/pull/2080)) +- Change Svm Panels Row Name ([#2097](https://github.com/NetApp/harvest/pull/2097)) +- Correct Unit In Panels With Added Testcase ([#2108](https://github.com/NetApp/harvest/pull/2108)) +- Rest Collector Incomplete Data If Retrieval Exceeds Return_timeout ([#2110](https://github.com/NetApp/harvest/pull/2110)) +- Storagegrid Should Honor `-Logtofile` Option ([#2119](https://github.com/NetApp/harvest/pull/2119)) +- Harvest Should Always Pass `Addr` Argument To Credentials_script ([#2128](https://github.com/NetApp/harvest/pull/2128)) +- Handle Difference Of Pollinstance And Polldata Records Via Exportable ([#2137](https://github.com/NetApp/harvest/pull/2137)) +- Cpu_busy Description In Cluster Dashboard ([#2141](https://github.com/NetApp/harvest/pull/2141)) +- Reduce Auto Support Log Noise When Collecting Process Info On Mac ([#2145](https://github.com/NetApp/harvest/pull/2145)) +- Correct The Flashpool Panel Units ([#2163](https://github.com/NetApp/harvest/pull/2163)) +- Handling Label Count When Matches Applied In Ems ([#2165](https://github.com/NetApp/harvest/pull/2165)) +- Volume Template Fix ([#2171](https://github.com/NetApp/harvest/pull/2171)) +- Harvest Should Retry Every Hour When Ontap Replies With An Api-R… ([#2181](https://github.com/NetApp/harvest/pull/2181)) +- Ciphers Query Was Giving Wrong Result In Promql ([#2188](https://github.com/NetApp/harvest/pull/2188)) +- S3 Dashboard Fails To Import In Grafana 8.5.15 ([#2191](https://github.com/NetApp/harvest/pull/2191)) +- Harvest Auto-Support Files Should Not Be World Writable ([#2193](https://github.com/NetApp/harvest/pull/2193)) +- Fix Key For Qtree 7Mode ([#2196](https://github.com/NetApp/harvest/pull/2196)) +- Check Existing Asup Dir Permission ([#2197](https://github.com/NetApp/harvest/pull/2197)) +- Import Dashboard Failure With Editor Role In Grafana ([#2206](https://github.com/NetApp/harvest/pull/2206)) +- When Using Credentials_file Make Sure Defaults Are Copied To Poller ([#2209](https://github.com/NetApp/harvest/pull/2209)) +- When Using Credentials_file Make Sure Defaults Are Copied To Poller ([#2215](https://github.com/NetApp/harvest/pull/2215)) +- Flashpool-Data Is Missing In Restperf ([#2217](https://github.com/NetApp/harvest/pull/2217)) +- Disable Nfs_clients.yaml Template By Default In Rest Collector ([#2219](https://github.com/NetApp/harvest/pull/2219)) +- Remove Duplicate Error Message ([#2222](https://github.com/NetApp/harvest/pull/2222)) +- Correct Svm Rest Template Based On Version ([#2239](https://github.com/NetApp/harvest/pull/2239)) +- Correct Shelf Metrics In 7Mode ([#2245](https://github.com/NetApp/harvest/pull/2245)) +- Remove Source_node Label From Snapmirror Zapi ([#2255](https://github.com/NetApp/harvest/pull/2255)) +- Added Version Check For Aggr-Object-Store-Get-Iter ([#2258](https://github.com/NetApp/harvest/pull/2258)) +- Volume Rest Template Based On Version ([#2263](https://github.com/NetApp/harvest/pull/2263)) +- Nfs Heatmap Per Cluster ([#2273](https://github.com/NetApp/harvest/pull/2273)) +- Make Poller Mandatory For Metrics Generation Cmd ([#2280](https://github.com/NetApp/harvest/pull/2280)) +- Handled When Metric Not Found In Plugin ([#2281](https://github.com/NetApp/harvest/pull/2281)) +- Disable Netconnections In Rest By Default ([#2283](https://github.com/NetApp/harvest/pull/2283)) +- Grafana Ask-For-Token Should Retry At Most 5 Times ([#2284](https://github.com/NetApp/harvest/pull/2284)) +- Match Object Name With Zapiperf For Cifs_vserver.yaml ([#2288](https://github.com/NetApp/harvest/pull/2288)) +- Add Bin Dir Check Before Removing Files ([#2289](https://github.com/NetApp/harvest/pull/2289)) +- Adding Log Forwarding Column In Compliance Table In Security Dashboard ([#2306](https://github.com/NetApp/harvest/pull/2306)) + +### :closed_book: Documentation +- Explain Bin/Grafana Import --Labels ([#2032](https://github.com/NetApp/harvest/pull/2032)) +- Update Release Checklist ([#2043](https://github.com/NetApp/harvest/pull/2043)) +- Update Docker Compose Generation Process To Remove Binary Dependencies ([#2046](https://github.com/NetApp/harvest/pull/2046)) +- Add Details About Volume Sis Stat Panel ([#2047](https://github.com/NetApp/harvest/pull/2047)) +- Add Harvest-Metrics Release Branch Creation For Release Steps ([#2050](https://github.com/NetApp/harvest/pull/2050)) +- Fix Rest Template Extend Instructions Path ([#2051](https://github.com/NetApp/harvest/pull/2051)) +- Fsx Does Not Support Headroom Dashboard ([#2131](https://github.com/NetApp/harvest/pull/2131)) +- Update Fsa Dashboard Doc ([#2159](https://github.com/NetApp/harvest/pull/2159)) +- Move K8 Podman Document To Documentation Site ([#2160](https://github.com/NetApp/harvest/pull/2160)) +- Clarify How To Tell Harvest To Continue Using The Zapi Protocol ([#2162](https://github.com/NetApp/harvest/pull/2162)) +- Clarify Generic Vs Custom Plugins ([#2166](https://github.com/NetApp/harvest/pull/2166)) +- Update Docker Docs Link To Doc Site ([#2186](https://github.com/NetApp/harvest/pull/2186)) +- Clarify Which Version Of Go Is Required ([#2214](https://github.com/NetApp/harvest/pull/2214)) +- Give Authentication Precedence Its Own Section ([#2226](https://github.com/NetApp/harvest/pull/2226)) +- Add Note About Workload Counter In Default Templates ([#2230](https://github.com/NetApp/harvest/pull/2230)) +- Simplify The Preparing Ontap Cdot Cluster Documentation ([#2231](https://github.com/NetApp/harvest/pull/2231)) +- Fix Ems Link ([#2244](https://github.com/NetApp/harvest/pull/2244)) +- Update Metric Generate Step Command ([#2279](https://github.com/NetApp/harvest/pull/2279)) +- Move Troubleshoot Docs To Doc Site ([#2287](https://github.com/NetApp/harvest/pull/2287)) +- Release 23.08 Metric Docs ([#2290](https://github.com/NetApp/harvest/pull/2290)) + +### :zap: Performance +- Improve Memory And Cpu Performance Of Restperf Collector ([#2053](https://github.com/NetApp/harvest/pull/2053)) +- Optimize Restperf Collector Pollinstance ([#2121](https://github.com/NetApp/harvest/pull/2121)) + +### :wrench: Testing +- Add Unit Test For Restperf ([#2044](https://github.com/NetApp/harvest/pull/2044)) +- Adding Ems Unit Tests ([#2052](https://github.com/NetApp/harvest/pull/2052)) +- Add Unit Test For Rest Collector ([#2062](https://github.com/NetApp/harvest/pull/2062)) +- Ensure Dashboard Time Is Now-3H ([#2275](https://github.com/NetApp/harvest/pull/2275)) + +### Styling +- Address All Lint Errors In Ci ([#2014](https://github.com/NetApp/harvest/pull/2014)) + +### Refactoring +- Move Unit Testing Json Parser To Common ([#2064](https://github.com/NetApp/harvest/pull/2064)) +- Dashboard Tests ([#2090](https://github.com/NetApp/harvest/pull/2090)) +- Harvest Dashboard Jsons Should Be Sorted By Key ([#2152](https://github.com/NetApp/harvest/pull/2152)) +- Eliminate Usages Of Time.sleep In Test Code ([#2182](https://github.com/NetApp/harvest/pull/2182)) +- Fix Inconsistent Pointer Receivers ([#2225](https://github.com/NetApp/harvest/pull/2225)) +- Reduce Asup Log Noise ([#2276](https://github.com/NetApp/harvest/pull/2276)) +- Increase Max Log File Size From 5Mb To 10Mb ([#2277](https://github.com/NetApp/harvest/pull/2277)) +- Add Cp Command In Dashboard Sort Test ([#2278](https://github.com/NetApp/harvest/pull/2278)) +- Code Cleanup ([#2282](https://github.com/NetApp/harvest/pull/2282)) + +### Miscellaneous +- Bump Github.com/Shirou/Gopsutil/V3 From 3.23.3 To 3.23.4 ([#2027](https://github.com/NetApp/harvest/pull/2027)) +- Bump Golang.org/X/Term From 0.7.0 To 0.8.0 ([#2056](https://github.com/NetApp/harvest/pull/2056)) +- Bump Golang.org/X/Sys From 0.7.0 To 0.8.0 ([#2057](https://github.com/NetApp/harvest/pull/2057)) +- Add Renovate Bot ([#2075](https://github.com/NetApp/harvest/pull/2075)) +- Update Module Github.com/Imdario/Mergo To V0.3.16 ([#2112](https://github.com/NetApp/harvest/pull/2112)) +- Update Renovate Bot ([#2116](https://github.com/NetApp/harvest/pull/2116)) +- Update Renovate Commit Prefix ([#2117](https://github.com/NetApp/harvest/pull/2117)) +- Update Module Github.com/Shirou/Gopsutil/V3 To V3.23.5 ([#2122](https://github.com/NetApp/harvest/pull/2122)) +- Update All Dependencies ([#2139](https://github.com/NetApp/harvest/pull/2139)) +- Update Module Github.com/Imdario/Mergo To V1 ([#2144](https://github.com/NetApp/harvest/pull/2144)) +- Upgrade Mergo Package ([#2157](https://github.com/NetApp/harvest/pull/2157)) +- Update Module Github.com/Shirou/Gopsutil/V3 To V3.23.6 ([#2174](https://github.com/NetApp/harvest/pull/2174)) +- Update All Dependencies ([#2176](https://github.com/NetApp/harvest/pull/2176)) +- Update Module Golang.org/X/Term To V0.10.0 ([#2183](https://github.com/NetApp/harvest/pull/2183)) +- Bump Go ([#2205](https://github.com/NetApp/harvest/pull/2205)) +- Update All Dependencies ([#2243](https://github.com/NetApp/harvest/pull/2243)) +- Bump Go ([#2253](https://github.com/NetApp/harvest/pull/2253)) +- Update All Dependencies ([#2261](https://github.com/NetApp/harvest/pull/2261)) +- Bump Go ([#2285](https://github.com/NetApp/harvest/pull/2285)) +- Update Module Github.com/Tidwall/Gjson To V1.16.0 ([#2286](https://github.com/NetApp/harvest/pull/2286)) + +### :hammer: CI +- Wait For Qos_volume Counters ([#2045](https://github.com/NetApp/harvest/pull/2045)) +- Update Docs For Nightly Builds ([#2058](https://github.com/NetApp/harvest/pull/2058)) +- Add Gh-Pages Fetch Before Mkdoc Deploy ([#2067](https://github.com/NetApp/harvest/pull/2067)) +- Configure Renovate ([#2074](https://github.com/NetApp/harvest/pull/2074)) +- Renovate Should Ignore Integration ([#2078](https://github.com/NetApp/harvest/pull/2078)) +- Renovate Should Run On A Schedule ([#2082](https://github.com/NetApp/harvest/pull/2082)) +- Renovate Group All Prs ([#2136](https://github.com/NetApp/harvest/pull/2136)) +- Ensure Exported Prometheus Metrics Are Unique ([#2173](https://github.com/NetApp/harvest/pull/2173)) +- Run Renovate Once In A Week ([#2185](https://github.com/NetApp/harvest/pull/2185)) +- Include Harvest Certification Tool ([#2241](https://github.com/NetApp/harvest/pull/2241)) +- Fix Local Ci Errors ([#2266](https://github.com/NetApp/harvest/pull/2266)) +- Remove Apt-Get Update ([#2271](https://github.com/NetApp/harvest/pull/2271)) + +--- + ## 23.05.0 / 2023-05-03 :pushpin: Highlights of this major release include: - :gem: Seven new dashboards: diff --git a/README.md b/README.md index 044c5bace..a90209544 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,11 @@ and GitHub [discussions](https://github.com/NetApp/harvest/discussions). Come jo :closed_book: https://netapp.github.io/harvest/ +# Videos + +- [Why Harvest](https://youtu.be/04-66_9egJc) +- [Harvest Quick Start: Docker Compose](https://youtu.be/4cbDKzwjGHI) + ---

diff --git a/pkg/changelog/main.go b/pkg/changelog/main.go index 0f1f7b559..032a5ce39 100644 --- a/pkg/changelog/main.go +++ b/pkg/changelog/main.go @@ -43,6 +43,59 @@ type cli struct { openIssues []string } +func (c *cli) makeDraft() { + const envName = "RELEASE" + releaseName, ok := os.LookupEnv(envName) + if ok { + releaseName = fmt.Sprintf("releaseHighlights_%s.md", releaseName) + } else { + releaseName = "highlights.md" + log.Warn(). + Str("environmentVariable", envName). + Str("highlightsName", releaseName). + Msg("environment variable does not exist. Using highlightsName") + } + _, err := os.Stat(releaseName) + if err == nil { + log.Error().Str("file", releaseName).Msg("Refuse to overwrite existing file") + return + } + out, err := os.Create(releaseName) + if err != nil { + log.Error().Err(err).Str("releaseName", releaseName).Msg("Failed to create releaseName file") + return + } + _, _ = out.WriteString(` +- :gem: Seven new dashboards: + - StorageGRID and ONTAP fabric pool + - Health + +- :star: Several of the existing dashboards include new panels in this release: + +- :ear_of_rice: Harvest includes new templates to collect: + +- :closed_book: Documentation additions + +## Announcements + +:bangbang: **IMPORTANT** NetApp moved their communities from Slack to [Discord](https://discord.gg/ZmmWPHTBHw), please join us [there](https://discordapp.com/channels/855068651522490400/1001963189124206732)! + +:bangbang: **IMPORTANT** If using Docker Compose and you want to keep your historical Prometheus data, please +read [how to migrate your Prometheus volume](https://github.com/NetApp/harvest/blob/main/docs/MigratePrometheusDocker.md) + +:bulb: **IMPORTANT** After upgrade, don't forget to re-import your dashboards, so you get all the new enhancements and fixes. You can import them via the 'bin/harvest grafana import' CLI, from the Grafana UI, or from the 'Maintenance > Reset Harvest Dashboards' button in NAbox. + +## Known Issues + +## Thanks to all the awesome contributors + +:metal: Thanks to all the people who've opened issues, asked questions on Discord, and contributed code or dashboards +this release: + +- @Falcon667 +`) +} + func (c *cli) makeChangelog() { highlights, err := os.ReadFile(c.highlights) if err != nil { @@ -165,7 +218,7 @@ func newPr(matches []string) pr { } func (c *cli) printChangelog(highlightBytes []byte) { - fmt.Printf("## %s / %s\n", c.title, time.Now().Format("2006-01-02")) + fmt.Printf("## %s / %s Release\n", c.title, time.Now().Format("2006-01-02")) fmt.Printf(":pushpin: Highlights of this major release include:\n") highlights := string(highlightBytes) highlights = strings.TrimSpace(highlights) @@ -240,11 +293,12 @@ type prType struct { } func (c *cli) initPrTypes() { - c.prOrder = []string{"feat", "fix", "doc", "test", "style", "refactor", "chore", "ci"} + c.prOrder = []string{"feat", "fix", "doc", "perf", "test", "style", "refactor", "chore", "ci"} c.addPrType(prType{id: "feat", summary: "features", header: ":rocket: Features"}) c.addPrType(prType{id: "fix", summary: "bug fixes", header: ":bug: Bug Fixes"}) c.addPrType(prType{id: "doc", summary: "documentation", header: ":closed_book: Documentation"}) + c.addPrType(prType{id: "perf", summary: "performance", header: ":zap: Performance"}) c.addPrType(prType{id: "test", summary: "testing", header: ":wrench: Testing"}) c.addPrType(prType{id: "style", summary: "styling", header: "Styling"}) c.addPrType(prType{id: "refactor", summary: "refactoring", header: "Refactoring"}) @@ -275,6 +329,14 @@ func (c *cli) Root() *cobra.Command { _ = r.MarkFlagRequired("title") _ = r.MarkFlagRequired("highlights") _ = r.MarkFlagRequired("releaseHighlights") + + r.AddCommand(&cobra.Command{ + Use: "new", + Short: "create draft release highlights", + Run: func(cmd *cobra.Command, args []string) { + c.makeDraft() + }, + }) return r } From 8f44053f9acf7ef90e7685c710b8da6818e436bb Mon Sep 17 00:00:00 2001 From: Rahul Date: Mon, 21 Aug 2023 23:49:56 +0530 Subject: [PATCH 37/40] feat: remove daemon dependency (#2195) * feat: remove daemon dependency --- Makefile | 4 -- cmd/harvest/harvest.go | 25 +++++++- cmd/tools/daemonize/daemonize.c | 109 -------------------------------- 3 files changed, 24 insertions(+), 114 deletions(-) delete mode 100644 cmd/tools/daemonize/daemonize.c diff --git a/Makefile b/Makefile index 355d75de7..920da1be6 100644 --- a/Makefile +++ b/Makefile @@ -129,10 +129,6 @@ harvest: deps @echo "Building harvest" @GOOS=$(GOOS) GOARCH=$(GOARCH) $(FLAGS) go build -trimpath -o bin -ldflags=$(LD_FLAGS) ./cmd/harvest ./cmd/poller - @# Build the daemonize for the pollers - @echo "Building daemonize" - @cd cmd/tools/daemonize; gcc daemonize.c -o ../../../bin/daemonize - @cp service/contrib/grafana bin; chmod +x bin/grafana ############################################################################### diff --git a/cmd/harvest/harvest.go b/cmd/harvest/harvest.go index a385cbd77..732dabeba 100644 --- a/cmd/harvest/harvest.go +++ b/cmd/harvest/harvest.go @@ -439,7 +439,30 @@ func startPoller(pollerName string, promPort int, opts *options) { os.Exit(0) } - cmd := exec.Command(path.Join(HarvestHomePath, "bin", "daemonize"), argv...) //nolint:gosec + // Set the Setsid attribute to true, which creates a new session for the child process + // This effectively detaches the child process from the parent process + cmd := exec.Command(argv[0], argv[1:]...) //nolint:gosec + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setsid: true, + } + + // Redirect standard file descriptors to /dev/null + devNull, err := os.OpenFile(os.DevNull, os.O_RDWR, 0) + if err != nil { + fmt.Println("Error opening /dev/null:", err) + os.Exit(1) + } + defer func() { + if err := devNull.Close(); err != nil { + fmt.Println("Error closing /dev/null:", err) + } + }() + + cmd.Stdin = devNull + cmd.Stdout = devNull + cmd.Stderr = devNull + + // Start the poller process in the background if err := cmd.Start(); err != nil { fmt.Println(err) os.Exit(1) diff --git a/cmd/tools/daemonize/daemonize.c b/cmd/tools/daemonize/daemonize.c deleted file mode 100644 index 7a33febdc..000000000 --- a/cmd/tools/daemonize/daemonize.c +++ /dev/null @@ -1,109 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -/* - Executes command as a daemon process. Unlike what - is done usually (two forks and exit), we perform - execv after second fork, such that the daemon has - it's own (pretty) cmdline identity. This is helpful to - trace status of pollers. - - Script is only intended to daemonize Harvest pollers. - Daemonizing other processes should be ok, as long as - the command line arguments are correct, but no - guarantee. - - The daemonize function is implemented in the quick - and dirty way: some system calls would need better - error-checking for a safer implementation. - - Usage: - ./daemonize [args...] - - Arguments: - - executable path to executable program - - args optional, passed to daemon unchanged -*/ - -int daemonize(char *bin, char *args[]) { - - // open syslog to send messages - openlog("harvest daemonize", LOG_PID, LOG_USER); - - if (fork() != 0) - return 0; // parent exits - // child continues... - - // get new session ID - if (setsid() == -1) { - syslog(LOG_ERR, "setsid: %s", strerror(errno)); - return -1; - } - - // second fork, so we are not session leader - if (fork() != 0) - return 0; - // (grand) child continues - - // clean file permissions - umask(0); - - // close FDs, if can't get max, choose reasonable number - int maxfds, fd; - if ((maxfds = sysconf(_SC_OPEN_MAX)) == -1) - maxfds = 256; - for (fd=0; fd [args...]\n"); - exit(0); - } - - // construct path to executable and arg vector - // this is done here, so errors are detected early on - char* daemon_argv[100]; - char path[100]; - int i; - - strcpy(path, argv[1]); - - daemon_argv[0] = argv[1]; - - for (i=1; i Date: Mon, 21 Aug 2023 13:44:51 -0400 Subject: [PATCH 38/40] feat: enable more golanglint linters --- .golangci.yml | 20 +- cmd/admin/admin.go | 12 +- cmd/collectors/commonutils_test.go | 72 ++--- cmd/collectors/ems/ems.go | 16 +- cmd/collectors/ems/ems_test.go | 10 +- cmd/collectors/rest/plugins/disk/disk.go | 5 +- cmd/collectors/rest/plugins/health/health.go | 9 +- cmd/collectors/rest/plugins/sensor/sensor.go | 12 +- cmd/collectors/rest/plugins/svm/svm.go | 32 +- cmd/collectors/rest/plugins/volume/volume.go | 42 +-- .../volumeanalytics/volumeanalytics.go | 4 +- cmd/collectors/rest/rest.go | 7 +- cmd/collectors/restperf/plugins/disk/disk.go | 10 +- cmd/collectors/restperf/plugins/fcvi/fcvi.go | 5 +- .../restperf/plugins/volumetag/volumetag.go | 5 +- cmd/collectors/restperf/restperf.go | 282 +++++++++--------- cmd/collectors/storagegrid/rest/client.go | 4 +- cmd/collectors/storagegrid/storagegrid.go | 3 +- cmd/collectors/zapi/collector/parsers.go | 11 +- cmd/collectors/zapi/collector/zapi.go | 19 +- cmd/collectors/zapi/plugins/sensor/sensor.go | 12 +- .../zapi/plugins/sensor/sensor_test.go | 8 +- .../plugins/snapmirror/snapmirror_test.go | 60 +--- cmd/collectors/zapi/plugins/svm/svm.go | 60 ++-- cmd/collectors/zapi/plugins/volume/volume.go | 5 +- cmd/collectors/zapiperf/plugins/disk/disk.go | 8 +- .../externalserviceoperation.go | 5 +- cmd/collectors/zapiperf/plugins/fcvi/fcvi.go | 5 +- .../zapiperf/plugins/headroom/headroom.go | 5 +- .../zapiperf/plugins/volumetag/volumetag.go | 5 +- cmd/collectors/zapiperf/zapiperf.go | 79 +++-- cmd/collectors/zapiperf/zapiperf_test.go | 2 +- cmd/exporters/influxdb/influxdb.go | 8 +- cmd/exporters/prometheus/httpd.go | 6 +- cmd/exporters/prometheus/prometheus.go | 2 +- cmd/harvest/harvest.go | 2 +- cmd/harvest/version/version.go | 3 +- cmd/poller/collector/asup.go | 3 +- cmd/poller/plugin/labelagent/label_agent.go | 6 +- .../plugin/labelagent/label_agent_test.go | 26 +- cmd/poller/plugin/labelagent/parse_rules.go | 18 +- cmd/poller/plugin/metricagent/metric_agent.go | 3 +- cmd/poller/plugin/plugin.go | 52 ++-- cmd/poller/plugin/test/plugin_test.go | 2 +- cmd/poller/poller.go | 21 +- cmd/poller/schedule/schedule.go | 97 +++--- cmd/tools/doctor/doctor.go | 3 +- cmd/tools/doctor/restZapiDiff.go | 9 +- cmd/tools/generate/counter.go | 4 +- cmd/tools/grafana/grafana.go | 4 +- cmd/tools/rest/client.go | 6 +- cmd/tools/rest/rest.go | 12 +- cmd/tools/template/template_test.go | 3 +- cmd/tools/zapi/export.go | 2 +- pkg/api/ontapi/zapi/client.go | 22 +- pkg/conf/conf.go | 3 +- pkg/logging/logger.go | 2 +- pkg/test/node_test.go | 2 +- pkg/tree/node/node.go | 29 +- pkg/tree/node/node_test.go | 2 +- pkg/util/util.go | 3 +- 61 files changed, 541 insertions(+), 648 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 84affe444..8a848b94f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,29 +1,44 @@ linters: disable-all: true enable: + - asasalint + - bidichk - bodyclose + - contextcheck + - dogsled - dupword + - durationcheck - errcheck + - errchkjson - errname - errorlint + - exhaustive - exportloopref + - gocheckcompilerdirectives - gosec - gosimple - govet - ineffassign - makezero + - mirror - nilerr - nolintlint - nonamedreturns - nosprintfhostport + - reassign + - revive - staticcheck - stylecheck - tenv - - thelper + - tenv + - testableexamples - typecheck - unconvert - unparam - unused + - usestdlibvars + - wastedassign + - zerologlint issues: max-issues-per-linter: 0 @@ -38,3 +53,6 @@ linters-settings: thelper: test: begin: false + gocritic: + disabled-tags: + - style diff --git a/cmd/admin/admin.go b/cmd/admin/admin.go index 41daf6845..e1d976c6e 100644 --- a/cmd/admin/admin.go +++ b/cmd/admin/admin.go @@ -98,19 +98,19 @@ func (a *Admin) APISD(w http.ResponseWriter, r *http.Request) { return } } - if r.Method == "PUT" { + if r.Method == http.MethodPut { a.apiPublish(w, r) - } else if r.Method == "GET" { - w.WriteHeader(200) + } else if r.Method == http.MethodGet { + w.WriteHeader(http.StatusOK) _, _ = w.Write(a.makeTargets()) } else { - w.WriteHeader(400) + w.WriteHeader(http.StatusBadRequest) } } func (a *Admin) setupLogger() { zerolog.SetGlobalLevel(zerolog.InfoLevel) - zerolog.ErrorStackMarshaler = logging.MarshalStack + zerolog.ErrorStackMarshaler = logging.MarshalStack //nolint:reassign a.logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}). With().Caller().Timestamp().Logger() @@ -128,7 +128,7 @@ func (a *Admin) apiPublish(w http.ResponseWriter, r *http.Request) { err := decoder.Decode(&publish) if err != nil { a.logger.Err(err).Msg("Unable to parse publish json") - w.WriteHeader(400) + w.WriteHeader(http.StatusBadRequest) return } a.pollerToPromAddr.Set(publish.Name, publish, a.expireAfter) diff --git a/cmd/collectors/commonutils_test.go b/cmd/collectors/commonutils_test.go index 56c63ec3a..d7d37f060 100644 --- a/cmd/collectors/commonutils_test.go +++ b/cmd/collectors/commonutils_test.go @@ -44,9 +44,7 @@ func testWithoutGroupType(t *testing.T, instance *matrix.Instance) { instance.SetLabel("group_type", "") UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "" && instance.GetLabel("protectionSourceType") == "" { - // OK - } else { + if instance.GetLabel("protectedBy") != "" || instance.GetLabel("protectionSourceType") != "" { t.Errorf("Labels protectedBy= %s, expected empty and protectionSourceType= %s, expected empty", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -57,9 +55,7 @@ func testSvmdr(t *testing.T, instance *matrix.Instance) { instance.SetLabel("source_volume", "") UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "storage_vm" && instance.GetLabel("protectionSourceType") == "storage_vm" { - // OK - } else { + if instance.GetLabel("protectedBy") != "storage_vm" || instance.GetLabel("protectionSourceType") != "storage_vm" { t.Errorf("Labels protectedBy= %s, expected: storage_vm and protectionSourceType= %s, expected: storage_vm", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -71,9 +67,7 @@ func testConstituentVolumeWithinSvmdr(t *testing.T, instance *matrix.Instance) { instance.SetLabel("destination_location", "test1") UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "storage_vm" && instance.GetLabel("protectionSourceType") == "volume" { - // OK - } else { + if instance.GetLabel("protectedBy") != "storage_vm" || instance.GetLabel("protectionSourceType") != "volume" { t.Errorf("Labels protectedBy= %s, expected: storage_vm and protectionSourceType= %s, expected: volume", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -83,9 +77,7 @@ func testCg(t *testing.T, instance *matrix.Instance) { instance.SetLabel("destination_location", "test123:/cg/") UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "cg" && instance.GetLabel("protectionSourceType") == "cg" { - // OK - } else { + if instance.GetLabel("protectedBy") != "cg" || instance.GetLabel("protectionSourceType") != "cg" { t.Errorf("Labels protectedBy= %s, expected: cg and protectionSourceType= %s, expected: cg", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -95,9 +87,7 @@ func testConstituentVolumeWithinCg(t *testing.T, instance *matrix.Instance) { instance.SetLabel("destination_location", "test123") UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "cg" && instance.GetLabel("protectionSourceType") == "volume" { - // OK - } else { + if instance.GetLabel("protectedBy") != "cg" || instance.GetLabel("protectionSourceType") != "volume" { t.Errorf("Labels protectedBy= %s, expected: cg and protectionSourceType= %s, expected: volume", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -106,9 +96,7 @@ func testNegativeCase1(t *testing.T, instance *matrix.Instance) { instance.SetLabel("group_type", "infinitevol") UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "volume" && instance.GetLabel("protectionSourceType") == "not_mapped" { - // OK - } else { + if instance.GetLabel("protectedBy") != "volume" || instance.GetLabel("protectionSourceType") != "not_mapped" { t.Errorf("Labels protectedBy= %s, expected: volume and protectionSourceType= %s, expected: not_mapped", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -120,9 +108,7 @@ func testNegativeCase2(t *testing.T, instance *matrix.Instance) { instance.SetLabel("destination_location", "test123:") UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "volume" && instance.GetLabel("protectionSourceType") == "not_mapped" { - // OK - } else { + if instance.GetLabel("protectedBy") != "volume" || instance.GetLabel("protectionSourceType") != "not_mapped" { t.Errorf("Labels protectedBy= %s, expected: volume and protectionSourceType= %s, expected: not_mapped", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -131,9 +117,7 @@ func testGroupTypeNone(t *testing.T, instance *matrix.Instance) { instance.SetLabel("group_type", "none") UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "volume" && instance.GetLabel("protectionSourceType") == "volume" { - // OK - } else { + if instance.GetLabel("protectedBy") != "volume" || instance.GetLabel("protectionSourceType") != "volume" { t.Errorf("Labels protectedBy= %s, expected: volume and protectionSourceType= %s, expected: volume", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -142,9 +126,7 @@ func testGroupTypeFlexgroup(t *testing.T, instance *matrix.Instance) { instance.SetLabel("group_type", "flexgroup") UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "volume" && instance.GetLabel("protectionSourceType") == "volume" { - // OK - } else { + if instance.GetLabel("protectedBy") != "volume" || instance.GetLabel("protectionSourceType") != "volume" { t.Errorf("Labels protectedBy= %s, expected: volume and protectionSourceType= %s, expected: volume", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -155,9 +137,7 @@ func testStrictSyncMirror(t *testing.T, instance *matrix.Instance) { instance.SetLabel("policy_type", "strict_sync_mirror") UpdateProtectedFields(instance) - if instance.GetLabel("derived_relationship_type") == "sync_mirror_strict" { - // OK - } else { + if instance.GetLabel("derived_relationship_type") != "sync_mirror_strict" { t.Errorf("Labels derived_relationship_type= %s, expected: sync_mirror_strict", instance.GetLabel("derived_relationship_type")) } } @@ -167,9 +147,7 @@ func testSyncMirror(t *testing.T, instance *matrix.Instance) { instance.SetLabel("policy_type", "sync_mirror") UpdateProtectedFields(instance) - if instance.GetLabel("derived_relationship_type") == "sync_mirror" { - // OK - } else { + if instance.GetLabel("derived_relationship_type") != "sync_mirror" { t.Errorf("Labels derived_relationship_type= %s, expected: sync_mirror", instance.GetLabel("derived_relationship_type")) } } @@ -179,9 +157,7 @@ func testMirrorVault(t *testing.T, instance *matrix.Instance) { instance.SetLabel("policy_type", "mirror_vault") UpdateProtectedFields(instance) - if instance.GetLabel("derived_relationship_type") == "mirror_vault" { - // OK - } else { + if instance.GetLabel("derived_relationship_type") != "mirror_vault" { t.Errorf("Labels derived_relationship_type= %s, expected: mirror_vault", instance.GetLabel("derived_relationship_type")) } } @@ -191,9 +167,7 @@ func testAutomatedFailover(t *testing.T, instance *matrix.Instance) { instance.SetLabel("policy_type", "automated_failover") UpdateProtectedFields(instance) - if instance.GetLabel("derived_relationship_type") == "sync_mirror" { - // OK - } else { + if instance.GetLabel("derived_relationship_type") != "sync_mirror" { t.Errorf("Labels derived_relationship_type= %s, expected: sync_mirror", instance.GetLabel("derived_relationship_type")) } } @@ -203,9 +177,7 @@ func testOtherPolicyType(t *testing.T, instance *matrix.Instance) { instance.SetLabel("policy_type", "vault") UpdateProtectedFields(instance) - if instance.GetLabel("derived_relationship_type") == "vault" { - // OK - } else { + if instance.GetLabel("derived_relationship_type") != "vault" { t.Errorf("Labels derived_relationship_type= %s, expected: vault", instance.GetLabel("derived_relationship_type")) } } @@ -215,9 +187,7 @@ func testWithNoPolicyType(t *testing.T, instance *matrix.Instance) { instance.SetLabel("policy_type", "") UpdateProtectedFields(instance) - if instance.GetLabel("derived_relationship_type") == "extended_data_protection" { - // OK - } else { + if instance.GetLabel("derived_relationship_type") != "extended_data_protection" { t.Errorf("Labels derived_relationship_type= %s, expected: extended_data_protection", instance.GetLabel("derived_relationship_type")) } } @@ -227,9 +197,7 @@ func testWithNoPolicyTypeNoRelationshipType(t *testing.T, instance *matrix.Insta instance.SetLabel("policy_type", "") UpdateProtectedFields(instance) - if instance.GetLabel("derived_relationship_type") == "" { - // OK - } else { + if instance.GetLabel("derived_relationship_type") != "" { t.Errorf("Labels derived_relationship_type= %s, expected: \"\"(empty)", instance.GetLabel("derived_relationship_type")) } } @@ -240,9 +208,7 @@ func testOlderTimestampThanDuration(t *testing.T) { duration := 5 * time.Minute isOlder := IsTimestampOlderThanDuration(time.Now(), timestamp, duration) - if isOlder { - // OK - } else { + if !isOlder { t.Errorf("timestamp= %f is older than duration %s", timestamp, duration.String()) } } @@ -252,9 +218,7 @@ func testNewerTimestampThanDuration(t *testing.T) { duration := 2 * time.Hour isOlder := IsTimestampOlderThanDuration(time.Now(), timestamp, duration) - if !isOlder { - // OK - } else { + if isOlder { t.Errorf("timestamp= %f is newer than duration %s", timestamp, duration.String()) } } diff --git a/cmd/collectors/ems/ems.go b/cmd/collectors/ems/ems.go index 530558e19..5468336bc 100644 --- a/cmd/collectors/ems/ems.go +++ b/cmd/collectors/ems/ems.go @@ -114,11 +114,7 @@ func (e *Ems) Init(a *collector.AbstractCollector) error { return err } - if err = e.InitMatrix(); err != nil { - return err - } - - return nil + return e.InitMatrix() } func (e *Ems) InitMatrix() error { @@ -359,7 +355,8 @@ func (e *Ems) PollData() (map[string]*matrix.Matrix, error) { } toTime := clusterTime.Unix() timeFilter := e.getTimeStampFilter(clusterTime) - filter := append(e.Filter, timeFilter) + filter := e.Filter + filter = append(filter, timeFilter) // build hrefs up to maxURLSize var hrefs []string @@ -534,13 +531,13 @@ func (e *Ems) HandleResults(result []gjson.Result, prop map[string][]*emsProp) ( } // Check matches at all same name ems - isMatch := false + var isMatch bool // Check matches at each ems - isMatchPs := false + var isMatchPs bool // Check instance count at all same name ems instanceLabelCount := uint64(0) // Check instance count at each ems - instanceLabelCountPs := uint64(0) + var instanceLabelCountPs uint64 // parse ems properties for the instance if ps, ok := prop[msgName]; ok { @@ -609,7 +606,6 @@ func (e *Ems) HandleResults(result []gjson.Result, prop map[string][]*emsProp) ( } } if !isMatchPs { - instanceLabelCountPs = 0 continue } diff --git a/cmd/collectors/ems/ems_test.go b/cmd/collectors/ems/ems_test.go index d75f65b0e..b66836850 100644 --- a/cmd/collectors/ems/ems_test.go +++ b/cmd/collectors/ems/ems_test.go @@ -57,7 +57,7 @@ func NewEms() *Ems { ac := collector.New("Ems", "Ems", &opts, emsParams(emsConfgPath), nil) e := &Ems{} if err := e.Init(ac); err != nil { - log.Fatal().Err(err) + log.Fatal().Err(err).Send() } // Changed the resolve_after for 2 issuing ems for auto resolve testing e.resolveAfter["LUN.offline"] = 1 * time.Second @@ -102,9 +102,7 @@ func (e *Ems) testBookendIssuingEms(t *testing.T, path string) { } // Test for matches - filter if generatedEmsName == "hm.alert.raised" { - if instance.GetLabel("alert_id") == "RaidLeftBehindAggrAlert" { - // OK - } else { + if instance.GetLabel("alert_id") != "RaidLeftBehindAggrAlert" { t.Errorf("Labels alert_id= %s, expected: RaidLeftBehindAggrAlert", instance.GetLabel("alert_id")) } } @@ -142,9 +140,7 @@ func (e *Ems) testBookendResolvingEms(t *testing.T, path string) { } // Test for matches - filter if generatedEmsName == "hm.alert.raised" { - if instance.GetLabel("alert_id") == "RaidLeftBehindAggrAlert" && ok && val == 0.0 { - // OK - } else { + if instance.GetLabel("alert_id") != "RaidLeftBehindAggrAlert" || !ok || val != 0.0 { t.Errorf("Labels alert_id= %s, expected: RaidLeftBehindAggrAlert, metric value = %f, expected: 0.0", instance.GetLabel("alert_id"), val) } } diff --git a/cmd/collectors/rest/plugins/disk/disk.go b/cmd/collectors/rest/plugins/disk/disk.go index 9a585b4c3..c2a94d63e 100644 --- a/cmd/collectors/rest/plugins/disk/disk.go +++ b/cmd/collectors/rest/plugins/disk/disk.go @@ -1,6 +1,5 @@ -/* - * Copyright NetApp Inc, 2021 All rights reserved - */ +// Copyright NetApp Inc, 2021 All rights reserved + package disk import ( diff --git a/cmd/collectors/rest/plugins/health/health.go b/cmd/collectors/rest/plugins/health/health.go index b2a3a2575..01eb40153 100644 --- a/cmd/collectors/rest/plugins/health/health.go +++ b/cmd/collectors/rest/plugins/health/health.go @@ -66,11 +66,7 @@ func (h *Health) Init() error { return err } - if err = h.client.Init(5); err != nil { - return err - } - - return nil + return h.client.Init(5) } func (h *Health) initAllMatrix() error { @@ -457,8 +453,7 @@ func (h *Health) collectSupportAlerts() { } toTime := clusterTime.Unix() timeFilter := h.getTimeStampFilter(clusterTime) - addFilter := []string{"suppress=false"} - filter := append(addFilter, timeFilter) + filter := append([]string{"suppress=false"}, timeFilter) records, err := h.getSupportAlerts(filter) if err != nil { diff --git a/cmd/collectors/rest/plugins/sensor/sensor.go b/cmd/collectors/rest/plugins/sensor/sensor.go index f78085eaa..fa0b241e5 100644 --- a/cmd/collectors/rest/plugins/sensor/sensor.go +++ b/cmd/collectors/rest/plugins/sensor/sensor.go @@ -245,31 +245,27 @@ func (my *Sensor) calculateEnvironmentMetrics(data *matrix.Matrix) ([]*matrix.Ma currentKey := currentKeys[i] voltageKey := voltageKeys[i] - //get values + // get values currentSensorValue := v.currentSensor[currentKey] voltageSensorValue := v.voltageSensor[voltageKey] // convert units if currentSensorValue.unit == "mA" { currentSensorValue.value = currentSensorValue.value / 1000 - } else if currentSensorValue.unit == "A" { - // do nothing - } else { + } else if currentSensorValue.unit != "A" { my.Logger.Warn().Str("unit", currentSensorValue.unit).Float64("value", currentSensorValue.value).Msg("unknown current unit") } if voltageSensorValue.unit == "mV" { voltageSensorValue.value = voltageSensorValue.value / 1000 - } else if voltageSensorValue.unit == "V" { - // do nothing - } else { + } else if voltageSensorValue.unit != "V" { my.Logger.Warn().Str("unit", voltageSensorValue.unit).Float64("value", voltageSensorValue.value).Msg("unknown voltage unit") } p := currentSensorValue.value * voltageSensorValue.value if !strings.EqualFold(voltageSensorValue.name, "in") && !strings.EqualFold(currentSensorValue.name, "in") { - p = p / 0.93 //If the sensor names to do NOT contain "IN" or "in", then we need to adjust the power to account for loss in the power supply. We will use 0.93 as the power supply efficiency factor for all systems. + p = p / 0.93 // If the sensor names to do NOT contain "IN" or "in", then we need to adjust the power to account for loss in the power supply. We will use 0.93 as the power supply efficiency factor for all systems. } sumPower += p diff --git a/cmd/collectors/rest/plugins/svm/svm.go b/cmd/collectors/rest/plugins/svm/svm.go index ef272348d..016d704a7 100644 --- a/cmd/collectors/rest/plugins/svm/svm.go +++ b/cmd/collectors/rest/plugins/svm/svm.go @@ -23,20 +23,20 @@ var weakCiphers = regexp.MustCompile("(.*)_cbc.*") type SVM struct { *plugin.AbstractPlugin - nsswitchInfo map[string]nsswitch + nsswitchInfo map[string]Nsswitch kerberosInfo map[string]string - fpolicyInfo map[string]fpolicy + fpolicyInfo map[string]Fpolicy iscsiServiceInfo map[string]string iscsiCredentialInfo map[string]string client *rest.Client } -type nsswitch struct { +type Nsswitch struct { nsdb []string nssource []string } -type fpolicy struct { +type Fpolicy struct { name string enable string } @@ -62,9 +62,9 @@ func (my *SVM) Init() error { if err = my.client.Init(5); err != nil { return err } - my.nsswitchInfo = make(map[string]nsswitch) + my.nsswitchInfo = make(map[string]Nsswitch) my.kerberosInfo = make(map[string]string) - my.fpolicyInfo = make(map[string]fpolicy) + my.fpolicyInfo = make(map[string]Fpolicy) my.iscsiServiceInfo = make(map[string]string) my.iscsiCredentialInfo = make(map[string]string) @@ -167,15 +167,15 @@ func (my *SVM) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) return nil, nil } -func (my *SVM) GetNSSwitchInfo(data *matrix.Matrix) (map[string]nsswitch, error) { +func (my *SVM) GetNSSwitchInfo(data *matrix.Matrix) (map[string]Nsswitch, error) { var ( - vserverNsswitchMap map[string]nsswitch - ns nsswitch + vserverNsswitchMap map[string]Nsswitch + ns Nsswitch ok bool ) - vserverNsswitchMap = make(map[string]nsswitch) + vserverNsswitchMap = make(map[string]Nsswitch) for _, svmInstance := range data.GetInstances() { svmName := svmInstance.GetLabel("svm") @@ -191,7 +191,7 @@ func (my *SVM) GetNSSwitchInfo(data *matrix.Matrix) (map[string]nsswitch, error) ns.nsdb = append(ns.nsdb, nsdb) ns.nssource = append(ns.nssource, nssourcelist...) } else { - ns = nsswitch{nsdb: []string{nsdb}, nssource: nssourcelist} + ns = Nsswitch{nsdb: []string{nsdb}, nssource: nssourcelist} } vserverNsswitchMap[svmName] = ns } @@ -231,14 +231,14 @@ func (my *SVM) GetKerberosConfig() (map[string]string, error) { return svmKerberosMap, nil } -func (my *SVM) GetFpolicy() (map[string]fpolicy, error) { +func (my *SVM) GetFpolicy() (map[string]Fpolicy, error) { var ( result []gjson.Result - svmFpolicyMap map[string]fpolicy + svmFpolicyMap map[string]Fpolicy err error ) - svmFpolicyMap = make(map[string]fpolicy) + svmFpolicyMap = make(map[string]Fpolicy) query := "api/protocols/fpolicy" fpolicyFields := []string{"svm.name", "policies.enabled", "policies.name"} href := rest.BuildHref("", strings.Join(fpolicyFields, ","), nil, "", "", "", "", query) @@ -252,11 +252,11 @@ func (my *SVM) GetFpolicy() (map[string]fpolicy, error) { fpolicyName := fpolicyData.Get("policies.name").String() svmName := fpolicyData.Get("svm.name").String() if _, ok := svmFpolicyMap[svmName]; !ok { - svmFpolicyMap[svmName] = fpolicy{name: fpolicyName, enable: fpolicyEnable} + svmFpolicyMap[svmName] = Fpolicy{name: fpolicyName, enable: fpolicyEnable} } else { // If svm is already present, update the status value only if it is false if svmFpolicyMap[svmName].enable == "false" { - svmFpolicyMap[svmName] = fpolicy{name: fpolicyName, enable: fpolicyEnable} + svmFpolicyMap[svmName] = Fpolicy{name: fpolicyName, enable: fpolicyEnable} } } } diff --git a/cmd/collectors/rest/plugins/volume/volume.go b/cmd/collectors/rest/plugins/volume/volume.go index e03c68b1b..966dc6df1 100644 --- a/cmd/collectors/rest/plugins/volume/volume.go +++ b/cmd/collectors/rest/plugins/volume/volume.go @@ -138,29 +138,29 @@ func (my *Volume) handleARWProtection(data *matrix.Matrix) { // If all volumes are in learning mode --> "Learning Mode" // Else indicates arwStatus for all volumes are enabled --> "Active Mode" for _, volume := range data.GetInstances() { - if arwState := volume.GetLabel("antiRansomwareState"); arwState == "" { - // Case where REST call don't return `antiRansomwareState` field, arwStatus show as 'Not Monitoring' + arwState := volume.GetLabel("antiRansomwareState") + if arwState == "" { + // Case where REST calls don't return `antiRansomwareState` field, arwStatus show as 'Not Monitoring' arwStatusValue = "Not Monitoring" break - } else { - if arwState == "disabled" { - arwStatusValue = "Not Monitoring" - break - } else if arwState == "dry_run" || arwState == "enable_paused" { - arwStartTime := volume.GetLabel("anti_ransomware_start_time") - if arwStartTime == "" || arwStatusValue == "Switch to Active Mode" { - continue - } - // If ARW startTime is more than 30 days old, which indicates that learning mode has been finished. - if arwStartTimeValue, err = time.Parse(time.RFC3339, arwStartTime); err != nil { - my.Logger.Error().Err(err).Msg("Failed to parse arw start time") - arwStartTimeValue = time.Now() - } - if time.Since(arwStartTimeValue).Hours() > HoursInMonth { - arwStatusValue = "Switch to Active Mode" - } else { - arwStatusValue = "Learning Mode" - } + } + if arwState == "disabled" { + arwStatusValue = "Not Monitoring" + break + } else if arwState == "dry_run" || arwState == "enable_paused" { + arwStartTime := volume.GetLabel("anti_ransomware_start_time") + if arwStartTime == "" || arwStatusValue == "Switch to Active Mode" { + continue + } + // If ARW startTime is more than 30 days old, which indicates that learning mode has been finished. + if arwStartTimeValue, err = time.Parse(time.RFC3339, arwStartTime); err != nil { + my.Logger.Error().Err(err).Msg("Failed to parse arw start time") + arwStartTimeValue = time.Now() + } + if time.Since(arwStartTimeValue).Hours() > HoursInMonth { + arwStatusValue = "Switch to Active Mode" + } else { + arwStatusValue = "Learning Mode" } } } diff --git a/cmd/collectors/rest/plugins/volumeanalytics/volumeanalytics.go b/cmd/collectors/rest/plugins/volumeanalytics/volumeanalytics.go index c23549f08..1b24d7312 100644 --- a/cmd/collectors/rest/plugins/volumeanalytics/volumeanalytics.go +++ b/cmd/collectors/rest/plugins/volumeanalytics/volumeanalytics.go @@ -176,7 +176,7 @@ func (v *VolumeAnalytics) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr } } } - if len(mtBytesUsedValues) == len(mtBytesUsedPercentages) && len(mtBytesUsedValues) == len(mtBytesUsedLabels) { + if len(mtBytesUsedValues) == len(mtBytesUsedPercentages) && len(mtBytesUsedValues) == len(mtBytesUsedLabels) { //nolint:gocritic for i, mv := range mtBytesUsedValues { key := "modified_value_" + mtBytesUsedLabels[i] @@ -212,7 +212,7 @@ func (v *VolumeAnalytics) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr } } - if len(atBytesUsedValues) == len(atBytesUsedPercentages) && len(atBytesUsedValues) == len(atBytesUsedLabels) { + if len(atBytesUsedValues) == len(atBytesUsedPercentages) && len(atBytesUsedValues) == len(atBytesUsedLabels) { //nolint:gocritic for i, av := range atBytesUsedValues { key := "access_value_" + atBytesUsedLabels[i] diff --git a/cmd/collectors/rest/rest.go b/cmd/collectors/rest/rest.go index 2314e908a..29cd05947 100644 --- a/cmd/collectors/rest/rest.go +++ b/cmd/collectors/rest/rest.go @@ -629,13 +629,16 @@ func (r *Rest) CollectAutoSupport(p *collector.Payload) { } if r.Object == "Node" || r.Name == "ems" { - nodeIds, err := r.getNodeUuids() + var ( + nodeIds []collector.ID + err error + ) + nodeIds, err = r.getNodeUuids() if err != nil { // log but don't return so the other info below is collected r.Logger.Error(). Err(err). Msg("Unable to get nodes.") - nodeIds = make([]collector.ID, 0) } info.Ids = nodeIds p.Nodes = &info diff --git a/cmd/collectors/restperf/plugins/disk/disk.go b/cmd/collectors/restperf/plugins/disk/disk.go index 03855a07d..3f1e49b63 100644 --- a/cmd/collectors/restperf/plugins/disk/disk.go +++ b/cmd/collectors/restperf/plugins/disk/disk.go @@ -125,7 +125,7 @@ func (d *Disk) Init() error { shelfMetric["frus => psu"] = []string{ "^^id => psu_id", "^installed => enabled", - //"^location", + // "^location", "^part_number", "^serial_number => serial", "^psu.model => type", @@ -456,7 +456,7 @@ func (d *Disk) calculateAggrPower(data *matrix.Matrix, output []*matrix.Matrix) for _, v1 := range v.disks { c := len(v1.aggregates) if c > 0 { - diskWithAggregateCount += 1 + diskWithAggregateCount++ } } if diskWithAggregateCount != 0 { @@ -731,14 +731,14 @@ func (d *Disk) initAggrPowerMatrix() { } func (d *Disk) initMaps() { - //reset shelf Power + // reset shelf Power d.ShelfMap = make(map[string]*shelf) - //reset diskmap + // reset diskmap d.diskMap = make(map[string]*disk) d.diskNameMap = make(map[string]*disk) - //reset aggrmap + // reset aggrmap d.aggrMap = make(map[string]*aggregate) } diff --git a/cmd/collectors/restperf/plugins/fcvi/fcvi.go b/cmd/collectors/restperf/plugins/fcvi/fcvi.go index 5274d211c..6dbba32ad 100644 --- a/cmd/collectors/restperf/plugins/fcvi/fcvi.go +++ b/cmd/collectors/restperf/plugins/fcvi/fcvi.go @@ -30,10 +30,7 @@ func (f *FCVI) Init() error { return err } - if err = f.client.Init(5); err != nil { - return err - } - return nil + return f.client.Init(5) } func (f *FCVI) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { diff --git a/cmd/collectors/restperf/plugins/volumetag/volumetag.go b/cmd/collectors/restperf/plugins/volumetag/volumetag.go index e91b7f58a..77628ea86 100644 --- a/cmd/collectors/restperf/plugins/volumetag/volumetag.go +++ b/cmd/collectors/restperf/plugins/volumetag/volumetag.go @@ -29,10 +29,7 @@ func (v *VolumeTag) Init() error { return err } - if err = v.client.Init(5); err != nil { - return err - } - return nil + return v.client.Init(5) } func (v *VolumeTag) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { diff --git a/cmd/collectors/restperf/restperf.go b/cmd/collectors/restperf/restperf.go index 121140c90..dbf65c1a8 100644 --- a/cmd/collectors/restperf/restperf.go +++ b/cmd/collectors/restperf/restperf.go @@ -137,20 +137,20 @@ func (r *RestPerf) Init(a *collector.AbstractCollector) error { func (r *RestPerf) InitQOSLabels() error { if isWorkloadObject(r.Prop.Query) || isWorkloadDetailObject(r.Prop.Query) { - if qosLabels := r.Params.GetChildS("qos_labels"); qosLabels == nil { + qosLabels := r.Params.GetChildS("qos_labels") + if qosLabels == nil { return errs.New(errs.ErrMissingParam, "qos_labels") - } else { - r.perfProp.qosLabels = make(map[string]string) - for _, label := range qosLabels.GetAllChildContentS() { - - display := strings.ReplaceAll(label, "-", "_") - before, after, found := strings.Cut(label, "=>") - if found { - label = strings.TrimSpace(before) - display = strings.TrimSpace(after) - } - r.perfProp.qosLabels[label] = display + } + r.perfProp.qosLabels = make(map[string]string) + for _, label := range qosLabels.GetAllChildContentS() { + + display := strings.ReplaceAll(label, "-", "_") + before, after, found := strings.Cut(label, "=>") + if found { + label = strings.TrimSpace(before) + display = strings.TrimSpace(after) } + r.perfProp.qosLabels[label] = display } } return nil @@ -158,7 +158,7 @@ func (r *RestPerf) InitQOSLabels() error { func (r *RestPerf) InitMatrix() error { mat := r.Matrix[r.Object] - //init perf properties + // init perf properties r.perfProp.latencyIoReqd = r.loadParamInt("latency_io_reqd", latencyIoReqd) r.perfProp.isCacheEmpty = true // overwrite from abstract collector @@ -433,7 +433,7 @@ func parseMetricResponses(instanceData gjson.Result, metric map[string]*rest2.Me ms := strings.Split(m, ",") for range ms { finalLabels = append(finalLabels, label+arrayKeyToken+subLabelSlice[vLen]) - vLen += 1 + vLen++ } if vLen > len(subLabelSlice) { return false @@ -502,7 +502,7 @@ func parseMetricResponse(instanceData gjson.Result, metric string) *metricRespon ms := strings.Split(m, ",") for range ms { finalLabels = append(finalLabels, label+arrayKeyToken+subLabelSlice[vLen]) - vLen += 1 + vLen++ } if vLen > len(subLabelSlice) { break @@ -580,32 +580,32 @@ func (r *RestPerf) processWorkLoadCounter() (map[string]*matrix.Matrix, error) { wait.SetExportable(false) visits.SetExportable(false) - if resourceMap := r.Params.GetChildS("resource_map"); resourceMap == nil { + resourceMap := r.Params.GetChildS("resource_map") + if resourceMap == nil { return nil, errs.New(errs.ErrMissingParam, "resource_map") - } else { - for _, x := range resourceMap.GetChildren() { - for _, wm := range workloadDetailMetrics { - name := x.GetNameS() + wm - resource := x.GetContentS() - - if m := mat.GetMetric(name); m != nil { - continue - } - if m, err := mat.NewMetricFloat64(name, wm); err != nil { - return nil, err - } else { - r.perfProp.counterInfo[name] = &counter{ - name: wm, - description: "", - counterType: r.perfProp.counterInfo[service.GetName()].counterType, - unit: r.perfProp.counterInfo[service.GetName()].unit, - denominator: "ops", - } - m.SetLabel("resource", resource) + } + for _, x := range resourceMap.GetChildren() { + for _, wm := range workloadDetailMetrics { + name := x.GetNameS() + wm + resource := x.GetContentS() - r.Logger.Debug().Str("name", name).Str("resource", resource).Msg("added workload latency metric") - } + if m := mat.GetMetric(name); m != nil { + continue } + m, err := mat.NewMetricFloat64(name, wm) + if err != nil { + return nil, err + } + r.perfProp.counterInfo[name] = &counter{ + name: wm, + description: "", + counterType: r.perfProp.counterInfo[service.GetName()].counterType, + unit: r.perfProp.counterInfo[service.GetName()].unit, + denominator: "ops", + } + m.SetLabel("resource", resource) + + r.Logger.Debug().Str("name", name).Str("resource", resource).Msg("added workload latency metric") } } } @@ -864,123 +864,121 @@ func (r *RestPerf) pollData(startTime time.Time, perfRecords []rest.PerfRecord) } } continue - } else { - if f.isArray { - labels := strings.Split(f.label, ",") - values := strings.Split(f.value, ",") - - if len(labels) != len(values) { - // warn & skip - r.Logger.Warn(). - Str("labels", f.label). - Str("value", f.value). - Msg("labels don't match parsed values") - continue - } - - // ONTAP does not have a `type` for histogram. Harvest tests the `desc` field to determine - // if a counter is a histogram - isHistogram = false - description := strings.ToLower(r.perfProp.counterInfo[name].description) - if len(labels) > 0 && strings.Contains(description, "histogram") { - key := name + ".bucket" - histogramMetric = curMat.GetMetric(key) - if histogramMetric != nil { - r.Logger.Trace().Str("metric", key).Msg("Updating array metric attributes") - } else { - histogramMetric, err = curMat.NewMetricFloat64(key, metric.Label) - if err != nil { - r.Logger.Error().Err(err).Str("key", key).Msg("unable to create histogram metric") - continue - } - } - histogramMetric.SetArray(true) - histogramMetric.SetExportable(metric.Exportable) - histogramMetric.SetBuckets(&labels) - isHistogram = true - } + } + if f.isArray { + labels := strings.Split(f.label, ",") + values := strings.Split(f.value, ",") + + if len(labels) != len(values) { + // warn & skip + r.Logger.Warn(). + Str("labels", f.label). + Str("value", f.value). + Msg("labels don't match parsed values") + continue + } - for i, label := range labels { - k := name + arrayKeyToken + label - metr, ok := curMat.GetMetrics()[k] - if !ok { - if metr, err = curMat.NewMetricFloat64(k, metric.Label); err != nil { - r.Logger.Error().Err(err). - Str("name", k). - Msg("NewMetricFloat64") - continue - } - if x := strings.Split(label, arrayKeyToken); len(x) == 2 { - metr.SetLabel("metric", x[0]) - metr.SetLabel("submetric", x[1]) - } else { - metr.SetLabel("metric", label) - } - // differentiate between array and normal counter - metr.SetArray(true) - metr.SetExportable(metric.Exportable) - if isHistogram { - // Save the index of this label so the labels can be exported in order - metr.SetLabel("comment", strconv.Itoa(i)) - // Save the bucket name so the flattened metrics can find their bucket when exported - metr.SetLabel("bucket", name+".bucket") - metr.SetHistogram(true) - } - } - if err = metr.SetValueString(instance, values[i]); err != nil { - r.Logger.Error(). - Err(err). - Str("name", name). - Str("label", label). - Str("value", values[i]). - Int("instIndex", instIndex). - Msg("Set value failed") + // ONTAP does not have a `type` for histogram. Harvest tests the `desc` field to determine + // if a counter is a histogram + isHistogram = false + description := strings.ToLower(r.perfProp.counterInfo[name].description) + if len(labels) > 0 && strings.Contains(description, "histogram") { + key := name + ".bucket" + histogramMetric = curMat.GetMetric(key) + if histogramMetric != nil { + r.Logger.Trace().Str("metric", key).Msg("Updating array metric attributes") + } else { + histogramMetric, err = curMat.NewMetricFloat64(key, metric.Label) + if err != nil { + r.Logger.Error().Err(err).Str("key", key).Msg("unable to create histogram metric") continue - } else { - r.Logger.Trace(). - Str("name", name). - Str("label", label). - Str("value", values[i]). - Int("instIndex", instIndex). - Msg("Set name.label = value") - count++ } } - } else { - metr, ok := curMat.GetMetrics()[name] + histogramMetric.SetArray(true) + histogramMetric.SetExportable(metric.Exportable) + histogramMetric.SetBuckets(&labels) + isHistogram = true + } + + for i, label := range labels { + k := name + arrayKeyToken + label + metr, ok := curMat.GetMetrics()[k] if !ok { - if metr, err = curMat.NewMetricFloat64(name, metric.Label); err != nil { + if metr, err = curMat.NewMetricFloat64(k, metric.Label); err != nil { r.Logger.Error().Err(err). - Str("name", name). - Int("instIndex", instIndex). + Str("name", k). Msg("NewMetricFloat64") + continue } - } - metr.SetExportable(metric.Exportable) - if c, err := strconv.ParseFloat(f.value, 64); err == nil { - if err = metr.SetValueFloat64(instance, c); err != nil { - r.Logger.Error().Err(err). - Str("key", metric.Name). - Str("metric", metric.Label). - Int("instIndex", instIndex). - Msg("Unable to set float key on metric") + if x := strings.Split(label, arrayKeyToken); len(x) == 2 { + metr.SetLabel("metric", x[0]) + metr.SetLabel("submetric", x[1]) } else { - r.Logger.Trace(). - Int("instIndex", instIndex). - Str("key", instanceKey). - Str("counter", name). - Str("value", f.value). - Msg("Set metric") + metr.SetLabel("metric", label) } - } else { + // differentiate between array and normal counter + metr.SetArray(true) + metr.SetExportable(metric.Exportable) + if isHistogram { + // Save the index of this label so the labels can be exported in order + metr.SetLabel("comment", strconv.Itoa(i)) + // Save the bucket name so the flattened metrics can find their bucket when exported + metr.SetLabel("bucket", name+".bucket") + metr.SetHistogram(true) + } + } + if err = metr.SetValueString(instance, values[i]); err != nil { + r.Logger.Error(). + Err(err). + Str("name", name). + Str("label", label). + Str("value", values[i]). + Int("instIndex", instIndex). + Msg("Set value failed") + continue + } + r.Logger.Trace(). + Str("name", name). + Str("label", label). + Str("value", values[i]). + Int("instIndex", instIndex). + Msg("Set name.label = value") + count++ + } + } else { + metr, ok := curMat.GetMetrics()[name] + if !ok { + if metr, err = curMat.NewMetricFloat64(name, metric.Label); err != nil { + r.Logger.Error().Err(err). + Str("name", name). + Int("instIndex", instIndex). + Msg("NewMetricFloat64") + } + } + metr.SetExportable(metric.Exportable) + if c, err := strconv.ParseFloat(f.value, 64); err == nil { + if err = metr.SetValueFloat64(instance, c); err != nil { r.Logger.Error().Err(err). Str("key", metric.Name). Str("metric", metric.Label). Int("instIndex", instIndex). - Msg("Unable to parse float value") + Msg("Unable to set float key on metric") + } else { + r.Logger.Trace(). + Int("instIndex", instIndex). + Str("key", instanceKey). + Str("counter", name). + Str("value", f.value). + Msg("Set metric") } - count++ + } else { + r.Logger.Error().Err(err). + Str("key", metric.Name). + Str("metric", metric.Label). + Int("instIndex", instIndex). + Msg("Unable to parse float value") } + count++ } } else { r.Logger.Warn().Str("counter", name).Msg("Counter is missing or unable to parse.") @@ -1049,8 +1047,10 @@ func (r *RestPerf) pollData(startTime time.Time, perfRecords []rest.PerfRecord) } // order metrics, such that those requiring base counters are processed last - orderedMetrics := append(orderedNonDenominatorMetrics, orderedDenominatorMetrics...) - orderedKeys := append(orderedNonDenominatorKeys, orderedDenominatorKeys...) + orderedMetrics := orderedNonDenominatorMetrics + orderedMetrics = append(orderedMetrics, orderedDenominatorMetrics...) + orderedKeys := orderedNonDenominatorKeys + orderedKeys = append(orderedKeys, orderedDenominatorKeys...) // Calculate timestamp delta first since many counters require it for postprocessing. // Timestamp has "raw" property, so it isn't post-processed automatically diff --git a/cmd/collectors/storagegrid/rest/client.go b/cmd/collectors/storagegrid/rest/client.go index 00a495ba8..d8feb4d08 100644 --- a/cmd/collectors/storagegrid/rest/client.go +++ b/cmd/collectors/storagegrid/rest/client.go @@ -241,7 +241,7 @@ func (c *Client) fetch() ([]byte, error) { //goland:noinspection GoUnhandledErrorResult defer response.Body.Close() - if response.StatusCode != 200 { + if response.StatusCode != http.StatusOK { if body, err = io.ReadAll(response.Body); err == nil { return nil, errs.NewStorageGridErr(response.StatusCode, body) } @@ -373,7 +373,7 @@ func (c *Client) fetchTokenWithAuthRetry() error { return err } - if response.StatusCode != 200 { + if response.StatusCode != http.StatusOK { return errs.NewStorageGridErr(response.StatusCode, body) } diff --git a/cmd/collectors/storagegrid/storagegrid.go b/cmd/collectors/storagegrid/storagegrid.go index f2362a533..1d20fa6da 100644 --- a/cmd/collectors/storagegrid/storagegrid.go +++ b/cmd/collectors/storagegrid/storagegrid.go @@ -136,9 +136,8 @@ func (s *StorageGrid) InitCache() error { func (s *StorageGrid) PollData() (map[string]*matrix.Matrix, error) { if s.Props.Query == "prometheus" { return s.pollPrometheusMetrics() - } else { - return s.pollRest() } + return s.pollRest() } func (s *StorageGrid) pollPrometheusMetrics() (map[string]*matrix.Matrix, error) { diff --git a/cmd/collectors/zapi/collector/parsers.go b/cmd/collectors/zapi/collector/parsers.go index 56f3d08be..b88c1eccb 100644 --- a/cmd/collectors/zapi/collector/parsers.go +++ b/cmd/collectors/zapi/collector/parsers.go @@ -14,8 +14,8 @@ import ( func ParseShortestPath(m *matrix.Matrix, l map[string]string) []string { - prefix := make([]string, 0) - keys := make([][]string, 0) + var prefix []string + var keys [][]string for key := range m.GetMetrics() { keys = append(keys, strings.Split(key, ".")) @@ -24,9 +24,9 @@ func ParseShortestPath(m *matrix.Matrix, l map[string]string) []string { keys = append(keys, strings.Split(key, ".")) } - max := util.MinLen(keys) + minLen := util.MinLen(keys) - for i := 0; i < max; i++ { + for i := 0; i < minLen; i++ { if util.AllSame(keys, i) { prefix = append(prefix, keys[0][i]) } else { @@ -93,7 +93,8 @@ func (z *Zapi) HandleCounter(path []string, content string) string { name = strings.TrimSpace(strings.TrimLeft(name, "^")) - fullPath = append(path, name) + fullPath = path + fullPath = append(fullPath, name) key = strings.Join(fullPath, ".") if display == "" { diff --git a/cmd/collectors/zapi/collector/zapi.go b/cmd/collectors/zapi/collector/zapi.go index af6a4c543..6436c01e4 100644 --- a/cmd/collectors/zapi/collector/zapi.go +++ b/cmd/collectors/zapi/collector/zapi.go @@ -275,10 +275,10 @@ func (z *Zapi) PollData() (map[string]*matrix.Matrix, error) { } fetch = func(instance *matrix.Instance, node *node.Node, path []string, isAppend bool) { - - newpath := append(path, node.GetNameS()) - key := strings.Join(newpath, ".") - z.Logger.Trace().Msgf(" > %s(%s)%s <%s%d%s> name=[%s%s%s%s] value=[%s%s%s]", color.Grey, newpath, color.End, color.Red, len(node.GetChildren()), color.End, color.Bold, color.Cyan, node.GetNameS(), color.End, color.Yellow, node.GetContentS(), color.End) + newPath := path + newPath = append(newPath, node.GetNameS()) + key := strings.Join(newPath, ".") + z.Logger.Trace().Msgf(" > %s(%s)%s <%s%d%s> name=[%s%s%s%s] value=[%s%s%s]", color.Grey, newPath, color.End, color.Red, len(node.GetChildren()), color.End, color.Bold, color.Cyan, node.GetNameS(), color.End, color.Yellow, node.GetContentS(), color.End) if value := node.GetContentS(); value != "" { if label, has := z.instanceLabelPaths[key]; has { @@ -312,9 +312,9 @@ func (z *Zapi) PollData() (map[string]*matrix.Matrix, error) { for _, child := range node.GetChildren() { if util.HasDuplicates(child.GetAllChildNamesS()) { z.Logger.Debug().Msgf("Array detected for %s", child.GetNameS()) - fetch(instance, child, newpath, true) + fetch(instance, child, newPath, true) } else { - fetch(instance, child, newpath, isAppend) + fetch(instance, child, newPath, isAppend) } } } @@ -483,13 +483,16 @@ func (z *Zapi) CollectAutoSupport(p *collector.Payload) { } if z.Object == "Node" { - nodeIds, err := z.getNodeUuids() + var ( + nodeIds []collector.ID + err error + ) + nodeIds, err = z.getNodeUuids() if err != nil { // log but don't return so the other info below is collected z.Logger.Error(). Err(err). Msg("Unable to get nodes.") - nodeIds = make([]collector.ID, 0) } info.Ids = nodeIds p.Nodes = &info diff --git a/cmd/collectors/zapi/plugins/sensor/sensor.go b/cmd/collectors/zapi/plugins/sensor/sensor.go index e715d8487..1b17dbaf3 100644 --- a/cmd/collectors/zapi/plugins/sensor/sensor.go +++ b/cmd/collectors/zapi/plugins/sensor/sensor.go @@ -249,31 +249,27 @@ func (my *Sensor) calculateEnvironmentMetrics(data *matrix.Matrix) ([]*matrix.Ma currentKey := currentKeys[i] voltageKey := voltageKeys[i] - //get values + // get values currentSensorValue := v.currentSensor[currentKey] voltageSensorValue := v.voltageSensor[voltageKey] // convert units if currentSensorValue.unit == "mA" { currentSensorValue.value = currentSensorValue.value / 1000 - } else if currentSensorValue.unit == "A" { - // do nothing - } else { + } else if currentSensorValue.unit != "A" { my.Logger.Warn().Str("node", key).Str("unit", currentSensorValue.unit).Float64("value", currentSensorValue.value).Msg("unknown current unit") } if voltageSensorValue.unit == "mV" { voltageSensorValue.value = voltageSensorValue.value / 1000 - } else if voltageSensorValue.unit == "V" { - // do nothing - } else { + } else if voltageSensorValue.unit != "V" { my.Logger.Warn().Str("node", key).Str("unit", voltageSensorValue.unit).Float64("value", voltageSensorValue.value).Msg("unknown voltage unit") } p := currentSensorValue.value * voltageSensorValue.value if !strings.EqualFold(voltageSensorValue.name, "in") && !strings.EqualFold(currentSensorValue.name, "in") { - p = p / 0.93 //If the sensor names to do NOT contain "IN" or "in", then we need to adjust the power to account for loss in the power supply. We will use 0.93 as the power supply efficiency factor for all systems. + p = p / 0.93 // If the sensor names to do NOT contain "IN" or "in", then we need to adjust the power to account for loss in the power supply. We will use 0.93 as the power supply efficiency factor for all systems. } sumPower += p diff --git a/cmd/collectors/zapi/plugins/sensor/sensor_test.go b/cmd/collectors/zapi/plugins/sensor/sensor_test.go index a4f7a3029..576585f9f 100644 --- a/cmd/collectors/zapi/plugins/sensor/sensor_test.go +++ b/cmd/collectors/zapi/plugins/sensor/sensor_test.go @@ -33,9 +33,9 @@ func init() { } fetch = func(instance *matrix.Instance, node *node.Node, path []string) { - - newpath := append(path, node.GetNameS()) - key := strings.Join(newpath, ".") + newPath := path + newPath = append(newPath, node.GetNameS()) + key := strings.Join(newPath, ".") if value := node.GetContentS(); value != "" { if label, has := instanceLabelPaths[key]; has { instance.SetLabel(label, value) @@ -45,7 +45,7 @@ func init() { } for _, child := range node.GetChildren() { - fetch(instance, child, newpath) + fetch(instance, child, newPath) } } diff --git a/cmd/collectors/zapi/plugins/snapmirror/snapmirror_test.go b/cmd/collectors/zapi/plugins/snapmirror/snapmirror_test.go index 00ef8967e..23254c7d2 100644 --- a/cmd/collectors/zapi/plugins/snapmirror/snapmirror_test.go +++ b/cmd/collectors/zapi/plugins/snapmirror/snapmirror_test.go @@ -34,9 +34,7 @@ func testWithoutGroupType(t *testing.T, instance *matrix.Instance) { instance.SetLabel("group_type", "") collectors.UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "" && instance.GetLabel("protectionSourceType") == "" { - // OK - } else { + if instance.GetLabel("protectedBy") != "" || instance.GetLabel("protectionSourceType") != "" { t.Errorf("Labels protectedBy= %s, expected empty and protectionSourceType= %s, expected empty", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -47,9 +45,7 @@ func testSvmdr(t *testing.T, instance *matrix.Instance) { instance.SetLabel("source_volume", "") collectors.UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "storage_vm" && instance.GetLabel("protectionSourceType") == "storage_vm" { - // OK - } else { + if instance.GetLabel("protectedBy") != "storage_vm" || instance.GetLabel("protectionSourceType") != "storage_vm" { t.Errorf("Labels protectedBy= %s, expected: storage_vm and protectionSourceType= %s, expected: storage_vm", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -61,9 +57,7 @@ func testConstituentVolumeWithinSvmdr(t *testing.T, instance *matrix.Instance) { instance.SetLabel("destination_location", "test1") collectors.UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "storage_vm" && instance.GetLabel("protectionSourceType") == "volume" { - // OK - } else { + if instance.GetLabel("protectedBy") != "storage_vm" || instance.GetLabel("protectionSourceType") != "volume" { t.Errorf("Labels protectedBy= %s, expected: storage_vm and protectionSourceType= %s, expected: volume", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -73,9 +67,7 @@ func testCg(t *testing.T, instance *matrix.Instance) { instance.SetLabel("destination_location", "test123:/cg/") collectors.UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "cg" && instance.GetLabel("protectionSourceType") == "cg" { - // OK - } else { + if instance.GetLabel("protectedBy") != "cg" || instance.GetLabel("protectionSourceType") != "cg" { t.Errorf("Labels protectedBy= %s, expected: cg and protectionSourceType= %s, expected: cg", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -85,9 +77,7 @@ func testConstituentVolumeWithinCg(t *testing.T, instance *matrix.Instance) { instance.SetLabel("destination_location", "test123") collectors.UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "cg" && instance.GetLabel("protectionSourceType") == "volume" { - // OK - } else { + if instance.GetLabel("protectedBy") != "cg" || instance.GetLabel("protectionSourceType") != "volume" { t.Errorf("Labels protectedBy= %s, expected: cg and protectionSourceType= %s, expected: volume", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -96,9 +86,7 @@ func testNegativeCase1(t *testing.T, instance *matrix.Instance) { instance.SetLabel("group_type", "infinitevol") collectors.UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "volume" && instance.GetLabel("protectionSourceType") == "not_mapped" { - // OK - } else { + if instance.GetLabel("protectedBy") != "volume" || instance.GetLabel("protectionSourceType") != "not_mapped" { t.Errorf("Labels protectedBy= %s, expected: volume and protectionSourceType= %s, expected: not_mapped", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -110,9 +98,7 @@ func testNegativeCase2(t *testing.T, instance *matrix.Instance) { instance.SetLabel("destination_location", "test123:") collectors.UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "volume" && instance.GetLabel("protectionSourceType") == "not_mapped" { - // OK - } else { + if instance.GetLabel("protectedBy") != "volume" || instance.GetLabel("protectionSourceType") != "not_mapped" { t.Errorf("Labels protectedBy= %s, expected: volume and protectionSourceType= %s, expected: not_mapped", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -121,9 +107,7 @@ func testGroupTypeNone(t *testing.T, instance *matrix.Instance) { instance.SetLabel("group_type", "none") collectors.UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "volume" && instance.GetLabel("protectionSourceType") == "volume" { - // OK - } else { + if instance.GetLabel("protectedBy") != "volume" || instance.GetLabel("protectionSourceType") != "volume" { t.Errorf("Labels protectedBy= %s, expected: volume and protectionSourceType= %s, expected: volume", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -132,9 +116,7 @@ func testGroupTypeFlexgroup(t *testing.T, instance *matrix.Instance) { instance.SetLabel("group_type", "flexgroup") collectors.UpdateProtectedFields(instance) - if instance.GetLabel("protectedBy") == "volume" && instance.GetLabel("protectionSourceType") == "volume" { - // OK - } else { + if instance.GetLabel("protectedBy") != "volume" || instance.GetLabel("protectionSourceType") != "volume" { t.Errorf("Labels protectedBy= %s, expected: volume and protectionSourceType= %s, expected: volume", instance.GetLabel("protectedBy"), instance.GetLabel("protectionSourceType")) } } @@ -145,9 +127,7 @@ func testStrictSyncMirror(t *testing.T, instance *matrix.Instance) { instance.SetLabel("policy_type", "strict_sync_mirror") collectors.UpdateProtectedFields(instance) - if instance.GetLabel("derived_relationship_type") == "sync_mirror_strict" { - // OK - } else { + if instance.GetLabel("derived_relationship_type") != "sync_mirror_strict" { t.Errorf("Labels derived_relationship_type= %s, expected: sync_mirror_strict", instance.GetLabel("derived_relationship_type")) } } @@ -157,9 +137,7 @@ func testSyncMirror(t *testing.T, instance *matrix.Instance) { instance.SetLabel("policy_type", "sync_mirror") collectors.UpdateProtectedFields(instance) - if instance.GetLabel("derived_relationship_type") == "sync_mirror" { - // OK - } else { + if instance.GetLabel("derived_relationship_type") != "sync_mirror" { t.Errorf("Labels derived_relationship_type= %s, expected: sync_mirror", instance.GetLabel("derived_relationship_type")) } } @@ -169,9 +147,7 @@ func testMirrorVault(t *testing.T, instance *matrix.Instance) { instance.SetLabel("policy_type", "mirror_vault") collectors.UpdateProtectedFields(instance) - if instance.GetLabel("derived_relationship_type") == "mirror_vault" { - // OK - } else { + if instance.GetLabel("derived_relationship_type") != "mirror_vault" { t.Errorf("Labels derived_relationship_type= %s, expected: mirror_vault", instance.GetLabel("derived_relationship_type")) } } @@ -181,9 +157,7 @@ func testAutomatedFailover(t *testing.T, instance *matrix.Instance) { instance.SetLabel("policy_type", "automated_failover") collectors.UpdateProtectedFields(instance) - if instance.GetLabel("derived_relationship_type") == "sync_mirror" { - // OK - } else { + if instance.GetLabel("derived_relationship_type") != "sync_mirror" { t.Errorf("Labels derived_relationship_type= %s, expected: sync_mirror", instance.GetLabel("derived_relationship_type")) } } @@ -193,9 +167,7 @@ func testOtherPolicyType(t *testing.T, instance *matrix.Instance) { instance.SetLabel("policy_type", "vault") collectors.UpdateProtectedFields(instance) - if instance.GetLabel("derived_relationship_type") == "vault" { - // OK - } else { + if instance.GetLabel("derived_relationship_type") != "vault" { t.Errorf("Labels derived_relationship_type= %s, expected: vault", instance.GetLabel("derived_relationship_type")) } } @@ -205,9 +177,7 @@ func testWithNoPolicyType(t *testing.T, instance *matrix.Instance) { instance.SetLabel("policy_type", "") collectors.UpdateProtectedFields(instance) - if instance.GetLabel("derived_relationship_type") == "extended_data_protection" { - // OK - } else { + if instance.GetLabel("derived_relationship_type") != "extended_data_protection" { t.Errorf("Labels derived_relationship_type= %s, expected: extended_data_protection", instance.GetLabel("derived_relationship_type")) } } diff --git a/cmd/collectors/zapi/plugins/svm/svm.go b/cmd/collectors/zapi/plugins/svm/svm.go index ad3d550c7..0214dc8ed 100644 --- a/cmd/collectors/zapi/plugins/svm/svm.go +++ b/cmd/collectors/zapi/plugins/svm/svm.go @@ -29,36 +29,36 @@ type SVM struct { batchSize string client *zapi.Client auditProtocols map[string]string - cifsProtocols map[string]cifsSecurity - nsswitchInfo map[string]nsswitch + cifsProtocols map[string]CifsSecurity + nsswitchInfo map[string]Nsswitch nisInfo map[string]string cifsEnabled map[string]bool nfsEnabled map[string]string - sshData map[string]sshInfo + sshData map[string]SSHInfo iscsiAuth map[string]string iscsiService map[string]string - fpolicyData map[string]fpolicy + fpolicyData map[string]Fpolicy ldapData map[string]string kerberosConfig map[string]string } -type nsswitch struct { +type Nsswitch struct { nsdb []string nssource []string } -type fpolicy struct { +type Fpolicy struct { name string enable string } -type cifsSecurity struct { +type CifsSecurity struct { cifsNtlmEnabled string smbEncryption string smbSigning string } -type sshInfo struct { +type SSHInfo struct { ciphers string isInsecure string } @@ -85,15 +85,15 @@ func (my *SVM) Init() error { } my.auditProtocols = make(map[string]string) - my.cifsProtocols = make(map[string]cifsSecurity) - my.nsswitchInfo = make(map[string]nsswitch) + my.cifsProtocols = make(map[string]CifsSecurity) + my.nsswitchInfo = make(map[string]Nsswitch) my.nisInfo = make(map[string]string) my.cifsEnabled = make(map[string]bool) my.nfsEnabled = make(map[string]string) - my.sshData = make(map[string]sshInfo) + my.sshData = make(map[string]SSHInfo) my.iscsiAuth = make(map[string]string) my.iscsiService = make(map[string]string) - my.fpolicyData = make(map[string]fpolicy) + my.fpolicyData = make(map[string]Fpolicy) my.ldapData = make(map[string]string) my.kerberosConfig = make(map[string]string) @@ -339,15 +339,15 @@ func (my *SVM) GetAuditProtocols() (map[string]string, error) { return vserverAuditEnableMap, nil } -func (my *SVM) GetCifsProtocols() (map[string]cifsSecurity, error) { +func (my *SVM) GetCifsProtocols() (map[string]CifsSecurity, error) { var ( result []*node.Node request *node.Node - vserverCifsDataMap map[string]cifsSecurity + vserverCifsDataMap map[string]CifsSecurity err error ) - vserverCifsDataMap = make(map[string]cifsSecurity) + vserverCifsDataMap = make(map[string]CifsSecurity) request = node.NewXMLS("cifs-security-get-iter") request.NewChildS("max-records", my.batchSize) @@ -365,22 +365,22 @@ func (my *SVM) GetCifsProtocols() (map[string]cifsSecurity, error) { smbSigning := cifsSecurityData.GetChildContentS("is-signing-required") smbEncryption := cifsSecurityData.GetChildContentS("is-smb-encryption-required") svmName := cifsSecurityData.GetChildContentS("vserver") - vserverCifsDataMap[svmName] = cifsSecurity{cifsNtlmEnabled: lmCompatibilityLevel, smbEncryption: smbEncryption, smbSigning: smbSigning} + vserverCifsDataMap[svmName] = CifsSecurity{cifsNtlmEnabled: lmCompatibilityLevel, smbEncryption: smbEncryption, smbSigning: smbSigning} } return vserverCifsDataMap, nil } -func (my *SVM) GetNSSwitchInfo() (map[string]nsswitch, error) { +func (my *SVM) GetNSSwitchInfo() (map[string]Nsswitch, error) { var ( result []*node.Node request *node.Node - vserverNsswitchMap map[string]nsswitch - ns nsswitch + vserverNsswitchMap map[string]Nsswitch + ns Nsswitch ok bool err error ) - vserverNsswitchMap = make(map[string]nsswitch) + vserverNsswitchMap = make(map[string]Nsswitch) request = node.NewXMLS("nameservice-nsswitch-get-iter") request.NewChildS("max-records", my.batchSize) @@ -403,7 +403,7 @@ func (my *SVM) GetNSSwitchInfo() (map[string]nsswitch, error) { ns.nsdb = append(ns.nsdb, nsdb) ns.nssource = append(ns.nssource, nssourcelist...) } else { - ns = nsswitch{nsdb: []string{nsdb}, nssource: nssourcelist} + ns = Nsswitch{nsdb: []string{nsdb}, nssource: nssourcelist} } vserverNsswitchMap[svmName] = ns } @@ -496,15 +496,15 @@ func (my *SVM) GetNfsEnabled() (map[string]string, error) { return vserverNfsMap, nil } -func (my *SVM) GetSSHData() (map[string]sshInfo, error) { +func (my *SVM) GetSSHData() (map[string]SSHInfo, error) { var ( result []*node.Node request *node.Node - sshMap map[string]sshInfo + sshMap map[string]SSHInfo err error ) - sshMap = make(map[string]sshInfo) + sshMap = make(map[string]SSHInfo) request = node.NewXMLS("security-ssh-get-iter") request.NewChildS("max-records", my.batchSize) @@ -523,7 +523,7 @@ func (my *SVM) GetSSHData() (map[string]sshInfo, error) { sort.Strings(sshList) ciphersVal := strings.Join(sshList, ",") insecured := weakCiphers.MatchString(ciphersVal) - sshMap[svmName] = sshInfo{ciphers: ciphersVal, isInsecure: strconv.FormatBool(insecured)} + sshMap[svmName] = SSHInfo{ciphers: ciphersVal, isInsecure: strconv.FormatBool(insecured)} } return sshMap, nil } @@ -598,15 +598,15 @@ func (my *SVM) GetIscsiService() (map[string]string, error) { return vserverIscsiServiceMap, nil } -func (my *SVM) GetFpolicy() (map[string]fpolicy, error) { +func (my *SVM) GetFpolicy() (map[string]Fpolicy, error) { var ( result []*node.Node request *node.Node - vserverFpolicyMap map[string]fpolicy + vserverFpolicyMap map[string]Fpolicy err error ) - vserverFpolicyMap = make(map[string]fpolicy) + vserverFpolicyMap = make(map[string]Fpolicy) request = node.NewXMLS("fpolicy-policy-status-get-iter") request.NewChildS("max-records", my.batchSize) @@ -624,11 +624,11 @@ func (my *SVM) GetFpolicy() (map[string]fpolicy, error) { enable := fpolicyData.GetChildContentS("status") svmName := fpolicyData.GetChildContentS("vserver") if _, ok := vserverFpolicyMap[svmName]; !ok { - vserverFpolicyMap[svmName] = fpolicy{name: name, enable: enable} + vserverFpolicyMap[svmName] = Fpolicy{name: name, enable: enable} } else { // If svm is already present, update the status value only if it is false if vserverFpolicyMap[svmName].enable == "false" { - vserverFpolicyMap[svmName] = fpolicy{name: name, enable: enable} + vserverFpolicyMap[svmName] = Fpolicy{name: name, enable: enable} } } } diff --git a/cmd/collectors/zapi/plugins/volume/volume.go b/cmd/collectors/zapi/plugins/volume/volume.go index a34cc33e8..66a5e5ec9 100644 --- a/cmd/collectors/zapi/plugins/volume/volume.go +++ b/cmd/collectors/zapi/plugins/volume/volume.go @@ -136,9 +136,8 @@ func (v *Volume) updateVolumeLabels(data *matrix.Matrix, volumeCloneMap map[stri if splitEstimateBytes, err = strconv.ParseFloat(vc.splitEstimate, 64); err != nil { v.Logger.Error().Err(err).Str("clone_split_estimate", vc.splitEstimate).Msg("parse clone_split_estimate") continue - } else { - splitEstimateBytes = splitEstimateBytes * 4 * 1024 } + splitEstimateBytes = splitEstimateBytes * 4 * 1024 if err = splitEstimate.SetValueFloat64(volume, splitEstimateBytes); err != nil { v.Logger.Error().Err(err).Str("clone_split_estimate", vc.splitEstimate).Msg("set clone_split_estimate") continue @@ -196,7 +195,7 @@ func (v *Volume) getEncryptedDisks() ([]string, error) { request := node.NewXMLS("disk-encrypt-get-iter") request.NewChildS("max-records", collectors.DefaultBatchSize) - //algorithm is -- Protection mode needs to be DATA or FULL + // algorithm is -- Protection mode needs to be DATA or FULL // Fetching rest of them and add as query := request.NewChildS("query", "") encryptInfoQuery := query.NewChildS("disk-encrypt-info", "") diff --git a/cmd/collectors/zapiperf/plugins/disk/disk.go b/cmd/collectors/zapiperf/plugins/disk/disk.go index e5dcd3d9f..7e4f27b39 100644 --- a/cmd/collectors/zapiperf/plugins/disk/disk.go +++ b/cmd/collectors/zapiperf/plugins/disk/disk.go @@ -235,13 +235,13 @@ func (d *Disk) initAggrPowerMatrix() { } func (d *Disk) initMaps() { - //reset shelf Power + // reset shelf Power d.ShelfMap = make(map[string]*shelf) - //reset diskmap + // reset diskmap d.diskMap = make(map[string]*disk) - //reset aggrmap + // reset aggrmap d.aggrMap = make(map[string]*aggregate) } @@ -356,7 +356,7 @@ func (d *Disk) calculateAggrPower(data *matrix.Matrix, output []*matrix.Matrix) for _, v1 := range v.disks { c := len(v1.aggregates) if c > 0 { - diskWithAggregateCount += 1 + diskWithAggregateCount++ } } if diskWithAggregateCount != 0 { diff --git a/cmd/collectors/zapiperf/plugins/externalserviceoperation/externalserviceoperation.go b/cmd/collectors/zapiperf/plugins/externalserviceoperation/externalserviceoperation.go index ef65ff41b..219602607 100644 --- a/cmd/collectors/zapiperf/plugins/externalserviceoperation/externalserviceoperation.go +++ b/cmd/collectors/zapiperf/plugins/externalserviceoperation/externalserviceoperation.go @@ -1,6 +1,5 @@ -/* - * Copyright NetApp Inc, 2023 All rights reserved - */ +// Copyright NetApp Inc, 2023 All rights reserved + package externalserviceoperation import ( diff --git a/cmd/collectors/zapiperf/plugins/fcvi/fcvi.go b/cmd/collectors/zapiperf/plugins/fcvi/fcvi.go index 6f7d8e882..46bfd2fa4 100644 --- a/cmd/collectors/zapiperf/plugins/fcvi/fcvi.go +++ b/cmd/collectors/zapiperf/plugins/fcvi/fcvi.go @@ -30,10 +30,7 @@ func (f *FCVI) Init() error { f.Logger.Error().Stack().Err(err).Msg("connecting") return err } - if err = f.client.Init(5); err != nil { - return err - } - return nil + return f.client.Init(5) } func (f *FCVI) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { diff --git a/cmd/collectors/zapiperf/plugins/headroom/headroom.go b/cmd/collectors/zapiperf/plugins/headroom/headroom.go index cefc0f14e..e4efee38b 100644 --- a/cmd/collectors/zapiperf/plugins/headroom/headroom.go +++ b/cmd/collectors/zapiperf/plugins/headroom/headroom.go @@ -1,6 +1,5 @@ -/* - * Copyright NetApp Inc, 2021 All rights reserved - */ +// Copyright NetApp Inc, 2021 All rights reserved + package headroom import ( diff --git a/cmd/collectors/zapiperf/plugins/volumetag/volumetag.go b/cmd/collectors/zapiperf/plugins/volumetag/volumetag.go index e7a4501e8..9dca02d91 100644 --- a/cmd/collectors/zapiperf/plugins/volumetag/volumetag.go +++ b/cmd/collectors/zapiperf/plugins/volumetag/volumetag.go @@ -29,10 +29,7 @@ func (v *VolumeTag) Init() error { v.Logger.Error().Stack().Err(err).Msg("connecting") return err } - if err = v.client.Init(5); err != nil { - return err - } - return nil + return v.client.Init(5) } func (v *VolumeTag) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { diff --git a/cmd/collectors/zapiperf/zapiperf.go b/cmd/collectors/zapiperf/zapiperf.go index 35bf4926e..a2ef30bac 100644 --- a/cmd/collectors/zapiperf/zapiperf.go +++ b/cmd/collectors/zapiperf/zapiperf.go @@ -264,14 +264,14 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) { // list of instance keys (instance names or uuids) for which // we will request counter data if z.Query == objWorkloadDetail || z.Query == objWorkloadDetailVolume { - if resourceMap := z.Params.GetChildS("resource_map"); resourceMap == nil { + resourceMap := z.Params.GetChildS("resource_map") + if resourceMap == nil { return nil, errs.New(errs.ErrMissingParam, "resource_map") - } else { - instanceKeys = make([]string, 0) - for _, layer := range resourceMap.GetAllChildNamesS() { - for key := range prevMat.GetInstances() { - instanceKeys = append(instanceKeys, key+"."+layer) - } + } + instanceKeys = make([]string, 0) + for _, layer := range resourceMap.GetAllChildNamesS() { + for key := range prevMat.GetInstances() { + instanceKeys = append(instanceKeys, key+"."+layer) } } } else { @@ -337,8 +337,7 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) { if err != nil { // if ONTAP complains about batch size, use a smaller batch size if strings.Contains(err.Error(), "resource limit exceeded") && z.batchSize > 100 { - z.Logger.Error().Err(err) - z.Logger.Info(). + z.Logger.Error().Err(err). Int("oldBatchSize", z.batchSize). Int("newBatchSize", z.batchSize-100). Msg("Changed batch_size") @@ -1109,48 +1108,48 @@ func (z *ZapiPerf) PollCounter() (map[string]*matrix.Matrix, error) { wait.SetExportable(false) visits.SetExportable(false) - if resourceMap := z.Params.GetChildS("resource_map"); resourceMap == nil { + resourceMap := z.Params.GetChildS("resource_map") + if resourceMap == nil { return nil, errs.New(errs.ErrMissingParam, "resource_map") - } else { - for _, x := range resourceMap.GetChildren() { - for _, wm := range workloadDetailMetrics { - - name := x.GetNameS() + wm - resource := x.GetContentS() + } + for _, x := range resourceMap.GetChildren() { + for _, wm := range workloadDetailMetrics { - if m := mat.GetMetric(name); m != nil { - oldMetrics.Remove(name) - continue - } - if m, err := mat.NewMetricFloat64(name, wm); err != nil { - return nil, err - } else { - m.SetLabel("resource", resource) - m.SetProperty(service.GetProperty()) - // base counter is the ops of the same resource - m.SetComment("ops") + name := x.GetNameS() + wm + resource := x.GetContentS() - oldMetrics.Remove(name) - z.Logger.Debug().Msgf("+ [%s] (=> %s) added workload latency metric", name, resource) - } + if m := mat.GetMetric(name); m != nil { + oldMetrics.Remove(name) + continue } + m, err := mat.NewMetricFloat64(name, wm) + if err != nil { + return nil, err + } + m.SetLabel("resource", resource) + m.SetProperty(service.GetProperty()) + // base counter is the ops of the same resource + m.SetComment("ops") + + oldMetrics.Remove(name) + z.Logger.Debug().Msgf("+ [%s] (=> %s) added workload latency metric", name, resource) } } } - if qosLabels := z.Params.GetChildS("qos_labels"); qosLabels == nil { + qosLabels := z.Params.GetChildS("qos_labels") + if qosLabels == nil { return nil, errs.New(errs.ErrMissingParam, "qos_labels") - } else { - z.qosLabels = make(map[string]string) - for _, label := range qosLabels.GetAllChildContentS() { + } + z.qosLabels = make(map[string]string) + for _, label := range qosLabels.GetAllChildContentS() { - display := strings.ReplaceAll(label, "-", "_") - if x := strings.Split(label, "=>"); len(x) == 2 { - label = strings.TrimSpace(x[0]) - display = strings.TrimSpace(x[1]) - } - z.qosLabels[label] = display + display := strings.ReplaceAll(label, "-", "_") + if x := strings.Split(label, "=>"); len(x) == 2 { + label = strings.TrimSpace(x[0]) + display = strings.TrimSpace(x[1]) } + z.qosLabels[label] = display } } diff --git a/cmd/collectors/zapiperf/zapiperf_test.go b/cmd/collectors/zapiperf/zapiperf_test.go index adf83d9b5..f4f8efbf9 100644 --- a/cmd/collectors/zapiperf/zapiperf_test.go +++ b/cmd/collectors/zapiperf/zapiperf_test.go @@ -48,7 +48,7 @@ func NewZapiPerf(object, path string) *ZapiPerf { ac := collector.New("Zapiperf", object, &opts, params(object, path), nil) z := &ZapiPerf{} if err := z.Init(ac); err != nil { - log.Fatal().Err(err) + log.Fatal().Err(err).Send() } z.Object = object diff --git a/cmd/exporters/influxdb/influxdb.go b/cmd/exporters/influxdb/influxdb.go index b9b683e00..137261d75 100644 --- a/cmd/exporters/influxdb/influxdb.go +++ b/cmd/exporters/influxdb/influxdb.go @@ -217,11 +217,11 @@ func (e *InfluxDB) Emit(data [][]byte) error { //goland:noinspection GoUnhandledErrorResult defer response.Body.Close() if response.StatusCode != expectedResponseCode { - if body, err := io.ReadAll(response.Body); err != nil { + body, err := io.ReadAll(response.Body) + if err != nil { return errs.New(errs.ErrAPIResponse, err.Error()) - } else { - return fmt.Errorf("%w: %s", errs.ErrAPIRequestRejected, string(body)) } + return fmt.Errorf("%w: %s", errs.ErrAPIRequestRejected, string(body)) } return nil } @@ -340,7 +340,7 @@ func (e *InfluxDB) Render(data *matrix.Matrix) ([][]byte, error) { e.Logger.Debug().Msgf("skip instance (%s), no field set parsed", key) } else if r, err := m.Render(); err == nil { rendered = append(rendered, []byte(r)) - //logger.Debug(e.Prefix, "M= [%s%s%s]", color.Blue, r, color.End) + // logger.Debug(e.Prefix, "M= [%s%s%s]", color.Blue, r, color.End) count += countTmp } else { e.Logger.Debug().Msg(err.Error()) diff --git a/cmd/exporters/prometheus/httpd.go b/cmd/exporters/prometheus/httpd.go index 7ffba1aac..9325944b7 100644 --- a/cmd/exporters/prometheus/httpd.go +++ b/cmd/exporters/prometheus/httpd.go @@ -96,7 +96,7 @@ func (p *Prometheus) checkAddr(addr string) bool { func (p *Prometheus) denyAccess(w http.ResponseWriter, r *http.Request) { p.Logger.Debug().Msgf("(httpd) denied request [%s] (%s)", r.RequestURI, r.RemoteAddr) - w.WriteHeader(403) + w.WriteHeader(http.StatusForbidden) w.Header().Set("content-type", "text/plain") _, err := w.Write([]byte("403 Forbidden")) if err != nil { @@ -145,7 +145,7 @@ func (p *Prometheus) ServeMetrics(w http.ResponseWriter, r *http.Request) { data = filterMetaTags(data) } - w.WriteHeader(200) + w.WriteHeader(http.StatusOK) w.Header().Set("content-type", "text/plain") _, err := w.Write(bytes.Join(data, []byte("\n"))) if err != nil { @@ -284,7 +284,7 @@ func (p *Prometheus) ServeInfo(w http.ResponseWriter, r *http.Request) { poller := p.Options.Poller bodyFlat := fmt.Sprintf(htmlTemplate, poller, poller, poller, numCollectors, numObjects, numMetrics, strings.Join(body, "\n\n")) - w.WriteHeader(200) + w.WriteHeader(http.StatusOK) w.Header().Set("content-type", "text/html") _, err := w.Write([]byte(bodyFlat)) if err != nil { diff --git a/cmd/exporters/prometheus/prometheus.go b/cmd/exporters/prometheus/prometheus.go index 55ac0b013..e09561ebc 100644 --- a/cmd/exporters/prometheus/prometheus.go +++ b/cmd/exporters/prometheus/prometheus.go @@ -585,7 +585,7 @@ func (p *Prometheus) normalizeHistogram(metric *matrix.Metric, ontap string, obj p.Logger.Trace().Str("num", num).Msg("Unable to convert to float64") return "" } - normal := 0.0 + var normal float64 switch unit { case "us": return num diff --git a/cmd/harvest/harvest.go b/cmd/harvest/harvest.go index 732dabeba..32945dfa7 100644 --- a/cmd/harvest/harvest.go +++ b/cmd/harvest/harvest.go @@ -1,5 +1,5 @@ /* - - Copyright NetApp Inc, 2021 All rights reserved +Copyright NetApp Inc, 2021 All rights reserved NetApp Harvest : the swiss-army-knife for datacenter monitoring diff --git a/cmd/harvest/version/version.go b/cmd/harvest/version/version.go index beecee9c6..041e54b75 100644 --- a/cmd/harvest/version/version.go +++ b/cmd/harvest/version/version.go @@ -90,9 +90,8 @@ func isNewerAvailable(current string, remote string) (bool, error) { } if currentVersion.GreaterThanOrEqual(remoteVersion) { return false, nil - } else { - return true, nil } + return true, nil } func latestRelease() (string, error) { diff --git a/cmd/poller/collector/asup.go b/cmd/poller/collector/asup.go index 9deeafa8f..9f55aa954 100644 --- a/cmd/poller/collector/asup.go +++ b/cmd/poller/collector/asup.go @@ -322,9 +322,8 @@ func attachMemory(msg *Payload) { cmdline, err := p.Cmdline() if err != nil { continue - } else { - pp.Cmdline = cmdline } + pp.Cmdline = cmdline if len(cmdline) == 0 { continue } diff --git a/cmd/poller/plugin/labelagent/label_agent.go b/cmd/poller/plugin/labelagent/label_agent.go index bd43e29a8..c5c83d6d1 100644 --- a/cmd/poller/plugin/labelagent/label_agent.go +++ b/cmd/poller/plugin/labelagent/label_agent.go @@ -311,9 +311,8 @@ func (a *LabelAgent) mapValueToNum(m *matrix.Matrix) error { if metric, err = m.NewMetricUint8(r.metric); err != nil { a.Logger.Error().Stack().Err(err).Msgf("valueToNumMapping: new metric [%s]:", r.metric) return err - } else { - metric.SetProperty("value_to_num mapping") } + metric.SetProperty("value_to_num mapping") } for key, instance := range m.GetInstances() { @@ -342,9 +341,8 @@ func (a *LabelAgent) mapValueToNumRegex(m *matrix.Matrix) error { if metric, err = m.NewMetricUint8(r.metric); err != nil { a.Logger.Error().Stack().Err(err).Msgf("valueToNumRegexMapping: new metric [%s]:", r.metric) return err - } else { - metric.SetProperty("value_to_num_regex mapping") } + metric.SetProperty("value_to_num_regex mapping") } for key, instance := range m.GetInstances() { diff --git a/cmd/poller/plugin/labelagent/label_agent_test.go b/cmd/poller/plugin/labelagent/label_agent_test.go index 14f10ec1d..27ec7998a 100644 --- a/cmd/poller/plugin/labelagent/label_agent_test.go +++ b/cmd/poller/plugin/labelagent/label_agent_test.go @@ -9,7 +9,7 @@ import ( "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" "testing" - //"github.com/netapp/harvest/v2/share/logger" + // "github.com/netapp/harvest/v2/share/logger" ) func newLabelAgent() *LabelAgent { @@ -83,9 +83,7 @@ func TestSplitSimpleRule(t *testing.T) { _ = p.splitSimple(m) t.Logf("after = [%s]\n", instance.GetLabels().String()) - if instance.GetLabel("C") == "c" && instance.GetLabel("D") == "d" { - // OK - } else { + if instance.GetLabel("C") != "c" || instance.GetLabel("D") != "d" { t.Error("Labels C and D don't have expected values") } } @@ -101,9 +99,7 @@ func TestSplitRegexRule(t *testing.T) { _ = p.splitRegex(m) t.Logf("after = [%s]\n", instance.GetLabels().String()) - if instance.GetLabel("A") == "A22" && instance.GetLabel("B") == "B333" { - // OK - } else { + if instance.GetLabel("A") != "A22" || instance.GetLabel("B") != "B333" { t.Error("Labels A and B don't have expected values") } } @@ -119,9 +115,7 @@ func TestSplitPairsRule(t *testing.T) { _ = p.splitPairs(m) t.Logf("after = [%s]\n", instance.GetLabels().String()) - if instance.GetLabel("owner") == "jack" && instance.GetLabel("contact") == "some@email" { - // OK - } else { + if instance.GetLabel("owner") != "jack" || instance.GetLabel("contact") != "some@email" { t.Error("Labels owner and contact don't have expected values") } } @@ -138,9 +132,7 @@ func TestJoinSimpleRule(t *testing.T) { _ = p.joinSimple(m) t.Logf("after = [%s]\n", instance.GetLabels().String()) - if instance.GetLabel("X") == "aaa_bbb" { - // OK - } else { + if instance.GetLabel("X") != "aaa_bbb" { t.Error("Label A does have expected value") } } @@ -156,9 +148,7 @@ func TestReplaceSimpleRule(t *testing.T) { _ = p.replaceSimple(m) t.Logf("after = [%s]\n", instance.GetLabels().String()) - if instance.GetLabel("A") == "X" && instance.GetLabel("B") == "bbb_X" { - // OK - } else { + if instance.GetLabel("A") != "X" || instance.GetLabel("B") != "bbb_X" { t.Error("Labels A and B don't have expected values") } } @@ -174,9 +164,7 @@ func TestReplaceRegexRule(t *testing.T) { _ = p.replaceRegex(m) t.Logf("after = [%s]\n", instance.GetLabels().String()) - if instance.GetLabel("B") == "abcDEF-12345-bbb" { - // OK - } else { + if instance.GetLabel("B") != "abcDEF-12345-bbb" { t.Error("Label B does not have expected value") } } diff --git a/cmd/poller/plugin/labelagent/parse_rules.go b/cmd/poller/plugin/labelagent/parse_rules.go index d085db17d..fbdb512fc 100644 --- a/cmd/poller/plugin/labelagent/parse_rules.go +++ b/cmd/poller/plugin/labelagent/parse_rules.go @@ -315,7 +315,7 @@ type replaceRegexRule struct { // example rule: //nolint:dupword -//node node `^(node)_(\d+)_.*$` `Node-$2` +// node node `^(node)_(\d+)_.*$` `Node-$2` // if node="node_10_dc2"; then: // node="Node-10" @@ -522,13 +522,13 @@ func (a *LabelAgent) parseValueToNumRule(rule string) { fields[4] = strings.TrimPrefix(strings.TrimSuffix(fields[4], "`"), "`") - if v, err := strconv.ParseUint(fields[4], 10, 8); err != nil { + v, err := strconv.ParseUint(fields[4], 10, 8) + if err != nil { a.Logger.Error().Stack().Err(err).Msgf("(value_to_num) parse default value (%s): ", fields[4]) return - } else { - r.hasDefault = true - r.defaultValue = uint8(v) } + r.hasDefault = true + r.defaultValue = uint8(v) } a.valueToNumRules = append(a.valueToNumRules, r) @@ -568,13 +568,13 @@ func (a *LabelAgent) parseValueToNumRegexRule(rule string) { if len(fields) == 5 { fields[4] = strings.TrimPrefix(strings.TrimSuffix(fields[4], "`"), "`") - if v, err := strconv.ParseUint(fields[4], 10, 8); err != nil { + v, err := strconv.ParseUint(fields[4], 10, 8) + if err != nil { a.Logger.Error().Stack().Err(err).Msgf("(value_to_num_regex) parse default value (%s): ", fields[4]) return - } else { - r.hasDefault = true - r.defaultValue = uint8(v) } + r.hasDefault = true + r.defaultValue = uint8(v) } a.valueToNumRegexRules = append(a.valueToNumRegexRules, r) diff --git a/cmd/poller/plugin/metricagent/metric_agent.go b/cmd/poller/plugin/metricagent/metric_agent.go index 7044c2694..8fa2c7060 100644 --- a/cmd/poller/plugin/metricagent/metric_agent.go +++ b/cmd/poller/plugin/metricagent/metric_agent.go @@ -68,9 +68,8 @@ func (a *MetricAgent) computeMetrics(m *matrix.Matrix) error { if metric, err = m.NewMetricFloat64(r.metric); err != nil { a.Logger.Error().Err(err).Str("metric", r.metric).Msg("Failed to create metric") return err - } else { - metric.SetProperty("compute_metric mapping") } + metric.SetProperty("compute_metric mapping") } for _, instance := range m.GetInstances() { diff --git a/cmd/poller/plugin/plugin.go b/cmd/poller/plugin/plugin.go index 9230cf7de..43d14a0a7 100644 --- a/cmd/poller/plugin/plugin.go +++ b/cmd/poller/plugin/plugin.go @@ -1,30 +1,28 @@ -//Copyright NetApp Inc, 2021 All rights reserved - -/* - Package plugin provides abstractions for plugins, as well as - a number of generic built-in plugins. Plugins allow to customize - and manipulate data from collectors and sometimes collect additional - data without changing the sourcecode of collectors. Multiple plugins - can be put in a pipeline, they are executed in the same order as they - are defined in the collector's config file. - Harvest architecture defines three types of plugins: - - **built-in** - Statically compiled, generic plugins. "Generic" means - the plugin is collector-agnostic. These plugins are - provided in this package. - - **generic** - These are generic plugins as well, but they are compiled - as shared objects and dynamically loaded. These plugins are - living in the directory src/plugins. - - **custom** - These plugins are collector-specific. Their source code should - reside inside the plugins/ subdirectory of the collector package. - Custom plugins have access to all the parameters of their parent - collector and should be therefore treated with great care. -*/ +// Copyright NetApp Inc, 2021 All rights reserved +// Package plugin provides abstractions for plugins, as well as +// a number of generic built-in plugins. Plugins allow customizing +// and manipulating data from collectors and sometimes collect additional +// data without changing the sourcecode of collectors. Multiple plugins +// can be put in a pipeline, they are executed in the same order as they +// are defined in the collector's config file. +// Harvest architecture defines three types of plugins: +// +// **built-in** +// Statically compiled, generic plugins. "Generic" means +// the plugin is collector-agnostic. These plugins are +// provided in this package. +// +// **generic** +// These are generic plugins as well, but they are compiled +// as shared objects and dynamically loaded. These plugins are +// living in the directory src/plugins. +// +// **custom** +// These plugins are collector-specific. Their source code should +// reside inside the plugins/ subdirectory of the collector package. +// Custom plugins have access to all the parameters of their parent +// collector and should be therefore treated with great care. + package plugin import ( diff --git a/cmd/poller/plugin/test/plugin_test.go b/cmd/poller/plugin/test/plugin_test.go index 51fa16926..c62c40613 100644 --- a/cmd/poller/plugin/test/plugin_test.go +++ b/cmd/poller/plugin/test/plugin_test.go @@ -90,7 +90,7 @@ func TestMultipleRule(t *testing.T) { instanceNo.SetLabel("B", "aaa bbb ccc") instanceNo.SetLabel("node", "nodeB") - results := make([]*matrix.Matrix, 0) + var results []*matrix.Matrix results = append(results, m) dataMap := map[string]*matrix.Matrix{ m.Object: m, diff --git a/cmd/poller/poller.go b/cmd/poller/poller.go index f6f937810..9f95d0cb6 100644 --- a/cmd/poller/poller.go +++ b/cmd/poller/poller.go @@ -135,7 +135,7 @@ func (p *Poller) Init() error { p.options = &args p.name = args.Poller - fileLoggingEnabled := false + var fileLoggingEnabled bool consoleLoggingEnabled := false zeroLogLevel := logging.GetZerologLevel(p.options.LogLevel) // if we are daemon, use file logging @@ -695,13 +695,13 @@ func (p *Poller) loadCollectorObject(ocs []objectCollector) error { // update metadata - if instance, err := p.metadata.NewInstance(name + "." + obj); err != nil { + instance, err := p.metadata.NewInstance(name + "." + obj) + if err != nil { return err - } else { - instance.SetLabel("type", "collector") - instance.SetLabel("name", name) - instance.SetLabel("target", obj) } + instance.SetLabel("type", "collector") + instance.SetLabel("name", name) + instance.SetLabel("target", obj) } return nil @@ -1031,7 +1031,7 @@ func (p *Poller) publishDetails() { return } p.client.CloseIdleConnections() - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK { txt := string(body) txt = txt[0:int(math.Min(float64(len(txt)), 48))] logger.Error(). @@ -1126,7 +1126,7 @@ func (p *Poller) negotiateAPI(c conf.Collector, checkZAPIs func() error) conf.Co switchToRest = true } - if he.StatusCode == 400 { + if he.StatusCode == http.StatusBadRequest { logger.Warn().Str("collector", c.Name).Msg("ZAPIs EOA. Use REST") switchToRest = true } @@ -1159,10 +1159,7 @@ func (p *Poller) doZAPIsExist() error { return err } - if err = connection.Init(2); err != nil { - return err - } - return nil + return connection.Init(2) } func startPoller(_ *cobra.Command, _ []string) { diff --git a/cmd/poller/schedule/schedule.go b/cmd/poller/schedule/schedule.go index 0cb546be6..bffdb9c7a 100644 --- a/cmd/poller/schedule/schedule.go +++ b/cmd/poller/schedule/schedule.go @@ -1,35 +1,34 @@ -/* - Copyright NetApp Inc, 2021 All rights reserved - - Package Schedule provides a mechanism to run tasks at fixed time interals. - It is intended to be used by collectors, but can be used by any other - package as well. Tasks can be dynamically pointed to the poll functions - of the collector. (This why poll functions of collectors are public and - have the same signature). - - At least one task should be added to Schedule before it can be used. - Tasks are yielded in the same order as added (FIFO). The intervals of tasks - can be safely changed any time. - - Create Schedule: - - Initialize empty Schedule with New(), - - Add tasks with NewTask() or NewTaskString(), - the task is marked as due immediately! - - Use Schedule (usually in a closed loop): - - iterate over all tasks with GetTasks() - - check if it's time to run the task with IsDue(task) - - run the task with task.Run() or run "manually" with task.Start() - - suspend the goroutine until another task is due Sleep()/Wait() - - The Schedule can enter standByMode when a critical task has failed. In this - scenario all tasks are stalled until the critical task has succeeded. This is - sometimes useful when a target system is unreachable and we have to wait - until it's up again. - - Schedule is meant to be used by at most one goroutine and is not - concurrent-safe. -*/ +// Copyright NetApp Inc, 2021 All rights reserved + +// Package Schedule provides a mechanism to run tasks at fixed time internals. +// It is intended to be used by collectors, but can be used by any other +// package as well. Tasks can be dynamically pointed to the poll functions +// of the collector. (This is why poll functions of collectors are public and +// have the same signature). +// +// At least one task should be added to Schedule before it can be used. +// Tasks are yielded in the same order as added (FIFO). The intervals of tasks +// can be safely changed any time. +// +// Create Schedule: +// - Initialize empty Schedule with New(), +// - Add tasks with NewTask() or NewTaskString(), +// the task is marked as due immediately! +// +// Use Schedule (usually in a closed loop): +// - iterate over all tasks with GetTasks() +// - check if it's time to run the task with IsDue(task) +// - run the task with task.Run() or run "manually" with task.Start() +// - suspend the goroutine until another task is due Sleep()/Wait() +// +// The Schedule can enter standByMode when a critical task has failed. In this +// scenario, all tasks are stalled until the critical task has succeeded. This is +// sometimes useful when a target system is unreachable, and we have to wait +// until it's up again. +// +// Schedule is meant to be used by at most one goroutine and is not +// concurrent-safe. + package schedule import ( @@ -39,7 +38,7 @@ import ( ) // Task represents a scheduled task -type task struct { +type Task struct { Name string // name of the task interval time.Duration // the schedule interval timer time.Time // last time task was executed @@ -51,49 +50,49 @@ type task struct { // Use this method if you are executing the task yourself and you need to register // when task started. If the task has a pointer to the executing function, use // Run() instead. -func (t *task) Start() { +func (t *Task) Start() { t.timer = time.Now() } // Run marks the task as started and executes it -func (t *task) Run() (map[string]*matrix.Matrix, error) { +func (t *Task) Run() (map[string]*matrix.Matrix, error) { t.Start() return t.foo() } // GetDuration tells duration of executing the task // it assumes that the task just completed -func (t *task) GetDuration() time.Duration { +func (t *Task) GetDuration() time.Duration { return time.Since(t.timer) } // GetInterval tells the scheduled interval of the task -func (t *task) GetInterval() time.Duration { +func (t *Task) GetInterval() time.Duration { return t.interval } // NextDue tells time until the task is due -func (t *task) NextDue() time.Duration { +func (t *Task) NextDue() time.Duration { return t.interval - time.Since(t.timer) } // IsDue tells whether it's time to run the task -func (t *task) IsDue() bool { +func (t *Task) IsDue() bool { return t.NextDue() <= 0 } // Schedule contains a collection of tasks and the current state of the schedule type Schedule struct { - tasks []*task // list of tasks that Schedule needs to run + tasks []*Task // list of tasks that Schedule needs to run standByMode bool // if true, Schedule waitsfor a stalled task - standByTask *task // stalled task in standByMode + standByTask *Task // stalled task in standByMode cachedInterval map[string]time.Duration // normal interval of the stalled tasks } // New creates and initializes an empty Schedule. func New() *Schedule { s := Schedule{} - s.tasks = make([]*task, 0) + s.tasks = make([]*Task, 0) s.standByMode = false s.cachedInterval = make(map[string]time.Duration) return &s @@ -106,14 +105,14 @@ func (s *Schedule) IsStandBy() bool { } // IsTaskStandBy tells if a task in schedule is in IsStandBy. -func (s *Schedule) IsTaskStandBy(t *task) bool { +func (s *Schedule) IsTaskStandBy(t *Task) bool { return t.Name == s.standByTask.Name } // SetStandByMode initializes StandbyMode: Schedule will suspend all tasks until // the critical task t has succeeded. The temporary interval i will be used for // the task until Schedule recovers to normal mode. -func (s *Schedule) SetStandByMode(t *task, i time.Duration) { +func (s *Schedule) SetStandByMode(t *Task, i time.Duration) { for _, x := range s.tasks { if x.Name == t.Name { s.standByTask = t @@ -142,7 +141,7 @@ func (s *Schedule) Recover() { t.timer = time.Now().Add(-t.interval) } } - //s.cachedInterval = nil + // s.cachedInterval = nil s.standByTask = nil s.standByMode = false return @@ -158,7 +157,7 @@ func (s *Schedule) Recover() { func (s *Schedule) NewTask(n string, i time.Duration, f func() (map[string]*matrix.Matrix, error), runNow bool, identifier string) error { if s.GetTask(n) == nil { if i > 0 { - t := &task{Name: n, interval: i, foo: f, identifier: identifier} + t := &Task{Name: n, interval: i, foo: f, identifier: identifier} s.cachedInterval[n] = t.interval // remember normal interval of task if runNow { t.timer = time.Now().Add(-i) // set to run immediately @@ -183,15 +182,15 @@ func (s *Schedule) NewTaskString(n, i string, f func() (map[string]*matrix.Matri } // GetTasks returns scheduled tasks -func (s *Schedule) GetTasks() []*task { +func (s *Schedule) GetTasks() []*Task { if !s.standByMode { return s.tasks } - return []*task{s.standByTask} + return []*Task{s.standByTask} } // GetTask returns the task named n or nil if it doesn't exist -func (s *Schedule) GetTask(n string) *task { +func (s *Schedule) GetTask(n string) *Task { for _, t := range s.tasks { if t.Name == n { return t diff --git a/cmd/tools/doctor/doctor.go b/cmd/tools/doctor/doctor.go index 19f6473c5..5ad102164 100644 --- a/cmd/tools/doctor/doctor.go +++ b/cmd/tools/doctor/doctor.go @@ -129,9 +129,8 @@ func checkAll(path string, contents []byte) { if anyFailed { os.Exit(1) - } else { - os.Exit(0) } + os.Exit(0) } // checkCollectorName checks if the collector names in the config struct are valid diff --git a/cmd/tools/doctor/restZapiDiff.go b/cmd/tools/doctor/restZapiDiff.go index 3e949fb8d..100911885 100644 --- a/cmd/tools/doctor/restZapiDiff.go +++ b/cmd/tools/doctor/restZapiDiff.go @@ -113,7 +113,7 @@ func metricPerfValueDiff(metricName string) { replacer := strings.NewReplacer("[", "", "]", "", "\"", "") zapiMetric := make(map[string]float64) restMetric := make(map[string]float64) - results := make([]gjson.Result, 0) + var results []gjson.Result keyIndexes := make([]int, 0) @@ -355,11 +355,10 @@ func metricValueDiff(metricName string) { replacer := strings.NewReplacer("[", "", "]", "", "\"", "") zapiMetric := make(map[string]float64) restMetric := make(map[string]float64) - results := make([]gjson.Result, 0) - - keyIndexes := make([]int, 0) + var results []gjson.Result + var keyIndexes []int - // These plugin generated metrics are node scoped. + // These plugin-generated metrics are node scoped. environmentSensorMetrics := strings.Join([]string{ "environment_sensor_average_ambient_temperature", "environment_sensor_average_fan_speed", diff --git a/cmd/tools/generate/counter.go b/cmd/tools/generate/counter.go index db892ec0f..45d5a478a 100644 --- a/cmd/tools/generate/counter.go +++ b/cmd/tools/generate/counter.go @@ -229,8 +229,8 @@ func handleZapiCounter(path []string, content string, object string) (string, st } name = strings.TrimSpace(strings.TrimLeft(name, "^")) - - fullPath = append(path, name) + fullPath = path + fullPath = append(fullPath, name) key = strings.Join(fullPath, ".") if display == "" { display = util.ParseZAPIDisplay(object, fullPath) diff --git a/cmd/tools/grafana/grafana.go b/cmd/tools/grafana/grafana.go index 75be7bbf8..fe5175649 100644 --- a/cmd/tools/grafana/grafana.go +++ b/cmd/tools/grafana/grafana.go @@ -217,7 +217,7 @@ func addLabel(content []byte, label string, labelMap map[string]string) []byte { // create a new list of vars and copy the existing ones into it, duplicate the first var since we're going to // overwrite it - newVars := make([]gjson.Result, 0) + var newVars []gjson.Result newVars = append(newVars, vars[0]) newVars = append(newVars, vars...) @@ -304,7 +304,7 @@ func toChainedVar(defStr string, label string) string { if firstParen == -1 { return "" } - if lastComma == -1 { + if lastComma == -1 { //nolint:revive // Case 1: There are not existing labels // label_values(datacenter) becomes label_values({foo=~"$Foo"}, datacenter) } else { diff --git a/cmd/tools/rest/client.go b/cmd/tools/rest/client.go index aa737db0c..13f6c019d 100644 --- a/cmd/tools/rest/client.go +++ b/cmd/tools/rest/client.go @@ -177,13 +177,13 @@ func (c *Client) invokeWithAuthRetry() ([]byte, error) { //goland:noinspection GoUnhandledErrorResult defer response.Body.Close() - if response.StatusCode != 200 { + if response.StatusCode != http.StatusOK { body, err2 := io.ReadAll(response.Body) if err2 != nil { return nil, errs.Rest(response.StatusCode, err2.Error(), 0, "") } - if response.StatusCode == 401 { + if response.StatusCode == http.StatusUnauthorized { return nil, errs.New(errs.ErrAuthFailed, response.Status) } @@ -271,7 +271,7 @@ func downloadSwagger(poller *conf.Poller, path string, url string, verbose bool) debugResp, _ := httputil.DumpResponse(response, false) fmt.Printf("RESPONSE: \n%s", debugResp) } - if response.StatusCode != 200 { + if response.StatusCode != http.StatusOK { return 0, fmt.Errorf("error making request. server response statusCode=[%d]", response.StatusCode) } n, err := io.Copy(out, response.Body) diff --git a/cmd/tools/rest/rest.go b/cmd/tools/rest/rest.go index 74e759e1d..34c7da99d 100644 --- a/cmd/tools/rest/rest.go +++ b/cmd/tools/rest/rest.go @@ -388,11 +388,11 @@ func Fetch(client *Client, href string) ([]gjson.Result, error) { return nil, err } if mr != "" { - if mri, err := strconv.Atoi(mr); err != nil { + mri, err := strconv.Atoi(mr) + if err != nil { return nil, err - } else { - maxRecords = mri } + maxRecords = mri downloadAll = maxRecords == 0 } } @@ -421,11 +421,11 @@ func FetchAnalytics(client *Client, href string) ([]gjson.Result, gjson.Result, return []gjson.Result{}, gjson.Result{}, err } if mr != "" { - if mri, err := strconv.Atoi(mr); err != nil { + mri, err := strconv.Atoi(mr) + if err != nil { return []gjson.Result{}, gjson.Result{}, err - } else { - maxRecords = mri } + maxRecords = mri } downloadAll = maxRecords == 0 } diff --git a/cmd/tools/template/template_test.go b/cmd/tools/template/template_test.go index 2181fb698..b87aa1a40 100644 --- a/cmd/tools/template/template_test.go +++ b/cmd/tools/template/template_test.go @@ -279,7 +279,8 @@ func TestExportLabelsExist(t *testing.T) { allLabelNames[m.right] = true } else { if isZapi { - zapiPaths := append(m.parents, m.left) + zapiPaths := m.parents + zapiPaths = append(zapiPaths, m.left) display := util.ParseZAPIDisplay(model.Object, zapiPaths) allLabelNames[display] = true } else { diff --git a/cmd/tools/zapi/export.go b/cmd/tools/zapi/export.go index ea53f9049..e01a93562 100644 --- a/cmd/tools/zapi/export.go +++ b/cmd/tools/zapi/export.go @@ -87,7 +87,7 @@ func exportCounters(item *node.Node, c *client.Client, args *Args) error { } fmt.Println("\n===========================================================================\n") */ - fp := make([]string, 0) + var fp []string harvestHomePath = conf.GetHarvestHomePath() fp = append(fp, harvestHomePath) diff --git a/pkg/api/ontapi/zapi/client.go b/pkg/api/ontapi/zapi/client.go index ba201f631..dbfb78ad4 100644 --- a/pkg/api/ontapi/zapi/client.go +++ b/pkg/api/ontapi/zapi/client.go @@ -311,11 +311,11 @@ func (c *Client) InvokeZapiCall(request *node.Node) ([]*node.Node, error) { // this method should only be called after building the request func (c *Client) Invoke(testFilePath string) (*node.Node, error) { if testFilePath != "" { - if testData, err := tree.ImportXML(testFilePath); err == nil { - return testData, nil - } else { + testData, err := tree.ImportXML(testFilePath) + if err != nil { return nil, err } + return testData, nil } result, _, _, err := c.invokeWithAuthRetry(false) return result, err @@ -328,11 +328,11 @@ func (c *Client) Invoke(testFilePath string) (*node.Node, error) { // Use the returned tag for subsequent calls to this method func (c *Client) InvokeBatchRequest(request *node.Node, tag string, testFilePath string) (*node.Node, string, error) { if testFilePath != "" && tag != "" { - if testData, err := tree.ImportXML(testFilePath); err == nil { - return testData, "", nil - } else { + testData, err := tree.ImportXML(testFilePath) + if err != nil { return nil, "", err } + return testData, "", nil } // wasteful of course, need to rewrite later @TODO results, tag, _, _, err := c.InvokeBatchWithTimers(request, tag) @@ -401,11 +401,11 @@ func (c *Client) InvokeRequest(request *node.Node) (*node.Node, error) { // This method should only be called after building the request func (c *Client) InvokeWithTimers(testFilePath string) (*node.Node, time.Duration, time.Duration, error) { if testFilePath != "" { - if testData, err := tree.ImportXML(testFilePath); err == nil { - return testData, 0, 0, nil - } else { + testData, err := tree.ImportXML(testFilePath) + if err != nil { return nil, 0, 0, err } + return testData, 0, 0, nil } return c.invokeWithAuthRetry(true) } @@ -489,8 +489,8 @@ func (c *Client) invoke(withTimers bool) (*node.Node, time.Duration, time.Durati responseT = time.Since(start) } - if response.StatusCode != 200 { - if response.StatusCode == 401 { + if response.StatusCode != http.StatusOK { + if response.StatusCode == http.StatusUnauthorized { return result, responseT, parseT, errs.New(errs.ErrAuthFailed, response.Status, errs.WithStatus(response.StatusCode)) } return result, responseT, parseT, errs.New(errs.ErrAPIResponse, response.Status, errs.WithStatus(response.StatusCode)) diff --git a/pkg/conf/conf.go b/pkg/conf/conf.go index a95f810fb..116d655f3 100644 --- a/pkg/conf/conf.go +++ b/pkg/conf/conf.go @@ -175,9 +175,8 @@ func GetDefaultHarvestConfigPath() string { configPath := os.Getenv("HARVEST_CONF") if configPath == "" { return "./" + HarvestYML - } else { - return path.Join(configPath, HarvestYML) } + return path.Join(configPath, HarvestYML) } // GetHarvestHomePath returns the value of the env var HARVEST_CONF or ./ diff --git a/pkg/logging/logger.go b/pkg/logging/logger.go index 33a400b80..94fe3ddea 100644 --- a/pkg/logging/logger.go +++ b/pkg/logging/logger.go @@ -107,7 +107,7 @@ func Configure(config LogConfig) *Logger { multiWriters := zerolog.MultiLevelWriter(writers...) zerolog.SetGlobalLevel(config.LogLevel) - zerolog.ErrorStackMarshaler = MarshalStack + zerolog.ErrorStackMarshaler = MarshalStack //nolint:reassign zerolog.CallerMarshalFunc = ShortFile zeroLogger := zerolog.New(multiWriters).With().Caller().Str(config.PrefixKey, config.PrefixValue).Timestamp().Logger() diff --git a/pkg/test/node_test.go b/pkg/test/node_test.go index ebe08c8e8..98aebb9be 100644 --- a/pkg/test/node_test.go +++ b/pkg/test/node_test.go @@ -70,7 +70,7 @@ func TestNode_MergeCollector(t *testing.T) { // object name overwrite want := "customLun" - got := "" + var got string if name := defaultTemplate.GetChildS("name"); name != nil { got = name.GetContentS() if got != want { diff --git a/pkg/tree/node/node.go b/pkg/tree/node/node.go index 16ac91ebb..96893f82b 100644 --- a/pkg/tree/node/node.go +++ b/pkg/tree/node/node.go @@ -402,20 +402,18 @@ func (n *Node) printN(depth int, b *strings.Builder) { } func (n *Node) SearchContent(prefix []string, paths [][]string) ([]string, bool) { - - //fmt.Printf("SearchContent: prefix=%v \t paths=%v\n", prefix, paths) - - var search func(*Node, []string) - - matches := make([]string, 0) + var ( + search func(*Node, []string) + matches []string + ) search = func(node *Node, currentPath []string) { var newPath []string if len(currentPath) > 0 || prefix[0] == node.GetNameS() { - newPath = append(currentPath, node.GetNameS()) + newPath = currentPath + newPath = append(newPath, node.GetNameS()) } else { - newPath = make([]string, len(currentPath)) - copy(newPath, currentPath) + newPath = slices.Clone(currentPath) } //fmt.Printf(" -> current_path=%v \t new_path=%v\n", currentPath, newPath) for _, path := range paths { @@ -440,17 +438,18 @@ func (n *Node) SearchContent(prefix []string, paths [][]string) ([]string, bool) func (n *Node) SearchChildren(path []string) []*Node { - var search func(*Node, []string) - - matches := make([]*Node, 0) + var ( + search func(*Node, []string) + matches []*Node + ) search = func(node *Node, currentPath []string) { var newPath []string if len(currentPath) > 0 || path[0] == node.GetNameS() { - newPath = append(currentPath, node.GetNameS()) + newPath = currentPath + newPath = append(newPath, node.GetNameS()) } else { - newPath = make([]string, len(currentPath)) - copy(newPath, currentPath) + newPath = slices.Clone(currentPath) } if slices.Equal(newPath, path) { matches = append(matches, node) diff --git a/pkg/tree/node/node_test.go b/pkg/tree/node/node_test.go index 3f406506c..7c4616bb7 100644 --- a/pkg/tree/node/node_test.go +++ b/pkg/tree/node/node_test.go @@ -40,7 +40,7 @@ func TestNode_FlatList(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - list := make([]string, 0) + var list []string tt.tree.FlatList(&list, "") if len(list) != tt.count { t.Errorf("flat list has size= %v, want %v", len(list), tt.count) diff --git a/pkg/util/util.go b/pkg/util/util.go index 8c63b2fcb..5d51f30e2 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -388,9 +388,8 @@ func HasDuplicates(slice []string) bool { for _, v := range slice { if encountered[v] { return true - } else { - encountered[v] = true } + encountered[v] = true } return false From 9c00fb8f8f2cb643336635d95e7c3fae16c25516 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Mon, 21 Aug 2023 14:23:27 -0400 Subject: [PATCH 39/40] feat: enable more golanglint linters --- .golangci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index 8a848b94f..22bab4fd3 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -30,7 +30,7 @@ linters: - staticcheck - stylecheck - tenv - - tenv + - thelper - testableexamples - typecheck - unconvert From ecaaa0bab29f734d6b04ab39a400c809187199a1 Mon Sep 17 00:00:00 2001 From: Chris Grindstaff Date: Tue, 22 Aug 2023 02:24:16 -0400 Subject: [PATCH 40/40] ci: create draft release highlights (#2314) --- .github/ISSUE_TEMPLATE/release-template.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release-template.md b/.github/ISSUE_TEMPLATE/release-template.md index 87305d6a2..3d75d13bb 100644 --- a/.github/ISSUE_TEMPLATE/release-template.md +++ b/.github/ISSUE_TEMPLATE/release-template.md @@ -45,7 +45,7 @@ git push origin release/$RELEASE - [ ] Create changelog - [ ] [Draft a new release](https://github.com/NetApp/harvest/releases). Use `v$RELEASE` for the tag and pick the release/$RELEASE branch. Click the `Generate release notes` button and double check, at the bottom of the release notes, that the commits are across the correct range. For example: `https://github.com/NetApp/harvest/compare/v22.11.1...v23.02.0` - [ ] Copy/paste the generated release notes and save them in a file `pbpaste > ghrn_$RELEASE.md` - - [ ] Hand-write list of release highlights `vi highlights_$RELEASE.md` ([example content](https://github.com/NetApp/harvest/blob/main/CHANGELOG.md#23020--2023-02-21)) + - [ ] Generate draft release highlights by executing `go run pkg/changelog/main.go new`. This will create a file named `releaseHighlights_$RELEASE.md`. Edit that file to add content ([example content](https://github.com/NetApp/harvest/blob/main/CHANGELOG.md#23020--2023-02-21)) - [ ] Ensure all notable features are highlighted - [ ] Ensure any breaking changes are highlighted - [ ] Ensure any deprecations are highlighted @@ -91,4 +91,4 @@ bin/harvest generate metrics --poller POLLERNAME ```bash mike deploy --push --update-aliases $SHORT latest ``` -- [ ] Merge Release Branch into Main +- [ ] Merge release branch into main. The PR should use the `chore: ` prefix