From 9ce2b0e34406f1d0d61a12c52bc4bf537b5e109b Mon Sep 17 00:00:00 2001 From: Marvin Date: Wed, 11 Mar 2020 09:58:29 -0500 Subject: [PATCH] Implement getclusterfullthreshold (#7) * initial commit for implementing GetClusterFullThreshold * added metrics for GetClusterFullThreshold api call --- pkg/prom/collector.go | 199 +++++++++++++++++++++ pkg/prom/metrics.go | 137 ++++++++++++++ pkg/solidfire/solidfire.go | 22 +++ pkg/solidfire/solidfire_test.go | 45 +++++ pkg/solidfire/types.go | 27 +++ test/fixtures/getclusterfullthreshold.json | 22 +++ 6 files changed, 452 insertions(+) create mode 100644 test/fixtures/getclusterfullthreshold.json diff --git a/pkg/prom/collector.go b/pkg/prom/collector.go index 77989d9..bd383f6 100644 --- a/pkg/prom/collector.go +++ b/pkg/prom/collector.go @@ -3,6 +3,7 @@ package prom import ( "math" "strconv" + "strings" "sync" log "github.com/amoghe/distillog" @@ -31,6 +32,13 @@ func sumHistogram(m map[float64]uint64) (r uint64) { return } +func strCompare(str1 string, str2 string) int { + if strings.Compare(strings.ToLower(str1), strings.ToLower(str2)) == 0 { + return 1 + } + return 0 +} + func (c *solidfireCollector) Describe(ch chan<- *prometheus.Desc) { ch <- MetricDescriptions.ScrapeSuccessDesc @@ -135,6 +143,23 @@ func (c *solidfireCollector) Describe(ch chan<- *prometheus.Desc) { ch <- MetricDescriptions.ClusterStatsWriteOps ch <- MetricDescriptions.ClusterStatsWriteOpsLastSample + ch <- MetricDescriptions.ClusterThresholdBlockFullness + ch <- MetricDescriptions.ClusterThresholdFullness + ch <- MetricDescriptions.ClusterThresholdMaxMetadataOverProvisionFactor + ch <- MetricDescriptions.ClusterThresholdMetadataFullness + ch <- MetricDescriptions.ClusterThresholdSliceReserveUsedThresholdPct + ch <- MetricDescriptions.ClusterThresholdStage2AwareThreshold + ch <- MetricDescriptions.ClusterThresholdStage2BlockThresholdBytes + ch <- MetricDescriptions.ClusterThresholdStage3BlockThresholdBytes + ch <- MetricDescriptions.ClusterThresholdStage3BlockThresholdPercent + ch <- MetricDescriptions.ClusterThresholdStage3LowThreshold + ch <- MetricDescriptions.ClusterThresholdStage4BlockThresholdBytes + ch <- MetricDescriptions.ClusterThresholdStage4CriticalThreshold + ch <- MetricDescriptions.ClusterThresholdStage5BlockThresholdBytes + ch <- MetricDescriptions.ClusterThresholdSumTotalClusterBytes + ch <- MetricDescriptions.ClusterThresholdSumTotalMetadataClusterBytes + ch <- MetricDescriptions.ClusterThresholdSumUsedClusterBytes + ch <- MetricDescriptions.ClusterThresholdSumUsedMetadataClusterBytes } func (c *solidfireCollector) Collect(ch chan<- prometheus.Metric) { @@ -946,6 +971,180 @@ func (c *solidfireCollector) Collect(ch chan<- prometheus.Metric) { clusterStats.Result.ClusterStats.WriteOpsLastSample, ) + clusterFullThreshold, err := c.client.GetClusterFullThreshold() + if err != nil { + scrapeSuccess = 0 + log.Errorln(err) + } + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdBlockFullness, + prometheus.GaugeValue, + float64(strCompare(clusterFullThreshold.Result.BlockFullness, "stage1Happy")), + "stage1Happy", + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdBlockFullness, + prometheus.GaugeValue, + float64(strCompare(clusterFullThreshold.Result.BlockFullness, "stage2Aware")), + "stage2Aware", + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdBlockFullness, + prometheus.GaugeValue, + float64(strCompare(clusterFullThreshold.Result.BlockFullness, "stage3Low")), + "stage3Low", + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdBlockFullness, + prometheus.GaugeValue, + float64(strCompare(clusterFullThreshold.Result.BlockFullness, "stage4Critical")), + "stage4Critical", + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdBlockFullness, + prometheus.GaugeValue, + float64(strCompare(clusterFullThreshold.Result.BlockFullness, "stage5CompletelyConsumed")), + "stage5CompletelyConsumed", + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdFullness, + prometheus.GaugeValue, + float64(strCompare(clusterFullThreshold.Result.Fullness, "blockFullness")), + "blockFullness", + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdFullness, + prometheus.GaugeValue, + float64(strCompare(clusterFullThreshold.Result.Fullness, "metadataFullness")), + "metadataFullness", + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdMaxMetadataOverProvisionFactor, + prometheus.GaugeValue, + clusterFullThreshold.Result.MaxMetadataOverProvisionFactor, + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdMetadataFullness, + prometheus.GaugeValue, + float64(strCompare(clusterFullThreshold.Result.MetadataFullness, "stage1Happy")), + "stage1Happy", + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdMetadataFullness, + prometheus.GaugeValue, + float64(strCompare(clusterFullThreshold.Result.MetadataFullness, "stage2Aware")), + "stage2Aware", + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdMetadataFullness, + prometheus.GaugeValue, + float64(strCompare(clusterFullThreshold.Result.MetadataFullness, "stage3Low")), + "stage3Low", + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdMetadataFullness, + prometheus.GaugeValue, + float64(strCompare(clusterFullThreshold.Result.MetadataFullness, "stage4Critical")), + "stage4Critical", + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdMetadataFullness, + prometheus.GaugeValue, + float64(strCompare(clusterFullThreshold.Result.MetadataFullness, "stage5CompletelyConsumed")), + "stage5CompletelyConsumed", + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdSliceReserveUsedThresholdPct, + prometheus.GaugeValue, + clusterFullThreshold.Result.SliceReserveUsedThresholdPct, + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdStage2AwareThreshold, + prometheus.GaugeValue, + clusterFullThreshold.Result.Stage2AwareThreshold, + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdStage2BlockThresholdBytes, + prometheus.GaugeValue, + clusterFullThreshold.Result.Stage2BlockThresholdBytes, + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdStage3BlockThresholdBytes, + prometheus.GaugeValue, + clusterFullThreshold.Result.Stage3BlockThresholdBytes, + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdStage3BlockThresholdPercent, + prometheus.GaugeValue, + clusterFullThreshold.Result.Stage3BlockThresholdPercent, + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdStage3LowThreshold, + prometheus.GaugeValue, + clusterFullThreshold.Result.Stage3LowThreshold, + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdStage4BlockThresholdBytes, + prometheus.GaugeValue, + clusterFullThreshold.Result.Stage4BlockThresholdBytes, + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdStage4CriticalThreshold, + prometheus.GaugeValue, + clusterFullThreshold.Result.Stage4CriticalThreshold, + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdStage5BlockThresholdBytes, + prometheus.GaugeValue, + clusterFullThreshold.Result.Stage5BlockThresholdBytes, + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdSumTotalClusterBytes, + prometheus.GaugeValue, + clusterFullThreshold.Result.SumTotalClusterBytes, + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdSumTotalMetadataClusterBytes, + prometheus.GaugeValue, + clusterFullThreshold.Result.SumTotalMetadataClusterBytes, + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdSumUsedClusterBytes, + prometheus.GaugeValue, + clusterFullThreshold.Result.SumUsedClusterBytes, + ) + + ch <- prometheus.MustNewConstMetric( + MetricDescriptions.ClusterThresholdSumUsedMetadataClusterBytes, + prometheus.GaugeValue, + clusterFullThreshold.Result.SumUsedMetadataClusterBytes, + ) + // Set scrape success metric to scrapeSuccess ch <- prometheus.MustNewConstMetric(MetricDescriptions.ScrapeSuccessDesc, prometheus.GaugeValue, scrapeSuccess) } diff --git a/pkg/prom/metrics.go b/pkg/prom/metrics.go index cc2e41d..dfc5bb3 100644 --- a/pkg/prom/metrics.go +++ b/pkg/prom/metrics.go @@ -124,6 +124,24 @@ type Descriptions struct { ClusterStatsWriteLatencyUSecTotal *prometheus.Desc ClusterStatsWriteOps *prometheus.Desc ClusterStatsWriteOpsLastSample *prometheus.Desc + + ClusterThresholdBlockFullness *prometheus.Desc + ClusterThresholdFullness *prometheus.Desc + ClusterThresholdMaxMetadataOverProvisionFactor *prometheus.Desc + ClusterThresholdMetadataFullness *prometheus.Desc + ClusterThresholdSliceReserveUsedThresholdPct *prometheus.Desc + ClusterThresholdStage2AwareThreshold *prometheus.Desc + ClusterThresholdStage2BlockThresholdBytes *prometheus.Desc + ClusterThresholdStage3BlockThresholdBytes *prometheus.Desc + ClusterThresholdStage3BlockThresholdPercent *prometheus.Desc + ClusterThresholdStage3LowThreshold *prometheus.Desc + ClusterThresholdStage4BlockThresholdBytes *prometheus.Desc + ClusterThresholdStage4CriticalThreshold *prometheus.Desc + ClusterThresholdStage5BlockThresholdBytes *prometheus.Desc + ClusterThresholdSumTotalClusterBytes *prometheus.Desc + ClusterThresholdSumTotalMetadataClusterBytes *prometheus.Desc + ClusterThresholdSumUsedClusterBytes *prometheus.Desc + ClusterThresholdSumUsedMetadataClusterBytes *prometheus.Desc } func NewMetricDescriptions(namespace string) *Descriptions { @@ -818,5 +836,124 @@ func NewMetricDescriptions(namespace string) *Descriptions { nil, ) + d.ClusterThresholdBlockFullness = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_block_fullness"), + "The current computed level of block fullness of the cluster.", + []string{"level"}, + nil, + ) + + d.ClusterThresholdFullness = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_fullness"), + "Reflects the highest level of fullness between 'blockFullness' and 'metadataFullness'.", + []string{"level"}, + nil, + ) + + d.ClusterThresholdMaxMetadataOverProvisionFactor = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_max_metadata_over_provision_factor"), + "A value representative of the number of times metadata space can be over provisioned relative to the amount of space available.", + nil, + nil, + ) + + d.ClusterThresholdMetadataFullness = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_metadata_fullness"), + "The current computed level of metadata fullness of the cluster.", + []string{"level"}, + nil, + ) + + d.ClusterThresholdSliceReserveUsedThresholdPct = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_slice_reserve_used_threshold_percentage"), + "Error condition. A system alert is triggered if the reserved slice utilization is greater than the sliceReserveUsedThresholdPct value returned.", + nil, + nil, + ) + + d.ClusterThresholdStage2AwareThreshold = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_stage2_aware_threshold"), + "Awareness condition. The value that is set for 'Stage 2' cluster threshold level.", + nil, + nil, + ) + + d.ClusterThresholdStage2BlockThresholdBytes = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_stage2_block_threshold_bytes"), + "Number of bytes being used by the cluster at which a stage2 condition will exist.", + nil, + nil, + ) + + d.ClusterThresholdStage3BlockThresholdBytes = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_stage3_block_threshold_bytes"), + "Number of bytes being used by the cluster at which a stage3 condition will exist.", + nil, + nil, + ) + + d.ClusterThresholdStage3BlockThresholdPercent = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_stage3_block_threshold_percentage"), + "Percent value set for stage3. At this percent full, a warning is posted in the Alerts log.", + nil, + nil, + ) + + d.ClusterThresholdStage3LowThreshold = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_stage3_low_threshold"), + "Error condition. The threshold at which a system alert is created due to low capacity on a cluster", + nil, + nil, + ) + + d.ClusterThresholdStage4BlockThresholdBytes = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_stage4_block_threshold_bytes"), + "Number of bytes being used by the cluster at which a stage4 condition will exist", + nil, + nil, + ) + + d.ClusterThresholdStage4CriticalThreshold = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_stage4_critical_threshold"), + "Error condition. The threshold at which a system alert is created to warn about critically low capacity on a cluster.", + nil, + nil, + ) + + d.ClusterThresholdStage5BlockThresholdBytes = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_stage5_block_threshold_bytes"), + "The number of bytes being used by the cluster at which a stage5 condition will exist.", + nil, + nil, + ) + + d.ClusterThresholdSumTotalClusterBytes = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_sum_total_cluster_bytes"), + "Physical capacity of the cluster, measured in bytes.", + nil, + nil, + ) + + d.ClusterThresholdSumTotalMetadataClusterBytes = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_sum_total_metadata_cluster_bytes"), + "Total amount of space that can be used to store metadata", + nil, + nil, + ) + + d.ClusterThresholdSumUsedClusterBytes = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_sum_used_cluster_bytes"), + "Number of bytes used on the cluster.", + nil, + nil, + ) + + d.ClusterThresholdSumUsedMetadataClusterBytes = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "cluster_threshold_sum_used_metadata_cluster_bytes"), + "Amount of space used on volume drives to store metadata.", + nil, + nil, + ) + return &d } diff --git a/pkg/solidfire/solidfire.go b/pkg/solidfire/solidfire.go index 6b9a3ff..95dd162 100644 --- a/pkg/solidfire/solidfire.go +++ b/pkg/solidfire/solidfire.go @@ -239,3 +239,25 @@ func (s *Client) GetClusterStats() (GetClusterStatsResponse, error) { } return r, nil } + +func (s *Client) GetClusterFullThreshold() (GetClusterFullThresholdResponse, error) { + payload := &RPCBody{ + Method: "GetClusterFullThreshold", + Params: GetClusterFullThresholdParams{}, + ID: 1, + } + + payloadBytes, err := json.Marshal(&payload) + r := GetClusterFullThresholdResponse{} + bodyBytes, err := doRpcCall(s, payloadBytes) + + if err != nil { + return r, err + } + err = json.Unmarshal(bodyBytes, &r) + + if err != nil { + return r, err + } + return r, nil +} diff --git a/pkg/solidfire/solidfire_test.go b/pkg/solidfire/solidfire_test.go index b1b4ba3..573a51f 100644 --- a/pkg/solidfire/solidfire_test.go +++ b/pkg/solidfire/solidfire_test.go @@ -395,3 +395,48 @@ func TestClient_GetClusterStats(t *testing.T) { }) } } + +func TestClient_GetClusterFullThreshold(t *testing.T) { + fixture, err := ioutil.ReadFile("../../test/fixtures/getclusterfullthreshold.json") + if err != nil { + panic(err) + } + tests := []struct { + name string + s solidfire.Client + want float64 + wantErr bool + }{ + { + name: "GetClusterFullThreshold Response should match fixture", + want: 3, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer gock.Off() + gock.Observe(gock.DumpRequest) + gock.New(sfHost). + Post(sfRPCEndpoint). + MatchType("json"). + JSON(solidfire.RPCBody{ + ID: 1, + Method: "GetClusterFullThreshold", + Params: solidfire.GetClusterFullThresholdParams{}}). + Reply(200). + BodyString(string(fixture)) + + gotRaw, err := sfClient.GetClusterFullThreshold() + got := gotRaw.Result.Stage2AwareThreshold + + if (err != nil) != tt.wantErr { + t.Errorf("Client.ListAllNodes() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Client.ListAllNodes() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/solidfire/types.go b/pkg/solidfire/types.go index f94213f..bb7a827 100644 --- a/pkg/solidfire/types.go +++ b/pkg/solidfire/types.go @@ -48,6 +48,10 @@ type GetClusterStatsRPCParams struct { // No params needed } +type GetClusterFullThresholdParams struct { + // No params needed +} + type ListVolumesResponse struct { ID int `json:"id"` Result struct { @@ -360,3 +364,26 @@ type GetClusterStatsResponse struct { } `json:"clusterStats"` } `json:"result"` } + +type GetClusterFullThresholdResponse struct { + ID int `json:"id"` + Result struct { + BlockFullness string `json:"blockFullness"` + Fullness string `json:"fullness"` + MaxMetadataOverProvisionFactor float64 `json:"maxMetadataOverProvisionFactor"` + MetadataFullness string `json:"metadataFullness"` + SliceReserveUsedThresholdPct float64 `json:"sliceReserveUsedThresholdPct"` + Stage2AwareThreshold float64 `json:"stage2AwareThreshold"` + Stage2BlockThresholdBytes float64 `json:"stage2BlockThresholdBytes"` + Stage3BlockThresholdBytes float64 `json:"stage3BlockThresholdBytes"` + Stage3BlockThresholdPercent float64 `json:"stage3BlockThresholdPercent"` + Stage3LowThreshold float64 `json:"stage3LowThreshold"` + Stage4BlockThresholdBytes float64 `json:"stage4BlockThresholdBytes"` + Stage4CriticalThreshold float64 `json:"stage4CriticalThreshold"` + Stage5BlockThresholdBytes float64 `json:"stage5BlockThresholdBytes"` + SumTotalClusterBytes float64 `json:"sumTotalClusterBytes"` + SumTotalMetadataClusterBytes float64 `json:"sumTotalMetadataClusterBytes"` + SumUsedClusterBytes float64 `json:"sumUsedClusterBytes"` + SumUsedMetadataClusterBytes float64 `json:"sumUsedMetadataClusterBytes"` + } `json:"result"` +} diff --git a/test/fixtures/getclusterfullthreshold.json b/test/fixtures/getclusterfullthreshold.json new file mode 100644 index 0000000..91f01b0 --- /dev/null +++ b/test/fixtures/getclusterfullthreshold.json @@ -0,0 +1,22 @@ +{ + "id": 1, + "result": { + "blockFullness": "stage1Happy", + "fullness": "stage3Low", + "maxMetadataOverProvisionFactor": 5, + "metadataFullness": "stage3Low", + "sliceReserveUsedThresholdPct": 5, + "stage2AwareThreshold": 3, + "stage2BlockThresholdBytes": 2640607661261, + "stage3BlockThresholdBytes": 8281905846682, + "stage3BlockThresholdPercent": 5, + "stage3LowThreshold": 2, + "stage4BlockThresholdBytes": 8641988709581, + "stage4CriticalThreshold": 1, + "stage5BlockThresholdBytes": 12002762096640, + "sumTotalClusterBytes": 12002762096640, + "sumTotalMetadataClusterBytes": 404849531289, + "sumUsedClusterBytes": 45553617581, + "sumUsedMetadataClusterBytes": 31703113728 + } +} \ No newline at end of file