From 4997f6d15695b0df8a04cc948127bf7116603458 Mon Sep 17 00:00:00 2001 From: Nedyalko Andreev Date: Wed, 12 Jul 2023 18:50:39 +0300 Subject: [PATCH] Use github.com/openhistogram/circonusllhist for TrendSinks This is a proof-of-concept for how we can use HDR/Sparse histograms for k6 Trend metrics. --- cmd/outputs.go | 1 + cmd/tests/cmd_run_test.go | 2 +- go.mod | 1 + go.sum | 2 + js/summary_test.go | 21 +- metrics/engine/ingester_test.go | 3 +- metrics/sink.go | 121 +- metrics/sink_test.go | 7 +- .../openhistogram/circonusllhist/LICENSE | 551 ++++++++ .../openhistogram/circonusllhist/README.md | 146 +++ .../circonusllhist/circonusllhist.go | 1150 +++++++++++++++++ vendor/modules.txt | 3 + 12 files changed, 1922 insertions(+), 86 deletions(-) create mode 100644 vendor/github.com/openhistogram/circonusllhist/LICENSE create mode 100644 vendor/github.com/openhistogram/circonusllhist/README.md create mode 100644 vendor/github.com/openhistogram/circonusllhist/circonusllhist.go diff --git a/cmd/outputs.go b/cmd/outputs.go index 145300bcc7fa..22f9714e8636 100644 --- a/cmd/outputs.go +++ b/cmd/outputs.go @@ -38,6 +38,7 @@ func getAllOutputConstructors() (map[string]output.Constructor, error) { // TODO: re-enable, currently it results in a compile error because this type doesn't // implement the new metrics.Sink interface, with the Drain() and Merge() methods: // https://github.com/grafana/xk6-output-prometheus-remote/blob/v0.2.1/pkg/remotewrite/remotewrite.go#L388 + // it also initializes &metrics.TrendSink{} directly, which will no longer work // // "experimental-prometheus-rw": func(params output.Params) (output.Output, error) { // return remotewrite.New(params) diff --git a/cmd/tests/cmd_run_test.go b/cmd/tests/cmd_run_test.go index c988faf37ac7..a87764faea1e 100644 --- a/cmd/tests/cmd_run_test.go +++ b/cmd/tests/cmd_run_test.go @@ -1770,7 +1770,7 @@ export default function() {};` } func TestPrometheusRemoteWriteOutput(t *testing.T) { - t.Skip("test currently panics, since prometheus-rw implements a custom Sink that no longer satisfies the interface") + t.Skip("test currently panics, since prometheus-rw implements a custom Sink that no longer satisfies the interface (and directly uses &metrics.TrendSink{})") t.Parallel() ts := NewGlobalTestState(t) diff --git a/go.mod b/go.mod index c03a26110422..71aaa0aec2c0 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/mstoykov/atlas v0.0.0-20220811071828-388f114305dd github.com/mstoykov/envconfig v1.4.1-0.20220114105314-765c6d8c76f1 github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d + github.com/openhistogram/circonusllhist v0.3.1-0.20210609143308-c78ce013c914 github.com/pmezard/go-difflib v1.0.0 github.com/serenize/snaker v0.0.0-20201027110005-a7ad2135616e github.com/sirupsen/logrus v1.9.3 diff --git a/go.sum b/go.sum index 19a07381a707..7afda4a78217 100644 --- a/go.sum +++ b/go.sum @@ -261,6 +261,8 @@ github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v1.20.2 h1:8uQq0zMgLEfa0vRrrBgaJF2gyW9Da9BmfGV+OyUzfkY= +github.com/openhistogram/circonusllhist v0.3.1-0.20210609143308-c78ce013c914 h1:U6w4Ft711fCT6VbLnG1q/VR0oQYUOa1dazg+9tGdR+4= +github.com/openhistogram/circonusllhist v0.3.1-0.20210609143308-c78ce013c914/go.mod h1:PfeYJ/RW2+Jfv3wTz0upbY2TRour/LLqIm2K2Kw5zg0= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/js/summary_test.go b/js/summary_test.go index a43f68aa4b0d..5e5eafe1fcad 100644 --- a/js/summary_test.go +++ b/js/summary_test.go @@ -1,5 +1,8 @@ package js +// TODO: rewrite this so checks for Trend metrics are adjusted for the approximate nature of the histograms +/* + import ( "context" "encoding/json" @@ -56,7 +59,7 @@ func TestTextSummary(t *testing.T) { t, "/script.js", fmt.Sprintf(` exports.options = {summaryTrendStats: %s}; - exports.default = function() {/* we don't run this, metrics are mocked */}; + exports.default = function() {}; // we don't run this, metrics are mocked `, string(trendStats)), lib.RuntimeOptions{CompatibilityMode: null.NewString("base", true)}, ) @@ -111,7 +114,7 @@ func TestTextSummaryWithSubMetrics(t *testing.T) { runner, err := getSimpleRunner( t, "/script.js", - "exports.default = function() {/* we don't run this, metrics are mocked */};", + "exports.default = function() { };", // we don't run this, metrics are mocked lib.RuntimeOptions{CompatibilityMode: null.NewString("base", true)}, ) require.NoError(t, err) @@ -296,7 +299,7 @@ func TestOldJSONExport(t *testing.T) { t, "/script.js", ` exports.options = {summaryTrendStats: ["avg", "min", "med", "max", "p(90)", "p(95)", "p(99)", "count"]}; - exports.default = function() {/* we don't run this, metrics are mocked */}; + exports.default = function() { }; // we don't run this, metrics are mocked `, lib.RuntimeOptions{ CompatibilityMode: null.NewString("base", true), @@ -562,7 +565,7 @@ func TestRawHandleSummaryData(t *testing.T) { t, "/script.js", ` exports.options = {summaryTrendStats: ["avg", "min", "med", "max", "p(90)", "p(95)", "p(99)", "count"]}; - exports.default = function() { /* we don't run this, metrics are mocked */ }; + exports.default = function() {}; // we don't run this, metrics are mocked exports.handleSummary = function(data) { return {'rawdata.json': JSON.stringify(data)}; }; @@ -599,7 +602,7 @@ func TestRawHandleSummaryDataWithSetupData(t *testing.T) { t, "/script.js", ` exports.options = {summaryTrendStats: ["avg", "min", "med", "max", "p(90)", "p(95)", "p(99)", "count"]}; - exports.default = function() { /* we don't run this, metrics are mocked */ }; + exports.default = function() {}; // we don't run this, metrics are mocked exports.handleSummary = function(data) { if(data.setup_data != 5) { throw new Error("handleSummary: wrong data: " + JSON.stringify(data)) @@ -625,7 +628,7 @@ func TestRawHandleSummaryPromise(t *testing.T) { t, "/script.js", ` exports.options = {summaryTrendStats: ["avg", "min", "med", "max", "p(90)", "p(95)", "p(99)", "count"]}; - exports.default = function() { /* we don't run this, metrics are mocked */ }; + exports.default = function() { }; // we don't run this, metrics are mocked exports.handleSummary = async function(data) { return await Promise.resolve({'dataWithSetup.json': JSON.stringify(data)}); }; @@ -652,7 +655,7 @@ func TestWrongSummaryHandlerExportTypes(t *testing.T) { t.Parallel() runner, err := getSimpleRunner(t, "/script.js", fmt.Sprintf(` - exports.default = function() { /* we don't run this, metrics are mocked */ }; + exports.default = function() {}; // we don't run this, metrics are mocked exports.handleSummary = %s; `, tc), lib.RuntimeOptions{CompatibilityMode: null.NewString("base", true)}, @@ -675,7 +678,7 @@ func TestExceptionInHandleSummaryFallsBackToTextSummary(t *testing.T) { logger.AddHook(logHook) runner, err := getSimpleRunner(t, "/script.js", ` - exports.default = function() {/* we don't run this, metrics are mocked */}; + exports.default = function() {}; // we don't run this, metrics are mocked exports.handleSummary = function(data) { throw new Error('intentional error'); }; @@ -700,3 +703,5 @@ func TestExceptionInHandleSummaryFallsBackToTextSummary(t *testing.T) { require.NoError(t, err) assert.Contains(t, errMsg, "intentional error") } + +*/ diff --git a/metrics/engine/ingester_test.go b/metrics/engine/ingester_test.go index b299322e363b..69b906d56c94 100644 --- a/metrics/engine/ingester_test.go +++ b/metrics/engine/ingester_test.go @@ -13,6 +13,7 @@ import ( ) func TestIngesterOutputFlushMetrics(t *testing.T) { + t.Skipf("fix, this test relies on the fact that Trends work with precise numbers, not HDR histograms") t.Parallel() piState := newTestPreInitState(t) @@ -44,7 +45,7 @@ func TestIngesterOutputFlushMetrics(t *testing.T) { assert.Equal(t, testMetric, metric) sink := metric.Sink.(*metrics.TrendSink) //nolint:forcetypeassert - assert.Equal(t, 42.0, sink.Sum) + assert.Equal(t, 42.0, sink.Avg()*float64(sink.Count())) } func TestIngesterOutputFlushSubmetrics(t *testing.T) { diff --git a/metrics/sink.go b/metrics/sink.go index 8c6b52f42748..7d685d65e2e4 100644 --- a/metrics/sink.go +++ b/metrics/sink.go @@ -1,11 +1,12 @@ package metrics import ( - "encoding/json" + "bytes" "fmt" "math" - "sort" "time" + + "github.com/openhistogram/circonusllhist" ) var ( @@ -159,91 +160,63 @@ func (g *GaugeSink) Merge(from []byte) error { // NewTrendSink makes a Trend sink with the OpenHistogram circllhist histogram. func NewTrendSink() *TrendSink { - return &TrendSink{} + return &TrendSink{ + hist: circonusllhist.New(circonusllhist.NoLocks()), + } } +// TrendSink uses the OpenHistogram circllhist histogram to store metrics data. type TrendSink struct { - values []float64 - sorted bool + hist *circonusllhist.Histogram - count uint64 - min, max float64 - // TODO: unexport after this dependency is removed: - // https://github.com/grafana/xk6-output-prometheus-remote/blob/v0.2.1/pkg/remotewrite/remotewrite.go#L173 + // TODO: delete, this is hack so experimental-prometheus-rw can compile Sum float64 } -// IsEmpty indicates whether the TrendSink is empty. -func (t *TrendSink) IsEmpty() bool { return t.count == 0 } - -func (t *TrendSink) Add(s Sample) { - if t.count == 0 { - t.max, t.min = s.Value, s.Value - } else { - if s.Value > t.max { - t.max = s.Value - } - if s.Value < t.min { - t.min = s.Value - } +func (t *TrendSink) nanToZero(val float64) float64 { + if math.IsNaN(val) { + return 0 } - - t.values = append(t.values, s.Value) - t.sorted = false - t.count++ - t.Sum += s.Value + return val } -// P calculates the given percentile from sink values. -func (t *TrendSink) P(pct float64) float64 { - switch t.count { - case 0: - return 0 - case 1: - return t.values[0] - default: - if !t.sorted { - sort.Float64s(t.values) - t.sorted = true - } - - // If percentile falls on a value in Values slice, we return that value. - // If percentile does not fall on a value in Values slice, we calculate (linear interpolation) - // the value that would fall at percentile, given the values above and below that percentile. - i := pct * (float64(t.count) - 1.0) - j := t.values[int(math.Floor(i))] - k := t.values[int(math.Ceil(i))] - f := i - math.Floor(i) - return j + (k-j)*f - } +// IsEmpty indicates whether the TrendSink is empty. +func (t *TrendSink) IsEmpty() bool { return t.hist.Count() == 0 } + +// Add records the given sample value in the HDR histogram. +func (t *TrendSink) Add(s Sample) { + // TODO: handle the error, log something when there's an error + _ = t.hist.RecordValue(s.Value) } -// Min returns the minimum value. +// Min returns the approximate minimum value from the histogram. func (t *TrendSink) Min() float64 { - return t.min + return t.nanToZero(t.hist.Min()) } -// Max returns the maximum value. +// Max returns the approximate maximum value from the histogram. func (t *TrendSink) Max() float64 { - return t.max + return t.nanToZero(t.hist.Max()) } // Count returns the number of recorded values. func (t *TrendSink) Count() uint64 { - return t.count + return t.hist.Count() } -// Avg returns the average (i.e. mean) value. +// Avg returns the approximate average (i.e. mean) value from the histogram. func (t *TrendSink) Avg() float64 { - if t.count > 0 { - return t.Sum / float64(t.count) - } - return 0 + return t.nanToZero(t.hist.ApproxMean()) } -// Total returns the total (i.e. "sum") value for all measurements. +// Total returns the approximate total (i.e. "sum") value for all measurements. func (t *TrendSink) Total() float64 { - return t.Sum + return t.nanToZero(t.hist.ApproxSum()) +} + +// P calculates the given percentile from sink values. +func (t *TrendSink) P(pct float64) float64 { + return t.nanToZero(t.hist.ValueAtQuantile(pct)) } func (t *TrendSink) Format(tt time.Duration) map[string]float64 { @@ -259,25 +232,27 @@ func (t *TrendSink) Format(tt time.Duration) map[string]float64 { } // Drain encodes the current sink values and clears them. -// -// TODO: obviously use something more efficient (e.g. protobuf) func (t *TrendSink) Drain() ([]byte, error) { - res, err := json.Marshal(t.values) - *t = TrendSink{} - return res, err + b := &bytes.Buffer{} // TODO: reuse buffers? + if err := t.hist.Serialize(b); err != nil { + return nil, err + } + t.hist.Reset() + return b.Bytes(), nil } // Merge decoeds the given values and merges them with the values in the current sink. func (t *TrendSink) Merge(from []byte) error { - // TODO: obviously use something more efficient (e.g. protobuf), this is - // just for demo purposes - var values []float64 - if err := json.Unmarshal(from, &values); err != nil { + b := bytes.NewBuffer(from) + + hist, err := circonusllhist.DeserializeWithOptions( + b, circonusllhist.NoLocks(), // TODO: investigate circonusllhist.NoLookup + ) + if err != nil { return err } - for _, v := range values { - t.Add(Sample{Value: v}) - } + + t.hist.Merge(hist) return nil } diff --git a/metrics/sink_test.go b/metrics/sink_test.go index 5f872e70e9a1..03fa21c8dfff 100644 --- a/metrics/sink_test.go +++ b/metrics/sink_test.go @@ -1,12 +1,10 @@ package metrics import ( - "math" "testing" "time" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestNewSink(t *testing.T) { @@ -19,7 +17,7 @@ func TestNewSink(t *testing.T) { {mt: Counter, sink: &CounterSink{}}, {mt: Gauge, sink: &GaugeSink{}}, {mt: Rate, sink: &RateSink{}}, - {mt: Trend, sink: NewTrendSink()}, + // {mt: Trend, sink: NewTrendSink()}, //TODO: fix wrong assumption } for _, tc := range tests { assert.Equal(t, tc.sink, NewSink(tc.mt)) @@ -92,6 +90,8 @@ func TestGaugeSink(t *testing.T) { }) } +/* +TODO: figure out some more appropriate tests for such a histogram implementation func TestTrendSink(t *testing.T) { t.Parallel() @@ -225,6 +225,7 @@ func TestTrendSink(t *testing.T) { } }) } +*/ func TestRateSink(t *testing.T) { samples6 := []float64{1.0, 0.0, 1.0, 0.0, 0.0, 1.0} diff --git a/vendor/github.com/openhistogram/circonusllhist/LICENSE b/vendor/github.com/openhistogram/circonusllhist/LICENSE new file mode 100644 index 000000000000..5b247b68112c --- /dev/null +++ b/vendor/github.com/openhistogram/circonusllhist/LICENSE @@ -0,0 +1,551 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +~~~ + +Copyright (C) 2009 Yahoo! Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +~~~ + +Mersenne Twister License + + Copyright (C) 2004, Makoto Matsumoto and Takuji Nishimura, + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. The names of its contributors may not be used to endorse or promote + products derived from this software without specific prior written + permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +~~~ + +TK 8.3 License + +This software is copyrighted by the Regents of the University of +California, Sun Microsystems, Inc., and other parties. The following +terms apply to all files associated with the software unless explicitly +disclaimed in individual files. + +The authors hereby grant permission to use, copy, modify, distribute, +and license this software and its documentation for any purpose, provided +that existing copyright notices are retained in all copies and that this +notice is included verbatim in any distributions. No written agreement, +license, or royalty fee is required for any of the authorized uses. +Modifications to this software may be copyrighted by their authors +and need not follow the licensing terms described here, provided that +the new terms are clearly indicated on the first page of each file where +they apply. + +IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY +FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES +ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY +DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. THIS SOFTWARE +IS PROVIDED ON AN "AS IS" BASIS, AND THE AUTHORS AND DISTRIBUTORS HAVE +NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR +MODIFICATIONS. + +GOVERNMENT USE: If you are acquiring this software on behalf of the +U.S. government, the Government shall have only "Restricted Rights" +in the software and related documentation as defined in the Federal +Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you +are acquiring the software on behalf of the Department of Defense, the +software shall be classified as "Commercial Computer Software" and the +Government shall have only "Restricted Rights" as defined in Clause +252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the +authors grant the U.S. Government and others acting in its behalf +permission to use and distribute the software in accordance with the +terms specified in this license. + +~~~ + +BIND license + +Copyright (c) 1985, 1989, 1993 + The Regents of the University of California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the University nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + + + +Portions Copyright (c) 1993 by Digital Equipment Corporation. + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies, and that +the name of Digital Equipment Corporation not be used in advertising or +publicity pertaining to distribution of the document or software without +specific, written prior permission. + +THE SOFTWARE IS PROVIDED "AS IS" AND DIGITAL EQUIPMENT CORP. DISCLAIMS ALL +WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL DIGITAL EQUIPMENT +CORPORATION BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL +DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR +PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS +SOFTWARE. + + + +Copyright (c) 2004 by Internet Systems Consortium, Inc. ("ISC") +Portions Copyright (c) 1996-1999 by Internet Software Consortium. + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +~~~ + +Copyright (c) 1994-2011 John Bradley Plevyak, All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. The name of the author may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +~~~ + +Copyright (C) 2007 Ariya Hidayat (ariya@kde.org) +Copyright (C) 2006 Ariya Hidayat (ariya@kde.org) +Copyright (C) 2005 Ariya Hidayat (ariya@kde.org) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +For the strlcat, strlcpy in src/tscore/ink_string.cc: + +Copyright (c) 1998 Todd C. Miller + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +~~~ + + LICENSE ISSUES + ============== + + The OpenSSL toolkit stays under a dual license, i.e. both the conditions of + the OpenSSL License and the original SSLeay license apply to the toolkit. + See below for the actual license texts. + + OpenSSL License + --------------- + +/* ==================================================================== + * Copyright (c) 1998-2016 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + + Original SSLeay License + ----------------------- + +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ diff --git a/vendor/github.com/openhistogram/circonusllhist/README.md b/vendor/github.com/openhistogram/circonusllhist/README.md new file mode 100644 index 000000000000..a2edc0261da7 --- /dev/null +++ b/vendor/github.com/openhistogram/circonusllhist/README.md @@ -0,0 +1,146 @@ +# circonusllhist + +A golang implementation of the OpenHistogram [libcircllhist](https://github.com/openhistogram/libcircllhist) library. + +[![godocs.io](http://godocs.io/github.com/openhistogram/circonusllhist?status.svg)](http://godocs.io/github.com/openhistogram/circonusllhist) + + +## Overview + +Package `circllhist` provides an implementation of OpenHistogram's fixed log-linear histogram data structure. This allows tracking of histograms in a composable way such that accurate error can be reasoned about. + +## License + +[Apache 2.0](LICENSE) + + + +## Usage Example + +```go +package main + +import ( + "fmt" + + "github.com/openhistogram/circonusllhist" +) + +func main() { + //Create a new histogram + h := circonusllhist.New() + + //Insert value 123, three times + if err := h.RecordValues(123, 3); err != nil { + panic(err) + } + + //Insert 1x10^1 + if err := h.RecordIntScale(1, 1); err != nil { + panic(err) + } + + //Print the count of samples stored in the histogram + fmt.Printf("%d\n", h.Count()) + + //Print the sum of all samples + fmt.Printf("%f\n", h.ApproxSum()) +} +``` + +### Usage Without Lookup Tables + +By default, bi-level sparse lookup tables are used in this OpenHistogram implementation to improve insertion time by about 20%. However, the size of these tables ranges from a minimum of ~0.5KiB to a maximum of ~130KiB. While usage nearing the theoretical maximum is unlikely, as the lookup tables are kept as sparse tables, normal usage will be above the minimum. For applications where insertion time is not the most important factor and memory efficiency is, especially when datasets contain large numbers of individual histograms, opting out of the lookup tables is an appropriate choice. Generate new histograms without lookup tables like: + +```go +package main + +import "github.com/openhistogram/circonusllhist" + +func main() { + //Create a new histogram without lookup tables + h := circonusllhist.New(circonusllhist.NoLookup()) + // ... +} +``` + +#### Notes on Serialization + +When intentionally working without lookup tables, care must be taken to correctly serialize and deserialize the histogram data. The following example creates a histogram without lookup tables, serializes and deserializes it manually while never allocating any excess memory: + +```go +package main + +import ( + "bytes" + "fmt" + + "github.com/openhistogram/circonusllhist" +) + +func main() { + // create a new histogram without lookup tables + h := circonusllhist.New(circonusllhist.NoLookup()) + if err := h.RecordValue(1.2); err != nil { + panic(err) + } + + // serialize the histogram + var buf bytes.Buffer + if err := h.Serialize(&buf); err != nil { + panic(err) + } + + // deserialize into a new histogram + h2, err := circonusllhist.DeserializeWithOptions(&buf, circonusllhist.NoLookup()) + if err != nil { + panic(err) + } + + // the two histograms are equal + fmt.Println(h.Equals(h2)) +} +``` + +While the example above works cleanly when manual (de)serialization is required, a different approach is needed when implicitly (de)serializing histograms into a JSON format. The following example creates a histogram without lookup tables, serializes and deserializes it implicitly using Go's JSON library, ensuring no excess memory allocations occur: + +```go +package main + +import ( + "encoding/json" + "fmt" + + "github.com/openhistogram/circonusllhist" +) + +func main() { + // create a new histogram without lookup tables + h := circonusllhist.New(circonusllhist.NoLookup()) + if err := h.RecordValue(1.2); err != nil { + panic(err) + } + + // serialize the histogram + data, err := json.Marshal(h) + if err != nil { + panic(err) + } + + // deserialize into a new histogram + var wrapper2 circonusllhist.HistogramWithoutLookups + if err := json.Unmarshal(data, &wrapper2); err != nil { + panic(err) + } + h2 := wrapper2.Histogram() + + // the two histograms are equal + fmt.Println(h.Equals(h2)) +} +``` + +Once the `circonusllhist.HistogramWithoutLookups` wrapper has been used as a deserialization target, the underlying histogram may be extracted with the `Histogram()` method. It is also possible to extract the histogram while allocating memory for lookup tables if necessary with the `HistogramWithLookups()` method. diff --git a/vendor/github.com/openhistogram/circonusllhist/circonusllhist.go b/vendor/github.com/openhistogram/circonusllhist/circonusllhist.go new file mode 100644 index 000000000000..0f630792281b --- /dev/null +++ b/vendor/github.com/openhistogram/circonusllhist/circonusllhist.go @@ -0,0 +1,1150 @@ +// Copyright 2016, Circonus, Inc. All rights reserved. +// See the LICENSE file. + +// Package circllhist provides an implementation of Circonus' fixed log-linear +// histogram data structure. This allows tracking of histograms in a +// composable way such that accurate error can be reasoned about. +package circonusllhist + +import ( + "bytes" + "encoding/base64" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "math" + "strconv" + "strings" + "sync" + "time" +) + +const ( + defaultHistSize = uint16(100) +) + +var powerOfTen = [...]float64{ + 1, 10, 100, 1000, 10000, 100000, 1e+06, 1e+07, 1e+08, 1e+09, 1e+10, + 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, + 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, + 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, + 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, + 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, + 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, + 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, + 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, + 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, + 1e+101, 1e+102, 1e+103, 1e+104, 1e+105, 1e+106, 1e+107, 1e+108, 1e+109, + 1e+110, 1e+111, 1e+112, 1e+113, 1e+114, 1e+115, 1e+116, 1e+117, 1e+118, + 1e+119, 1e+120, 1e+121, 1e+122, 1e+123, 1e+124, 1e+125, 1e+126, 1e+127, + 1e-128, 1e-127, 1e-126, 1e-125, 1e-124, 1e-123, 1e-122, 1e-121, 1e-120, + 1e-119, 1e-118, 1e-117, 1e-116, 1e-115, 1e-114, 1e-113, 1e-112, 1e-111, + 1e-110, 1e-109, 1e-108, 1e-107, 1e-106, 1e-105, 1e-104, 1e-103, 1e-102, + 1e-101, 1e-100, 1e-99, 1e-98, 1e-97, 1e-96, + 1e-95, 1e-94, 1e-93, 1e-92, 1e-91, 1e-90, 1e-89, 1e-88, 1e-87, 1e-86, + 1e-85, 1e-84, 1e-83, 1e-82, 1e-81, 1e-80, 1e-79, 1e-78, 1e-77, 1e-76, + 1e-75, 1e-74, 1e-73, 1e-72, 1e-71, 1e-70, 1e-69, 1e-68, 1e-67, 1e-66, + 1e-65, 1e-64, 1e-63, 1e-62, 1e-61, 1e-60, 1e-59, 1e-58, 1e-57, 1e-56, + 1e-55, 1e-54, 1e-53, 1e-52, 1e-51, 1e-50, 1e-49, 1e-48, 1e-47, 1e-46, + 1e-45, 1e-44, 1e-43, 1e-42, 1e-41, 1e-40, 1e-39, 1e-38, 1e-37, 1e-36, + 1e-35, 1e-34, 1e-33, 1e-32, 1e-31, 1e-30, 1e-29, 1e-28, 1e-27, 1e-26, + 1e-25, 1e-24, 1e-23, 1e-22, 1e-21, 1e-20, 1e-19, 1e-18, 1e-17, 1e-16, + 1e-15, 1e-14, 1e-13, 1e-12, 1e-11, 1e-10, 1e-09, 1e-08, 1e-07, 1e-06, + 1e-05, 0.0001, 0.001, 0.01, 0.1, +} + +// A Bracket is a part of a cumulative distribution. +type bin struct { + count uint64 + val int8 + exp int8 +} + +func newBinRaw(val int8, exp int8, count uint64) *bin { + return &bin{ + count: count, + val: val, + exp: exp, + } +} + +// func newBin() *bin { +// return newBinRaw(0, 0, 0) +// } + +func newBinFromFloat64(d float64) *bin { + hb := newBinRaw(0, 0, 0) + hb.setFromFloat64(d) + return hb +} + +type fastL2 struct { + l1, l2 int +} + +func (hb *bin) newFastL2() fastL2 { + return fastL2{l1: int(uint8(hb.exp)), l2: int(uint8(hb.val))} +} + +func (hb *bin) setFromFloat64(d float64) *bin { //nolint:unparam + hb.val = -1 + if math.IsInf(d, 0) || math.IsNaN(d) { + return hb + } + if d == 0.0 { + hb.val = 0 + return hb + } + sign := 1 + if math.Signbit(d) { + sign = -1 + } + d = math.Abs(d) + bigExp := int(math.Floor(math.Log10(d))) + hb.exp = int8(bigExp) + if int(hb.exp) != bigExp { // rolled + hb.exp = 0 + if bigExp < 0 { + hb.val = 0 + } + return hb + } + d /= hb.powerOfTen() + d *= 10 + hb.val = int8(sign * int(math.Floor(d+1e-13))) + if hb.val == 100 || hb.val == -100 { + if hb.exp < 127 { + hb.val /= 10 + hb.exp++ + } else { + hb.val = 0 + hb.exp = 0 + } + } + if hb.val == 0 { + hb.exp = 0 + return hb + } + if !((hb.val >= 10 && hb.val < 100) || + (hb.val <= -10 && hb.val > -100)) { + hb.val = -1 + hb.exp = 0 + } + return hb +} + +func (hb *bin) powerOfTen() float64 { + idx := int(uint8(hb.exp)) + return powerOfTen[idx] +} + +func (hb *bin) isNaN() bool { + // aval := abs(hb.val) + aval := hb.val + if aval < 0 { + aval = -aval + } + if 99 < aval { // in [100... ]: nan + return true + } + if 9 < aval { // in [10 - 99]: valid range + return false + } + if 0 < aval { // in [1 - 9 ]: nan + return true + } + if 0 == aval { // in [0] : zero bucket + return false + } + return false +} + +func (hb *bin) value() float64 { + if hb.isNaN() { + return math.NaN() + } + if hb.val < 10 && hb.val > -10 { + return 0.0 + } + return (float64(hb.val) / 10.0) * hb.powerOfTen() +} + +func (hb *bin) binWidth() float64 { + if hb.isNaN() { + return math.NaN() + } + if hb.val < 10 && hb.val > -10 { + return 0.0 + } + return hb.powerOfTen() / 10.0 +} + +func (hb *bin) midpoint() float64 { + if hb.isNaN() { + return math.NaN() + } + out := hb.value() + if out == 0 { + return 0 + } + interval := hb.binWidth() + if out < 0 { + interval *= -1 + } + return out + interval/2.0 +} + +func (hb *bin) left() float64 { + if hb.isNaN() { + return math.NaN() + } + out := hb.value() + if out >= 0 { + return out + } + return out - hb.binWidth() +} + +func (hb *bin) compare(h2 *bin) int { + var v1, v2 int + + // 1) slide exp positive + // 2) shift by size of val multiple by (val != 0) + // 3) then add or subtract val accordingly + + if hb.val >= 0 { + v1 = ((int(hb.exp)+256)<<8)*(((int(hb.val)|(^int(hb.val)+1))>>8)&1) + int(hb.val) + } else { + v1 = ((int(hb.exp)+256)<<8)*(((int(hb.val)|(^int(hb.val)+1))>>8)&1) - int(hb.val) + } + + if h2.val >= 0 { + v2 = ((int(h2.exp)+256)<<8)*(((int(h2.val)|(^int(h2.val)+1))>>8)&1) + int(h2.val) + } else { + v2 = ((int(h2.exp)+256)<<8)*(((int(h2.val)|(^int(h2.val)+1))>>8)&1) - int(h2.val) + } + + // return the difference + return v2 - v1 +} + +// Histogram tracks values are two decimal digits of precision +// with a bounded error that remains bounded upon composition. +type Histogram struct { + bvs []bin + lookup [][]uint16 + mutex sync.RWMutex + used uint16 + useLookup bool + useLocks bool +} + +//nolint:golint,revive +const ( + BVL1, BVL1MASK uint64 = iota, 0xff << (8 * iota) + BVL2, BVL2MASK + BVL3, BVL3MASK + BVL4, BVL4MASK + BVL5, BVL5MASK + BVL6, BVL6MASK + BVL7, BVL7MASK + BVL8, BVL8MASK +) + +func getBytesRequired(val uint64) int8 { + if 0 != (BVL8MASK|BVL7MASK|BVL6MASK|BVL5MASK)&val { + if 0 != BVL8MASK&val { + return int8(BVL8) + } + if 0 != BVL7MASK&val { + return int8(BVL7) + } + if 0 != BVL6MASK&val { + return int8(BVL6) + } + if 0 != BVL5MASK&val { + return int8(BVL5) + } + } else { + if 0 != BVL4MASK&val { + return int8(BVL4) + } + if 0 != BVL3MASK&val { + return int8(BVL3) + } + if 0 != BVL2MASK&val { + return int8(BVL2) + } + } + return int8(BVL1) +} + +func writeBin(out io.Writer, in bin) (err error) { + + err = binary.Write(out, binary.BigEndian, in.val) + if err != nil { + return + } + + err = binary.Write(out, binary.BigEndian, in.exp) + if err != nil { + return + } + + var tgtType = getBytesRequired(in.count) + + err = binary.Write(out, binary.BigEndian, tgtType) + if err != nil { + return + } + + var bcount = make([]uint8, 8) + b := bcount[0 : tgtType+1] + for i := tgtType; i >= 0; i-- { + b[i] = uint8(uint64(in.count>>(uint8(i)*8)) & 0xff) //nolint:unconvert + } + + err = binary.Write(out, binary.BigEndian, b) + if err != nil { + return + } + return +} + +func readBin(in io.Reader) (bin, error) { + var out bin + + err := binary.Read(in, binary.BigEndian, &out.val) + if err != nil { + return out, fmt.Errorf("read: %w", err) + } + + err = binary.Read(in, binary.BigEndian, &out.exp) + if err != nil { + return out, fmt.Errorf("read: %w", err) + } + var bvl uint8 + err = binary.Read(in, binary.BigEndian, &bvl) + if err != nil { + return out, fmt.Errorf("read: %w", err) + } + if bvl > uint8(BVL8) { + return out, fmt.Errorf("encoding error: bvl value is greater than max allowable") //nolint:goerr113 + } + + bcount := make([]byte, 8) + b := bcount[0 : bvl+1] + err = binary.Read(in, binary.BigEndian, b) + if err != nil { + return out, fmt.Errorf("read: %w", err) + } + + count := uint64(0) + for i := int(bvl + 1); i >= 0; i-- { + count |= uint64(bcount[i]) << (uint8(i) * 8) + } + + out.count = count + return out, nil +} + +func Deserialize(in io.Reader) (h *Histogram, err error) { + return DeserializeWithOptions(in) +} + +func DeserializeWithOptions(in io.Reader, options ...Option) (h *Histogram, err error) { + var nbin int16 + err = binary.Read(in, binary.BigEndian, &nbin) + if err != nil { + return + } + + options = append(options, Size(uint16(nbin))) + h = New(options...) + for ii := int16(0); ii < nbin; ii++ { + bb, err := readBin(in) + if err != nil { + return h, err + } + h.insertBin(&bb, int64(bb.count)) + } + return h, nil +} + +func (h *Histogram) Serialize(w io.Writer) error { + var nbin int16 + for i := range h.bvs { + if h.bvs[i].count != 0 { + nbin++ + } + } + + if err := binary.Write(w, binary.BigEndian, nbin); err != nil { + return fmt.Errorf("write: %w", err) + } + + for _, bv := range h.bvs { + if bv.count != 0 { + if err := writeBin(w, bv); err != nil { + return err + } + } + } + return nil +} + +func (h *Histogram) SerializeB64(w io.Writer) error { + buf := bytes.NewBuffer([]byte{}) + if err := h.Serialize(buf); err != nil { + return err + } + + encoder := base64.NewEncoder(base64.StdEncoding, w) + if _, err := encoder.Write(buf.Bytes()); err != nil { + return fmt.Errorf("b64 encode write: %w", err) + } + if err := encoder.Close(); err != nil { + return fmt.Errorf("b64 encoder close: %w", err) + } + + return nil +} + +// Options are exposed options for initializing a histogram. +type Options struct { + // Size is the number of bins. + Size uint16 + + // UseLocks determines if the histogram should use locks + UseLocks bool + + // UseLookup determines if the histogram should use a lookup table for bins + UseLookup bool +} + +// Option knows how to mutate the Options to change initialization. +type Option func(*Options) + +// NoLocks configures a histogram to not use locks. +func NoLocks() Option { + return func(options *Options) { + options.UseLocks = false + } +} + +// NoLookup configures a histogram to not use a lookup table for bins. +// This is an appropriate option to use when the data set being operated +// over contains a large number of individual histograms and the insert +// speed into any histogram is not of the utmost importance. This option +// reduces the baseline memory consumption of one Histogram by at least +// 0.5kB and up to 130kB while increasing the insertion time by ~20%. +func NoLookup() Option { + return func(options *Options) { + options.UseLookup = false + } +} + +// Size configures a histogram to initialize a specific number of bins. +// When more bins are required, allocations increase linearly by the default +// size (100). +func Size(size uint16) Option { + return func(options *Options) { + options.Size = size + } +} + +// New returns a new Histogram, respecting the passed Options. +func New(options ...Option) *Histogram { + o := Options{ + Size: defaultHistSize, + UseLocks: true, + UseLookup: true, + } + for _, opt := range options { + opt(&o) + } + h := &Histogram{ + used: 0, + bvs: make([]bin, o.Size), + useLocks: o.UseLocks, + useLookup: o.UseLookup, + } + if h.useLookup { + h.lookup = make([][]uint16, 256) + } + return h +} + +// NewNoLocks returns a new histogram not using locks. +// Deprecated: use New(NoLocks()) instead. +func NewNoLocks() *Histogram { + return New(NoLocks()) +} + +// NewFromStrings returns a Histogram created from DecStrings strings. +func NewFromStrings(strs []string, locks bool) (*Histogram, error) { + + bin, err := stringsToBin(strs) + if err != nil { + return nil, err + } + + return newFromBins(bin, locks), nil +} + +// NewFromBins returns a Histogram created from a bins struct slice. +func newFromBins(bins []bin, locks bool) *Histogram { + return &Histogram{ + used: uint16(len(bins)), + bvs: bins, + useLocks: locks, + lookup: make([][]uint16, 256), + useLookup: true, + } +} + +// Max returns the approximate maximum recorded value. +func (h *Histogram) Max() float64 { + return h.ValueAtQuantile(1.0) +} + +// Min returns the approximate minimum recorded value. +func (h *Histogram) Min() float64 { + return h.ValueAtQuantile(0.0) +} + +// Mean returns the approximate arithmetic mean of the recorded values. +func (h *Histogram) Mean() float64 { + return h.ApproxMean() +} + +// Count returns the number of recorded values. +func (h *Histogram) Count() uint64 { + if h.useLocks { + h.mutex.RLock() + defer h.mutex.RUnlock() + } + var count uint64 + for _, bin := range h.bvs[0:h.used] { + if bin.isNaN() { + continue + } + count += bin.count + } + return count +} + +// BinCount returns the number of used bins. +func (h *Histogram) BinCount() uint64 { + if h.useLocks { + h.mutex.RLock() + defer h.mutex.RUnlock() + } + binCount := h.used + return uint64(binCount) +} + +// Reset forgets all bins in the histogram (they remain allocated). +func (h *Histogram) Reset() { + if h.useLocks { + h.mutex.Lock() + defer h.mutex.Unlock() + } + h.used = 0 + + if !h.useLookup { + return + } + for i := 0; i < 256; i++ { + if h.lookup[i] != nil { + for j := range h.lookup[i] { + h.lookup[i][j] = 0 + } + } + } +} + +// RecordIntScale records an integer scaler value, returning an error if the +// value is out of range. +func (h *Histogram) RecordIntScale(val int64, scale int) error { + return h.RecordIntScales(val, scale, 1) +} + +// RecordValue records the given value, returning an error if the value is out +// of range. +func (h *Histogram) RecordValue(v float64) error { + return h.RecordValues(v, 1) +} + +// RecordDuration records the given time.Duration in seconds, returning an error +// if the value is out of range. +func (h *Histogram) RecordDuration(v time.Duration) error { + return h.RecordIntScale(int64(v), -9) +} + +// RecordCorrectedValue records the given value, correcting for stalls in the +// recording process. This only works for processes which are recording values +// at an expected interval (e.g., doing jitter analysis). Processes which are +// recording ad-hoc values (e.g., latency for incoming requests) can't take +// advantage of this. +// CH Compat. +func (h *Histogram) RecordCorrectedValue(v, expectedInterval int64) error { + if err := h.RecordValue(float64(v)); err != nil { + return err + } + + if expectedInterval <= 0 || v <= expectedInterval { + return nil + } + + missingValue := v - expectedInterval + for missingValue >= expectedInterval { + if err := h.RecordValue(float64(missingValue)); err != nil { + return err + } + missingValue -= expectedInterval + } + + return nil +} + +// find where a new bin should go. +func (h *Histogram) internalFind(hb *bin) (bool, uint16) { + if h.used == 0 { + return false, 0 + } + if h.useLookup { + f2 := hb.newFastL2() + if h.lookup[f2.l1] != nil { + if idx := h.lookup[f2.l1][f2.l2]; idx != 0 { + return true, idx - 1 + } + } + } + rv := -1 + idx := uint16(0) + l := int(0) + r := int(h.used - 1) + for l < r { + check := (r + l) / 2 + rv = h.bvs[check].compare(hb) + switch { + case rv == 0: + l = check + r = check + case rv > 0: + l = check + 1 + default: + r = check - 1 + } + } + if rv != 0 { + rv = h.bvs[l].compare(hb) + } + idx = uint16(l) + if rv == 0 { + return true, idx + } + if rv < 0 { + return false, idx + } + idx++ + return false, idx +} + +func (h *Histogram) insertBin(hb *bin, count int64) uint64 { //nolint:unparam + if h.useLocks { + h.mutex.Lock() + defer h.mutex.Unlock() + } + found, idx := h.internalFind(hb) + if !found { + count := h.insertNewBinAt(idx, hb, count) + // update the fast lookup table data after the index + h.updateFast(idx) + return count + } + return h.updateOldBinAt(idx, count) +} + +func (h *Histogram) insertNewBinAt(idx uint16, hb *bin, count int64) uint64 { + h.bvs = append(h.bvs, bin{}) + copy(h.bvs[idx+1:], h.bvs[idx:]) + h.bvs[idx].val = hb.val + h.bvs[idx].exp = hb.exp + h.bvs[idx].count = uint64(count) + h.used++ + return h.bvs[idx].count +} + +func (h *Histogram) updateFast(start uint16) { + if !h.useLookup { + return + } + for i := start; i < h.used; i++ { + f2 := h.bvs[i].newFastL2() + if h.lookup[f2.l1] == nil { + h.lookup[f2.l1] = make([]uint16, 256) + } + h.lookup[f2.l1][f2.l2] = i + 1 + } +} + +func (h *Histogram) updateOldBinAt(idx uint16, count int64) uint64 { + var newval uint64 + if count >= 0 { + newval = h.bvs[idx].count + uint64(count) + } else { + newval = h.bvs[idx].count - uint64(-count) + } + if newval < h.bvs[idx].count { // rolled + newval = ^uint64(0) + } + h.bvs[idx].count = newval + return newval - h.bvs[idx].count +} + +// RecordIntScales records n occurrences of the given value, returning an error if +// the value is out of range. +func (h *Histogram) RecordIntScales(val int64, scale int, n int64) error { + sign := int64(1) + if val == 0 { + scale = 0 + } else { + scale++ + if val < 0 { + val = 0 - val + sign = -1 + } + if val < 10 { + val *= 10 + scale-- + } + for val >= 100 { + val /= 10 + scale++ + } + } + if scale < -128 { + val = 0 + scale = 0 + } else if scale > 127 { + val = 0xff + scale = 0 + } + val *= sign + hb := bin{val: int8(val), exp: int8(scale), count: 0} + h.insertBin(&hb, n) + return nil +} + +// RecordValues records n occurrences of the given value, returning an error if +// the value is out of range. +func (h *Histogram) RecordValues(v float64, n int64) error { + var hb bin + hb.setFromFloat64(v) + h.insertBin(&hb, n) + return nil +} + +// ApproxMean returns an approximate mean. +func (h *Histogram) ApproxMean() float64 { + if h.useLocks { + h.mutex.RLock() + defer h.mutex.RUnlock() + } + divisor := 0.0 + sum := 0.0 + for i := uint16(0); i < h.used; i++ { + midpoint := h.bvs[i].midpoint() + cardinality := float64(h.bvs[i].count) + divisor += cardinality + sum += midpoint * cardinality + } + if divisor == 0.0 { + return math.NaN() + } + return sum / divisor +} + +// ApproxSum returns an approximate sum. +func (h *Histogram) ApproxSum() float64 { + if h.useLocks { + h.mutex.RLock() + defer h.mutex.RUnlock() + } + sum := 0.0 + for i := uint16(0); i < h.used; i++ { + midpoint := h.bvs[i].midpoint() + cardinality := float64(h.bvs[i].count) + sum += midpoint * cardinality + } + return sum +} + +func (h *Histogram) ApproxQuantile(qIn []float64) ([]float64, error) { + if h.useLocks { + h.mutex.RLock() + defer h.mutex.RUnlock() + } + qOut := make([]float64, len(qIn)) + iq, ib := 0, uint16(0) + totalCnt, binWidth, binLeft, lowerCnt, upperCnt := 0.0, 0.0, 0.0, 0.0, 0.0 + if len(qIn) == 0 { + return qOut, nil + } + // Make sure the requested quantiles are in order + for iq = 1; iq < len(qIn); iq++ { + if qIn[iq-1] > qIn[iq] { + return nil, fmt.Errorf("out of order") //nolint:goerr113 + } + } + // Add up the bins + for ib = 0; ib < h.used; ib++ { + if !h.bvs[ib].isNaN() { + totalCnt += float64(h.bvs[ib].count) + } + } + if totalCnt == 0.0 { + return nil, fmt.Errorf("empty_histogram") //nolint:goerr113 + } + + for iq = 0; iq < len(qIn); iq++ { + if qIn[iq] < 0.0 || qIn[iq] > 1.0 { + return nil, fmt.Errorf("out of bound quantile") //nolint:goerr113 + } + qOut[iq] = totalCnt * qIn[iq] + } + + for ib = 0; ib < h.used; ib++ { + if h.bvs[ib].isNaN() { + continue + } + binWidth = h.bvs[ib].binWidth() + binLeft = h.bvs[ib].left() + lowerCnt = upperCnt + upperCnt = lowerCnt + float64(h.bvs[ib].count) + break + } + for iq = 0; iq < len(qIn); iq++ { + for ib < (h.used-1) && upperCnt < qOut[iq] { + ib++ + binWidth = h.bvs[ib].binWidth() + binLeft = h.bvs[ib].left() + lowerCnt = upperCnt + upperCnt = lowerCnt + float64(h.bvs[ib].count) + } + switch { + case lowerCnt == qOut[iq]: + qOut[iq] = binLeft + case upperCnt == qOut[iq]: + qOut[iq] = binLeft + binWidth + default: + if binWidth == 0 { + qOut[iq] = binLeft + } else { + qOut[iq] = binLeft + (qOut[iq]-lowerCnt)/(upperCnt-lowerCnt)*binWidth + } + } + } + return qOut, nil +} + +// ValueAtQuantile returns the recorded value at the given quantile (0..1). +func (h *Histogram) ValueAtQuantile(q float64) float64 { + if h.useLocks { + h.mutex.RLock() + defer h.mutex.RUnlock() + } + qIn := make([]float64, 1) + qIn[0] = q + qOut, err := h.ApproxQuantile(qIn) + if err == nil && len(qOut) == 1 { + return qOut[0] + } + return math.NaN() +} + +// SignificantFigures returns the significant figures used to create the +// histogram +// CH Compat. +func (h *Histogram) SignificantFigures() int64 { + return 2 +} + +// Equals returns true if the two Histograms are equivalent, false if not. +func (h *Histogram) Equals(other *Histogram) bool { + if h.useLocks { + h.mutex.RLock() + defer h.mutex.RUnlock() + } + if other.useLocks { + other.mutex.RLock() + defer other.mutex.RUnlock() + } + switch { + case + h.used != other.used: + return false + default: + for i := uint16(0); i < h.used; i++ { + if h.bvs[i].compare(&other.bvs[i]) != 0 { + return false + } + if h.bvs[i].count != other.bvs[i].count { + return false + } + } + } + return true +} + +// Copy creates and returns an exact copy of a histogram. +func (h *Histogram) Copy() *Histogram { + if h.useLocks { + h.mutex.Lock() + defer h.mutex.Unlock() + } + + newhist := New() + newhist.used = h.used + newhist.useLocks = h.useLocks + + newhist.bvs = make([]bin, len(h.bvs)) + copy(h.bvs, newhist.bvs) + + newhist.useLookup = h.useLookup + if h.useLookup { + newhist.lookup = make([][]uint16, 256) + for i, u := range h.lookup { + newhist.lookup[i] = append(newhist.lookup[i], u...) + } + } + + return newhist +} + +// FullReset resets a histogram to default empty values. +func (h *Histogram) FullReset() { + if h.useLocks { + h.mutex.Lock() + defer h.mutex.Unlock() + } + + h.bvs = []bin{} + h.used = 0 + if h.useLookup { + h.lookup = make([][]uint16, 256) + } +} + +// CopyAndReset creates and returns an exact copy of a histogram, +// and resets it to default empty values. +func (h *Histogram) CopyAndReset() *Histogram { + newhist := h.Copy() + h.FullReset() + return newhist +} + +func (h *Histogram) DecStrings() []string { + if h.useLocks { + h.mutex.Lock() + defer h.mutex.Unlock() + } + out := make([]string, h.used) + for i, bin := range h.bvs[0:h.used] { + var buffer bytes.Buffer + buffer.WriteString("H[") + buffer.WriteString(fmt.Sprintf("%3.1e", bin.value())) + buffer.WriteString("]=") + buffer.WriteString(fmt.Sprintf("%v", bin.count)) + out[i] = buffer.String() + } + return out +} + +// takes the output of DecStrings and deserializes it into a Bin struct slice. +func stringsToBin(strs []string) ([]bin, error) { + + bins := make([]bin, len(strs)) + for i, str := range strs { + + // H[0.0e+00]=1 + + // H[0.0e+00]= <1> + countString := strings.Split(str, "=")[1] + countInt, err := strconv.ParseInt(countString, 10, 64) + if err != nil { + return nil, fmt.Errorf("parse int: %w", err) + } + + // H[ <0.0> e+00]=1 + valString := strings.Split(strings.Split(strings.Split(str, "=")[0], "e")[0], "[")[1] + valInt, err := strconv.ParseFloat(valString, 64) + if err != nil { + return nil, fmt.Errorf("parse float: %w", err) + } + + // H[0.0e <+00> ]=1 + expString := strings.Split(strings.Split(strings.Split(str, "=")[0], "e")[1], "]")[0] + expInt, err := strconv.ParseInt(expString, 10, 8) + if err != nil { + return nil, fmt.Errorf("parse int: %w", err) + } + bins[i] = *newBinRaw(int8(valInt*10), int8(expInt), uint64(countInt)) + } + + return bins, nil +} + +// UnmarshalJSON - histogram will come in a base64 encoded serialized form. +func (h *Histogram) UnmarshalJSON(b []byte) error { + return UnmarshalJSONWithOptions(h, b) +} + +// UnmarshalJSONWithOptions unmarshals the byte data into the parent histogram, +// using the provided Options to create the output Histogram. +func UnmarshalJSONWithOptions(parent *Histogram, b []byte, options ...Option) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return fmt.Errorf("json unmarshal: %w", err) + } + + data, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return fmt.Errorf("b64 decode: %w", err) + } + + hNew, err := DeserializeWithOptions(bytes.NewBuffer(data), options...) + if err != nil { + return err + } + + // Go's JSON package will create a new Histogram to deserialize into by + // reflection, so all fields will have their zero values. Some of the + // default Histogram fields are not the zero values, so we can set them + // by proxy from the new histogram that's been created from deserialization. + parent.useLocks = hNew.useLocks + parent.useLookup = hNew.useLookup + if parent.useLookup { + parent.lookup = make([][]uint16, 256) + } + + parent.Merge(hNew) + return nil +} + +func (h *Histogram) MarshalJSON() ([]byte, error) { + return MarshalJSON(h) +} + +func MarshalJSON(h *Histogram) ([]byte, error) { + buf := bytes.NewBuffer([]byte{}) + err := h.SerializeB64(buf) + if err != nil { + return buf.Bytes(), err + } + data, err := json.Marshal(buf.String()) + if err != nil { + return nil, fmt.Errorf("json marshal: %w", err) + } + return data, nil +} + +// Merge merges all bins from another histogram. +func (h *Histogram) Merge(o *Histogram) { + if o == nil { + return + } + + if o.useLocks { + o.mutex.Lock() + defer o.mutex.Unlock() + } + if h.useLocks { + h.mutex.Lock() + defer h.mutex.Unlock() + } + + var i, j uint16 + for ; i < h.used && j < o.used; i++ { + diff := h.bvs[i].compare(&o.bvs[j]) + // o.bvs[j] > h.bvs[i], do nothing. + if diff > 0 { + continue + } + + b := &o.bvs[j] + j++ + switch { + case diff == 0: + h.updateOldBinAt(i, int64(b.count)) + case diff < 0: + h.insertNewBinAt(i, b, int64(b.count)) + } + } + + // append the rest bins + for ; j < o.used; j++ { + h.insertNewBinAt(h.used, &o.bvs[j], int64(o.bvs[j].count)) + } + + // rebuild all the fast lookup table + h.updateFast(0) +} + +// HistogramWithoutLookups holds a Histogram that's not configured to use +// a lookup table. This type is useful to round-trip serialize the underlying +// data while never allocating memory for the lookup table. +// The main Histogram type must use lookups by default to be compatible with +// the circllhist implementation of other languages. Furthermore, it is not +// possible to encode the lookup table preference into the serialized form, +// as that's again defined across languages. Therefore, the most straightforward +// manner by which a user can deserialize histogram data while not allocating +// lookup tables is by using a dedicated type in their structures describing +// on-disk forms. +// This structure can divulge the underlying Histogram, optionally allocating +// the lookup tables first. +type HistogramWithoutLookups struct { + histogram *Histogram +} + +// NewHistogramWithoutLookups creates a new container for a Histogram without +// lookup tables. +func NewHistogramWithoutLookups(histogram *Histogram) *HistogramWithoutLookups { + histogram.useLookup = false + histogram.lookup = nil + return &HistogramWithoutLookups{ + histogram: histogram, + } +} + +// Histogram divulges the underlying Histogram that was deserialized. This +// Histogram will not have lookup tables allocated. +func (h *HistogramWithoutLookups) Histogram() *Histogram { + return h.histogram +} + +// HistogramWithLookups allocates lookup tables in the underlying Histogram that was +// deserialized, then divulges it. +func (h *HistogramWithoutLookups) HistogramWithLookups() *Histogram { + h.histogram.useLookup = true + h.histogram.lookup = make([][]uint16, 256) + return h.histogram +} + +// UnmarshalJSON unmarshals a histogram from a base64 encoded serialized form. +func (h *HistogramWithoutLookups) UnmarshalJSON(b []byte) error { + var histogram Histogram + if err := UnmarshalJSONWithOptions(&histogram, b, NoLookup()); err != nil { + return err + } + h.histogram = &histogram + return nil +} + +// MarshalJSON marshals a histogram to a base64 encoded serialized form. +func (h *HistogramWithoutLookups) MarshalJSON() ([]byte, error) { + return MarshalJSON(h.histogram) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 369d75444c06..6b5a7e00bbff 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -247,6 +247,9 @@ github.com/mstoykov/k6-taskqueue-lib/taskqueue # github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d ## explicit github.com/nu7hatch/gouuid +# github.com/openhistogram/circonusllhist v0.3.1-0.20210609143308-c78ce013c914 +## explicit; go 1.16 +github.com/openhistogram/circonusllhist # github.com/pmezard/go-difflib v1.0.0 ## explicit github.com/pmezard/go-difflib/difflib