diff --git a/bcs-services/bcs-monitor/cmd/bcs-monitor/storegw.go b/bcs-services/bcs-monitor/cmd/bcs-monitor/storegw.go index 022caac944..bc957e2e61 100644 --- a/bcs-services/bcs-monitor/cmd/bcs-monitor/storegw.go +++ b/bcs-services/bcs-monitor/cmd/bcs-monitor/storegw.go @@ -114,7 +114,7 @@ func runStoreGW(ctx context.Context, g *run.Group, opt *option) error { if err != nil { return errors.Wrap(err, "building gRPC client") } - endpoints = query.NewEndpointSet(kitLogger, opt.reg, + endpoints = query.NewEndpointSet(time.Now, kitLogger, opt.reg, func() (specs []*query.GRPCEndpointSpec) { for _, addr := range gw.GetStoreAddrs() { specs = append(specs, query.NewGRPCEndpointSpec(addr, true)) @@ -123,6 +123,7 @@ func runStoreGW(ctx context.Context, g *run.Group, opt *option) error { }, dialOpts, time.Second*30, + time.Second*30, ) var cancel context.CancelFunc @@ -159,10 +160,10 @@ func runStoreGW(ctx context.Context, g *run.Group, opt *option) error { func registryProxyStore(g *run.Group, kitLogger gokit.Logger, opt *option, endpoints *query.EndpointSet) { proxyStore := store.NewProxyStore(kitLogger, opt.reg, endpoints.GetStoreClients, component.Query, nil, - time.Minute*2) + time.Minute*2, store.LazyRetrieval) grpcProbe := prober.NewGRPC() grpcSrv := grpcserver.New(kitLogger, opt.reg, nil, nil, nil, component.Store, grpcProbe, - grpcserver.WithServer(store.RegisterStoreServer(proxyStore)), + grpcserver.WithServer(store.RegisterStoreServer(proxyStore, kitLogger)), grpcserver.WithListen(utils.GetListenAddr(bindAddress, grpcPort)), grpcserver.WithGracePeriod(time.Duration(0)), grpcserver.WithMaxConnAge(time.Minute*5), // 5分钟主动重连, pod 扩容等需要 diff --git a/bcs-services/bcs-monitor/go.mod b/bcs-services/bcs-monitor/go.mod index 9a06c857c3..dd0b9f50f1 100644 --- a/bcs-services/bcs-monitor/go.mod +++ b/bcs-services/bcs-monitor/go.mod @@ -10,12 +10,13 @@ require ( github.com/TencentBlueKing/bkmonitor-kits v0.2.0 github.com/chonla/format v0.0.0-20220105105701-1119f4a3f36f github.com/dustin/go-humanize v1.0.0 + github.com/efficientgo/core v1.0.0-rc.2 github.com/fsnotify/fsnotify v1.6.0 github.com/gin-contrib/cors v1.3.1 github.com/gin-contrib/requestid v0.0.4 github.com/gin-contrib/sse v0.1.0 github.com/gin-gonic/gin v1.9.0 - github.com/go-kit/log v0.2.0 + github.com/go-kit/log v0.2.1 github.com/go-micro/plugins/v4/registry/etcd v1.1.0 github.com/go-redis/redis/v8 v8.11.5 github.com/go-resty/resty/v2 v2.7.0 @@ -31,8 +32,8 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.46.0 github.com/prometheus-operator/prometheus-operator/pkg/client v0.46.0 - github.com/prometheus/client_golang v1.12.1 - github.com/prometheus/common v0.34.0 + github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/common v0.39.1-0.20230202092144-f9c1994be032 github.com/prometheus/prometheus v1.8.2-0.20220308163432-03831554a519 github.com/spf13/cobra v1.7.0 github.com/spf13/viper v1.12.0 @@ -40,7 +41,7 @@ require ( github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 github.com/swaggo/gin-swagger v1.4.3 github.com/swaggo/swag v1.8.1 - github.com/thanos-io/thanos v0.26.0 + github.com/thanos-io/thanos v0.31.0 go-micro.dev/v4 v4.10.2 go.uber.org/automaxprocs v1.5.1 golang.org/x/sync v0.2.0 @@ -49,9 +50,9 @@ require ( gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/mysql v1.5.1 gorm.io/gorm v1.25.1 - k8s.io/api v0.23.5 - k8s.io/apimachinery v0.23.5 - k8s.io/client-go v0.23.5 + k8s.io/api v0.26.1 + k8s.io/apimachinery v0.26.1 + k8s.io/client-go v0.26.1 k8s.io/klog v1.0.0 k8s.io/klog/v2 v2.90.1 ) @@ -59,46 +60,84 @@ require ( require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/BurntSushi/toml v1.3.2 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.32.3 // indirect github.com/TencentBlueKing/bk-audit-go-sdk v0.0.5 // indirect + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/bytedance/sonic v1.8.0 // indirect github.com/cenkalti/backoff/v4 v4.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/fatih/color v1.15.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/frankban/quicktest v1.14.4 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-openapi/analysis v0.21.4 // indirect + github.com/go-openapi/errors v0.20.3 // indirect + github.com/go-openapi/loads v0.21.2 // indirect + github.com/go-openapi/strfmt v0.21.3 // indirect + github.com/go-openapi/validate v0.22.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect - github.com/go-stack/stack v1.8.0 // indirect github.com/goccy/go-json v0.10.0 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/klauspost/compress v1.13.6 // indirect - github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/klauspost/compress v1.15.9 // indirect + github.com/klauspost/cpuid/v2 v2.1.0 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/matryer/is v1.4.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect + github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/onsi/ginkgo/v2 v2.9.2 // indirect github.com/onsi/gomega v1.27.6 // indirect github.com/openzipkin/zipkin-go v0.3.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/prometheus/alertmanager v0.25.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/rueian/rueidis v0.0.93 // indirect + github.com/shirou/gopsutil/v3 v3.22.9 // indirect + github.com/thanos-community/promql-engine v0.0.0-20230224075812-ae04bbea7613 // indirect + github.com/thanos-io/objstore v0.0.0-20230201072718-11ffbc490204 // indirect + github.com/tklauser/go-sysconf v0.3.10 // indirect + github.com/tklauser/numcpus v0.4.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.0.2 // indirect - github.com/xdg-go/stringprep v1.0.2 // indirect + github.com/xdg-go/scram v1.1.1 // indirect + github.com/xdg-go/stringprep v1.0.3 // indirect + github.com/xhit/go-str2duration/v2 v2.1.0 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0 // indirect - go.opentelemetry.io/otel/exporters/jaeger v1.3.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.39.0 // indirect + go.opentelemetry.io/contrib/propagators/autoprop v0.38.0 // indirect + go.opentelemetry.io/contrib/propagators/aws v1.13.0 // indirect + go.opentelemetry.io/contrib/propagators/b3 v1.13.0 // indirect + go.opentelemetry.io/contrib/propagators/jaeger v1.13.0 // indirect + go.opentelemetry.io/contrib/samplers/jaegerremote v0.7.0 // indirect + go.opentelemetry.io/otel/exporters/jaeger v1.12.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.12.0 // indirect go.opentelemetry.io/otel/exporters/zipkin v1.3.0 // indirect + go.opentelemetry.io/otel/metric v0.36.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go4.org/intern v0.0.0-20220617035311-6925f38cc365 // indirect + go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect + golang.org/x/exp v0.0.0-20230124195608-d38c7dcee874 // indirect sigs.k8s.io/controller-runtime v0.11.2 // indirect ) require ( cloud.google.com/go/compute v1.18.0 // indirect cloud.google.com/go/trace v1.8.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.0.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.8.3 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/Masterminds/semver/v3 v3.1.1 // indirect github.com/Microsoft/go-winio v0.6.0 // indirect @@ -106,11 +145,9 @@ require ( github.com/ProtonMail/go-crypto v0.0.0-20220824120805-4b6e5c587895 // indirect github.com/TencentBlueKing/iam-go-sdk v0.0.8 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect - github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/armon/go-radix v1.0.0 // indirect - github.com/aws/aws-sdk-go v1.42.31 // indirect - github.com/benbjohnson/clock v1.3.0 // indirect + github.com/aws/aws-sdk-go v1.44.187 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bitly/go-simplejson v0.5.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect @@ -119,24 +156,24 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cloudflare/circl v1.2.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/coreos/go-systemd/v22 v22.4.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dennwc/varint v1.0.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/edsrzf/mmap-go v1.1.0 // indirect - github.com/efficientgo/tools/extkingpin v0.0.0-20210609125236-d73259166f20 // indirect - github.com/elastic/go-sysinfo v1.1.1 // indirect + github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd // indirect + github.com/elastic/go-sysinfo v1.8.1 // indirect github.com/elastic/go-windows v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect - github.com/felixge/fgprof v0.9.1 // indirect + github.com/felixge/fgprof v0.9.2 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/go-git/go-git/v5 v5.4.2 // indirect github.com/go-kit/kit v0.12.0 // indirect - github.com/go-logfmt/logfmt v0.5.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect @@ -144,19 +181,18 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/gogo/googleapis v1.4.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/gogo/status v1.1.0 // indirect + github.com/gogo/status v1.1.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect + github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect - github.com/googleapis/gnostic v0.5.5 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/kit/v2 v2.0.0-20201002093600-73cf2ae9d891 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/hashicorp/golang-lru v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -169,8 +205,8 @@ require ( github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect - github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743 // indirect - github.com/lightstep/lightstep-tracer-go v0.18.1 // indirect + github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20210210170715-a8dfcb80d3a7 // indirect + github.com/lightstep/lightstep-tracer-go v0.25.0 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect @@ -181,14 +217,13 @@ require ( github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/oklog/ulid v1.3.1 // indirect - github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect github.com/parnurzeal/gorequest v0.2.16 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common/sigv4 v0.1.0 // indirect - github.com/prometheus/exporter-toolkit v0.7.1 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/exporter-toolkit v0.8.2 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect github.com/sergi/go-diff v1.2.0 // indirect @@ -212,14 +247,14 @@ require ( go.etcd.io/etcd/api/v3 v3.5.4 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect go.etcd.io/etcd/client/v3 v3.5.4 // indirect - go.mongodb.org/mongo-driver v1.7.5 + go.mongodb.org/mongo-driver v1.11.0 go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/propagators/ot v1.4.0 // indirect + go.opentelemetry.io/contrib/propagators/ot v1.13.0 // indirect go.opentelemetry.io/otel v1.14.0 go.opentelemetry.io/otel/bridge/opentracing v1.14.0 // indirect go.opentelemetry.io/otel/sdk v1.14.0 // indirect go.opentelemetry.io/otel/trace v1.14.0 - go.uber.org/atomic v1.9.0 // indirect + go.uber.org/atomic v1.10.0 // indirect go.uber.org/goleak v1.2.1 // indirect golang.org/x/crypto v0.9.0 // indirect golang.org/x/mod v0.10.0 // indirect @@ -228,9 +263,8 @@ require ( golang.org/x/sys v0.8.0 // indirect golang.org/x/term v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect - golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect + golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.3 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.110.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect @@ -243,34 +277,35 @@ require ( helm.sh/helm/v3 v3.8.2 // indirect howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect k8s.io/apiextensions-apiserver v0.23.5 // indirect - k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect - k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect + k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715 // indirect + k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect moul.io/http2curl v1.0.0 // indirect - sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) require ( - github.com/OneOfOne/xxhash v1.2.6 // indirect github.com/clusternet/clusternet v0.13.0 github.com/go-logr/logr v1.2.4 // indirect - github.com/go-openapi/spec v0.20.6 // indirect - github.com/go-openapi/swag v0.21.1 // indirect + github.com/go-openapi/spec v0.20.7 // indirect + github.com/go-openapi/swag v0.22.3 // indirect github.com/go-playground/validator/v10 v10.11.2 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect - github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/urfave/cli/v2 v2.8.1 // indirect - go.uber.org/multierr v1.8.0 // indirect + go.uber.org/multierr v1.9.0 // indirect go.uber.org/zap v1.24.0 // indirect google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect ) replace ( github.com/go-resty/resty/v2 => github.com/ifooth/resty/v2 v2.0.0-20230223083514-3015979960de - github.com/prometheus/prometheus => github.com/prometheus/prometheus v1.8.2-0.20220308163432-03831554a519 - github.com/thanos-io/thanos => github.com/ifooth/thanos v0.26.1-0.20230707020703-bac1f168813b + // from thanos v0.31.0 + github.com/prometheus/prometheus => github.com/prometheus/prometheus v0.42.0 + github.com/thanos-io/thanos => github.com/LidolLxf/thanos v0.31.1-0.20230711084146-2b9427366eb0 github.com/vimeo/galaxycache => github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e + // from thanos v0.31.0 + gopkg.in/alecthomas/kingpin.v2 => github.com/alecthomas/kingpin/v2 v2.3.2 ) diff --git a/bcs-services/bcs-monitor/pkg/api/metrics/cluster_test.go b/bcs-services/bcs-monitor/pkg/api/metrics/cluster_test.go index c2c513bf79..88e1afefa2 100644 --- a/bcs-services/bcs-monitor/pkg/api/metrics/cluster_test.go +++ b/bcs-services/bcs-monitor/pkg/api/metrics/cluster_test.go @@ -13,12 +13,16 @@ package metrics import ( + "context" "fmt" + "net/http" "sync" "testing" + "time" "github.com/stretchr/testify/assert" + "github.com/Tencent/bk-bcs/bcs-services/bcs-monitor/pkg/component/promclient" "github.com/Tencent/bk-bcs/bcs-services/bcs-monitor/pkg/rest" bcstesting "github.com/Tencent/bk-bcs/bcs-services/bcs-monitor/pkg/testing" ) @@ -42,3 +46,24 @@ func TestGetClusterOverview(t *testing.T) { } wg.Wait() } + +// QueryRange的基准测试,需要启动query, storegw +// 需要配置环境变量 eg:export TEST_CONFIG_FILE=D:/project_config/bcs-monitor/bcs_monitor.yml +func BenchmarkQueryRange(b *testing.B) { + for i := 0; i < b.N; i++ { + ctx := context.Background() + header := http.Header{} + // query配置的启动端口 + rawURL := "http://127.0.0.1:10902" + // 前提是先把数据打入prometheus,才能查得到 + promql := "myMetric{cluster_id=\"1\", provider=\"PROMETHEUS\", bcs_cluster_id=\"1\"}" + endTime := time.Now() + startTime := endTime.Add(-time.Hour) + // 默认只返回 60 个点 + stepTime := endTime.Sub(startTime) / 60 + _, err := promclient.QueryRange(ctx, rawURL, header, promql, startTime, endTime, stepTime) + if err != nil { + fmt.Println(err) + } + } +} diff --git a/bcs-services/bcs-monitor/pkg/query/constants.go b/bcs-services/bcs-monitor/pkg/query/constants.go index 75ba43fd48..62b83595d6 100644 --- a/bcs-services/bcs-monitor/pkg/query/constants.go +++ b/bcs-services/bcs-monitor/pkg/query/constants.go @@ -32,6 +32,7 @@ const ( unhealthyStoreTimeout = time.Minute * 5 // 健康检查的时间 storeResponseTimeout = time.Second * 20 // 查询store的超时时间 defaultEvaluationInterval = time.Minute * 1 // 自查询的默认处理间隔。这里用不到 + endpointInfoTimeout = time.Minute * 5 // thanos端点超时时间 ) var queryReplicaLabels = []string{"prometheus_replica"} diff --git a/bcs-services/bcs-monitor/pkg/query/discovery_client.go b/bcs-services/bcs-monitor/pkg/query/discovery_client.go index b04918814c..c15d7bd278 100644 --- a/bcs-services/bcs-monitor/pkg/query/discovery_client.go +++ b/bcs-services/bcs-monitor/pkg/query/discovery_client.go @@ -161,6 +161,7 @@ func resolveStoreProvider(ctxm context.Context, g *run.Group, dnsStoreProvider * func getEndpoints(kitLogger gokit.Logger, reg *prometheus.Registry, strictStoreList []string, dnsStoreProvider *dns.Provider, dialOpts []grpc.DialOption) *query.EndpointSet { return query.NewEndpointSet( + time.Now, kitLogger, reg, func() (specs []*query.GRPCEndpointSpec) { @@ -177,6 +178,8 @@ func getEndpoints(kitLogger gokit.Logger, reg *prometheus.Registry, strictStoreL }, dialOpts, unhealthyStoreTimeout, + endpointInfoTimeout, // thanos端点超时时间 + ) } @@ -373,7 +376,7 @@ func NewHTTPSDClient(ctx context.Context, kitLogger gokit.Logger, conf httpdisco // Run File Service Discovery and update the store set when the files are modified. u.Host = addr conf.URL = u.String() - sdClient, err := httpdiscovery.NewDiscovery(&conf, kitLogger) + sdClient, err := httpdiscovery.NewDiscovery(&conf, kitLogger, nil) if err != nil { return nil, err } diff --git a/bcs-services/bcs-monitor/pkg/query/engine.go b/bcs-services/bcs-monitor/pkg/query/engine.go index 38cc3db6fd..4ddfe5f3c1 100644 --- a/bcs-services/bcs-monitor/pkg/query/engine.go +++ b/bcs-services/bcs-monitor/pkg/query/engine.go @@ -32,7 +32,7 @@ import ( func NewQueryableCreator(reg *prometheus.Registry, kitLogger gokit.Logger, discoveryClient *DiscoveryClient) query.QueryableCreator { proxy := store.NewProxyStore(kitLogger, reg, discoveryClient.Endpoints().GetStoreClients, component.Query, nil, - storeResponseTimeout) + storeResponseTimeout, store.LazyRetrieval) queryableCreator := query.NewQueryableCreator( kitLogger, diff --git a/bcs-services/bcs-monitor/pkg/query/query.go b/bcs-services/bcs-monitor/pkg/query/query.go index db72cc0424..d5b4a9216d 100644 --- a/bcs-services/bcs-monitor/pkg/query/query.go +++ b/bcs-services/bcs-monitor/pkg/query/query.go @@ -16,6 +16,7 @@ package query import ( "context" + "math" "path" "time" @@ -26,7 +27,9 @@ import ( "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/route" + "github.com/prometheus/prometheus/promql" v1 "github.com/thanos-io/thanos/pkg/api/query" + "github.com/thanos-io/thanos/pkg/compact/downsample" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/extprom" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" @@ -35,6 +38,7 @@ import ( "github.com/thanos-io/thanos/pkg/prober" "github.com/thanos-io/thanos/pkg/query" httpserver "github.com/thanos-io/thanos/pkg/server/http" + "github.com/thanos-io/thanos/pkg/store" "github.com/thanos-io/thanos/pkg/ui" "github.com/Tencent/bk-bcs/bcs-services/bcs-monitor/pkg/config" @@ -113,11 +117,25 @@ func NewQueryAPI( prefix = path.Join(config.G.Web.RoutePrefix, config.QueryServicePrefix) } ui.NewQueryUI(kitLogger, discoveryClient.Endpoints(), prefix, "", "").Register(router, ins) - + engineOpts := promql.EngineOpts{ + Logger: kitLogger, + Reg: reg, + MaxSamples: math.MaxInt32, + Timeout: queryTimeout, + LookbackDelta: lookbackDelta, + NoStepSubqueryIntervalFn: func(int64) int64 { + return defaultEvaluationInterval.Milliseconds() + }, + } + lookBackDeltaCreator := LookBackDeltaFactory(engineOpts, dynamicLookbackDelta) + queryTelemetryDurationQuantiles := []float64{0.1, 0.25, 0.75, 1.25, 1.75, 2.5, 3, 5, 10} + queryTelemetrySamplesQuantiles := []int64{100, 1000, 10000, 100000, 1000000} + queryTelemetrySeriesQuantiles := []int64{10, 100, 1000, 10000, 100000} api := v1.NewQueryAPI( kitLogger, discoveryClient.Endpoints().GetEndpointStatus, - queryEngine, + queryEngine(math.MaxInt32), + lookBackDeltaCreator, queryableCreator, NewEmptyRuleClient(), NewEmptyTargetClient(), @@ -138,7 +156,13 @@ func NewQueryAPI( false, // disableCORS gate.New( extprom.WrapRegistererWithPrefix("bcs_monitor_query_concurrent_", reg), - maxConcurrentQueries, + maxConcurrentQueries, "bcs_monitor_query", + ), + store.NewSeriesStatsAggregator( + reg, + queryTelemetryDurationQuantiles, + queryTelemetrySamplesQuantiles, + queryTelemetrySeriesQuantiles, ), reg, ) @@ -183,3 +207,38 @@ func (a *QueryAPI) Close(err error) { a.statusProber.NotReady(err) a.srv.Shutdown(err) } + +// LookBackDeltaFactory creates from 1 to 3 lookback deltas depending on +// dynamicLookbackDelta and eo.LookbackDelta and returns a function +// that returns appropriate lookback delta for given maxSourceResolutionMillis. +func LookBackDeltaFactory(eo promql.EngineOpts, dynamicLookbackDelta bool) func(int64) time.Duration { + resolutions := []int64{downsample.ResLevel0} + if dynamicLookbackDelta { + resolutions = []int64{downsample.ResLevel0, downsample.ResLevel1, downsample.ResLevel2} + } + var ( + lds = make([]time.Duration, len(resolutions)) + ld = eo.LookbackDelta.Milliseconds() + ) + + lookBackDelta := eo.LookbackDelta + for i, r := range resolutions { + if ld < r { + lookBackDelta = time.Duration(r) * time.Millisecond + } + + lds[i] = lookBackDelta + } + return func(maxSourceResolutionMillis int64) time.Duration { + for i := len(resolutions) - 1; i >= 1; i-- { + left := resolutions[i-1] + if resolutions[i-1] < ld { + left = ld + } + if left < maxSourceResolutionMillis { + return lds[i] + } + } + return lds[0] + } +} diff --git a/bcs-services/bcs-monitor/pkg/storegw/clientutil/clientutil_test.go b/bcs-services/bcs-monitor/pkg/storegw/clientutil/clientutil_test.go index cf501b0947..ec3d723e51 100644 --- a/bcs-services/bcs-monitor/pkg/storegw/clientutil/clientutil_test.go +++ b/bcs-services/bcs-monitor/pkg/storegw/clientutil/clientutil_test.go @@ -16,10 +16,10 @@ package clientutil import ( "testing" + "github.com/efficientgo/core/testutil" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/prompb" "github.com/thanos-io/thanos/pkg/store/storepb" - "github.com/thanos-io/thanos/pkg/testutil" ) func TestGetCluterID(t *testing.T) { diff --git a/bcs-services/bcs-monitor/pkg/storegw/store.go b/bcs-services/bcs-monitor/pkg/storegw/store.go index d403ab9ab4..355941e17c 100644 --- a/bcs-services/bcs-monitor/pkg/storegw/store.go +++ b/bcs-services/bcs-monitor/pkg/storegw/store.go @@ -96,7 +96,7 @@ func NewStore(ctx context.Context, logger log.Logger, reg *prometheus.Registry, nilLogger := &NilLogger{} g := grpcserver.New(nilLogger, _reg, nil, nil, nil, component.Store, grpcProbe, - grpcserver.WithServer(store.RegisterStoreServer(storeSvr)), + grpcserver.WithServer(store.RegisterStoreServer(storeSvr, logger)), grpcserver.WithListen(address), grpcserver.WithGracePeriod(time.Duration(0)), )