Skip to content

Commit

Permalink
agent: push metrics to S3 (#898)
Browse files Browse the repository at this point in the history 
Add s3 billing destination. 

Part of a neondatabase/cloud#9825

---------

Signed-off-by: Oleg Vasilev <[email protected]>
  • Loading branch information
@Omrigan
Omrigan authored May 8, 2024
1 parent 669bb52 commit da5dbf2
Show file tree
Hide file tree
Showing 15 changed files with 697 additions and 46 deletions.
1 change: 1 addition & 0 deletions .github/workflows/e2e-test.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ jobs:

- name: Install dependencies
run: |
sudo apt install -y python3-venv
make e2e-tools
echo $(pwd)/bin >> $GITHUB_PATH
Expand Down
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,4 @@ testbin/*
neonvm/hack/kernel/vmlinuz

rendered_manifests
.venv
7 changes: 6 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -351,7 +351,7 @@ k3d-load: k3d # Push docker images to the k3d cluster.
##@ End-to-End tests

.PHONE: e2e-tools
e2e-tools: k3d kind kubectl kuttl ## Donwnload tools for e2e tests locally if necessary.
e2e-tools: k3d kind kubectl kuttl python-init ## Donwnload tools for e2e tests locally if necessary.

.PHONE: e2e
e2e: check-local-context e2e-tools ## Run e2e kuttl tests
Expand Down Expand Up @@ -461,3 +461,8 @@ $(K3D): $(LOCALBIN)
.PHONY: cert-manager
cert-manager: check-local-context kubectl ## install cert-manager to cluster
$(KUBECTL) apply -f https://github.com/cert-manager/cert-manager/releases/latest/download/cert-manager.yaml

.PHONY: python-init
python-init:
python3 -m venv tests/e2e/.venv
tests/e2e/.venv/bin/pip install -r requirements.txt
17 changes: 17 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ replace (

require (
github.com/alessio/shellescape v1.4.1
github.com/aws/aws-sdk-go-v2/config v1.15.12
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
github.com/cilium/cilium v1.12.14
github.com/containerd/cgroups/v3 v3.0.1
github.com/containernetworking/cni v1.1.1
Expand Down Expand Up @@ -92,6 +94,21 @@ require (
github.com/BurntSushi/toml v1.2.1 // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/NYTimes/gziphandler v1.1.1 // indirect
github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.12.7 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.10 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.8 // indirect
github.com/aws/smithy-go v1.20.2 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
Expand Down
42 changes: 42 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,45 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5
github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/aws/aws-sdk-go-v2 v1.16.6/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw=
github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
github.com/aws/aws-sdk-go-v2/config v1.15.12 h1:D4mdf0cOSmZRgJe0DDOd1Qm6tkwHJ7r5i1lz0asa+AA=
github.com/aws/aws-sdk-go-v2/config v1.15.12/go.mod h1:oxRNnH11J580bxDEXyfTqfB3Auo2fxzhV052LD4HnyA=
github.com/aws/aws-sdk-go-v2/credentials v1.12.7 h1:e2DcCR0gP+T2zVj5eQPMQoRdxo+vd2p9BkpJ72BdyzA=
github.com/aws/aws-sdk-go-v2/credentials v1.12.7/go.mod h1:8b1nSHdDaKLho9VEK+K8WivifA/2K5pPm4sfI21NlQ8=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7 h1:8yi2ORCwXpXEPnj0vP3DjYhejwDQD/5klgBoxXcKOxY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.7/go.mod h1:81k6q0UUZj6AdQZ1E/VQ27cLrTUpJGraZR6/hVHRxjE=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.13/go.mod h1:wLLesU+LdMZDM3U0PP9vZXJW39zmD/7L4nY2pSrYZ/g=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.7/go.mod h1:93Uot80ddyVzSl//xEJreNKMhxntr71WtR3v/A1cRYk=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14 h1:bJv4Y9QOiW0GZPStgLgpGrpdfRDSR3XM4V4M3YCQRZo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.14/go.mod h1:R1HF8ZDdcRFfAGF+13En4LSHi2IrrNuPQCaxgWCeGyY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.7/go.mod h1:HvVdEh/x4jsPBsjNvDy+MH3CDCPy4gTZEzFe2r4uJY8=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
github.com/aws/aws-sdk-go-v2/service/sso v1.11.10 h1:icon5WWg9Yg5nkB0pJF6bfKw6M0xozukeGKSNKtnqzw=
github.com/aws/aws-sdk-go-v2/service/sso v1.11.10/go.mod h1:UHxA35uPrCykRySBV5iSPZhZRlYnWSS2c/aaZVsoU94=
github.com/aws/aws-sdk-go-v2/service/sts v1.16.8 h1:GLGfpqX+1bmjNvUJkwB1ZaDpNFXQwJ3z9RkQDA58OBY=
github.com/aws/aws-sdk-go-v2/service/sts v1.16.8/go.mod h1:50YdFq1WIuxA0AGrygvYGucnNYrG24WYzu5fNp7lMgY=
github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
Expand Down Expand Up @@ -271,6 +310,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
Expand Down Expand Up @@ -313,6 +353,8 @@ github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf
github.com/ishidawataru/sctp v0.0.0-20210707070123-9a39160e9062 h1:G1+wBT0dwjIrBdLy0MIG0i+E4CQxEnedHXdauJEIH6g=
github.com/ishidawataru/sctp v0.0.0-20210707070123-9a39160e9062/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
Expand Down
51 changes: 44 additions & 7 deletions pkg/agent/billing/billing.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,19 @@ type Config struct {

type ClientsConfig struct {
HTTP *HTTPClientConfig `json:"http"`
S3 *S3ClientConfig `json:"s3"`
}

type HTTPClientConfig struct {
BaseClientConfig
URL string `json:"url"`
}

type S3ClientConfig struct {
BaseClientConfig
billing.S3ClientConfig
}

type BaseClientConfig struct {
PushEverySeconds uint `json:"pushEverySeconds"`
PushRequestTimeoutSeconds uint `json:"pushRequestTimeoutSeconds"`
Expand Down Expand Up @@ -80,24 +86,55 @@ type vmMetricsSeconds struct {
activeTime time.Duration
}

func RunBillingMetricsCollector(
backgroundCtx context.Context,
func StartBillingMetricsCollector(
ctx context.Context,
parentLogger *zap.Logger,
conf *Config,
store VMStoreForNode,
metrics PromMetrics,
) {
) error {
logger := parentLogger.Named("billing")

var clients []clientInfo

if c := conf.Clients.HTTP; c != nil {
clients = append(clients, clientInfo{
client: billing.NewClient(c.URL, http.DefaultClient),
client: billing.NewHTTPClient(c.URL, http.DefaultClient),
name: "http",
config: c.BaseClientConfig,
})
}
if c := conf.Clients.S3; c != nil {
client, err := billing.NewS3Client(ctx, c.S3ClientConfig)
if err != nil {
return fmt.Errorf("Failed to create S3 client: %w", err)
}
logger.Info("Created S3 client",
zap.String("bucket", c.Bucket),
zap.String("region", c.Region),
zap.String("prefixInBucket", c.PrefixInBucket),
zap.String("endpoint", c.Endpoint),
)
clients = append(clients, clientInfo{
client: client,
name: "s3",
config: c.BaseClientConfig,
})
}

logger := parentLogger.Named("billing")
// TODO: catch panics here, bubble those into a clean-ish shutdown.
go runBillingMetricsCollector(ctx, logger, conf, store, metrics, clients)
return nil
}

func runBillingMetricsCollector(
ctx context.Context,
logger *zap.Logger,
conf *Config,
store VMStoreForNode,
metrics PromMetrics,
clients []clientInfo,
) {

collectTicker := time.NewTicker(time.Second * time.Duration(conf.CollectEverySeconds))
defer collectTicker.Stop()
Expand Down Expand Up @@ -141,15 +178,15 @@ func RunBillingMetricsCollector(
select {
case <-collectTicker.C:
logger.Info("Collecting billing state")
if store.Stopped() && backgroundCtx.Err() == nil {
if store.Stopped() && ctx.Err() == nil {
err := errors.New("VM store stopped but background context is still live")
logger.Panic("Validation check failed", zap.Error(err))
}
state.collect(logger, store, metrics)
case <-accumulateTicker.C:
logger.Info("Creating billing batch")
state.drainEnqueue(logger, conf, billing.GetHostname(), queueWriters)
case <-backgroundCtx.Done():
case <-ctx.Done():
return
}
}
Expand Down
10 changes: 6 additions & 4 deletions pkg/agent/billing/send.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,13 +112,13 @@ func (s eventSender) sendAllCurrentEvents(logger *zap.Logger) {
return
}

traceID := s.client.GenerateTraceID()
traceID := billing.GenerateTraceID()

logger.Info(
"Pushing billing events",
zap.Int("count", count),
zap.String("traceID", string(traceID)),
zap.String("url", s.client.URL),
s.client.LogFields(),
)

reqStart := time.Now()
Expand All @@ -138,7 +138,7 @@ func (s eventSender) sendAllCurrentEvents(logger *zap.Logger) {
zap.Int("count", count),
zap.Duration("after", reqDuration),
zap.String("traceID", string(traceID)),
zap.String("url", s.client.URL),
s.client.LogFields(),
zap.Int("total", total),
zap.Duration("totalTime", time.Since(startTime)),
zap.Error(err),
Expand All @@ -151,6 +151,8 @@ func (s eventSender) sendAllCurrentEvents(logger *zap.Logger) {
rootErr = "JSON marshaling"
case billing.UnexpectedStatusCodeError:
rootErr = fmt.Sprintf("HTTP code %d", e.StatusCode)
case billing.S3Error:
rootErr = "S3 error"
default:
rootErr = util.RootError(err).Error()
}
Expand All @@ -170,7 +172,7 @@ func (s eventSender) sendAllCurrentEvents(logger *zap.Logger) {
zap.Int("count", count),
zap.Duration("after", reqDuration),
zap.String("traceID", string(traceID)),
zap.String("url", s.client.URL),
s.client.LogFields(),
zap.Int("total", total),
zap.Duration("totalTime", currentTotalTime),
)
Expand Down
21 changes: 17 additions & 4 deletions pkg/agent/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,14 +163,27 @@ func (c *Config) validate() error {
zeroTmpl = "field %q cannot be zero"
)

validateBaseBillingConfig := func(cfg *billing.BaseClientConfig, key string) {
erc.Whenf(ec, cfg.PushEverySeconds == 0, zeroTmpl, fmt.Sprintf("%s.pushEverySeconds", key))
erc.Whenf(ec, cfg.PushRequestTimeoutSeconds == 0, zeroTmpl, fmt.Sprintf("%s.pushRequestTimeoutSeconds", key))
erc.Whenf(ec, cfg.MaxBatchSize == 0, zeroTmpl, fmt.Sprintf("%s.maxBatchSize", key))
}

erc.Whenf(ec, c.Billing.ActiveTimeMetricName == "", emptyTmpl, ".billing.activeTimeMetricName")
erc.Whenf(ec, c.Billing.CPUMetricName == "", emptyTmpl, ".billing.cpuMetricName")
erc.Whenf(ec, c.Billing.CollectEverySeconds == 0, zeroTmpl, ".billing.collectEverySeconds")
erc.Whenf(ec, c.Billing.AccumulateEverySeconds == 0, zeroTmpl, ".billing.accumulateEverySeconds")
erc.Whenf(ec, c.Billing.Clients.HTTP != nil && c.Billing.Clients.HTTP.PushEverySeconds == 0, zeroTmpl, ".billing.clients.http.pushEverySeconds")
erc.Whenf(ec, c.Billing.Clients.HTTP != nil && c.Billing.Clients.HTTP.PushRequestTimeoutSeconds == 0, zeroTmpl, ".billing.clients.http.pushRequestTimeoutSeconds")
erc.Whenf(ec, c.Billing.Clients.HTTP != nil && c.Billing.Clients.HTTP.MaxBatchSize == 0, zeroTmpl, ".billing.clients.http.maxBatchSize")
erc.Whenf(ec, c.Billing.Clients.HTTP != nil && c.Billing.Clients.HTTP.URL == "", emptyTmpl, ".billing.clients.http.url")
if c.Billing.Clients.HTTP != nil {
validateBaseBillingConfig(&c.Billing.Clients.HTTP.BaseClientConfig, ".billing.clients.http")
erc.Whenf(ec, c.Billing.Clients.HTTP.URL == "", emptyTmpl, ".billing.clients.http.url")
}
if c.Billing.Clients.S3 != nil {
validateBaseBillingConfig(&c.Billing.Clients.S3.BaseClientConfig, "billing.clients.s3")
erc.Whenf(ec, c.Billing.Clients.S3.Bucket == "", emptyTmpl, ".billing.clients.s3.bucket")
erc.Whenf(ec, c.Billing.Clients.S3.Region == "", emptyTmpl, ".billing.clients.s3.region")
erc.Whenf(ec, c.Billing.Clients.S3.PrefixInBucket == "", emptyTmpl, ".billing.clients.s3.prefixInBucket")
erc.Whenf(ec, c.Billing.Clients.S3.Endpoint == "", emptyTmpl, ".billing.clients.s3.endpoint")
}
erc.Whenf(ec, c.DumpState != nil && c.DumpState.Port == 0, zeroTmpl, ".dumpState.port")
erc.Whenf(ec, c.DumpState != nil && c.DumpState.TimeoutSeconds == 0, zeroTmpl, ".dumpState.timeoutSeconds")
erc.Whenf(ec, c.Metrics.Port == 0, zeroTmpl, ".metrics.port")
Expand Down
6 changes: 4 additions & 2 deletions pkg/agent/entrypoint.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,10 @@ func (r MainRunner) Run(logger *zap.Logger, ctx context.Context) error {
metrics := billing.NewPromMetrics()
metrics.MustRegister(globalPromReg)

// TODO: catch panics here, bubble those into a clean-ish shutdown.
go billing.RunBillingMetricsCollector(ctx, logger, &r.Config.Billing, storeForNode, metrics)
err = billing.StartBillingMetricsCollector(ctx, logger, &r.Config.Billing, storeForNode, metrics)
if err != nil {
return fmt.Errorf("error starting billing metrics collector: %w", err)
}

promLogger := logger.Named("prometheus")
if err := util.StartPrometheusMetricsServer(ctx, promLogger.Named("global"), 9100, globalPromReg); err != nil {
Expand Down
Loading

1 comment on commit da5dbf2

@sharnoff
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚀 🚀 🚀

Please sign in to comment.