From d8877a71fc1faac18539cd29fa736deccaf4ea92 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 5 Mar 2024 14:56:14 +0100 Subject: [PATCH 1/8] lp metrics: collect datasources and console options (#2870) --- cmd/crowdsec/crowdsec.go | 31 ++++++++++++++++++------------- cmd/crowdsec/main.go | 19 +++++++++++-------- cmd/crowdsec/serve.go | 14 +++++++------- pkg/csconfig/console.go | 29 +++++++++++++++++++++++++++++ test/bats/01_crowdsec.bats | 2 +- test/bats/01_crowdsec_lapi.bats | 2 +- 6 files changed, 67 insertions(+), 30 deletions(-) diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index d4cd2d3cf74..0d7d454edf2 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -23,39 +23,42 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) { +// initCrowdsec prepares the log processor service +func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, []acquisition.DataSource, error) { var err error if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil { - return nil, fmt.Errorf("while loading context: %w", err) + return nil, nil, fmt.Errorf("while loading context: %w", err) } // Start loading configs csParsers := parser.NewParsers(hub) if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { - return nil, fmt.Errorf("while loading parsers: %w", err) + return nil, nil, fmt.Errorf("while loading parsers: %w", err) } if err := LoadBuckets(cConfig, hub); err != nil { - return nil, fmt.Errorf("while loading scenarios: %w", err) + return nil, nil, fmt.Errorf("while loading scenarios: %w", err) } if err := appsec.LoadAppsecRules(hub); err != nil { - return nil, fmt.Errorf("while loading appsec rules: %w", err) + return nil, nil, fmt.Errorf("while loading appsec rules: %w", err) } - if err := LoadAcquisition(cConfig); err != nil { - return nil, fmt.Errorf("while loading acquisition config: %w", err) + datasources, err := LoadAcquisition(cConfig) + if err != nil { + return nil, nil, fmt.Errorf("while loading acquisition config: %w", err) } - return csParsers, nil + return csParsers, datasources, nil } -func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.Hub) error { +// runCrowdsec starts the log processor service +func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.Hub, datasources []acquisition.DataSource) error { inputEventChan = make(chan types.Event) inputLineChan = make(chan types.Event) - //start go-routines for parsing, buckets pour and outputs. + // start go-routines for parsing, buckets pour and outputs. parserWg := &sync.WaitGroup{} parsersTomb.Go(func() error { @@ -65,7 +68,8 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.H parsersTomb.Go(func() error { defer trace.CatchPanic("crowdsec/runParse") - if err := runParse(inputLineChan, inputEventChan, *parsers.Ctx, parsers.Nodes); err != nil { //this error will never happen as parser.Parse is not able to return errors + if err := runParse(inputLineChan, inputEventChan, *parsers.Ctx, parsers.Nodes); err != nil { + // this error will never happen as parser.Parse is not able to return errors log.Fatalf("starting parse error : %s", err) return err } @@ -161,7 +165,8 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.H return nil } -func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, agentReady chan bool) { +// serveCrowdsec wraps the log processor service +func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, datasources []acquisition.DataSource, agentReady chan bool) { crowdsecTomb.Go(func() error { defer trace.CatchPanic("crowdsec/serveCrowdsec") @@ -171,7 +176,7 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub log.Debugf("running agent after %s ms", time.Since(crowdsecT0)) agentReady <- true - if err := runCrowdsec(cConfig, parsers, hub); err != nil { + if err := runCrowdsec(cConfig, parsers, hub, datasources); err != nil { log.Fatalf("unable to start crowdsec routines: %s", err) } }() diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index 2040141bb3e..7f3070b5f29 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -1,6 +1,7 @@ package main import ( + "errors" "flag" "fmt" _ "net/http/pprof" @@ -10,7 +11,6 @@ import ( "strings" "time" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -95,7 +95,7 @@ func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error { holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent) if err != nil { - return fmt.Errorf("scenario loading failed: %v", err) + return fmt.Errorf("scenario loading failed: %w", err) } if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled { @@ -107,7 +107,7 @@ func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error { return nil } -func LoadAcquisition(cConfig *csconfig.Config) error { +func LoadAcquisition(cConfig *csconfig.Config) ([]acquisition.DataSource, error) { var err error if flags.SingleFileType != "" && flags.OneShotDSN != "" { @@ -116,20 +116,20 @@ func LoadAcquisition(cConfig *csconfig.Config) error { dataSources, err = acquisition.LoadAcquisitionFromDSN(flags.OneShotDSN, flags.Labels, flags.Transform) if err != nil { - return errors.Wrapf(err, "failed to configure datasource for %s", flags.OneShotDSN) + return nil, fmt.Errorf("failed to configure datasource for %s: %w", flags.OneShotDSN, err) } } else { dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec) if err != nil { - return err + return nil, err } } if len(dataSources) == 0 { - return fmt.Errorf("no datasource enabled") + return nil, errors.New("no datasource enabled") } - return nil + return dataSources, nil } var ( @@ -272,7 +272,7 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo } if cConfig.DisableAPI && cConfig.DisableAgent { - return nil, errors.New("You must run at least the API Server or crowdsec") + return nil, errors.New("you must run at least the API Server or crowdsec") } if flags.OneShotDSN != "" && flags.SingleFileType == "" { @@ -360,11 +360,14 @@ func main() { if err != nil { log.Fatalf("could not create CPU profile: %s", err) } + log.Infof("CPU profile will be written to %s", flags.CpuProfile) + if err := pprof.StartCPUProfile(f); err != nil { f.Close() log.Fatalf("could not start CPU profile: %s", err) } + defer f.Close() defer pprof.StopCPUProfile() } diff --git a/cmd/crowdsec/serve.go b/cmd/crowdsec/serve.go index 22f65b927a0..c8ccd4d5d70 100644 --- a/cmd/crowdsec/serve.go +++ b/cmd/crowdsec/serve.go @@ -86,7 +86,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { return nil, fmt.Errorf("while loading hub index: %w", err) } - csParsers, err := initCrowdsec(cConfig, hub) + csParsers, datasources, err := initCrowdsec(cConfig, hub) if err != nil { return nil, fmt.Errorf("unable to init crowdsec: %w", err) } @@ -103,7 +103,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { } agentReady := make(chan bool, 1) - serveCrowdsec(csParsers, cConfig, hub, agentReady) + serveCrowdsec(csParsers, cConfig, hub, datasources, agentReady) } log.Printf("Reload is finished") @@ -230,7 +230,7 @@ func drainChan(c chan types.Event) { for { select { case _, ok := <-c: - if !ok { //closed + if !ok { // closed return } default: @@ -256,8 +256,8 @@ func HandleSignals(cConfig *csconfig.Config) error { exitChan := make(chan error) - //Always try to stop CPU profiling to avoid passing flags around - //It's a noop if profiling is not enabled + // Always try to stop CPU profiling to avoid passing flags around + // It's a noop if profiling is not enabled defer pprof.StopCPUProfile() go func() { @@ -369,14 +369,14 @@ func Serve(cConfig *csconfig.Config, agentReady chan bool) error { return fmt.Errorf("while loading hub index: %w", err) } - csParsers, err := initCrowdsec(cConfig, hub) + csParsers, datasources, err := initCrowdsec(cConfig, hub) if err != nil { return fmt.Errorf("crowdsec init: %w", err) } // if it's just linting, we're done if !flags.TestMode { - serveCrowdsec(csParsers, cConfig, hub, agentReady) + serveCrowdsec(csParsers, cConfig, hub, datasources, agentReady) } else { agentReady <- true } diff --git a/pkg/csconfig/console.go b/pkg/csconfig/console.go index 01e74a94db4..4c14f5f7d49 100644 --- a/pkg/csconfig/console.go +++ b/pkg/csconfig/console.go @@ -37,6 +37,35 @@ type ConsoleConfig struct { ShareContext *bool `yaml:"share_context"` } +func (c *ConsoleConfig) EnabledOptions() []string { + ret := []string{} + if c == nil { + return ret + } + + if c.ShareCustomScenarios != nil && *c.ShareCustomScenarios { + ret = append(ret, SEND_CUSTOM_SCENARIOS) + } + + if c.ShareTaintedScenarios != nil && *c.ShareTaintedScenarios { + ret = append(ret, SEND_TAINTED_SCENARIOS) + } + + if c.ShareManualDecisions != nil && *c.ShareManualDecisions { + ret = append(ret, SEND_MANUAL_SCENARIOS) + } + + if c.ConsoleManagement != nil && *c.ConsoleManagement { + ret = append(ret, CONSOLE_MANAGEMENT) + } + + if c.ShareContext != nil && *c.ShareContext { + ret = append(ret, SEND_CONTEXT) + } + + return ret +} + func (c *ConsoleConfig) IsPAPIEnabled() bool { if c == nil || c.ConsoleManagement == nil { return false diff --git a/test/bats/01_crowdsec.bats b/test/bats/01_crowdsec.bats index a585930e34c..7051b4d33a3 100644 --- a/test/bats/01_crowdsec.bats +++ b/test/bats/01_crowdsec.bats @@ -38,7 +38,7 @@ teardown() { @test "crowdsec (no api and no agent)" { rune -0 wait-for \ - --err "You must run at least the API Server or crowdsec" \ + --err "you must run at least the API Server or crowdsec" \ "${CROWDSEC}" -no-api -no-cs } diff --git a/test/bats/01_crowdsec_lapi.bats b/test/bats/01_crowdsec_lapi.bats index 4819d724fea..233340e500f 100644 --- a/test/bats/01_crowdsec_lapi.bats +++ b/test/bats/01_crowdsec_lapi.bats @@ -28,7 +28,7 @@ teardown() { @test "lapi (.api.server.enable=false)" { rune -0 config_set '.api.server.enable=false' rune -1 "${CROWDSEC}" -no-cs - assert_stderr --partial "You must run at least the API Server or crowdsec" + assert_stderr --partial "you must run at least the API Server or crowdsec" } @test "lapi (no .api.server.listen_uri)" { From 5356ccc6cd138faa0f41fabaa5733bb94d9fb017 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:42:57 +0100 Subject: [PATCH 2/8] cron: spread server load when upgrading hub and data files (#2873) --- config/crowdsec.cron.daily | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/crowdsec.cron.daily b/config/crowdsec.cron.daily index 1c110df38fc..9c488d29884 100644 --- a/config/crowdsec.cron.daily +++ b/config/crowdsec.cron.daily @@ -2,12 +2,13 @@ test -x /usr/bin/cscli || exit 0 +# splay hub upgrade and crowdsec reload +sleep "$(seq 1 300 | shuf -n 1)" + /usr/bin/cscli --error hub update upgraded=$(/usr/bin/cscli --error hub upgrade) if [ -n "$upgraded" ]; then - # splay initial metrics push - sleep $(seq 1 90 | shuf -n 1) systemctl reload crowdsec fi From e611d01c90f3e57d591f3875c2e6c629cba9c68a Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Wed, 6 Mar 2024 14:27:05 +0100 Subject: [PATCH 3/8] cscli: hide hashed api keys (#2874) * cscli: hide hashed api keys * lint --- docker/test/tests/test_bouncer.py | 3 --- pkg/database/bouncers.go | 32 +++++++++++++++++++----------- pkg/database/ent/bouncer.go | 5 ++--- pkg/database/ent/schema/bouncer.go | 2 +- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/docker/test/tests/test_bouncer.py b/docker/test/tests/test_bouncer.py index 1324c3bd38c..98b86de858c 100644 --- a/docker/test/tests/test_bouncer.py +++ b/docker/test/tests/test_bouncer.py @@ -36,8 +36,6 @@ def test_register_bouncer_env(crowdsec, flavor): bouncer1, bouncer2 = j assert bouncer1['name'] == 'bouncer1name' assert bouncer2['name'] == 'bouncer2name' - assert bouncer1['api_key'] == hex512('bouncer1key') - assert bouncer2['api_key'] == hex512('bouncer2key') # add a second bouncer at runtime res = cs.cont.exec_run('cscli bouncers add bouncer3name -k bouncer3key') @@ -48,7 +46,6 @@ def test_register_bouncer_env(crowdsec, flavor): assert len(j) == 3 bouncer3 = j[2] assert bouncer3['name'] == 'bouncer3name' - assert bouncer3['api_key'] == hex512('bouncer3key') # remove all bouncers res = cs.cont.exec_run('cscli bouncers delete bouncer1name bouncer2name bouncer3name') diff --git a/pkg/database/bouncers.go b/pkg/database/bouncers.go index 496b9b6cc9c..2cc6b9dcb47 100644 --- a/pkg/database/bouncers.go +++ b/pkg/database/bouncers.go @@ -33,6 +33,7 @@ func (c *Client) ListBouncers() ([]*ent.Bouncer, error) { if err != nil { return nil, errors.Wrapf(QueryFail, "listing bouncers: %s", err) } + return result, nil } @@ -48,8 +49,10 @@ func (c *Client) CreateBouncer(name string, ipAddr string, apiKey string, authTy if ent.IsConstraintError(err) { return nil, fmt.Errorf("bouncer %s already exists", name) } - return nil, fmt.Errorf("unable to create bouncer: %s", err) + + return nil, fmt.Errorf("unable to create bouncer: %w", err) } + return bouncer, nil } @@ -63,7 +66,7 @@ func (c *Client) DeleteBouncer(name string) error { } if nbDeleted == 0 { - return fmt.Errorf("bouncer doesn't exist") + return errors.New("bouncer doesn't exist") } return nil @@ -74,36 +77,41 @@ func (c *Client) BulkDeleteBouncers(bouncers []*ent.Bouncer) (int, error) { for i, b := range bouncers { ids[i] = b.ID } + nbDeleted, err := c.Ent.Bouncer.Delete().Where(bouncer.IDIn(ids...)).Exec(c.CTX) if err != nil { - return nbDeleted, fmt.Errorf("unable to delete bouncers: %s", err) + return nbDeleted, fmt.Errorf("unable to delete bouncers: %w", err) } + return nbDeleted, nil } -func (c *Client) UpdateBouncerLastPull(lastPull time.Time, ID int) error { - _, err := c.Ent.Bouncer.UpdateOneID(ID). +func (c *Client) UpdateBouncerLastPull(lastPull time.Time, id int) error { + _, err := c.Ent.Bouncer.UpdateOneID(id). SetLastPull(lastPull). Save(c.CTX) if err != nil { - return fmt.Errorf("unable to update machine last pull in database: %s", err) + return fmt.Errorf("unable to update machine last pull in database: %w", err) } + return nil } -func (c *Client) UpdateBouncerIP(ipAddr string, ID int) error { - _, err := c.Ent.Bouncer.UpdateOneID(ID).SetIPAddress(ipAddr).Save(c.CTX) +func (c *Client) UpdateBouncerIP(ipAddr string, id int) error { + _, err := c.Ent.Bouncer.UpdateOneID(id).SetIPAddress(ipAddr).Save(c.CTX) if err != nil { - return fmt.Errorf("unable to update bouncer ip address in database: %s", err) + return fmt.Errorf("unable to update bouncer ip address in database: %w", err) } + return nil } -func (c *Client) UpdateBouncerTypeAndVersion(bType string, version string, ID int) error { - _, err := c.Ent.Bouncer.UpdateOneID(ID).SetVersion(version).SetType(bType).Save(c.CTX) +func (c *Client) UpdateBouncerTypeAndVersion(bType string, version string, id int) error { + _, err := c.Ent.Bouncer.UpdateOneID(id).SetVersion(version).SetType(bType).Save(c.CTX) if err != nil { - return fmt.Errorf("unable to update bouncer type and version in database: %s", err) + return fmt.Errorf("unable to update bouncer type and version in database: %w", err) } + return nil } diff --git a/pkg/database/ent/bouncer.go b/pkg/database/ent/bouncer.go index fe189c3817e..203f49a432d 100644 --- a/pkg/database/ent/bouncer.go +++ b/pkg/database/ent/bouncer.go @@ -24,7 +24,7 @@ type Bouncer struct { // Name holds the value of the "name" field. Name string `json:"name"` // APIKey holds the value of the "api_key" field. - APIKey string `json:"api_key"` + APIKey string `json:"-"` // Revoked holds the value of the "revoked" field. Revoked bool `json:"revoked"` // IPAddress holds the value of the "ip_address" field. @@ -193,8 +193,7 @@ func (b *Bouncer) String() string { builder.WriteString("name=") builder.WriteString(b.Name) builder.WriteString(", ") - builder.WriteString("api_key=") - builder.WriteString(b.APIKey) + builder.WriteString("api_key=") builder.WriteString(", ") builder.WriteString("revoked=") builder.WriteString(fmt.Sprintf("%v", b.Revoked)) diff --git a/pkg/database/ent/schema/bouncer.go b/pkg/database/ent/schema/bouncer.go index c3081291254..986a1bf3ba8 100644 --- a/pkg/database/ent/schema/bouncer.go +++ b/pkg/database/ent/schema/bouncer.go @@ -21,7 +21,7 @@ func (Bouncer) Fields() []ent.Field { Default(types.UtcNow). UpdateDefault(types.UtcNow).Nillable().Optional().StructTag(`json:"updated_at"`), field.String("name").Unique().StructTag(`json:"name"`), - field.String("api_key").StructTag(`json:"api_key"`), // hash of api_key + field.String("api_key").Sensitive(), // hash of api_key field.Bool("revoked").StructTag(`json:"revoked"`), field.String("ip_address").Default("").Optional().StructTag(`json:"ip_address"`), field.String("type").Optional().StructTag(`json:"type"`), From 98560d0cf56d9c56366065b9a40a915844238468 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:29:10 +0100 Subject: [PATCH 4/8] bin/crowdsec: avoid writing errors twice when log_media=stdout (#2876) * bin/crowdsec: avoid writing errors twice when log_media=stdout simpler, correct hook usage * lint --- cmd/crowdsec/api.go | 2 +- cmd/crowdsec/fatalhook.go | 28 +++++++++++++++++++ cmd/crowdsec/hook.go | 43 ------------------------------ cmd/crowdsec/main.go | 21 ++++++++++----- cmd/crowdsec/metrics.go | 6 ++--- cmd/crowdsec/output.go | 8 +++--- cmd/crowdsec/parse.go | 2 +- cmd/crowdsec/pour.go | 19 ++++++++----- cmd/crowdsec/run_in_svc.go | 4 +-- cmd/crowdsec/run_in_svc_windows.go | 4 +-- 10 files changed, 68 insertions(+), 69 deletions(-) create mode 100644 cmd/crowdsec/fatalhook.go delete mode 100644 cmd/crowdsec/hook.go diff --git a/cmd/crowdsec/api.go b/cmd/crowdsec/api.go index 4ac5c3ce96f..995345a25e4 100644 --- a/cmd/crowdsec/api.go +++ b/cmd/crowdsec/api.go @@ -1,11 +1,11 @@ package main import ( + "errors" "fmt" "runtime" "time" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/crowdsecurity/go-cs-lib/trace" diff --git a/cmd/crowdsec/fatalhook.go b/cmd/crowdsec/fatalhook.go new file mode 100644 index 00000000000..84a57406a21 --- /dev/null +++ b/cmd/crowdsec/fatalhook.go @@ -0,0 +1,28 @@ +package main + +import ( + "io" + + log "github.com/sirupsen/logrus" +) + +// FatalHook is used to log fatal messages to stderr when the rest goes to a file +type FatalHook struct { + Writer io.Writer + LogLevels []log.Level +} + +func (hook *FatalHook) Fire(entry *log.Entry) error { + line, err := entry.String() + if err != nil { + return err + } + + _, err = hook.Writer.Write([]byte(line)) + + return err +} + +func (hook *FatalHook) Levels() []log.Level { + return hook.LogLevels +} diff --git a/cmd/crowdsec/hook.go b/cmd/crowdsec/hook.go deleted file mode 100644 index 28515d9e474..00000000000 --- a/cmd/crowdsec/hook.go +++ /dev/null @@ -1,43 +0,0 @@ -package main - -import ( - "io" - "os" - - log "github.com/sirupsen/logrus" -) - -type ConditionalHook struct { - Writer io.Writer - LogLevels []log.Level - Enabled bool -} - -func (hook *ConditionalHook) Fire(entry *log.Entry) error { - if hook.Enabled { - line, err := entry.String() - if err != nil { - return err - } - - _, err = hook.Writer.Write([]byte(line)) - - return err - } - - return nil -} - -func (hook *ConditionalHook) Levels() []log.Level { - return hook.LogLevels -} - -// The primal logging hook is set up before parsing config.yaml. -// Once config.yaml is parsed, the primal hook is disabled if the -// configured logger is writing to stderr. Otherwise it's used to -// report fatal errors and panics to stderr in addition to the log file. -var primalHook = &ConditionalHook{ - Writer: os.Stderr, - LogLevels: []log.Level{log.FatalLevel, log.PanicLevel}, - Enabled: true, -} diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index 7f3070b5f29..70f7d48dce4 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -72,7 +72,7 @@ type Flags struct { DisableCAPI bool Transform string OrderEvent bool - CpuProfile string + CPUProfile string } type labelsMap map[string]string @@ -181,7 +181,7 @@ func (f *Flags) Parse() { } flag.StringVar(&dumpFolder, "dump-data", "", "dump parsers/buckets raw outputs") - flag.StringVar(&f.CpuProfile, "cpu-profile", "", "write cpu profile to file") + flag.StringVar(&f.CPUProfile, "cpu-profile", "", "write cpu profile to file") flag.Parse() } @@ -249,7 +249,12 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo return nil, err } - primalHook.Enabled = (cConfig.Common.LogMedia != "stdout") + if cConfig.Common.LogMedia != "stdout" { + log.AddHook(&FatalHook{ + Writer: os.Stderr, + LogLevels: []log.Level{log.FatalLevel, log.PanicLevel}, + }) + } if err := csconfig.LoadFeatureFlagsFile(configFile, log.StandardLogger()); err != nil { return nil, err @@ -323,7 +328,9 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo var crowdsecT0 time.Time func main() { - log.AddHook(primalHook) + // The initial log level is INFO, even if the user provided an -error or -warning flag + // because we need feature flags before parsing cli flags + log.SetFormatter(&log.TextFormatter{TimestampFormat: time.RFC3339, FullTimestamp: true}) if err := fflag.RegisterAllFeatures(); err != nil { log.Fatalf("failed to register features: %s", err) @@ -355,13 +362,13 @@ func main() { os.Exit(0) } - if flags.CpuProfile != "" { - f, err := os.Create(flags.CpuProfile) + if flags.CPUProfile != "" { + f, err := os.Create(flags.CPUProfile) if err != nil { log.Fatalf("could not create CPU profile: %s", err) } - log.Infof("CPU profile will be written to %s", flags.CpuProfile) + log.Infof("CPU profile will be written to %s", flags.CPUProfile) if err := pprof.StartCPUProfile(f); err != nil { f.Close() diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index 563bb56bfc9..aed43db00c8 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -104,12 +104,12 @@ func computeDynamicMetrics(next http.Handler, dbClient *database.Client) http.Ha return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // catch panics here because they are not handled by servePrometheus defer trace.CatchPanic("crowdsec/computeDynamicMetrics") - //update cache metrics (stash) + // update cache metrics (stash) cache.UpdateCacheMetrics() - //update cache metrics (regexp) + // update cache metrics (regexp) exprhelpers.UpdateRegexpCacheMetrics() - //decision metrics are only relevant for LAPI + // decision metrics are only relevant for LAPI if dbClient == nil { next.ServeHTTP(w, r) return diff --git a/cmd/crowdsec/output.go b/cmd/crowdsec/output.go index c4a2c0b6ac1..ac05b502e52 100644 --- a/cmd/crowdsec/output.go +++ b/cmd/crowdsec/output.go @@ -27,7 +27,7 @@ func dedupAlerts(alerts []types.RuntimeAlert) ([]*models.Alert, error) { } for k, src := range alert.Sources { - refsrc := *alert.Alert //copy + refsrc := *alert.Alert // copy log.Tracef("source[%s]", k) @@ -81,7 +81,7 @@ LOOP: cacheMutex.Unlock() if err := PushAlerts(cachecopy, client); err != nil { log.Errorf("while pushing to api : %s", err) - //just push back the events to the queue + // just push back the events to the queue cacheMutex.Lock() cache = append(cache, cachecopy...) cacheMutex.Unlock() @@ -110,8 +110,8 @@ LOOP: return fmt.Errorf("postoverflow failed: %w", err) } log.Printf("%s", *event.Overflow.Alert.Message) - //if the Alert is nil, it's to signal bucket is ready for GC, don't track this - //dump after postoveflow processing to avoid missing whitelist info + // if the Alert is nil, it's to signal bucket is ready for GC, don't track this + // dump after postoveflow processing to avoid missing whitelist info if dumpStates && event.Overflow.Alert != nil { if bucketOverflows == nil { bucketOverflows = make([]types.Event, 0) diff --git a/cmd/crowdsec/parse.go b/cmd/crowdsec/parse.go index c62eeb5869d..53c9ee65d4f 100644 --- a/cmd/crowdsec/parse.go +++ b/cmd/crowdsec/parse.go @@ -11,7 +11,6 @@ import ( ) func runParse(input chan types.Event, output chan types.Event, parserCTX parser.UnixParserCtx, nodes []parser.Node) error { - LOOP: for { select { @@ -56,5 +55,6 @@ LOOP: output <- parsed } } + return nil } diff --git a/cmd/crowdsec/pour.go b/cmd/crowdsec/pour.go index 3f717e3975d..388c7a6c1b3 100644 --- a/cmd/crowdsec/pour.go +++ b/cmd/crowdsec/pour.go @@ -4,27 +4,30 @@ import ( "fmt" "time" + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/prometheus/client_golang/prometheus" - log "github.com/sirupsen/logrus" ) func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets, cConfig *csconfig.Config) error { count := 0 + for { - //bucket is now ready + // bucket is now ready select { case <-bucketsTomb.Dying(): log.Infof("Bucket routine exiting") return nil case parsed := <-input: startTime := time.Now() + count++ if count%5000 == 0 { log.Infof("%d existing buckets", leaky.LeakyRoutineCount) - //when in forensics mode, garbage collect buckets + // when in forensics mode, garbage collect buckets if cConfig.Crowdsec.BucketsGCEnabled { if parsed.MarshaledTime != "" { z := &time.Time{} @@ -32,26 +35,30 @@ func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *lea log.Warningf("Failed to unmarshal time from event '%s' : %s", parsed.MarshaledTime, err) } else { log.Warning("Starting buckets garbage collection ...") + if err = leaky.GarbageCollectBuckets(*z, buckets); err != nil { - return fmt.Errorf("failed to start bucket GC : %s", err) + return fmt.Errorf("failed to start bucket GC : %w", err) } } } } } - //here we can bucketify with parsed + // here we can bucketify with parsed poured, err := leaky.PourItemToHolders(parsed, holders, buckets) if err != nil { log.Errorf("bucketify failed for: %v", parsed) continue } + elapsed := time.Since(startTime) globalPourHistogram.With(prometheus.Labels{"type": parsed.Line.Module, "source": parsed.Line.Src}).Observe(elapsed.Seconds()) + if poured { globalBucketPourOk.Inc() } else { globalBucketPourKo.Inc() } + if len(parsed.MarshaledTime) != 0 { if err := lastProcessedItem.UnmarshalText([]byte(parsed.MarshaledTime)); err != nil { log.Warningf("failed to unmarshal time from event : %s", err) diff --git a/cmd/crowdsec/run_in_svc.go b/cmd/crowdsec/run_in_svc.go index 5a8bc9a6cd3..58f4cdf005d 100644 --- a/cmd/crowdsec/run_in_svc.go +++ b/cmd/crowdsec/run_in_svc.go @@ -23,8 +23,8 @@ func StartRunSvc() error { defer trace.CatchPanic("crowdsec/StartRunSvc") - //Always try to stop CPU profiling to avoid passing flags around - //It's a noop if profiling is not enabled + // Always try to stop CPU profiling to avoid passing flags around + // It's a noop if profiling is not enabled defer pprof.StopCPUProfile() if cConfig, err = LoadConfig(flags.ConfigFile, flags.DisableAgent, flags.DisableAPI, false); err != nil { diff --git a/cmd/crowdsec/run_in_svc_windows.go b/cmd/crowdsec/run_in_svc_windows.go index 7845e9c58b5..c0aa18d7fc6 100644 --- a/cmd/crowdsec/run_in_svc_windows.go +++ b/cmd/crowdsec/run_in_svc_windows.go @@ -20,8 +20,8 @@ func StartRunSvc() error { defer trace.CatchPanic("crowdsec/StartRunSvc") - //Always try to stop CPU profiling to avoid passing flags around - //It's a noop if profiling is not enabled + // Always try to stop CPU profiling to avoid passing flags around + // It's a noop if profiling is not enabled defer pprof.StopCPUProfile() isRunninginService, err := svc.IsWindowsService() From 5731491b4e0948e5011e47a378a14f9a86e46b40 Mon Sep 17 00:00:00 2001 From: blotus Date: Thu, 7 Mar 2024 14:04:50 +0100 Subject: [PATCH 5/8] Auto detect if reading logs or storing sqlite db on a network share (#2241) --- pkg/acquisition/modules/file/file.go | 44 +++++++++-- pkg/csconfig/api_test.go | 1 + pkg/csconfig/database.go | 36 ++++++++- pkg/csconfig/database_test.go | 7 +- pkg/types/getfstype.go | 112 +++++++++++++++++++++++++++ pkg/types/getfstype_windows.go | 53 +++++++++++++ pkg/types/utils.go | 10 +++ 7 files changed, 251 insertions(+), 12 deletions(-) create mode 100644 pkg/types/getfstype.go create mode 100644 pkg/types/getfstype_windows.go diff --git a/pkg/acquisition/modules/file/file.go b/pkg/acquisition/modules/file/file.go index 4ea9466d457..9ab418a8442 100644 --- a/pkg/acquisition/modules/file/file.go +++ b/pkg/acquisition/modules/file/file.go @@ -38,9 +38,9 @@ type FileConfiguration struct { Filenames []string ExcludeRegexps []string `yaml:"exclude_regexps"` Filename string - ForceInotify bool `yaml:"force_inotify"` - MaxBufferSize int `yaml:"max_buffer_size"` - PollWithoutInotify bool `yaml:"poll_without_inotify"` + ForceInotify bool `yaml:"force_inotify"` + MaxBufferSize int `yaml:"max_buffer_size"` + PollWithoutInotify *bool `yaml:"poll_without_inotify"` configuration.DataSourceCommonCfg `yaml:",inline"` } @@ -330,7 +330,22 @@ func (f *FileSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) er continue } - tail, err := tail.TailFile(file, tail.Config{ReOpen: true, Follow: true, Poll: f.config.PollWithoutInotify, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekEnd}, Logger: log.NewEntry(log.StandardLogger())}) + inotifyPoll := true + if f.config.PollWithoutInotify != nil { + inotifyPoll = *f.config.PollWithoutInotify + } else { + networkFS, fsType, err := types.IsNetworkFS(file) + if err != nil { + f.logger.Warningf("Could not get fs type for %s : %s", file, err) + } + f.logger.Debugf("fs for %s is network: %t (%s)", file, networkFS, fsType) + if networkFS { + f.logger.Warnf("Disabling inotify poll on %s as it is on a network share. You can manually set poll_without_inotify to true to make this message disappear, or to false to enforce inotify poll", file) + inotifyPoll = false + } + } + + tail, err := tail.TailFile(file, tail.Config{ReOpen: true, Follow: true, Poll: inotifyPoll, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekEnd}, Logger: log.NewEntry(log.StandardLogger())}) if err != nil { f.logger.Errorf("Could not start tailing file %s : %s", file, err) continue @@ -413,8 +428,27 @@ func (f *FileSource) monitorNewFiles(out chan types.Event, t *tomb.Tomb) error { f.logger.Errorf("unable to close %s : %s", event.Name, err) continue } + + inotifyPoll := true + if f.config.PollWithoutInotify != nil { + inotifyPoll = *f.config.PollWithoutInotify + } else { + if f.config.PollWithoutInotify != nil { + inotifyPoll = *f.config.PollWithoutInotify + } else { + networkFS, fsType, err := types.IsNetworkFS(event.Name) + if err != nil { + f.logger.Warningf("Could not get fs type for %s : %s", event.Name, err) + } + f.logger.Debugf("fs for %s is network: %t (%s)", event.Name, networkFS, fsType) + if networkFS { + inotifyPoll = false + } + } + } + //Slightly different parameters for Location, as we want to read the first lines of the newly created file - tail, err := tail.TailFile(event.Name, tail.Config{ReOpen: true, Follow: true, Poll: f.config.PollWithoutInotify, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart}}) + tail, err := tail.TailFile(event.Name, tail.Config{ReOpen: true, Follow: true, Poll: inotifyPoll, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart}}) if err != nil { logger.Errorf("Could not start tailing file %s : %s", event.Name, err) break diff --git a/pkg/csconfig/api_test.go b/pkg/csconfig/api_test.go index b6febd4d450..463b7c1b2ec 100644 --- a/pkg/csconfig/api_test.go +++ b/pkg/csconfig/api_test.go @@ -194,6 +194,7 @@ func TestLoadAPIServer(t *testing.T) { DbPath: "./testdata/test.db", Type: "sqlite", MaxOpenConns: ptr.Of(DEFAULT_MAX_OPEN_CONNS), + UseWal: ptr.Of(true), // autodetected DecisionBulkSize: defaultDecisionBulkSize, }, ConsoleConfigPath: DefaultConfigPath("console.yaml"), diff --git a/pkg/csconfig/database.go b/pkg/csconfig/database.go index 2df2207859d..a7bc57eefdc 100644 --- a/pkg/csconfig/database.go +++ b/pkg/csconfig/database.go @@ -3,12 +3,15 @@ package csconfig import ( "errors" "fmt" + "path/filepath" "time" "entgo.io/ent/dialect" log "github.com/sirupsen/logrus" "github.com/crowdsecurity/go-cs-lib/ptr" + + "github.com/crowdsecurity/crowdsec/pkg/types" ) const ( @@ -69,6 +72,35 @@ func (c *Config) LoadDBConfig(inCli bool) error { c.DbConfig.MaxOpenConns = ptr.Of(DEFAULT_MAX_OPEN_CONNS) } + if !inCli && c.DbConfig.Type == "sqlite" { + if c.DbConfig.UseWal == nil { + dbDir := filepath.Dir(c.DbConfig.DbPath) + isNetwork, fsType, err := types.IsNetworkFS(dbDir) + if err != nil { + log.Warnf("unable to determine if database is on network filesystem: %s", err) + log.Warning("You are using sqlite without WAL, this can have a performance impact. If you do not store the database in a network share, set db_config.use_wal to true. Set explicitly to false to disable this warning.") + return nil + } + if isNetwork { + log.Debugf("database is on network filesystem (%s), setting useWal to false", fsType) + c.DbConfig.UseWal = ptr.Of(false) + } else { + log.Debugf("database is on local filesystem (%s), setting useWal to true", fsType) + c.DbConfig.UseWal = ptr.Of(true) + } + } else if *c.DbConfig.UseWal { + dbDir := filepath.Dir(c.DbConfig.DbPath) + isNetwork, fsType, err := types.IsNetworkFS(dbDir) + if err != nil { + log.Warnf("unable to determine if database is on network filesystem: %s", err) + return nil + } + if isNetwork { + log.Warnf("database seems to be stored on a network share (%s), but useWal is set to true. Proceed at your own risk.", fsType) + } + } + } + if c.DbConfig.DecisionBulkSize == 0 { log.Tracef("No decision_bulk_size value provided, using default value of %d", defaultDecisionBulkSize) c.DbConfig.DecisionBulkSize = defaultDecisionBulkSize @@ -79,10 +111,6 @@ func (c *Config) LoadDBConfig(inCli bool) error { c.DbConfig.DecisionBulkSize = maxDecisionBulkSize } - if !inCli && c.DbConfig.Type == "sqlite" && c.DbConfig.UseWal == nil { - log.Warning("You are using sqlite without WAL, this can have a performance impact. If you do not store the database in a network share, set db_config.use_wal to true. Set explicitly to false to disable this warning.") - } - return nil } diff --git a/pkg/csconfig/database_test.go b/pkg/csconfig/database_test.go index a946025799d..c7741baf038 100644 --- a/pkg/csconfig/database_test.go +++ b/pkg/csconfig/database_test.go @@ -30,9 +30,10 @@ func TestLoadDBConfig(t *testing.T) { }, }, expected: &DatabaseCfg{ - Type: "sqlite", - DbPath: "./testdata/test.db", - MaxOpenConns: ptr.Of(10), + Type: "sqlite", + DbPath: "./testdata/test.db", + MaxOpenConns: ptr.Of(10), + UseWal: ptr.Of(true), DecisionBulkSize: defaultDecisionBulkSize, }, }, diff --git a/pkg/types/getfstype.go b/pkg/types/getfstype.go new file mode 100644 index 00000000000..4a54fc9481e --- /dev/null +++ b/pkg/types/getfstype.go @@ -0,0 +1,112 @@ +//go:build !windows + +package types + +import ( + "fmt" + "syscall" +) + +// Generated with `man statfs | grep _MAGIC | awk '{split(tolower($1),a,"_"); print $2 ": \"" a[1] "\","}'` +// ext2/3/4 duplicates removed to just have ext4 +// XIAFS removed as well +var fsTypeMapping map[int]string = map[int]string{ + 0xadf5: "adfs", + 0xadff: "affs", + 0x5346414f: "afs", + 0x09041934: "anon", + 0x0187: "autofs", + 0x62646576: "bdevfs", + 0x42465331: "befs", + 0x1badface: "bfs", + 0x42494e4d: "binfmtfs", + 0xcafe4a11: "bpf", + 0x9123683e: "btrfs", + 0x73727279: "btrfs", + 0x27e0eb: "cgroup", + 0x63677270: "cgroup2", + 0xff534d42: "cifs", + 0x73757245: "coda", + 0x012ff7b7: "coh", + 0x28cd3d45: "cramfs", + 0x64626720: "debugfs", + 0x1373: "devfs", + 0x1cd1: "devpts", + 0xf15f: "ecryptfs", + 0xde5e81e4: "efivarfs", + 0x00414a53: "efs", + 0x137d: "ext", + 0xef51: "ext2", + 0xef53: "ext4", + 0xf2f52010: "f2fs", + 0x65735546: "fuse", + 0xbad1dea: "futexfs", + 0x4244: "hfs", + 0x00c0ffee: "hostfs", + 0xf995e849: "hpfs", + 0x958458f6: "hugetlbfs", + 0x9660: "isofs", + 0x72b6: "jffs2", + 0x3153464a: "jfs", + 0x137f: "minix", + 0x138f: "minix", + 0x2468: "minix2", + 0x2478: "minix2", + 0x4d5a: "minix3", + 0x19800202: "mqueue", + 0x4d44: "msdos", + 0x11307854: "mtd", + 0x564c: "ncp", + 0x6969: "nfs", + 0x3434: "nilfs", + 0x6e736673: "nsfs", + 0x5346544e: "ntfs", + 0x7461636f: "ocfs2", + 0x9fa1: "openprom", + 0x794c7630: "overlayfs", + 0x50495045: "pipefs", + 0x9fa0: "proc", + 0x6165676c: "pstorefs", + 0x002f: "qnx4", + 0x68191122: "qnx6", + 0x858458f6: "ramfs", + 0x52654973: "reiserfs", + 0x7275: "romfs", + 0x73636673: "securityfs", + 0xf97cff8c: "selinux", + 0x43415d53: "smack", + 0x517b: "smb", + 0xfe534d42: "smb2", + 0x534f434b: "sockfs", + 0x73717368: "squashfs", + 0x62656572: "sysfs", + 0x012ff7b6: "sysv2", + 0x012ff7b5: "sysv4", + 0x01021994: "tmpfs", + 0x74726163: "tracefs", + 0x15013346: "udf", + 0x00011954: "ufs", + 0x9fa2: "usbdevice", + 0x01021997: "v9fs", + 0xa501fcf5: "vxfs", + 0xabba1974: "xenfs", + 0x012ff7b4: "xenix", + 0x58465342: "xfs", +} + +func GetFSType(path string) (string, error) { + var buf syscall.Statfs_t + + err := syscall.Statfs(path, &buf) + + if err != nil { + return "", err + } + + fsType, ok := fsTypeMapping[int(buf.Type)] + if !ok { + return "", fmt.Errorf("unknown fstype %d", buf.Type) + } + + return fsType, nil +} diff --git a/pkg/types/getfstype_windows.go b/pkg/types/getfstype_windows.go new file mode 100644 index 00000000000..03d8fffd48d --- /dev/null +++ b/pkg/types/getfstype_windows.go @@ -0,0 +1,53 @@ +package types + +import ( + "path/filepath" + "syscall" + "unsafe" +) + +func GetFSType(path string) (string, error) { + kernel32, err := syscall.LoadLibrary("kernel32.dll") + if err != nil { + return "", err + } + defer syscall.FreeLibrary(kernel32) + + getVolumeInformation, err := syscall.GetProcAddress(kernel32, "GetVolumeInformationW") + if err != nil { + return "", err + } + + // Convert relative path to absolute path + absPath, err := filepath.Abs(path) + if err != nil { + return "", err + } + + // Get the root path of the volume + volumeRoot := filepath.VolumeName(absPath) + "\\" + + volumeRootPtr, _ := syscall.UTF16PtrFromString(volumeRoot) + + var ( + fileSystemNameBuffer = make([]uint16, 260) + nFileSystemNameSize = uint32(len(fileSystemNameBuffer)) + ) + + ret, _, err := syscall.SyscallN(getVolumeInformation, + uintptr(unsafe.Pointer(volumeRootPtr)), + 0, + 0, + 0, + 0, + 0, + uintptr(unsafe.Pointer(&fileSystemNameBuffer[0])), + uintptr(nFileSystemNameSize), + 0) + + if ret == 0 { + return "", err + } + + return syscall.UTF16ToString(fileSystemNameBuffer), nil +} diff --git a/pkg/types/utils.go b/pkg/types/utils.go index e42c36d8aeb..712d44ba12d 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -3,6 +3,7 @@ package types import ( "fmt" "path/filepath" + "strings" "time" log "github.com/sirupsen/logrus" @@ -67,3 +68,12 @@ func ConfigureLogger(clog *log.Logger) error { func UtcNow() time.Time { return time.Now().UTC() } + +func IsNetworkFS(path string) (bool, string, error) { + fsType, err := GetFSType(path) + if err != nil { + return false, "", err + } + fsType = strings.ToLower(fsType) + return fsType == "nfs" || fsType == "cifs" || fsType == "smb" || fsType == "smb2", fsType, nil +} From 8108e4156d1f564d7acc950e9b49488da5021c17 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:25:25 +0100 Subject: [PATCH 6/8] CI: "make generate" target; use ent 0.12.5 (#2871) * CI: "make generate" target; pin tool versions * use ent 0.12.5 * fix make help * fix model generation target; re-run swagger --- Makefile | 4 ++ go.mod | 2 +- go.sum | 6 +- pkg/database/ent/alert_update.go | 16 ++++++ pkg/database/ent/bouncer_update.go | 48 ++++++++++++++++ pkg/database/ent/client.go | 11 +++- pkg/database/ent/configitem_update.go | 32 +++++++++++ pkg/database/ent/decision_update.go | 80 +++++++++++++++++++++++++++ pkg/database/ent/event_update.go | 32 +++++++++++ pkg/database/ent/generate.go | 2 +- pkg/database/ent/lock_update.go | 16 ++++++ pkg/database/ent/machine_update.go | 48 ++++++++++++++++ pkg/database/ent/meta_update.go | 32 +++++++++++ pkg/database/ent/runtime/runtime.go | 4 +- pkg/models/add_alerts_request.go | 5 ++ pkg/models/alert.go | 11 ++++ pkg/models/generate.go | 4 ++ pkg/models/get_alerts_response.go | 5 ++ pkg/models/get_decisions_response.go | 5 ++ pkg/models/meta.go | 5 ++ pkg/models/metrics.go | 10 ++++ 21 files changed, 367 insertions(+), 11 deletions(-) create mode 100644 pkg/models/generate.go diff --git a/Makefile b/Makefile index 5d656165fa8..3f271c54ca4 100644 --- a/Makefile +++ b/Makefile @@ -202,6 +202,10 @@ cscli: goversion ## Build cscli crowdsec: goversion ## Build crowdsec @$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS) +.PHONY: generate +generate: ## Generate code for the database and APIs + $(GO) generate ./pkg/database/ent + $(GO) generate ./pkg/models .PHONY: testclean testclean: bats-clean ## Remove test artifacts diff --git a/go.mod b/go.mod index c2d6ca2c148..06e0275c82b 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ go 1.21 // toolchain go1.21.3 require ( - entgo.io/ent v0.12.4 + entgo.io/ent v0.12.5 github.com/AlecAivazis/survey/v2 v2.3.7 github.com/Masterminds/semver/v3 v3.2.1 github.com/Masterminds/sprig/v3 v3.2.3 diff --git a/go.sum b/go.sum index 7e860300089..35ab5813fca 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ ariga.io/atlas v0.14.1-0.20230918065911-83ad451a4935 h1:JnYs/y8RJ3+MiIUp+3RgyyeO ariga.io/atlas v0.14.1-0.20230918065911-83ad451a4935/go.mod h1:isZrlzJ5cpoCoKFoY9knZug7Lq4pP1cm8g3XciLZ0Pw= bitbucket.org/creachadair/stringset v0.0.9 h1:L4vld9nzPt90UZNrXjNelTshD74ps4P5NGs3Iq6yN3o= bitbucket.org/creachadair/stringset v0.0.9/go.mod h1:t+4WcQ4+PXTa8aQdNKe40ZP6iwesoMFWAxPGd3UGjyY= -entgo.io/ent v0.12.4 h1:LddPnAyxls/O7DTXZvUGDj0NZIdGSu317+aoNLJWbD8= -entgo.io/ent v0.12.4/go.mod h1:Y3JVAjtlIk8xVZYSn3t3mf8xlZIn5SAOXZQxD6kKI+Q= +entgo.io/ent v0.12.5 h1:KREM5E4CSoej4zeGa88Ou/gfturAnpUv0mzAjch1sj4= +entgo.io/ent v0.12.5/go.mod h1:Y3JVAjtlIk8xVZYSn3t3mf8xlZIn5SAOXZQxD6kKI+Q= github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= @@ -540,8 +540,6 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= diff --git a/pkg/database/ent/alert_update.go b/pkg/database/ent/alert_update.go index 0e41ba18109..f8a4d108527 100644 --- a/pkg/database/ent/alert_update.go +++ b/pkg/database/ent/alert_update.go @@ -62,6 +62,14 @@ func (au *AlertUpdate) SetScenario(s string) *AlertUpdate { return au } +// SetNillableScenario sets the "scenario" field if the given value is not nil. +func (au *AlertUpdate) SetNillableScenario(s *string) *AlertUpdate { + if s != nil { + au.SetScenario(*s) + } + return au +} + // SetBucketId sets the "bucketId" field. func (au *AlertUpdate) SetBucketId(s string) *AlertUpdate { au.mutation.SetBucketId(s) @@ -1029,6 +1037,14 @@ func (auo *AlertUpdateOne) SetScenario(s string) *AlertUpdateOne { return auo } +// SetNillableScenario sets the "scenario" field if the given value is not nil. +func (auo *AlertUpdateOne) SetNillableScenario(s *string) *AlertUpdateOne { + if s != nil { + auo.SetScenario(*s) + } + return auo +} + // SetBucketId sets the "bucketId" field. func (auo *AlertUpdateOne) SetBucketId(s string) *AlertUpdateOne { auo.mutation.SetBucketId(s) diff --git a/pkg/database/ent/bouncer_update.go b/pkg/database/ent/bouncer_update.go index f7e71eb315e..b3f5e1a5540 100644 --- a/pkg/database/ent/bouncer_update.go +++ b/pkg/database/ent/bouncer_update.go @@ -58,18 +58,42 @@ func (bu *BouncerUpdate) SetName(s string) *BouncerUpdate { return bu } +// SetNillableName sets the "name" field if the given value is not nil. +func (bu *BouncerUpdate) SetNillableName(s *string) *BouncerUpdate { + if s != nil { + bu.SetName(*s) + } + return bu +} + // SetAPIKey sets the "api_key" field. func (bu *BouncerUpdate) SetAPIKey(s string) *BouncerUpdate { bu.mutation.SetAPIKey(s) return bu } +// SetNillableAPIKey sets the "api_key" field if the given value is not nil. +func (bu *BouncerUpdate) SetNillableAPIKey(s *string) *BouncerUpdate { + if s != nil { + bu.SetAPIKey(*s) + } + return bu +} + // SetRevoked sets the "revoked" field. func (bu *BouncerUpdate) SetRevoked(b bool) *BouncerUpdate { bu.mutation.SetRevoked(b) return bu } +// SetNillableRevoked sets the "revoked" field if the given value is not nil. +func (bu *BouncerUpdate) SetNillableRevoked(b *bool) *BouncerUpdate { + if b != nil { + bu.SetRevoked(*b) + } + return bu +} + // SetIPAddress sets the "ip_address" field. func (bu *BouncerUpdate) SetIPAddress(s string) *BouncerUpdate { bu.mutation.SetIPAddress(s) @@ -333,18 +357,42 @@ func (buo *BouncerUpdateOne) SetName(s string) *BouncerUpdateOne { return buo } +// SetNillableName sets the "name" field if the given value is not nil. +func (buo *BouncerUpdateOne) SetNillableName(s *string) *BouncerUpdateOne { + if s != nil { + buo.SetName(*s) + } + return buo +} + // SetAPIKey sets the "api_key" field. func (buo *BouncerUpdateOne) SetAPIKey(s string) *BouncerUpdateOne { buo.mutation.SetAPIKey(s) return buo } +// SetNillableAPIKey sets the "api_key" field if the given value is not nil. +func (buo *BouncerUpdateOne) SetNillableAPIKey(s *string) *BouncerUpdateOne { + if s != nil { + buo.SetAPIKey(*s) + } + return buo +} + // SetRevoked sets the "revoked" field. func (buo *BouncerUpdateOne) SetRevoked(b bool) *BouncerUpdateOne { buo.mutation.SetRevoked(b) return buo } +// SetNillableRevoked sets the "revoked" field if the given value is not nil. +func (buo *BouncerUpdateOne) SetNillableRevoked(b *bool) *BouncerUpdateOne { + if b != nil { + buo.SetRevoked(*b) + } + return buo +} + // SetIPAddress sets the "ip_address" field. func (buo *BouncerUpdateOne) SetIPAddress(s string) *BouncerUpdateOne { buo.mutation.SetIPAddress(s) diff --git a/pkg/database/ent/client.go b/pkg/database/ent/client.go index 006d52ef9ba..5318109ed42 100644 --- a/pkg/database/ent/client.go +++ b/pkg/database/ent/client.go @@ -50,9 +50,7 @@ type Client struct { // NewClient creates a new client configured with the given options. func NewClient(opts ...Option) *Client { - cfg := config{log: log.Println, hooks: &hooks{}, inters: &inters{}} - cfg.options(opts...) - client := &Client{config: cfg} + client := &Client{config: newConfig(opts...)} client.init() return client } @@ -87,6 +85,13 @@ type ( Option func(*config) ) +// newConfig creates a new config for the client. +func newConfig(opts ...Option) config { + cfg := config{log: log.Println, hooks: &hooks{}, inters: &inters{}} + cfg.options(opts...) + return cfg +} + // options applies the options on the config object. func (c *config) options(opts ...Option) { for _, opt := range opts { diff --git a/pkg/database/ent/configitem_update.go b/pkg/database/ent/configitem_update.go index 0db3a0b5233..11fb0755191 100644 --- a/pkg/database/ent/configitem_update.go +++ b/pkg/database/ent/configitem_update.go @@ -58,12 +58,28 @@ func (ciu *ConfigItemUpdate) SetName(s string) *ConfigItemUpdate { return ciu } +// SetNillableName sets the "name" field if the given value is not nil. +func (ciu *ConfigItemUpdate) SetNillableName(s *string) *ConfigItemUpdate { + if s != nil { + ciu.SetName(*s) + } + return ciu +} + // SetValue sets the "value" field. func (ciu *ConfigItemUpdate) SetValue(s string) *ConfigItemUpdate { ciu.mutation.SetValue(s) return ciu } +// SetNillableValue sets the "value" field if the given value is not nil. +func (ciu *ConfigItemUpdate) SetNillableValue(s *string) *ConfigItemUpdate { + if s != nil { + ciu.SetValue(*s) + } + return ciu +} + // Mutation returns the ConfigItemMutation object of the builder. func (ciu *ConfigItemUpdate) Mutation() *ConfigItemMutation { return ciu.mutation @@ -186,12 +202,28 @@ func (ciuo *ConfigItemUpdateOne) SetName(s string) *ConfigItemUpdateOne { return ciuo } +// SetNillableName sets the "name" field if the given value is not nil. +func (ciuo *ConfigItemUpdateOne) SetNillableName(s *string) *ConfigItemUpdateOne { + if s != nil { + ciuo.SetName(*s) + } + return ciuo +} + // SetValue sets the "value" field. func (ciuo *ConfigItemUpdateOne) SetValue(s string) *ConfigItemUpdateOne { ciuo.mutation.SetValue(s) return ciuo } +// SetNillableValue sets the "value" field if the given value is not nil. +func (ciuo *ConfigItemUpdateOne) SetNillableValue(s *string) *ConfigItemUpdateOne { + if s != nil { + ciuo.SetValue(*s) + } + return ciuo +} + // Mutation returns the ConfigItemMutation object of the builder. func (ciuo *ConfigItemUpdateOne) Mutation() *ConfigItemMutation { return ciuo.mutation diff --git a/pkg/database/ent/decision_update.go b/pkg/database/ent/decision_update.go index 1b62cc54c30..182457e9f63 100644 --- a/pkg/database/ent/decision_update.go +++ b/pkg/database/ent/decision_update.go @@ -79,12 +79,28 @@ func (du *DecisionUpdate) SetScenario(s string) *DecisionUpdate { return du } +// SetNillableScenario sets the "scenario" field if the given value is not nil. +func (du *DecisionUpdate) SetNillableScenario(s *string) *DecisionUpdate { + if s != nil { + du.SetScenario(*s) + } + return du +} + // SetType sets the "type" field. func (du *DecisionUpdate) SetType(s string) *DecisionUpdate { du.mutation.SetType(s) return du } +// SetNillableType sets the "type" field if the given value is not nil. +func (du *DecisionUpdate) SetNillableType(s *string) *DecisionUpdate { + if s != nil { + du.SetType(*s) + } + return du +} + // SetStartIP sets the "start_ip" field. func (du *DecisionUpdate) SetStartIP(i int64) *DecisionUpdate { du.mutation.ResetStartIP() @@ -226,18 +242,42 @@ func (du *DecisionUpdate) SetScope(s string) *DecisionUpdate { return du } +// SetNillableScope sets the "scope" field if the given value is not nil. +func (du *DecisionUpdate) SetNillableScope(s *string) *DecisionUpdate { + if s != nil { + du.SetScope(*s) + } + return du +} + // SetValue sets the "value" field. func (du *DecisionUpdate) SetValue(s string) *DecisionUpdate { du.mutation.SetValue(s) return du } +// SetNillableValue sets the "value" field if the given value is not nil. +func (du *DecisionUpdate) SetNillableValue(s *string) *DecisionUpdate { + if s != nil { + du.SetValue(*s) + } + return du +} + // SetOrigin sets the "origin" field. func (du *DecisionUpdate) SetOrigin(s string) *DecisionUpdate { du.mutation.SetOrigin(s) return du } +// SetNillableOrigin sets the "origin" field if the given value is not nil. +func (du *DecisionUpdate) SetNillableOrigin(s *string) *DecisionUpdate { + if s != nil { + du.SetOrigin(*s) + } + return du +} + // SetSimulated sets the "simulated" field. func (du *DecisionUpdate) SetSimulated(b bool) *DecisionUpdate { du.mutation.SetSimulated(b) @@ -557,12 +597,28 @@ func (duo *DecisionUpdateOne) SetScenario(s string) *DecisionUpdateOne { return duo } +// SetNillableScenario sets the "scenario" field if the given value is not nil. +func (duo *DecisionUpdateOne) SetNillableScenario(s *string) *DecisionUpdateOne { + if s != nil { + duo.SetScenario(*s) + } + return duo +} + // SetType sets the "type" field. func (duo *DecisionUpdateOne) SetType(s string) *DecisionUpdateOne { duo.mutation.SetType(s) return duo } +// SetNillableType sets the "type" field if the given value is not nil. +func (duo *DecisionUpdateOne) SetNillableType(s *string) *DecisionUpdateOne { + if s != nil { + duo.SetType(*s) + } + return duo +} + // SetStartIP sets the "start_ip" field. func (duo *DecisionUpdateOne) SetStartIP(i int64) *DecisionUpdateOne { duo.mutation.ResetStartIP() @@ -704,18 +760,42 @@ func (duo *DecisionUpdateOne) SetScope(s string) *DecisionUpdateOne { return duo } +// SetNillableScope sets the "scope" field if the given value is not nil. +func (duo *DecisionUpdateOne) SetNillableScope(s *string) *DecisionUpdateOne { + if s != nil { + duo.SetScope(*s) + } + return duo +} + // SetValue sets the "value" field. func (duo *DecisionUpdateOne) SetValue(s string) *DecisionUpdateOne { duo.mutation.SetValue(s) return duo } +// SetNillableValue sets the "value" field if the given value is not nil. +func (duo *DecisionUpdateOne) SetNillableValue(s *string) *DecisionUpdateOne { + if s != nil { + duo.SetValue(*s) + } + return duo +} + // SetOrigin sets the "origin" field. func (duo *DecisionUpdateOne) SetOrigin(s string) *DecisionUpdateOne { duo.mutation.SetOrigin(s) return duo } +// SetNillableOrigin sets the "origin" field if the given value is not nil. +func (duo *DecisionUpdateOne) SetNillableOrigin(s *string) *DecisionUpdateOne { + if s != nil { + duo.SetOrigin(*s) + } + return duo +} + // SetSimulated sets the "simulated" field. func (duo *DecisionUpdateOne) SetSimulated(b bool) *DecisionUpdateOne { duo.mutation.SetSimulated(b) diff --git a/pkg/database/ent/event_update.go b/pkg/database/ent/event_update.go index db748101519..a06178f79af 100644 --- a/pkg/database/ent/event_update.go +++ b/pkg/database/ent/event_update.go @@ -59,12 +59,28 @@ func (eu *EventUpdate) SetTime(t time.Time) *EventUpdate { return eu } +// SetNillableTime sets the "time" field if the given value is not nil. +func (eu *EventUpdate) SetNillableTime(t *time.Time) *EventUpdate { + if t != nil { + eu.SetTime(*t) + } + return eu +} + // SetSerialized sets the "serialized" field. func (eu *EventUpdate) SetSerialized(s string) *EventUpdate { eu.mutation.SetSerialized(s) return eu } +// SetNillableSerialized sets the "serialized" field if the given value is not nil. +func (eu *EventUpdate) SetNillableSerialized(s *string) *EventUpdate { + if s != nil { + eu.SetSerialized(*s) + } + return eu +} + // SetAlertEvents sets the "alert_events" field. func (eu *EventUpdate) SetAlertEvents(i int) *EventUpdate { eu.mutation.SetAlertEvents(i) @@ -274,12 +290,28 @@ func (euo *EventUpdateOne) SetTime(t time.Time) *EventUpdateOne { return euo } +// SetNillableTime sets the "time" field if the given value is not nil. +func (euo *EventUpdateOne) SetNillableTime(t *time.Time) *EventUpdateOne { + if t != nil { + euo.SetTime(*t) + } + return euo +} + // SetSerialized sets the "serialized" field. func (euo *EventUpdateOne) SetSerialized(s string) *EventUpdateOne { euo.mutation.SetSerialized(s) return euo } +// SetNillableSerialized sets the "serialized" field if the given value is not nil. +func (euo *EventUpdateOne) SetNillableSerialized(s *string) *EventUpdateOne { + if s != nil { + euo.SetSerialized(*s) + } + return euo +} + // SetAlertEvents sets the "alert_events" field. func (euo *EventUpdateOne) SetAlertEvents(i int) *EventUpdateOne { euo.mutation.SetAlertEvents(i) diff --git a/pkg/database/ent/generate.go b/pkg/database/ent/generate.go index 9f3a916c7a4..5f4b39eec90 100644 --- a/pkg/database/ent/generate.go +++ b/pkg/database/ent/generate.go @@ -1,4 +1,4 @@ package ent -//go:generate go run -mod=mod entgo.io/ent/cmd/ent generate ./schema +//go:generate go run -mod=mod entgo.io/ent/cmd/ent@v0.12.5 generate ./schema diff --git a/pkg/database/ent/lock_update.go b/pkg/database/ent/lock_update.go index f4deda6e3a8..dc61dfdfde1 100644 --- a/pkg/database/ent/lock_update.go +++ b/pkg/database/ent/lock_update.go @@ -34,6 +34,14 @@ func (lu *LockUpdate) SetName(s string) *LockUpdate { return lu } +// SetNillableName sets the "name" field if the given value is not nil. +func (lu *LockUpdate) SetNillableName(s *string) *LockUpdate { + if s != nil { + lu.SetName(*s) + } + return lu +} + // SetCreatedAt sets the "created_at" field. func (lu *LockUpdate) SetCreatedAt(t time.Time) *LockUpdate { lu.mutation.SetCreatedAt(t) @@ -121,6 +129,14 @@ func (luo *LockUpdateOne) SetName(s string) *LockUpdateOne { return luo } +// SetNillableName sets the "name" field if the given value is not nil. +func (luo *LockUpdateOne) SetNillableName(s *string) *LockUpdateOne { + if s != nil { + luo.SetName(*s) + } + return luo +} + // SetCreatedAt sets the "created_at" field. func (luo *LockUpdateOne) SetCreatedAt(t time.Time) *LockUpdateOne { luo.mutation.SetCreatedAt(t) diff --git a/pkg/database/ent/machine_update.go b/pkg/database/ent/machine_update.go index eb517081174..1f87ac04d6f 100644 --- a/pkg/database/ent/machine_update.go +++ b/pkg/database/ent/machine_update.go @@ -83,18 +83,42 @@ func (mu *MachineUpdate) SetMachineId(s string) *MachineUpdate { return mu } +// SetNillableMachineId sets the "machineId" field if the given value is not nil. +func (mu *MachineUpdate) SetNillableMachineId(s *string) *MachineUpdate { + if s != nil { + mu.SetMachineId(*s) + } + return mu +} + // SetPassword sets the "password" field. func (mu *MachineUpdate) SetPassword(s string) *MachineUpdate { mu.mutation.SetPassword(s) return mu } +// SetNillablePassword sets the "password" field if the given value is not nil. +func (mu *MachineUpdate) SetNillablePassword(s *string) *MachineUpdate { + if s != nil { + mu.SetPassword(*s) + } + return mu +} + // SetIpAddress sets the "ipAddress" field. func (mu *MachineUpdate) SetIpAddress(s string) *MachineUpdate { mu.mutation.SetIpAddress(s) return mu } +// SetNillableIpAddress sets the "ipAddress" field if the given value is not nil. +func (mu *MachineUpdate) SetNillableIpAddress(s *string) *MachineUpdate { + if s != nil { + mu.SetIpAddress(*s) + } + return mu +} + // SetScenarios sets the "scenarios" field. func (mu *MachineUpdate) SetScenarios(s string) *MachineUpdate { mu.mutation.SetScenarios(s) @@ -470,18 +494,42 @@ func (muo *MachineUpdateOne) SetMachineId(s string) *MachineUpdateOne { return muo } +// SetNillableMachineId sets the "machineId" field if the given value is not nil. +func (muo *MachineUpdateOne) SetNillableMachineId(s *string) *MachineUpdateOne { + if s != nil { + muo.SetMachineId(*s) + } + return muo +} + // SetPassword sets the "password" field. func (muo *MachineUpdateOne) SetPassword(s string) *MachineUpdateOne { muo.mutation.SetPassword(s) return muo } +// SetNillablePassword sets the "password" field if the given value is not nil. +func (muo *MachineUpdateOne) SetNillablePassword(s *string) *MachineUpdateOne { + if s != nil { + muo.SetPassword(*s) + } + return muo +} + // SetIpAddress sets the "ipAddress" field. func (muo *MachineUpdateOne) SetIpAddress(s string) *MachineUpdateOne { muo.mutation.SetIpAddress(s) return muo } +// SetNillableIpAddress sets the "ipAddress" field if the given value is not nil. +func (muo *MachineUpdateOne) SetNillableIpAddress(s *string) *MachineUpdateOne { + if s != nil { + muo.SetIpAddress(*s) + } + return muo +} + // SetScenarios sets the "scenarios" field. func (muo *MachineUpdateOne) SetScenarios(s string) *MachineUpdateOne { muo.mutation.SetScenarios(s) diff --git a/pkg/database/ent/meta_update.go b/pkg/database/ent/meta_update.go index 8071c4f0df5..a1379faa130 100644 --- a/pkg/database/ent/meta_update.go +++ b/pkg/database/ent/meta_update.go @@ -59,12 +59,28 @@ func (mu *MetaUpdate) SetKey(s string) *MetaUpdate { return mu } +// SetNillableKey sets the "key" field if the given value is not nil. +func (mu *MetaUpdate) SetNillableKey(s *string) *MetaUpdate { + if s != nil { + mu.SetKey(*s) + } + return mu +} + // SetValue sets the "value" field. func (mu *MetaUpdate) SetValue(s string) *MetaUpdate { mu.mutation.SetValue(s) return mu } +// SetNillableValue sets the "value" field if the given value is not nil. +func (mu *MetaUpdate) SetNillableValue(s *string) *MetaUpdate { + if s != nil { + mu.SetValue(*s) + } + return mu +} + // SetAlertMetas sets the "alert_metas" field. func (mu *MetaUpdate) SetAlertMetas(i int) *MetaUpdate { mu.mutation.SetAlertMetas(i) @@ -274,12 +290,28 @@ func (muo *MetaUpdateOne) SetKey(s string) *MetaUpdateOne { return muo } +// SetNillableKey sets the "key" field if the given value is not nil. +func (muo *MetaUpdateOne) SetNillableKey(s *string) *MetaUpdateOne { + if s != nil { + muo.SetKey(*s) + } + return muo +} + // SetValue sets the "value" field. func (muo *MetaUpdateOne) SetValue(s string) *MetaUpdateOne { muo.mutation.SetValue(s) return muo } +// SetNillableValue sets the "value" field if the given value is not nil. +func (muo *MetaUpdateOne) SetNillableValue(s *string) *MetaUpdateOne { + if s != nil { + muo.SetValue(*s) + } + return muo +} + // SetAlertMetas sets the "alert_metas" field. func (muo *MetaUpdateOne) SetAlertMetas(i int) *MetaUpdateOne { muo.mutation.SetAlertMetas(i) diff --git a/pkg/database/ent/runtime/runtime.go b/pkg/database/ent/runtime/runtime.go index 2a645f624d7..d10a2fb5459 100644 --- a/pkg/database/ent/runtime/runtime.go +++ b/pkg/database/ent/runtime/runtime.go @@ -5,6 +5,6 @@ package runtime // The schema-stitching logic is generated in github.com/crowdsecurity/crowdsec/pkg/database/ent/runtime.go const ( - Version = "v0.12.4" // Version of ent codegen. - Sum = "h1:LddPnAyxls/O7DTXZvUGDj0NZIdGSu317+aoNLJWbD8=" // Sum of ent codegen. + Version = "v0.12.5" // Version of ent codegen. + Sum = "h1:KREM5E4CSoej4zeGa88Ou/gfturAnpUv0mzAjch1sj4=" // Sum of ent codegen. ) diff --git a/pkg/models/add_alerts_request.go b/pkg/models/add_alerts_request.go index fd7246be066..a69934ef770 100644 --- a/pkg/models/add_alerts_request.go +++ b/pkg/models/add_alerts_request.go @@ -54,6 +54,11 @@ func (m AddAlertsRequest) ContextValidate(ctx context.Context, formats strfmt.Re for i := 0; i < len(m); i++ { if m[i] != nil { + + if swag.IsZero(m[i]) { // not required + return nil + } + if err := m[i].ContextValidate(ctx, formats); err != nil { if ve, ok := err.(*errors.Validation); ok { return ve.ValidateName(strconv.Itoa(i)) diff --git a/pkg/models/alert.go b/pkg/models/alert.go index ec769a1fbb1..895f5ad76e1 100644 --- a/pkg/models/alert.go +++ b/pkg/models/alert.go @@ -399,6 +399,11 @@ func (m *Alert) contextValidateDecisions(ctx context.Context, formats strfmt.Reg for i := 0; i < len(m.Decisions); i++ { if m.Decisions[i] != nil { + + if swag.IsZero(m.Decisions[i]) { // not required + return nil + } + if err := m.Decisions[i].ContextValidate(ctx, formats); err != nil { if ve, ok := err.(*errors.Validation); ok { return ve.ValidateName("decisions" + "." + strconv.Itoa(i)) @@ -419,6 +424,11 @@ func (m *Alert) contextValidateEvents(ctx context.Context, formats strfmt.Regist for i := 0; i < len(m.Events); i++ { if m.Events[i] != nil { + + if swag.IsZero(m.Events[i]) { // not required + return nil + } + if err := m.Events[i].ContextValidate(ctx, formats); err != nil { if ve, ok := err.(*errors.Validation); ok { return ve.ValidateName("events" + "." + strconv.Itoa(i)) @@ -469,6 +479,7 @@ func (m *Alert) contextValidateMeta(ctx context.Context, formats strfmt.Registry func (m *Alert) contextValidateSource(ctx context.Context, formats strfmt.Registry) error { if m.Source != nil { + if err := m.Source.ContextValidate(ctx, formats); err != nil { if ve, ok := err.(*errors.Validation); ok { return ve.ValidateName("source") diff --git a/pkg/models/generate.go b/pkg/models/generate.go new file mode 100644 index 00000000000..ccacc409ab5 --- /dev/null +++ b/pkg/models/generate.go @@ -0,0 +1,4 @@ +package models + +//go:generate go run -mod=mod github.com/go-swagger/go-swagger/cmd/swagger@v0.30.5 generate model --spec=./localapi_swagger.yaml --target=../ + diff --git a/pkg/models/get_alerts_response.go b/pkg/models/get_alerts_response.go index 41b9d5afdbd..d4ea36e02c5 100644 --- a/pkg/models/get_alerts_response.go +++ b/pkg/models/get_alerts_response.go @@ -54,6 +54,11 @@ func (m GetAlertsResponse) ContextValidate(ctx context.Context, formats strfmt.R for i := 0; i < len(m); i++ { if m[i] != nil { + + if swag.IsZero(m[i]) { // not required + return nil + } + if err := m[i].ContextValidate(ctx, formats); err != nil { if ve, ok := err.(*errors.Validation); ok { return ve.ValidateName(strconv.Itoa(i)) diff --git a/pkg/models/get_decisions_response.go b/pkg/models/get_decisions_response.go index b65b950fc58..19437dc9b38 100644 --- a/pkg/models/get_decisions_response.go +++ b/pkg/models/get_decisions_response.go @@ -54,6 +54,11 @@ func (m GetDecisionsResponse) ContextValidate(ctx context.Context, formats strfm for i := 0; i < len(m); i++ { if m[i] != nil { + + if swag.IsZero(m[i]) { // not required + return nil + } + if err := m[i].ContextValidate(ctx, formats); err != nil { if ve, ok := err.(*errors.Validation); ok { return ve.ValidateName(strconv.Itoa(i)) diff --git a/pkg/models/meta.go b/pkg/models/meta.go index 6ad20856d6a..df5ae3c6285 100644 --- a/pkg/models/meta.go +++ b/pkg/models/meta.go @@ -56,6 +56,11 @@ func (m Meta) ContextValidate(ctx context.Context, formats strfmt.Registry) erro for i := 0; i < len(m); i++ { if m[i] != nil { + + if swag.IsZero(m[i]) { // not required + return nil + } + if err := m[i].ContextValidate(ctx, formats); err != nil { if ve, ok := err.(*errors.Validation); ok { return ve.ValidateName(strconv.Itoa(i)) diff --git a/pkg/models/metrics.go b/pkg/models/metrics.go index 573678d1f84..7fbb91c63e4 100644 --- a/pkg/models/metrics.go +++ b/pkg/models/metrics.go @@ -141,6 +141,11 @@ func (m *Metrics) contextValidateBouncers(ctx context.Context, formats strfmt.Re for i := 0; i < len(m.Bouncers); i++ { if m.Bouncers[i] != nil { + + if swag.IsZero(m.Bouncers[i]) { // not required + return nil + } + if err := m.Bouncers[i].ContextValidate(ctx, formats); err != nil { if ve, ok := err.(*errors.Validation); ok { return ve.ValidateName("bouncers" + "." + strconv.Itoa(i)) @@ -161,6 +166,11 @@ func (m *Metrics) contextValidateMachines(ctx context.Context, formats strfmt.Re for i := 0; i < len(m.Machines); i++ { if m.Machines[i] != nil { + + if swag.IsZero(m.Machines[i]) { // not required + return nil + } + if err := m.Machines[i].ContextValidate(ctx, formats); err != nil { if ve, ok := err.(*errors.Validation); ok { return ve.ValidateName("machines" + "." + strconv.Itoa(i)) From 1eab943ec224e63d15fce31ef55961741f9b4077 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:36:28 +0100 Subject: [PATCH 7/8] crowdsec: remove warning if prometheus port is taken during cold logs processing (#2857) i.e. remove a "Warning: port is already in use" because it's probably LAPI --- cmd/crowdsec/crowdsec.go | 2 +- cmd/crowdsec/main.go | 4 ++++ cmd/crowdsec/metrics.go | 5 ++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 0d7d454edf2..37a12b9d3cb 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -280,7 +280,7 @@ func waitOnTomb() { case <-acquisTomb.Dead(): /*if it's acquisition dying it means that we were in "cat" mode. while shutting down, we need to give time for all buckets to process in flight data*/ - log.Warning("Acquisition is finished, shutting down") + log.Info("Acquisition is finished, shutting down") /* While it might make sense to want to shut-down parser/buckets/etc. as soon as acquisition is finished, we might have some pending buckets: buckets that overflowed, but whose LeakRoutine are still alive because they diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index 70f7d48dce4..8950790480a 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -75,6 +75,10 @@ type Flags struct { CPUProfile string } +func (f *Flags) haveTimeMachine() bool { + return f.OneShotDSN != "" +} + type labelsMap map[string]string func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error { diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index aed43db00c8..682968bb75a 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -196,6 +196,9 @@ func servePrometheus(config *csconfig.PrometheusCfg, dbClient *database.Client, log.Debugf("serving metrics after %s ms", time.Since(crowdsecT0)) if err := http.ListenAndServe(fmt.Sprintf("%s:%d", config.ListenAddr, config.ListenPort), nil); err != nil { - log.Warningf("prometheus: %s", err) + // in time machine, we most likely have the LAPI using the port + if !flags.haveTimeMachine() { + log.Warningf("prometheus: %s", err) + } } } From 6c5e8afde9ab8f751bd4c35c42e66dd114354279 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Fri, 8 Mar 2024 10:55:30 +0100 Subject: [PATCH 8/8] pkg/cwhub: download data assets to temporary files to avoid partial fetch (#2879) --- pkg/cwhub/dataset.go | 22 ++++++++++++++++++---- pkg/cwhub/dataset_test.go | 8 ++++---- pkg/cwhub/errors.go | 2 +- pkg/cwhub/hub.go | 3 ++- pkg/cwhub/sync.go | 15 ++++++++------- 5 files changed, 33 insertions(+), 17 deletions(-) diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go index c900752b8b3..4612f357626 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/cwhub/dataset.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "os" + "path/filepath" "time" "github.com/sirupsen/logrus" @@ -31,19 +32,32 @@ func downloadFile(url string, destPath string) error { return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url) } - file, err := os.Create(destPath) + tmpFile, err := os.CreateTemp(filepath.Dir(destPath), filepath.Base(destPath)+".*.tmp") if err != nil { return err } - defer file.Close() + + tmpFileName := tmpFile.Name() + defer func() { + tmpFile.Close() + os.Remove(tmpFileName) + }() // avoid reading the whole file in memory - _, err = io.Copy(file, resp.Body) + _, err = io.Copy(tmpFile, resp.Body) if err != nil { return err } - if err = file.Sync(); err != nil { + if err = tmpFile.Sync(); err != nil { + return err + } + + if err = tmpFile.Close(); err != nil { + return err + } + + if err = os.Rename(tmpFileName, destPath); err != nil { return err } diff --git a/pkg/cwhub/dataset_test.go b/pkg/cwhub/dataset_test.go index f23f4878285..93d3e3bf01e 100644 --- a/pkg/cwhub/dataset_test.go +++ b/pkg/cwhub/dataset_test.go @@ -16,7 +16,7 @@ func TestDownloadFile(t *testing.T) { httpmock.Activate() defer httpmock.DeactivateAndReset() - //OK + // OK httpmock.RegisterResponder( "GET", "https://example.com/xx", @@ -36,15 +36,15 @@ func TestDownloadFile(t *testing.T) { assert.Equal(t, "example content oneoneone", string(content)) require.NoError(t, err) - //bad uri + // bad uri err = downloadFile("https://zz.com", examplePath) require.Error(t, err) - //404 + // 404 err = downloadFile("https://example.com/x", examplePath) require.Error(t, err) - //bad target + // bad target err = downloadFile("https://example.com/xx", "") require.Error(t, err) } diff --git a/pkg/cwhub/errors.go b/pkg/cwhub/errors.go index 789c2eced7b..f1e779b5476 100644 --- a/pkg/cwhub/errors.go +++ b/pkg/cwhub/errors.go @@ -6,7 +6,7 @@ import ( ) var ( - // ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor. + // ErrNilRemoteHub is returned when trying to download with a local-only configuration. ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers") ) diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 21a19bc4526..44e24020d03 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -3,6 +3,7 @@ package cwhub import ( "bytes" "encoding/json" + "errors" "fmt" "io" "os" @@ -34,7 +35,7 @@ func (h *Hub) GetDataDir() string { // All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set. func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool, logger *logrus.Logger) (*Hub, error) { if local == nil { - return nil, fmt.Errorf("no hub configuration found") + return nil, errors.New("no hub configuration found") } if logger == nil { diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index 8ce91dc2193..cb7bf37867c 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -77,9 +77,9 @@ func (h *Hub) getItemFileInfo(path string, logger *logrus.Logger) (*itemFileInfo if strings.HasPrefix(path, hubDir) { logger.Tracef("in hub dir") - //.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml - //.../hub/scenarios/crowdsec/ssh_bf.yaml - //.../hub/profiles/crowdsec/linux.yaml + // .../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml + // .../hub/scenarios/crowdsec/ssh_bf.yaml + // .../hub/profiles/crowdsec/linux.yaml if len(subs) < 4 { return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) } @@ -93,13 +93,14 @@ func (h *Hub) getItemFileInfo(path string, logger *logrus.Logger) (*itemFileInfo } } else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec//... logger.Tracef("in install dir") + if len(subs) < 3 { return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) } - ///.../config/parser/stage/file.yaml - ///.../config/postoverflow/stage/file.yaml - ///.../config/scenarios/scenar.yaml - ///.../config/collections/linux.yaml //file is empty + // .../config/parser/stage/file.yaml + // .../config/postoverflow/stage/file.yaml + // .../config/scenarios/scenar.yaml + // .../config/collections/linux.yaml //file is empty ret = &itemFileInfo{ inhub: false, fname: subs[len(subs)-1],