(maint) Common CLI and config implementation for engine hosts

.deadcode-ignore

@@ -0,0 +1,24 @@
+adapterhelpers/shared_tests.go:19:6: unreachable func: PtrInt32
+adapterhelpers/shared_tests.go:27:6: unreachable func: PtrFloat32
+adapterhelpers/shared_tests.go:31:6: unreachable func: PtrFloat64
+adapterhelpers/shared_tests.go:35:6: unreachable func: PtrTime
+adapterhelpers/shared_tests.go:39:6: unreachable func: PtrBool
+adapterhelpers/shared_tests.go:73:21: unreachable func: VPCConfig.Cleanup
+adapterhelpers/shared_tests.go:77:21: unreachable func: VPCConfig.RunCleanup
+adapterhelpers/shared_tests.go:88:21: unreachable func: VPCConfig.Fetch
+adapterhelpers/shared_tests.go:121:21: unreachable func: VPCConfig.CreateGateway
+adapterhelpers/shared_tests.go:198:6: unreachable func: retry
+adapterhelpers/shared_tests.go:221:21: unreachable func: QueryTests.Execute
+adapterhelpers/shared_tests.go:238:6: unreachable func: lirMatches
+adapterhelpers/shared_tests.go:246:6: unreachable func: CheckQuery
+adapterhelpers/util.go:180:18: unreachable func: E2ETest.Run
+adapterhelpers/util.go:326:6: unreachable func: GetAutoConfig
+adapters/rds.go:25:24: unreachable func: mockRdsClient.DescribeDBClusterParameterGroups
+adapters/rds.go:29:24: unreachable func: mockRdsClient.DescribeDBClusterParameters
+adapters/rds.go:33:24: unreachable func: mockRdsClient.ListTagsForResource
+adapters/rds.go:44:24: unreachable func: mockRdsClient.DescribeDBClusters
+adapters/rds.go:48:24: unreachable func: mockRdsClient.DescribeDBInstances
+adapters/rds.go:52:24: unreachable func: mockRdsClient.DescribeDBSubnetGroups
+adapters/rds.go:56:24: unreachable func: mockRdsClient.DescribeOptionGroups
+adapters/rds.go:60:24: unreachable func: mockRdsClient.DescribeDBParameterGroups
+adapters/rds.go:64:24: unreachable func: mockRdsClient.DescribeDBParameters

.github/workflows/test-build.yml

@@ -50,6 +50,12 @@ jobs:
       - name: Vet
         run: go vet ./...
 
+      - name: find deadcode
+        run: |
+          go install golang.org/x/tools/cmd/deadcode@latest
+          deadcode . > deadcode.txt
+          diff .deadcode-ignore deadcode.txt
+
       # get .golangci.yml from github.com/overmindtech/golangci-lint_config
       - name: Get .golangci.yml from github.com/overmindtech/golangci-lint_configs
         run: |

cmd/root.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/getsentry/sentry-go"
-	"github.com/google/uuid"
 	"github.com/nats-io/jwt/v2"
 	"github.com/nats-io/nkeys"
 	"github.com/overmindtech/aws-source/proc"
@@ -48,20 +47,14 @@ var rootCmd = &cobra.Command{
 				panic(err)
 			}
 		}()
-
-		var err error
-
 		// Get srcman supplied config
 		natsServers := viper.GetStringSlice("nats-servers")
 		natsJWT := viper.GetString("nats-jwt")
 		natsNKeySeed := viper.GetString("nats-nkey-seed")
-		maxParallel := viper.GetInt("max-parallel")
 		apiKey := viper.GetString("api-key")
 		app := viper.GetString("app")
 		healthCheckPort := viper.GetInt("health-check-port")
 		natsConnectionName := viper.GetString("nats-connection-name")
-		sourceName := viper.GetString("source-name")
-		sourceUUIDString := viper.GetString("source-uuid")
 
 		awsAuthConfig := proc.AwsAuthConfig{
 			Strategy:        viper.GetString("aws-access-strategy"),
@@ -73,7 +66,7 @@ var rootCmd = &cobra.Command{
 			AutoConfig:      viper.GetBool("auto-config"),
 		}
 
-		err = viper.UnmarshalKey("aws-regions", &awsAuthConfig.Regions)
+		err := viper.UnmarshalKey("aws-regions", &awsAuthConfig.Regions)
 		if err != nil {
 			log.WithError(err).Fatal("Could not parse aws-regions")
 		}
@@ -83,12 +76,17 @@ var rootCmd = &cobra.Command{
 			natsNKeySeedLog = "[REDACTED]"
 		}
 
+		engineConfig, err := discovery.EngineConfigFromViper("aws", tracing.ServiceVersion)
+		if err != nil {
+			log.WithError(err).Fatal("Could not create engine config")
+		}
+
 		log.WithFields(log.Fields{
 			"nats-servers":         natsServers,
 			"nats-jwt":             natsJWT,
 			"nats-nkey-seed":       natsNKeySeedLog,
 			"nats-connection-name": natsConnectionName,
-			"max-parallel":         maxParallel,
+			"max-parallel":         engineConfig.MaxParallelExecutions,
 			"aws-regions":          awsAuthConfig.Regions,
 			"aws-access-strategy":  awsAuthConfig.Strategy,
 			"aws-external-id":      awsAuthConfig.ExternalID,
@@ -97,21 +95,10 @@ var rootCmd = &cobra.Command{
 			"auto-config":          awsAuthConfig.AutoConfig,
 			"health-check-port":    healthCheckPort,
 			"app":                  app,
-			"source-name":          sourceName,
-			"source-uuid":          sourceUUIDString,
+			"source-name":          engineConfig.SourceName,
+			"source-uuid":          engineConfig.SourceUUID,
 		}).Info("Got config")
 
-		var sourceUUID uuid.UUID
-		if sourceUUIDString == "" {
-			sourceUUID = uuid.New()
-		} else {
-			sourceUUID, err = uuid.Parse(sourceUUIDString)
-
-			if err != nil {
-				log.WithError(err).Fatal("Could not parse source UUID")
-			}
-		}
-
 		// Determine the required Overmind URLs
 		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 		defer cancel()
@@ -179,14 +166,12 @@ var rootCmd = &cobra.Command{
 			log.WithError(err).Fatal("Could not create AWS configs")
 		}
 
+		// Initialize the engine
 		e, err := proc.InitializeAwsSourceEngine(
 			rateLimitContext,
-			sourceName,
-			tracing.ServiceVersion,
-			sourceUUID,
+			engineConfig,
 			natsOptions,
 			heartbeatOptions,
-			maxParallel,
 			999_999, // Very high max retries as it'll time out after 15min anyway
 			configs...,
 		)
@@ -284,6 +269,9 @@ func init() {
 		log.WithError(err).Fatal("Could not determine hostname for use in NATS connection name and source name")
 	}
 
+	// add engine flags
+	discovery.AddEngineFlags(rootCmd)
+
 	// General config options
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "/etc/srcman/config/source.yaml", "config file path")
 	rootCmd.PersistentFlags().StringVar(&logLevel, "log", "info", "Set the log level. Valid values: panic, fatal, error, warn, info, debug, trace")
@@ -295,16 +283,6 @@ func init() {
 	rootCmd.PersistentFlags().String("nats-nkey-seed", "", "The NKey seed which corresponds to the NATS JWT e.g. SUAFK6QUC...")
 	rootCmd.PersistentFlags().String("nats-connection-name", hostname, "The name that the source should use to connect to NATS")
 
-	rootCmd.PersistentFlags().String("api-key", "", "The API key to use to authenticate to the Overmind API")
-	// Support API Keys in the environment
-	err = viper.BindEnv("api-key", "OVM_API_KEY", "API_KEY")
-	if err != nil {
-		log.WithError(err).Fatal("could not bind api key to env")
-	}
-
-	rootCmd.PersistentFlags().String("app", "https://app.overmind.tech", "The URL of the Overmind app to use")
-	rootCmd.PersistentFlags().Int("max-parallel", 2_000, "Max number of requests to run in parallel")
-
 	// Custom flags for this source
 	rootCmd.PersistentFlags().String("aws-access-strategy", "defaults", "The strategy to use to access this customer's AWS account. Valid values: 'access-key', 'external-id', 'sso-profile', 'defaults'. Default: 'defaults'.")
 	rootCmd.PersistentFlags().String("aws-access-key-id", "", "The ID of the access key to use")
@@ -315,8 +293,6 @@ func init() {
 	rootCmd.PersistentFlags().String("aws-regions", "", "Comma-separated list of AWS regions that this source should operate in")
 	rootCmd.PersistentFlags().BoolP("auto-config", "a", false, "Use the local AWS config, the same as the AWS CLI could use. This can be set up with \"aws configure\"")
 	rootCmd.PersistentFlags().IntP("health-check-port", "", 8080, "The port that the health check should run on")
-	rootCmd.PersistentFlags().String("source-name", fmt.Sprintf("aws-source-%v", hostname), "The name of the source")
-	rootCmd.PersistentFlags().String("source-uuid", "", "The UUID of the source, is this is blank it will be auto-generated. This is used in heartbeats and shouldn't be supplied usually")
 
 	// tracing
 	rootCmd.PersistentFlags().String("honeycomb-api-key", "", "If specified, configures opentelemetry libraries to submit traces to honeycomb")

go.mod

@@ -36,7 +36,7 @@ require (
 	github.com/micahhausler/aws-iam-policy v0.4.2
 	github.com/nats-io/jwt/v2 v2.7.2
 	github.com/nats-io/nkeys v0.4.7
-	github.com/overmindtech/discovery v0.29.2
+	github.com/overmindtech/discovery v0.30.0
 	github.com/overmindtech/sdp-go v0.96.0
 	github.com/overmindtech/sdpcache v1.6.4
 	github.com/sirupsen/logrus v1.9.3
@@ -82,12 +82,12 @@ require (
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/google/btree v1.1.2 // indirect
-	github.com/google/uuid v1.6.0
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/nats-io/nats.go v1.37.0 // indirect

go.sum

@@ -130,8 +130,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
-github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -146,16 +146,16 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/nats-io/jwt/v2 v2.7.2 h1:SCRjfDLJ2q8naXp8YlGJJS5/yj3wGSODFYVi4nnwVMw=
 github.com/nats-io/jwt/v2 v2.7.2/go.mod h1:kB6QUmqHG6Wdrzj0KP2L+OX4xiTPBeV+NHVstFaATXU=
-github.com/nats-io/nats-server/v2 v2.10.21 h1:gfG6T06wBdI25XyY2IsauarOc2srWoFxxfsOKjrzoRA=
-github.com/nats-io/nats-server/v2 v2.10.21/go.mod h1:I1YxSAEWbXCfy0bthwvNb5X43WwIWMz7gx5ZVPDr5Rc=
+github.com/nats-io/nats-server/v2 v2.10.22 h1:Yt63BGu2c3DdMoBZNcR6pjGQwk/asrKU7VX846ibxDA=
+github.com/nats-io/nats-server/v2 v2.10.22/go.mod h1:X/m1ye9NYansUXYFrbcDwUi/blHkrgHh2rgCJaakonk=
 github.com/nats-io/nats.go v1.37.0 h1:07rauXbVnnJvv1gfIyghFEo6lUcYRY0WXc3x7x0vUxE=
 github.com/nats-io/nats.go v1.37.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
 github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
 github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
-github.com/overmindtech/discovery v0.29.2 h1:bRCaWG8T9RfgVURwvrP86XMTjXhP5RXXvMKx9FTLYEA=
-github.com/overmindtech/discovery v0.29.2/go.mod h1:MUDUqB0vM7xJ9wQdpyNWFPmqtXNjh6QfYhQ4g7QIRmU=
+github.com/overmindtech/discovery v0.30.0 h1:cZoYfHtLIzYTAtJO0r8+SAmtFdY+RaozD9hhOJ+6gro=
+github.com/overmindtech/discovery v0.30.0/go.mod h1:rYAyBHfdvK57fUusO4+Qh/YkKMBvbxv6HYWNo23Dhe8=
 github.com/overmindtech/sdp-go v0.96.0 h1:4rgO8VSkS4Kh/Yv8IluFrXD5c7rU2rgACO/xLh6QLII=
 github.com/overmindtech/sdp-go v0.96.0/go.mod h1:6PPU8IBPWeMVXe1UW/LHb/hbwZPR0ndfjHRTbNQJy2w=
 github.com/overmindtech/sdpcache v1.6.4 h1:MJoYBDqDE3s8FrRzZ0RPgFiH39HWI/Mv2ImH1NdLT8k=
@@ -214,8 +214,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0 h1:K0XaT3DwHAcV4nKLzcQ
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.31.0/go.mod h1:B5Ki776z/MBnVha1Nzwp5arlzBbE3+1jk+pGmaP5HME=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 h1:lUsI2TYsQw2r1IASwoROaCnjdj2cvC2+Jbxvk6nHnWU=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0/go.mod h1:2HpZxxQurfGxJlJDblybejHB6RX6pmExPNe517hREw4=
-go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.30.0 h1:kn1BudCgwtE7PxLqcZkErpD8GKqLZ6BSzeW9QihQJeM=
-go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.30.0/go.mod h1:ljkUDtAMdleoi9tIG1R6dJUpVwDcYjw3J2Q6Q/SuiC0=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0 h1:UGZ1QwZWY67Z6BmckTU+9Rxn04m2bD3gD6Mk0OIOCPk=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.31.0/go.mod h1:fcwWuDuaObkkChiDlhEpSq9+X1C0omv+s5mBtToAQ64=
 go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
 go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
 go.opentelemetry.io/otel/schema v0.0.7 h1:UslTvUtbFGmaxlpL1Z+aROrjsP7BRWt06Xxm5Ng/kAU=
@@ -245,8 +245,8 @@ golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
 golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
 golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
-golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
-golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
+golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 h1:T6rh4haD3GVYsgEfWExoCZA2o2FmbNyKpTuAxbEFPTg=
 google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:wp2WsuBYj6j8wUdo3ToZsdxxixbvQNAHqVJrTgi5E5M=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 h1:QCqS/PdaHTSWGvupk2F/ehwHtGc0/GYkT+3GAcR1CCc=

proc/proc.go

@@ -32,7 +32,6 @@ import (
 	awssns "github.com/aws/aws-sdk-go-v2/service/sns"
 	awssqs "github.com/aws/aws-sdk-go-v2/service/sqs"
 	"github.com/cenkalti/backoff/v4"
-	"github.com/google/uuid"
 	"github.com/sourcegraph/conc/pool"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -198,8 +197,8 @@ func CreateAWSConfigs(awsAuthConfig AwsAuthConfig) ([]aws.Config, error) {
 // engine, and an error if any. The context provided will be used for the rate
 // limit buckets and should not be cancelled until the source is shut down. AWS
 // configs should be provided for each region that is enabled
-func InitializeAwsSourceEngine(ctx context.Context, name string, version string, engineUUID uuid.UUID, natsOptions auth.NATSOptions, heartbeatOptions *discovery.HeartbeatOptions, maxParallel int, maxRetries uint64, configs ...aws.Config) (*discovery.Engine, error) {
-	e, err := discovery.NewEngine()
+func InitializeAwsSourceEngine(ctx context.Context, ec *discovery.EngineConfig, natsOptions auth.NATSOptions, heartbeatOptions *discovery.HeartbeatOptions, maxRetries uint64, configs ...aws.Config) (*discovery.Engine, error) {
+	e, err := discovery.NewEngine(ec)
 	if err != nil {
 		return nil, fmt.Errorf("error initializing Engine: %w", err)
 	}
@@ -215,22 +214,13 @@ func InitializeAwsSourceEngine(ctx context.Context, name string, version string,
 		e.HeartbeatOptions = heartbeatOptions
 	}
 
-	e.Name = "aws-source"
 	e.NATSOptions = &natsOptions
-	e.MaxParallelExecutions = maxParallel
-	e.Version = version
-	e.Name = name
-	e.UUID = engineUUID
-	e.Type = "aws"
-
 	e.StartSendingHeartbeats(ctx)
-
 	if len(configs) == 0 {
 		return nil, errors.New("No configs specified")
 	}
 
 	var globalDone atomic.Bool
-
 	var b backoff.BackOff
 	b = backoff.NewExponentialBackOff(
 		backoff.WithMaxInterval(30*time.Second),