From afa694dbc9ad65058bc0905595962c10fcf4b0f8 Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Wed, 11 Oct 2023 15:28:34 +0300 Subject: [PATCH] Move export options to config file with env read support (#392) * Move export options to config file with env read support * misc option update * misc update * readme update --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- .gitignore | 5 ++- README.md | 13 ++------ go.mod | 1 + go.sum | 5 +++ internal/runner/options.go | 64 +++++++++++++++++++++++++++++++++----- pkg/logger/logger.go | 5 +-- pkg/logger/options.go | 12 +++++++ 7 files changed, 84 insertions(+), 21 deletions(-) create mode 100644 pkg/logger/options.go diff --git a/.gitignore b/.gitignore index 08376315..669e6c04 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,7 @@ cmd/proxify/logs/* cmd/replay/replay cmd/mitmrelay/mitmrelay *.exe -dist/* \ No newline at end of file +dist/* + +.vscode +.devcontainer \ No newline at end of file diff --git a/README.md b/README.md index d061744f..04956c3b 100644 --- a/README.md +++ b/README.md @@ -96,19 +96,12 @@ PROXY: -c int Number of requests before switching to the next upstream proxy (default 1) EXPORT: - -max-size int Max export data size (request/responses will be truncated) (default 9223372036854775807) - -elastic-address string elasticsearch address (ip:port) - -elastic-ssl enable elasticsearch ssl - -elastic-ssl-verification enable elasticsearch ssl verification - -elastic-username string elasticsearch username - -elastic-password string elasticsearch password - -elastic-index string elasticsearch index name (default "proxify") - -kafka-address string address of kafka broker (ip:port) - -kafka-topic string kafka topic to publish messages on (default "proxify") + -max-size int Max export data size (request/responses will be truncated) (default 9223372036854775807) CONFIGURATION: -config string path to the proxify configuration file - -config-directory string override the default config path ($home/.config/proxify) (default "$home/.config/proxify") + -ec, -export-config string proxify export module configuration file ($HOME/.config/proxify/export-config.yaml) + -config-directory string override the default config path ($HOME/.config/proxify) -cert-cache-size int Number of certificates to cache (default 256) -a, -allow string[] Allowed list of IP/CIDR's to be proxied -d, -deny string[] Denied list of IP/CIDR's to be proxied diff --git a/go.mod b/go.mod index f48b4fa5..c707037b 100644 --- a/go.mod +++ b/go.mod @@ -129,4 +129,5 @@ require ( google.golang.org/grpc v1.38.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect + gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 26e62b4c..e8798d90 100644 --- a/go.sum +++ b/go.sum @@ -77,6 +77,7 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk= @@ -169,6 +170,7 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= @@ -499,6 +501,7 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -506,6 +509,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/runner/options.go b/internal/runner/options.go index 5a565800..00151188 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -1,6 +1,7 @@ package runner import ( + "fmt" "math" "os" "path/filepath" @@ -9,12 +10,15 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/proxify/pkg/logger" "github.com/projectdiscovery/proxify/pkg/logger/elastic" "github.com/projectdiscovery/proxify/pkg/logger/kafka" "github.com/projectdiscovery/proxify/pkg/types" + errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" permissionutil "github.com/projectdiscovery/utils/permission" updateutils "github.com/projectdiscovery/utils/update" + "gopkg.in/yaml.v2" ) var ( @@ -25,6 +29,7 @@ var ( type Options struct { OutputDirectory string OutputFile string // for storing the jsonl output + LoggerConfig string ConfigDir string CertCacheSize int Verbosity types.Verbosity @@ -104,18 +109,11 @@ func ParseOptions() (*Options, error) { flagSet.CreateGroup("export", "Export", flagSet.IntVar(&options.MaxSize, "max-size", math.MaxInt, "Max export data size (request/responses will be truncated)"), - flagSet.StringVar(&options.Elastic.Addr, "elastic-address", "", "elasticsearch address (ip:port)"), - flagSet.BoolVar(&options.Elastic.SSL, "elastic-ssl", false, "enable elasticsearch ssl"), - flagSet.BoolVar(&options.Elastic.SSLVerification, "elastic-ssl-verification", false, "enable elasticsearch ssl verification"), - flagSet.StringVar(&options.Elastic.Username, "elastic-username", "", "elasticsearch username"), - flagSet.StringVar(&options.Elastic.Password, "elastic-password", "", "elasticsearch password"), - flagSet.StringVar(&options.Elastic.IndexName, "elastic-index", "proxify", "elasticsearch index name"), - flagSet.StringVar(&options.Kafka.Addr, "kafka-address", "", "address of kafka broker (ip:port)"), - flagSet.StringVar(&options.Kafka.Topic, "kafka-topic", "proxify", "kafka topic to publish messages on"), ) flagSet.CreateGroup("configuration", "Configuration", flagSet.StringVar(&cfgFile, "config", "", "path to the proxify configuration file"), + flagSet.StringVarP(&options.LoggerConfig, "export-config", "ec", filepath.Join(homeDir, ".config", "proxify", logger.LoggerConfigFilename), "proxify export module configuration file"), flagSet.StringVar(&options.ConfigDir, "config-directory", filepath.Join(homeDir, ".config", "proxify"), "override the default config path ($home/.config/proxify)"), flagSet.IntVar(&options.CertCacheSize, "cert-cache-size", 256, "Number of certificates to cache"), flagSet.StringSliceVarP(&options.Allow, "allow", "a", nil, "Allowed list of IP/CIDR's to be proxied", goflags.FileNormalizedStringSliceOptions), @@ -151,6 +149,14 @@ func ParseOptions() (*Options, error) { } } + if err := options.createLoggerConfigIfNotExists(); err != nil { + return nil, err + } + + if err := options.parseLoggerConfig(); err != nil { + return nil, err + } + // Read the inputs and configure the logging options.configureVerbosity(silent, verbose, veryVerbose) options.configureOutput() @@ -237,3 +243,45 @@ func (options *Options) configureOutput() { gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true)) } } + +// createLoggerConfigIfNotExists creates export-config if it doesn't exists +func (options *Options) createLoggerConfigIfNotExists() error { + if fileutil.FileExists(options.LoggerConfig) { + return nil + } + + config := &logger.Config{ + Elastic: elastic.Options{}, + Kafka: kafka.Options{}, + } + loggerConfigFile, err := os.Create(options.LoggerConfig) + if err != nil { + return errorutil.NewWithErr(err).Msgf("could not create config file") + } + defer loggerConfigFile.Close() + + err = yaml.NewEncoder(loggerConfigFile).Encode(config) + return err +} + +// parseLoggerConfig parses the logger configuration file +func (options *Options) parseLoggerConfig() error { + var config logger.Config + + data, err := os.ReadFile(options.LoggerConfig) + if err != nil { + return err + } + + expandedData := os.ExpandEnv(string(data)) + err = yaml.Unmarshal([]byte(expandedData), &config) + if err != nil { + return err + } + fmt.Println(expandedData) + + options.Kafka = config.Kafka + options.Elastic = config.Elastic + + return nil +} diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index aad507ca..da5de107 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -22,8 +22,9 @@ import ( ) const ( - dataWithNewLine = "%s\n" - dataWithoutNewLine = "%s" + dataWithNewLine = "%s\n" + dataWithoutNewLine = "%s" + LoggerConfigFilename = "export-config.yaml" ) type OptionsLogger struct { diff --git a/pkg/logger/options.go b/pkg/logger/options.go new file mode 100644 index 00000000..7ecdee52 --- /dev/null +++ b/pkg/logger/options.go @@ -0,0 +1,12 @@ +package logger + +import ( + "github.com/projectdiscovery/proxify/pkg/logger/elastic" + "github.com/projectdiscovery/proxify/pkg/logger/kafka" +) + +// Config is a configuration file for proxify logger module +type Config struct { + Kafka kafka.Options `yaml:"kafka"` + Elastic elastic.Options `yaml:"elastic"` +}