Move export options to config file with env read support (#392)

* Move export options to config file with env read support

* misc option update

* misc update

* readme update

---------

Co-authored-by: sandeep <[email protected]>

.gitignore

@@ -5,4 +5,7 @@ cmd/proxify/logs/*
 cmd/replay/replay
 cmd/mitmrelay/mitmrelay
 *.exe
-dist/*
+dist/*
+
+.vscode
+.devcontainer

README.md

@@ -96,19 +96,12 @@ PROXY:
    -c int                       Number of requests before switching to the next upstream proxy (default 1)
 
 EXPORT:
-   -max-size int              Max export data size (request/responses will be truncated) (default 9223372036854775807)
-   -elastic-address string    elasticsearch address (ip:port)
-   -elastic-ssl               enable elasticsearch ssl
-   -elastic-ssl-verification  enable elasticsearch ssl verification
-   -elastic-username string   elasticsearch username
-   -elastic-password string   elasticsearch password
-   -elastic-index string      elasticsearch index name (default "proxify")
-   -kafka-address string      address of kafka broker (ip:port)
-   -kafka-topic string        kafka topic to publish messages on (default "proxify")
+   -max-size int  Max export data size (request/responses will be truncated) (default 9223372036854775807)
 
 CONFIGURATION:
    -config string              path to the proxify configuration file
-   -config-directory string    override the default config path ($home/.config/proxify) (default "$home/.config/proxify")
+   -ec, -export-config string  proxify export module configuration file ($HOME/.config/proxify/export-config.yaml)
+   -config-directory string    override the default config path ($HOME/.config/proxify)
    -cert-cache-size int        Number of certificates to cache (default 256)
    -a, -allow string[]         Allowed list of IP/CIDR's to be proxied
    -d, -deny string[]          Denied list of IP/CIDR's to be proxied

go.mod

@@ -129,4 +129,5 @@ require (
 	google.golang.org/grpc v1.38.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0
 )

go.sum

@@ -77,6 +77,7 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
 github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=
+github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk=
@@ -169,6 +170,7 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
@@ -499,13 +501,16 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o=
 gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/runner/options.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"fmt"
 	"math"
 	"os"
 	"path/filepath"
@@ -9,12 +10,15 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/formatter"
 	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/proxify/pkg/logger"
 	"github.com/projectdiscovery/proxify/pkg/logger/elastic"
 	"github.com/projectdiscovery/proxify/pkg/logger/kafka"
 	"github.com/projectdiscovery/proxify/pkg/types"
+	errorutil "github.com/projectdiscovery/utils/errors"
 	fileutil "github.com/projectdiscovery/utils/file"
 	permissionutil "github.com/projectdiscovery/utils/permission"
 	updateutils "github.com/projectdiscovery/utils/update"
+	"gopkg.in/yaml.v2"
 )
 
 var (
@@ -25,6 +29,7 @@ var (
 type Options struct {
 	OutputDirectory             string
 	OutputFile                  string // for storing the jsonl output
+	LoggerConfig                string
 	ConfigDir                   string
 	CertCacheSize               int
 	Verbosity                   types.Verbosity
@@ -104,18 +109,11 @@ func ParseOptions() (*Options, error) {
 
 	flagSet.CreateGroup("export", "Export",
 		flagSet.IntVar(&options.MaxSize, "max-size", math.MaxInt, "Max export data size (request/responses will be truncated)"),
-		flagSet.StringVar(&options.Elastic.Addr, "elastic-address", "", "elasticsearch address (ip:port)"),
-		flagSet.BoolVar(&options.Elastic.SSL, "elastic-ssl", false, "enable elasticsearch ssl"),
-		flagSet.BoolVar(&options.Elastic.SSLVerification, "elastic-ssl-verification", false, "enable elasticsearch ssl verification"),
-		flagSet.StringVar(&options.Elastic.Username, "elastic-username", "", "elasticsearch username"),
-		flagSet.StringVar(&options.Elastic.Password, "elastic-password", "", "elasticsearch password"),
-		flagSet.StringVar(&options.Elastic.IndexName, "elastic-index", "proxify", "elasticsearch index name"),
-		flagSet.StringVar(&options.Kafka.Addr, "kafka-address", "", "address of kafka broker (ip:port)"),
-		flagSet.StringVar(&options.Kafka.Topic, "kafka-topic", "proxify", "kafka topic to publish messages on"),
 	)
 
 	flagSet.CreateGroup("configuration", "Configuration",
 		flagSet.StringVar(&cfgFile, "config", "", "path to the proxify configuration file"),
+		flagSet.StringVarP(&options.LoggerConfig, "export-config", "ec", filepath.Join(homeDir, ".config", "proxify", logger.LoggerConfigFilename), "proxify export module configuration file"),
 		flagSet.StringVar(&options.ConfigDir, "config-directory", filepath.Join(homeDir, ".config", "proxify"), "override the default config path ($home/.config/proxify)"),
 		flagSet.IntVar(&options.CertCacheSize, "cert-cache-size", 256, "Number of certificates to cache"),
 		flagSet.StringSliceVarP(&options.Allow, "allow", "a", nil, "Allowed list of IP/CIDR's to be proxied", goflags.FileNormalizedStringSliceOptions),
@@ -151,6 +149,14 @@ func ParseOptions() (*Options, error) {
 		}
 	}
 
+	if err := options.createLoggerConfigIfNotExists(); err != nil {
+		return nil, err
+	}
+
+	if err := options.parseLoggerConfig(); err != nil {
+		return nil, err
+	}
+
 	// Read the inputs and configure the logging
 	options.configureVerbosity(silent, verbose, veryVerbose)
 	options.configureOutput()
@@ -237,3 +243,45 @@ func (options *Options) configureOutput() {
 		gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true))
 	}
 }
+
+// createLoggerConfigIfNotExists creates export-config if it doesn't exists
+func (options *Options) createLoggerConfigIfNotExists() error {
+	if fileutil.FileExists(options.LoggerConfig) {
+		return nil
+	}
+
+	config := &logger.Config{
+		Elastic: elastic.Options{},
+		Kafka:   kafka.Options{},
+	}
+	loggerConfigFile, err := os.Create(options.LoggerConfig)
+	if err != nil {
+		return errorutil.NewWithErr(err).Msgf("could not create config file")
+	}
+	defer loggerConfigFile.Close()
+
+	err = yaml.NewEncoder(loggerConfigFile).Encode(config)
+	return err
+}
+
+// parseLoggerConfig parses the logger configuration file
+func (options *Options) parseLoggerConfig() error {
+	var config logger.Config
+
+	data, err := os.ReadFile(options.LoggerConfig)
+	if err != nil {
+		return err
+	}
+
+	expandedData := os.ExpandEnv(string(data))
+	err = yaml.Unmarshal([]byte(expandedData), &config)
+	if err != nil {
+		return err
+	}
+	fmt.Println(expandedData)
+
+	options.Kafka = config.Kafka
+	options.Elastic = config.Elastic
+
+	return nil
+}

pkg/logger/logger.go

@@ -22,8 +22,9 @@ import (
 )
 
 const (
-	dataWithNewLine    = "%s\n"
-	dataWithoutNewLine = "%s"
+	dataWithNewLine      = "%s\n"
+	dataWithoutNewLine   = "%s"
+	LoggerConfigFilename = "export-config.yaml"
 )
 
 type OptionsLogger struct {

pkg/logger/options.go

@@ -0,0 +1,12 @@
+package logger
+
+import (
+	"github.com/projectdiscovery/proxify/pkg/logger/elastic"
+	"github.com/projectdiscovery/proxify/pkg/logger/kafka"
+)
+
+// Config is a configuration file for proxify logger module
+type Config struct {
+	Kafka   kafka.Options   `yaml:"kafka"`
+	Elastic elastic.Options `yaml:"elastic"`
+}