diff --git a/.travis.yml b/.travis.yml index 08617dd..56c6df0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,6 @@ matrix: before_script: - GO_FILES=$(find . -iname '*.go' -type f | grep -v /vendor/) # All the .go files, excluding vendor/ - go get honnef.co/go/tools/cmd/staticcheck # Badass static analyzer/linter - - go get github.com/fzipp/gocyclo - go get github.com/mitchellh/gox - go get github.com/golang/dep/cmd/dep @@ -24,7 +23,6 @@ script: - go test -v -race ./... # Run all the tests with the race detector enabled - go vet ./... # go vet is the official Go static analyzer - staticcheck ./... # "go vet on steroids" + linter - - gocyclo -over 19 $GO_FILES # forbid code with huge functions - make release # build release binaries deploy: diff --git a/cmd/defang/main.go b/cmd/defang/main.go index bd98950..776f833 100644 --- a/cmd/defang/main.go +++ b/cmd/defang/main.go @@ -129,7 +129,6 @@ func (d app) defangIOCs() (output string) { } if govalidator.IsURL(l) && !govalidator.IsIPv4(l) { - // extract links without defanging if d.Config.extract { output += l + "\n" @@ -151,7 +150,6 @@ func (d app) defangIOCs() (output string) { for _, l := range ips { if govalidator.IsIPv4(l) { - // extract IPs without defanging if d.Config.extract { output += l + "\n" diff --git a/defang_url.go b/defang_url.go index 44aed56..bcec22c 100644 --- a/defang_url.go +++ b/defang_url.go @@ -6,8 +6,8 @@ import ( "regexp" "strings" - "github.com/DNSFilter/tldextract" "github.com/asaskevich/govalidator" + "github.com/jakewarren/tldomains" "github.com/pkg/errors" ) @@ -27,7 +27,6 @@ const ( // URL defangs an IPv4 address func URL(rawURL interface{}) (string, error) { - var ( input string output string @@ -44,7 +43,7 @@ func URL(rawURL interface{}) (string, error) { } // if the url doesn't have a scheme, add a temporary one so net/url can parse it properly - if !strings.HasPrefix(input, "http") { + if !strings.HasPrefix(input, "http://") && !strings.HasPrefix(input, "https://") { input = "http://" + input inputWithoutScheme = true } @@ -55,8 +54,7 @@ func URL(rawURL interface{}) (string, error) { } cache := os.TempDir() + "/tld.cache" - extract, err := tldextract.New(cache, false) - + extract, err := tldomains.New(cache) if err != nil { return "", errors.Wrap(err, "error initializing tldextract") } @@ -65,24 +63,25 @@ func URL(rawURL interface{}) (string, error) { if u.Scheme != "" { defangedScheme = strings.Replace(u.Scheme, "http", "hxxp", -1) } - host := extract.Extract(input) + + host := extract.Parse(u.Host) var defangedHost string - if len(host.Sub) > 0 { - defangedHost = host.Sub + "." + if len(host.Subdomain) > 0 { + defangedHost = host.Subdomain + "." } // if the tld has a period, defang there - if strings.Contains(host.Tld, ".") { - defangedHost += host.Root + "." + strings.Replace(host.Tld, ".", "[.]", -1) - } else if len(host.Tld) > 0 { - defangedHost += host.Root + "[.]" + host.Tld - } else if govalidator.IsIPv4(host.Root) { - ip, ipErr := IPv4(host.Root) + if govalidator.IsIPv4(u.Host) { + ip, ipErr := IPv4(u.Host) if ipErr != nil { return "", errors.New("error defanging IPv4 URL") } - defangedHost += ip + defangedHost = ip + } else if strings.Contains(host.Suffix, ".") { + defangedHost += host.Root + "." + strings.Replace(host.Suffix, ".", "[.]", -1) + } else if len(host.Suffix) > 0 { + defangedHost += host.Root + "[.]" + host.Suffix } else { return "", errors.New("error defanging URL") } @@ -110,7 +109,6 @@ func URL(rawURL interface{}) (string, error) { // URLWithMask defangs a URL and applies the specified defanging scheme func URLWithMask(rawURL interface{}, m Mask) (string, error) { - output, err := URL(rawURL) if err != nil { return "", err @@ -132,5 +130,4 @@ func URLWithMask(rawURL interface{}, m Mask) (string, error) { re := regexp.MustCompile(`^hxxp`) return re.ReplaceAllString(output, maskStr), nil - }