diff --git a/README.md b/README.md index f930322..4f86442 100644 --- a/README.md +++ b/README.md @@ -20,15 +20,15 @@ URLFinder更专注于提取页面中的JS与URL链接，提取的数据更完善 ## 功能说明 -1.提取页面与JS中的JS及URL链接（URL深入一层，JS深入三层 防止抓偏），以及部分敏感信息 +1.提取页面与JS中的JS及URL链接，以及部分敏感信息 2.提取到的链接会显示状态码、响应大小、标题等（带cookie操作时请使用-m 3 安全模式，防止误操作） 3.提取批量URL -4.yml配置Headers请求头、代理 +4.yml配置Headers请求头、代理、抓取规则等 5.结果导出到csv、json、html -6.记录抓取来源，便于手动分析（-o 导出才有） +6.记录抓取来源，便于手动分析（-o 导出才有） 7.指定抓取域名 -8.指定baseurl路径(指定目录拼接) -9.设置代理 +8.指定baseurl路径(指定目录拼接) +9.使用代理ip 10.对404链接Fuzz（测试版，有问题提issue） 结果会优先显示输入的url顶级域名，其他域名不做区分显示在 other @@ -36,14 +36,12 @@ URLFinder更专注于提取页面中的JS与URL链接，提取的数据更完善 ## 使用截图 -[![0.jpg](https://github.com/pingc0y/URLFinder/img/0.jpg)](https://github.com/pingc0y/URLFinder/img/1.jpg) -[![1.jpg](https://github.com/pingc0y/URLFinder/img/1.jpg)](https://github.com/pingc0y/URLFinder/img/2.jpg) -[![2.jpg](https://github.com/pingc0y/URLFinder/img/2.jpg)](https://github.com/pingc0y/URLFinder/img/3.jpg) -[![3.jpg](https://github.com/pingc0y/URLFinder/img/3.jpg)](https://github.com/pingc0y/URLFinder/img/4.jpg) -[![4.jpg](https://github.com/pingc0y/URLFinder/img/4.jpg)](https://github.com/pingc0y/URLFinder/img/5.jpg) -[![5.jpg](https://github.com/pingc0y/URLFinder/img/5.jpg)](https://github.com/pingc0y/URLFinder/img/6.jpg) - - +[![0.jpg](https://github.com/pingc0y/URLFinder/raw/master/img/0.jpg)](https://github.com/pingc0y/URLFinder/raw/master/img/0.jpg) +[![1.jpg](https://github.com/pingc0y/URLFinder/raw/master/img/1.jpg)](https://github.com/pingc0y/URLFinder/raw/master/img/1.jpg) +[![2.jpg](https://github.com/pingc0y/URLFinder/raw/master/img/2.jpg)](https://github.com/pingc0y/URLFinder/raw/master/img/2.jpg) +[![3.jpg](https://github.com/pingc0y/URLFinder/raw/master/img/3.jpg)](https://github.com/pingc0y/URLFinder/raw/master/img/3.jpg) +[![4.jpg](https://github.com/pingc0y/URLFinder/raw/master/img/4.jpg)](https://github.com/pingc0y/URLFinder/raw/master/img/4.jpg) +[![5.jpg](https://github.com/pingc0y/URLFinder/raw/master/img/5.jpg)](https://github.com/pingc0y/URLFinder/raw/master/img/5.jpg) ## 使用教程 单url时使用 @@ -67,7 +65,7 @@ URLFinder.exe -s all -m 2 -f url.txt -o d:/ -i 加载yaml配置文件（不存在时，会在当前目录创建一个默认yaml配置文件） -m 抓取模式： 1 正常抓取（默认） - 2 深入抓取 （url只深入一层，防止抓偏） + 2 深入抓取 （URL深入一层 JS深入三层 防止抓偏） 3 安全深入抓取（过滤delete，remove等敏感路由） -o 结果导出到csv文件，需指定导出文件目录（.代表当前目录） -s 显示指定状态码，all为显示全部 @@ -91,34 +89,38 @@ go build -ldflags "-s -w" -o ./URLFinder-windows-amd64.exe SET CGO_ENABLED=0 SET GOOS=windows SET GOARCH=386 -go build -ldflags "-s -w" -o ../URLFinder-windows-386.exe +go build -ldflags "-s -w" -o ./URLFinder-windows-386.exe SET CGO_ENABLED=0 SET GOOS=linux SET GOARCH=amd64 -go build -ldflags "-s -w" -o ../URLFinder-linux-amd64 +go build -ldflags "-s -w" -o ./URLFinder-linux-amd64 SET CGO_ENABLED=0 SET GOOS=linux SET GOARCH=arm64 -go build -ldflags "-s -w" -o ../URLFinder-linux-arm64 +go build -ldflags "-s -w" -o ./URLFinder-linux-arm64 SET CGO_ENABLED=0 SET GOOS=linux SET GOARCH=386 -go build -ldflags "-s -w" -o ../URLFinder-linux-386 +go build -ldflags "-s -w" -o ./URLFinder-linux-386 SET CGO_ENABLED=0 SET GOOS=darwin SET GOARCH=amd64 -go build -ldflags "-s -w" -o ../URLFinder-macos-amd64 +go build -ldflags "-s -w" -o ./URLFinder-macos-amd64 SET CGO_ENABLED=0 SET GOOS=darwin SET GOARCH=arm64 -go build -ldflags "-s -w" -o ../URLFinder-macos-arm64 +go build -ldflags "-s -w" -o ./URLFinder-macos-arm64 ``` ## 更新说明 +2023/2/3 +新增 域名信息展示 +变化 -i配置文件可配置抓取规则等 + 2023/1/29 新增 -b 设置baseurl路径 新增 -o json、html格式导出 diff --git a/URLFinder-linux-386 b/URLFinder-linux-386 index 94e583a..545147d 100644 Binary files a/URLFinder-linux-386 and b/URLFinder-linux-386 differ diff --git a/URLFinder-linux-amd64 b/URLFinder-linux-amd64 index ff64942..3b1d26c 100644 Binary files a/URLFinder-linux-amd64 and b/URLFinder-linux-amd64 differ diff --git a/URLFinder-linux-arm64 b/URLFinder-linux-arm64 index 5bacd99..1424135 100644 Binary files a/URLFinder-linux-arm64 and b/URLFinder-linux-arm64 differ diff --git a/URLFinder-macos-amd64 b/URLFinder-macos-amd64 index ff64942..3b1d26c 100644 Binary files a/URLFinder-macos-amd64 and b/URLFinder-macos-amd64 differ diff --git a/URLFinder-macos-arm64 b/URLFinder-macos-arm64 index 31e4245..7a547ee 100644 Binary files a/URLFinder-macos-arm64 and b/URLFinder-macos-arm64 differ diff --git a/URLFinder-windows-386.exe b/URLFinder-windows-386.exe index 60ab89b..a07e8bf 100644 Binary files a/URLFinder-windows-386.exe and b/URLFinder-windows-386.exe differ diff --git a/URLFinder-windows-amd64.exe b/URLFinder-windows-amd64.exe index 2283612..84b6e64 100644 Binary files a/URLFinder-windows-amd64.exe and b/URLFinder-windows-amd64.exe differ diff --git a/cmd/cmd.go b/cmd/cmd.go new file mode 100644 index 0000000..ebcd7f2 --- /dev/null +++ b/cmd/cmd.go @@ -0,0 +1,62 @@ +package cmd + +import ( + "flag" + "fmt" + "github.com/gookit/color" + "os" +) + +var ( + h bool + I bool + M int + S string + U string + D string + C string + A string + b string + F string + O string + X string + T = 50 + Z int +) + +func init() { + flag.StringVar(&A, "a", "", "set user-agent\n设置user-agent请求头") + flag.StringVar(&b, "b", "", "set baseurl\n设置baseurl路径") + flag.StringVar(&C, "c", "", "set cookie\n设置cookie") + flag.StringVar(&D, "d", "", "set domainName\n指定获取的域名") + flag.StringVar(&F, "f", "", "set urlFile\n批量抓取url,指定文件路径") + flag.BoolVar(&h, "h", false, "this help\n帮助信息") + flag.BoolVar(&I, "i", false, "set configFile\n加载yaml配置文件（不存在时，会在当前目录创建一个默认yaml配置文件）") + flag.IntVar(&M, "m", 1, "set mode\n抓取模式 \n 1 normal\n 正常抓取（默认） \n 2 thorough\n 深入抓取 （url深入一层,js深入三层，防止抓偏） \n 3 security\n 安全深入抓取（过滤delete，remove等敏感路由） \n ") + flag.StringVar(&O, "o", "", "set outFile\n结果导出到csv文件，需指定导出文件目录（.代表当前目录）") + flag.StringVar(&S, "s", "", "set Status\n显示指定状态码，all为显示全部（多个状态码用,隔开）") + flag.IntVar(&T, "t", 50, "set thread\n设置线程数（默认50）\n") + flag.StringVar(&U, "u", "", "set Url\n目标URL") + flag.StringVar(&X, "x", "", "set httpProxy\n设置代理,格式: http://username:password@127.0.0.1:8809") + flag.IntVar(&Z, "z", 0, "set Fuzz\n对404链接进行fuzz(只对主域名下的链接生效,需要与-s一起使用） \n 1 decreasing\n 目录递减fuzz \n 2 2combination\n 2级目录组合fuzz（适合少量链接使用） \n 3 3combination\n 3级目录组合fuzz（适合少量链接使用） \n") + + // 改变默认的 Usage + flag.Usage = usage +} +func usage() { + fmt.Fprintf(os.Stderr, `Usage: URLFinder [-a user-agent] [-b baseurl] [-c cookie] [-d domainName] [-f urlFile] [-h help] [-i configFile] [-m mode] [-o outFile] [-s Status] [-t thread] [-u Url] [-x httpProxy] [-z fuzz] + +Options: +`) + flag.PrintDefaults() +} + +func Parse() { + color.LightCyan.Println(" __ __ ___ _ _ \n /\\ /\\ /__\\ / / / __(_)_ __ __| | ___ _ __ \n/ / \\ \\/ \\/// / / _\\ | | '_ \\ / _` |/ _ \\ '__|\n\\ \\_/ / _ \\ /___ / | | | | | (_| | __/ | \n \\___/\\/ \\_\\____\\/ |_|_| |_|\\__,_|\\___|_| \n\nBy: pingc0y\nUpdateTime: 2023/2/3\nGithub: https://github.com/pingc0y/URLFinder \n") + flag.Parse() + if h || (U == "" && F == "") { + flag.Usage() + os.Exit(0) + } + +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..9e2aca8 --- /dev/null +++ b/config/config.go @@ -0,0 +1,109 @@ +package config + +import ( + "fmt" + "github.com/pingc0y/URLFinder/cmd" + "github.com/pingc0y/URLFinder/mode" + "gopkg.in/yaml.v3" + "os" + "strings" + "sync" +) + +var Conf mode.Config +var Progress = 1 +var FuzzNum int + +var ( + Risks = []string{"remove", "delete", "insert", "update", "logout"} + + JsFuzzPath = []string{ + "login.js", + "app.js", + "main.js", + "config.js", + "admin.js", + "info.js", + "open.js", + "user.js", + "input.js", + "list.js", + "upload.js", + } + JsFind = []string{ + ".(https{0,1}:[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{2,250}?[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{3}[.]js)", + "[\",',‘,“]\\s{0,6}(/{0,1}[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{2,250}?[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{3}[.]js)", + "=\\s{0,6}[\",',’,”]{0,1}\\s{0,6}(/{0,1}[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{2,250}?[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{3}[.]js)", + } + UrlFind = []string{ + "[\",',‘,“]\\s{0,6}(https{0,1}:[-a-zA-Z0-9()@:%_\\+.~#?&//=]{2,250}?)\\s{0,6}[\",',‘,“]", + "=\\s{0,6}(https{0,1}:[-a-zA-Z0-9()@:%_\\+.~#?&//=]{2,250})", + "[\",',‘,“]\\s{0,6}([#,.]{0,2}/[-a-zA-Z0-9()@:%_\\+.~#?&//=]{2,250}?)\\s{0,6}[\",',‘,“]", + "\"([-a-zA-Z0-9()@:%_\\+.~#?&//=]+?[/]{1}[-a-zA-Z0-9()@:%_\\+.~#?&//=]+?)\"", + "href\\s{0,6}=\\s{0,6}[\",',‘,“]{0,1}\\s{0,6}([-a-zA-Z0-9()@:%_\\+.~#?&//=]{2,250})|action\\s{0,6}=\\s{0,6}[\",',‘,“]{0,1}\\s{0,6}([-a-zA-Z0-9()@:%_\\+.~#?&//=]{2,250})", + } + + JsFiler = []string{ + "www\\.w3\\.org", + "example\\.com", + } + UrlFiler = []string{ + "\\.js\\?|\\.css\\?|\\.jpeg\\?|\\.jpg\\?|\\.png\\?|.gif\\?|www\\.w3\\.org|example\\.com|\\<|\\>|\\{|\\}|\\[|\\]|\\||\\^|;|/js/|\\.src|\\.replace|\\.url|\\.att|\\.href|location\\.href|javascript:|location:|application/x-www-form-urlencoded|\\.createObject|:location|\\.path|\\*#__PURE__\\*|\\*\\$0\\*|\\n", + ".*\\.js$|.*\\.css$|.*\\.scss$|.*,$|.*\\.jpeg$|.*\\.jpg$|.*\\.png&|.*\\.gif&|.*\\.ico$|.*\\.svg$|.*\\.vue$|.*\\.ts$", + } + + Phone = []string{"['\"](1(3([0-35-9]\\d|4[1-8])|4[14-9]\\d|5([\\d]\\d|7[1-79])|66\\d|7[2-35-8]\\d|8\\d{2}|9[89]\\d)\\d{7})['\"]"} + Email = []string{"['\"]([\\w!#$%&'*+=?^_`{|}~-]+(?:\\.[\\w!#$%&'*+=?^_`{|}~-]+)*@(?:[\\w](?:[\\w-]*[\\w])?\\.)+[\\w](?:[\\w-]*[\\w])?)['\"]"} + IDcard = []string{"['\"]((\\d{8}(0\\d|10|11|12)([0-2]\\d|30|31)\\d{3}$)|(\\d{6}(18|19|20)\\d{2}(0[1-9]|10|11|12)([0-2]\\d|30|31)\\d{3}(\\d|X|x)))['\"]"} + Jwt = []string{"['\"](ey[A-Za-z0-9_-]{10,}\\.[A-Za-z0-9._-]{10,}|ey[A-Za-z0-9_\\/+-]{10,}\\.[A-Za-z0-9._\\/+-]{10,})['\"]"} +) + +var ( + Lock sync.Mutex + Wg sync.WaitGroup + Mux sync.Mutex + Ch = make(chan int, 50) + Jsch = make(chan int, 50/2) + Urlch = make(chan int, 50/2) +) + +// 读取配置文件 +func GetConfig(path string) { + con := &mode.Config{} + if f, err := os.Open(path); err != nil { + if strings.Contains(err.Error(), "The system cannot find the file specified") || strings.Contains(err.Error(), "no such file or directory") { + con.Headers = map[string]string{"Cookie": cmd.C, "User-Agent": `Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36 SE 2.X MetaSr 1.0`, "Accept": "*/*"} + con.Proxy = "" + con.JsFind = JsFind + con.UrlFind = UrlFind + con.JsFiler = JsFiler + con.UrlFiler = UrlFiler + con.JsFuzzPath = JsFuzzPath + con.InfoFind = map[string][]string{"Phone": Phone, "Email": Email, "IDcard": IDcard, "Jwt": Jwt} + data, err2 := yaml.Marshal(con) + err2 = os.WriteFile(path, data, 0644) + if err2 != nil { + fmt.Println(err) + } else { + fmt.Println("未找到配置文件,已在当面目录下创建配置文件: config.yaml") + } + } else { + fmt.Println("配置文件错误,请尝试重新生成配置文件") + fmt.Println(err) + } + os.Exit(1) + } else { + yaml.NewDecoder(f).Decode(con) + Conf = *con + JsFind = con.JsFind + UrlFind = con.UrlFind + JsFiler = con.JsFiler + UrlFiler = con.UrlFiler + JsFuzzPath = con.JsFuzzPath + Phone = con.InfoFind["Phone"] + Email = con.InfoFind["Email"] + IDcard = con.InfoFind["IDcard"] + Jwt = con.InfoFind["Jwt"] + } + +} diff --git a/crawler/crawler.go b/crawler/crawler.go new file mode 100644 index 0000000..b548b08 --- /dev/null +++ b/crawler/crawler.go @@ -0,0 +1,133 @@ +package crawler + +import ( + "crypto/tls" + "fmt" + "github.com/pingc0y/URLFinder/cmd" + "github.com/pingc0y/URLFinder/config" + "github.com/pingc0y/URLFinder/result" + "github.com/pingc0y/URLFinder/util" + "io" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "time" +) + +// 蜘蛛抓取页面内容 +func Spider(u string, num int) { + var is bool + fmt.Printf("\rSpider %d", config.Progress) + config.Mux.Lock() + config.Progress++ + config.Mux.Unlock() + //标记完成 + defer func() { + config.Wg.Done() + if !is { + <-config.Ch + } + + }() + u, _ = url.QueryUnescape(u) + if num > 1 && cmd.D != "" && !strings.Contains(u, cmd.D) { + return + } + if GetEndUrl(u) { + return + } + if cmd.M == 3 { + for _, v := range config.Risks { + if strings.Contains(u, v) { + return + } + } + } + AppendEndUrl(u) + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + + //配置代理 + if cmd.X != "" { + proxyUrl, parseErr := url.Parse(config.Conf.Proxy) + if parseErr != nil { + fmt.Println("代理地址错误: \n" + parseErr.Error()) + os.Exit(1) + } + tr.Proxy = http.ProxyURL(proxyUrl) + } + //加载yaml配置(proxy) + if cmd.I { + util.SetProxyConfig(tr) + } + client := &http.Client{Timeout: 10 * time.Second, Transport: tr} + request, err := http.NewRequest("GET", u, nil) + if err != nil { + return + } + //增加header选项 + request.Header.Add("Cookie", cmd.C) + request.Header.Add("User-Agent", util.GetUserAgent()) + request.Header.Add("Accept", "*/*") + //加载yaml配置（headers） + if cmd.I { + util.SetHeadersConfig(&request.Header) + } + //处理返回结果 + response, err := client.Do(request) + if err != nil { + return + } else { + defer response.Body.Close() + + } + + //提取url用于拼接其他url或js + dataBytes, err := io.ReadAll(response.Body) + if err != nil { + return + } + path := response.Request.URL.Path + host := response.Request.URL.Host + scheme := response.Request.URL.Scheme + source := scheme + "://" + host + path + + //字节数组 转换成 字符串 + result := string(dataBytes) + //处理base标签 + re := regexp.MustCompile("base.{1,5}href.{1,5}(http.+?//[^\\s]+?)[\",',‘,“]") + base := re.FindAllStringSubmatch(result, -1) + if len(base) > 0 { + host = regexp.MustCompile("http.*?//([^/]+)").FindAllStringSubmatch(base[0][1], -1)[0][1] + scheme = regexp.MustCompile("(http.*?)://").FindAllStringSubmatch(base[0][1], -1)[0][1] + paths := regexp.MustCompile("http.*?//.*?(/.*)").FindAllStringSubmatch(base[0][1], -1) + if len(paths) > 0 { + path = paths[0][1] + } else { + path = "/" + } + } + <-config.Ch + is = true + + //提取js + jsFind(result, host, scheme, path, source, num) + //提取url + urlFind(result, host, scheme, path, source, num) + //提取信息 + infoFind(result, source) + +} + +// 打印Validate进度 +func PrintProgress() { + num := len(result.ResultJs) + len(result.ResultUrl) + fmt.Printf("\rValidate %.0f%%", float64(config.Progress+1)/float64(num+1)*100) + config.Mux.Lock() + config.Progress++ + config.Mux.Unlock() +} diff --git a/crawler/filter.go b/crawler/filter.go new file mode 100644 index 0000000..2c42f3a --- /dev/null +++ b/crawler/filter.go @@ -0,0 +1,74 @@ +package crawler + +import ( + "github.com/pingc0y/URLFinder/config" + "net/url" + "regexp" + "strings" +) + +// 过滤JS +func jsFilter(str [][]string) [][]string { + + //对不需要的数据过滤 + for i := range str { + str[i][0], _ = url.QueryUnescape(str[i][1]) + str[i][0] = strings.Replace(str[i][0], " ", "", -1) + str[i][0] = strings.Replace(str[i][0], "\\/", "/", -1) + str[i][0] = strings.Replace(str[i][0], "%3A", ":", -1) + str[i][0] = strings.Replace(str[i][0], "%2F", "/", -1) + + //去除不是.js的链接 + if !strings.HasSuffix(str[i][0], ".js") && !strings.Contains(str[i][0], ".js?") { + str[i][0] = "" + } + + //过滤配置的黑名单 + for i2 := range config.JsFiler { + re := regexp.MustCompile(config.JsFiler[i2]) + is := re.MatchString(str[i][0]) + if is { + str[i][0] = "" + } + } + + } + return str + +} + +// 过滤URL +func urlFilter(str [][]string) [][]string { + + //对不需要的数据过滤 + for i := range str { + str[i][0], _ = url.QueryUnescape(str[i][1]) + str[i][0] = strings.Replace(str[i][0], " ", "", -1) + str[i][0] = strings.Replace(str[i][0], "\\/", "/", -1) + str[i][0] = strings.Replace(str[i][0], "%3A", ":", -1) + str[i][0] = strings.Replace(str[i][0], "%2F", "/", -1) + + //去除不存在字符串和数字的url，判断为错误数据 + match, _ := regexp.MatchString("[a-zA-Z]+|[0-9]+", str[i][0]) + if !match { + str[i][0] = "" + } + + //对抓到的域名做处理 + re := regexp.MustCompile("([a-z0-9\\-]+\\.)+([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?").FindAllString(str[i][0], 1) + if len(re) != 0 && !strings.HasPrefix(str[i][0], "http") && !strings.HasPrefix(str[i][0], "/") { + str[i][0] = "http://" + str[i][0] + } + + //过滤配置的黑名单 + for i2 := range config.UrlFiler { + re := regexp.MustCompile(config.UrlFiler[i2]) + is := re.MatchString(str[i][0]) + if is { + str[i][0] = "" + } + } + + } + return str +} diff --git a/crawler/find.go b/crawler/find.go new file mode 100644 index 0000000..d04abf8 --- /dev/null +++ b/crawler/find.go @@ -0,0 +1,195 @@ +package crawler + +import ( + "github.com/pingc0y/URLFinder/cmd" + "github.com/pingc0y/URLFinder/config" + "github.com/pingc0y/URLFinder/mode" + "github.com/pingc0y/URLFinder/result" + "regexp" + "strings" +) + +// 分析内容中的js +func jsFind(cont, host, scheme, path, source string, num int) { + var cata string + care := regexp.MustCompile("/.*/{1}|/") + catae := care.FindAllString(path, -1) + if len(catae) == 0 { + cata = "/" + } else { + cata = catae[0] + } + //js匹配正则 + host = scheme + "://" + host + for _, re := range config.JsFind { + reg := regexp.MustCompile(re) + jss := reg.FindAllStringSubmatch(cont, -1) + //return + jss = jsFilter(jss) + //循环提取js放到结果中 + for _, js := range jss { + if js[0] == "" { + continue + } + if strings.HasPrefix(js[0], "https:") || strings.HasPrefix(js[0], "http:") { + AppendJs(js[0], source) + if num < 5 && (cmd.M == 2 || cmd.M == 3) { + config.Wg.Add(1) + config.Ch <- 1 + go Spider(js[0], num+1) + } + } else if strings.HasPrefix(js[0], "//") { + AppendJs(scheme+":"+js[0], source) + if num < 5 && (cmd.M == 2 || cmd.M == 3) { + config.Wg.Add(1) + config.Ch <- 1 + go Spider(scheme+":"+js[0], num+1) + } + + } else if strings.HasPrefix(js[0], "/") { + AppendJs(host+js[0], source) + if num < 5 && (cmd.M == 2 || cmd.M == 3) { + config.Wg.Add(1) + config.Ch <- 1 + go Spider(host+js[0], num+1) + } + } else if strings.HasPrefix(js[0], "./") { + AppendJs(host+"/"+js[0], source) + if num < 5 && (cmd.M == 2 || cmd.M == 3) { + config.Wg.Add(1) + config.Ch <- 1 + go Spider(host+"/"+js[0], num+1) + } + } else { + AppendJs(host+cata+js[0], source) + if num < 5 && (cmd.M == 2 || cmd.M == 3) { + config.Wg.Add(1) + config.Ch <- 1 + go Spider(host+cata+js[0], num+1) + } + } + } + + } + +} + +// 分析内容中的url +func urlFind(cont, host, scheme, path, source string, num int) { + var cata string + care := regexp.MustCompile("/.*/{1}|/") + catae := care.FindAllString(path, -1) + if len(catae) == 0 { + cata = "/" + } else { + cata = catae[0] + } + host = scheme + "://" + host + + //url匹配正则 + + for _, re := range config.UrlFind { + reg := regexp.MustCompile(re) + urls := reg.FindAllStringSubmatch(cont, -1) + //fmt.Println(urls) + urls = urlFilter(urls) + + //循环提取url放到结果中 + for _, url := range urls { + if url[0] == "" { + continue + } + if strings.HasPrefix(url[0], "https:") || strings.HasPrefix(url[0], "http:") { + AppendUrl(url[0], source) + if num < 2 && (cmd.M == 2 || cmd.M == 3) { + config.Wg.Add(1) + config.Ch <- 1 + go Spider(url[0], num+1) + } + } else if strings.HasPrefix(url[0], "//") { + AppendUrl(scheme+":"+url[0], source) + if num < 2 && (cmd.M == 2 || cmd.M == 3) { + config.Wg.Add(1) + config.Ch <- 1 + go Spider(scheme+":"+url[0], num+1) + } + } else if strings.HasPrefix(url[0], "/") { + urlz := "" + if cmd.D != "" { + urlz = cmd.D + url[0] + } else { + urlz = host + url[0] + } + AppendUrl(urlz, source) + if num < 2 && (cmd.M == 2 || cmd.M == 3) { + config.Wg.Add(1) + config.Ch <- 1 + go Spider(urlz, num+1) + } + } else if !strings.HasSuffix(source, ".js") { + urlz := "" + if cmd.D != "" { + if strings.HasSuffix(cmd.D, "/") { + urlz = cmd.D + url[0] + } else { + urlz = cmd.D + "/" + url[0] + } + } else { + urlz = host + cata + url[0] + } + AppendUrl(urlz, source) + if num < 2 && (cmd.M == 2 || cmd.M == 3) { + config.Wg.Add(1) + config.Ch <- 1 + go Spider(urlz, num+1) + } + } else if strings.HasSuffix(source, ".js") { + AppendUrl(result.Jsinurl[host+path]+url[0], source) + if num < 2 && (cmd.M == 2 || cmd.M == 3) { + config.Wg.Add(1) + config.Ch <- 1 + go Spider(result.Jsinurl[host+path]+url[0], num+1) + } + } + } + } +} + +// 分析内容中的敏感信息 +func infoFind(cont, source string) { + info := mode.Info{} + //手机号码 + for i := range config.Phone { + phones := regexp.MustCompile(config.Phone[i]).FindAllStringSubmatch(cont, -1) + for i := range phones { + info.Phone = append(info.Phone, phones[i][1]) + } + } + + for i := range config.Email { + emails := regexp.MustCompile(config.Email[i]).FindAllStringSubmatch(cont, -1) + for i := range emails { + info.Email = append(info.Email, emails[i][1]) + } + } + + for i := range config.IDcard { + IDcards := regexp.MustCompile(config.IDcard[i]).FindAllStringSubmatch(cont, -1) + for i := range IDcards { + info.IDcard = append(info.IDcard, IDcards[i][1]) + } + } + + for i := range config.Jwt { + Jwts := regexp.MustCompile(config.Jwt[i]).FindAllStringSubmatch(cont, -1) + for i := range Jwts { + info.JWT = append(info.JWT, Jwts[i][1]) + } + } + + info.Source = source + if len(info.Phone) != 0 || len(info.IDcard) != 0 || len(info.JWT) != 0 || len(info.Email) != 0 { + AppendInfo(info) + } + +} diff --git a/crawler/fuzz/jsFuzz.go b/crawler/fuzz/jsFuzz.go new file mode 100644 index 0000000..80fb61b --- /dev/null +++ b/crawler/fuzz/jsFuzz.go @@ -0,0 +1,33 @@ +package fuzz + +import ( + "github.com/pingc0y/URLFinder/config" + "github.com/pingc0y/URLFinder/mode" + "github.com/pingc0y/URLFinder/result" + "github.com/pingc0y/URLFinder/util" + "regexp" +) + +func JsFuzz() { + + paths := []string{} + for i := range result.ResultJs { + re := regexp.MustCompile("(.+/)[^/]+.js").FindAllStringSubmatch(result.ResultJs[i].Url, -1) + if len(re) != 0 { + paths = append(paths, re[0][1]) + } + re2 := regexp.MustCompile("(https{0,1}://([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?/)").FindAllStringSubmatch(result.ResultJs[i].Url, -1) + if len(re2) != 0 { + paths = append(paths, re2[0][1]) + } + } + paths = util.UniqueArr(paths) + for i := range paths { + for i2 := range config.JsFuzzPath { + result.ResultJs = append(result.ResultJs, mode.Link{ + Url: paths[i] + config.JsFuzzPath[i2], + Source: "Fuzz", + }) + } + } +} diff --git a/src/fuzz.go b/crawler/fuzz/urlFuzz.go similarity index 64% rename from src/fuzz.go rename to crawler/fuzz/urlFuzz.go index 2c3920b..2797e8d 100644 --- a/src/fuzz.go +++ b/crawler/fuzz/urlFuzz.go @@ -1,8 +1,13 @@ -package main +package fuzz import ( "crypto/tls" "fmt" + "github.com/pingc0y/URLFinder/cmd" + "github.com/pingc0y/URLFinder/config" + "github.com/pingc0y/URLFinder/mode" + "github.com/pingc0y/URLFinder/result" + "github.com/pingc0y/URLFinder/util" "io" "net/http" "net/url" @@ -13,23 +18,23 @@ import ( "time" ) -// fuzz -func fuzz() { +// Fuzz +func UrlFuzz() { var host string re := regexp.MustCompile("([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") - hosts := re.FindAllString(u, 1) + hosts := re.FindAllString(cmd.U, 1) if len(hosts) == 0 { - host = u + host = cmd.U } else { host = hosts[0] } - if d != "" { - host = d + if cmd.D != "" { + host = cmd.D } - disposes, _ := urlDispose(append(resultUrl, Link{Url: u, Status: "200", Size: "0"}), host, "") - if z == 2 || z == 3 { + disposes, _ := util.UrlDispose(append(result.ResultUrl, mode.Link{Url: cmd.U, Status: "200", Size: "0"}), host, "") + if cmd.Z == 2 || cmd.Z == 3 { fuzz2(disposes) - } else if z != 0 { + } else if cmd.Z != 0 { fuzz1(disposes) } fmt.Println("\rFuzz OK ") @@ -38,12 +43,12 @@ func fuzz() { // fuzz请求 func fuzzGet(u string) { defer func() { - wg.Done() - <-ch - printFuzz() + config.Wg.Done() + <-config.Ch + util.PrintFuzz() }() - if m == 3 { - for _, v := range risks { + if cmd.M == 3 { + for _, v := range config.Risks { if strings.Contains(u, v) { return } @@ -53,8 +58,8 @@ func fuzzGet(u string) { TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } //配置代理 - if x != "" { - proxyUrl, parseErr := url.Parse(conf.Proxy) + if cmd.X != "" { + proxyUrl, parseErr := url.Parse(config.Conf.Proxy) if parseErr != nil { fmt.Println("代理地址错误: \n" + parseErr.Error()) os.Exit(1) @@ -62,8 +67,8 @@ func fuzzGet(u string) { tr.Proxy = http.ProxyURL(proxyUrl) } //加载yaml配置(proxy) - if I { - SetProxyConfig(tr) + if cmd.I { + util.SetProxyConfig(tr) } client := &http.Client{Timeout: 10 * time.Second, Transport: tr} request, err := http.NewRequest("GET", u, nil) @@ -71,12 +76,12 @@ func fuzzGet(u string) { return } //增加header选项 - request.Header.Add("Cookie", c) - request.Header.Add("User-Agent", ua) + request.Header.Add("Cookie", cmd.C) + request.Header.Add("User-Agent", util.GetUserAgent()) request.Header.Add("Accept", "*/*") //加载yaml配置 - if I { - SetHeadersConfig(&request.Header) + if cmd.I { + util.SetHeadersConfig(&request.Header) } //处理返回结果 response, err := client.Do(request) @@ -86,7 +91,7 @@ func fuzzGet(u string) { defer response.Body.Close() } code := response.StatusCode - if strings.Contains(s, strconv.Itoa(code)) || s == "all" { + if strings.Contains(cmd.S, strconv.Itoa(code)) || cmd.S == "all" { var length int dataBytes, err := io.ReadAll(response.Body) if err != nil { @@ -97,17 +102,17 @@ func fuzzGet(u string) { body := string(dataBytes) re := regexp.MustCompile("(.*?)") title := re.FindAllStringSubmatch(body, -1) - lock.Lock() + config.Lock.Lock() if len(title) != 0 { - fuzzs = append(fuzzs, Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Title: title[0][1], Source: "Fuzz"}) + result.Fuzzs = append(result.Fuzzs, mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Title: title[0][1], Source: "Fuzz"}) } else { - fuzzs = append(fuzzs, Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Title: "", Source: "Fuzz"}) + result.Fuzzs = append(result.Fuzzs, mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Title: "", Source: "Fuzz"}) } - lock.Unlock() + config.Lock.Unlock() } } -func fuzz1(disposes []Link) { +func fuzz1(disposes []mode.Link) { dispose404 := []string{} for _, v := range disposes { if v.Status == "404" { @@ -154,21 +159,21 @@ func fuzz1(disposes []Link) { } } } - fuzz1s = uniqueArr(fuzz1s) - fuzzNum = len(fuzz1s) - progress = 1 - fmt.Printf("Start %d Fuzz...\n", fuzzNum) + fuzz1s = util.UniqueArr(fuzz1s) + config.FuzzNum = len(fuzz1s) + config.Progress = 1 + fmt.Printf("Start %d Fuzz...\n", config.FuzzNum) fmt.Printf("\r ") for _, v := range fuzz1s { - wg.Add(1) - ch <- 1 + config.Wg.Add(1) + config.Ch <- 1 go fuzzGet(v) } - wg.Wait() - fuzzs = del404(fuzzs) + config.Wg.Wait() + result.Fuzzs = util.Del404(result.Fuzzs) } -func fuzz2(disposes []Link) { +func fuzz2(disposes []mode.Link) { disposex := []string{} dispose404 := []string{} for _, v := range disposes { @@ -187,20 +192,20 @@ func fuzz2(disposes []Link) { } } - dispose, _ := pathExtract(disposex) - _, targets := pathExtract(dispose404) + dispose, _ := util.PathExtract(disposex) + _, targets := util.PathExtract(dispose404) - fuzzNum = len(dispose) * len(targets) - progress = 1 + config.FuzzNum = len(dispose) * len(targets) + config.Progress = 1 fmt.Printf("Start %d Fuzz...\n", len(dispose)*len(targets)) fmt.Printf("\r ") for _, v := range dispose { for _, vv := range targets { - wg.Add(1) - ch <- 1 + config.Wg.Add(1) + config.Ch <- 1 go fuzzGet(v + vv) } } - wg.Wait() - fuzzs = del404(fuzzs) + config.Wg.Wait() + result.Fuzzs = util.Del404(result.Fuzzs) } diff --git a/crawler/run.go b/crawler/run.go new file mode 100644 index 0000000..3985434 --- /dev/null +++ b/crawler/run.go @@ -0,0 +1,196 @@ +package crawler + +import ( + "bufio" + "fmt" + "github.com/gookit/color" + "github.com/pingc0y/URLFinder/cmd" + "github.com/pingc0y/URLFinder/config" + "github.com/pingc0y/URLFinder/crawler/fuzz" + "github.com/pingc0y/URLFinder/mode" + "github.com/pingc0y/URLFinder/result" + "github.com/pingc0y/URLFinder/util" + "io" + "os" + "regexp" + "strings" + "time" +) + +func start(u string) { + result.Jsinurl = make(map[string]string) + result.Jstourl = make(map[string]string) + result.Urltourl = make(map[string]string) + result.Infos = []mode.Info{} + fmt.Println("Start Spider URL: " + color.LightBlue.Sprintf(u)) + config.Wg.Add(1) + config.Ch <- 1 + go Spider(u, 1) + config.Wg.Wait() + config.Progress = 1 + fmt.Printf("\rSpider OK \n") + result.ResultUrl = util.RemoveRepeatElement(result.ResultUrl) + result.ResultJs = util.RemoveRepeatElement(result.ResultJs) + if cmd.S != "" { + fmt.Printf("Start %d Validate...\n", len(result.ResultUrl)+len(result.ResultJs)) + fmt.Printf("\r ") + fuzz.JsFuzz() + //验证JS状态 + for i, s := range result.ResultJs { + config.Wg.Add(1) + config.Jsch <- 1 + go JsState(s.Url, i, result.ResultJs[i].Source) + } + //验证URL状态 + for i, s := range result.ResultUrl { + config.Wg.Add(1) + config.Urlch <- 1 + go UrlState(s.Url, i) + } + config.Wg.Wait() + + time.Sleep(1 * time.Second) + fmt.Printf("\r ") + fmt.Printf("\rValidate OK \n") + + if cmd.Z != 0 { + fuzz.UrlFuzz() + time.Sleep(1 * time.Second) + } + } + AddSource() + + //打印还是输出 + if len(cmd.O) > 0 { + result.OutFileJson() + result.OutFileCsv() + result.OutFileHtml() + } else { + result.Print() + } +} + +func Run() { + + if cmd.O != "" { + if !util.IsDir(cmd.O) { + return + } + } + if cmd.I { + config.GetConfig("config.yaml") + } else { + + } + if cmd.T != 50 { + config.Ch = make(chan int, cmd.T+1) + config.Jsch = make(chan int, cmd.T/2+1) + config.Urlch = make(chan int, cmd.T/2+1) + } + if cmd.F != "" { + // 创建句柄 + fi, err := os.Open(cmd.F) + if err != nil { + panic(err) + } + r := bufio.NewReader(fi) // 创建 Reader + for { + result.ResultJs = nil + result.ResultUrl = nil + result.EndUrl = nil + result.Domains = nil + + lineBytes, err := r.ReadBytes('\n') + //去掉字符串首尾空白字符，返回字符串 + line := strings.TrimSpace(string(lineBytes)) + cmd.U = line + start(cmd.U) + + if err == io.EOF { + break + } + fmt.Println("----------------------------------------") + + } + return + } + start(cmd.U) +} + +func AppendJs(url string, urltjs string) { + config.Lock.Lock() + defer config.Lock.Unlock() + url = strings.Replace(url, "/./", "/", -1) + for _, eachItem := range result.ResultJs { + if eachItem.Url == url { + return + } + } + result.ResultJs = append(result.ResultJs, mode.Link{Url: url}) + if strings.HasSuffix(urltjs, ".js") { + result.Jsinurl[url] = result.Jsinurl[urltjs] + } else { + re := regexp.MustCompile("[a-zA-z]+://[^\\s]*/|[a-zA-z]+://[^\\s]*") + u := re.FindAllStringSubmatch(urltjs, -1) + result.Jsinurl[url] = u[0][0] + } + if cmd.O != "" { + result.Jstourl[url] = urltjs + } + +} + +func AppendUrl(url string, urlturl string) { + config.Lock.Lock() + defer config.Lock.Unlock() + url = strings.Replace(url, "/./", "/", -1) + for _, eachItem := range result.ResultUrl { + if eachItem.Url == url { + return + } + } + result.ResultUrl = append(result.ResultUrl, mode.Link{Url: url}) + if cmd.O != "" { + result.Urltourl[url] = urlturl + } +} + +func AppendInfo(info mode.Info) { + config.Lock.Lock() + defer config.Lock.Unlock() + result.Infos = append(result.Infos, info) +} + +func AppendEndUrl(url string) { + config.Lock.Lock() + defer config.Lock.Unlock() + for _, eachItem := range result.EndUrl { + if eachItem == url { + return + } + } + result.EndUrl = append(result.EndUrl, url) + +} + +func GetEndUrl(url string) bool { + config.Lock.Lock() + defer config.Lock.Unlock() + for _, eachItem := range result.EndUrl { + if eachItem == url { + return true + } + } + return false + +} + +func AddSource() { + for i := range result.ResultJs { + result.ResultJs[i].Source = result.Jstourl[result.ResultJs[i].Url] + } + for i := range result.ResultUrl { + result.ResultUrl[i].Source = result.Urltourl[result.ResultUrl[i].Url] + } + +} diff --git a/crawler/state.go b/crawler/state.go new file mode 100644 index 0000000..ab5c29c --- /dev/null +++ b/crawler/state.go @@ -0,0 +1,187 @@ +package crawler + +import ( + "crypto/tls" + "fmt" + "github.com/pingc0y/URLFinder/cmd" + "github.com/pingc0y/URLFinder/config" + "github.com/pingc0y/URLFinder/mode" + "github.com/pingc0y/URLFinder/result" + "github.com/pingc0y/URLFinder/util" + "io" + "net/http" + "net/url" + "os" + "regexp" + "strconv" + "strings" + "time" +) + +// 检测js访问状态码 +func JsState(u string, i int, sou string) { + defer func() { + config.Wg.Done() + <-config.Jsch + PrintProgress() + }() + if cmd.S == "" { + result.ResultJs[i].Url = u + return + } + if cmd.M == 3 { + for _, v := range config.Risks { + if strings.Contains(u, v) { + result.ResultJs[i] = mode.Link{Url: u, Status: "疑似危险路由"} + return + } + } + } + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + //配置代理 + if cmd.X != "" { + proxyUrl, parseErr := url.Parse(config.Conf.Proxy) + if parseErr != nil { + fmt.Println("代理地址错误: \n" + parseErr.Error()) + os.Exit(1) + } + tr.Proxy = http.ProxyURL(proxyUrl) + } + //加载yaml配置(proxy) + if cmd.I { + util.SetProxyConfig(tr) + } + client := &http.Client{Timeout: 15 * time.Second, Transport: tr} + request, err := http.NewRequest("GET", u, nil) + if err != nil { + result.ResultJs[i].Url = "" + return + } + + //增加header选项 + request.Header.Add("Cookie", cmd.C) + request.Header.Add("User-Agent", util.GetUserAgent()) + request.Header.Add("Accept", "*/*") + //加载yaml配置 + if cmd.I { + util.SetHeadersConfig(&request.Header) + } + + //处理返回结果 + response, err := client.Do(request) + if err != nil { + if strings.Contains(err.Error(), "Client.Timeout") && cmd.S == "" { + result.ResultJs[i] = mode.Link{Url: u, Status: "timeout", Size: "0"} + + } else { + result.ResultJs[i].Url = "" + } + return + } else { + defer response.Body.Close() + } + + code := response.StatusCode + if strings.Contains(cmd.S, strconv.Itoa(code)) || cmd.S == "all" && (sou != "Fuzz" && code == 200) { + var length int + dataBytes, err := io.ReadAll(response.Body) + if err != nil { + length = 0 + } else { + length = len(dataBytes) + } + result.ResultJs[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length)} + } else { + result.ResultJs[i].Url = "" + } +} + +// 检测url访问状态码 +func UrlState(u string, i int) { + + defer func() { + config.Wg.Done() + <-config.Urlch + PrintProgress() + }() + if cmd.S == "" { + result.ResultUrl[i].Url = u + return + } + if cmd.M == 3 { + for _, v := range config.Risks { + if strings.Contains(u, v) { + result.ResultUrl[i] = mode.Link{Url: u, Status: "0", Size: "0", Title: "疑似危险路由，已跳过验证"} + return + } + } + } + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + //配置代理 + if cmd.X != "" { + proxyUrl, parseErr := url.Parse(config.Conf.Proxy) + if parseErr != nil { + fmt.Println("代理地址错误: \n" + parseErr.Error()) + os.Exit(1) + } + tr.Proxy = http.ProxyURL(proxyUrl) + } + //加载yaml配置(proxy) + if cmd.I { + util.SetProxyConfig(tr) + } + client := &http.Client{Timeout: 15 * time.Second, Transport: tr} + request, err := http.NewRequest("GET", u, nil) + if err != nil { + result.ResultUrl[i].Url = "" + return + } + //增加header选项 + request.Header.Add("Cookie", cmd.C) + request.Header.Add("User-Agent", util.GetUserAgent()) + request.Header.Add("Accept", "*/*") + //加载yaml配置 + if cmd.I { + util.SetHeadersConfig(&request.Header) + } + //处理返回结果 + response, err := client.Do(request) + + if err != nil { + if strings.Contains(err.Error(), "Client.Timeout") && cmd.S == "all" { + result.ResultUrl[i] = mode.Link{Url: u, Status: "timeout", Size: "0"} + } else { + result.ResultUrl[i].Url = "" + } + return + } else { + defer response.Body.Close() + } + + code := response.StatusCode + if strings.Contains(cmd.S, strconv.Itoa(code)) || cmd.S == "all" { + var length int + dataBytes, err := io.ReadAll(response.Body) + if err != nil { + length = 0 + } else { + length = len(dataBytes) + } + body := string(dataBytes) + re := regexp.MustCompile("<[tT]itle>(.*?)") + title := re.FindAllStringSubmatch(body, -1) + if len(title) != 0 { + result.ResultUrl[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Title: title[0][1]} + } else { + result.ResultUrl[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length)} + } + } else { + result.ResultUrl[i].Url = "" + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..440e39a --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module github.com/pingc0y/URLFinder + +go 1.19 + +require ( + github.com/gookit/color v1.5.2 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect + golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 // indirect +) diff --git a/main.go b/main.go new file mode 100644 index 0000000..88fd258 --- /dev/null +++ b/main.go @@ -0,0 +1,11 @@ +package main + +import ( + "github.com/pingc0y/URLFinder/cmd" + "github.com/pingc0y/URLFinder/crawler" +) + +func main() { + cmd.Parse() + crawler.Run() +} diff --git a/mode/mode.go b/mode/mode.go new file mode 100644 index 0000000..e795d1e --- /dev/null +++ b/mode/mode.go @@ -0,0 +1,31 @@ +package mode + +type Config struct { + Headers map[string]string `yaml:"headers"` + Proxy string `yaml:"proxy"` + + JsFind []string `yaml:"jsFind"` + UrlFind []string `yaml:"urlFind"` + InfoFind map[string][]string `yaml:"infoFiler"` + + JsFiler []string `yaml:"jsFiler"` + UrlFiler []string `yaml:"urlFiler"` + + JsFuzzPath []string `yaml:"jsFuzzPath"` +} + +type Link struct { + Url string + Status string + Size string + Title string + Source string +} + +type Info struct { + Phone []string + Email []string + IDcard []string + JWT []string + Source string +} diff --git a/result/report.html b/result/report.html new file mode 100644 index 0000000..56e4d17 --- /dev/null +++ b/result/report.html @@ -0,0 +1,956 @@ + + + + + + + URLFinder + + + + +
+
+
+
+

+ URLFinder Report + +

+
+
+
+ +
+
+
+
+

JS to {urlHost}

+
+
+
+
+
+
+
+
+
+ +
+ + + + + + + + + + + + + {JS} + + +
+ +
+ URL +
+ + 		+ +
+ Status +
+ + 		+ +
+ Size +
+ + 		+ +
+ Title +
+ + 		+ +
+ Source +
+ +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+

JS to Other

+
+
+
+
+
+
+
+
+
+ +
+ + + + + + + + + + + + + {JSOther} + + +
+ +
+ URL +
+ + 		+ +
+ Status +
+ + 		+ +
+ Size +
+ + 		+ +
+ Title +
+ + 		+ +
+ Source +
+ +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+

URL to {urlHost}

+
+
+
+
+
+
+
+
+
+ +
+ + + + + + + + + + + + + {URL} + + +
+ +
+ URL +
+ + 		+ +
+ Status +
+ + 		+ +
+ Size +
+ + 		+ +
+ Title +
+ + 		+ +
+ Source +
+ +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+

URL to Other

+
+
+
+
+
+
+
+
+
+ +
+ + + + + + + + + + + + + {URLOther} + + +
+ +
+ URL +
+ + 		+ +
+ Status +
+ + 		+ +
+ Size +
+ + 		+ +
+ Title +
+ + 		+ +
+ Source +
+ +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+

Fuzz

+
+
+
+
+
+
+
+
+
+ +
+ + + + + + + + + + + + {Fuzz} + + +
+ +
+ URL +
+ + 		+ +
+ Status +
+ + 		+ +
+ Size +
+ + 		+ +
+ Title +
+ + 		+ +
+ Source +
+ +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+

Domains

+
+
+
+
+
+
+
+
+
+ +
+ + + + + + + + {Domains} + +
+ +
+ Domain +
+ +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+

Info

+
+
+
+
+
+
+
+
+
+ +
+ + + + + + + + + + + {Info} + +
+ +
+ Type +
+ + 		+ +
+ Value +
+ + 		+ +
+ Source +
+ +
+
+
+
+
+
+
+
+
+
+ + +
+ +
+ +
+
+ + + + \ No newline at end of file diff --git a/src/result.go b/result/result.go similarity index 53% rename from src/result.go rename to result/result.go index 8e68a40..2bee2d6 100644 --- a/src/result.go +++ b/result/result.go @@ -1,12 +1,14 @@ -package main +package result import ( "bufio" _ "embed" - "encoding/json" "fmt" "github.com/gookit/color" + "github.com/pingc0y/URLFinder/cmd" + "github.com/pingc0y/URLFinder/mode" + "github.com/pingc0y/URLFinder/util" "log" "os" "regexp" @@ -17,7 +19,20 @@ import ( //go:embed report.html var html string -func outHtmlString(link Link) string { +var ( + ResultJs []mode.Link + ResultUrl []mode.Link + Fuzzs []mode.Link + Infos []mode.Info + + EndUrl []string + Jsinurl map[string]string + Jstourl map[string]string + Urltourl map[string]string + Domains []string +) + +func outHtmlString(link mode.Link) string { ht := ` @@ -33,7 +48,7 @@ func outHtmlString(link Link) string { ` + link.Title + ` - + ` + link.Source + ` ` @@ -56,40 +71,49 @@ func outHtmlInfoString(ty, val, sou string) string { return ht } +func outHtmlDomainString(domain string) string { + ht := ` + + ` + domain + ` + + ` + return ht +} + // 导出csv -func outFileCsv() { - addSource() +func OutFileCsv() { //获取域名 var host string re := regexp.MustCompile("([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") - hosts := re.FindAllString(u, 1) + hosts := re.FindAllString(cmd.U, 1) if len(hosts) == 0 { - host = u + host = cmd.U } else { host = hosts[0] } //抓取的域名优先排序 - if s != "" { - resultUrl = SelectSort(resultUrl) - resultJs = SelectSort(resultJs) + if cmd.S != "" { + ResultUrl = util.SelectSort(ResultUrl) + ResultJs = util.SelectSort(ResultJs) } - resultJsHost, resultJsOther := urlDispose(resultJs, host, getHost(u)) - resultUrlHost, resultUrlOther := urlDispose(resultUrl, host, getHost(u)) + ResultJsHost, ResultJsOther := util.UrlDispose(ResultJs, host, util.GetHost(cmd.U)) + ResultUrlHost, ResultUrlOther := util.UrlDispose(ResultUrl, host, util.GetHost(cmd.U)) + Domains = util.GetDomains(util.MergeArray(ResultJs, ResultUrl)) //输出到文件 if strings.Contains(host, ":") { host = strings.Replace(host, ":", "：", -1) } //在当前文件夹创建文件夹 - err := os.MkdirAll(o+"/"+host, 0644) + err := os.MkdirAll(cmd.O+"/"+host, 0644) if err != nil { - fmt.Printf(o+"/"+host+" 目录创建失败 ：%s", err) + fmt.Printf(cmd.O+"/"+host+" 目录创建失败 ：%s", err) return } //多相同url处理 - fileName := o + "/" + host + "/" + host + ".csv" - for fileNum := 1; exists(fileName); fileNum++ { - fileName = o + "/" + host + "/" + host + "(" + strconv.Itoa(fileNum) + ").csv" + fileName := cmd.O + "/" + host + "/" + host + ".csv" + for fileNum := 1; util.Exists(fileName); fileNum++ { + fileName = cmd.O + "/" + host + "/" + host + "(" + strconv.Itoa(fileNum) + ").csv" } file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, os.ModePerm) @@ -102,121 +126,127 @@ func outFileCsv() { defer file.Close() writer := bufio.NewWriter(file) - if s == "" { + if cmd.S == "" { writer.WriteString("url,Source\n") } else { writer.WriteString("url,Status,Size,Title,Source\n") } - if d == "" { - writer.WriteString(strconv.Itoa(len(resultJsHost)) + " JS to " + getHost(u) + "\n") + if cmd.D == "" { + writer.WriteString(strconv.Itoa(len(ResultJsHost)) + " JS to " + util.GetHost(cmd.U) + "\n") } else { - writer.WriteString(strconv.Itoa(len(resultJsHost)+len(resultJsOther)) + " JS to " + d + "\n") + writer.WriteString(strconv.Itoa(len(ResultJsHost)+len(ResultJsOther)) + " JS to " + cmd.D + "\n") } - for _, j := range resultJsHost { - if s != "" { + for _, j := range ResultJsHost { + if cmd.S != "" { writer.WriteString(fmt.Sprintf("\"%s\",\"%s\",\"%s\",,\"%s\"\n", j.Url, j.Status, j.Size, j.Source)) } else { writer.WriteString(fmt.Sprintf("\"%s\",\"%s\"\n", j.Url, j.Source)) } } - if d == "" { - writer.WriteString("\n" + strconv.Itoa(len(resultJsOther)) + " JS to Other\n") + if cmd.D == "" { + writer.WriteString("\n" + strconv.Itoa(len(ResultJsOther)) + " JS to Other\n") } - for _, j := range resultJsOther { - if s != "" { + for _, j := range ResultJsOther { + if cmd.S != "" { writer.WriteString(fmt.Sprintf("\"%s\",\"%s\",\"%s\",,\"%s\"\n", j.Url, j.Status, j.Size, j.Source)) } else { writer.WriteString(fmt.Sprintf("\"%s\",\"%s\"\n", j.Url, j.Source)) } } writer.WriteString("\n\n") - if d == "" { - writer.WriteString(strconv.Itoa(len(resultUrlHost)) + " URL to " + getHost(u) + "\n") + if cmd.D == "" { + writer.WriteString(strconv.Itoa(len(ResultUrlHost)) + " URL to " + util.GetHost(cmd.U) + "\n") } else { - writer.WriteString(strconv.Itoa(len(resultUrlHost)+len(resultUrlOther)) + " URL to " + d + "\n") + writer.WriteString(strconv.Itoa(len(ResultUrlHost)+len(ResultUrlOther)) + " URL to " + cmd.D + "\n") } - for _, u := range resultUrlHost { - if s != "" { + for _, u := range ResultUrlHost { + if cmd.S != "" { writer.WriteString(fmt.Sprintf("\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"\n", u.Url, u.Status, u.Size, u.Title, u.Source)) } else { writer.WriteString(fmt.Sprintf("\"%s\",\"%s\",\n", u.Url, u.Source)) } } - if d == "" { - writer.WriteString("\n" + strconv.Itoa(len(resultUrlOther)) + " URL to Other\n") + if cmd.D == "" { + writer.WriteString("\n" + strconv.Itoa(len(ResultUrlOther)) + " URL to Other\n") } - for _, u := range resultUrlOther { - if s != "" { + for _, u := range ResultUrlOther { + if cmd.S != "" { writer.WriteString(fmt.Sprintf("\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"\n", u.Url, u.Status, u.Size, u.Title, u.Source)) } else { writer.WriteString(fmt.Sprintf("\"%s\",\"%s\"\n", u.Url, u.Source)) } } - if s != "" && z != 0 { - writer.WriteString("\n" + strconv.Itoa(len(fuzzs)) + " URL to Fuzz\n") - fuzzs = SelectSort(fuzzs) - for _, u := range fuzzs { + if cmd.S != "" && cmd.Z != 0 { + writer.WriteString("\n" + strconv.Itoa(len(Fuzzs)) + " URL to Fuzz\n") + Fuzzs = util.SelectSort(Fuzzs) + for _, u := range Fuzzs { writer.WriteString(fmt.Sprintf("\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"\n", u.Url, u.Status, u.Size, u.Title, "Fuzz")) } } + writer.WriteString("\n" + strconv.Itoa(len(Domains)) + " Domain\n") + for _, u := range Domains { + writer.WriteString(fmt.Sprintf("\"%s\"\n", u)) + } + writer.WriteString("\n Phone \n") - for i := range infos { - for i2 := range infos[i].Phone { - writer.WriteString(fmt.Sprintf("\"%s\",\"%s\"\n", infos[i].Phone[i2], infos[i].Source)) + for i := range Infos { + for i2 := range Infos[i].Phone { + writer.WriteString(fmt.Sprintf("\"%s\",\"%s\"\n", Infos[i].Phone[i2], Infos[i].Source)) } } writer.WriteString("\n Email \n") - for i := range infos { - for i2 := range infos[i].Email { - writer.WriteString(fmt.Sprintf("\"%s\",\"%s\"\n", infos[i].Email[i2], infos[i].Source)) + for i := range Infos { + for i2 := range Infos[i].Email { + writer.WriteString(fmt.Sprintf("\"%s\",\"%s\"\n", Infos[i].Email[i2], Infos[i].Source)) } } writer.WriteString("\n IDcard \n") - for i := range infos { - for i2 := range infos[i].IDcard { - writer.WriteString(fmt.Sprintf("\"%s\",\"%s\"\n", infos[i].IDcard[i2], infos[i].Source)) + for i := range Infos { + for i2 := range Infos[i].IDcard { + writer.WriteString(fmt.Sprintf("\"%s\",\"%s\"\n", Infos[i].IDcard[i2], Infos[i].Source)) } } writer.WriteString("\n JWT \n") - for i := range infos { - for i2 := range infos[i].JWT { - writer.WriteString(fmt.Sprintf("\"%s\",\"%s\"\n", infos[i].JWT[i2], infos[i].Source)) + for i := range Infos { + for i2 := range Infos[i].JWT { + writer.WriteString(fmt.Sprintf("\"%s\",\"%s\"\n", Infos[i].JWT[i2], Infos[i].Source)) } } writer.Flush() //内容是先写到缓存对，所以需要调用flush将缓存对数据真正写到文件中 - fmt.Println(strconv.Itoa(len(resultJsHost)+len(resultJsOther))+"JS + "+strconv.Itoa(len(resultUrlHost)+len(resultUrlOther))+"URL --> ", file.Name()) + fmt.Println(strconv.Itoa(len(ResultJsHost)+len(ResultJsOther))+"JS + "+strconv.Itoa(len(ResultUrlHost)+len(ResultUrlOther))+"URL --> ", file.Name()) return } // 导出json -func outFileJson() { - addSource() +func OutFileJson() { jsons := make(map[string]interface{}) var info map[string][]map[string]string //获取域名 var host string re := regexp.MustCompile("([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") - hosts := re.FindAllString(u, 1) + hosts := re.FindAllString(cmd.U, 1) if len(hosts) == 0 { - host = u + host = cmd.U } else { host = hosts[0] } //抓取的域名优先排序 - if s != "" { - resultUrl = SelectSort(resultUrl) - resultJs = SelectSort(resultJs) + if cmd.S != "" { + ResultUrl = util.SelectSort(ResultUrl) + ResultJs = util.SelectSort(ResultJs) } - resultJsHost, resultJsOther := urlDispose(resultJs, host, getHost(u)) - resultUrlHost, resultUrlOther := urlDispose(resultUrl, host, getHost(u)) - if len(infos) > 0 { + ResultJsHost, ResultJsOther := util.UrlDispose(ResultJs, host, util.GetHost(cmd.U)) + ResultUrlHost, ResultUrlOther := util.UrlDispose(ResultUrl, host, util.GetHost(cmd.U)) + Domains = util.GetDomains(util.MergeArray(ResultJs, ResultUrl)) + + if len(Infos) > 0 { info = make(map[string][]map[string]string) info["IDcard"] = nil info["JWT"] = nil @@ -224,18 +254,18 @@ func outFileJson() { info["Phone"] = nil } - for i := range infos { - for i2 := range infos[i].IDcard { - info["IDcard"] = append(info["IDcard"], map[string]string{"IDcard": infos[i].JWT[i2], "Source": infos[i].Source}) + for i := range Infos { + for i2 := range Infos[i].IDcard { + info["IDcard"] = append(info["IDcard"], map[string]string{"IDcard": Infos[i].JWT[i2], "Source": Infos[i].Source}) } - for i2 := range infos[i].JWT { - info["JWT"] = append(info["JWT"], map[string]string{"JWT": infos[i].JWT[i2], "Source": infos[i].Source}) + for i2 := range Infos[i].JWT { + info["JWT"] = append(info["JWT"], map[string]string{"JWT": Infos[i].JWT[i2], "Source": Infos[i].Source}) } - for i2 := range infos[i].Email { - info["Email"] = append(info["Email"], map[string]string{"Email": infos[i].Email[i2], "Source": infos[i].Source}) + for i2 := range Infos[i].Email { + info["Email"] = append(info["Email"], map[string]string{"Email": Infos[i].Email[i2], "Source": Infos[i].Source}) } - for i2 := range infos[i].Phone { - info["Phone"] = append(info["Phone"], map[string]string{"Phone": infos[i].Phone[i2], "Source": infos[i].Source}) + for i2 := range Infos[i].Phone { + info["Phone"] = append(info["Phone"], map[string]string{"Phone": Infos[i].Phone[i2], "Source": Infos[i].Source}) } } @@ -244,15 +274,15 @@ func outFileJson() { host = strings.Replace(host, ":", "：", -1) } //在当前文件夹创建文件夹 - err := os.MkdirAll(o+"/"+host, 0644) + err := os.MkdirAll(cmd.O+"/"+host, 0644) if err != nil { - fmt.Printf(o+"/"+host+" 目录创建失败 ：%s", err) + fmt.Printf(cmd.O+"/"+host+" 目录创建失败 ：%s", err) return } //多相同url处理 - fileName := o + "/" + host + "/" + host + ".json" - for fileNum := 1; exists(fileName); fileNum++ { - fileName = o + "/" + host + "/" + host + "(" + strconv.Itoa(fileNum) + ").json" + fileName := cmd.O + "/" + host + "/" + host + ".json" + for fileNum := 1; util.Exists(fileName); fileNum++ { + fileName = cmd.O + "/" + host + "/" + host + "(" + strconv.Itoa(fileNum) + ").json" } file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, 0644) @@ -260,17 +290,18 @@ func outFileJson() { log.Printf("创建失败：%s", err) return } - if d == "" { - jsons["jsOther"] = resultJsOther - jsons["urlOther"] = resultUrlOther + if cmd.D == "" { + jsons["jsOther"] = ResultJsOther + jsons["urlOther"] = ResultUrlOther } - jsons["js"] = resultJsHost - jsons["url"] = resultUrlHost + jsons["js"] = ResultJsHost + jsons["url"] = ResultUrlHost jsons["info"] = info - jsons["fuzz"] = fuzzs - if s != "" && z != 0 { - fuzzs = SelectSort(fuzzs) - jsons["fuzz"] = fuzzs + jsons["fuzz"] = Fuzzs + jsons["domain"] = Domains + if cmd.S != "" && cmd.Z != 0 { + Fuzzs = util.SelectSort(Fuzzs) + jsons["fuzz"] = Fuzzs } defer file.Close() @@ -288,44 +319,44 @@ func outFileJson() { if err != nil { log.Println("json保存失败:", err) } - fmt.Println(strconv.Itoa(len(resultJsHost)+len(resultJsOther))+"JS + "+strconv.Itoa(len(resultUrlHost)+len(resultUrlOther))+"URL --> ", file.Name()) + fmt.Println(strconv.Itoa(len(ResultJsHost)+len(ResultJsOther))+"JS + "+strconv.Itoa(len(ResultUrlHost)+len(ResultUrlOther))+"URL --> ", file.Name()) return } // 导出html -func outFileHtml() { - addSource() +func OutFileHtml() { //获取域名 var host string re := regexp.MustCompile("([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") - hosts := re.FindAllString(u, 1) + hosts := re.FindAllString(cmd.U, 1) if len(hosts) == 0 { - host = u + host = cmd.U } else { host = hosts[0] } //抓取的域名优先排序 - if s != "" { - resultUrl = SelectSort(resultUrl) - resultJs = SelectSort(resultJs) + if cmd.S != "" { + ResultUrl = util.SelectSort(ResultUrl) + ResultJs = util.SelectSort(ResultJs) } - resultJsHost, resultJsOther := urlDispose(resultJs, host, getHost(u)) - resultUrlHost, resultUrlOther := urlDispose(resultUrl, host, getHost(u)) + ResultJsHost, ResultJsOther := util.UrlDispose(ResultJs, host, util.GetHost(cmd.U)) + ResultUrlHost, ResultUrlOther := util.UrlDispose(ResultUrl, host, util.GetHost(cmd.U)) + Domains = util.GetDomains(util.MergeArray(ResultJs, ResultUrl)) //输出到文件 if strings.Contains(host, ":") { host = strings.Replace(host, ":", "：", -1) } //在当前文件夹创建文件夹 - err := os.MkdirAll(o+"/"+host, 0644) + err := os.MkdirAll(cmd.O+"/"+host, 0644) if err != nil { - fmt.Printf(o+"/"+host+" 目录创建失败 ：%s", err) + fmt.Printf(cmd.O+"/"+host+" 目录创建失败 ：%s", err) return } //多相同url处理 - fileName := o + "/" + host + "/" + host + ".html" - for fileNum := 1; exists(fileName); fileNum++ { - fileName = o + "/" + host + "/" + host + "(" + strconv.Itoa(fileNum) + ").html" + fileName := cmd.O + "/" + host + "/" + host + ".html" + for fileNum := 1; util.Exists(fileName); fileNum++ { + fileName = cmd.O + "/" + host + "/" + host + "(" + strconv.Itoa(fileNum) + ").html" } file, err := os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY, os.ModePerm) @@ -339,116 +370,122 @@ func outFileHtml() { writer := bufio.NewWriter(file) - if d == "" { - html = strings.Replace(html, "{urlHost}", getHost(u), -1) + if cmd.D == "" { + html = strings.Replace(html, "{urlHost}", util.GetHost(cmd.U), -1) } else { - html = strings.Replace(html, "{urlHost}", d, -1) + html = strings.Replace(html, "{urlHost}", cmd.D, -1) } - var resultJsHostStr string - for _, j := range resultJsHost { - resultJsHostStr += outHtmlString(j) + var ResultJsHostStr string + for _, j := range ResultJsHost { + ResultJsHostStr += outHtmlString(j) } - html = strings.Replace(html, "{JS}", resultJsHostStr, -1) + html = strings.Replace(html, "{JS}", ResultJsHostStr, -1) - var resultJsOtherStr string - for _, j := range resultJsOther { - resultJsOtherStr += outHtmlString(j) + var ResultJsOtherStr string + for _, j := range ResultJsOther { + ResultJsOtherStr += outHtmlString(j) } - html = strings.Replace(html, "{JSOther}", resultJsOtherStr, -1) + html = strings.Replace(html, "{JSOther}", ResultJsOtherStr, -1) - var resultUrlHostStr string - for _, u := range resultUrlHost { - resultUrlHostStr += outHtmlString(u) + var ResultUrlHostStr string + for _, u := range ResultUrlHost { + ResultUrlHostStr += outHtmlString(u) } - html = strings.Replace(html, "{URL}", resultUrlHostStr, -1) + html = strings.Replace(html, "{URL}", ResultUrlHostStr, -1) - var resultUrlOtherStr string - for _, u := range resultUrlOther { - resultUrlOtherStr += outHtmlString(u) + var ResultUrlOtherStr string + for _, u := range ResultUrlOther { + ResultUrlOtherStr += outHtmlString(u) } - html = strings.Replace(html, "{URLOther}", resultUrlOtherStr, -1) + html = strings.Replace(html, "{URLOther}", ResultUrlOtherStr, -1) - var fuzzsStr string - if s != "" && z != 0 { - fuzzs = SelectSort(fuzzs) - for _, u := range fuzzs { - fuzzsStr += outHtmlString(u) + var FuzzsStr string + if cmd.S != "" && cmd.Z != 0 { + Fuzzs = util.SelectSort(Fuzzs) + for _, u := range Fuzzs { + FuzzsStr += outHtmlString(u) } } - html = strings.Replace(html, "{Fuzz}", fuzzsStr, -1) + html = strings.Replace(html, "{Fuzz}", FuzzsStr, -1) - var infoStr string - for i := range infos { - for i2 := range infos[i].Phone { - infoStr += outHtmlInfoString("Phone", infos[i].Phone[i2], infos[i].Source) + var DomainsStr string + for _, u := range Domains { + DomainsStr += outHtmlDomainString(u) + } + html = strings.Replace(html, "{Domains}", DomainsStr, -1) + + var Infostr string + for i := range Infos { + for i2 := range Infos[i].Phone { + Infostr += outHtmlInfoString("Phone", Infos[i].Phone[i2], Infos[i].Source) } } - for i := range infos { - for i2 := range infos[i].Email { - infoStr += outHtmlInfoString("Email", infos[i].Email[i2], infos[i].Source) + for i := range Infos { + for i2 := range Infos[i].Email { + Infostr += outHtmlInfoString("Email", Infos[i].Email[i2], Infos[i].Source) } } - for i := range infos { - for i2 := range infos[i].IDcard { - infoStr += outHtmlInfoString("IDcard", infos[i].IDcard[i2], infos[i].Source) + for i := range Infos { + for i2 := range Infos[i].IDcard { + Infostr += outHtmlInfoString("IDcard", Infos[i].IDcard[i2], Infos[i].Source) } } - for i := range infos { - for i2 := range infos[i].JWT { - infoStr += outHtmlInfoString("JWT", infos[i].JWT[i2], infos[i].Source) + for i := range Infos { + for i2 := range Infos[i].JWT { + Infostr += outHtmlInfoString("JWT", Infos[i].JWT[i2], Infos[i].Source) } } - html = strings.Replace(html, "{Info}", infoStr, -1) + html = strings.Replace(html, "{Info}", Infostr, -1) writer.WriteString(html) writer.Flush() //内容是先写到缓存对，所以需要调用flush将缓存对数据真正写到文件中 - fmt.Println(strconv.Itoa(len(resultJsHost)+len(resultJsOther))+"JS + "+strconv.Itoa(len(resultUrlHost)+len(resultUrlOther))+"URL --> ", file.Name()) + fmt.Println(strconv.Itoa(len(ResultJsHost)+len(ResultJsOther))+"JS + "+strconv.Itoa(len(ResultUrlHost)+len(ResultUrlOther))+"URL --> ", file.Name()) return } // 打印 -func print() { +func Print() { //获取域名 var host string re := regexp.MustCompile("([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") - hosts := re.FindAllString(u, 1) + hosts := re.FindAllString(cmd.U, 1) if len(hosts) == 0 { - host = u + host = cmd.U } else { host = hosts[0] } //打印JS - if s != "" { - resultJs = SelectSort(resultJs) - resultUrl = SelectSort(resultUrl) + if cmd.S != "" { + ResultJs = util.SelectSort(ResultJs) + ResultUrl = util.SelectSort(ResultUrl) } //抓取的域名优先排序 - resultJsHost, resultJsOther := urlDispose(resultJs, host, getHost(u)) - resultUrlHost, resultUrlOther := urlDispose(resultUrl, host, getHost(u)) - + ResultJsHost, ResultJsOther := util.UrlDispose(ResultJs, host, util.GetHost(cmd.U)) + ResultUrlHost, ResultUrlOther := util.UrlDispose(ResultUrl, host, util.GetHost(cmd.U)) + Domains = util.GetDomains(util.MergeArray(ResultJs, ResultUrl)) var ulen string - if len(resultUrl) != 0 { + if len(ResultUrl) != 0 { uleni := 0 - for _, u := range resultUrl { + for _, u := range ResultUrl { uleni += len(u.Url) } - ulen = strconv.Itoa(uleni/len(resultUrl) + 10) + ulen = strconv.Itoa(uleni/len(ResultUrl) + 10) } var jlen string - if len(resultJs) != 0 { + if len(ResultJs) != 0 { jleni := 0 - for _, j := range resultJs { + for _, j := range ResultJs { jleni += len(j.Url) } - jlen = strconv.Itoa(jleni/len(resultJs) + 10) + jlen = strconv.Itoa(jleni/len(ResultJs) + 10) } - if d == "" { - fmt.Println(strconv.Itoa(len(resultJsHost)) + " JS to " + getHost(u)) + if cmd.D == "" { + fmt.Println(strconv.Itoa(len(ResultJsHost)) + " JS to " + util.GetHost(cmd.U)) } else { - fmt.Println(strconv.Itoa(len(resultJsHost)+len(resultJsOther)) + " JS to " + d) + fmt.Println(strconv.Itoa(len(ResultJsHost)+len(ResultJsOther)) + " JS to " + cmd.D) } - for _, j := range resultJsHost { - if s != "" { + for _, j := range ResultJsHost { + if cmd.S != "" { if strings.HasPrefix(j.Status, "2") { fmt.Printf(color.LightBlue.Sprintf("%-"+jlen+"s", j.Url) + color.LightGreen.Sprintf(" [ Status: %s, Size: %s ]\n", j.Status, j.Size)) } else if strings.HasPrefix(j.Status, "3") { @@ -456,15 +493,15 @@ func print() { } else { fmt.Printf(color.LightBlue.Sprintf("%-"+jlen+"s", j.Url) + color.LightRed.Sprintf(" [ Status: %s, Size: %s ]\n", j.Status, j.Size)) } - } else if s == "" { + } else if cmd.S == "" { fmt.Printf(color.LightBlue.Sprintf(j.Url) + "\n") } } - if d == "" { - fmt.Println("\n" + strconv.Itoa(len(resultJsOther)) + " JS to Other") + if cmd.D == "" { + fmt.Println("\n" + strconv.Itoa(len(ResultJsOther)) + " JS to Other") } - for _, j := range resultJsOther { - if s != "" { + for _, j := range ResultJsOther { + if cmd.S != "" { if strings.HasPrefix(j.Status, "2") { fmt.Printf(color.LightBlue.Sprintf("%-"+jlen+"s", j.Url) + color.LightGreen.Sprintf(" [ Status: %s, Size: %s ]\n", j.Status, j.Size)) } else if strings.HasPrefix(j.Status, "3") { @@ -477,17 +514,16 @@ func print() { } } - //打印URL - fmt.Println("\n\n") + fmt.Println("\n ") - if d == "" { - fmt.Println(strconv.Itoa(len(resultUrlHost)) + " URL to " + getHost(u)) + if cmd.D == "" { + fmt.Println(strconv.Itoa(len(ResultUrlHost)) + " URL to " + util.GetHost(cmd.U)) } else { - fmt.Println(strconv.Itoa(len(resultUrlHost)+len(resultUrlOther)) + " URL to " + d) + fmt.Println(strconv.Itoa(len(ResultUrlHost)+len(ResultUrlOther)) + " URL to " + cmd.D) } - for _, u := range resultUrlHost { - if s != "" && len(u.Title) != 0 { + for _, u := range ResultUrlHost { + if cmd.S != "" && len(u.Title) != 0 { if u.Status == "疑似危险路由" { fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightGreen.Sprintf(" [ %s ]\n", u.Status)) } else if strings.HasPrefix(u.Status, "2") { @@ -497,7 +533,7 @@ func print() { } else { fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightRed.Sprintf(" [ Status: %s, Size: %s, Title: %s ]\n", u.Status, u.Size, u.Title)) } - } else if s != "" { + } else if cmd.S != "" { if strings.HasPrefix(u.Status, "2") { fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightGreen.Sprintf(" [ Status: %s, Size: %s ]\n", u.Status, u.Size)) } else if strings.HasPrefix(u.Status, "3") { @@ -509,11 +545,11 @@ func print() { fmt.Printf(color.LightBlue.Sprintf(u.Url) + "\n") } } - if d == "" { - fmt.Println("\n" + strconv.Itoa(len(resultUrlOther)) + " URL to Other") + if cmd.D == "" { + fmt.Println("\n" + strconv.Itoa(len(ResultUrlOther)) + " URL to Other") } - for _, u := range resultUrlOther { - if s != "" && len(u.Title) != 0 { + for _, u := range ResultUrlOther { + if cmd.S != "" && len(u.Title) != 0 { if u.Status == "疑似危险路由" { fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightGreen.Sprintf(" [ %s ]\n", u.Status)) } else if strings.HasPrefix(u.Status, "2") { @@ -523,7 +559,7 @@ func print() { } else { fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightRed.Sprintf(" [ Status: %s, Size: %s, Title: %s ]\n", u.Status, u.Size, u.Title)) } - } else if s != "" { + } else if cmd.S != "" { if strings.HasPrefix(u.Status, "2") { fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightGreen.Sprintf(" [ Status: %s, Size: %s ]\n", u.Status, u.Size)) } else if strings.HasPrefix(u.Status, "3") { @@ -536,10 +572,10 @@ func print() { } } - if s != "" && z != 0 { - fmt.Println("\n" + strconv.Itoa(len(fuzzs)) + " URL to Fuzz") - fuzzs = SelectSort(fuzzs) - for _, u := range fuzzs { + if cmd.S != "" && cmd.Z != 0 { + fmt.Println("\n" + strconv.Itoa(len(Fuzzs)) + " URL to Fuzz") + Fuzzs = util.SelectSort(Fuzzs) + for _, u := range Fuzzs { if len(u.Title) != 0 { if u.Status == "疑似危险路由" { fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightGreen.Sprintf(" [ %s ]\n", u.Status)) @@ -561,29 +597,38 @@ func print() { } } } - fmt.Println("\n Phone ") - for i := range infos { - for i2 := range infos[i].Phone { - fmt.Printf(color.LightBlue.Sprintf("%-10s", infos[i].Phone[i2]) + color.LightGreen.Sprintf(" [ %s ]\n", infos[i].Source)) - } + fmt.Println("\n" + strconv.Itoa(len(Domains)) + " Domain") + for _, u := range Domains { + fmt.Printf(color.LightBlue.Sprintf("%s \n", u)) + } - fmt.Println("\n Email ") - for i := range infos { - for i2 := range infos[i].Email { - fmt.Printf(color.LightBlue.Sprintf("%-10s", infos[i].Email[i2])) + + if len(Infos) > 0 { + fmt.Println("\n Phone ") + for i := range Infos { + for i2 := range Infos[i].Phone { + fmt.Printf(color.LightBlue.Sprintf("%-10s", Infos[i].Phone[i2]) + color.LightGreen.Sprintf(" [ %s ]\n", Infos[i].Source)) + } } - } - fmt.Println("\n IDcard ") - for i := range infos { - for i2 := range infos[i].IDcard { - fmt.Printf(color.LightBlue.Sprintf("%-10s", infos[i].IDcard[i2])) + fmt.Println("\n Email ") + for i := range Infos { + for i2 := range Infos[i].Email { + fmt.Printf(color.LightBlue.Sprintf("%-10s", Infos[i].Email[i2]) + color.LightGreen.Sprintf(" [ %s ]\n", Infos[i].Source)) + } } - } - fmt.Println("\n JWT ") - for i := range infos { - for i2 := range infos[i].JWT { - fmt.Printf(color.LightBlue.Sprintf("%-10s", infos[i].JWT[i2])) + fmt.Println("\n IDcard ") + for i := range Infos { + for i2 := range Infos[i].IDcard { + fmt.Printf(color.LightBlue.Sprintf("%-10s", Infos[i].IDcard[i2]) + color.LightGreen.Sprintf(" [ %s ]\n", Infos[i].Source)) + } } + fmt.Println("\n JWT ") + for i := range Infos { + for i2 := range Infos[i].JWT { + fmt.Printf(color.LightBlue.Sprintf("%-10s", Infos[i].JWT[i2]) + color.LightGreen.Sprintf(" [ %s ]\n", Infos[i].Source)) + } + } + } } diff --git a/src/config.yaml b/src/config.yaml deleted file mode 100644 index 06acccd..0000000 --- a/src/config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -headers: - Accept: "*/*" - Cookie: "" - User-Agent: "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" -proxy: "" - - - - - diff --git a/src/crawler.go b/src/crawler.go deleted file mode 100644 index 3728fa8..0000000 --- a/src/crawler.go +++ /dev/null @@ -1,593 +0,0 @@ -package main - -import ( - "crypto/tls" - "fmt" - "io" - "net/http" - "net/url" - "os" - "regexp" - "strconv" - "strings" - "time" -) - -type config struct { - Headers map[string]string `yaml:"headers"` - Proxy string `yaml:"proxy"` -} - -var ( - risks = []string{"remove", "delete", "insert", "update", "logout"} - fuzzs = []Link{} - fuzzNum int -) -var ua = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" - -var conf config - -// 蜘蛛抓取页面内容 -func spider(u string, num int) { - var is bool - fmt.Printf("\rSpider %d", progress) - mux.Lock() - progress++ - mux.Unlock() - //标记完成 - defer func() { - wg.Done() - if !is { - <-ch - } - - }() - u, _ = url.QueryUnescape(u) - if num > 1 && d != "" && !strings.Contains(u, d) { - return - } - if getEndUrl(u) { - return - } - if m == 3 { - for _, v := range risks { - if strings.Contains(u, v) { - return - } - } - } - appendEndUrl(u) - - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - - //配置代理 - if x != "" { - proxyUrl, parseErr := url.Parse(conf.Proxy) - if parseErr != nil { - fmt.Println("代理地址错误: \n" + parseErr.Error()) - os.Exit(1) - } - tr.Proxy = http.ProxyURL(proxyUrl) - } - //加载yaml配置(proxy) - if I { - SetProxyConfig(tr) - } - client := &http.Client{Timeout: 10 * time.Second, Transport: tr} - request, err := http.NewRequest("GET", u, nil) - if err != nil { - return - } - //增加header选项 - request.Header.Add("Cookie", c) - request.Header.Add("User-Agent", ua) - request.Header.Add("Accept", "*/*") - //加载yaml配置（headers） - if I { - SetHeadersConfig(&request.Header) - } - //处理返回结果 - response, err := client.Do(request) - if err != nil { - return - } else { - defer response.Body.Close() - - } - - //提取url用于拼接其他url或js - dataBytes, err := io.ReadAll(response.Body) - if err != nil { - return - } - path := response.Request.URL.Path - host := response.Request.URL.Host - scheme := response.Request.URL.Scheme - source := scheme + "://" + host + path - - //字节数组 转换成 字符串 - result := string(dataBytes) - //处理base标签 - re := regexp.MustCompile("base.{1,5}href.{1,5}(http.+?//[^\\s]+?)[\",',‘,“]") - base := re.FindAllStringSubmatch(result, -1) - if len(base) > 0 { - host = regexp.MustCompile("http.*?//([^/]+)").FindAllStringSubmatch(base[0][1], -1)[0][1] - scheme = regexp.MustCompile("(http.*?)://").FindAllStringSubmatch(base[0][1], -1)[0][1] - paths := regexp.MustCompile("http.*?//.*?(/.*)").FindAllStringSubmatch(base[0][1], -1) - if len(paths) > 0 { - path = paths[0][1] - } else { - path = "/" - } - } - <-ch - is = true - - //提取js - jsFind(result, host, scheme, path, source, num) - //提取url - urlFind(result, host, scheme, path, source, num) - //提取信息 - infoFind(result, source) - -} - -// 分析内容中的js -func jsFind(cont, host, scheme, path, source string, num int) { - var cata string - care := regexp.MustCompile("/.*/{1}|/") - catae := care.FindAllString(path, -1) - if len(catae) == 0 { - cata = "/" - } else { - cata = catae[0] - } - //js匹配正则 - res := []string{ - ".(https{0,1}:[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{2,250}?[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{3}[.]js)", - "[\",',‘,“]\\s{0,6}(/{0,1}[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{2,250}?[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{3}[.]js)", - "=\\s{0,6}[\",',’,”]{0,1}\\s{0,6}(/{0,1}[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{2,250}?[-a-zA-Z0-9（）@:%_\\+.~#?&//=]{3}[.]js)", - } - host = scheme + "://" + host - for _, re := range res { - reg := regexp.MustCompile(re) - jss := reg.FindAllStringSubmatch(cont, -1) - //return - jss = jsFilter(jss) - //循环提取js放到结果中 - for _, js := range jss { - if js[0] == "" { - continue - } - if strings.HasPrefix(js[0], "https:") || strings.HasPrefix(js[0], "http:") { - appendJs(js[0], source) - if num < 5 && (m == 2 || m == 3) { - wg.Add(1) - ch <- 1 - go spider(js[0], num+1) - } - } else if strings.HasPrefix(js[0], "//") { - appendJs(scheme+":"+js[0], source) - if num < 5 && (m == 2 || m == 3) { - wg.Add(1) - ch <- 1 - go spider(scheme+":"+js[0], num+1) - } - - } else if strings.HasPrefix(js[0], "/") { - appendJs(host+js[0], source) - if num < 5 && (m == 2 || m == 3) { - wg.Add(1) - ch <- 1 - go spider(host+js[0], num+1) - } - } else if strings.HasPrefix(js[0], "./") { - appendJs(host+"/"+js[0], source) - if num < 5 && (m == 2 || m == 3) { - wg.Add(1) - ch <- 1 - go spider(host+"/"+js[0], num+1) - } - } else { - appendJs(host+cata+js[0], source) - if num < 5 && (m == 2 || m == 3) { - wg.Add(1) - ch <- 1 - go spider(host+cata+js[0], num+1) - } - } - } - - } - -} - -// 分析内容中的url -func urlFind(cont, host, scheme, path, source string, num int) { - var cata string - care := regexp.MustCompile("/.*/{1}|/") - catae := care.FindAllString(path, -1) - if len(catae) == 0 { - cata = "/" - } else { - cata = catae[0] - } - host = scheme + "://" + host - - //url匹配正则 - res := []string{ - "[\",',‘,“]\\s{0,6}(https{0,1}:[-a-zA-Z0-9()@:%_\\+.~#?&//=]{2,250}?)\\s{0,6}[\",',‘,“]", - "=\\s{0,6}(https{0,1}:[-a-zA-Z0-9()@:%_\\+.~#?&//=]{2,250})", - "[\",',‘,“]\\s{0,6}([#,.]{0,2}/[-a-zA-Z0-9()@:%_\\+.~#?&//=]{2,250}?)\\s{0,6}[\",',‘,“]", - "\"([-a-zA-Z0-9()@:%_\\+.~#?&//=]+?[/]{1}[-a-zA-Z0-9()@:%_\\+.~#?&//=]+?)\"", - "href\\s{0,6}=\\s{0,6}[\",',‘,“]{0,1}\\s{0,6}([-a-zA-Z0-9()@:%_\\+.~#?&//=]{2,250})|action\\s{0,6}=\\s{0,6}[\",',‘,“]{0,1}\\s{0,6}([-a-zA-Z0-9()@:%_\\+.~#?&//=]{2,250})", - } - for _, re := range res { - reg := regexp.MustCompile(re) - urls := reg.FindAllStringSubmatch(cont, -1) - //fmt.Println(urls) - urls = urlFilter(urls) - - //循环提取url放到结果中 - for _, url := range urls { - if url[0] == "" { - continue - } - if strings.HasPrefix(url[0], "https:") || strings.HasPrefix(url[0], "http:") { - appendUrl(url[0], source) - if num < 2 && (m == 2 || m == 3) { - wg.Add(1) - ch <- 1 - go spider(url[0], num+1) - } - } else if strings.HasPrefix(url[0], "//") { - appendUrl(scheme+":"+url[0], source) - if num < 2 && (m == 2 || m == 3) { - wg.Add(1) - ch <- 1 - go spider(scheme+":"+url[0], num+1) - } - } else if strings.HasPrefix(url[0], "/") { - urlz := "" - if b != "" { - urlz = b + url[0] - } else { - urlz = host + url[0] - } - appendUrl(urlz, source) - if num < 2 && (m == 2 || m == 3) { - wg.Add(1) - ch <- 1 - go spider(urlz, num+1) - } - } else if !strings.HasSuffix(source, ".js") { - urlz := "" - if b != "" { - if strings.HasSuffix(b, "/") { - urlz = b + url[0] - } else { - urlz = b + "/" + url[0] - } - } else { - urlz = host + cata + url[0] - } - appendUrl(urlz, source) - if num < 2 && (m == 2 || m == 3) { - wg.Add(1) - ch <- 1 - go spider(urlz, num+1) - } - } else if strings.HasSuffix(source, ".js") { - appendUrl(jsinurl[host+path]+url[0], source) - if num < 2 && (m == 2 || m == 3) { - wg.Add(1) - ch <- 1 - go spider(jsinurl[host+path]+url[0], num+1) - } - } - } - } -} - -// 分析内容中的敏感信息 -func infoFind(cont, source string) { - //手机号码 - phone := "['\"](1(3([0-35-9]\\d|4[1-8])|4[14-9]\\d|5([\\d]\\d|7[1-79])|66\\d|7[2-35-8]\\d|8\\d{2}|9[89]\\d)\\d{7})['\"]" - email := "['\"]([\\w!#$%&'*+/=?^_`{|}~-]+(?:\\.[\\w!#$%&'*+/=?^_`{|}~-]+)*@(?:[\\w](?:[\\w-]*[\\w])?\\.)+[\\w](?:[\\w-]*[\\w])?)['\"]" - IDcard := "['\"]((\\d{8}(0\\d|10|11|12)([0-2]\\d|30|31)\\d{3}$)|(\\d{6}(18|19|20)\\d{2}(0[1-9]|10|11|12)([0-2]\\d|30|31)\\d{3}(\\d|X|x)))['\"]" - jwt := "['\"](ey[A-Za-z0-9_-]{10,}\\.[A-Za-z0-9._-]{10,}|ey[A-Za-z0-9_\\/+-]{10,}\\.[A-Za-z0-9._\\/+-]{10,})['\"]" - phones := regexp.MustCompile(phone).FindAllStringSubmatch(cont, -1) - emails := regexp.MustCompile(email).FindAllStringSubmatch(cont, -1) - IDcards := regexp.MustCompile(IDcard).FindAllStringSubmatch(cont, -1) - jwts := regexp.MustCompile(jwt).FindAllStringSubmatch(cont, -1) - info := Info{} - for i := range phones { - info.Phone = append(info.Phone, phones[i][1]) - } - for i := range emails { - info.Email = append(info.Email, emails[i][1]) - } - for i := range IDcards { - info.IDcard = append(info.IDcard, IDcards[i][1]) - } - for i := range jwts { - info.JWT = append(info.JWT, jwts[i][1]) - } - info.Source = source - if len(info.Phone) != 0 || len(info.IDcard) != 0 || len(info.JWT) != 0 || len(info.Email) != 0 { - appendInfo(info) - } - -} - -// 过滤JS -func jsFilter(str [][]string) [][]string { - - //对不需要的数据过滤 - for i := range str { - str[i][0], _ = url.QueryUnescape(str[i][1]) - str[i][0] = strings.Replace(str[i][0], " ", "", -1) - str[i][0] = strings.Replace(str[i][0], "\\/", "/", -1) - str[i][0] = strings.Replace(str[i][0], "%3A", ":", -1) - str[i][0] = strings.Replace(str[i][0], "%2F", "/", -1) - - match, _ := regexp.MatchString("[.]js", str[i][0]) - if !match { - str[i][0] = "" - } - //过滤指定字段 - fstr := []string{"www.w3.org", "example.com", "github.com"} - for _, v := range fstr { - if strings.Contains(str[i][0], v) { - str[i][0] = "" - } - } - - } - return str - -} - -// jsfuzz -func jsFuzz() { - jsFuzzFile := []string{ - "login.js", - "app.js", - "main.js", - "config.js", - "admin.js", - "info.js", - "open.js", - "user.js", - "input.js", - "list.js", - "upload.js"} - paths := []string{} - for i := range resultJs { - re := regexp.MustCompile("(.+/)[^/]+.js").FindAllStringSubmatch(resultJs[i].Url, -1) - if len(re) != 0 { - paths = append(paths, re[0][1]) - } - } - paths = uniqueArr(paths) - for i := range paths { - for i2 := range jsFuzzFile { - resultJs = append(resultJs, Link{ - Url: paths[i] + jsFuzzFile[i2], - Source: "Fuzz", - }) - } - } -} - -// 过滤URL -func urlFilter(str [][]string) [][]string { - - //对不需要的数据过滤 - for i := range str { - str[i][0], _ = url.QueryUnescape(str[i][1]) - str[i][0] = strings.Replace(str[i][0], " ", "", -1) - str[i][0] = strings.Replace(str[i][0], "\\/", "/", -1) - str[i][0] = strings.Replace(str[i][0], "%3A", ":", -1) - str[i][0] = strings.Replace(str[i][0], "%2F", "/", -1) - - //过滤包含指定内容 - fstr := []string{".js?", ".css?", ".jpeg?", ".jpg?", ".png?", ".gif?", "github.com", "www.w3.org", "example.com", "<", ">", "{", "}", "[", "]", "|", "^", ";", "/js/", ".src", ".replace", ".url", ".att", ".href", "location.href", "javascript:", "location:", ".createObject", ":location", ".path", "*#__PURE__*", "*$0*", "\\n"} - for _, v := range fstr { - if strings.Contains(str[i][0], v) { - str[i][0] = "" - } - } - - match, _ := regexp.MatchString("[a-zA-Z]+|[0-9]+", str[i][0]) - if !match { - str[i][0] = "" - } - //过滤指定后缀 - zstr := []string{".js", ".css", ".scss", ",", ".jpeg", ".jpg", ".png", ".gif", ".ico", ".svg", ".vue", ".ts"} - - for _, v := range zstr { - if strings.HasSuffix(str[i][0], v) { - str[i][0] = "" - } - } - //对抓到的域名做处理 - re := regexp.MustCompile("([a-z0-9\\-]+\\.)+([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?").FindAllString(str[i][0], 1) - if len(re) != 0 && !strings.HasPrefix(str[i][0], "http") && !strings.HasPrefix(str[i][0], "/") { - str[i][0] = "http://" + str[i][0] - } - - } - return str -} - -// 检测js访问状态码 -func jsState(u string, i int, sou string) { - defer func() { - wg.Done() - <-jsch - printProgress() - }() - if s == "" { - resultJs[i].Url = u - return - } - if m == 3 { - for _, v := range risks { - if strings.Contains(u, v) { - resultJs[i] = Link{Url: u, Status: "疑似危险路由"} - return - } - } - } - - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - //配置代理 - //配置代理 - if x != "" { - proxyUrl, parseErr := url.Parse(conf.Proxy) - if parseErr != nil { - fmt.Println("代理地址错误: \n" + parseErr.Error()) - os.Exit(1) - } - tr.Proxy = http.ProxyURL(proxyUrl) - } - //加载yaml配置(proxy) - if I { - SetProxyConfig(tr) - } - client := &http.Client{Timeout: 15 * time.Second, Transport: tr} - request, err := http.NewRequest("GET", u, nil) - if err != nil { - resultJs[i].Url = "" - return - } - - //增加header选项 - request.Header.Add("Cookie", c) - request.Header.Add("User-Agent", ua) - request.Header.Add("Accept", "*/*") - //加载yaml配置 - if I { - SetHeadersConfig(&request.Header) - } - - //处理返回结果 - response, err := client.Do(request) - if err != nil { - if strings.Contains(err.Error(), "Client.Timeout") && s == "" { - resultJs[i] = Link{Url: u, Status: "timeout", Size: "0"} - - } else { - resultJs[i].Url = "" - } - return - } else { - defer response.Body.Close() - } - - code := response.StatusCode - if strings.Contains(s, strconv.Itoa(code)) || s == "all" && (sou != "Fuzz" && code == 200) { - var length int - dataBytes, err := io.ReadAll(response.Body) - if err != nil { - length = 0 - } else { - length = len(dataBytes) - } - resultJs[i] = Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length)} - } else { - resultJs[i].Url = "" - } -} - -// 检测url访问状态码 -func urlState(u string, i int) { - - defer func() { - wg.Done() - <-urlch - printProgress() - }() - if s == "" { - resultUrl[i].Url = u - return - } - if m == 3 { - for _, v := range risks { - if strings.Contains(u, v) { - resultUrl[i] = Link{Url: u, Status: "0", Size: "0", Title: "疑似危险路由，已跳过验证"} - return - } - } - } - - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - //配置代理 - if x != "" { - proxyUrl, parseErr := url.Parse(conf.Proxy) - if parseErr != nil { - fmt.Println("代理地址错误: \n" + parseErr.Error()) - os.Exit(1) - } - tr.Proxy = http.ProxyURL(proxyUrl) - } - //加载yaml配置(proxy) - if I { - SetProxyConfig(tr) - } - client := &http.Client{Timeout: 15 * time.Second, Transport: tr} - request, err := http.NewRequest("GET", u, nil) - if err != nil { - resultUrl[i].Url = "" - return - } - //增加header选项 - request.Header.Add("Cookie", c) - request.Header.Add("User-Agent", ua) - request.Header.Add("Accept", "*/*") - //加载yaml配置 - if I { - SetHeadersConfig(&request.Header) - } - //处理返回结果 - response, err := client.Do(request) - - if err != nil { - if strings.Contains(err.Error(), "Client.Timeout") && s == "all" { - resultUrl[i] = Link{Url: u, Status: "timeout", Size: "0"} - } else { - resultUrl[i].Url = "" - } - return - } else { - defer response.Body.Close() - } - - code := response.StatusCode - if strings.Contains(s, strconv.Itoa(code)) || s == "all" { - var length int - dataBytes, err := io.ReadAll(response.Body) - if err != nil { - length = 0 - } else { - length = len(dataBytes) - } - body := string(dataBytes) - re := regexp.MustCompile("<[tT]itle>(.*?)") - title := re.FindAllStringSubmatch(body, -1) - if len(title) != 0 { - resultUrl[i] = Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Title: title[0][1]} - } else { - resultUrl[i] = Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length)} - } - } else { - resultUrl[i].Url = "" - } -} diff --git a/src/main.go b/src/main.go deleted file mode 100644 index 017a9b1..0000000 --- a/src/main.go +++ /dev/null @@ -1,204 +0,0 @@ -package main - -import ( - "bufio" - "flag" - "fmt" - "github.com/gookit/color" - "io" - "os" - "strings" - "sync" - "time" -) - -var ( - lock sync.Mutex - wg sync.WaitGroup - mux sync.Mutex - ch = make(chan int, t) - jsch = make(chan int, t/2) - urlch = make(chan int, t/2) -) - -type Link struct { - Url string - Status string - Size string - Title string - Source string -} - -type Info struct { - Phone []string - Email []string - IDcard []string - JWT []string - Source string -} - -var progress = 1 -var ( - resultJs []Link - resultUrl []Link - infos []Info - endUrl []string - jsinurl map[string]string - jstourl map[string]string - urltourl map[string]string -) - -var ( - h bool - I bool - m int - s string - u string - d string - c string - a string - b string - f string - o string - x string - t = 50 - z int -) - -func init() { - flag.StringVar(&a, "a", "", "set user-agent\n设置user-agent请求头") - flag.StringVar(&b, "b", "", "set baseurl\n设置baseurl路径") - flag.StringVar(&c, "c", "", "set cookie\n设置cookie") - flag.StringVar(&d, "d", "", "set domainName\n指定获取的域名") - flag.StringVar(&f, "f", "", "set urlFile\n批量抓取url,指定文件路径") - flag.BoolVar(&h, "h", false, "this help\n帮助信息") - flag.BoolVar(&I, "i", false, "set configFile\n加载yaml配置文件（不存在时，会在当前目录创建一个默认yaml配置文件）") - flag.IntVar(&m, "m", 1, "set mode\n抓取模式 \n 1 normal\n 正常抓取（默认） \n 2 thorough\n 深入抓取 （url只深入一层，防止抓偏） \n 3 security\n 安全深入抓取（过滤delete，remove等敏感路由） \n ") - flag.StringVar(&o, "o", "", "set outFile\n结果导出到csv文件，需指定导出文件目录（.代表当前目录）") - flag.StringVar(&s, "s", "", "set Status\n显示指定状态码，all为显示全部（多个状态码用,隔开）") - flag.IntVar(&t, "t", 50, "set thread\n设置线程数（默认50）\n") - flag.StringVar(&u, "u", "", "set Url\n目标URL") - flag.StringVar(&x, "x", "", "set httpProxy\n设置代理,格式: http://username:password@127.0.0.1:8809") - flag.IntVar(&z, "z", 0, "set Fuzz\n对404链接进行fuzz(只对主域名下的链接生效,需要与-s一起使用） \n 1 decreasing\n 目录递减fuzz \n 2 2combination\n 2级目录组合fuzz（适合少量链接使用） \n 3 3combination\n 3级目录组合fuzz（适合少量链接使用） \n") - - // 改变默认的 Usage - flag.Usage = usage -} -func usage() { - fmt.Fprintf(os.Stderr, `URLFinder 2023/1/29 by pingc0y -Usage: URLFinder [-a user-agent] [-b baseurl] [-c cookie] [-d domainName] [-f urlFile] [-h help] [-i configFile] [-m mode] [-o outFile] [-s Status] [-t thread] [-u Url] [-x httpProxy] [-z fuzz] - -Options: -`) - flag.PrintDefaults() -} - -func main() { - flag.Parse() - if h { - flag.Usage() - return - } - if u == "" && f == "" { - flag.Usage() - return - } - color.LightCyan.Println(" __ __ ___ _ _ \n /\\ /\\ /__\\ / / / __(_)_ __ __| | ___ _ __ \n/ / \\ \\/ \\/// / / _\\ | | '_ \\ / _` |/ _ \\ '__|\n\\ \\_/ / _ \\ /___ / | | | | | (_| | __/ | \n \\___/\\/ \\_\\____\\/ |_|_| |_|\\__,_|\\___|_| \n\nBy: pingc0y\nUpdateTime: 2023/1/29\nGithub: https://github.com/pingc0y/URLFinder \n") - if a != "" { - ua = a - } - if o != "" { - if !IsDir(o) { - return - } - } - if I { - GetConfig("config.yaml") - } - if t != 50 { - ch = make(chan int, t+1) - jsch = make(chan int, t/2+1) - urlch = make(chan int, t/2+1) - } - if f != "" { - // 创建句柄 - fi, err := os.Open(f) - if err != nil { - panic(err) - } - r := bufio.NewReader(fi) // 创建 Reader - for { - resultJs = nil - resultUrl = nil - endUrl = nil - - lineBytes, err := r.ReadBytes('\n') - //去掉字符串首尾空白字符，返回字符串 - line := strings.TrimSpace(string(lineBytes)) - u = line - start(u) - - if err == io.EOF { - break - } - fmt.Println("----------------------------------------") - - } - return - } - start(u) -} - -func start(u string) { - - jsinurl = make(map[string]string) - jstourl = make(map[string]string) - urltourl = make(map[string]string) - infos = []Info{} - fmt.Println("Start Spider URL: " + color.LightBlue.Sprintf(u)) - wg.Add(1) - ch <- 1 - go spider(u, 1) - wg.Wait() - progress = 1 - fmt.Printf("\rSpider OK \n") - resultUrl = RemoveRepeatElement(resultUrl) - resultJs = RemoveRepeatElement(resultJs) - if s != "" { - fmt.Printf("Start %d Validate...\n", len(resultUrl)+len(resultJs)) - fmt.Printf("\r ") - jsFuzz() - //验证JS状态 - for i, s := range resultJs { - wg.Add(1) - jsch <- 1 - go jsState(s.Url, i, resultJs[i].Source) - } - //验证URL状态 - for i, s := range resultUrl { - wg.Add(1) - urlch <- 1 - go urlState(s.Url, i) - } - wg.Wait() - - time.Sleep(1 * time.Second) - fmt.Printf("\r ") - fmt.Printf("\rValidate OK \n") - - if z != 0 { - fuzz() - time.Sleep(1 * time.Second) - } - } - - //打印还是输出 - if len(o) > 0 { - outFileJson() - outFileCsv() - outFileHtml() - - } else { - print() - } -} diff --git a/src/report.html b/src/report.html deleted file mode 100644 index bf1334c..0000000 --- a/src/report.html +++ /dev/null @@ -1,795 +0,0 @@ - - - - - - - URLFinder - - - - -
-
-
-
-

- URLFinder Report - -

-
-
-
- -
-
-
-
-

JS to {urlHost}

-
-
-
-
-
-
-
-
-
- -
- - - - - - - - - - - - - {JS} - - -
- -
- URL -
- - 		- -
- Status -
- - 		- -
- Size -
- - 		- -
- Title -
- - 		- -
- Source -
- -
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-

JS to Other

-
-
-
-
-
-
-
-
-
- -
- - - - - - - - - - - - - {JSOther} - - -
- -
- URL -
- - 		- -
- Status -
- - 		- -
- Size -
- - 		- -
- Title -
- - 		- -
- Source -
- -
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-

URL to {urlHost}

-
-
-
-
-
-
-
-
-
- -
- - - - - - - - - - - - - {URL} - - -
- -
- URL -
- - 		- -
- Status -
- - 		- -
- Size -
- - 		- -
- Title -
- - 		- -
- Source -
- -
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-

URL to Other

-
-
-
-
-
-
-
-
-
- -
- - - - - - - - - - - - - {URLOther} - - -
- -
- URL -
- - 		- -
- Status -
- - 		- -
- Size -
- - 		- -
- Title -
- - 		- -
- Source -
- -
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-

Fuzz

-
-
-
-
-
-
-
-
-
- -
- - - - - - - - - - - - {Fuzz} - - -
- -
- URL -
- - 		- -
- Status -
- - 		- -
- Size -
- - 		- -
- Title -
- - 		- -
- Source -
- -
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-

Info

-
-
-
-
-
-
-
-
-
- -
- - - - - - - - - - - {Info} - -
- -
- Type -
- - 		- -
- Value -
- - 		- -
- Source -
- -
-
-
-
-
-
-
-
-
-
- - -
- -
- -
-
- - - - \ No newline at end of file diff --git a/src/utils.go b/util/utils.go similarity index 55% rename from src/utils.go rename to util/utils.go index 98fa9e5..1c5945f 100644 --- a/src/utils.go +++ b/util/utils.go @@ -1,8 +1,11 @@ -package main +package util import ( "fmt" - "gopkg.in/yaml.v2" + "github.com/pingc0y/URLFinder/cmd" + "github.com/pingc0y/URLFinder/config" + "github.com/pingc0y/URLFinder/mode" + "math/rand" "net/http" "net/url" "os" @@ -21,8 +24,18 @@ func IsDir(path string) bool { return s.IsDir() } +// MergeArray 合并数组 +func MergeArray(dest []mode.Link, src []mode.Link) (result []mode.Link) { + result = make([]mode.Link, len(dest)+len(src)) + //将第一个数组传入result + copy(result, dest) + //将第二个数组接在尾部，也就是 len(dest): + copy(result[len(dest):], src) + return +} + // 对结果进行状态码排序 -func SelectSort(arr []Link) []Link { +func SelectSort(arr []mode.Link) []mode.Link { length := len(arr) var sort []int for _, v := range arr { @@ -53,10 +66,10 @@ func SelectSort(arr []Link) []Link { } // 对结果进行URL排序 -func urlDispose(arr []Link, url, host string) ([]Link, []Link) { - var urls []Link - var urlts []Link - var other []Link +func UrlDispose(arr []mode.Link, url, host string) ([]mode.Link, []mode.Link) { + var urls []mode.Link + var urlts []mode.Link + var other []mode.Link for _, v := range arr { if strings.Contains(v.Url, url) { urls = append(urls, v) @@ -76,8 +89,29 @@ func urlDispose(arr []Link, url, host string) ([]Link, []Link) { return RemoveRepeatElement(urls), RemoveRepeatElement(other) } +// 处理Headers配置 +func SetHeadersConfig(header *http.Header) *http.Header { + for k, v := range config.Conf.Headers { + header.Add(k, v) + } + return header +} + +// 设置proxy配置 +func SetProxyConfig(tr *http.Transport) *http.Transport { + if len(config.Conf.Proxy) > 0 { + proxyUrl, parseErr := url.Parse(config.Conf.Proxy) + if parseErr != nil { + fmt.Println("代理地址错误: \n" + parseErr.Error()) + os.Exit(1) + } + tr.Proxy = http.ProxyURL(proxyUrl) + } + return tr +} + // 提取顶级域名 -func getHost(u string) string { +func GetHost(u string) string { re := regexp.MustCompile("([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") var host string @@ -113,15 +147,15 @@ func getHost(u string) string { } // 去重+去除错误url -func RemoveRepeatElement(list []Link) []Link { +func RemoveRepeatElement(list []mode.Link) []mode.Link { // 创建一个临时map用来存储数组元素 temp := make(map[string]bool) - var list2 []Link + var list2 []mode.Link index := 0 for _, v := range list { //处理-d参数 - if d != "" { + if cmd.D != "" { v.Url = domainNameFilter(v.Url) } if len(v.Url) > 10 { @@ -143,68 +177,12 @@ func RemoveRepeatElement(list []Link) []Link { return list2 } -// 读取配置文件 -func GetConfig(path string) { - con := &config{} - if f, err := os.Open(path); err != nil { - if strings.Contains(err.Error(), "The system cannot find the file specified") || strings.Contains(err.Error(), "no such file or directory") { - - con.Headers = map[string]string{"Cookie": c, "User-Agent": ua, "Accept": "*/*"} - con.Proxy = "" - data, err2 := yaml.Marshal(con) - err2 = os.WriteFile(path, data, 0644) - if err2 != nil { - fmt.Println(err) - } else { - fmt.Println("未找到配置文件,已在当面目录下创建配置文件: config.yaml") - } - } else { - fmt.Println("配置文件错误,请尝试重新生成配置文件") - fmt.Println(err) - } - os.Exit(1) - } else { - yaml.NewDecoder(f).Decode(con) - conf = *con - } -} - -// 处理Headers配置 -func SetHeadersConfig(header *http.Header) *http.Header { - for k, v := range conf.Headers { - header.Add(k, v) - } - return header -} - -// proxy -func SetProxyConfig(tr *http.Transport) *http.Transport { - if len(conf.Proxy) > 0 { - proxyUrl, parseErr := url.Parse(conf.Proxy) - if parseErr != nil { - fmt.Println("代理地址错误: \n" + parseErr.Error()) - os.Exit(1) - } - tr.Proxy = http.ProxyURL(proxyUrl) - } - return tr -} - -// 打印Validate进度 -func printProgress() { - num := len(resultJs) + len(resultUrl) - fmt.Printf("\rValidate %.0f%%", float64(progress+1)/float64(num+1)*100) - mux.Lock() - progress++ - mux.Unlock() -} - // 打印Fuzz进度 -func printFuzz() { - fmt.Printf("\rFuzz %.0f%%", float64(progress+1)/float64(fuzzNum+1)*100) - mux.Lock() - progress++ - mux.Unlock() +func PrintFuzz() { + fmt.Printf("\rFuzz %.0f%%", float64(config.Progress+1)/float64(config.FuzzNum+1)*100) + config.Mux.Lock() + config.Progress++ + config.Mux.Unlock() } // 处理-d @@ -212,7 +190,7 @@ func domainNameFilter(url string) string { re := regexp.MustCompile("://([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") hosts := re.FindAllString(url, 1) if len(hosts) != 0 { - if !strings.Contains(hosts[0], d) { + if !strings.Contains(hosts[0], cmd.D) { url = "" } } @@ -220,7 +198,7 @@ func domainNameFilter(url string) string { } // 文件是否存在 -func exists(path string) bool { +func Exists(path string) bool { _, err := os.Stat(path) //os.Stat获取文件信息 if err != nil { if os.IsExist(err) { @@ -232,7 +210,7 @@ func exists(path string) bool { } // 数组去重 -func uniqueArr(arr []string) []string { +func UniqueArr(arr []string) []string { newArr := make([]string, 0) tempArr := make(map[string]bool, len(newArr)) for _, v := range arr { @@ -244,8 +222,20 @@ func uniqueArr(arr []string) []string { return newArr } +func GetDomains(lis []mode.Link) []string { + var urls []string + for i := range lis { + re := regexp.MustCompile("([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") + hosts := re.FindAllString(lis[i].Url, 1) + if len(hosts) > 0 { + urls = append(urls, hosts[0]) + } + } + return UniqueArr(urls) +} + // 提取fuzz的目录结构 -func pathExtract(urls []string) ([]string, []string) { +func PathExtract(urls []string) ([]string, []string) { var catalogues []string var targets []string if len(urls) == 0 { @@ -270,8 +260,8 @@ func pathExtract(urls []string) ([]string, []string) { } targets = append(targets, "upload") - catalogues = uniqueArr(catalogues) - targets = uniqueArr(targets) + catalogues = UniqueArr(catalogues) + targets = UniqueArr(targets) url1 := catalogues url2 := []string{} url3 := []string{} @@ -283,7 +273,7 @@ func pathExtract(urls []string) ([]string, []string) { } } } - if z == 3 { + if cmd.Z == 3 { for _, v1 := range url1 { for _, v3 := range url2 { if !strings.Contains(v3, v1) { @@ -295,14 +285,14 @@ func pathExtract(urls []string) ([]string, []string) { for i := range url1 { url1[i] = "/" + url1[i] } - if z == 3 { + if cmd.Z == 3 { path = make([]string, len(url1)+len(url2)+len(url3)) } else { path = make([]string, len(url1)+len(url2)) } copy(path, url1) copy(path[len(url1):], url2) - if z == 3 { + if cmd.Z == 3 { copy(path[len(url1)+len(url2):], url3) } for i := range path { @@ -313,7 +303,7 @@ func pathExtract(urls []string) ([]string, []string) { } // 去除状态码非404的404链接 -func del404(urls []Link) []Link { +func Del404(urls []mode.Link) []mode.Link { is := make(map[int]int) //根据长度分别存放 for _, v := range urls { @@ -324,7 +314,7 @@ func del404(urls []Link) []Link { is[len(v.Size)] = 1 } } - res := []Link{} + res := []mode.Link{} //如果某个长度的数量大于全部的3分之2，那么就判定它是404页面 for i, v := range is { if v > len(urls)/2 { @@ -339,80 +329,40 @@ func del404(urls []Link) []Link { } -func appendJs(url string, urltjs string) { - lock.Lock() - defer lock.Unlock() - url = strings.Replace(url, "/./", "/", -1) - for _, eachItem := range resultJs { - if eachItem.Url == url { - return - } - } - resultJs = append(resultJs, Link{Url: url}) - if strings.HasSuffix(urltjs, ".js") { - jsinurl[url] = jsinurl[urltjs] - } else { - re := regexp.MustCompile("[a-zA-z]+://[^\\s]*/|[a-zA-z]+://[^\\s]*") - u := re.FindAllStringSubmatch(urltjs, -1) - jsinurl[url] = u[0][0] - } - if o != "" { - jstourl[url] = urltjs - } - -} - -func appendUrl(url string, urlturl string) { - lock.Lock() - defer lock.Unlock() - url = strings.Replace(url, "/./", "/", -1) - for _, eachItem := range resultUrl { - if eachItem.Url == url { - return - } - } - resultUrl = append(resultUrl, Link{Url: url}) - if o != "" { - urltourl[url] = urlturl - } -} - -func appendInfo(info Info) { - lock.Lock() - defer lock.Unlock() - infos = append(infos, info) -} - -func appendEndUrl(url string) { - lock.Lock() - defer lock.Unlock() - for _, eachItem := range endUrl { - if eachItem == url { - return - } - } - endUrl = append(endUrl, url) - -} - -func getEndUrl(url string) bool { - lock.Lock() - defer lock.Unlock() - for _, eachItem := range endUrl { - if eachItem == url { - return true - } - } - return false - -} +var ( + + // for each request, a random UA will be selected from this list + uas = [...]string{ + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.68 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.61 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.71 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.71 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.62 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.107 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.121 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.88 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.71 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.72 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.94 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.98 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.98 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.63 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.95 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.106 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.5304.87 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.82 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.46", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36 SE 2.X MetaSr 1.0", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3883.400 QQBrowser/10.8.4559.400", + } + + nuas = len(uas) +) -func addSource() { - for i := range resultJs { - resultJs[i].Source = jstourl[resultJs[i].Url] +func GetUserAgent() string { + if cmd.A == "" { + cmd.A = uas[rand.Intn(nuas)] } - for i := range resultUrl { - resultUrl[i].Source = urltourl[resultUrl[i].Url] - } - + return cmd.A }