From fe552d7350d89280994bc455c7328810c52db1fe Mon Sep 17 00:00:00 2001 From: pingc0y Date: Fri, 5 May 2023 18:18:09 +0800 Subject: [PATCH] =?UTF-8?q?2023/5/5=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/release.yaml | 38 +++++++++++++++ .goreleaser.yaml | 45 ++++++++++++++++++ README.md | 5 ++ cmd/cmd.go | 2 +- crawler/crawler.go | 13 +++--- crawler/run.go | 23 +++++++-- crawler/state.go | 33 ++++++++++--- mode/mode.go | 11 +++-- result/report.html | 85 +++++++++++++++++++++++++--------- result/result.go | 37 ++++++++------- 10 files changed, 233 insertions(+), 59 deletions(-) create mode 100644 .github/workflows/release.yaml create mode 100644 .goreleaser.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..630d7dc --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,38 @@ +name: goreleaser + +on: + push: + # run only against tags + tags: + - '*' + +permissions: + contents: write + # packages: write + # issues: write + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - run: git fetch --force --tags + - uses: actions/setup-go@v3 + with: + go-version: '>=1.20.2' + cache: true + # More assembly might be required: Docker logins, GPG, etc. It all depends + # on your needs. + - uses: goreleaser/goreleaser-action@v4 + with: + # either 'goreleaser' (default) or 'goreleaser-pro': + distribution: goreleaser + version: latest + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Your GoReleaser Pro key, if you are using the 'goreleaser-pro' + # distribution: + # GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} \ No newline at end of file diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..7abc125 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,45 @@ +# This is an example .goreleaser.yml file with some sensible defaults. +# Make sure to check the documentation at https://goreleaser.com +before: + hooks: + # You may remove this if you don't use go modules. + - go mod tidy + # you may remove this if you don't need go generate + - go generate ./... +builds: + - env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin + +archives: + - format: tar.gz + # this name template makes the OS and Arch compatible with the results of uname. + name_template: >- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} + # use zip for windows archives + format_overrides: + - goos: windows + format: zip +checksum: + name_template: 'checksums.txt' +snapshot: + name_template: "{{ incpatch .Version }}-next" +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' + +# The lines beneath this are called `modelines`. See `:help modeline` +# Feel free to remove those if you don't want/use them. +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +# vim: set ts=2 sw=2 tw=0 fo=cnqoj diff --git a/README.md b/README.md index 0f7e5bf..7886aa6 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,11 @@ SET GOARCH=arm64 go build -ldflags "-s -w" -o ./URLFinder-macos-arm64 ``` ## 更新说明 +2023/5/5 +修复 多个任务时html结果混乱 +新增 结果添加302跳转信息 +变化 未获取到数据时不打印与输出结果 + 2023/4/22 修复 已知bug 变化 -d 改为正则表达式 diff --git a/cmd/cmd.go b/cmd/cmd.go index 0eb6823..a1236e7 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -54,6 +54,6 @@ Options: } func Parse() { - color.LightCyan.Println(" __ __ ___ _ _ \n /\\ /\\ /__\\ / / / __(_)_ __ __| | ___ _ __ \n/ / \\ \\/ \\/// / / _\\ | | '_ \\ / _` |/ _ \\ '__|\n\\ \\_/ / _ \\ /___ / | | | | | (_| | __/ | \n \\___/\\/ \\_\\____\\/ |_|_| |_|\\__,_|\\___|_| \n\nBy: pingc0y\nUpdateTime: 2023/4/22\nGithub: https://github.com/pingc0y/URLFinder \n") + color.LightCyan.Println(" __ __ ___ _ _ \n /\\ /\\ /__\\ / / / __(_)_ __ __| | ___ _ __ \n/ / \\ \\/ \\/// / / _\\ | | '_ \\ / _` |/ _ \\ '__|\n\\ \\_/ / _ \\ /___ / | | | | | (_| | __/ | \n \\___/\\/ \\_\\____\\/ |_|_| |_|\\__,_|\\___|_| \n\nBy: pingc0y\nUpdateTime: 2023/5/5\nGithub: https://github.com/pingc0y/URLFinder \n") flag.Parse() } diff --git a/crawler/crawler.go b/crawler/crawler.go index c1a891b..bb8f30f 100644 --- a/crawler/crawler.go +++ b/crawler/crawler.go @@ -20,18 +20,19 @@ import ( // 蜘蛛抓取页面内容 func Spider(u string, num int) { var is bool - fmt.Printf("\rStart %d Spider...", config.Progress) - config.Mux.Lock() - config.Progress++ - config.Mux.Unlock() - //标记完成 defer func() { config.Wg.Done() if !is { <-config.Ch } - }() + + fmt.Printf("\rStart %d Spider...", config.Progress) + config.Mux.Lock() + config.Progress++ + config.Mux.Unlock() + //标记完成 + u, _ = url.QueryUnescape(u) if num > 1 && cmd.D != "" && !regexp.MustCompile(cmd.D).MatchString(u) { return diff --git a/crawler/run.go b/crawler/run.go index fcb8e9f..9892dab 100644 --- a/crawler/run.go +++ b/crawler/run.go @@ -60,18 +60,22 @@ func start(u string) { } func Res() { + if len(result.ResultJs) == 0 && len(result.ResultUrl) == 0 { + fmt.Println("未获取到数据") + return + } //打印还是输出 if len(cmd.O) > 0 { result.OutFileJson() result.OutFileCsv() result.OutFileHtml() } else { + UrlToRedirect() result.Print() } } func Run() { - if cmd.O != "" { if !util.IsDir(cmd.O) { return @@ -85,7 +89,7 @@ func Run() { os.Exit(0) } if cmd.U == "" && cmd.F == "" && cmd.FF == "" { - fmt.Println("至少使用 -u 或 -f 指定一个url") + fmt.Println("至少使用 -u -f -ff 指定一个url") os.Exit(0) } if cmd.U != "" && !regexp.MustCompile("https{0,1}://").MatchString(cmd.U) { @@ -107,7 +111,6 @@ func Run() { } r := bufio.NewReader(fi) // 创建 Reader for { - lineBytes, err := r.ReadBytes('\n') //去掉字符串首尾空白字符,返回字符串 if len(lineBytes) > 5 { @@ -232,6 +235,20 @@ func AddSource() { } +func UrlToRedirect() { + for i := range result.ResultJs { + if result.ResultJs[i].Status == "302" { + result.ResultJs[i].Url = result.ResultJs[i].Url + " -> " + result.ResultJs[i].Redirect + } + } + for i := range result.ResultUrl { + if result.ResultUrl[i].Status == "302" { + result.ResultUrl[i].Url = result.ResultUrl[i].Url + " -> " + result.ResultUrl[i].Redirect + } + } + +} + func Initialization() { result.ResultJs = []mode.Link{} result.ResultUrl = []mode.Link{} diff --git a/crawler/state.go b/crawler/state.go index 3c44548..66e1741 100644 --- a/crawler/state.go +++ b/crawler/state.go @@ -54,7 +54,15 @@ func JsState(u string, i int, sou string) { if cmd.I { util.SetProxyConfig(tr) } - client := &http.Client{Timeout: 15 * time.Second, Transport: tr} + var redirect string + client := &http.Client{Timeout: 15 * time.Second, Transport: tr, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > 0 { + redirect = req.URL.String() + } + return nil + }, + } request, err := http.NewRequest("GET", u, nil) if err != nil { result.ResultJs[i].Url = "" @@ -94,7 +102,10 @@ func JsState(u string, i int, sou string) { } else { length = len(dataBytes) } - result.ResultJs[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length)} + if redirect != "" { + code = 302 + } + result.ResultJs[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Redirect: redirect} } else { result.ResultJs[i].Url = "" } @@ -102,7 +113,6 @@ func JsState(u string, i int, sou string) { // 检测url访问状态码 func UrlState(u string, i int) { - defer func() { config.Wg.Done() <-config.Urlch @@ -138,7 +148,15 @@ func UrlState(u string, i int) { if cmd.I { util.SetProxyConfig(tr) } - client := &http.Client{Timeout: 15 * time.Second, Transport: tr} + var redirect string + client := &http.Client{Timeout: 15 * time.Second, Transport: tr, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > 0 { + redirect = req.URL.String() + } + return nil + }, + } request, err := http.NewRequest("GET", u, nil) if err != nil { result.ResultUrl[i].Url = "" @@ -182,10 +200,13 @@ func UrlState(u string, i int) { body := string(dataBytes) re := regexp.MustCompile("<[tT]itle>(.*?)") title := re.FindAllStringSubmatch(body, -1) + if redirect != "" { + code = 302 + } if len(title) != 0 { - result.ResultUrl[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Title: title[0][1]} + result.ResultUrl[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Title: title[0][1], Redirect: redirect} } else { - result.ResultUrl[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length)} + result.ResultUrl[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Redirect: redirect} } } else { result.ResultUrl[i].Url = "" diff --git a/mode/mode.go b/mode/mode.go index bfce493..5bcb6e6 100644 --- a/mode/mode.go +++ b/mode/mode.go @@ -15,11 +15,12 @@ type Config struct { } type Link struct { - Url string - Status string - Size string - Title string - Source string + Url string + Status string + Size string + Title string + Redirect string + Source string } type Info struct { diff --git a/result/report.html b/result/report.html index 56e4d17..14b06ab 100644 --- a/result/report.html +++ b/result/report.html @@ -451,7 +451,7 @@

JS to {urlHost}

+ style="min-width:100px;">
Status @@ -460,7 +460,7 @@

JS to {urlHost}

+ style="min-width:100px;">
Size @@ -478,14 +478,22 @@

JS to {urlHost}

+ style="min-width:400px;"> + +
+ Redirect +
+ + +
Source
- @@ -534,7 +542,7 @@

JS to Other

+ style="min-width:100px;">
Status @@ -543,7 +551,7 @@

JS to Other

+ style="min-width:100px;">
Size @@ -561,14 +569,22 @@

JS to Other

+ style="min-width:400px;"> + +
+ Redirect +
+ + +
Source
- @@ -617,7 +633,7 @@

URL to {urlHost}

+ style="min-width:100px;">
Status @@ -626,7 +642,7 @@

URL to {urlHost}

+ style="min-width:100px;">
Size @@ -644,14 +660,22 @@

URL to {urlHost}

+ style="min-width:400px;"> + +
+ Redirect +
+ + +
Source
- @@ -700,7 +724,7 @@

URL to Other

+ style="min-width:100px;">
Status @@ -709,7 +733,7 @@

URL to Other

+ style="min-width:100px;">
Size @@ -727,14 +751,22 @@

URL to Other

+ style="min-width:400px;"> + +
+ Redirect +
+ + +
Source
- @@ -783,7 +815,7 @@

Fuzz

+ style="min-width:100px;">
Status @@ -792,7 +824,7 @@

Fuzz

+ style="min-width:100px;">
Size @@ -810,13 +842,23 @@

Fuzz

+ style="min-width:400px;">
- Source + Redirect
+ + + +
+ Source +
+ + @@ -910,7 +952,7 @@

Info

+ style="min-width:100px;">
Value @@ -926,7 +968,6 @@

Info

 - diff --git a/result/result.go b/result/result.go index a25f3fe..302adf6 100644 --- a/result/result.go +++ b/result/result.go @@ -49,6 +49,10 @@ func outHtmlString(link mode.Link) string { ` + link.Title + ` + + + ` + link.Redirect + ` + ` + link.Source + ` @@ -128,7 +132,7 @@ func OutFileCsv() { if cmd.S == "" { resultWriter.Write([]string{"url", "Source"}) } else { - resultWriter.Write([]string{"url", "Status", "Size", "Title", "Source"}) + resultWriter.Write([]string{"url", "Status", "Size", "Title", "Redirect", "Source"}) } if cmd.D == "" { resultWriter.Write([]string{strconv.Itoa(len(ResultJsHost)) + " JS to " + util.GetHost(cmd.U)}) @@ -138,7 +142,7 @@ func OutFileCsv() { for _, j := range ResultJsHost { if cmd.S != "" { - resultWriter.Write([]string{j.Url, j.Status, j.Size, "", j.Source}) + resultWriter.Write([]string{j.Url, j.Status, j.Size, "", j.Redirect, j.Source}) } else { resultWriter.Write([]string{j.Url, j.Source}) } @@ -150,7 +154,7 @@ func OutFileCsv() { } for _, j := range ResultJsOther { if cmd.S != "" { - resultWriter.Write([]string{j.Url, j.Status, j.Size, "", j.Source}) + resultWriter.Write([]string{j.Url, j.Status, j.Size, "", j.Redirect, j.Source}) } else { resultWriter.Write([]string{j.Url, j.Source}) } @@ -165,7 +169,7 @@ func OutFileCsv() { for _, u := range ResultUrlHost { if cmd.S != "" { - resultWriter.Write([]string{u.Url, u.Status, u.Size, u.Title, u.Source}) + resultWriter.Write([]string{u.Url, u.Status, u.Size, u.Title, u.Redirect, u.Source}) } else { resultWriter.Write([]string{u.Url, u.Source}) } @@ -176,7 +180,7 @@ func OutFileCsv() { } for _, u := range ResultUrlOther { if cmd.S != "" { - resultWriter.Write([]string{u.Url, u.Status, u.Size, u.Title, u.Source}) + resultWriter.Write([]string{u.Url, u.Status, u.Size, u.Title, u.Redirect, u.Source}) } else { resultWriter.Write([]string{u.Url, u.Source}) } @@ -186,7 +190,7 @@ func OutFileCsv() { resultWriter.Write([]string{strconv.Itoa(len(Fuzzs)) + " URL to Fuzz"}) Fuzzs = util.SelectSort(Fuzzs) for _, u := range Fuzzs { - resultWriter.Write([]string{u.Url, u.Status, u.Size, u.Title, "Fuzz"}) + resultWriter.Write([]string{u.Url, u.Status, u.Size, u.Title, u.Redirect, "Fuzz"}) } } resultWriter.Write([]string{""}) @@ -347,6 +351,7 @@ func OutFileJson() { // 导出html func OutFileHtml() { + htmlTemp := html //获取域名 var host string re := regexp.MustCompile("([a-z0-9\\-]+\\.)*([a-z0-9\\-]+\\.[a-z0-9\\-]+)(:[0-9]+)?") @@ -393,33 +398,33 @@ func OutFileHtml() { writer := bufio.NewWriter(file) if cmd.D == "" { - html = strings.Replace(html, "{urlHost}", util.GetHost(cmd.U), -1) + htmlTemp = strings.Replace(htmlTemp, "{urlHost}", util.GetHost(cmd.U), -1) } else { - html = strings.Replace(html, "{urlHost}", cmd.D, -1) + htmlTemp = strings.Replace(htmlTemp, "{urlHost}", cmd.D, -1) } var ResultJsHostStr string for _, j := range ResultJsHost { ResultJsHostStr += outHtmlString(j) } - html = strings.Replace(html, "{JS}", ResultJsHostStr, -1) + htmlTemp = strings.Replace(htmlTemp, "{JS}", ResultJsHostStr, -1) var ResultJsOtherStr string for _, j := range ResultJsOther { ResultJsOtherStr += outHtmlString(j) } - html = strings.Replace(html, "{JSOther}", ResultJsOtherStr, -1) + htmlTemp = strings.Replace(htmlTemp, "{JSOther}", ResultJsOtherStr, -1) var ResultUrlHostStr string for _, u := range ResultUrlHost { ResultUrlHostStr += outHtmlString(u) } - html = strings.Replace(html, "{URL}", ResultUrlHostStr, -1) + htmlTemp = strings.Replace(htmlTemp, "{URL}", ResultUrlHostStr, -1) var ResultUrlOtherStr string for _, u := range ResultUrlOther { ResultUrlOtherStr += outHtmlString(u) } - html = strings.Replace(html, "{URLOther}", ResultUrlOtherStr, -1) + htmlTemp = strings.Replace(htmlTemp, "{URLOther}", ResultUrlOtherStr, -1) var FuzzsStr string if cmd.S != "" && cmd.Z != 0 { @@ -428,13 +433,13 @@ func OutFileHtml() { FuzzsStr += outHtmlString(u) } } - html = strings.Replace(html, "{Fuzz}", FuzzsStr, -1) + htmlTemp = strings.Replace(htmlTemp, "{Fuzz}", FuzzsStr, -1) var DomainsStr string for _, u := range Domains { DomainsStr += outHtmlDomainString(u) } - html = strings.Replace(html, "{Domains}", DomainsStr, -1) + htmlTemp = strings.Replace(htmlTemp, "{Domains}", DomainsStr, -1) var Infostr string for i := range Infos { @@ -462,8 +467,8 @@ func OutFileHtml() { Infostr += outHtmlInfoString("Other", Infos[i].Other[i2], Infos[i].Source) } } - html = strings.Replace(html, "{Info}", Infostr, -1) - writer.WriteString(html) + htmlTemp = strings.Replace(htmlTemp, "{Info}", Infostr, -1) + writer.WriteString(htmlTemp) writer.Flush() //内容是先写到缓存对,所以需要调用flush将缓存对数据真正写到文件中 fmt.Println(strconv.Itoa(len(ResultJsHost)+len(ResultJsOther))+"JS + "+strconv.Itoa(len(ResultUrlHost)+len(ResultUrlOther))+"URL --> ", file.Name()) return