2023/5/5更新

.github/workflows/release.yaml

@@ -0,0 +1,38 @@
+name: goreleaser
+
+on:
+  push:
+    # run only against tags
+    tags:
+      - '*'
+
+permissions:
+  contents: write
+  # packages: write
+  # issues: write
+
+jobs:
+  goreleaser:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - run: git fetch --force --tags
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '>=1.20.2'
+          cache: true
+      # More assembly might be required: Docker logins, GPG, etc. It all depends
+      # on your needs.
+      - uses: goreleaser/goreleaser-action@v4
+        with:
+          # either 'goreleaser' (default) or 'goreleaser-pro':
+          distribution: goreleaser
+          version: latest
+          args: release --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # Your GoReleaser Pro key, if you are using the 'goreleaser-pro'
+          # distribution:
+          # GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}

.goreleaser.yaml

@@ -0,0 +1,45 @@
+# This is an example .goreleaser.yml file with some sensible defaults.
+# Make sure to check the documentation at https://goreleaser.com
+before:
+  hooks:
+    # You may remove this if you don't use go modules.
+    - go mod tidy
+    # you may remove this if you don't need go generate
+    - go generate ./...
+builds:
+  - env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+
+archives:
+  - format: tar.gz
+    # this name template makes the OS and Arch compatible with the results of uname.
+    name_template: >-
+      {{ .ProjectName }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+    # use zip for windows archives
+    format_overrides:
+      - goos: windows
+        format: zip
+checksum:
+  name_template: 'checksums.txt'
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
+
+# The lines beneath this are called `modelines`. See `:help modeline`
+# Feel free to remove those if you don't want/use them.
+# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
+# vim: set ts=2 sw=2 tw=0 fo=cnqoj

README.md

@@ -118,6 +118,11 @@ SET GOARCH=arm64
 go build -ldflags "-s -w" -o ./URLFinder-macos-arm64
 ```
 ## 更新说明  
+2023/5/5   
+修复 多个任务时html结果混乱  
+新增 结果添加302跳转信息  
+变化 未获取到数据时不打印与输出结果  
+
 2023/4/22   
 修复 已知bug  
 变化 -d 改为正则表达式  

cmd/cmd.go

@@ -54,6 +54,6 @@ Options:
 }
 
 func Parse() {
-	color.LightCyan.Println("         __   __   ___ _           _           \n /\\ /\\  /__\\ / /  / __(_)_ __   __| | ___ _ __ \n/ / \\ \\/ \\/// /  / _\\ | | '_ \\ / _` |/ _ \\ '__|\n\\ \\_/ / _  \\ /___ /   | | | | | (_| |  __/ |   \n \\___/\\/ \\_\\____\\/    |_|_| |_|\\__,_|\\___|_|     \n\nBy: pingc0y\nUpdateTime: 2023/4/22\nGithub: https://github.com/pingc0y/URLFinder \n")
+	color.LightCyan.Println("         __   __   ___ _           _           \n /\\ /\\  /__\\ / /  / __(_)_ __   __| | ___ _ __ \n/ / \\ \\/ \\/// /  / _\\ | | '_ \\ / _` |/ _ \\ '__|\n\\ \\_/ / _  \\ /___ /   | | | | | (_| |  __/ |   \n \\___/\\/ \\_\\____\\/    |_|_| |_|\\__,_|\\___|_|     \n\nBy: pingc0y\nUpdateTime: 2023/5/5\nGithub: https://github.com/pingc0y/URLFinder \n")
 	flag.Parse()
 }

crawler/crawler.go

@@ -20,18 +20,19 @@ import (
 // 蜘蛛抓取页面内容
 func Spider(u string, num int) {
 	var is bool
-	fmt.Printf("\rStart %d Spider...", config.Progress)
-	config.Mux.Lock()
-	config.Progress++
-	config.Mux.Unlock()
-	//标记完成
 	defer func() {
 		config.Wg.Done()
 		if !is {
 			<-config.Ch
 		}
-
 	}()
+
+	fmt.Printf("\rStart %d Spider...", config.Progress)
+	config.Mux.Lock()
+	config.Progress++
+	config.Mux.Unlock()
+	//标记完成
+
 	u, _ = url.QueryUnescape(u)
 	if num > 1 && cmd.D != "" && !regexp.MustCompile(cmd.D).MatchString(u) {
 		return

crawler/run.go

@@ -60,18 +60,22 @@ func start(u string) {
 }
 
 func Res() {
+	if len(result.ResultJs) == 0 && len(result.ResultUrl) == 0 {
+		fmt.Println("未获取到数据")
+		return
+	}
 	//打印还是输出
 	if len(cmd.O) > 0 {
 		result.OutFileJson()
 		result.OutFileCsv()
 		result.OutFileHtml()
 	} else {
+		UrlToRedirect()
 		result.Print()
 	}
 }
 
 func Run() {
-
 	if cmd.O != "" {
 		if !util.IsDir(cmd.O) {
 			return
@@ -85,7 +89,7 @@ func Run() {
 		os.Exit(0)
 	}
 	if cmd.U == "" && cmd.F == "" && cmd.FF == "" {
-		fmt.Println("至少使用 -u 或 -f 指定一个url")
+		fmt.Println("至少使用 -u -f -ff 指定一个url")
 		os.Exit(0)
 	}
 	if cmd.U != "" && !regexp.MustCompile("https{0,1}://").MatchString(cmd.U) {
@@ -107,7 +111,6 @@ func Run() {
 		}
 		r := bufio.NewReader(fi) // 创建 Reader
 		for {
-
 			lineBytes, err := r.ReadBytes('\n')
 			//去掉字符串首尾空白字符,返回字符串
 			if len(lineBytes) > 5 {
@@ -232,6 +235,20 @@ func AddSource() {
 
 }
 
+func UrlToRedirect() {
+	for i := range result.ResultJs {
+		if result.ResultJs[i].Status == "302" {
+			result.ResultJs[i].Url = result.ResultJs[i].Url + " -> " + result.ResultJs[i].Redirect
+		}
+	}
+	for i := range result.ResultUrl {
+		if result.ResultUrl[i].Status == "302" {
+			result.ResultUrl[i].Url = result.ResultUrl[i].Url + " -> " + result.ResultUrl[i].Redirect
+		}
+	}
+
+}
+
 func Initialization() {
 	result.ResultJs = []mode.Link{}
 	result.ResultUrl = []mode.Link{}

crawler/state.go

@@ -54,7 +54,15 @@ func JsState(u string, i int, sou string) {
 	if cmd.I {
 		util.SetProxyConfig(tr)
 	}
-	client := &http.Client{Timeout: 15 * time.Second, Transport: tr}
+	var redirect string
+	client := &http.Client{Timeout: 15 * time.Second, Transport: tr,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if len(via) > 0 {
+				redirect = req.URL.String()
+			}
+			return nil
+		},
+	}
 	request, err := http.NewRequest("GET", u, nil)
 	if err != nil {
 		result.ResultJs[i].Url = ""
@@ -94,15 +102,17 @@ func JsState(u string, i int, sou string) {
 		} else {
 			length = len(dataBytes)
 		}
-		result.ResultJs[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length)}
+		if redirect != "" {
+			code = 302
+		}
+		result.ResultJs[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Redirect: redirect}
 	} else {
 		result.ResultJs[i].Url = ""
 	}
 }
 
 // 检测url访问状态码
 func UrlState(u string, i int) {
-
 	defer func() {
 		config.Wg.Done()
 		<-config.Urlch
@@ -138,7 +148,15 @@ func UrlState(u string, i int) {
 	if cmd.I {
 		util.SetProxyConfig(tr)
 	}
-	client := &http.Client{Timeout: 15 * time.Second, Transport: tr}
+	var redirect string
+	client := &http.Client{Timeout: 15 * time.Second, Transport: tr,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if len(via) > 0 {
+				redirect = req.URL.String()
+			}
+			return nil
+		},
+	}
 	request, err := http.NewRequest("GET", u, nil)
 	if err != nil {
 		result.ResultUrl[i].Url = ""
@@ -182,10 +200,13 @@ func UrlState(u string, i int) {
 		body := string(dataBytes)
 		re := regexp.MustCompile("<[tT]itle>(.*?)</[tT]itle>")
 		title := re.FindAllStringSubmatch(body, -1)
+		if redirect != "" {
+			code = 302
+		}
 		if len(title) != 0 {
-			result.ResultUrl[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Title: title[0][1]}
+			result.ResultUrl[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Title: title[0][1], Redirect: redirect}
 		} else {
-			result.ResultUrl[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length)}
+			result.ResultUrl[i] = mode.Link{Url: u, Status: strconv.Itoa(code), Size: strconv.Itoa(length), Redirect: redirect}
 		}
 	} else {
 		result.ResultUrl[i].Url = ""

mode/mode.go

@@ -15,11 +15,12 @@ type Config struct {
 }
 
 type Link struct {
-	Url    string
-	Status string
-	Size   string
-	Title  string
-	Source string
+	Url      string
+	Status   string
+	Size     string
+	Title    string
+	Redirect string
+	Source   string
 }
 
 type Info struct {