2023/2/21更新

README.md

@@ -117,6 +117,10 @@ SET GOARCH=arm64
 go build -ldflags "-s -w" -o ./URLFinder-macos-arm64
 ```
 ## 更新说明  
+2023/2/21   
+修复 已知bug  
+
+
 2023/2/3   
 新增 域名信息展示  
 变化 -i配置文件可配置抓取规则等   

cmd/cmd.go

@@ -16,7 +16,7 @@ var (
 	D string
 	C string
 	A string
-	b string
+	B string
 	F string
 	O string
 	X string
@@ -26,7 +26,7 @@ var (
 
 func init() {
 	flag.StringVar(&A, "a", "", "set user-agent\n设置user-agent请求头")
-	flag.StringVar(&b, "b", "", "set baseurl\n设置baseurl路径")
+	flag.StringVar(&B, "b", "", "set baseurl\n设置baseurl路径")
 	flag.StringVar(&C, "c", "", "set cookie\n设置cookie")
 	flag.StringVar(&D, "d", "", "set domainName\n指定获取的域名")
 	flag.StringVar(&F, "f", "", "set urlFile\n批量抓取url,指定文件路径")
@@ -52,7 +52,7 @@ Options:
 }
 
 func Parse() {
-	color.LightCyan.Println("         __   __   ___ _           _           \n /\\ /\\  /__\\ / /  / __(_)_ __   __| | ___ _ __ \n/ / \\ \\/ \\/// /  / _\\ | | '_ \\ / _` |/ _ \\ '__|\n\\ \\_/ / _  \\ /___ /   | | | | | (_| |  __/ |   \n \\___/\\/ \\_\\____\\/    |_|_| |_|\\__,_|\\___|_|     \n\nBy: pingc0y\nUpdateTime: 2023/2/3\nGithub: https://github.com/pingc0y/URLFinder \n")
+	color.LightCyan.Println("         __   __   ___ _           _           \n /\\ /\\  /__\\ / /  / __(_)_ __   __| | ___ _ __ \n/ / \\ \\/ \\/// /  / _\\ | | '_ \\ / _` |/ _ \\ '__|\n\\ \\_/ / _  \\ /___ /   | | | | | (_| |  __/ |   \n \\___/\\/ \\_\\____\\/    |_|_| |_|\\__,_|\\___|_|     \n\nBy: pingc0y\nUpdateTime: 2023/2/21\nGithub: https://github.com/pingc0y/URLFinder \n")
 	flag.Parse()
 	if h || (U == "" && F == "") {
 		flag.Usage()

config/config.go

@@ -49,7 +49,7 @@ var (
 	}
 	UrlFiler = []string{
 		"\\.js\\?|\\.css\\?|\\.jpeg\\?|\\.jpg\\?|\\.png\\?|.gif\\?|www\\.w3\\.org|example\\.com|\\<|\\>|\\{|\\}|\\[|\\]|\\||\\^|;|/js/|\\.src|\\.replace|\\.url|\\.att|\\.href|location\\.href|javascript:|location:|application/x-www-form-urlencoded|\\.createObject|:location|\\.path|\\*#__PURE__\\*|\\*\\$0\\*|\\n",
-		".*\\.js$|.*\\.css$|.*\\.scss$|.*,$|.*\\.jpeg$|.*\\.jpg$|.*\\.png&|.*\\.gif&|.*\\.ico$|.*\\.svg$|.*\\.vue$|.*\\.ts$",
+		".*\\.js$|.*\\.css$|.*\\.scss$|.*,$|.*\\.jpeg$|.*\\.jpg$|.*\\.png$|.*\\.gif$|.*\\.ico$|.*\\.svg$|.*\\.vue$|.*\\.ts$",
 	}
 
 	Phone  = []string{"['\"](1(3([0-35-9]\\d|4[1-8])|4[14-9]\\d|5([\\d]\\d|7[1-79])|66\\d|7[2-35-8]\\d|8\\d{2}|9[89]\\d)\\d{7})['\"]"}

crawler/crawler.go

@@ -1,6 +1,7 @@
 package crawler
 
 import (
+	"compress/gzip"
 	"crypto/tls"
 	"fmt"
 	"github.com/pingc0y/URLFinder/cmd"
@@ -50,33 +51,36 @@ func Spider(u string, num int) {
 	tr := &http.Transport{
 		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 	}
-
 	//配置代理
 	if cmd.X != "" {
-		proxyUrl, parseErr := url.Parse(config.Conf.Proxy)
+		proxyUrl, parseErr := url.Parse(cmd.X)
 		if parseErr != nil {
 			fmt.Println("代理地址错误: \n" + parseErr.Error())
 			os.Exit(1)
 		}
 		tr.Proxy = http.ProxyURL(proxyUrl)
-	}
-	//加载yaml配置(proxy)
-	if cmd.I {
+	} else if cmd.I {
+		//加载yaml配置
 		util.SetProxyConfig(tr)
 	}
 	client := &http.Client{Timeout: 10 * time.Second, Transport: tr}
 	request, err := http.NewRequest("GET", u, nil)
 	if err != nil {
 		return
 	}
-	//增加header选项
-	request.Header.Add("Cookie", cmd.C)
+
+	request.Header.Add("Accept-Encoding", "gzip") //使用gzip压缩传输数据让访问更快
 	request.Header.Add("User-Agent", util.GetUserAgent())
 	request.Header.Add("Accept", "*/*")
-	//加载yaml配置（headers）
+	//增加header选项
+	if cmd.C != "" {
+		request.Header.Add("Cookie", cmd.C)
+	}
+	//加载yaml配置(headers)
 	if cmd.I {
 		util.SetHeadersConfig(&request.Header)
 	}
+
 	//处理返回结果
 	response, err := client.Do(request)
 	if err != nil {
@@ -85,19 +89,32 @@ func Spider(u string, num int) {
 		defer response.Body.Close()
 
 	}
-
-	//提取url用于拼接其他url或js
-	dataBytes, err := io.ReadAll(response.Body)
-	if err != nil {
-		return
+	result := ""
+	//解压
+	if response.Header.Get("Content-Encoding") == "gzip" {
+		reader, err := gzip.NewReader(response.Body) // gzip解压缩
+		if err != nil {
+			return
+		}
+		defer reader.Close()
+		con, err := io.ReadAll(reader)
+		if err != nil {
+			return
+		}
+		result = string(con)
+	} else {
+		//提取url用于拼接其他url或js
+		dataBytes, err := io.ReadAll(response.Body)
+		if err != nil {
+			return
+		}
+		//字节数组 转换成 字符串
+		result = string(dataBytes)
 	}
 	path := response.Request.URL.Path
 	host := response.Request.URL.Host
 	scheme := response.Request.URL.Scheme
 	source := scheme + "://" + host + path
-
-	//字节数组 转换成 字符串
-	result := string(dataBytes)
 	//处理base标签
 	re := regexp.MustCompile("base.{1,5}href.{1,5}(http.+?//[^\\s]+?)[\",',‘,“]")
 	base := re.FindAllStringSubmatch(result, -1)

crawler/filter.go

@@ -29,6 +29,7 @@ func jsFilter(str [][]string) [][]string {
 			is := re.MatchString(str[i][0])
 			if is {
 				str[i][0] = ""
+				break
 			}
 		}
 
@@ -66,6 +67,7 @@ func urlFilter(str [][]string) [][]string {
 			is := re.MatchString(str[i][0])
 			if is {
 				str[i][0] = ""
+				break
 			}
 		}
 

crawler/find.go

@@ -115,8 +115,8 @@ func urlFind(cont, host, scheme, path, source string, num int) {
 				}
 			} else if strings.HasPrefix(url[0], "/") {
 				urlz := ""
-				if cmd.D != "" {
-					urlz = cmd.D + url[0]
+				if cmd.B != "" {
+					urlz = cmd.B + url[0]
 				} else {
 					urlz = host + url[0]
 				}
@@ -128,11 +128,11 @@ func urlFind(cont, host, scheme, path, source string, num int) {
 				}
 			} else if !strings.HasSuffix(source, ".js") {
 				urlz := ""
-				if cmd.D != "" {
-					if strings.HasSuffix(cmd.D, "/") {
-						urlz = cmd.D + url[0]
+				if cmd.B != "" {
+					if strings.HasSuffix(cmd.B, "/") {
+						urlz = cmd.B + url[0]
 					} else {
-						urlz = cmd.D + "/" + url[0]
+						urlz = cmd.B + "/" + url[0]
 					}
 				} else {
 					urlz = host + cata + url[0]

result/result.go

@@ -10,6 +10,7 @@ import (
 	"github.com/pingc0y/URLFinder/mode"
 	"github.com/pingc0y/URLFinder/util"
 	"log"
+	"net/url"
 	"os"
 	"regexp"
 	"strconv"
@@ -523,6 +524,7 @@ func Print() {
 	}
 
 	for _, u := range ResultUrlHost {
+		u.Url, _ = url.QueryUnescape(u.Url)
 		if cmd.S != "" && len(u.Title) != 0 {
 			if u.Status == "疑似危险路由" {
 				fmt.Printf(color.LightBlue.Sprintf("%-"+ulen+"s", u.Url) + color.LightGreen.Sprintf(" [ %s ]\n", u.Status))