update

SiJiDo · Jun 7, 2022 · c9cc8dc · c9cc8dc
1 parent 5e1b195
commit c9cc8dc
Show file tree

Hide file tree

Showing 8 changed files with 153 additions and 90 deletions.
diff --git a/go.mod b/go.mod
@@ -3,6 +3,16 @@ module IEyes
 go 1.15
 
 require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/antchfx/htmlquery v1.2.5 // indirect
+	github.com/antchfx/xmlquery v1.3.11 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/gocolly/colly v1.2.0
+	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/kirinlabs/HttpRequest v1.1.1
+	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
+	github.com/temoto/robotstxt v1.1.2 // indirect
+	golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect
+	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 )
diff --git a/go.sum b/go.sum
@@ -1,6 +1,59 @@
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/antchfx/htmlquery v1.2.5 h1:1lXnx46/1wtv1E/kzmH8vrfMuUKYgkdDBA9pIdMJnk4=
+github.com/antchfx/htmlquery v1.2.5/go.mod h1:2MCVBzYVafPBmKbrmwB9F5xdd+IEgRY61ci2oOsOQVw=
+github.com/antchfx/xmlquery v1.3.11 h1:8aRK7l3+dJjL8ZmwgVzG5AXysrP7Mss2424tfntKWKY=
+github.com/antchfx/xmlquery v1.3.11/go.mod h1:ywPcYkN0GvURUxXpUujaMVvuLSOYQBzoSfHKfAYezCE=
+github.com/antchfx/xpath v1.2.1 h1:qhp4EW6aCOVr5XIkT+l6LJ9ck/JsUH/yyauNgTQkBF8=
+github.com/antchfx/xpath v1.2.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
+github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
+github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
 github.com/kirinlabs/HttpRequest v1.1.1 h1:eBbFzpRd/Y7vQhRY30frHK3yAJiT1wDlB31Ryzyklc0=
 github.com/kirinlabs/HttpRequest v1.1.1/go.mod h1:XV38fA4rXZox83tlEV9KIQ7Cdsut319x6NGzVLuRlB8=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
+github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
+github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8=
+golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
diff --git a/main/IEyes_0.1.0-SNAPSHOT-5e1b195_windows_x64.tar.gz b/main/IEyes_0.1.0-SNAPSHOT-5e1b195_windows_x64.tar.gz
diff --git a/main/IEyes_0.1.0-SNAPSHOT-5e1b195_windows_x64/IEye.exe b/main/IEyes_0.1.0-SNAPSHOT-5e1b195_windows_x64/IEye.exe
diff --git a/main/basefunc.go b/main/basefunc.go
@@ -7,99 +7,97 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/gocolly/colly"
 	"github.com/kirinlabs/HttpRequest"
 )
 
-//获取当前页面的id和公司
 func GetPage(auth_token string, name string, page int) ([]string, []string) {
-	//开始获取页面的公司和id号
-	fmt.Println("[+]开始获取页面的公司")
-
-	split1 := "/span></div></div></div><div class=\"search-item sv-search-company  \""
-	split2 := "&card_name="
-	split3 := "&card_type=公司&card_id="
-	spilit_id1 := "&card_type=公司&card_id="
-	spilit_id2 := "&item=公司&"
 	var idlist []string
 	var infolist []string
 
+	defer func() {
+		if err := recover(); err != nil {
+			fmt.Println("[-]爬取异常,程序退出")
+			os.Exit(0)
+		}
+	}()
+
 	for p := 1; p <= page; p++ {
 
 		url1 := "https://www.tianyancha.com/search/p" + strconv.Itoa(p) + "?key=" + url.QueryEscape(name)
-		req := HttpRequest.NewRequest()
-		req.SetCookies(map[string]string{
-			"auth_token": auth_token,
-		})
 
-		resp, err := req.Get(url1)
+		//初始化爬虫
+		c := colly.NewCollector(
+			colly.UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:101.0) Gecko/20100101 Firefox/101.0"))
 
-		if err != nil {
-			fmt.Println(err)
-			os.Exit(1)
-		}
+		c.OnRequest(func(r *colly.Request) {
+			// Request头部设定
+			r.Headers.Set("Connection", "keep-alive")
+			r.Headers.Set("Accept", "*/*")
+			r.Headers.Set("Cookie", "auth_token="+auth_token)
+			r.Headers.Set("Accept-Language", "zh-CN, zh;q=0.9")
 
-		body, err := resp.Body()
-		r := strings.Split(string(body), split1)
-		flag := false
-
-		for s := range r {
-			var info string
-			var id string
-			if flag == false {
-				info = strings.Split(r[s], "<div class=\"xcx-qrcode\" tyc-xcx-qrcode></div><div class=\"info\">")[1]
-				info = strings.Split(info, "</div><div class=\"bottom\"><span>")[0]
-				fmt.Println(info)
-				flag = true
-			} else {
-				info = strings.Split(r[s], split2)[1]
-				info = strings.Split(info, split3)[0]
-				fmt.Println(info)
+			fmt.Println("Visiting", r.URL)
+		})
+
+		c.OnHTML("a", func(e *colly.HTMLElement) {
+			if e.Attr("class") == "index_alink__zcia5 link-click" {
+
+				info := string(e.Text)
+				link := strings.Split(e.Attr("href"), "/")
+				id := link[len(link)-1]
+				idlist = append(idlist, id)
+				infolist = append(infolist, info)
 			}
-			id = strings.Split(r[s], spilit_id1)[1]
-			id = strings.Split(id, spilit_id2)[0]
+		})
 
-			idlist = append(idlist, id)
-			infolist = append(infolist, info)
-		}
+		c.Visit(url1)
 	}
-	fmt.Println()
+
 	return idlist, infolist
 }
 
 func GetFirstCompany(auth_token string, name string) ([]string, []string) {
 
 	//开始获取页面的公司和id号
-	fmt.Println("[+]开始准备获取当前公司名称")
+	url1 := "https://www.tianyancha.com/search?key=" + url.QueryEscape(name)
+	id := ""
+	info := ""
 
-	split1 := "/span></div></div></div><div class=\"search-item sv-search-company  \""
-	split2 := "<div class=\"xcx-qrcode\" tyc-xcx-qrcode></div><div class=\"info\">"
-	split3 := "</div><div class=\"bottom\"><span>"
-	spilit_id1 := "&card_type=公司&card_id="
-	spilit_id2 := "&item=公司&"
+	fmt.Println("[+]开始准备获取当前公司名称")
 
-	url1 := "https://www.tianyancha.com/search?key=" + url.QueryEscape(name)
-	req := HttpRequest.NewRequest()
-	req.SetCookies(map[string]string{
-		"auth_token": auth_token,
-	})
+	defer func() {
+		if err := recover(); err != nil {
+			fmt.Println("[-]爬取异常,程序退出")
+			os.Exit(0)
+		}
+	}()
 
-	resp, err := req.Get(url1)
+	//初始化爬虫
+	c := colly.NewCollector(
+		colly.UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:101.0) Gecko/20100101 Firefox/101.0"))
 
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
+	c.OnRequest(func(r *colly.Request) {
+		// Request头部设定
+		r.Headers.Set("Connection", "keep-alive")
+		r.Headers.Set("Accept", "*/*")
+		r.Headers.Set("Cookie", "auth_token="+auth_token)
+		r.Headers.Set("Accept-Language", "zh-CN, zh;q=0.9")
 
-	body, err := resp.Body()
+		fmt.Println("Visiting", r.URL)
+	})
 
-	info := strings.Split(string(body), split2)[1]
-	info = strings.Split(info, split3)[0]
+	c.OnHTML("a", func(e *colly.HTMLElement) {
+		if e.Attr("class") == "index_alink__zcia5 link-click" {
+			if id == "" && info == "" {
+				info = string(e.Text)
+				link := strings.Split(e.Attr("href"), "/")
+				id = link[len(link)-1]
+			}
+		}
+	})
 
-	id := strings.Split(string(body), split1)[0]
-	id = strings.Split(string(id), spilit_id1)[1]
-	id = strings.Split(string(id), spilit_id2)[0]
-	fmt.Println(info)
-	fmt.Println()
+	c.Visit(url1)
 
 	return []string{id}, []string{info}
 

diff --git a/main/config.yaml b/main/config.yaml
@@ -1 +1 @@
-auth_token: "eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiIxNTk4NDYzNDYwOSIsImlhdCI6MTY0MjY2NzUyOSwiZXhwIjoxNjc0MjAzNTI5fQ.WCVqrlNXSjZcIWmUw2OmGXkw76rDHungg0-auM3Qmv_AoetxKI-iable2ljT-X47nAOPhndu-gF6mhQ_0258WQ"
+auth_token: ""
diff --git a/main/main.go b/main/main.go
@@ -119,26 +119,26 @@ func main() {
 	domainlist = RemoveRepeatedElement(domainlist)
 	applist = RemoveRepeatedElement(applist)
 	weixinlist = RemoveRepeatedElement(weixinlist)
-
-	//输出汇总结果
-	fmt.Println()
-	fmt.Println("[+]域名资产汇总")
-	fmt.Println("==================================")
-
-	for i := range domainlist {
-		fmt.Println(domainlist[i])
-	}
-	fmt.Println()
-	fmt.Println("[+]app资产汇总")
-	fmt.Println("==================================")
-	for i := range applist {
-		fmt.Println(applist[i])
-	}
-	fmt.Println()
-	fmt.Println("[+]微信公众号资产汇总")
-	fmt.Println("==================================")
-	for i := range weixinlist {
-		fmt.Println(weixinlist[i])
+	if child == true {
+		//输出汇总结果
+		fmt.Println()
+		fmt.Println("[+]域名资产汇总")
+		fmt.Println("==================================")
+
+		for i := range domainlist {
+			fmt.Println(domainlist[i])
+		}
+		fmt.Println()
+		fmt.Println("[+]app资产汇总")
+		fmt.Println("==================================")
+		for i := range applist {
+			fmt.Println(applist[i])
+		}
+		fmt.Println()
+		fmt.Println("[+]微信公众号资产汇总")
+		fmt.Println("==================================")
+		for i := range weixinlist {
+			fmt.Println(weixinlist[i])
+		}
 	}
-
 }
diff --git a/readme.md b/readme.md
@@ -44,6 +44,10 @@ Usage of ./IEyes:
 ./IEyes -n 哔哩哔哩
 ```
 
+![img1](img/img1.png)
+
+
+
 例子2
 查询目标公司和子公司，并且持股比例大于50%的
 
@@ -64,6 +68,4 @@ Usage of ./IEyes:
 ./IEyes -page 2 -n 北京通信管理局
 ```
 
-![img1](img/img1.png)
-
-![img2](img/img2.png)
+![img2](/Users/mi0/workplace/go-test/img/img2.png)