diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 00dca0c..f120a80 100755 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: build +name: build 🔨 on: push: @@ -12,16 +12,16 @@ jobs: runs-on: ubuntu-latest steps: - - name: Set up Go - uses: actions/setup-go@v2 + name: Check out code + uses: actions/checkout@v3 with: - go-version: 1.17 - + fetch-depth: 0 - - name: Check out code - uses: actions/checkout@v2 - + name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '>=1.17.0' - name: Build run: go build -v . - working-directory: cmd/sigrawl3r/ \ No newline at end of file + working-directory: ./cmd/hqcrawl3r \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 369bbe8..6368ab4 100755 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,9 +1,9 @@ -name: release +name: release 🎉 on: create: tags: - - v* + - v*.*.* workflow_dispatch: jobs: @@ -11,18 +11,16 @@ jobs: name: release runs-on: ubuntu-latest steps: - - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.17 - - - name: "Check out code" - uses: actions/checkout@v2 + name: Check out code + uses: actions/checkout@v3 with: fetch-depth: 0 - + - + name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '>=1.17.0' - name: Get dependencies run: | @@ -31,12 +29,12 @@ jobs: curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh dep ensure fi - - name: "Create release on GitHub" - uses: goreleaser/goreleaser-action@v2 + uses: goreleaser/goreleaser-action@v3 env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" with: args: "release --rm-dist" version: latest + distribution: goreleaser \ No newline at end of file diff --git a/.goreleaser.yaml b/.goreleaser.yaml index a3eb535..de4f891 100755 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,7 +1,7 @@ builds: - - binary: sigrawl3r - main: cmd/sigrawl3r/main.go + binary: hqcrawl3r + main: cmd/hqcrawl3r/main.go goos: - linux - windows diff --git a/README.md b/README.md index 3be25ef..75f1886 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# sigrawl3r +# hqcrawl3r -[![release](https://img.shields.io/github/release/signedsecurity/sigrawl3r?style=flat&color=0040ff)](https://github.com/signedsecurity/sigrawl3r/releases) [![maintenance](https://img.shields.io/badge/maintained%3F-yes-0040ff.svg)](https://github.com/signedsecurity/sigrawl3r) [![open issues](https://img.shields.io/github/issues-raw/signedsecurity/sigrawl3r.svg?style=flat&color=0040ff)](https://github.com/signedsecurity/sigrawl3r/issues?q=is:issue+is:open) [![closed issues](https://img.shields.io/github/issues-closed-raw/signedsecurity/sigrawl3r.svg?style=flat&color=0040ff)](https://github.com/signedsecurity/sigrawl3r/issues?q=is:issue+is:closed) [![license](https://img.shields.io/badge/license-MIT-gray.svg?colorB=0040FF)](https://github.com/signedsecurity/sigrawl3r/blob/master/LICENSE) [![twitter](https://img.shields.io/badge/twitter-@signedsecurity-0040ff.svg)](https://twitter.com/signedsecurity) +[![release](https://img.shields.io/github/release/hueristiq/hqcrawl3r?style=flat&color=0040ff)](https://github.com/hueristiq/hqcrawl3r/releases) [![maintenance](https://img.shields.io/badge/maintained%3F-yes-0040ff.svg)](https://github.com/hueristiq/hqcrawl3r) [![open issues](https://img.shields.io/github/issues-raw/hueristiq/hqcrawl3r.svg?style=flat&color=0040ff)](https://github.com/hueristiq/hqcrawl3r/issues?q=is:issue+is:open) [![closed issues](https://img.shields.io/github/issues-closed-raw/hueristiq/hqcrawl3r.svg?style=flat&color=0040ff)](https://github.com/hueristiq/hqcrawl3r/issues?q=is:issue+is:closed) [![license](https://img.shields.io/badge/license-MIT-gray.svg?colorB=0040FF)](https://github.com/hueristiq/hqcrawl3r/blob/master/LICENSE) [![twitter](https://img.shields.io/badge/twitter-@itshueristiq-0040ff.svg)](https://twitter.com/itshueristiq) A fast web crawler. @@ -25,19 +25,19 @@ A fast web crawler. ## Usage ```bash -sigrawl3r -h +hqcrawl3r -h ``` ```text - _ _ _____ - ___(_) __ _ _ __ __ ___ _| |___ / _ __ -/ __| |/ _` | '__/ _` \ \ /\ / / | |_ \| '__| -\__ \ | (_| | | | (_| |\ V V /| |___) | | -|___/_|\__, |_| \__,_| \_/\_/ |_|____/|_| v1.1.0 - |___/ + _ _ _____ +| |__ __ _ ___ _ __ __ ___ _| |___ / _ __ +| '_ \ / _` |/ __| '__/ _` \ \ /\ / / | |_ \| '__| +| | | | (_| | (__| | | (_| |\ V V /| |___) | | +|_| |_|\__, |\___|_| \__,_| \_/\_/ |_|____/|_| v1.1.0 + |_| USAGE: - sigrawl3r [OPTIONS] + hqcrawl3r [OPTIONS] OPTIONS: -c, --concurrency Maximum concurrent requests for matching domains (default: 5) @@ -47,7 +47,8 @@ OPTIONS: --headless If true the browser will be displayed while crawling Note: Requires '-r, --render' flag Note: Usage to show browser: '--headless=false' (default true) - -H, --headers Custom headers separated by two semi-colons. E.g. -h 'Cookie: foo=bar;;Referer: http://example.com/' + -H, --headers Custom headers separated by two semi-colons. + E.g. -h 'Cookie: foo=bar;;Referer: http://example.com/' --include-subs Extend scope to include subdomains (default: false) --no-color Enable no color mode (default: false) -p, --proxy Proxy URL (e.g: http://127.0.0.1:8080) @@ -68,26 +69,26 @@ OPTIONS: #### From Binary -You can download the pre-built binary for your platform from this repository's [releases](https://github.com/signedsecurity/sigrawl3r/releases/) page, extract, then move it to your `$PATH`and you're ready to go. +You can download the pre-built binary for your platform from this repository's [releases](https://github.com/hueristiq/hqcrawl3r/releases/) page, extract, then move it to your `$PATH`and you're ready to go. #### From Source -sigrawl3r requires **go1.17+** to install successfully. Run the following command to get the repo +hqcrawl3r requires **go1.17+** to install successfully. Run the following command to get the repo ```bash -go install github.com/signedsecurity/sigrawl3r/cmd/sigrawl3r@latest +go install github.com/hueristiq/hqcrawl3r/cmd/hqcrawl3r@latest ``` #### From Github ```bash -git clone https://github.com/signedsecurity/sigrawl3r.git && \ -cd sigrawl3r/cmd/sigrawl3r/ && \ +git clone https://github.com/hueristiq/hqcrawl3r.git && \ +cd hqcrawl3r/cmd/hqcrawl3r/ && \ go build . && \ -mv sigrawl3r /usr/local/bin/ && \ -sigrawl3r -h +mv hqcrawl3r /usr/local/bin/ && \ +hqcrawl3r -h ``` ## Contribution -[Issues](https://github.com/signedsecurity/sigrawl3r/issues) and [Pull Requests](https://github.com/signedsecurity/sigrawl3r/pulls) are welcome! \ No newline at end of file +[Issues](https://github.com/hueristiq/hqcrawl3r/issues) and [Pull Requests](https://github.com/hueristiq/hqcrawl3r/pulls) are welcome! \ No newline at end of file diff --git a/cmd/sigrawl3r/main.go b/cmd/hqcrawl3r/main.go similarity index 96% rename from cmd/sigrawl3r/main.go rename to cmd/hqcrawl3r/main.go index 3854867..52eca1a 100755 --- a/cmd/sigrawl3r/main.go +++ b/cmd/hqcrawl3r/main.go @@ -9,10 +9,10 @@ import ( "sync" "github.com/enenumxela/urlx/pkg/urlx" + "github.com/hueristiq/hqcrawl3r/internal/configuration" + "github.com/hueristiq/hqcrawl3r/internal/crawler" + "github.com/hueristiq/hqcrawl3r/internal/utils/io" "github.com/logrusorgru/aurora/v3" - "github.com/signedsecurity/sigrawl3r/internal/configuration" - "github.com/signedsecurity/sigrawl3r/internal/crawler" - "github.com/signedsecurity/sigrawl3r/internal/utils/io" ) var ( @@ -60,7 +60,7 @@ func init() { displayBanner() h := "USAGE:\n" - h += " sigrawl3r [OPTIONS]\n" + h += " hqcrawl3r [OPTIONS]\n" h += "\nOPTIONS:\n" h += fmt.Sprintf(" -c, --concurrency Maximum concurrent requests for matching domains (default: %d)\n", configuration.DefaultConcurrency) @@ -86,6 +86,7 @@ func init() { h += " `web` for a random web user-agent\n" h += " `mobile` for a random mobile user-agent\n" h += " or you can set your special user-agent\n" + h += "\n" fmt.Fprint(os.Stderr, h) } diff --git a/go.mod b/go.mod index de40fe1..1afcf91 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ -module github.com/signedsecurity/sigrawl3r +module github.com/hueristiq/hqcrawl3r -go 1.17 +go 1.18 require ( - github.com/chromedp/chromedp v0.8.0 + github.com/chromedp/chromedp v0.8.3 github.com/enenumxela/urlx v0.0.0-20220330083946-3fddfff8acee github.com/gocolly/colly/v2 v2.1.0 github.com/logrusorgru/aurora/v3 v3.0.0 @@ -11,27 +11,27 @@ require ( ) require ( - github.com/PuerkitoBio/goquery v1.8.0 // indirect - github.com/andybalholm/cascadia v1.3.1 // indirect - github.com/antchfx/htmlquery v1.2.4 // indirect - github.com/antchfx/xmlquery v1.3.10 // indirect - github.com/antchfx/xpath v1.2.0 // indirect - github.com/chromedp/cdproto v0.0.0-20220321060548-7bc2623472b3 // indirect + github.com/PuerkitoBio/goquery v1.5.1 // indirect + github.com/andybalholm/cascadia v1.2.0 // indirect + github.com/antchfx/htmlquery v1.2.3 // indirect + github.com/antchfx/xmlquery v1.2.4 // indirect + github.com/antchfx/xpath v1.1.8 // indirect + github.com/chromedp/cdproto v0.0.0-20220725225757-5988d9195a6c // indirect github.com/chromedp/sysutil v1.0.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.1.0 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.4.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect - github.com/temoto/robotstxt v1.1.2 // indirect + github.com/temoto/robotstxt v1.1.1 // indirect golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect golang.org/x/text v0.3.7 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.0 // indirect + google.golang.org/appengine v1.6.6 // indirect + google.golang.org/protobuf v1.24.0 // indirect ) diff --git a/go.sum b/go.sum index 696ec21..cd63520 100644 --- a/go.sum +++ b/go.sum @@ -1,27 +1,22 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= -github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= -github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE= github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= -github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M= github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= -github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494= -github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc= +github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4= github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM= -github.com/antchfx/xmlquery v1.3.10 h1:U2yMwr8U0KmGM2iDG2Ky/3LfxNsiK4uw1bSBkeMO9+g= -github.com/antchfx/xmlquery v1.3.10/go.mod h1:wojC/BxjEkjJt6dPiAqUzoXO5nIMWtxHS8PD8TmN4ks= github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/antchfx/xpath v1.1.8 h1:PcL6bIX42Px5usSx6xRYw/wjB3wYGkj0MJ9MBzEKVgk= github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= -github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8= -github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chromedp/cdproto v0.0.0-20220321060548-7bc2623472b3 h1:ilF7R875ORf246k6nU2/6fRIbtAFYjPOOtwZJTQvusM= -github.com/chromedp/cdproto v0.0.0-20220321060548-7bc2623472b3/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0= -github.com/chromedp/chromedp v0.8.0 h1:+Cufl+QWWfbvyylGCtAUt34A2EI/kqxRM3wGHXMabU4= -github.com/chromedp/chromedp v0.8.0/go.mod h1:odCVV9o9i7HUKwHMFz9Y7T6s4Kbcz4GOyPlwKWopI9Q= +github.com/chromedp/cdproto v0.0.0-20220725225757-5988d9195a6c h1:Gm+DujZPVAtQNTLhbg5PExjRNfhdTCSMLvJ/pFfY4aY= +github.com/chromedp/cdproto v0.0.0-20220725225757-5988d9195a6c/go.mod h1:5Y4sD/eXpwrChIuxhSr/G20n9CdbCmoerOHnuAf0Zr0= +github.com/chromedp/chromedp v0.8.3 h1:UwOY+fhC5Vv3uKgRpnvilCbWs/QPz8ciFwRB0q6pH8k= +github.com/chromedp/chromedp v0.8.3/go.mod h1:9YfKSJnBNeP77vKecv+DNx2/Tcb+6Gli0d1aZPw/xbk= github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -40,14 +35,12 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.1.0 h1:7RFti/xnNkMJnrK7D1yQ/iCIB5OrrY/54/H930kIbHA= github.com/gobwas/ws v1.1.0/go.mod h1:nzvNcVha5eUziGrbxFCo6qFIojQHjJV5cLYIbezhfL0= -github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs= github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -58,16 +51,13 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -78,7 +68,6 @@ github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxA github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/orisano/pixelmatch v0.0.0-20210112091706-4fa4c7ba91d5 h1:1SoBaSPudixRecmlHXb/GxmaD3fLMtHIDN13QujwQuc= -github.com/orisano/pixelmatch v0.0.0-20210112091706-4fa4c7ba91d5/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/oxffaa/gopher-parse-sitemap v0.0.0-20191021113419-005d2eb1def4 h1:2vmb32OdDhjZf2ETGDlr9n8RYXx7c+jXPxMiPbwnA+8= github.com/oxffaa/gopher-parse-sitemap v0.0.0-20191021113419-005d2eb1def4/go.mod h1:2JQx4jDHmWrbABvpOayg/+OTU6ehN0IyK2EHzceXpJo= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -90,12 +79,10 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA= github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= -github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= -github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -110,8 +97,6 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -122,17 +107,11 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -145,9 +124,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IV golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= @@ -162,10 +140,7 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/configuration/configuration.go b/internal/configuration/configuration.go index 96c70e8..89382fa 100644 --- a/internal/configuration/configuration.go +++ b/internal/configuration/configuration.go @@ -15,12 +15,12 @@ const ( var ( BANNER string = aurora.Sprintf(aurora.BrightBlue(` - _ _ _____ - ___(_) __ _ _ __ __ ___ _| |___ / _ __ -/ __| |/ _`+"`"+` | '__/ _`+"`"+` \ \ /\ / / | |_ \| '__| -\__ \ | (_| | | | (_| |\ V V /| |___) | | -|___/_|\__, |_| \__,_| \_/\_/ |_|____/|_| %s - |___/ + _ _ _____ +| |__ __ _ ___ _ __ __ ___ _| |___ / _ __ +| '_ \ / _`+"`"+` |/ __| '__/ _`+"`"+` \ \ /\ / / | |_ \| '__| +| | | | (_| | (__| | | (_| |\ V V /| |___) | | +|_| |_|\__, |\___|_| \__,_| \_/\_/ |_|____/|_| %s + |_| `).Bold(), aurora.BrightRed(VERSION).Bold()) ) diff --git a/internal/crawler/crawler.go b/internal/crawler/crawler.go index 3a41cad..28bcc0b 100755 --- a/internal/crawler/crawler.go +++ b/internal/crawler/crawler.go @@ -16,8 +16,8 @@ import ( "github.com/gocolly/colly/v2" "github.com/gocolly/colly/v2/debug" "github.com/gocolly/colly/v2/extensions" - "github.com/signedsecurity/sigrawl3r/internal/browser" - "github.com/signedsecurity/sigrawl3r/internal/configuration" + "github.com/hueristiq/hqcrawl3r/internal/browser" + "github.com/hueristiq/hqcrawl3r/internal/configuration" ) type Crawler struct {