From 2812513a2b043de9f46a59dec68bd52b09499410 Mon Sep 17 00:00:00 2001 From: "Alex Munene (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Thu, 1 Feb 2024 19:38:06 +0300 Subject: [PATCH 01/26] chore(*): - --- README.md | 4 ++-- go.mod | 6 +++--- go.sum | 6 ++++++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d1cdd36..57bf2ac 100644 --- a/README.md +++ b/README.md @@ -99,8 +99,8 @@ go install -v github.com/hueristiq/xsubfind3r/cmd/xsubfind3r@latest ```bash sudo mv xsubfind3r /usr/local/bin/ ``` - > [!NOTE] - > Windows users can follow [How to: Add Tool Locations to the PATH Environment Variable](https://msdn.microsoft.com/en-us/library/office/ee537574(v=office.14).aspx) in order to add `xsubfind3r` to their `PATH`. + + Windows users can follow [How to: Add Tool Locations to the PATH Environment Variable](https://msdn.microsoft.com/en-us/library/office/ee537574(v=office.14).aspx) in order to add `xsubfind3r` to their `PATH`. > [!CAUTION] diff --git a/go.mod b/go.mod index e49ecb6..3faf026 100644 --- a/go.mod +++ b/go.mod @@ -16,9 +16,9 @@ require ( require ( github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440 // indirect - golang.org/x/net v0.19.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index b3d4f5b..e0a72e0 100644 --- a/go.sum +++ b/go.sum @@ -37,10 +37,16 @@ github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJ github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 63db00f4e2986db3cd10690d2e6ced8d140a0cc1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Feb 2024 05:06:34 +0000 Subject: [PATCH 02/26] chore(deps): bump golangci/golangci-lint-action from 3 to 4 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 3 to 4. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v3...v4) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/lint-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 24a0512..e3b1b98 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -36,7 +36,7 @@ jobs: fetch-depth: 0 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v4 with: version: v1.54.2 args: --timeout 5m From de2c45ff2986e7ed5915a61130b79971b9aa0016 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 05:41:34 +0000 Subject: [PATCH 03/26] chore(deps): bump golangci/golangci-lint-action from 4 to 6 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 4 to 6. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v4...v6) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/lint-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index e3b1b98..cf7c209 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -36,7 +36,7 @@ jobs: fetch-depth: 0 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v4 + uses: golangci/golangci-lint-action@v6 with: version: v1.54.2 args: --timeout 5m From 21e239378fee991c941fe2688b976c9ffde5c0da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 05:05:00 +0000 Subject: [PATCH 04/26] chore(deps): bump goreleaser/goreleaser-action from 5 to 6 Bumps [goreleaser/goreleaser-action](https://github.com/goreleaser/goreleaser-action) from 5 to 6. - [Release notes](https://github.com/goreleaser/goreleaser-action/releases) - [Commits](https://github.com/goreleaser/goreleaser-action/compare/v5...v6) --- updated-dependencies: - dependency-name: goreleaser/goreleaser-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3ce6366..c09fdc2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: fetch-depth: 0 - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v5 + uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser version: latest From 62e2227d2d54be1df905ab4f5c9c30144abbd873 Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Sun, 22 Sep 2024 13:30:14 +0300 Subject: [PATCH 05/26] chore(*): - --- go.mod | 12 ++++++------ go.sum | 30 ++++++++++++------------------ 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 3faf026..b3a4823 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module github.com/hueristiq/xsubfind3r go 1.21 require ( - dario.cat/mergo v1.0.0 + dario.cat/mergo v1.0.1 github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f github.com/logrusorgru/aurora/v3 v3.0.0 - github.com/spf13/cast v1.6.0 + github.com/spf13/cast v1.7.0 github.com/spf13/pflag v1.0.5 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 gopkg.in/yaml.v3 v3.0.1 @@ -16,9 +16,9 @@ require ( require ( github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440 // indirect - golang.org/x/net v0.20.0 // indirect - golang.org/x/sys v0.16.0 // indirect - golang.org/x/term v0.16.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/term v0.24.0 // indirect + golang.org/x/text v0.18.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index e0a72e0..8022d28 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -27,28 +27,22 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= -github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 13a5d3419f20c6228f154ef7585b1bd5146489d2 Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Sun, 22 Sep 2024 13:37:28 +0300 Subject: [PATCH 06/26] chore(*): - --- README.md | 2 -- internal/configuration/configuration.go | 5 +---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 57bf2ac..ef1f005 100644 --- a/README.md +++ b/README.md @@ -177,8 +177,6 @@ __ _____ _ _| |__ / _(_)_ __ __| |___ / _ __ /_/\_\___/\__,_|_.__/|_| |_|_| |_|\__,_|____/|_| v0.7.0 - with <3 by Hueristiq Open Source - USAGE: xsubfind3r [OPTIONS] diff --git a/internal/configuration/configuration.go b/internal/configuration/configuration.go index 15b4c85..b4aead6 100644 --- a/internal/configuration/configuration.go +++ b/internal/configuration/configuration.go @@ -58,11 +58,8 @@ __ _____ _ _| |__ / _(_)_ __ __| |___ / _ __ \ \/ / __| | | | '_ \| |_| | '_ \ / _`+"`"+` | |_ \| '__| > <\__ \ |_| | |_) | _| | | | | (_| |___) | | /_/\_\___/\__,_|_.__/|_| |_|_| |_|\__,_|____/|_| - %s - - %s`).Bold(), + %s`).Bold(), aurora.BrightRed("v"+VERSION).Bold(), - aurora.BrightYellow("with <3 by Hueristiq Open Source").Italic(), ) UserDotConfigDirectoryPath = func() (userDotConfig string) { var err error From c9ff4edfff7fae0e64882802d2485eda8fa1886f Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Sun, 22 Sep 2024 13:38:47 +0300 Subject: [PATCH 07/26] chore(*): - --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5335b0c..1703acf 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ GOFLAGS := -v LDFLAGS := -s -w ifneq ($(shell go env GOOS),darwin) -LDFLAGS := -extldflags "-static" + LDFLAGS := -extldflags "-static" endif GOLANGCILINTCMD=golangci-lint From 39b8639e617f22b53cd4976b3cd332a41bbe0ae9 Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Sun, 22 Sep 2024 18:30:20 +0300 Subject: [PATCH 08/26] build(*): Add setup on Docker option --- .github/workflows/dockerhub-push.yaml | 46 +++++++++ Dockerfile | 59 +++++++++++ Makefile | 128 +++++++++++++++++++++--- README.md | 24 +++++ internal/configuration/configuration.go | 4 +- 5 files changed, 245 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/dockerhub-push.yaml create mode 100644 Dockerfile diff --git a/.github/workflows/dockerhub-push.yaml b/.github/workflows/dockerhub-push.yaml new file mode 100644 index 0000000..2c101fe --- /dev/null +++ b/.github/workflows/dockerhub-push.yaml @@ -0,0 +1,46 @@ +name: 🌥 Docker Push + +on: + workflow_run: + workflows: ["🎉 Release"] + types: + - completed + workflow_dispatch: + +jobs: + push: + runs-on: ubuntu-latest-16-cores + steps: + - + name: Checkout + uses: actions/checkout@v3 + + - + name: Get Github tag + id: meta + run: | + curl --silent "https://api.github.com/repos/hueristiq/xsubfind3r/releases/latest" | jq -r .tag_name | xargs -I {} echo TAG={} >> $GITHUB_OUTPUT + + - + name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - + name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - + name: Build and push + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64,linux/arm64,linux/arm + push: true + tags: hueristiq/xsubfind3r:latest,hueristiq/xsubfind3r:${{ steps.meta.outputs.TAG }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b93a567 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,59 @@ +# Use the official Golang image version 1.21 with the Alpine distribution as the base image for the build stage. +# This multi-stage build starts with the "build-stage" stage where the Go application will be compiled. +FROM golang:1.21-alpine AS build-stage + +# Perform system updates and install necessary packages. +# - `apk --no-cache update`: Updates the Alpine package repository without caching index files. +# - `apk --no-cache upgrade`: Upgrades all installed packages to the latest available versions. +# - `apk --no-cache add`: Installs additional required packages: +# - `ca-certificates`: For managing CA certificates for secure communication. +# - `curl`: For making HTTP requests (can be used to download files or for health checks). +# - `gcc` and `g++`: The GNU Compiler Collection used for compiling C and C++ code, essential for building Go applications. +# - `git`: Required for downloading Go modules that reference external repositories. +# - `make`: Utility for automating build processes and running the `Makefile`. +RUN < [!NOTE] +> Run `xsubfind3r` using the Docker image with a local config file: +> +>```bash +>docker run -v $CONFIG/xsubfind3r:/root/.config/xsubfind3r -t hueristiq/xsubfind3r -h +>``` + ## Usage To display help message for `xsubfind3r` use the `-h` flag: diff --git a/internal/configuration/configuration.go b/internal/configuration/configuration.go index b4aead6..42a8f1b 100644 --- a/internal/configuration/configuration.go +++ b/internal/configuration/configuration.go @@ -46,8 +46,8 @@ func (cfg *Configuration) Write(path string) (err error) { } const ( - NAME string = "xsubfind3r" - VERSION string = "0.7.0" + NAME = "xsubfind3r" + VERSION = "0.7.0" ) var ( From bdde936b50091af86c71d2085de38c727b3bddff Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Sun, 22 Sep 2024 18:34:16 +0300 Subject: [PATCH 09/26] docs(README): - --- README.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 05292f0..52e0057 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ * [Features](#features) * [Installation](#installation) - * [Install on Docker (With Docker Installed)](#install-on-docker-with-docker-installed) * [Install release binaries (Without Go Installed)](#install-release-binaries-without-go-installed) + * [Install on Docker (With Docker Installed)](#install-on-docker-with-docker-installed) * [Install source (With Go Installed)](#install-source-with-go-installed) * [`go install ...`](#go-install) * [`go build ...` the development Version](#go-build--the-development-version) @@ -29,22 +29,6 @@ ## Installation -### Install on Docker (With Docker Installed) - -To install `xsubfind3r` on docker: - -* Pull the docker image using: - - ```bash - docker pull hueristiq/xsubfind3r:latest - ``` - -* Run `xsubfind3r` using the image: - - ```bash - docker run hueristiq/xsubfind3r:latest -h - ``` - ### Install release binaries (Without Go Installed) Visit the [releases page](https://github.com/hueristiq/xsubfind3r/releases) and find the appropriate archive for your operating system and architecture. Download the archive from your browser or copy its URL and retrieve it with `wget` or `curl`: @@ -86,6 +70,22 @@ sudo mv xsubfind3r /usr/local/bin/ > [!NOTE] > Windows users can follow [How to: Add Tool Locations to the PATH Environment Variable](https://msdn.microsoft.com/en-us/library/office/ee537574(v=office.14).aspx) in order to add `xsubfind3r` to their `PATH`. +### Install on Docker (With Docker Installed) + +To install `xsubfind3r` on docker: + +* Pull the docker image using: + + ```bash + docker pull hueristiq/xsubfind3r:latest + ``` + +* Run `xsubfind3r` using the image: + + ```bash + docker run hueristiq/xsubfind3r:latest -h + ``` + ### Install source (With Go Installed) Before you install from source, you need to make sure that Go is installed on your system. You can install Go by following the official instructions for your operating system. For this, we will assume that Go is already installed. From b824b740ac06db307fc810ecf827b09c1384be76 Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Sun, 22 Sep 2024 18:45:03 +0300 Subject: [PATCH 10/26] chore(*): bump go from 1.21 to 1.23 --- .github/workflows/build-test.yml | 2 +- .github/workflows/lint-test.yml | 2 +- .github/workflows/release.yml | 2 +- Dockerfile | 4 ++-- go.mod | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 0b637c1..00fec91 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -27,7 +27,7 @@ jobs: name: Set up Go uses: actions/setup-go@v5 with: - go-version: '>=1.21' + go-version: '>=1.23' - name: Checkout the repository uses: actions/checkout@v4 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index cf7c209..10ff608 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -27,7 +27,7 @@ jobs: name: Set up Go uses: actions/setup-go@v5 with: - go-version: '>=1.21' + go-version: '>=1.23' cache: false - name: Checkout the repository diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c09fdc2..ebfb12f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ jobs: name: Set up Go uses: actions/setup-go@v5 with: - go-version: '>=1.21' + go-version: '>=1.23' - name: Checkout the repository uses: actions/checkout@v4 diff --git a/Dockerfile b/Dockerfile index b93a567..ea1e960 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -# Use the official Golang image version 1.21 with the Alpine distribution as the base image for the build stage. +# Use the official Golang image version 1.23 with the Alpine distribution as the base image for the build stage. # This multi-stage build starts with the "build-stage" stage where the Go application will be compiled. -FROM golang:1.21-alpine AS build-stage +FROM golang:1.23.1-alpine3.20 AS build-stage # Perform system updates and install necessary packages. # - `apk --no-cache update`: Updates the Alpine package repository without caching index files. diff --git a/go.mod b/go.mod index b3a4823..ffb8bee 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/hueristiq/xsubfind3r -go 1.21 +go 1.23 require ( dario.cat/mergo v1.0.1 From da86964cea531c8dd00c889f53093aa048a32d82 Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Sun, 22 Sep 2024 18:46:32 +0300 Subject: [PATCH 11/26] docs(README): - --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 52e0057..9170e0f 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ To install `xsubfind3r` on docker: * Run `xsubfind3r` using the image: ```bash - docker run hueristiq/xsubfind3r:latest -h + docker run --rm hueristiq/xsubfind3r:latest -h ``` ### Install source (With Go Installed) @@ -180,7 +180,7 @@ keys: > Run `xsubfind3r` using the Docker image with a local config file: > >```bash ->docker run -v $CONFIG/xsubfind3r:/root/.config/xsubfind3r -t hueristiq/xsubfind3r -h +>docker run --rm -v $CONFIG/xsubfind3r:/root/.config/xsubfind3r -t hueristiq/xsubfind3r -h >``` ## Usage From 9551dc8dc23835516e0c18797df2f5b1a5304aef Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Sun, 22 Sep 2024 22:54:29 +0300 Subject: [PATCH 12/26] fix(*): outputDirectory -> output-directory --- cmd/xsubfind3r/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/xsubfind3r/main.go b/cmd/xsubfind3r/main.go index 16ac260..9d7d29a 100644 --- a/cmd/xsubfind3r/main.go +++ b/cmd/xsubfind3r/main.go @@ -45,7 +45,7 @@ func init() { pflag.StringSliceVarP(&sourcesToExclude, "exclude-sources", "e", []string{}, "") pflag.BoolVar(&monochrome, "monochrome", false, "") pflag.StringVarP(&output, "output", "o", "", "") - pflag.StringVarP(&outputDirectory, "outputDirectory", "O", "", "") + pflag.StringVarP(&outputDirectory, "output-directory", "O", "", "") pflag.BoolVarP(&silent, "silent", "s", false, "") pflag.BoolVarP(&verbose, "verbose", "v", false, "") From 375c50d2ff3703501fc47ab27974bf944d104fc1 Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Sun, 22 Sep 2024 23:10:25 +0300 Subject: [PATCH 13/26] ci(DockerHub): - --- .github/workflows/dockerhub-push.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dockerhub-push.yaml b/.github/workflows/dockerhub-push.yaml index 2c101fe..1c0eb4a 100644 --- a/.github/workflows/dockerhub-push.yaml +++ b/.github/workflows/dockerhub-push.yaml @@ -1,4 +1,4 @@ -name: 🌥 Docker Push +name: 🐋 DockerHub Push on: workflow_run: @@ -33,8 +33,8 @@ jobs: name: Login to DockerHub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push From 542eca78fff5c90459c20ea03e67ac62b899182a Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Sun, 22 Sep 2024 23:19:56 +0300 Subject: [PATCH 14/26] docs(README): - --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9170e0f..c1bc14b 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ keys: > Run `xsubfind3r` using the Docker image with a local config file: > >```bash ->docker run --rm -v $CONFIG/xsubfind3r:/root/.config/xsubfind3r -t hueristiq/xsubfind3r -h +>docker run --rm -v $HOME/.config/xsubfind3r:/root/.config/xsubfind3r hueristiq/xsubfind3r:latest -h >``` ## Usage From 223f58f22604acbd89538ccb201d870ca50c2aef Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Sun, 22 Sep 2024 23:40:52 +0300 Subject: [PATCH 15/26] build(Docker): - --- .dockerignore | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ae1ecbc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.github +bin +.gitignore +.golangci.yaml +.goreleaser.yaml +CONTRIBUTING.md +LICENSE +README.md \ No newline at end of file From b92b47a0a6ab4754a878740a4df0ef0c3e6ce46f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 05:16:53 +0000 Subject: [PATCH 16/26] chore(deps): bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/dockerhub-push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yaml b/.github/workflows/dockerhub-push.yaml index 1c0eb4a..f2eefe0 100644 --- a/.github/workflows/dockerhub-push.yaml +++ b/.github/workflows/dockerhub-push.yaml @@ -13,7 +13,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get Github tag From 2513d48c5af17943dca3e706e49478ee4ffab796 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 05:16:55 +0000 Subject: [PATCH 17/26] chore(deps): bump docker/build-push-action from 4 to 6 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4 to 6. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v4...v6) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/dockerhub-push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yaml b/.github/workflows/dockerhub-push.yaml index 1c0eb4a..a88fd3f 100644 --- a/.github/workflows/dockerhub-push.yaml +++ b/.github/workflows/dockerhub-push.yaml @@ -38,7 +38,7 @@ jobs: - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64,linux/arm64,linux/arm From 0ff26117f090ac3a8892cd21d96371928a490755 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 05:16:57 +0000 Subject: [PATCH 18/26] chore(deps): bump docker/login-action from 2 to 3 Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/dockerhub-push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yaml b/.github/workflows/dockerhub-push.yaml index 1c0eb4a..8dbc028 100644 --- a/.github/workflows/dockerhub-push.yaml +++ b/.github/workflows/dockerhub-push.yaml @@ -31,7 +31,7 @@ jobs: - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} From 8ee31018707580122774ac12511a77484da97034 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 05:16:58 +0000 Subject: [PATCH 19/26] chore(deps): bump docker/setup-qemu-action from 2 to 3 Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/dockerhub-push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yaml b/.github/workflows/dockerhub-push.yaml index 1c0eb4a..27ceded 100644 --- a/.github/workflows/dockerhub-push.yaml +++ b/.github/workflows/dockerhub-push.yaml @@ -23,7 +23,7 @@ jobs: - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx From 3cb81feac65527596e187830c1483a1af00f4bab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 05:17:00 +0000 Subject: [PATCH 20/26] chore(deps): bump docker/setup-buildx-action from 2 to 3 Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/dockerhub-push.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dockerhub-push.yaml b/.github/workflows/dockerhub-push.yaml index 1c0eb4a..f54620b 100644 --- a/.github/workflows/dockerhub-push.yaml +++ b/.github/workflows/dockerhub-push.yaml @@ -27,7 +27,7 @@ jobs: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub From 54d5aaa17e10ce9b3c7893925cdde23101880cdb Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 23 Sep 2024 19:34:00 +0300 Subject: [PATCH 21/26] fix(wayback): Pagination error Remove the number of pages checking it errors out with 400 status code. Getting subdomains form wayback is working but there some issues with pagination not working as expected. --- pkg/httpclient/client.go | 1 + pkg/scraper/sources/wayback/wayback.go | 43 ++------------------------ 2 files changed, 4 insertions(+), 40 deletions(-) diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go index 020819b..d85c3fd 100644 --- a/pkg/httpclient/client.go +++ b/pkg/httpclient/client.go @@ -16,6 +16,7 @@ var client *hqgohttp.Client func init() { options := hqgohttp.DefaultOptionsSpraying + options.Timeout = 0 client, _ = hqgohttp.New(options) } diff --git a/pkg/scraper/sources/wayback/wayback.go b/pkg/scraper/sources/wayback/wayback.go index 62432ad..63fb170 100644 --- a/pkg/scraper/sources/wayback/wayback.go +++ b/pkg/scraper/sources/wayback/wayback.go @@ -21,43 +21,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source var err error - getPagesReqURL := fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey&showNumPages=true", domain) - - var getPagesRes *http.Response - - getPagesRes, err = httpclient.SimpleGet(getPagesReqURL) - if err != nil { - result := sources.Result{ - Type: sources.Error, - Source: source.Name(), - Error: err, - } - - results <- result - - httpclient.DiscardResponse(getPagesRes) - - return - } - - var pages uint - - if err = json.NewDecoder(getPagesRes.Body).Decode(&pages); err != nil { - result := sources.Result{ - Type: sources.Error, - Source: source.Name(), - Error: err, - } - - results <- result - - getPagesRes.Body.Close() - - return - } - - getPagesRes.Body.Close() - var regex *regexp.Regexp regex, err = extractor.New(domain) @@ -73,8 +36,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source return } - for page := uint(0); page < pages; page++ { - getURLsReqURL := fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&collapse=urlkey&fl=original&page=%d", domain, page) + for page := uint(0); ; page++ { + getURLsReqURL := fmt.Sprintf("https://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&collapse=urlkey&fl=original&pageSize=100&page=%d", domain, page) var getURLsRes *http.Response @@ -112,7 +75,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source getURLsRes.Body.Close() // check if there's results, wayback's pagination response - // is not always correct when using a filter + // is not always correct if len(getURLsResData) == 0 { break } From d3cb2af2db7747727a76b55b78e3843835f2752b Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 23 Sep 2024 20:13:46 +0300 Subject: [PATCH 22/26] feat(wayback): Add rate limiting --- go.mod | 1 + go.sum | 2 ++ pkg/scraper/sources/wayback/wayback.go | 7 +++++++ 3 files changed, 10 insertions(+) diff --git a/go.mod b/go.mod index ffb8bee..d6ed345 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23 require ( dario.cat/mergo v1.0.1 github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead + github.com/hueristiq/hqgolimit v0.0.0-20230623113203-3e14552a97f8 github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f github.com/logrusorgru/aurora/v3 v3.0.0 github.com/spf13/cast v1.7.0 diff --git a/go.sum b/go.sum index 8022d28..e231645 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead h1:Iep2G2h3hSwc7w0qr1iVVAptgXqjn7DRXVQ33luPmhk= github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead/go.mod h1:Faf/mOhyfNnLIfhoYj2vfPrjt0nKBr4WaU+OQ0C7r6U= +github.com/hueristiq/hqgolimit v0.0.0-20230623113203-3e14552a97f8 h1:r4ze6pX8H//X4SJEIcn8wHPgAhaGKEaa44lyHh1epXY= +github.com/hueristiq/hqgolimit v0.0.0-20230623113203-3e14552a97f8/go.mod h1:CzhJzxz2rv/NMKNz5b4eKFh1epdcED05YTHT32NFyrI= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f h1:JAgZOIJ+UbkENpRiOTlfg51CW0UNrUkgwLjUGiH+x9g= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f/go.mod h1:S5J3E3Azva5+JKv67uc+Hh3XwLDvkVYDGjEaMTFrIqg= github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440 h1:dpHAa9c74HgAXkZ2WPd84q2cCiF76eluuSGRw7bk7To= diff --git a/pkg/scraper/sources/wayback/wayback.go b/pkg/scraper/sources/wayback/wayback.go index 63fb170..7bf47df 100644 --- a/pkg/scraper/sources/wayback/wayback.go +++ b/pkg/scraper/sources/wayback/wayback.go @@ -6,6 +6,7 @@ import ( "net/http" "regexp" + "github.com/hueristiq/hqgolimit" "github.com/hueristiq/xsubfind3r/pkg/extractor" "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" @@ -13,6 +14,10 @@ import ( type Source struct{} +var limiter = hqgolimit.New(&hqgolimit.Options{ + RequestsPerMinute: 40, +}) + func (source *Source) Run(_ *sources.Configuration, domain string) <-chan sources.Result { results := make(chan sources.Result) @@ -37,6 +42,8 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source } for page := uint(0); ; page++ { + limiter.Wait() + getURLsReqURL := fmt.Sprintf("https://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&collapse=urlkey&fl=original&pageSize=100&page=%d", domain, page) var getURLsRes *http.Response From 491bb4fe381f5dcbf33712a1d70f38897a1c0d26 Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 23 Sep 2024 21:33:31 +0300 Subject: [PATCH 23/26] feat(*): Read config from environment variables --- README.md | 19 +++++++++++++++-- cmd/xsubfind3r/main.go | 37 ++++++++++++++++++++------------ go.mod | 15 +++++++++++++ go.sum | 48 +++++++++++++++++++++++++++++++++++++++--- 4 files changed, 100 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index c1bc14b..85be2b0 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ go install -v github.com/hueristiq/xsubfind3r/cmd/xsubfind3r@latest ## Post Installation -`xsubfind3r` will work right after [installation](#installation). However, **[BeVigil](https://bevigil.com)**, **[BufferOver](https://tls.bufferover.run/)**, **[BuiltWith](https://api.builtwith.com/domain-api)**, **[Certspotter](https://sslmate.com/ct_search_api/)**, **[Chaos](https://chaos.hueristiq.io/#/)**, **[Fullhunt](https://fullhunt.io/)**, **[Github](https://github.com)**, **[Intelligence X](https://intelx.io)**, **[LeakIX](https://leakix.net)** and **[Shodan](https://shodan.io/)** require API keys to work, **[URLScan](https://urlscan.io)** supports API key but not required. The API keys are stored in the `$HOME/.config/xsubfind3r/config.yaml` file - created upon first run - and uses the YAML format. Multiple API keys can be specified for each of these source from which one of them will be used. +`xsubfind3r` will work right after [installation](#installation). However, **[BeVigil](https://bevigil.com)**, **[BufferOver](https://tls.bufferover.run/)**, **[BuiltWith](https://api.builtwith.com/domain-api)**, **[Certspotter](https://sslmate.com/ct_search_api/)**, **[Chaos](https://chaos.hueristiq.io/#/)**, **[Fullhunt](https://fullhunt.io/)**, **[Github](https://github.com)**, **[Intelligence X](https://intelx.io)**, **[LeakIX](https://leakix.net)** and **[Shodan](https://shodan.io/)** require API keys to work, **[URLScan](https://urlscan.io)** supports API key but not required. The API keys are stored in the `$HOME/.config/xsubfind3r/config.yaml` file, created upon first run, and uses the YAML format, or supplied via environment variables. Multiple API keys can be specified for each of these source from which one of them will be used. Example `config.yaml`: @@ -177,12 +177,27 @@ keys: ``` > [!NOTE] -> Run `xsubfind3r` using the Docker image with a local config file: +> To run `xsubfind3r` using docker with a local config file: > >```bash >docker run --rm -v $HOME/.config/xsubfind3r:/root/.config/xsubfind3r hueristiq/xsubfind3r:latest -h >``` +Example environmet variables: + +```bash +XURLFIND3R_KEYS_BEVIGIL=awA5nvpKU3N8ygkZ +XURLFIND3R_KEYS_BUFFEROVER=COx9GBnhz63hcF1hlBtLb4KAdlzJly1d8xeovTjK +XURLFIND3R_KEYS_BUILTWITH=7fcbaec4-dc49-472c-b837-3896cb255823 +XURLFIND3R_KEYS_CHAOS=d23a554bbc1aabb208c9acfbd2dd41ce7fc9db39asdsd54bbc1aabb208c9acfb +XURLFIND3R_KEYS_FULLHUNT=0d9652ce-516c-4315-b589-9b241ee6dc24 +XURLFIND3R_KEYS_GITHUB=asdsd54bbc1aabb208c9acfbd2dd41ce7fc9db39,d23a554bbc1aabb208c9acfbd2dd41ce7fc9db39 +XURLFIND3R_KEYS_INTELX=2.intelx.io:00000000-0000-0000-0000-000000000000 +XURLFIND3R_KEYS_LEAKIX=xhDsgKejYTUWVNLn9R6f8afhsG6h6KM69lqEBoMJbfcvDk1v +XURLFIND3R_KEYS_SHODAN=AAAAClP1bJJSRMEYJazgwhJKrggRwKA +XURLFIND3R_KEYS_URLSCAN=d4c85d34-e425-446e-d4ab-f5a3412acbe8 +``` + ## Usage To display help message for `xsubfind3r` use the `-h` flag: diff --git a/cmd/xsubfind3r/main.go b/cmd/xsubfind3r/main.go index 9d7d29a..a3676c3 100644 --- a/cmd/xsubfind3r/main.go +++ b/cmd/xsubfind3r/main.go @@ -3,6 +3,7 @@ package main import ( "bufio" "fmt" + "log" "os" "path/filepath" "reflect" @@ -17,6 +18,7 @@ import ( "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" "github.com/logrusorgru/aurora/v3" "github.com/spf13/pflag" + "github.com/spf13/viper" ) var ( @@ -84,6 +86,20 @@ func init() { pflag.Parse() + // Initialize configuration management (...with viper) + if err := configuration.CreateUpdate(configurationFilePath); err != nil { + hqgolog.Fatal().Msg(err.Error()) + } + + viper.SetConfigFile(configurationFilePath) + viper.AutomaticEnv() + viper.SetEnvPrefix("XURLFIND3R") + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + + if err := viper.ReadInConfig(); err != nil { + log.Fatalln(err) + } + // Initialize logger (hqgolog) hqgolog.DefaultLogger.SetMaxLevel(levels.LevelInfo) @@ -95,11 +111,6 @@ func init() { Colorize: !monochrome, })) - // Create or Update configuration - if err := configuration.CreateUpdate(configurationFilePath); err != nil { - hqgolog.Fatal().Msg(err.Error()) - } - au = aurora.NewAurora(!monochrome) } @@ -111,30 +122,28 @@ func main() { var err error - var config configuration.Configuration + var cfg *configuration.Configuration - // read in configuration. - config, err = configuration.Read(configurationFilePath) - if err != nil { + if err := viper.Unmarshal(&cfg); err != nil { hqgolog.Fatal().Msg(err.Error()) } // if `--sources`: List suported sources & exit. if listSources { hqgolog.Print().Msg("") - hqgolog.Info().Msgf("listing, %v, current supported sources.", au.Underline(strconv.Itoa(len(config.Sources))).Bold()) + hqgolog.Info().Msgf("listing, %v, current supported sources.", au.Underline(strconv.Itoa(len(cfg.Sources))).Bold()) hqgolog.Info().Msgf("sources marked with %v take in key(s) or token(s).", au.Underline("*").Bold()) hqgolog.Print().Msg("") needsKey := make(map[string]interface{}) - keysElem := reflect.ValueOf(&config.Keys).Elem() + keysElem := reflect.ValueOf(&cfg.Keys).Elem() for i := 0; i < keysElem.NumField(); i++ { needsKey[strings.ToLower(keysElem.Type().Field(i).Name)] = keysElem.Field(i).Interface() } - for index := range config.Sources { - source := config.Sources[index] + for index := range cfg.Sources { + source := cfg.Sources[index] if _, ok := needsKey[source]; ok { hqgolog.Print().Msgf("> %s *", source) @@ -193,7 +202,7 @@ func main() { options := &scraper.Options{ SourcesToUSe: sourcesToUse, SourcesToExclude: sourcesToExclude, - Keys: config.Keys, + Keys: cfg.Keys, } spr := scraper.New(options) diff --git a/go.mod b/go.mod index d6ed345..01173fd 100644 --- a/go.mod +++ b/go.mod @@ -10,16 +10,31 @@ require ( github.com/logrusorgru/aurora/v3 v3.0.0 github.com/spf13/cast v1.7.0 github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.19.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/term v0.24.0 // indirect golang.org/x/text v0.18.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index e231645..b9834bb 100644 --- a/go.sum +++ b/go.sum @@ -2,12 +2,18 @@ dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead h1:Iep2G2h3hSwc7w0qr1iVVAptgXqjn7DRXVQ33luPmhk= github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead/go.mod h1:Faf/mOhyfNnLIfhoYj2vfPrjt0nKBr4WaU+OQ0C7r6U= github.com/hueristiq/hqgolimit v0.0.0-20230623113203-3e14552a97f8 h1:r4ze6pX8H//X4SJEIcn8wHPgAhaGKEaa44lyHh1epXY= @@ -25,18 +31,51 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4= github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= @@ -48,5 +87,8 @@ golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 4be195986e22fce8cf077e7a49d358c4485d1b3e Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 23 Sep 2024 21:35:49 +0300 Subject: [PATCH 24/26] chore(*): - --- go.mod | 9 ++++----- go.sum | 33 ++++++++++----------------------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 01173fd..8634234 100644 --- a/go.mod +++ b/go.mod @@ -22,15 +22,14 @@ require ( github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect - github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.9.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/term v0.24.0 // indirect diff --git a/go.sum b/go.sum index b9834bb..617c0dc 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,6 @@ dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -10,8 +9,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead h1:Iep2G2h3hSwc7w0qr1iVVAptgXqjn7DRXVQ33luPmhk= @@ -35,15 +34,14 @@ github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0V github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= -github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= -github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= +github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= @@ -56,26 +54,16 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= @@ -89,6 +77,5 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From abae6033bdaf10e8514420dc7f504b0ead2d4c52 Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 23 Sep 2024 21:59:38 +0300 Subject: [PATCH 25/26] docs(README): Revise the README --- README.md | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 85be2b0..67a8af6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![made with go](https://img.shields.io/badge/made%20with-Go-1E90FF.svg) [![go report card](https://goreportcard.com/badge/github.com/hueristiq/xsubfind3r)](https://goreportcard.com/report/github.com/hueristiq/xsubfind3r) [![release](https://img.shields.io/github/release/hueristiq/xsubfind3r?style=flat&color=1E90FF)](https://github.com/hueristiq/xsubfind3r/releases) [![open issues](https://img.shields.io/github/issues-raw/hueristiq/xsubfind3r.svg?style=flat&color=1E90FF)](https://github.com/hueristiq/xsubfind3r/issues?q=is:issue+is:open) [![closed issues](https://img.shields.io/github/issues-closed-raw/hueristiq/xsubfind3r.svg?style=flat&color=1E90FF)](https://github.com/hueristiq/xsubfind3r/issues?q=is:issue+is:closed) [![license](https://img.shields.io/badge/license-MIT-gray.svg?color=1E90FF)](https://github.com/hueristiq/xsubfind3r/blob/master/LICENSE) ![maintenance](https://img.shields.io/badge/maintained%3F-yes-1E90FF.svg) [![contribution](https://img.shields.io/badge/contributions-welcome-1E90FF.svg)](https://github.com/hueristiq/xsubfind3r/blob/master/CONTRIBUTING.md) -`xsubfind3r` is a command-line interface (CLI) based passive subdomain discovery utility. It is designed to efficiently identify known subdomains of given domains by tapping into a multitude of curated online passive sources. +`xsubfind3r` is a command-line utility designed to help you discover subdomains for a given domain in a simple, efficient way. It works by gathering information from a variety of passive sources, meaning it doesn't interact directly with the target but instead gathers data that is already publicly available. This makes it a powerful tool for security researchers, IT professionals, and anyone who wants to know more about the subdomains associated with a domain. ## Resource @@ -12,7 +12,7 @@ * [Install on Docker (With Docker Installed)](#install-on-docker-with-docker-installed) * [Install source (With Go Installed)](#install-source-with-go-installed) * [`go install ...`](#go-install) - * [`go build ...` the development Version](#go-build--the-development-version) + * [`go build ...`](#go-build) * [Post Installation](#post-installation) * [Usage](#usage) * [Contributing](#contributing) @@ -23,9 +23,10 @@ ## Features -* Fetches subdomains from curated passive sources to maximize results. -* Supports `stdin` and `stdout` for easy integration into workflows. -* Cross-Platform (Windows, Linux & macOS). +* **Wide Coverage:** Fetches subdomains from multiple online passive sources to provide extensive results. +* **Flexible Output:** Supports silent mode for only showing results and offers various output options for saving the results. +* **Easy to Integrate:** Supports `stdin` and `stdout` for easy integration in automated workflows. +* **Cross-Platform:** Works on Windows, Linux, and macOS. ## Installation @@ -72,7 +73,7 @@ sudo mv xsubfind3r /usr/local/bin/ ### Install on Docker (With Docker Installed) -To install `xsubfind3r` on docker: +If you have Docker installed, you can use `xsubfind3r` using it's image: * Pull the docker image using: @@ -88,7 +89,7 @@ To install `xsubfind3r` on docker: ### Install source (With Go Installed) -Before you install from source, you need to make sure that Go is installed on your system. You can install Go by following the official instructions for your operating system. For this, we will assume that Go is already installed. +Before you install from source, you need to make sure that Go is installed on your system. You can install Go by following the [official instructions](https://go.dev/doc/install) for your operating system. For this, we will assume that Go is already installed. #### `go install ...` @@ -96,7 +97,7 @@ Before you install from source, you need to make sure that Go is installed on yo go install -v github.com/hueristiq/xsubfind3r/cmd/xsubfind3r@latest ``` -#### `go build ...` the development Version +#### `go build ...` * Clone the repository @@ -200,13 +201,13 @@ XURLFIND3R_KEYS_URLSCAN=d4c85d34-e425-446e-d4ab-f5a3412acbe8 ## Usage -To display help message for `xsubfind3r` use the `-h` flag: +To start using `xsubfind3r`, open your terminal and run the following command for a list of options: ```bash xsubfind3r -h ``` -help message: +Here's what the help message looks like: ```text _ __ _ _ _____ @@ -242,24 +243,32 @@ OUTPUT: -v, --verbose bool display verbose output ``` +For example, to discover subdomains for `example.com`: + +```bash +xsubfind3r -d example.com +``` + +You can also use multiple domains by separating them with commas or providing a list from a file. + ## Contributing -[Issues](https://github.com/hueristiq/xsubfind3r/issues) and [Pull Requests](https://github.com/hueristiq/xsubfind3r/pulls) are welcome! **Check out the [contribution guidelines](https://github.com/hueristiq/xsubfind3r/blob/master/CONTRIBUTING.md).** +We welcome contributions! Feel free to submit [Pull Requests](https://github.com/hueristiq/xsubfind3r/pulls) or report [Issues](https://github.com/hueristiq/xsubfind3r/issues). For more details, check out the [contribution guidelines](https://github.com/hueristiq/xsubfind3r/blob/master/CONTRIBUTING.md). ## Licensing -This utility is distributed under the [MIT license](https://github.com/hueristiq/xsubfind3r/blob/master/LICENSE). +This utility is licensed under the [MIT license](https://opensource.org/license/mit). You are free to use, modify, and distribute it, as long as you follow the terms of the license. You can find the full license text in the repository - [Full MIT license text](https://github.com/hueristiq/xsubfind3r/blob/master/LICENSE). ## Credits ### Contributors -Thanks to the amazing [contributors](https://github.com/hueristiq/xsubfind3r/graphs/contributors) for keeping this project alive. +A huge thanks to all the contributors who have helped make `xsubfind3r` what it is today! [![contributors](https://contrib.rocks/image?repo=hueristiq/xsubfind3r&max=500)](https://github.com/hueristiq/xsubfind3r/graphs/contributors) ### Similar Projects -Thanks to similar open source projects - check them out, may fit in your workflow. +If you're interested in more utilities like this, check out: [subfinder](https://github.com/projectdiscovery/subfinder) ◇ [assetfinder](https://github.com/tomnomnom/assetfinder) \ No newline at end of file From 220643fbe7461ad43d4319d086636c8591989ed2 Mon Sep 17 00:00:00 2001 From: "Alex (@enenumxela)" <62714471+enenumxela@users.noreply.github.com> Date: Mon, 23 Sep 2024 22:36:59 +0300 Subject: [PATCH 26/26] chore(*): Bump up version to 0.8.0 --- README.md | 4 ++-- internal/configuration/configuration.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 67a8af6..0e982eb 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ Example `config.yaml`: > The keys/tokens below are invalid and used as examples, use your own keys/tokens! ```yaml -version: 0.7.0 +version: 0.8.0 sources: - alienvault - anubis @@ -215,7 +215,7 @@ __ _____ _ _| |__ / _(_)_ __ __| |___ / _ __ \ \/ / __| | | | '_ \| |_| | '_ \ / _` | |_ \| '__| > <\__ \ |_| | |_) | _| | | | | (_| |___) | | /_/\_\___/\__,_|_.__/|_| |_|_| |_|\__,_|____/|_| - v0.7.0 + v0.8.0 USAGE: xsubfind3r [OPTIONS] diff --git a/internal/configuration/configuration.go b/internal/configuration/configuration.go index 42a8f1b..eaa6d7d 100644 --- a/internal/configuration/configuration.go +++ b/internal/configuration/configuration.go @@ -47,7 +47,7 @@ func (cfg *Configuration) Write(path string) (err error) { const ( NAME = "xsubfind3r" - VERSION = "0.7.0" + VERSION = "0.8.0" ) var (