diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ae1ecbc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.github +bin +.gitignore +.golangci.yaml +.goreleaser.yaml +CONTRIBUTING.md +LICENSE +README.md \ No newline at end of file diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 0b637c1..00fec91 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -27,7 +27,7 @@ jobs: name: Set up Go uses: actions/setup-go@v5 with: - go-version: '>=1.21' + go-version: '>=1.23' - name: Checkout the repository uses: actions/checkout@v4 diff --git a/.github/workflows/dockerhub-push.yaml b/.github/workflows/dockerhub-push.yaml new file mode 100644 index 0000000..b671f6a --- /dev/null +++ b/.github/workflows/dockerhub-push.yaml @@ -0,0 +1,46 @@ +name: 🐋 DockerHub Push + +on: + workflow_run: + workflows: ["🎉 Release"] + types: + - completed + workflow_dispatch: + +jobs: + push: + runs-on: ubuntu-latest-16-cores + steps: + - + name: Checkout + uses: actions/checkout@v4 + + - + name: Get Github tag + id: meta + run: | + curl --silent "https://api.github.com/repos/hueristiq/xsubfind3r/releases/latest" | jq -r .tag_name | xargs -I {} echo TAG={} >> $GITHUB_OUTPUT + + - + name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - + name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - + name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64,linux/arm + push: true + tags: hueristiq/xsubfind3r:latest,hueristiq/xsubfind3r:${{ steps.meta.outputs.TAG }} \ No newline at end of file diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 24a0512..10ff608 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -27,7 +27,7 @@ jobs: name: Set up Go uses: actions/setup-go@v5 with: - go-version: '>=1.21' + go-version: '>=1.23' cache: false - name: Checkout the repository @@ -36,7 +36,7 @@ jobs: fetch-depth: 0 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v6 with: version: v1.54.2 args: --timeout 5m diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3ce6366..ebfb12f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ jobs: name: Set up Go uses: actions/setup-go@v5 with: - go-version: '>=1.21' + go-version: '>=1.23' - name: Checkout the repository uses: actions/checkout@v4 @@ -24,7 +24,7 @@ jobs: fetch-depth: 0 - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v5 + uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser version: latest diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ea1e960 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,59 @@ +# Use the official Golang image version 1.23 with the Alpine distribution as the base image for the build stage. +# This multi-stage build starts with the "build-stage" stage where the Go application will be compiled. +FROM golang:1.23.1-alpine3.20 AS build-stage + +# Perform system updates and install necessary packages. +# - `apk --no-cache update`: Updates the Alpine package repository without caching index files. +# - `apk --no-cache upgrade`: Upgrades all installed packages to the latest available versions. +# - `apk --no-cache add`: Installs additional required packages: +# - `ca-certificates`: For managing CA certificates for secure communication. +# - `curl`: For making HTTP requests (can be used to download files or for health checks). +# - `gcc` and `g++`: The GNU Compiler Collection used for compiling C and C++ code, essential for building Go applications. +# - `git`: Required for downloading Go modules that reference external repositories. +# - `make`: Utility for automating build processes and running the `Makefile`. +RUN < [!NOTE] > Windows users can follow [How to: Add Tool Locations to the PATH Environment Variable](https://msdn.microsoft.com/en-us/library/office/ee537574(v=office.14).aspx) in order to add `xsubfind3r` to their `PATH`. +### Install on Docker (With Docker Installed) + +If you have Docker installed, you can use `xsubfind3r` using it's image: + +* Pull the docker image using: + + ```bash + docker pull hueristiq/xsubfind3r:latest + ``` + +* Run `xsubfind3r` using the image: + + ```bash + docker run --rm hueristiq/xsubfind3r:latest -h + ``` + ### Install source (With Go Installed) -Before you install from source, you need to make sure that Go is installed on your system. You can install Go by following the official instructions for your operating system. For this, we will assume that Go is already installed. +Before you install from source, you need to make sure that Go is installed on your system. You can install Go by following the [official instructions](https://go.dev/doc/install) for your operating system. For this, we will assume that Go is already installed. #### `go install ...` @@ -79,7 +97,7 @@ Before you install from source, you need to make sure that Go is installed on yo go install -v github.com/hueristiq/xsubfind3r/cmd/xsubfind3r@latest ``` -#### `go build ...` the development Version +#### `go build ...` * Clone the repository @@ -99,8 +117,8 @@ go install -v github.com/hueristiq/xsubfind3r/cmd/xsubfind3r@latest ```bash sudo mv xsubfind3r /usr/local/bin/ ``` - > [!NOTE] - > Windows users can follow [How to: Add Tool Locations to the PATH Environment Variable](https://msdn.microsoft.com/en-us/library/office/ee537574(v=office.14).aspx) in order to add `xsubfind3r` to their `PATH`. + + Windows users can follow [How to: Add Tool Locations to the PATH Environment Variable](https://msdn.microsoft.com/en-us/library/office/ee537574(v=office.14).aspx) in order to add `xsubfind3r` to their `PATH`. > [!CAUTION] @@ -108,7 +126,7 @@ go install -v github.com/hueristiq/xsubfind3r/cmd/xsubfind3r@latest ## Post Installation -`xsubfind3r` will work right after [installation](#installation). However, **[BeVigil](https://bevigil.com)**, **[BufferOver](https://tls.bufferover.run/)**, **[BuiltWith](https://api.builtwith.com/domain-api)**, **[Certspotter](https://sslmate.com/ct_search_api/)**, **[Chaos](https://chaos.hueristiq.io/#/)**, **[Fullhunt](https://fullhunt.io/)**, **[Github](https://github.com)**, **[Intelligence X](https://intelx.io)**, **[LeakIX](https://leakix.net)** and **[Shodan](https://shodan.io/)** require API keys to work, **[URLScan](https://urlscan.io)** supports API key but not required. The API keys are stored in the `$HOME/.config/xsubfind3r/config.yaml` file - created upon first run - and uses the YAML format. Multiple API keys can be specified for each of these source from which one of them will be used. +`xsubfind3r` will work right after [installation](#installation). However, **[BeVigil](https://bevigil.com)**, **[BufferOver](https://tls.bufferover.run/)**, **[BuiltWith](https://api.builtwith.com/domain-api)**, **[Certspotter](https://sslmate.com/ct_search_api/)**, **[Chaos](https://chaos.hueristiq.io/#/)**, **[Fullhunt](https://fullhunt.io/)**, **[Github](https://github.com)**, **[Intelligence X](https://intelx.io)**, **[LeakIX](https://leakix.net)** and **[Shodan](https://shodan.io/)** require API keys to work, **[URLScan](https://urlscan.io)** supports API key but not required. The API keys are stored in the `$HOME/.config/xsubfind3r/config.yaml` file, created upon first run, and uses the YAML format, or supplied via environment variables. Multiple API keys can be specified for each of these source from which one of them will be used. Example `config.yaml`: @@ -116,7 +134,7 @@ Example `config.yaml`: > The keys/tokens below are invalid and used as examples, use your own keys/tokens! ```yaml -version: 0.7.0 +version: 0.8.0 sources: - alienvault - anubis @@ -159,15 +177,37 @@ keys: - d4c85d34-e425-446e-d4ab-f5a3412acbe8 ``` +> [!NOTE] +> To run `xsubfind3r` using docker with a local config file: +> +>```bash +>docker run --rm -v $HOME/.config/xsubfind3r:/root/.config/xsubfind3r hueristiq/xsubfind3r:latest -h +>``` + +Example environmet variables: + +```bash +XURLFIND3R_KEYS_BEVIGIL=awA5nvpKU3N8ygkZ +XURLFIND3R_KEYS_BUFFEROVER=COx9GBnhz63hcF1hlBtLb4KAdlzJly1d8xeovTjK +XURLFIND3R_KEYS_BUILTWITH=7fcbaec4-dc49-472c-b837-3896cb255823 +XURLFIND3R_KEYS_CHAOS=d23a554bbc1aabb208c9acfbd2dd41ce7fc9db39asdsd54bbc1aabb208c9acfb +XURLFIND3R_KEYS_FULLHUNT=0d9652ce-516c-4315-b589-9b241ee6dc24 +XURLFIND3R_KEYS_GITHUB=asdsd54bbc1aabb208c9acfbd2dd41ce7fc9db39,d23a554bbc1aabb208c9acfbd2dd41ce7fc9db39 +XURLFIND3R_KEYS_INTELX=2.intelx.io:00000000-0000-0000-0000-000000000000 +XURLFIND3R_KEYS_LEAKIX=xhDsgKejYTUWVNLn9R6f8afhsG6h6KM69lqEBoMJbfcvDk1v +XURLFIND3R_KEYS_SHODAN=AAAAClP1bJJSRMEYJazgwhJKrggRwKA +XURLFIND3R_KEYS_URLSCAN=d4c85d34-e425-446e-d4ab-f5a3412acbe8 +``` + ## Usage -To display help message for `xsubfind3r` use the `-h` flag: +To start using `xsubfind3r`, open your terminal and run the following command for a list of options: ```bash xsubfind3r -h ``` -help message: +Here's what the help message looks like: ```text _ __ _ _ _____ @@ -175,9 +215,7 @@ __ _____ _ _| |__ / _(_)_ __ __| |___ / _ __ \ \/ / __| | | | '_ \| |_| | '_ \ / _` | |_ \| '__| > <\__ \ |_| | |_) | _| | | | | (_| |___) | | /_/\_\___/\__,_|_.__/|_| |_|_| |_|\__,_|____/|_| - v0.7.0 - - with <3 by Hueristiq Open Source + v0.8.0 USAGE: xsubfind3r [OPTIONS] @@ -205,24 +243,32 @@ OUTPUT: -v, --verbose bool display verbose output ``` +For example, to discover subdomains for `example.com`: + +```bash +xsubfind3r -d example.com +``` + +You can also use multiple domains by separating them with commas or providing a list from a file. + ## Contributing -[Issues](https://github.com/hueristiq/xsubfind3r/issues) and [Pull Requests](https://github.com/hueristiq/xsubfind3r/pulls) are welcome! **Check out the [contribution guidelines](https://github.com/hueristiq/xsubfind3r/blob/master/CONTRIBUTING.md).** +We welcome contributions! Feel free to submit [Pull Requests](https://github.com/hueristiq/xsubfind3r/pulls) or report [Issues](https://github.com/hueristiq/xsubfind3r/issues). For more details, check out the [contribution guidelines](https://github.com/hueristiq/xsubfind3r/blob/master/CONTRIBUTING.md). ## Licensing -This utility is distributed under the [MIT license](https://github.com/hueristiq/xsubfind3r/blob/master/LICENSE). +This utility is licensed under the [MIT license](https://opensource.org/license/mit). You are free to use, modify, and distribute it, as long as you follow the terms of the license. You can find the full license text in the repository - [Full MIT license text](https://github.com/hueristiq/xsubfind3r/blob/master/LICENSE). ## Credits ### Contributors -Thanks to the amazing [contributors](https://github.com/hueristiq/xsubfind3r/graphs/contributors) for keeping this project alive. +A huge thanks to all the contributors who have helped make `xsubfind3r` what it is today! [![contributors](https://contrib.rocks/image?repo=hueristiq/xsubfind3r&max=500)](https://github.com/hueristiq/xsubfind3r/graphs/contributors) ### Similar Projects -Thanks to similar open source projects - check them out, may fit in your workflow. +If you're interested in more utilities like this, check out: [subfinder](https://github.com/projectdiscovery/subfinder) ◇ [assetfinder](https://github.com/tomnomnom/assetfinder) \ No newline at end of file diff --git a/cmd/xsubfind3r/main.go b/cmd/xsubfind3r/main.go index 16ac260..a3676c3 100644 --- a/cmd/xsubfind3r/main.go +++ b/cmd/xsubfind3r/main.go @@ -3,6 +3,7 @@ package main import ( "bufio" "fmt" + "log" "os" "path/filepath" "reflect" @@ -17,6 +18,7 @@ import ( "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" "github.com/logrusorgru/aurora/v3" "github.com/spf13/pflag" + "github.com/spf13/viper" ) var ( @@ -45,7 +47,7 @@ func init() { pflag.StringSliceVarP(&sourcesToExclude, "exclude-sources", "e", []string{}, "") pflag.BoolVar(&monochrome, "monochrome", false, "") pflag.StringVarP(&output, "output", "o", "", "") - pflag.StringVarP(&outputDirectory, "outputDirectory", "O", "", "") + pflag.StringVarP(&outputDirectory, "output-directory", "O", "", "") pflag.BoolVarP(&silent, "silent", "s", false, "") pflag.BoolVarP(&verbose, "verbose", "v", false, "") @@ -84,6 +86,20 @@ func init() { pflag.Parse() + // Initialize configuration management (...with viper) + if err := configuration.CreateUpdate(configurationFilePath); err != nil { + hqgolog.Fatal().Msg(err.Error()) + } + + viper.SetConfigFile(configurationFilePath) + viper.AutomaticEnv() + viper.SetEnvPrefix("XURLFIND3R") + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + + if err := viper.ReadInConfig(); err != nil { + log.Fatalln(err) + } + // Initialize logger (hqgolog) hqgolog.DefaultLogger.SetMaxLevel(levels.LevelInfo) @@ -95,11 +111,6 @@ func init() { Colorize: !monochrome, })) - // Create or Update configuration - if err := configuration.CreateUpdate(configurationFilePath); err != nil { - hqgolog.Fatal().Msg(err.Error()) - } - au = aurora.NewAurora(!monochrome) } @@ -111,30 +122,28 @@ func main() { var err error - var config configuration.Configuration + var cfg *configuration.Configuration - // read in configuration. - config, err = configuration.Read(configurationFilePath) - if err != nil { + if err := viper.Unmarshal(&cfg); err != nil { hqgolog.Fatal().Msg(err.Error()) } // if `--sources`: List suported sources & exit. if listSources { hqgolog.Print().Msg("") - hqgolog.Info().Msgf("listing, %v, current supported sources.", au.Underline(strconv.Itoa(len(config.Sources))).Bold()) + hqgolog.Info().Msgf("listing, %v, current supported sources.", au.Underline(strconv.Itoa(len(cfg.Sources))).Bold()) hqgolog.Info().Msgf("sources marked with %v take in key(s) or token(s).", au.Underline("*").Bold()) hqgolog.Print().Msg("") needsKey := make(map[string]interface{}) - keysElem := reflect.ValueOf(&config.Keys).Elem() + keysElem := reflect.ValueOf(&cfg.Keys).Elem() for i := 0; i < keysElem.NumField(); i++ { needsKey[strings.ToLower(keysElem.Type().Field(i).Name)] = keysElem.Field(i).Interface() } - for index := range config.Sources { - source := config.Sources[index] + for index := range cfg.Sources { + source := cfg.Sources[index] if _, ok := needsKey[source]; ok { hqgolog.Print().Msgf("> %s *", source) @@ -193,7 +202,7 @@ func main() { options := &scraper.Options{ SourcesToUSe: sourcesToUse, SourcesToExclude: sourcesToExclude, - Keys: config.Keys, + Keys: cfg.Keys, } spr := scraper.New(options) diff --git a/go.mod b/go.mod index e49ecb6..8634234 100644 --- a/go.mod +++ b/go.mod @@ -1,24 +1,39 @@ module github.com/hueristiq/xsubfind3r -go 1.21 +go 1.23 require ( - dario.cat/mergo v1.0.0 + dario.cat/mergo v1.0.1 github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead + github.com/hueristiq/hqgolimit v0.0.0-20230623113203-3e14552a97f8 github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f github.com/logrusorgru/aurora/v3 v3.0.0 - github.com/spf13/cast v1.6.0 + github.com/spf13/cast v1.7.0 github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.19.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440 // indirect - golang.org/x/net v0.19.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect - golang.org/x/text v0.14.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/sagikazarmark/locafero v0.6.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/term v0.24.0 // indirect + golang.org/x/text v0.18.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index b3d4f5b..617c0dc 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,22 @@ -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead h1:Iep2G2h3hSwc7w0qr1iVVAptgXqjn7DRXVQ33luPmhk= github.com/hueristiq/hqgohttp v0.0.0-20231024010818-fdb48fa4aead/go.mod h1:Faf/mOhyfNnLIfhoYj2vfPrjt0nKBr4WaU+OQ0C7r6U= +github.com/hueristiq/hqgolimit v0.0.0-20230623113203-3e14552a97f8 h1:r4ze6pX8H//X4SJEIcn8wHPgAhaGKEaa44lyHh1epXY= +github.com/hueristiq/hqgolimit v0.0.0-20230623113203-3e14552a97f8/go.mod h1:CzhJzxz2rv/NMKNz5b4eKFh1epdcED05YTHT32NFyrI= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f h1:JAgZOIJ+UbkENpRiOTlfg51CW0UNrUkgwLjUGiH+x9g= github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f/go.mod h1:S5J3E3Azva5+JKv67uc+Hh3XwLDvkVYDGjEaMTFrIqg= github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440 h1:dpHAa9c74HgAXkZ2WPd84q2cCiF76eluuSGRw7bk7To= @@ -23,28 +30,52 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4= github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= -github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= +github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/configuration/configuration.go b/internal/configuration/configuration.go index 15b4c85..eaa6d7d 100644 --- a/internal/configuration/configuration.go +++ b/internal/configuration/configuration.go @@ -46,8 +46,8 @@ func (cfg *Configuration) Write(path string) (err error) { } const ( - NAME string = "xsubfind3r" - VERSION string = "0.7.0" + NAME = "xsubfind3r" + VERSION = "0.8.0" ) var ( @@ -58,11 +58,8 @@ __ _____ _ _| |__ / _(_)_ __ __| |___ / _ __ \ \/ / __| | | | '_ \| |_| | '_ \ / _`+"`"+` | |_ \| '__| > <\__ \ |_| | |_) | _| | | | | (_| |___) | | /_/\_\___/\__,_|_.__/|_| |_|_| |_|\__,_|____/|_| - %s - - %s`).Bold(), + %s`).Bold(), aurora.BrightRed("v"+VERSION).Bold(), - aurora.BrightYellow("with <3 by Hueristiq Open Source").Italic(), ) UserDotConfigDirectoryPath = func() (userDotConfig string) { var err error diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go index 020819b..d85c3fd 100644 --- a/pkg/httpclient/client.go +++ b/pkg/httpclient/client.go @@ -16,6 +16,7 @@ var client *hqgohttp.Client func init() { options := hqgohttp.DefaultOptionsSpraying + options.Timeout = 0 client, _ = hqgohttp.New(options) } diff --git a/pkg/scraper/sources/wayback/wayback.go b/pkg/scraper/sources/wayback/wayback.go index 62432ad..7bf47df 100644 --- a/pkg/scraper/sources/wayback/wayback.go +++ b/pkg/scraper/sources/wayback/wayback.go @@ -6,6 +6,7 @@ import ( "net/http" "regexp" + "github.com/hueristiq/hqgolimit" "github.com/hueristiq/xsubfind3r/pkg/extractor" "github.com/hueristiq/xsubfind3r/pkg/httpclient" "github.com/hueristiq/xsubfind3r/pkg/scraper/sources" @@ -13,6 +14,10 @@ import ( type Source struct{} +var limiter = hqgolimit.New(&hqgolimit.Options{ + RequestsPerMinute: 40, +}) + func (source *Source) Run(_ *sources.Configuration, domain string) <-chan sources.Result { results := make(chan sources.Result) @@ -21,43 +26,6 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source var err error - getPagesReqURL := fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey&showNumPages=true", domain) - - var getPagesRes *http.Response - - getPagesRes, err = httpclient.SimpleGet(getPagesReqURL) - if err != nil { - result := sources.Result{ - Type: sources.Error, - Source: source.Name(), - Error: err, - } - - results <- result - - httpclient.DiscardResponse(getPagesRes) - - return - } - - var pages uint - - if err = json.NewDecoder(getPagesRes.Body).Decode(&pages); err != nil { - result := sources.Result{ - Type: sources.Error, - Source: source.Name(), - Error: err, - } - - results <- result - - getPagesRes.Body.Close() - - return - } - - getPagesRes.Body.Close() - var regex *regexp.Regexp regex, err = extractor.New(domain) @@ -73,8 +41,10 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source return } - for page := uint(0); page < pages; page++ { - getURLsReqURL := fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&collapse=urlkey&fl=original&page=%d", domain, page) + for page := uint(0); ; page++ { + limiter.Wait() + + getURLsReqURL := fmt.Sprintf("https://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&collapse=urlkey&fl=original&pageSize=100&page=%d", domain, page) var getURLsRes *http.Response @@ -112,7 +82,7 @@ func (source *Source) Run(_ *sources.Configuration, domain string) <-chan source getURLsRes.Body.Close() // check if there's results, wayback's pagination response - // is not always correct when using a filter + // is not always correct if len(getURLsResData) == 0 { break }