From d6c96a4ee88d0984023789ec1665dbd1080703e6 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto <25837540+dwisiswant0@users.noreply.github.com> Date: Mon, 27 Jan 2025 16:42:34 +0700 Subject: [PATCH 1/3] ci(perf-test): use 16 cores CPU (#5987) * ci(perf-test): use 16 cores CPU Signed-off-by: Dwi Siswanto * ci(perf-test): add `flags` matrix Signed-off-by: Dwi Siswanto * ci(perf-test): add flamegraph action Signed-off-by: Dwi Siswanto * ci(perf-test): debug Signed-off-by: Dwi Siswanto * ci(perf-test): add input `name` for flamegraph action Signed-off-by: Dwi Siswanto * ci(perf-test): run w/o `-race` Since it is irrelevant for the context of perf tests and doesn't show any noticeable difference (ref: https://github.com/projectdiscovery/nuclei/actions/runs/12805564634). Signed-off-by: Dwi Siswanto * misc update --------- Signed-off-by: Dwi Siswanto Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- .github/workflows/perf-test.yaml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/perf-test.yaml b/.github/workflows/perf-test.yaml index 732e7a7de1..f4c327fe04 100644 --- a/.github/workflows/perf-test.yaml +++ b/.github/workflows/perf-test.yaml @@ -10,16 +10,26 @@ jobs: strategy: matrix: count: [50, 100, 150] - runs-on: ubuntu-latest + runs-on: ubuntu-latest-16-cores if: github.repository == 'projectdiscovery/nuclei' env: LIST_FILE: "/tmp/targets-${{ matrix.count }}.txt" + PROFILE_MEM: "/tmp/nuclei-perf-test-${{ matrix.count }}" steps: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go@v1 - run: make verify - name: Generate list run: for i in {1..${{ matrix.count }}}; do echo "https://scanme.sh/?_=${i}" >> "${LIST_FILE}"; done - - run: go run -race . -l "${LIST_FILE}" + - run: NUCLEI_ARGS=host-error-stats go run . -l "${LIST_FILE}" -profile-mem="${PROFILE_MEM}" working-directory: cmd/nuclei/ - + - uses: projectdiscovery/actions/flamegraph@v1 + id: flamegraph + with: + profile: "${{ env.PROFILE_MEM }}.prof" + name: "nuclei-perf-test-${{ matrix.count }}" + continue-on-error: true + - if: ${{ steps.flamegraph.outputs.message == '' }} + run: echo "::notice::${FLAMEGRAPH_URL}" + env: + FLAMEGRAPH_URL: ${{ steps.flamegraph.outputs.url }} From ceab5964b71b3d853eaed80b0e3499610929e483 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 15:12:52 +0530 Subject: [PATCH 2/3] chore(deps): bump the modules group with 13 updates (#6007) Bumps the modules group with 13 updates: | Package | From | To | | --- | --- | --- | | [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) | `0.2.15` | `0.3.0` | | [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) | `0.0.76` | `0.0.77` | | [github.com/projectdiscovery/rawhttp](https://github.com/projectdiscovery/rawhttp) | `0.1.83` | `0.1.84` | | [github.com/projectdiscovery/retryablehttp-go](https://github.com/projectdiscovery/retryablehttp-go) | `1.0.96` | `1.0.97` | | [github.com/projectdiscovery/dsl](https://github.com/projectdiscovery/dsl) | `0.3.12` | `0.3.13` | | [github.com/projectdiscovery/goflags](https://github.com/projectdiscovery/goflags) | `0.1.68` | `0.1.69` | | [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) | `1.1.41` | `1.1.42` | | [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) | `0.0.69` | `0.0.70` | | [github.com/projectdiscovery/tlsx](https://github.com/projectdiscovery/tlsx) | `1.1.8` | `1.1.9` | | [github.com/projectdiscovery/useragent](https://github.com/projectdiscovery/useragent) | `0.0.88` | `0.0.89` | | [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) | `0.4.7` | `0.4.8` | | [github.com/projectdiscovery/wappalyzergo](https://github.com/projectdiscovery/wappalyzergo) | `0.2.12` | `0.2.13` | | [github.com/projectdiscovery/networkpolicy](https://github.com/projectdiscovery/networkpolicy) | `0.1.1` | `0.1.2` | Updates `github.com/projectdiscovery/fastdialer` from 0.2.15 to 0.3.0 - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.2.15...v0.3.0) Updates `github.com/projectdiscovery/hmap` from 0.0.76 to 0.0.77 - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.76...v0.0.77) Updates `github.com/projectdiscovery/rawhttp` from 0.1.83 to 0.1.84 - [Release notes](https://github.com/projectdiscovery/rawhttp/releases) - [Commits](https://github.com/projectdiscovery/rawhttp/compare/v0.1.83...v0.1.84) Updates `github.com/projectdiscovery/retryablehttp-go` from 1.0.96 to 1.0.97 - [Release notes](https://github.com/projectdiscovery/retryablehttp-go/releases) - [Commits](https://github.com/projectdiscovery/retryablehttp-go/compare/v1.0.96...v1.0.97) Updates `github.com/projectdiscovery/dsl` from 0.3.12 to 0.3.13 - [Release notes](https://github.com/projectdiscovery/dsl/releases) - [Commits](https://github.com/projectdiscovery/dsl/compare/v0.3.12...v0.3.13) Updates `github.com/projectdiscovery/goflags` from 0.1.68 to 0.1.69 - [Release notes](https://github.com/projectdiscovery/goflags/releases) - [Commits](https://github.com/projectdiscovery/goflags/compare/v0.1.68...v0.1.69) Updates `github.com/projectdiscovery/gologger` from 1.1.41 to 1.1.42 - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.41...v1.1.42) Updates `github.com/projectdiscovery/ratelimit` from 0.0.69 to 0.0.70 - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.69...v0.0.70) Updates `github.com/projectdiscovery/tlsx` from 1.1.8 to 1.1.9 - [Release notes](https://github.com/projectdiscovery/tlsx/releases) - [Changelog](https://github.com/projectdiscovery/tlsx/blob/main/.goreleaser.yml) - [Commits](https://github.com/projectdiscovery/tlsx/compare/v1.1.8...v1.1.9) Updates `github.com/projectdiscovery/useragent` from 0.0.88 to 0.0.89 - [Release notes](https://github.com/projectdiscovery/useragent/releases) - [Commits](https://github.com/projectdiscovery/useragent/compare/v0.0.88...v0.0.89) Updates `github.com/projectdiscovery/utils` from 0.4.7 to 0.4.8 - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Changelog](https://github.com/projectdiscovery/utils/blob/main/CHANGELOG.md) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.4.7...v0.4.8) Updates `github.com/projectdiscovery/wappalyzergo` from 0.2.12 to 0.2.13 - [Release notes](https://github.com/projectdiscovery/wappalyzergo/releases) - [Commits](https://github.com/projectdiscovery/wappalyzergo/compare/v0.2.12...v0.2.13) Updates `github.com/projectdiscovery/networkpolicy` from 0.1.1 to 0.1.2 - [Release notes](https://github.com/projectdiscovery/networkpolicy/releases) - [Commits](https://github.com/projectdiscovery/networkpolicy/compare/v0.1.1...v0.1.2) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-minor dependency-group: modules - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/rawhttp dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/retryablehttp-go dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/dsl dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/goflags dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/gologger dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/tlsx dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/useragent dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/wappalyzergo dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules - dependency-name: github.com/projectdiscovery/networkpolicy dependency-type: direct:production update-type: version-update:semver-patch dependency-group: modules ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 39 +++++++++++++++--------------- go.sum | 76 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 58 insertions(+), 57 deletions(-) diff --git a/go.mod b/go.mod index ada74d1a8d..d10ed4519b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/projectdiscovery/nuclei/v3 go 1.21.0 +toolchain go1.22.5 require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible @@ -16,16 +17,16 @@ require ( github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible - github.com/miekg/dns v1.1.59 + github.com/miekg/dns v1.1.62 github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.1.1 - github.com/projectdiscovery/fastdialer v0.2.15 - github.com/projectdiscovery/hmap v0.0.76 + github.com/projectdiscovery/fastdialer v0.3.0 + github.com/projectdiscovery/hmap v0.0.77 github.com/projectdiscovery/interactsh v1.2.3 - github.com/projectdiscovery/rawhttp v0.1.83 + github.com/projectdiscovery/rawhttp v0.1.84 github.com/projectdiscovery/retryabledns v1.0.94 - github.com/projectdiscovery/retryablehttp-go v1.0.96 + github.com/projectdiscovery/retryablehttp-go v1.0.97 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -81,24 +82,24 @@ require ( github.com/microsoft/go-mssqldb v1.6.0 github.com/ory/dockertest/v3 v3.10.0 github.com/praetorian-inc/fingerprintx v1.1.9 - github.com/projectdiscovery/dsl v0.3.12 + github.com/projectdiscovery/dsl v0.3.13 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb - github.com/projectdiscovery/goflags v0.1.68 - github.com/projectdiscovery/gologger v1.1.41 + github.com/projectdiscovery/goflags v0.1.69 + github.com/projectdiscovery/gologger v1.1.42 github.com/projectdiscovery/gostruct v0.0.2 github.com/projectdiscovery/gozero v0.0.3 github.com/projectdiscovery/httpx v1.6.9 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 - github.com/projectdiscovery/ratelimit v0.0.69 + github.com/projectdiscovery/ratelimit v0.0.70 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 - github.com/projectdiscovery/tlsx v1.1.8 + github.com/projectdiscovery/tlsx v1.1.9 github.com/projectdiscovery/uncover v1.0.9 - github.com/projectdiscovery/useragent v0.0.88 - github.com/projectdiscovery/utils v0.4.7 - github.com/projectdiscovery/wappalyzergo v0.2.12 + github.com/projectdiscovery/useragent v0.0.89 + github.com/projectdiscovery/utils v0.4.8 + github.com/projectdiscovery/wappalyzergo v0.2.13 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 @@ -155,7 +156,7 @@ require ( github.com/felixge/fgprof v0.9.5 // indirect github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect - github.com/gaissmai/bart v0.9.5 // indirect + github.com/gaissmai/bart v0.17.8 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/gin-gonic/gin v1.9.1 // indirect @@ -222,7 +223,7 @@ require ( github.com/skeema/knownhosts v1.3.0 // indirect github.com/tidwall/btree v1.7.0 // indirect github.com/tidwall/buntdb v1.3.1 // indirect - github.com/tidwall/gjson v1.17.1 // indirect + github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect @@ -297,7 +298,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/networkpolicy v0.1.1 + github.com/projectdiscovery/networkpolicy v0.1.2 github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect @@ -315,11 +316,11 @@ require ( go.uber.org/zap v1.25.0 // indirect goftp.io/server/v2 v2.0.1 // indirect golang.org/x/crypto v0.32.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 - golang.org/x/mod v0.19.0 // indirect + golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 + golang.org/x/mod v0.22.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/time v0.6.0 // indirect - golang.org/x/tools v0.23.0 + golang.org/x/tools v0.29.0 google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect diff --git a/go.sum b/go.sum index dc77965636..d44034127e 100644 --- a/go.sum +++ b/go.sum @@ -346,8 +346,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/gaissmai/bart v0.9.5 h1:vy+r4Px6bjZ+v2QYXAsg63vpz9IfzdW146A8Cn4GPIo= -github.com/gaissmai/bart v0.9.5/go.mod h1:KHeYECXQiBjTzQz/om2tqn3sZF1J7hw9m6z41ftj3fg= +github.com/gaissmai/bart v0.17.8 h1:REVOltckoqNqNrdhKYgA4THrufgzr2CuPKKXsSQMRhw= +github.com/gaissmai/bart v0.17.8/go.mod h1:JCPkH/Xt5bSPCKDc6OpzkhSCeib8BIxu3kthzZwcl6w= github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/getkin/kin-openapi v0.126.0 h1:c2cSgLnAsS0xYfKsgt5oBV6MYRM/giU8/RtwUY4wyfY= @@ -737,8 +737,8 @@ github.com/microsoft/go-mssqldb v1.6.0 h1:mM3gYdVwEPFrlg/Dvr2DNVEgYFG7L42l+dGc67 github.com/microsoft/go-mssqldb v1.6.0/go.mod h1:00mDtPbeQCRGC1HwOOR5K/gr30P1NcEG0vx6Kbv2aJU= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= -github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= +github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ= +github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ= github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= @@ -856,26 +856,26 @@ github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/ github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc= github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB72JIg66c8wE= github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= -github.com/projectdiscovery/dsl v0.3.12 h1:nJMfO9UBh9dDDhrveIKPhtwALWR4HJa2S4iHxrNKv54= -github.com/projectdiscovery/dsl v0.3.12/go.mod h1:fQPk2z7kdDAWQPLIxi4XTEd/ai1biwQCGG3qsO0dUUU= -github.com/projectdiscovery/fastdialer v0.2.15 h1:mq79KOx+XXvXfid2FyRy7vYzFT9uWZbCwJ9XxrHiytA= -github.com/projectdiscovery/fastdialer v0.2.15/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ= +github.com/projectdiscovery/dsl v0.3.13 h1:3/2Cd7bl5qS3bpVIKK4Qns2f8/XKPlj82SsztHplHg0= +github.com/projectdiscovery/dsl v0.3.13/go.mod h1:nWkkkfreA+P0nc0O9lalqu5lieIirSYNAD4W9daKd+U= +github.com/projectdiscovery/fastdialer v0.3.0 h1:/wMptjdsrAU/wiaA/U3lSgYGaYCGJH6xm0mLei6oMxk= +github.com/projectdiscovery/fastdialer v0.3.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk= github.com/projectdiscovery/freeport v0.0.7/go.mod h1:cOhWKvNBe9xM6dFJ3RrrLvJ5vXx2NQ36SecuwjenV2k= github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg= github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY= -github.com/projectdiscovery/goflags v0.1.68 h1:o+uROASO6hNkLEU127/SXjUyGC0wCjiyGZedKejhW8k= -github.com/projectdiscovery/goflags v0.1.68/go.mod h1:WmVQzaIzh8xFZ3Akz8RzA6yfULkJdbWWgWu+5hZD9cg= -github.com/projectdiscovery/gologger v1.1.41 h1:s5iAOhYDoKnUay5CtvhFpZdGzQKXeARJbpQubThT1Bs= -github.com/projectdiscovery/gologger v1.1.41/go.mod h1:68ltkt+p8WquBsi4CQ2Z2nfrDXhuraC7GQbtH15B0PI= +github.com/projectdiscovery/goflags v0.1.69 h1:F9ANzRhVOlj+0/yS8xEKJ1TCXtcb/nXxVJPJylZYUXg= +github.com/projectdiscovery/goflags v0.1.69/go.mod h1:bKs3HhVh6woQfSFK6bFv2qydi1tx5UQ3nAQv9mYh2v4= +github.com/projectdiscovery/gologger v1.1.42 h1:wTF52VEhJCtZk9GDNrlnAYBrqfm5FqXv/FImkNfdOSE= +github.com/projectdiscovery/gologger v1.1.42/go.mod h1:hf6vcNeKgj+BHfWQpMV4UBsfE8d+9wauWvcI31ZwuXs= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.3 h1:tsYkrSvWw4WdIUJyisd4MB1vRiw1X57TuVVk3p8Z3G8= github.com/projectdiscovery/gozero v0.0.3/go.mod h1:MpJ37Dsh94gy2EKqaemdeh+CzduGVB2SDfhr6Upsjew= -github.com/projectdiscovery/hmap v0.0.76 h1:m13e18HEJYkCddcnHucn0XxO/tG8Av6TX+U1dw+PQzQ= -github.com/projectdiscovery/hmap v0.0.76/go.mod h1:AmLVOcAQKX4Iq4vTE/07LONuctMNUOveofI5EHb4oMQ= +github.com/projectdiscovery/hmap v0.0.77 h1:pI7pmW+CN19LAkMsSFKC0K1SK1+sonto2hwte/7eOaQ= +github.com/projectdiscovery/hmap v0.0.77/go.mod h1:LkV8r/enq9G9HnY2YzMpU4CCmFz3dstOlhmeaObVQfk= github.com/projectdiscovery/httpx v1.6.9 h1:ihyFclesLjvQpiJpRIlAYeebapyIbOI/arDAvvy1ES8= github.com/projectdiscovery/httpx v1.6.9/go.mod h1:zQtX5CtcDYXzIRWne1ztCVtqG0sXCnx84tFwfMHoB8Q= github.com/projectdiscovery/interactsh v1.2.3 h1:5fWNJQy0+X0+7PoK9z3stxk58xZw3QwG+vXKLKs9tE8= @@ -888,32 +888,32 @@ github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQt github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw1pfT6bg35NiN7yd1XKtJap5Nk6lMwQ0RNi8= github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc= -github.com/projectdiscovery/networkpolicy v0.1.1 h1:iv9gECukD5KAZp98KVh+T3TEPTkY6dr3sKsdbh9XyZU= -github.com/projectdiscovery/networkpolicy v0.1.1/go.mod h1:/Hg2ieLewSe/BagFF+UYXAQo3NwmVMq16MSAl492XkU= -github.com/projectdiscovery/ratelimit v0.0.69 h1:co8QoB/hyIW9HwT/oA5aa1ENAdg8CCasMzBb9cbfxY8= -github.com/projectdiscovery/ratelimit v0.0.69/go.mod h1:5inCjvHaTFMVlS2ac6FHJa0XGGRmF1p+GWdyGrJzSkI= -github.com/projectdiscovery/rawhttp v0.1.83 h1:+ON7DRlFclrcdgtSCZeC4Z4IvA53LN/MIc7Xn7g7CC0= -github.com/projectdiscovery/rawhttp v0.1.83/go.mod h1:I79YDyBuRieOjrEs5GqUHEoRJXt7FU/SGoAVFn7GWxY= +github.com/projectdiscovery/networkpolicy v0.1.2 h1:5ff6SM6H6tRgp4kzmq4+/T8F0S7lom+QdXeNsLx8Mb4= +github.com/projectdiscovery/networkpolicy v0.1.2/go.mod h1:lJ6Vlr5/CkJEusm/ZIWi/W6yS6sKnwrLuROpDfI2Z5Y= +github.com/projectdiscovery/ratelimit v0.0.70 h1:SxFQcIKO3hppmEn9MOaDiqX2NXceji0vd8ER+eCHQjc= +github.com/projectdiscovery/ratelimit v0.0.70/go.mod h1:jg253i7eeKBIV5QpTpQv6+lZXr53XmKGBLS3dwlmRWM= +github.com/projectdiscovery/rawhttp v0.1.84 h1:91ctBCABSR2dIMj+hoQz3E0EsmPDNv2YmsjMOwjyJnk= +github.com/projectdiscovery/rawhttp v0.1.84/go.mod h1:F3mb+qlkh/pI+CiaoddmMmgeD2TSPM7WUMcm9DU9KBY= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.94 h1:MvxtRcmvxhxikxT7p/E40hcYRWRiL5fg/JQ8bpBaz+0= github.com/projectdiscovery/retryabledns v1.0.94/go.mod h1:croGTyMM4yNlrSWA/X7xNe3c0c7mDmCdbm8goLd8Bak= -github.com/projectdiscovery/retryablehttp-go v1.0.96 h1:/88LMgEGVx5rpqdLYIqxyJHBluuREKxJxPVIWi1zOlA= -github.com/projectdiscovery/retryablehttp-go v1.0.96/go.mod h1:YIN8SpckCijt5J1AV+nQApUWyCC3Y+8nAwDfK/RwhWs= +github.com/projectdiscovery/retryablehttp-go v1.0.97 h1:6nee/vJjiZP3vOhyqLcpSADM3vqmcC2QOvaMIo+dKWQ= +github.com/projectdiscovery/retryablehttp-go v1.0.97/go.mod h1:ZvwB6IsIHf0YlovcEQufZ6OTluyWfxRd360SrKd9fPk= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/tlsx v1.1.8 h1:Y+VkOp6JmUBb4tci1Fbz9U7ELEQ2irFhm+HS58tHruM= -github.com/projectdiscovery/tlsx v1.1.8/go.mod h1:6u/dbLuMsLzmux58AWnAB24qh2+Trk0auCK2I9B17Vo= +github.com/projectdiscovery/tlsx v1.1.9 h1:DhErhHCO2+toF5DEX7qe3pkwrIrYlex3F/mzHEUiZHg= +github.com/projectdiscovery/tlsx v1.1.9/go.mod h1:Jy+r38WyYjapQWaffiKGdYm4Ksdrt8BWtsqA2rUospU= github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0JprejqLSopN9Y= github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U= -github.com/projectdiscovery/useragent v0.0.88 h1:B0QPPGkCiXQ+APzjaaLaX3+01dKuFINZoLjnXUtUrZo= -github.com/projectdiscovery/useragent v0.0.88/go.mod h1:FJg7Rjedoszrcbz6Rhtgh6bnVrcwj3kzJ+p75UE7dME= -github.com/projectdiscovery/utils v0.4.7 h1:kw0BCox/G4eCaz5MW0U4HP6z/n0K4X7qv7v1wNuXSLA= -github.com/projectdiscovery/utils v0.4.7/go.mod h1:B4k3whXzeBD+HrlvlqCv+KNgcAuzrTIL+OADkk0z8Bw= -github.com/projectdiscovery/wappalyzergo v0.2.12 h1:A3oBpnEbTHOa3Q9m4w/5LLXsmCEiu0mJcwyjf3M9xnc= -github.com/projectdiscovery/wappalyzergo v0.2.12/go.mod h1:3vtvQCSYpU+Ilk0qy09WYT9BH0Stut5Qon7KJJ78GKw= +github.com/projectdiscovery/useragent v0.0.89 h1:j/ryhBjScEAX4Ktfzdxea6hvxrL3OpsxBCEJ4o1D73g= +github.com/projectdiscovery/useragent v0.0.89/go.mod h1:ZAYEFo2ZhtCmeuj1hj0E/ZoMm8K91jJ3ct3qrHkbX1E= +github.com/projectdiscovery/utils v0.4.8 h1:/Xd38fP8xc6kifZayjrhcYALenJrjO3sHO7lg+I8ZGk= +github.com/projectdiscovery/utils v0.4.8/go.mod h1:S314NzLcXVCbLbwYCoorAJYcnZEwv7Uhw2d3aF5fJ4s= +github.com/projectdiscovery/wappalyzergo v0.2.13 h1:yrzmcT/Zq+H+qpbAjHKKFVVQA7uei+sAYlmS2HLpkGI= +github.com/projectdiscovery/wappalyzergo v0.2.13/go.mod h1:3vtvQCSYpU+Ilk0qy09WYT9BH0Stut5Qon7KJJ78GKw= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -1045,8 +1045,8 @@ github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EU github.com/tidwall/buntdb v1.3.1 h1:HKoDF01/aBhl9RjYtbaLnvX9/OuenwvQiC3OP1CcL4o= github.com/tidwall/buntdb v1.3.1/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= -github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= @@ -1235,8 +1235,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1259,8 +1259,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1510,8 +1510,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= +golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From d699c278cd1f889a8bbee53ab9fe4ca470a0c856 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto <25837540+dwisiswant0@users.noreply.github.com> Date: Mon, 27 Jan 2025 16:52:11 +0700 Subject: [PATCH 3/3] fix(installer): handle removal of deleted templates during update (#5998) * fix(installer): handle removal of deleted templates during update Signed-off-by: Dwi Siswanto * chore(installer): no log for non-existent deleted templates err Signed-off-by: Dwi Siswanto * feat(installer): purge empty dirs after removing deleted templates Signed-off-by: Dwi Siswanto * Revert "feat(installer): purge empty dirs after removing deleted templates" This reverts commit 8175e2a291b7bbfdae8c8ab55ba54703c1958e99. * fix(installer): use semicolons as delimiters Signed-off-by: Dwi Siswanto * feat(installer): add mods count in `*templateUpdateResults.String` Signed-off-by: Dwi Siswanto --------- Signed-off-by: Dwi Siswanto --- pkg/installer/template.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/pkg/installer/template.go b/pkg/installer/template.go index 37a959cf1e..26d09f9f2f 100644 --- a/pkg/installer/template.go +++ b/pkg/installer/template.go @@ -45,10 +45,15 @@ type templateUpdateResults struct { func (t *templateUpdateResults) String() string { var buff bytes.Buffer data := [][]string{ - {strconv.Itoa(t.totalCount), strconv.Itoa(len(t.additions)), strconv.Itoa(len(t.deletions))}, + { + strconv.Itoa(t.totalCount), + strconv.Itoa(len(t.additions)), + strconv.Itoa(len(t.modifications)), + strconv.Itoa(len(t.deletions)), + }, } table := tablewriter.NewWriter(&buff) - table.SetHeader([]string{"Total", "Added", "Removed"}) + table.SetHeader([]string{"Total", "Added", "Modified", "Removed"}) for _, v := range data { table.Append(v) } @@ -151,6 +156,13 @@ func (t *TemplateManager) updateTemplatesAt(dir string) error { // summarize all changes results := t.summarizeChanges(oldchecksums, newchecksums) + // remove deleted templates + for _, deletion := range results.deletions { + if err := os.Remove(deletion); err != nil && !os.IsNotExist(err) { + gologger.Warning().Msgf("failed to remove deleted template %s: %s", deletion, err) + } + } + // print summary if results.totalCount > 0 { gologger.Info().Msgf("Successfully updated nuclei-templates (%v) to %s. GoodLuck!", ghrd.Latest.GetTagName(), dir) @@ -343,7 +355,7 @@ func (t *TemplateManager) getChecksumFromDir(dir string) (map[string]string, err checksums, err := os.ReadFile(checksumFilePath) if err == nil { allChecksums := make(map[string]string) - for _, v := range strings.Split(string(checksums), "\n") { + for _, v := range strings.Split(string(checksums), ";") { v = strings.TrimSpace(v) tmparr := strings.Split(v, ",") if len(tmparr) != 2 { @@ -366,7 +378,10 @@ func (t *TemplateManager) writeChecksumFileInDir(dir string) error { } var buff bytes.Buffer for k, v := range checksumMap { - buff.WriteString(k + "," + v) + buff.WriteString(k) + buff.WriteString(",") + buff.WriteString(v) + buff.WriteString(";") } return os.WriteFile(config.DefaultConfig.GetChecksumFilePath(), buff.Bytes(), checkSumFilePerm) }