Merge branch 'dev'

projectdiscovery · Mar 1, 2024 · 0a08bcf · 0a08bcf
2 parents 58beb91 + 4a783e5
commit 0a08bcf
Show file tree

Hide file tree

Showing 9 changed files with 143 additions and 72 deletions.
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -30,6 +30,9 @@ jobs:
       - name: Test
         run: go test ./...
         working-directory: .
+        env:
+          PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
+
 
       - name: Running example
         run: go run .
@@ -39,6 +42,7 @@ jobs:
         if: runner.os == 'Linux' || runner.os == 'macOS'
         env:
           GH_ACTION: true
+          PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
         run: bash run.sh
         working-directory: integration_tests/
 
@@ -47,6 +51,7 @@ jobs:
         env:
           GH_ACTION: true
           MSYS_NO_PATHCONV: true
+          PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
         run: bash run.sh
         working-directory: integration_tests/
 

diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml
@@ -28,3 +28,5 @@ jobs:
           chmod +x run.sh
           bash run.sh ${{ matrix.os }}
         working-directory: cmd/functional-test
+        env:
+          PDCP_API_KEY: "${{ secrets.PDCP_API_KEY }}"
diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go
@@ -16,6 +16,7 @@ import (
 	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate"
 	"github.com/projectdiscovery/httpx/common/httputilz"
+	"github.com/projectdiscovery/networkpolicy"
 	"github.com/projectdiscovery/rawhttp"
 	retryablehttp "github.com/projectdiscovery/retryablehttp-go"
 	"github.com/projectdiscovery/useragent"
@@ -37,6 +38,7 @@ type HTTPX struct {
 	CustomHeaders map[string]string
 	cdn           *cdncheck.Client
 	Dialer        *fastdialer.Dialer
+	NetworkPolicy *networkpolicy.NetworkPolicy
 }
 
 // New httpx instance
@@ -50,8 +52,10 @@ func New(options *Options) (*HTTPX, error) {
 		fastdialerOpts.EnableFallback = false
 	}
 
-	fastdialerOpts.Deny = options.Deny
-	fastdialerOpts.Allow = options.Allow
+	if options.NetworkPolicy != nil {
+		httpx.NetworkPolicy = options.NetworkPolicy
+		fastdialerOpts.WithNetworkPolicyOptions = options.NetworkPolicy.Options
+	}
 	fastdialerOpts.WithDialerHistory = true
 	fastdialerOpts.WithZTLS = options.ZTLS
 	if len(options.Resolvers) > 0 {

diff --git a/common/httpx/option.go b/common/httpx/option.go
@@ -4,6 +4,8 @@ import (
 	"net/http"
 	"strings"
 	"time"
+
+	"github.com/projectdiscovery/networkpolicy"
 )
 
 // Options contains configuration options for the client
@@ -35,15 +37,14 @@ type Options struct {
 	VHostIgnoreNumberOfWords  bool
 	VHostIgnoreNumberOfLines  bool
 	VHostStripHTML            bool
-	Allow                     []string
-	Deny                      []string
 	MaxResponseBodySizeToSave int64
 	MaxResponseBodySizeToRead int64
 	UnsafeURI                 string
 	Resolvers                 []string
 	customCookies             []*http.Cookie
 	SniName                   string
 	TlsImpersonate            bool
+	NetworkPolicy             *networkpolicy.NetworkPolicy
 }
 
 // DefaultOptions contains the default options

diff --git a/go.mod b/go.mod
@@ -3,57 +3,53 @@ module github.com/projectdiscovery/httpx
 go 1.21
 
 require (
+	github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057
+	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/akrylysov/pogreb v0.10.1 // indirect
+	github.com/corona10/goimagehash v1.1.0
+	github.com/go-faker/faker/v4 v4.1.1
+	github.com/go-rod/rod v0.114.0
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf
+	github.com/hdm/jarm-go v0.0.7
+	github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/logrusorgru/aurora v2.0.3+incompatible
+	github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6
 	github.com/microcosm-cc/bluemonday v1.0.26
 	github.com/miekg/dns v1.1.56 // indirect
+	github.com/mitchellh/mapstructure v1.5.0
 	github.com/pkg/errors v0.9.1
+	github.com/projectdiscovery/asnmap v1.1.0
 	github.com/projectdiscovery/cdncheck v1.0.9
 	github.com/projectdiscovery/clistats v0.0.20
+	github.com/projectdiscovery/dsl v0.0.41
+	github.com/projectdiscovery/fastdialer v0.0.60
 	github.com/projectdiscovery/fdmax v0.0.4
 	github.com/projectdiscovery/goconfig v0.0.1
-	github.com/projectdiscovery/goflags v0.1.39
+	github.com/projectdiscovery/goflags v0.1.41
 	github.com/projectdiscovery/gologger v1.1.12
-	github.com/projectdiscovery/hmap v0.0.37
+	github.com/projectdiscovery/hmap v0.0.40
 	github.com/projectdiscovery/mapcidr v1.1.16
-	github.com/projectdiscovery/rawhttp v0.1.37
-	github.com/projectdiscovery/retryablehttp-go v1.0.47
-	github.com/projectdiscovery/wappalyzergo v0.0.109
-	github.com/remeh/sizedwaitgroup v1.0.0
-	github.com/rs/xid v1.5.0
-	go.etcd.io/bbolt v1.3.7 // indirect
-	golang.org/x/net v0.17.0
-	golang.org/x/sys v0.16.0 // indirect
-	golang.org/x/text v0.14.0
-)
-
-require github.com/spaolacci/murmur3 v1.1.0
-
-require (
-	github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057
-	github.com/PuerkitoBio/goquery v1.8.1
-	github.com/corona10/goimagehash v1.1.0
-	github.com/go-faker/faker/v4 v4.1.1
-	github.com/go-rod/rod v0.114.0
-	github.com/hdm/jarm-go v0.0.7
-	github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056
-	github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6
-	github.com/mitchellh/mapstructure v1.5.0
-	github.com/projectdiscovery/asnmap v1.0.6
-	github.com/projectdiscovery/dsl v0.0.41
-	github.com/projectdiscovery/fastdialer v0.0.58
 	github.com/projectdiscovery/networkpolicy v0.0.7
 	github.com/projectdiscovery/ratelimit v0.0.27
+	github.com/projectdiscovery/rawhttp v0.1.39
+	github.com/projectdiscovery/retryablehttp-go v1.0.49
 	github.com/projectdiscovery/tlsx v1.1.6
-	github.com/projectdiscovery/useragent v0.0.37
-	github.com/projectdiscovery/utils v0.0.77
+	github.com/projectdiscovery/useragent v0.0.38
+	github.com/projectdiscovery/utils v0.0.81
+	github.com/projectdiscovery/wappalyzergo v0.0.109
+	github.com/remeh/sizedwaitgroup v1.0.0
+	github.com/rs/xid v1.5.0
+	github.com/spaolacci/murmur3 v1.1.0
 	github.com/stretchr/testify v1.8.4
 	github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968
+	go.etcd.io/bbolt v1.3.7 // indirect
 	go.uber.org/multierr v1.11.0
 	golang.org/x/exp v0.0.0-20230810033253-352e893a4cad
+	golang.org/x/net v0.17.0
+	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/text v0.14.0
 )
 
 require (
@@ -74,13 +70,13 @@ require (
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/denisbrodbeck/machineid v1.0.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/dlclark/regexp2 v1.8.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/gaukas/godicttls v0.0.4 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/certificate-transparency-go v1.1.4 // indirect
 	github.com/google/go-github/v30 v30.1.0 // indirect
@@ -93,6 +89,7 @@ require (
 	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/kljensen/snowball v0.8.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
@@ -107,16 +104,20 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pierrec/lz4/v4 v4.1.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/projectdiscovery/blackrock v0.0.1 // indirect
 	github.com/projectdiscovery/freeport v0.0.5 // indirect
 	github.com/projectdiscovery/gostruct v0.0.2 // indirect
-	github.com/projectdiscovery/retryabledns v1.0.55 // indirect
+	github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect
+	github.com/projectdiscovery/retryabledns v1.0.57 // indirect
 	github.com/projectdiscovery/stringsutil v0.0.2 // indirect
 	github.com/quic-go/quic-go v0.37.7 // indirect
 	github.com/refraction-networking/utls v1.5.4 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	github.com/sashabaranov/go-openai v1.14.2 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.7 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
 	github.com/tidwall/btree v1.6.0 // indirect
@@ -127,6 +128,8 @@ require (
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/rtred v0.1.2 // indirect
 	github.com/tidwall/tinyqueue v0.1.1 // indirect
+	github.com/tklauser/go-sysconf v0.3.11 // indirect
+	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
 	github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db // indirect
@@ -139,10 +142,12 @@ require (
 	github.com/ysmood/leakless v0.8.0 // indirect
 	github.com/yuin/goldmark v1.5.4 // indirect
 	github.com/yuin/goldmark-emoji v1.0.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/oauth2 v0.11.0 // indirect
+	golang.org/x/term v0.16.0 // indirect
 	golang.org/x/tools v0.13.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect