From 06e1f66b81bba7293ecd1263a81649219fc3f0a6 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 31 Jan 2023 16:01:02 +0100 Subject: [PATCH 01/10] docker cleanup --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 62962ad3..39bfcfd8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,3 @@ -# Base FROM golang:1.19.5-alpine AS builder RUN apk add --no-cache git From 2d24b9bab788c9db23c935c7ec3634fff21e608d Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 31 Jan 2023 16:04:00 +0100 Subject: [PATCH 02/10] simplified flagset syntax --- internal/runner/options.go | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/internal/runner/options.go b/internal/runner/options.go index 16d6aaa8..cc8d6939 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -53,21 +53,21 @@ func ParseOptions() *Options { flagSet := goflags.NewFlagSet() flagSet.SetDescription(`Swiss Army Knife Proxy for rapid deployments. Supports multiple operations such as request/response dump,filtering and manipulation via DSL language, upstream HTTP/Socks5 proxy`) - createGroup(flagSet, "output", "Output", + flagSet.CreateGroup("output", "Output", // Todo: flagSet.BoolVar(&options.Dump, "dump", true, "Dump HTTP requests/response to output file"), flagSet.StringVarP(&options.OutputDirectory, "output", "o", "logs", "Output Directory to store HTTP proxy logs"), flagSet.BoolVar(&options.DumpRequest, "dump-req", false, "Dump only HTTP requests to output file"), flagSet.BoolVar(&options.DumpResponse, "dump-resp", false, "Dump only HTTP responses to output file"), ) - createGroup(flagSet, "filter", "Filter", + flagSet.CreateGroup("filter", "Filter", flagSet.StringSliceVarP(&options.RequestDSL, "request-dsl", "req-fd", nil, "Request Filter DSL", goflags.StringSliceOptions), flagSet.StringSliceVarP(&options.ResponseDSL, "response-dsl", "resp-fd", nil, "Response Filter DSL", goflags.StringSliceOptions), flagSet.StringSliceVarP(&options.RequestMatchReplaceDSL, "request-match-replace-dsl", "req-mrd", nil, "Request Match-Replace DSL", goflags.StringSliceOptions), flagSet.StringSliceVarP(&options.ResponseMatchReplaceDSL, "response-match-replace-dsl", "resp-mrd", nil, "Response Match-Replace DSL", goflags.StringSliceOptions), ) - createGroup(flagSet, "network", "Network", + flagSet.CreateGroup("network", "Network", flagSet.StringVarP(&options.ListenAddrHTTP, "http-addr", "ha", "127.0.0.1:8888", "Listening HTTP IP and Port address (ip:port)"), flagSet.StringVarP(&options.ListenAddrSocks5, "socks-addr", "sa", "127.0.0.1:10080", "Listening SOCKS IP and Port address (ip:port)"), flagSet.StringVarP(&options.ListenDNSAddr, "dns-addr", "da", "", "Listening DNS IP and Port address (ip:port)"), @@ -75,13 +75,13 @@ func ParseOptions() *Options { flagSet.StringVarP(&options.DNSFallbackResolver, "resolver", "r", "", "Custom DNS resolvers to use (ip:port)"), ) - createGroup(flagSet, "proxy", "Proxy", + flagSet.CreateGroup("proxy", "Proxy", flagSet.StringSliceVarP(&options.UpstreamHTTPProxies, "http-proxy", "hp", nil, "Upstream HTTP Proxies (eg http://proxy-ip:proxy-port)", goflags.NormalizedStringSliceOptions), flagSet.StringSliceVarP(&options.UpstreamSocks5Proxies, "socks5-proxy", "sp", nil, "Upstream SOCKS5 Proxies (eg socks5://proxy-ip:proxy-port)", goflags.NormalizedStringSliceOptions), flagSet.IntVar(&options.UpstreamProxyRequestsNumber, "c", 1, "Number of requests before switching to the next upstream proxy"), ) - createGroup(flagSet, "export", "Export", + flagSet.CreateGroup("export", "Export", flagSet.StringVar(&options.Elastic.Addr, "elastic-address", "", "elasticsearch address (ip:port)"), flagSet.BoolVar(&options.Elastic.SSL, "elastic-ssl", false, "enable elasticsearch ssl"), flagSet.BoolVar(&options.Elastic.SSLVerification, "elastic-ssl-verification", false, "enable elasticsearch ssl verification"), @@ -92,7 +92,7 @@ func ParseOptions() *Options { flagSet.StringVar(&options.Kafka.Topic, "kafka-topic", "proxify", "kafka topic to publish messages on"), ) - createGroup(flagSet, "configuration", "Configuration", + flagSet.CreateGroup("configuration", "Configuration", // Todo: default config file support (homeDir/.config/proxify/config.yaml) flagSet.StringVar(&options.Directory, "config", path.Join(homeDir, ".config", "proxify"), "Directory for storing program information"), flagSet.IntVar(&options.CertCacheSize, "cert-cache-size", 256, "Number of certificates to cache"), @@ -101,7 +101,7 @@ func ParseOptions() *Options { ) silent, verbose, veryVerbose := false, false, false - createGroup(flagSet, "debug", "debug", + flagSet.CreateGroup("debug", "debug", flagSet.BoolVarP(&options.NoColor, "no-color", "nc", true, "No Color"), flagSet.BoolVar(&options.Version, "version", false, "Version"), flagSet.BoolVar(&silent, "silent", false, "Silent"), @@ -154,10 +154,3 @@ func (options *Options) configureOutput() { gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true)) } } - -func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) { - flagSet.SetGroup(groupName, description) - for _, currentFlag := range flags { - currentFlag.Group(groupName) - } -} From 0f678a4e3a45a76f8bf015383189c6cd567a2f95 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 31 Jan 2023 16:04:12 +0100 Subject: [PATCH 03/10] simplified graceful exit --- cmd/proxify/proxify.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/proxify/proxify.go b/cmd/proxify/proxify.go index 5e5cdb98..51a8503a 100644 --- a/cmd/proxify/proxify.go +++ b/cmd/proxify/proxify.go @@ -21,19 +21,17 @@ func main() { // Setup close handler go func() { - c := make(chan os.Signal,1) //added size 1 to channel buffer + c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) - go func() { - <-c + for range c { fmt.Println("\r- Ctrl+C pressed in Terminal") proxifyRunner.Close() os.Exit(0) - }() + } }() err = proxifyRunner.Run() if err != nil { gologger.Fatal().Msgf("Could not run proxify: %s\n", err) } - } From 61b5e282e8da7f290b50f6730f9bbc5aedc31b7e Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 31 Jan 2023 16:07:02 +0100 Subject: [PATCH 04/10] removing redundant tests on push --- .github/workflows/build-test.yml | 1 - .github/workflows/codeql-analysis.yml | 1 - .github/workflows/lint-test.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index f3fc9daa..e95251fc 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -1,7 +1,6 @@ name: 🔨 Build Test on: - push: pull_request: workflow_dispatch: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2bbccfd4..9f533f8d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -2,7 +2,6 @@ name: 🚨 CodeQL Analysis on: workflow_dispatch: - push: pull_request: branches: - dev diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 21c5b533..ebb10384 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -1,6 +1,5 @@ name: 🙏🏻 Lint Test on: - push: pull_request: workflow_dispatch: From 6cb6ba89863cd3e3e93b98a35e72fdda0fc24061 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 31 Jan 2023 16:51:24 +0100 Subject: [PATCH 05/10] adding data truncation --- internal/runner/options.go | 2 ++ pkg/logger/elastic/elasticsearch.go | 20 ++++++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/internal/runner/options.go b/internal/runner/options.go index cc8d6939..bcaf800a 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -1,6 +1,7 @@ package runner import ( + "math" "os" "path" @@ -87,6 +88,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Elastic.SSLVerification, "elastic-ssl-verification", false, "enable elasticsearch ssl verification"), flagSet.StringVar(&options.Elastic.Username, "elastic-username", "", "elasticsearch username"), flagSet.StringVar(&options.Elastic.Password, "elastic-password", "", "elasticsearch password"), + flagSet.IntVar(&options.Elastic.MaxFieldSize, "elastic-max-field-size", math.MaxInt, "elasticsearch max field size"), flagSet.StringVar(&options.Elastic.IndexName, "elastic-index", "proxify", "elasticsearch index name"), flagSet.StringVar(&options.Kafka.Addr, "kafka-address", "", "address of kafka broker (ip:port)"), flagSet.StringVar(&options.Kafka.Topic, "kafka-topic", "proxify", "kafka topic to publish messages on"), diff --git a/pkg/logger/elastic/elasticsearch.go b/pkg/logger/elastic/elasticsearch.go index 07e0ebaf..877b402f 100644 --- a/pkg/logger/elastic/elasticsearch.go +++ b/pkg/logger/elastic/elasticsearch.go @@ -29,11 +29,14 @@ type Options struct { Password string `yaml:"password"` // IndexName is the name of the elasticsearch index IndexName string `yaml:"index-name"` + // MaxFieldSize sets a maximum limit to the field length + MaxFieldSize int `yaml:"max-field-size"` } // Client type for elasticsearch type Client struct { index string + options *Options esClient *elasticsearch.Client } @@ -57,10 +60,12 @@ func New(option *Options) (*Client, error) { if err != nil { return nil, errors.Wrap(err, "error creating elasticsearch client") } - return &Client{ + client := &Client{ esClient: elasticsearchClient, index: option.IndexName, - }, nil + options: option, + } + return client, nil } @@ -69,12 +74,12 @@ func (c *Client) Save(data types.OutputData) error { var doc map[string]interface{} if data.Userdata.HasResponse { doc = map[string]interface{}{ - "response": data.DataString, + "response": truncate(data.DataString, c.options.MaxFieldSize), "timestamp": time.Now().Format(time.RFC3339), } } else { doc = map[string]interface{}{ - "request": data.DataString, + "request": truncate(data.DataString, c.options.MaxFieldSize), "timestamp": time.Now().Format(time.RFC3339), } } @@ -103,3 +108,10 @@ func (c *Client) Save(data types.OutputData) error { res.Body.Close() return er } + +func truncate(data string, maxSize int) string { + if maxSize > 0 && len(data) > maxSize { + return data[:maxSize] + } + return data +} From 7b79a0ec56d85d014f9a83f11cb6c033eebc6626 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 31 Jan 2023 17:44:46 +0100 Subject: [PATCH 06/10] fixing path => filepath --- internal/runner/options.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/runner/options.go b/internal/runner/options.go index bcaf800a..68cda184 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -3,7 +3,7 @@ package runner import ( "math" "os" - "path" + "path/filepath" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" @@ -96,7 +96,7 @@ func ParseOptions() *Options { flagSet.CreateGroup("configuration", "Configuration", // Todo: default config file support (homeDir/.config/proxify/config.yaml) - flagSet.StringVar(&options.Directory, "config", path.Join(homeDir, ".config", "proxify"), "Directory for storing program information"), + flagSet.StringVar(&options.Directory, "config", filepath.Join(homeDir, ".config", "proxify"), "Directory for storing program information"), flagSet.IntVar(&options.CertCacheSize, "cert-cache-size", 256, "Number of certificates to cache"), flagSet.StringSliceVarP(&options.Allow, "allow", "a", nil, "Allowed list of IP/CIDR's to be proxied", goflags.FileNormalizedStringSliceOptions), flagSet.StringSliceVarP(&options.Deny, "deny", "d", nil, "Denied list of IP/CIDR's to be proxied", goflags.FileNormalizedStringSliceOptions), From e7864610a7b342efa8a30ba352f0831a92db13fb Mon Sep 17 00:00:00 2001 From: xm1k3 Date: Sun, 19 Feb 2023 09:03:40 +0100 Subject: [PATCH 07/10] small change: only added -elastic-max-field-size to readme --- README.md | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 1e22c638..d975eb93 100644 --- a/README.md +++ b/README.md @@ -72,10 +72,10 @@ OUTPUT: -dump-resp Dump only HTTP responses to output file FILTER: - -req-fd, -request-dsl string Request Filter DSL - -resp-fd, -response-dsl string Response Filter DSL - -req-mrd, -request-match-replace-dsl string Request Match-Replace DSL - -resp-mrd, -response-match-replace-dsl string Response Match-Replace DSL + -req-fd, -request-dsl string[] Request Filter DSL + -resp-fd, -response-dsl string[] Response Filter DSL + -req-mrd, -request-match-replace-dsl string[] Request Match-Replace DSL + -resp-mrd, -response-match-replace-dsl string[] Response Match-Replace DSL NETWORK: -ha, -http-addr string Listening HTTP IP and Port address (ip:port) (default "127.0.0.1:8888") @@ -85,32 +85,33 @@ NETWORK: -r, -resolver string Custom DNS resolvers to use (ip:port) PROXY: - -hp, -http-proxy string Upstream HTTP Proxies (eg http://proxy-ip:proxy-port - -sp, -socks5-proxy string Upstream SOCKS5 Proxies (eg socks5://proxy-ip:proxy-port) - -c int Number of requests before switching to the next upstream proxy (default 1) + -hp, -http-proxy string[] Upstream HTTP Proxies (eg http://proxy-ip:proxy-port) + -sp, -socks5-proxy string[] Upstream SOCKS5 Proxies (eg socks5://proxy-ip:proxy-port) + -c int Number of requests before switching to the next upstream proxy (default 1) EXPORT: - -elastic-address string elasticsearch address (ip:port) - -elastic-ssl enable elasticsearch ssl - -elastic-ssl-verification enable elasticsearch ssl verification - -elastic-username string elasticsearch username - -elastic-password string elasticsearch password - -elastic-index string elasticsearch index name (default "proxify") - -kafka-address string address of kafka broker (ip:port) - -kafka-topic string kafka topic to publish messages on (default "proxify") + -elastic-address string elasticsearch address (ip:port) + -elastic-ssl enable elasticsearch ssl + -elastic-ssl-verification enable elasticsearch ssl verification + -elastic-username string elasticsearch username + -elastic-password string elasticsearch password + -elastic-max-field-size int elasticsearch max field size (default 9223372036854775807) + -elastic-index string elasticsearch index name (default "proxify") + -kafka-address string address of kafka broker (ip:port) + -kafka-topic string kafka topic to publish messages on (default "proxify") CONFIGURATION: - -config string Directory for storing program information (default "/Users/geekboy/.config/proxify") + -config string Directory for storing program information (default "$HOME/.config/proxify") -cert-cache-size int Number of certificates to cache (default 256) - -allow string Allowed list of IP/CIDR's to be proxied - -deny string Denied list of IP/CIDR's to be proxied + -a, -allow string[] Allowed list of IP/CIDR's to be proxied + -d, -deny string[] Denied list of IP/CIDR's to be proxied DEBUG: - -nc, -no-color No Color (default true) - -version Version - -silent Silent - -v, -verbose Verbose - -vv, -very-verbose Very Verbose + -nc, -no-color No Color (default true) + -version Version + -silent Silent + -v, -verbose Verbose + -vv, -very-verbose Very Verbose ``` ### Running Proxify From d08a46c43f81fc59b36916c73f8c02573187b1ef Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 24 Feb 2023 16:56:47 +0100 Subject: [PATCH 08/10] making export size generic --- go.mod | 2 +- go.sum | 2 ++ internal/runner/options.go | 3 ++- internal/runner/runner.go | 1 + pkg/logger/elastic/elasticsearch.go | 13 ++----------- pkg/logger/logger.go | 5 +++++ proxy.go | 2 ++ 7 files changed, 15 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index ce15c52e..42be0fbb 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/projectdiscovery/goflags v0.1.7 github.com/projectdiscovery/gologger v1.1.8 github.com/projectdiscovery/tinydns v0.0.1 - github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d + github.com/projectdiscovery/utils v0.0.11 github.com/rs/xid v1.4.0 golang.org/x/net v0.7.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index a97ee292..f66e9981 100644 --- a/go.sum +++ b/go.sum @@ -441,6 +441,8 @@ github.com/projectdiscovery/tinydns v0.0.1 h1:Ls5TAlMJoCfEObpsC7S+Hg4WLyF5wiN+O1 github.com/projectdiscovery/tinydns v0.0.1/go.mod h1:xPZcaMje/MWozKc5Sdln3XWptINYoWqCBXgLeSrNB1Q= github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d h1:iB/n2/NL4oh1IaEcqX6pBxj0WHfYN7finzNOKVNVISM= github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= +github.com/projectdiscovery/utils v0.0.11 h1:JwyW3Fp7iF6d3XjzsaC+Gz38EROYuXlMxm7re7LyGvk= +github.com/projectdiscovery/utils v0.0.11/go.mod h1:0AqTr9qCXU7dD+c1hItoVY2xhr4gLr3U4/ABiQvhW1U= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= diff --git a/internal/runner/options.go b/internal/runner/options.go index 8dcb49c1..2b3882ee 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -41,6 +41,7 @@ type Options struct { Elastic elastic.Options Kafka kafka.Options PassThrough goflags.StringSlice // Passthrough items list + MaxSize int } func ParseOptions() *Options { @@ -84,12 +85,12 @@ func ParseOptions() *Options { ) flagSet.CreateGroup("export", "Export", + flagSet.IntVar(&options.MaxSize, "max-size", math.MaxInt, "Max export data size (request/responses will be truncated)"), flagSet.StringVar(&options.Elastic.Addr, "elastic-address", "", "elasticsearch address (ip:port)"), flagSet.BoolVar(&options.Elastic.SSL, "elastic-ssl", false, "enable elasticsearch ssl"), flagSet.BoolVar(&options.Elastic.SSLVerification, "elastic-ssl-verification", false, "enable elasticsearch ssl verification"), flagSet.StringVar(&options.Elastic.Username, "elastic-username", "", "elasticsearch username"), flagSet.StringVar(&options.Elastic.Password, "elastic-password", "", "elasticsearch password"), - flagSet.IntVar(&options.Elastic.MaxFieldSize, "elastic-max-field-size", math.MaxInt, "elasticsearch max field size"), flagSet.StringVar(&options.Elastic.IndexName, "elastic-index", "proxify", "elasticsearch index name"), flagSet.StringVar(&options.Kafka.Addr, "kafka-address", "", "address of kafka broker (ip:port)"), flagSet.StringVar(&options.Kafka.Topic, "kafka-topic", "proxify", "kafka topic to publish messages on"), diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c890d17e..1da2ca5c 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -35,6 +35,7 @@ func NewRunner(options *Options) (*Runner, error) { ResponseMatchReplaceDSL: options.ResponseMatchReplaceDSL, DumpRequest: options.DumpRequest, DumpResponse: options.DumpResponse, + MaxSize: options.MaxSize, UpstreamProxyRequestsNumber: options.UpstreamProxyRequestsNumber, Elastic: &options.Elastic, Kafka: &options.Kafka, diff --git a/pkg/logger/elastic/elasticsearch.go b/pkg/logger/elastic/elasticsearch.go index 877b402f..18a5b305 100644 --- a/pkg/logger/elastic/elasticsearch.go +++ b/pkg/logger/elastic/elasticsearch.go @@ -29,8 +29,6 @@ type Options struct { Password string `yaml:"password"` // IndexName is the name of the elasticsearch index IndexName string `yaml:"index-name"` - // MaxFieldSize sets a maximum limit to the field length - MaxFieldSize int `yaml:"max-field-size"` } // Client type for elasticsearch @@ -74,12 +72,12 @@ func (c *Client) Save(data types.OutputData) error { var doc map[string]interface{} if data.Userdata.HasResponse { doc = map[string]interface{}{ - "response": truncate(data.DataString, c.options.MaxFieldSize), + "response": data.DataString, "timestamp": time.Now().Format(time.RFC3339), } } else { doc = map[string]interface{}{ - "request": truncate(data.DataString, c.options.MaxFieldSize), + "request": data.DataString, "timestamp": time.Now().Format(time.RFC3339), } } @@ -108,10 +106,3 @@ func (c *Client) Save(data types.OutputData) error { res.Body.Close() return er } - -func truncate(data string, maxSize int) string { - if maxSize > 0 && len(data) > maxSize { - return data[:maxSize] - } - return data -} diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 2b0440ce..461d2812 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -28,6 +28,7 @@ type OptionsLogger struct { OutputFolder string DumpRequest bool DumpResponse bool + MaxSize int Elastic *elastic.Options Kafka *kafka.Options } @@ -112,6 +113,10 @@ func (l *Logger) AsyncWrite() { outputdata.DataString = fmt.Sprintf(outputdata.Format, outputdata.Data) + if l.options.MaxSize > 0 { + outputdata.DataString = stringsutil.Truncate(outputdata.DataString, l.options.MaxSize) + } + for _, store := range l.Store { err := store.Save(outputdata) if err != nil { diff --git a/proxy.go b/proxy.go index 03e203aa..862dbeec 100644 --- a/proxy.go +++ b/proxy.go @@ -42,6 +42,7 @@ type OnConnectFunc func(string, *goproxy.ProxyCtx) (*goproxy.ConnectAction, stri type Options struct { DumpRequest bool DumpResponse bool + MaxSize int Verbosity types.Verbosity CertCacheSize int Directory string @@ -377,6 +378,7 @@ func NewProxy(options *Options) (*Proxy, error) { OutputFolder: options.OutputDirectory, DumpRequest: options.DumpRequest, DumpResponse: options.DumpResponse, + MaxSize: options.MaxSize, Elastic: options.Elastic, Kafka: options.Kafka, }) From f8e7147680ddfe046fd687d95e66c94568a547a6 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 24 Feb 2023 17:01:57 +0100 Subject: [PATCH 09/10] bumping go version + readme update --- .github/workflows/build-test.yml | 2 +- .github/workflows/lint-test.yml | 2 +- .github/workflows/release-binary.yml | 2 +- .github/workflows/sonarcloud.yml | 2 +- README.md | 18 +++++++++--------- go.mod | 2 +- go.sum | 2 -- 7 files changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index e95251fc..b7a9a81b 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.19 - name: Check out code uses: actions/checkout@v3 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index ebb10384..e098957a 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.19 - name: Run golangci-lint uses: golangci/golangci-lint-action@v3.4.0 diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index c4408a33..7da4791d 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -18,7 +18,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.19 - name: "Create release on GitHub" uses: goreleaser/goreleaser-action@v4 diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 20e7e48f..b88eab46 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -20,7 +20,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.19 - name: Run unit Tests run: | diff --git a/README.md b/README.md index 93c1122c..fdd5de0e 100644 --- a/README.md +++ b/README.md @@ -90,15 +90,15 @@ PROXY: -c int Number of requests before switching to the next upstream proxy (default 1) EXPORT: - -elastic-address string elasticsearch address (ip:port) - -elastic-ssl enable elasticsearch ssl - -elastic-ssl-verification enable elasticsearch ssl verification - -elastic-username string elasticsearch username - -elastic-password string elasticsearch password - -elastic-max-field-size int elasticsearch max field size (default 9223372036854775807) - -elastic-index string elasticsearch index name (default "proxify") - -kafka-address string address of kafka broker (ip:port) - -kafka-topic string kafka topic to publish messages on (default "proxify") + -max-size int Max export data size (request/responses will be truncated) (default 9223372036854775807) + -elastic-address string elasticsearch address (ip:port) + -elastic-ssl enable elasticsearch ssl + -elastic-ssl-verification enable elasticsearch ssl verification + -elastic-username string elasticsearch username + -elastic-password string elasticsearch password + -elastic-index string elasticsearch index name (default "proxify") + -kafka-address string address of kafka broker (ip:port) + -kafka-topic string kafka topic to publish messages on (default "proxify") CONFIGURATION: -config string Directory for storing program information (default "$HOME/.config/proxify") diff --git a/go.mod b/go.mod index 42be0fbb..b7980610 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/projectdiscovery/proxify -go 1.18 +go 1.19 require ( github.com/Knetic/govaluate v3.0.0+incompatible diff --git a/go.sum b/go.sum index f66e9981..a74dfbbb 100644 --- a/go.sum +++ b/go.sum @@ -439,8 +439,6 @@ github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZA github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= github.com/projectdiscovery/tinydns v0.0.1 h1:Ls5TAlMJoCfEObpsC7S+Hg4WLyF5wiN+O1xNgEIjnEc= github.com/projectdiscovery/tinydns v0.0.1/go.mod h1:xPZcaMje/MWozKc5Sdln3XWptINYoWqCBXgLeSrNB1Q= -github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d h1:iB/n2/NL4oh1IaEcqX6pBxj0WHfYN7finzNOKVNVISM= -github.com/projectdiscovery/utils v0.0.4-0.20230117135930-7371ae6a739d/go.mod h1:PCwA5YuCYWPgHaGiZmr53/SA9iGQmAnw7DSHuhr8VPQ= github.com/projectdiscovery/utils v0.0.11 h1:JwyW3Fp7iF6d3XjzsaC+Gz38EROYuXlMxm7re7LyGvk= github.com/projectdiscovery/utils v0.0.11/go.mod h1:0AqTr9qCXU7dD+c1hItoVY2xhr4gLr3U4/ABiQvhW1U= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= From 53ba2ed6763451f3086354dd20b4115cbb37affb Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Sun, 26 Feb 2023 23:52:18 +0530 Subject: [PATCH 10/10] Delete sonarcloud.yml --- .github/workflows/sonarcloud.yml | 38 -------------------------------- 1 file changed, 38 deletions(-) delete mode 100644 .github/workflows/sonarcloud.yml diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml deleted file mode 100644 index b88eab46..00000000 --- a/.github/workflows/sonarcloud.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: 👮🏼‍♂️ Sonarcloud -on: - push: - branches: - - master - - dev - pull_request: - types: [opened, synchronize, reopened] - workflow_dispatch: - -jobs: - sonarcloud: - name: SonarCloud - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - - - name: "Set up Go" - uses: actions/setup-go@v3 - with: - go-version: 1.19 - - - name: Run unit Tests - run: | - go test -coverprofile=./cov.out ./... - - - name: Run Gosec Security Scanner - uses: securego/gosec@master - with: - args: '-no-fail -fmt=sonarqube -out report.json ./...' - - - name: SonarCloud Scan - uses: SonarSource/sonarcloud-github-action@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file