Max elastic exported request/response size

.github/workflows/build-test.yml

@@ -1,7 +1,6 @@
 name: 🔨 Build Test
 
 on:
-  push:
   pull_request:
   workflow_dispatch:
 

.github/workflows/codeql-analysis.yml

@@ -2,7 +2,6 @@ name: 🚨 CodeQL Analysis
 
 on:
   workflow_dispatch:
-  push:
   pull_request:
     branches:
       - dev

.github/workflows/lint-test.yml

@@ -1,6 +1,5 @@
 name: 🙏🏻 Lint Test
 on:
-  push:
   pull_request:
   workflow_dispatch:
 

README.md

@@ -72,10 +72,10 @@ OUTPUT:
    -dump-resp          Dump only HTTP responses to output file
 
 FILTER:
-   -req-fd, -request-dsl string                   Request Filter DSL
-   -resp-fd, -response-dsl string                 Response Filter DSL
-   -req-mrd, -request-match-replace-dsl string    Request Match-Replace DSL
-   -resp-mrd, -response-match-replace-dsl string  Response Match-Replace DSL
+   -req-fd, -request-dsl string[]                   Request Filter DSL
+   -resp-fd, -response-dsl string[]                 Response Filter DSL
+   -req-mrd, -request-match-replace-dsl string[]    Request Match-Replace DSL
+   -resp-mrd, -response-match-replace-dsl string[]  Response Match-Replace DSL
 
 NETWORK:
    -ha, -http-addr string    Listening HTTP IP and Port address (ip:port) (default "127.0.0.1:8888")
@@ -85,32 +85,33 @@ NETWORK:
    -r, -resolver string      Custom DNS resolvers to use (ip:port)
 
 PROXY:
-   -hp, -http-proxy string    Upstream HTTP Proxies (eg http://proxy-ip:proxy-port
-   -sp, -socks5-proxy string  Upstream SOCKS5 Proxies (eg socks5://proxy-ip:proxy-port)
-   -c int                     Number of requests before switching to the next upstream proxy (default 1)
+   -hp, -http-proxy string[]    Upstream HTTP Proxies (eg http://proxy-ip:proxy-port)
+   -sp, -socks5-proxy string[]  Upstream SOCKS5 Proxies (eg socks5://proxy-ip:proxy-port)
+   -c int                       Number of requests before switching to the next upstream proxy (default 1)
 
 EXPORT:
-   -elastic-address string    elasticsearch address (ip:port)
-   -elastic-ssl               enable elasticsearch ssl
-   -elastic-ssl-verification  enable elasticsearch ssl verification
-   -elastic-username string   elasticsearch username
-   -elastic-password string   elasticsearch password
-   -elastic-index string      elasticsearch index name (default "proxify")
-   -kafka-address string      address of kafka broker (ip:port)
-   -kafka-topic string        kafka topic to publish messages on (default "proxify")
+   -elastic-address string      elasticsearch address (ip:port)
+   -elastic-ssl                 enable elasticsearch ssl
+   -elastic-ssl-verification    enable elasticsearch ssl verification
+   -elastic-username string     elasticsearch username
+   -elastic-password string     elasticsearch password
+   -elastic-max-field-size int  elasticsearch max field size (default 9223372036854775807)
+   -elastic-index string        elasticsearch index name (default "proxify")
+   -kafka-address string        address of kafka broker (ip:port)
+   -kafka-topic string          kafka topic to publish messages on (default "proxify")
 
 CONFIGURATION:
-   -config string        Directory for storing program information (default "/Users/geekboy/.config/proxify")
+   -config string        Directory for storing program information (default "$HOME/.config/proxify")
    -cert-cache-size int  Number of certificates to cache (default 256)
-   -allow string         Allowed list of IP/CIDR's to be proxied
-   -deny string          Denied list of IP/CIDR's to be proxied
+   -a, -allow string[]   Allowed list of IP/CIDR's to be proxied
+   -d, -deny string[]    Denied list of IP/CIDR's to be proxied
 
 DEBUG:
-   -nc, -no-color       No Color (default true)
-   -version             Version
-   -silent              Silent
-   -v, -verbose         Verbose
-   -vv, -very-verbose   Very Verbose
+   -nc, -no-color      No Color (default true)
+   -version            Version
+   -silent             Silent
+   -v, -verbose        Verbose
+   -vv, -very-verbose  Very Verbose
 ```
 
 ### Running Proxify

cmd/proxify/proxify.go

@@ -21,19 +21,17 @@ func main() {
 
 	// Setup close handler
 	go func() {
-		c := make(chan os.Signal,1) //added size 1 to channel buffer
+		c := make(chan os.Signal, 1)
 		signal.Notify(c, os.Interrupt, syscall.SIGTERM)
-		go func() {
-			<-c
+		for range c {
 			fmt.Println("\r- Ctrl+C pressed in Terminal")
 			proxifyRunner.Close()
 			os.Exit(0)
-		}()
+		}
 	}()
 
 	err = proxifyRunner.Run()
 	if err != nil {
 		gologger.Fatal().Msgf("Could not run proxify: %s\n", err)
 	}
-
 }

internal/runner/options.go

@@ -1,8 +1,9 @@
 package runner
 
 import (
+	"math"
 	"os"
-	"path"
+	"path/filepath"
 
 	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
@@ -53,55 +54,56 @@ func ParseOptions() *Options {
 	flagSet := goflags.NewFlagSet()
 	flagSet.SetDescription(`Swiss Army Knife Proxy for rapid deployments. Supports multiple operations such as request/response dump,filtering and manipulation via DSL language, upstream HTTP/Socks5 proxy`)
 
-	createGroup(flagSet, "output", "Output",
+	flagSet.CreateGroup("output", "Output",
 		// Todo:	flagSet.BoolVar(&options.Dump, "dump", true, "Dump HTTP requests/response to output file"),
 		flagSet.StringVarP(&options.OutputDirectory, "output", "o", "logs", "Output Directory to store HTTP proxy logs"),
 		flagSet.BoolVar(&options.DumpRequest, "dump-req", false, "Dump only HTTP requests to output file"),
 		flagSet.BoolVar(&options.DumpResponse, "dump-resp", false, "Dump only HTTP responses to output file"),
 	)
 
-	createGroup(flagSet, "filter", "Filter",
+	flagSet.CreateGroup("filter", "Filter",
 		flagSet.StringSliceVarP(&options.RequestDSL, "request-dsl", "req-fd", nil, "Request Filter DSL", goflags.StringSliceOptions),
 		flagSet.StringSliceVarP(&options.ResponseDSL, "response-dsl", "resp-fd", nil, "Response Filter DSL", goflags.StringSliceOptions),
 		flagSet.StringSliceVarP(&options.RequestMatchReplaceDSL, "request-match-replace-dsl", "req-mrd", nil, "Request Match-Replace DSL", goflags.StringSliceOptions),
 		flagSet.StringSliceVarP(&options.ResponseMatchReplaceDSL, "response-match-replace-dsl", "resp-mrd", nil, "Response Match-Replace DSL", goflags.StringSliceOptions),
 	)
 
-	createGroup(flagSet, "network", "Network",
+	flagSet.CreateGroup("network", "Network",
 		flagSet.StringVarP(&options.ListenAddrHTTP, "http-addr", "ha", "127.0.0.1:8888", "Listening HTTP IP and Port address (ip:port)"),
 		flagSet.StringVarP(&options.ListenAddrSocks5, "socks-addr", "sa", "127.0.0.1:10080", "Listening SOCKS IP and Port address (ip:port)"),
 		flagSet.StringVarP(&options.ListenDNSAddr, "dns-addr", "da", "", "Listening DNS IP and Port address (ip:port)"),
 		flagSet.StringVarP(&options.DNSMapping, "dns-mapping", "dm", "", "Domain to IP DNS mapping (eg domain:ip,domain:ip,..)"),
 		flagSet.StringVarP(&options.DNSFallbackResolver, "resolver", "r", "", "Custom DNS resolvers to use (ip:port)"),
 	)
 
-	createGroup(flagSet, "proxy", "Proxy",
+	flagSet.CreateGroup("proxy", "Proxy",
 		flagSet.StringSliceVarP(&options.UpstreamHTTPProxies, "http-proxy", "hp", nil, "Upstream HTTP Proxies (eg http://proxy-ip:proxy-port)", goflags.NormalizedStringSliceOptions),
 		flagSet.StringSliceVarP(&options.UpstreamSocks5Proxies, "socks5-proxy", "sp", nil, "Upstream SOCKS5 Proxies (eg socks5://proxy-ip:proxy-port)", goflags.NormalizedStringSliceOptions),
 		flagSet.IntVar(&options.UpstreamProxyRequestsNumber, "c", 1, "Number of requests before switching to the next upstream proxy"),
 	)
 
-	createGroup(flagSet, "export", "Export",
+	flagSet.CreateGroup("export", "Export",
 		flagSet.StringVar(&options.Elastic.Addr, "elastic-address", "", "elasticsearch address (ip:port)"),
 		flagSet.BoolVar(&options.Elastic.SSL, "elastic-ssl", false, "enable elasticsearch ssl"),
 		flagSet.BoolVar(&options.Elastic.SSLVerification, "elastic-ssl-verification", false, "enable elasticsearch ssl verification"),
 		flagSet.StringVar(&options.Elastic.Username, "elastic-username", "", "elasticsearch username"),
 		flagSet.StringVar(&options.Elastic.Password, "elastic-password", "", "elasticsearch password"),
+		flagSet.IntVar(&options.Elastic.MaxFieldSize, "elastic-max-field-size", math.MaxInt, "elasticsearch max field size"),
 		flagSet.StringVar(&options.Elastic.IndexName, "elastic-index", "proxify", "elasticsearch index name"),
 		flagSet.StringVar(&options.Kafka.Addr, "kafka-address", "", "address of kafka broker (ip:port)"),
 		flagSet.StringVar(&options.Kafka.Topic, "kafka-topic", "proxify", "kafka topic to publish messages on"),
 	)
 
-	createGroup(flagSet, "configuration", "Configuration",
+	flagSet.CreateGroup("configuration", "Configuration",
 		// Todo: default config file support (homeDir/.config/proxify/config.yaml)
-		flagSet.StringVar(&options.Directory, "config", path.Join(homeDir, ".config", "proxify"), "Directory for storing program information"),
+		flagSet.StringVar(&options.Directory, "config", filepath.Join(homeDir, ".config", "proxify"), "Directory for storing program information"),
 		flagSet.IntVar(&options.CertCacheSize, "cert-cache-size", 256, "Number of certificates to cache"),
 		flagSet.StringSliceVarP(&options.Allow, "allow", "a", nil, "Allowed list of IP/CIDR's to be proxied", goflags.FileNormalizedStringSliceOptions),
 		flagSet.StringSliceVarP(&options.Deny, "deny", "d", nil, "Denied list of IP/CIDR's to be proxied", goflags.FileNormalizedStringSliceOptions),
 	)
 
 	silent, verbose, veryVerbose := false, false, false
-	createGroup(flagSet, "debug", "debug",
+	flagSet.CreateGroup("debug", "debug",
 		flagSet.BoolVarP(&options.NoColor, "no-color", "nc", true, "No Color"),
 		flagSet.BoolVar(&options.Version, "version", false, "Version"),
 		flagSet.BoolVar(&silent, "silent", false, "Silent"),
@@ -154,10 +156,3 @@ func (options *Options) configureOutput() {
 		gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true))
 	}
 }
-
-func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) {
-	flagSet.SetGroup(groupName, description)
-	for _, currentFlag := range flags {
-		currentFlag.Group(groupName)
-	}
-}

pkg/logger/elastic/elasticsearch.go

@@ -29,11 +29,14 @@ type Options struct {
 	Password string `yaml:"password"`
 	// IndexName is the name of the elasticsearch index
 	IndexName string `yaml:"index-name"`
+	// MaxFieldSize sets a maximum limit to the field length
+	MaxFieldSize int `yaml:"max-field-size"`
 }
 
 // Client type for elasticsearch
 type Client struct {
 	index    string
+	options  *Options
 	esClient *elasticsearch.Client
 }
 
@@ -57,10 +60,12 @@ func New(option *Options) (*Client, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "error creating elasticsearch client")
 	}
-	return &Client{
+	client := &Client{
 		esClient: elasticsearchClient,
 		index:    option.IndexName,
-	}, nil
+		options:  option,
+	}
+	return client, nil
 
 }
 
@@ -69,12 +74,12 @@ func (c *Client) Save(data types.OutputData) error {
 	var doc map[string]interface{}
 	if data.Userdata.HasResponse {
 		doc = map[string]interface{}{
-			"response":  data.DataString,
+			"response":  truncate(data.DataString, c.options.MaxFieldSize),
 			"timestamp": time.Now().Format(time.RFC3339),
 		}
 	} else {
 		doc = map[string]interface{}{
-			"request":   data.DataString,
+			"request":   truncate(data.DataString, c.options.MaxFieldSize),
 			"timestamp": time.Now().Format(time.RFC3339),
 		}
 	}
@@ -103,3 +108,10 @@ func (c *Client) Save(data types.OutputData) error {
 	res.Body.Close()
 	return er
 }
+
+func truncate(data string, maxSize int) string {
+	if maxSize > 0 && len(data) > maxSize {
+		return data[:maxSize]
+	}
+	return data
+}