Grow the Generic Embedded Config container to fit large config (#3209)

With very large artifacts, the embedded config space reserved in the
binary may be exceeded. In this case it should still be possible to use
the Generic Collector container to hold large embedded artifact
definitions.

This PR allows larger configs to be embedded in the GenericCollector
container.

actions/client_info.go

@@ -12,7 +12,7 @@ import (
 
 // Return essential information about the client used for indexing
 // etc. This augments the interrogation workflow via the
-// Server.Internal.ClientInfo artifact. We send this message tothe
+// Server.Internal.ClientInfo artifact. We send this message to the
 // server periodically to avoid having to issue Generic.Client.Info
 // hunts all the time.
 func GetClientInfo(

artifacts/definitions/Server/Utils/CreateCollector.yaml

@@ -470,6 +470,8 @@ sources:
                dict(name="SleepDuration", type="int", default="0"),
                dict(name="ToolName"),
                dict(name="ToolInfo"),
+               dict(name="TemporaryOnly", type="bool"),
+               dict(name="Version"),
                dict(name="IsExecutable", type="bool", default="Y"),
             ) AS parameters,
             (

config/embedded.go

@@ -0,0 +1,84 @@
+package config
+
+import (
+	"bytes"
+	"compress/zlib"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/Velocidex/yaml/v2"
+	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+)
+
+func ExtractEmbeddedConfig(
+	embedded_file string) (*config_proto.Config, error) {
+
+	fd, err := os.Open(embedded_file)
+	if err != nil {
+		return nil, err
+	}
+
+	// Read a lot of the file into memory so we can extract the
+	// configuration. This solution only loads the first 10mb into
+	// memory which should be sufficient for most practical config
+	// files. If there are embedded binaries they will not be read and
+	// will be ignored at this stage (thay can be extracted with the
+	// 'me' accessor).
+	buf, err := ioutil.ReadAll(io.LimitReader(fd, 10*1024*1024))
+	if err != nil {
+		return nil, err
+	}
+
+	// Find the embedded marker in the buffer.
+	match := embedded_re.FindIndex(buf)
+	if match == nil {
+		return nil, noEmbeddedConfig
+	}
+
+	embedded_string := buf[match[0]:]
+	return decode_embedded_config(embedded_string)
+}
+
+func read_embedded_config() (*config_proto.Config, error) {
+	return decode_embedded_config(FileConfigDefaultYaml)
+}
+
+func decode_embedded_config(encoded_string []byte) (*config_proto.Config, error) {
+	// Get the first line which is never disturbed
+	idx := bytes.IndexByte(encoded_string, '\n')
+
+	if len(encoded_string) < idx+10 {
+		return nil, noEmbeddedConfig
+	}
+
+	// If the following line still starts with # then the file is not
+	// repacked - the repacker will replace all further data with the
+	// compressed string.
+	if encoded_string[idx+1] == '#' {
+		return nil, noEmbeddedConfig
+	}
+
+	// Decompress the rest of the data - note that zlib will ignore
+	// any padding anyway because the zlib header already contains the
+	// length of the compressed data so it is safe to just feed it the
+	// whole string here.
+	r, err := zlib.NewReader(bytes.NewReader(encoded_string[idx+1:]))
+	if err != nil {
+		return nil, err
+	}
+
+	b := &bytes.Buffer{}
+	_, err = io.Copy(b, r)
+	if err != nil {
+		return nil, err
+	}
+	r.Close()
+
+	result := &config_proto.Config{}
+	err = yaml.Unmarshal(b.Bytes(), result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}

config/loader.go

@@ -1,10 +1,7 @@
 package config
 
 import (
-	"bytes"
-	"compress/zlib"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
 	"os/user"
@@ -351,7 +348,6 @@ func (self *Loader) WithEmbedded(embedded_file string) *Loader {
 				EmbeddedFile, err = os.Executable()
 				return result, err
 			}
-
 			// Ensure the "me" accessor uses this file for embedded zip.
 			full_path, err := filepath.Abs(embedded_file)
 			if err != nil {
@@ -360,31 +356,12 @@ func (self *Loader) WithEmbedded(embedded_file string) *Loader {
 
 			EmbeddedFile = full_path
 
-			fd, err := os.Open(full_path)
-			if err != nil {
-				return nil, err
-			}
-
-			buf := make([]byte, len(FileConfigDefaultYaml)+1024)
-			n, err := fd.Read(buf)
-			if err != nil {
-				return nil, err
-			}
-
-			buf = buf[:n]
-
-			// Find the embedded marker in the buffer.
-			match := embedded_re.FindIndex(buf)
-			if match == nil {
-				return nil, noEmbeddedConfig
-			}
-
-			embedded_string := buf[match[0]:]
-			result, err := decode_embedded_config(embedded_string)
+			result, err := ExtractEmbeddedConfig(full_path)
 			if err == nil {
-				self.Log("Loaded embedded config from %v", embedded_file)
+				self.Log("Loaded embedded config from %v", full_path)
 			}
 			return result, err
+
 		}})
 	return self
 }
@@ -547,49 +524,6 @@ func (self *Loader) LoadAndValidate() (*config_proto.Config, error) {
 	return nil, errors.New("Unable to load config from any source.")
 }
 
-func read_embedded_config() (*config_proto.Config, error) {
-	return decode_embedded_config(FileConfigDefaultYaml)
-}
-
-func decode_embedded_config(encoded_string []byte) (*config_proto.Config, error) {
-	// Get the first line which is never disturbed
-	idx := bytes.IndexByte(encoded_string, '\n')
-
-	if len(encoded_string) < idx+10 {
-		return nil, noEmbeddedConfig
-	}
-
-	// If the following line still starts with # then the file is not
-	// repacked - the repacker will replace all further data with the
-	// compressed string.
-	if encoded_string[idx+1] == '#' {
-		return nil, noEmbeddedConfig
-	}
-
-	// Decompress the rest of the data - note that zlib will ignore
-	// any padding anyway because the zlib header already contains the
-	// length of the compressed data so it is safe to just feed it the
-	// whole string here.
-	r, err := zlib.NewReader(bytes.NewReader(encoded_string[idx+1:]))
-	if err != nil {
-		return nil, err
-	}
-
-	b := &bytes.Buffer{}
-	_, err = io.Copy(b, r)
-	if err != nil {
-		return nil, err
-	}
-	r.Close()
-
-	result := &config_proto.Config{}
-	err = yaml.Unmarshal(b.Bytes(), result)
-	if err != nil {
-		return nil, err
-	}
-	return result, nil
-}
-
 func read_config_from_file(filename string) (*config_proto.Config, error) {
 	result := &config_proto.Config{}
 

config/proto/config.proto

@@ -262,7 +262,7 @@ message ClientConfig {
     uint64 default_server_flow_stats_update = 39;
 
     // Clients will send a Server.Internal.ClientInfo message to the
-    // server every this many seconds.This helps to keep the server
+    // server every this many seconds. This helps to keep the server
     // info up to date about each client. This should not be sent too
     // frequently. The default is 1 day (86400 seconds).
     int64 client_info_update_time = 40;

services/interrogation/interrogation.go

@@ -275,12 +275,12 @@ func modifyRecord(ctx context.Context,
 	label_array, ok := row.GetStrings("Labels")
 	if ok {
 		client_info.Labels = append(client_info.Labels, label_array...)
+		client_info.Labels = utils.Uniquify(client_info.Labels)
 	}
 
 	mac_addresses, ok := row.GetStrings("MACAddresses")
 	if ok {
-		client_info.MacAddresses = mac_addresses
-		client_info.MacAddresses = utils.Uniquify(client_info.MacAddresses)
+		client_info.MacAddresses = utils.Uniquify(mac_addresses)
 	}
 
 	if client_info.FirstSeenAt == 0 {

utils/utils.go

@@ -153,6 +153,20 @@ func SlicesEqual(a []string, b []string) bool {
 	return true
 }
 
+func BytesEqual(a []byte, b []byte) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	for idx, a_item := range a {
+		if a_item != b[idx] {
+			return false
+		}
+	}
+
+	return true
+}
+
 func ToString(x interface{}) string {
 	switch t := x.(type) {
 	case string:

vql/tools/repack.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"io/ioutil"
 	"regexp"
 	"strings"
@@ -41,6 +40,7 @@ import (
 )
 
 var (
+	generic_re      = []byte(`#!/bin/sh`)
 	embedded_re     = regexp.MustCompile(`#{3}<Begin Embedded Config>\r?\n`)
 	embedded_msi_re = regexp.MustCompile(`## Velociraptor client configuration`)
 )
@@ -126,8 +126,10 @@ func (self RepackFunction) Call(ctx context.Context,
 	}
 	w.Close()
 
-	if b.Len() > len(config.FileConfigDefaultYaml)-40 {
-		return fmt.Errorf("config file is too large to embed.")
+	exe_bytes, err = resizeEmbeddedSize(exe_bytes, b.Len())
+	if err != nil {
+		scope.Log("ERROR:client_repack: %v", err)
+		return vfilter.Null{}
 	}
 
 	compressed_config_data := b.Bytes()
@@ -245,6 +247,31 @@ func readExeFile(
 	return exe_bytes[:n], nil
 }
 
+func resizeEmbeddedSize(
+	exe_bytes []byte, required_size int) ([]byte, error) {
+	if len(exe_bytes) < 100 {
+		return nil, errors.New("Binary is too small to resize")
+	}
+
+	// Are we dealing with the generic collector? It has an unlimited
+	// size so we can just increase it to the required size.
+	if utils.BytesEqual(exe_bytes[:len(generic_re)], generic_re) {
+		resize_bytes := make([]byte, len(exe_bytes)+required_size)
+		for i := 0; i < len(exe_bytes); i++ {
+			resize_bytes[i] = exe_bytes[i]
+		}
+		return resize_bytes, nil
+	}
+
+	// For real binaries we have limited space determined by the
+	// compiled in placeholder.
+	if required_size > len(config.FileConfigDefaultYaml)-40 {
+		return nil, errors.New("config file is too large to embed.")
+	}
+
+	return exe_bytes, nil
+}
+
 func RepackMSI(
 	ctx context.Context,
 	scope vfilter.Scope, upload_name string,