Grow the Generic Embedded Config container to fit large config

actions/client_info.go

@@ -12,7 +12,7 @@ import (
 
 // Return essential information about the client used for indexing
 // etc. This augments the interrogation workflow via the
-// Server.Internal.ClientInfo artifact. We send this message tothe
+// Server.Internal.ClientInfo artifact. We send this message to the
 // server periodically to avoid having to issue Generic.Client.Info
 // hunts all the time.
 func GetClientInfo(

artifacts/definitions/Server/Utils/CreateCollector.yaml

@@ -470,6 +470,8 @@ sources:
                dict(name="SleepDuration", type="int", default="0"),
                dict(name="ToolName"),
                dict(name="ToolInfo"),
+               dict(name="TemporaryOnly", type="bool"),
+               dict(name="Version"),
                dict(name="IsExecutable", type="bool", default="Y"),
             ) AS parameters,
             (

config/embedded.go

@@ -0,0 +1,84 @@
+package config
+
+import (
+	"bytes"
+	"compress/zlib"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/Velocidex/yaml/v2"
+	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+)
+
+func ExtractEmbeddedConfig(
+	embedded_file string) (*config_proto.Config, error) {
+
+	fd, err := os.Open(embedded_file)
+	if err != nil {
+		return nil, err
+	}
+
+	// Read a lot of the file into memory so we can extract the
+	// configuration. This solution only loads the first 10mb into
+	// memory which should be sufficient for most practical config
+	// files. If there are embedded binaries they will not be read and
+	// will be ignored at this stage (thay can be extracted with the
+	// 'me' accessor).
+	buf, err := ioutil.ReadAll(io.LimitReader(fd, 10*1024*1024))
+	if err != nil {
+		return nil, err
+	}
+
+	// Find the embedded marker in the buffer.
+	match := embedded_re.FindIndex(buf)
+	if match == nil {
+		return nil, noEmbeddedConfig
+	}
+
+	embedded_string := buf[match[0]:]
+	return decode_embedded_config(embedded_string)
+}
+
+func read_embedded_config() (*config_proto.Config, error) {
+	return decode_embedded_config(FileConfigDefaultYaml)
+}
+
+func decode_embedded_config(encoded_string []byte) (*config_proto.Config, error) {
+	// Get the first line which is never disturbed
+	idx := bytes.IndexByte(encoded_string, '\n')
+
+	if len(encoded_string) < idx+10 {
+		return nil, noEmbeddedConfig
+	}
+
+	// If the following line still starts with # then the file is not
+	// repacked - the repacker will replace all further data with the
+	// compressed string.
+	if encoded_string[idx+1] == '#' {
+		return nil, noEmbeddedConfig
+	}
+
+	// Decompress the rest of the data - note that zlib will ignore
+	// any padding anyway because the zlib header already contains the
+	// length of the compressed data so it is safe to just feed it the
+	// whole string here.
+	r, err := zlib.NewReader(bytes.NewReader(encoded_string[idx+1:]))
+	if err != nil {
+		return nil, err
+	}
+
+	b := &bytes.Buffer{}
+	_, err = io.Copy(b, r)
+	if err != nil {
+		return nil, err
+	}
+	r.Close()
+
+	result := &config_proto.Config{}
+	err = yaml.Unmarshal(b.Bytes(), result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}

config/loader.go

@@ -1,10 +1,7 @@
 package config
 
 import (
-	"bytes"
-	"compress/zlib"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
 	"os/user"
@@ -351,7 +348,6 @@ func (self *Loader) WithEmbedded(embedded_file string) *Loader {
 				EmbeddedFile, err = os.Executable()
 				return result, err
 			}
-
 			// Ensure the "me" accessor uses this file for embedded zip.
 			full_path, err := filepath.Abs(embedded_file)
 			if err != nil {
@@ -360,31 +356,12 @@ func (self *Loader) WithEmbedded(embedded_file string) *Loader {
 
 			EmbeddedFile = full_path
 
-			fd, err := os.Open(full_path)
-			if err != nil {
-				return nil, err
-			}
-
-			buf := make([]byte, len(FileConfigDefaultYaml)+1024)
-			n, err := fd.Read(buf)
-			if err != nil {
-				return nil, err
-			}
-
-			buf = buf[:n]
-
-			// Find the embedded marker in the buffer.
-			match := embedded_re.FindIndex(buf)
-			if match == nil {
-				return nil, noEmbeddedConfig
-			}
-
-			embedded_string := buf[match[0]:]
-			result, err := decode_embedded_config(embedded_string)
+			result, err := ExtractEmbeddedConfig(full_path)
 			if err == nil {
-				self.Log("Loaded embedded config from %v", embedded_file)
+				self.Log("Loaded embedded config from %v", full_path)
 			}
 			return result, err
+
 		}})
 	return self
 }
@@ -547,49 +524,6 @@ func (self *Loader) LoadAndValidate() (*config_proto.Config, error) {
 	return nil, errors.New("Unable to load config from any source.")
 }
 
-func read_embedded_config() (*config_proto.Config, error) {
-	return decode_embedded_config(FileConfigDefaultYaml)
-}
-
-func decode_embedded_config(encoded_string []byte) (*config_proto.Config, error) {
-	// Get the first line which is never disturbed
-	idx := bytes.IndexByte(encoded_string, '\n')
-
-	if len(encoded_string) < idx+10 {
-		return nil, noEmbeddedConfig
-	}
-
-	// If the following line still starts with # then the file is not
-	// repacked - the repacker will replace all further data with the
-	// compressed string.
-	if encoded_string[idx+1] == '#' {
-		return nil, noEmbeddedConfig
-	}
-
-	// Decompress the rest of the data - note that zlib will ignore
-	// any padding anyway because the zlib header already contains the
-	// length of the compressed data so it is safe to just feed it the
-	// whole string here.
-	r, err := zlib.NewReader(bytes.NewReader(encoded_string[idx+1:]))
-	if err != nil {
-		return nil, err
-	}
-
-	b := &bytes.Buffer{}
-	_, err = io.Copy(b, r)
-	if err != nil {
-		return nil, err
-	}
-	r.Close()
-
-	result := &config_proto.Config{}
-	err = yaml.Unmarshal(b.Bytes(), result)
-	if err != nil {
-		return nil, err
-	}
-	return result, nil
-}
-
 func read_config_from_file(filename string) (*config_proto.Config, error) {
 	result := &config_proto.Config{}
 

config/proto/config.proto

@@ -262,7 +262,7 @@ message ClientConfig {
     uint64 default_server_flow_stats_update = 39;
 
     // Clients will send a Server.Internal.ClientInfo message to the
-    // server every this many seconds.This helps to keep the server
+    // server every this many seconds. This helps to keep the server
     // info up to date about each client. This should not be sent too
     // frequently. The default is 1 day (86400 seconds).
     int64 client_info_update_time = 40;

services/interrogation/interrogation.go

@@ -264,12 +264,12 @@ func modifyRecord(ctx context.Context,
 	label_array, ok := row.GetStrings("Labels")
 	if ok {
 		client_info.Labels = append(client_info.Labels, label_array...)
+		client_info.Labels = utils.Uniquify(client_info.Labels)
 	}
 
 	mac_addresses, ok := row.GetStrings("MACAddresses")
 	if ok {
-		client_info.MacAddresses = mac_addresses
-		client_info.MacAddresses = utils.Uniquify(client_info.MacAddresses)
+		client_info.MacAddresses = utils.Uniquify(mac_addresses)
 	}
 
 	if client_info.FirstSeenAt == 0 {

utils/utils.go

@@ -153,6 +153,20 @@ func SlicesEqual(a []string, b []string) bool {
 	return true
 }
 
+func BytesEqual(a []byte, b []byte) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	for idx, a_item := range a {
+		if a_item != b[idx] {
+			return false
+		}
+	}
+
+	return true
+}
+
 func ToString(x interface{}) string {
 	switch t := x.(type) {
 	case string:

vql/tools/repack.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
-	"fmt"
 	"io/ioutil"
 	"regexp"
 	"strings"
@@ -41,6 +40,7 @@ import (
 )
 
 var (
+	generic_re      = []byte(`#!/bin/sh`)
 	embedded_re     = regexp.MustCompile(`#{3}<Begin Embedded Config>\r?\n`)
 	embedded_msi_re = regexp.MustCompile(`## Velociraptor client configuration`)
 )
@@ -126,8 +126,10 @@ func (self RepackFunction) Call(ctx context.Context,
 	}
 	w.Close()
 
-	if b.Len() > len(config.FileConfigDefaultYaml)-40 {
-		return fmt.Errorf("config file is too large to embed.")
+	exe_bytes, err = resizeEmbeddedSize(exe_bytes, b.Len())
+	if err != nil {
+		scope.Log("ERROR:client_repack: %v", err)
+		return vfilter.Null{}
 	}
 
 	compressed_config_data := b.Bytes()
@@ -245,6 +247,31 @@ func readExeFile(
 	return exe_bytes[:n], nil
 }
 
+func resizeEmbeddedSize(
+	exe_bytes []byte, required_size int) ([]byte, error) {
+	if len(exe_bytes) < 100 {
+		return nil, errors.New("Binary is too small to resize")
+	}
+
+	// Are we dealing with the generic collector? It has an unlimited
+	// size so we can just increase it to the required size.
+	if utils.BytesEqual(exe_bytes[:len(generic_re)], generic_re) {
+		resize_bytes := make([]byte, len(exe_bytes)+required_size)
+		for i := 0; i < len(exe_bytes); i++ {
+			resize_bytes[i] = exe_bytes[i]
+		}
+		return resize_bytes, nil
+	}
+
+	// For real binaries we have limited space determined by the
+	// compiled in placeholder.
+	if required_size > len(config.FileConfigDefaultYaml)-40 {
+		return nil, errors.New("config file is too large to embed.")
+	}
+
+	return exe_bytes, nil
+}
+
 func RepackMSI(
 	ctx context.Context,
 	scope vfilter.Scope, upload_name string,