From d678648021df8d38f37fb49e8d6981a7f67e7521 Mon Sep 17 00:00:00 2001 From: Rebecca Mahany-Horton Date: Wed, 3 Jul 2024 10:58:29 -0400 Subject: [PATCH] [Kolide ATC] Construct KATC tables and add support for Firefox extension data (#1763) --- ee/katc/config.go | 120 ++++++++++++ ee/katc/config_test.go | 100 ++++++++++ ee/katc/deserialize_firefox.go | 275 +++++++++++++++++++++++++++ ee/katc/snappy.go | 26 +++ ee/katc/sqlite.go | 117 ++++++++++++ ee/katc/table.go | 172 +++++++++++++++++ ee/katc/table_test.go | 332 +++++++++++++++++++++++++++++++++ pkg/osquery/table/table.go | 22 +-- 8 files changed, 1150 insertions(+), 14 deletions(-) create mode 100644 ee/katc/config.go create mode 100644 ee/katc/config_test.go create mode 100644 ee/katc/deserialize_firefox.go create mode 100644 ee/katc/snappy.go create mode 100644 ee/katc/sqlite.go create mode 100644 ee/katc/table.go create mode 100644 ee/katc/table_test.go diff --git a/ee/katc/config.go b/ee/katc/config.go new file mode 100644 index 000000000..6d4470167 --- /dev/null +++ b/ee/katc/config.go @@ -0,0 +1,120 @@ +package katc + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "runtime" + + "github.com/osquery/osquery-go" + "github.com/osquery/osquery-go/plugin/table" +) + +// katcSourceType defines a source of data for a KATC table. The `name` is the +// identifier parsed from the JSON KATC config, and the `dataFunc` is the function +// that performs the query against the source. +type katcSourceType struct { + name string + dataFunc func(ctx context.Context, slogger *slog.Logger, path string, query string, sourceConstraints *table.ConstraintList) ([]sourceData, error) +} + +// sourceData holds the result of calling `katcSourceType.dataFunc`. It maps the +// source to the query results. (A config may have wildcards in the source, +// allowing for querying against multiple sources.) +type sourceData struct { + path string + rows []map[string][]byte +} + +const ( + sqliteSourceType = "sqlite" +) + +func (kst *katcSourceType) UnmarshalJSON(data []byte) error { + var s string + err := json.Unmarshal(data, &s) + if err != nil { + return fmt.Errorf("unmarshalling string: %w", err) + } + + switch s { + case sqliteSourceType: + kst.name = sqliteSourceType + kst.dataFunc = sqliteData + return nil + default: + return fmt.Errorf("unknown table type %s", s) + } +} + +// rowTransformStep defines an operation performed against a row of data +// returned from a source. The `name` is the identifier parsed from the +// JSON KATC config. +type rowTransformStep struct { + name string + transformFunc func(ctx context.Context, slogger *slog.Logger, row map[string][]byte) (map[string][]byte, error) +} + +const ( + snappyDecodeTransformStep = "snappy" + deserializeFirefoxTransformStep = "deserialize_firefox" +) + +func (r *rowTransformStep) UnmarshalJSON(data []byte) error { + var s string + err := json.Unmarshal(data, &s) + if err != nil { + return fmt.Errorf("unmarshalling string: %w", err) + } + + switch s { + case snappyDecodeTransformStep: + r.name = snappyDecodeTransformStep + r.transformFunc = snappyDecode + return nil + case deserializeFirefoxTransformStep: + r.name = deserializeFirefoxTransformStep + r.transformFunc = deserializeFirefox + return nil + default: + return fmt.Errorf("unknown data processing step %s", s) + } +} + +// katcTableConfig is the configuration for a specific KATC table. The control server +// sends down these configurations. +type katcTableConfig struct { + SourceType katcSourceType `json:"source_type"` + Source string `json:"source"` // Describes how to connect to source (e.g. path to db) -- % and _ wildcards supported + Platform string `json:"platform"` + Columns []string `json:"columns"` + Query string `json:"query"` // Query to run against `path` + RowTransformSteps []rowTransformStep `json:"row_transform_steps"` +} + +// ConstructKATCTables takes stored configuration of KATC tables, parses the configuration, +// and returns the constructed tables. +func ConstructKATCTables(config map[string]string, slogger *slog.Logger) []osquery.OsqueryPlugin { + plugins := make([]osquery.OsqueryPlugin, 0) + for tableName, tableConfigStr := range config { + var cfg katcTableConfig + if err := json.Unmarshal([]byte(tableConfigStr), &cfg); err != nil { + slogger.Log(context.TODO(), slog.LevelWarn, + "unable to unmarshal config for Kolide ATC table, skipping", + "table_name", tableName, + "err", err, + ) + continue + } + + if cfg.Platform != runtime.GOOS { + continue + } + + t, columns := newKatcTable(tableName, cfg, slogger) + plugins = append(plugins, table.NewPlugin(tableName, columns, t.generate)) + } + + return plugins +} diff --git a/ee/katc/config_test.go b/ee/katc/config_test.go new file mode 100644 index 000000000..e7ce530e1 --- /dev/null +++ b/ee/katc/config_test.go @@ -0,0 +1,100 @@ +package katc + +import ( + _ "embed" + "fmt" + "runtime" + "testing" + + "github.com/kolide/launcher/pkg/log/multislogger" + "github.com/stretchr/testify/require" +) + +func TestConstructKATCTables(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + testCaseName string + katcConfig map[string]string + expectedPluginCount int + }{ + { + testCaseName: "snappy_sqlite", + katcConfig: map[string]string{ + "kolide_snappy_sqlite_test": fmt.Sprintf(`{ + "source_type": "sqlite", + "platform": "%s", + "columns": ["data"], + "source": "/some/path/to/db.sqlite", + "query": "SELECT data FROM object_data JOIN object_store ON (object_data.object_store_id = object_store.id) WHERE object_store.name=\"testtable\";", + "row_transform_steps": ["snappy"] + }`, runtime.GOOS), + }, + expectedPluginCount: 1, + }, + { + testCaseName: "multiple plugins", + katcConfig: map[string]string{ + "test_1": fmt.Sprintf(`{ + "source_type": "sqlite", + "platform": "%s", + "columns": ["data"], + "source": "/some/path/to/db.sqlite", + "query": "SELECT data FROM object_data;", + "row_transform_steps": ["snappy"] + }`, runtime.GOOS), + "test_2": fmt.Sprintf(`{ + "source_type": "sqlite", + "platform": "%s", + "columns": ["col1", "col2"], + "source": "/some/path/to/a/different/db.sqlite", + "query": "SELECT col1, col2 FROM some_table;", + "row_transform_steps": [] + }`, runtime.GOOS), + }, + expectedPluginCount: 2, + }, + { + testCaseName: "malformed config", + katcConfig: map[string]string{ + "malformed_test": "this is not a config", + }, + expectedPluginCount: 0, + }, + { + testCaseName: "invalid table source", + katcConfig: map[string]string{ + "kolide_snappy_test": fmt.Sprintf(`{ + "source_type": "unknown_source", + "platform": "%s", + "columns": ["data"], + "source": "/some/path/to/db.sqlite", + "query": "SELECT data FROM object_data;" + }`, runtime.GOOS), + }, + expectedPluginCount: 0, + }, + { + testCaseName: "invalid data processing step type", + katcConfig: map[string]string{ + "kolide_snappy_test": fmt.Sprintf(`{ + "source_type": "sqlite", + "platform": "%s", + "columns": ["data"], + "source": "/some/path/to/db.sqlite", + "query": "SELECT data FROM object_data;", + "row_transform_steps": ["unknown_step"] + }`, runtime.GOOS), + }, + expectedPluginCount: 0, + }, + } { + tt := tt + t.Run(tt.testCaseName, func(t *testing.T) { + t.Parallel() + + plugins := ConstructKATCTables(tt.katcConfig, multislogger.NewNopLogger()) + require.Equal(t, tt.expectedPluginCount, len(plugins), "unexpected number of plugins") + }) + } +} diff --git a/ee/katc/deserialize_firefox.go b/ee/katc/deserialize_firefox.go new file mode 100644 index 000000000..1a50975c3 --- /dev/null +++ b/ee/katc/deserialize_firefox.go @@ -0,0 +1,275 @@ +package katc + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "strconv" + + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/transform" +) + +const ( + tagHeader uint32 = 0xfff10000 + tagNull uint32 = 0xffff0000 + tagUndefined uint32 = 0xffff0001 + tagBoolean uint32 = 0xffff0002 + tagInt32 uint32 = 0xffff0003 + tagString uint32 = 0xffff0004 + tagArrayObject uint32 = 0xffff0007 + tagObjectObject uint32 = 0xffff0008 + tagBooleanObject uint32 = 0xffff000a + tagStringObject uint32 = 0xffff000b + tagEndOfKeys uint32 = 0xffff0013 +) + +// deserializeFirefox deserializes a JS object that has been stored by Firefox +// in IndexedDB sqlite-backed databases. +// References: +// * https://stackoverflow.com/a/59923297 +// * https://searchfox.org/mozilla-central/source/js/src/vm/StructuredClone.cpp (see especially JSStructuredCloneReader::read) +func deserializeFirefox(ctx context.Context, slogger *slog.Logger, row map[string][]byte) (map[string][]byte, error) { + // IndexedDB data is stored by key "data" pointing to the serialized object. We want to + // extract that serialized object, and discard the top-level "data" key. + data, ok := row["data"] + if !ok { + return nil, errors.New("row missing top-level data key") + } + + srcReader := bytes.NewReader(data) + + // First, read the header + firstTag, _, err := nextPair(srcReader) + if err != nil { + return nil, fmt.Errorf("reading header pair: %w", err) + } + if firstTag != tagHeader { + return nil, fmt.Errorf("unknown header tag %x", firstTag) + } + + // Next up should be our top-level object + objectTag, _, err := nextPair(srcReader) + if err != nil { + return nil, fmt.Errorf("reading top-level object tag: %w", err) + } + if objectTag != tagObjectObject { + return nil, fmt.Errorf("object not found after header: expected %x, got %x", tagObjectObject, objectTag) + } + + // Read all entries in our object + resultObj, err := deserializeObject(srcReader) + if err != nil { + return nil, fmt.Errorf("reading top-level object: %w", err) + } + + return resultObj, nil +} + +// nextPair returns the next (tag, data) pair from `srcReader`. +func nextPair(srcReader io.ByteReader) (uint32, uint32, error) { + // Tags and data are written as a singular little-endian uint64 value. + // For example, the pair (`tagBoolean`, 1) is written as 01 00 00 00 02 00 FF FF, + // where 0xffff0002 is `tagBoolean`. + // To read the pair, we read the next 8 bytes in reverse order, treating the + // first four as the tag and the next four as the data. + var err error + pairBytes := make([]byte, 8) + for i := 7; i >= 0; i -= 1 { + pairBytes[i], err = srcReader.ReadByte() + if err != nil { + return 0, 0, fmt.Errorf("reading byte in pair: %w", err) + } + } + + return binary.BigEndian.Uint32(pairBytes[0:4]), binary.BigEndian.Uint32(pairBytes[4:]), nil +} + +// deserializeObject deserializes the next object from `srcReader`. +func deserializeObject(srcReader io.ByteReader) (map[string][]byte, error) { + resultObj := make(map[string][]byte, 0) + + for { + nextObjTag, nextObjData, err := nextPair(srcReader) + if err != nil { + return nil, fmt.Errorf("reading next pair in object: %w", err) + } + + if nextObjTag == tagEndOfKeys { + // All done! Return object + break + } + + // Read key + if nextObjTag != tagString { + return nil, fmt.Errorf("unsupported key type %x", nextObjTag) + } + nextKey, err := deserializeString(nextObjData, srcReader) + if err != nil { + return nil, fmt.Errorf("reading string for tag %x: %w", nextObjTag, err) + } + nextKeyStr := string(nextKey) + + // Read value + valTag, valData, err := nextPair(srcReader) + if err != nil { + return nil, fmt.Errorf("reading next pair for value in object: %w", err) + } + + switch valTag { + case tagInt32: + resultObj[nextKeyStr] = []byte(strconv.Itoa(int(valData))) + case tagString, tagStringObject: + str, err := deserializeString(valData, srcReader) + if err != nil { + return nil, fmt.Errorf("reading string for key %s: %w", nextKeyStr, err) + } + resultObj[nextKeyStr] = str + case tagObjectObject: + obj, err := deserializeNestedObject(srcReader) + if err != nil { + return nil, fmt.Errorf("reading object for key %s: %w", nextKeyStr, err) + } + resultObj[nextKeyStr] = obj + case tagArrayObject: + arr, err := deserializeArray(valData, srcReader) + if err != nil { + return nil, fmt.Errorf("reading array for key %s: %w", nextKeyStr, err) + } + resultObj[nextKeyStr] = arr + case tagNull, tagUndefined: + resultObj[nextKeyStr] = nil + default: + return nil, fmt.Errorf("cannot process %s: unknown tag type %x", nextKeyStr, valTag) + } + } + + return resultObj, nil +} + +func deserializeString(strData uint32, srcReader io.ByteReader) ([]byte, error) { + strLen := strData & bitMask(31) + isAscii := strData & (1 << 31) + + if isAscii != 0 { + return deserializeAsciiString(strLen, srcReader) + } + + return deserializeUtf16String(strLen, srcReader) +} + +func deserializeAsciiString(strLen uint32, srcReader io.ByteReader) ([]byte, error) { + // Read bytes for string + var i uint32 + var err error + strBytes := make([]byte, strLen) + for i = 0; i < strLen; i += 1 { + strBytes[i], err = srcReader.ReadByte() + if err != nil { + return nil, fmt.Errorf("reading byte in string: %w", err) + } + } + + // Now, read padding and discard -- data is stored in 8-byte words + bytesIntoNextWord := strLen % 8 + if bytesIntoNextWord > 0 { + paddingLen := 8 - bytesIntoNextWord + for i = 0; i < paddingLen; i += 1 { + _, _ = srcReader.ReadByte() + } + } + + return strBytes, nil +} + +func deserializeUtf16String(strLen uint32, srcReader io.ByteReader) ([]byte, error) { + // Two bytes per char + lenToRead := strLen * 2 + var i uint32 + var err error + strBytes := make([]byte, lenToRead) + for i = 0; i < lenToRead; i += 1 { + strBytes[i], err = srcReader.ReadByte() + if err != nil { + return nil, fmt.Errorf("reading byte in string: %w", err) + } + } + + // Now, read padding and discard -- data is stored in 8-byte words + bytesIntoNextWord := lenToRead % 8 + if bytesIntoNextWord > 0 { + paddingLen := 8 - bytesIntoNextWord + for i = 0; i < paddingLen; i += 1 { + _, _ = srcReader.ReadByte() + } + } + + // Decode string + utf16Reader := transform.NewReader(bytes.NewReader(strBytes), unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewDecoder()) + decoded, err := io.ReadAll(utf16Reader) + if err != nil { + return nil, fmt.Errorf("decoding: %w", err) + } + return decoded, nil +} + +func deserializeArray(arrayLength uint32, srcReader io.ByteReader) ([]byte, error) { + resultArr := make([]any, arrayLength) + + // We discard the next pair before reading the array. + _, _, _ = nextPair(srcReader) + + for i := 0; i < int(arrayLength); i += 1 { + itemTag, _, err := nextPair(srcReader) + if err != nil { + return nil, fmt.Errorf("reading item at index %d in array: %w", i, err) + } + + switch itemTag { + case tagObjectObject: + obj, err := deserializeNestedObject(srcReader) + if err != nil { + return nil, fmt.Errorf("reading object at index %d in array: %w", i, err) + } + resultArr[i] = string(obj) // cast to string so it's readable when marshalled again below + default: + return nil, fmt.Errorf("cannot process item at index %d in array: unsupported tag type %x", i, itemTag) + } + } + + arrBytes, err := json.Marshal(resultArr) + if err != nil { + return nil, fmt.Errorf("marshalling array: %w", err) + } + + return arrBytes, nil +} + +func deserializeNestedObject(srcReader io.ByteReader) ([]byte, error) { + nestedObj, err := deserializeObject(srcReader) + if err != nil { + return nil, fmt.Errorf("deserializing nested object: %w", err) + } + + // Make nested object values readable -- cast []byte to string + readableNestedObj := make(map[string]string) + for k, v := range nestedObj { + readableNestedObj[k] = string(v) + } + + resultObj, err := json.Marshal(readableNestedObj) + if err != nil { + return nil, fmt.Errorf("marshalling nested object: %w", err) + } + + return resultObj, nil +} + +func bitMask(n uint32) uint32 { + return (1 << n) - 1 +} diff --git a/ee/katc/snappy.go b/ee/katc/snappy.go new file mode 100644 index 000000000..6b9cece99 --- /dev/null +++ b/ee/katc/snappy.go @@ -0,0 +1,26 @@ +package katc + +import ( + "context" + "fmt" + "log/slog" + + "github.com/golang/snappy" +) + +// snappyDecode is a dataProcessingStep that decodes data compressed with snappy. +// We use this to decode data retrieved from Firefox IndexedDB sqlite-backed databases. +func snappyDecode(ctx context.Context, _ *slog.Logger, row map[string][]byte) (map[string][]byte, error) { + decodedRow := make(map[string][]byte) + + for k, v := range row { + decodedResultBytes, err := snappy.Decode(nil, v) + if err != nil { + return nil, fmt.Errorf("decoding data for key %s: %w", k, err) + } + + decodedRow[k] = decodedResultBytes + } + + return decodedRow, nil +} diff --git a/ee/katc/sqlite.go b/ee/katc/sqlite.go new file mode 100644 index 000000000..6ddeea639 --- /dev/null +++ b/ee/katc/sqlite.go @@ -0,0 +1,117 @@ +package katc + +import ( + "context" + "database/sql" + "fmt" + "log/slog" + "path/filepath" + "strings" + + "github.com/osquery/osquery-go/plugin/table" + _ "modernc.org/sqlite" +) + +// sqliteData is the dataFunc for sqlite KATC tables +func sqliteData(ctx context.Context, slogger *slog.Logger, sourcePattern string, query string, sourceConstraints *table.ConstraintList) ([]sourceData, error) { + pathPattern := sourcePatternToGlobbablePattern(sourcePattern) + sqliteDbs, err := filepath.Glob(pathPattern) + if err != nil { + return nil, fmt.Errorf("globbing for files with pattern %s: %w", pathPattern, err) + } + + results := make([]sourceData, 0) + for _, sqliteDb := range sqliteDbs { + // Check to make sure `sqliteDb` adheres to sourceConstraints + valid, err := checkSourceConstraints(sqliteDb, sourceConstraints) + if err != nil { + return nil, fmt.Errorf("checking source path constraints: %w", err) + } + if !valid { + continue + } + + rowsFromDb, err := querySqliteDb(ctx, slogger, sqliteDb, query) + if err != nil { + return nil, fmt.Errorf("querying %s: %w", sqliteDb, err) + } + results = append(results, sourceData{ + path: sqliteDb, + rows: rowsFromDb, + }) + } + + return results, nil +} + +// sourcePatternToGlobbablePattern translates the source pattern, which adheres to LIKE +// sqlite syntax for consistency with other osquery tables, into a pattern that can be +// accepted by filepath.Glob. +func sourcePatternToGlobbablePattern(sourcePattern string) string { + // % matches zero or more characters in LIKE, corresponds to * in glob syntax + globbablePattern := strings.Replace(sourcePattern, "%", `*`, -1) + // _ matches a single character in LIKE, corresponds to ? in glob syntax + globbablePattern = strings.Replace(globbablePattern, "_", `?`, -1) + return globbablePattern +} + +// querySqliteDb queries the database at the given path, returning rows of results +func querySqliteDb(ctx context.Context, slogger *slog.Logger, path string, query string) ([]map[string][]byte, error) { + dsn := fmt.Sprintf("file:%s?mode=ro", path) + conn, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("opening sqlite db: %w", err) + } + defer func() { + if err := conn.Close(); err != nil { + slogger.Log(ctx, slog.LevelWarn, + "closing sqlite db after query", + "err", err, + ) + } + }() + + rows, err := conn.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("running query: %w", err) + } + defer func() { + if err := rows.Close(); err != nil { + slogger.Log(ctx, slog.LevelWarn, + "closing rows after scanning results", + "err", err, + ) + } + }() + + results := make([]map[string][]byte, 0) + + // Fetch columns so we know how many values per row we will scan + columns, err := rows.Columns() + if err != nil { + return nil, fmt.Errorf("getting columns from query result: %w", err) + } + + // Prepare scan destination + rawResult := make([][]byte, len(columns)) + scanDest := make([]any, len(columns)) + for i := 0; i < len(columns); i += 1 { + scanDest[i] = &rawResult[i] + } + + // Scan all rows + for rows.Next() { + if err := rows.Scan(scanDest...); err != nil { + return nil, fmt.Errorf("scanning query results: %w", err) + } + + row := make(map[string][]byte) + for i := 0; i < len(columns); i += 1 { + row[columns[i]] = rawResult[i] + } + + results = append(results, row) + } + + return results, nil +} diff --git a/ee/katc/table.go b/ee/katc/table.go new file mode 100644 index 000000000..d3aa31d0f --- /dev/null +++ b/ee/katc/table.go @@ -0,0 +1,172 @@ +package katc + +import ( + "context" + "fmt" + "log/slog" + "regexp" + "strings" + + "github.com/osquery/osquery-go/plugin/table" +) + +const sourceColumnName = "source" + +// katcTable is a Kolide ATC table. It queries the source and transforms the response data +// per the configuration in its `cfg`. +type katcTable struct { + cfg katcTableConfig + columnLookup map[string]struct{} + slogger *slog.Logger +} + +// newKatcTable returns a new table with the given `cfg`, as well as the osquery columns for that table. +func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) (*katcTable, []table.ColumnDefinition) { + columns := []table.ColumnDefinition{ + { + Name: sourceColumnName, + Type: table.ColumnTypeText, + }, + } + columnLookup := map[string]struct{}{ + sourceColumnName: {}, + } + for i := 0; i < len(cfg.Columns); i += 1 { + columns = append(columns, table.ColumnDefinition{ + Name: cfg.Columns[i], + Type: table.ColumnTypeText, + }) + columnLookup[cfg.Columns[i]] = struct{}{} + } + + return &katcTable{ + cfg: cfg, + columnLookup: columnLookup, + slogger: slogger.With( + "table_name", tableName, + "table_type", cfg.SourceType, + "table_source", cfg.Source, + ), + }, columns +} + +// generate handles queries against a KATC table. +func (k *katcTable) generate(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { + // Fetch data from our table source + dataRaw, err := k.cfg.SourceType.dataFunc(ctx, k.slogger, k.cfg.Source, k.cfg.Query, getSourceConstraint(queryContext)) + if err != nil { + return nil, fmt.Errorf("fetching data: %w", err) + } + + // Process data + transformedResults := make([]map[string]string, 0) + for _, s := range dataRaw { + for _, dataRawRow := range s.rows { + // Make sure source is included in row data + rowData := map[string]string{ + sourceColumnName: s.path, + } + + // Run any needed transformations on the row data + for _, step := range k.cfg.RowTransformSteps { + dataRawRow, err = step.transformFunc(ctx, k.slogger, dataRawRow) + if err != nil { + return nil, fmt.Errorf("running transform func %s: %w", step.name, err) + } + } + + // After transformations have been applied, we can cast the data from []byte + // to string to return to osquery. + for key, val := range dataRawRow { + rowData[key] = string(val) + } + transformedResults = append(transformedResults, rowData) + } + } + + // Now, filter data to ensure we only return columns in k.columnLookup + filteredResults := make([]map[string]string, 0) + for _, row := range transformedResults { + filteredRow := make(map[string]string) + for column, data := range row { + if _, expectedColumn := k.columnLookup[column]; !expectedColumn { + // Silently discard the column+data + continue + } + + filteredRow[column] = data + } + + filteredResults = append(filteredResults, filteredRow) + } + + return filteredResults, nil +} + +// getSourceConstraint retrieves any constraints against the `source` column +func getSourceConstraint(queryContext table.QueryContext) *table.ConstraintList { + sourceConstraint, sourceConstraintExists := queryContext.Constraints[sourceColumnName] + if sourceConstraintExists { + return &sourceConstraint + } + return nil +} + +// checkSourceConstraints validates whether a given `source` matches the given constraints. +func checkSourceConstraints(source string, sourceConstraints *table.ConstraintList) (bool, error) { + if sourceConstraints == nil { + return true, nil + } + + for _, sourceConstraint := range sourceConstraints.Constraints { + switch sourceConstraint.Operator { + case table.OperatorEquals: + if source != sourceConstraint.Expression { + return false, nil + } + case table.OperatorLike: + // Transform the expression into a regex to test if we have a match. + likeRegexpStr := regexp.QuoteMeta(sourceConstraint.Expression) + // % matches zero or more characters + likeRegexpStr = strings.Replace(likeRegexpStr, "%", `.*`, -1) + // _ matches a single character + likeRegexpStr = strings.Replace(likeRegexpStr, "_", `.`, -1) + // LIKE is case-insensitive + likeRegexpStr = `(?i)` + likeRegexpStr + r, err := regexp.Compile(likeRegexpStr) + if err != nil { + return false, fmt.Errorf("invalid LIKE statement: %w", err) + } + if !r.MatchString(source) { + return false, nil + } + case table.OperatorGlob: + // Transform the expression into a regex to test if we have a match. + // Unlike LIKE, GLOB is case-sensitive. + globRegexpStr := regexp.QuoteMeta(sourceConstraint.Expression) + // * matches zero or more characters + globRegexpStr = strings.Replace(globRegexpStr, `\*`, `.*`, -1) + // ? matches a single character + globRegexpStr = strings.Replace(globRegexpStr, `\?`, `.`, -1) + r, err := regexp.Compile(globRegexpStr) + if err != nil { + return false, fmt.Errorf("invalid GLOB statement: %w", err) + } + if !r.MatchString(source) { + return false, nil + } + case table.OperatorRegexp: + r, err := regexp.Compile(sourceConstraint.Expression) + if err != nil { + return false, fmt.Errorf("invalid regex: %w", err) + } + if !r.MatchString(source) { + return false, nil + } + default: + return false, fmt.Errorf("operator %v not valid source constraint", sourceConstraint.Operator) + } + } + + return true, nil +} diff --git a/ee/katc/table_test.go b/ee/katc/table_test.go new file mode 100644 index 000000000..2c3465d09 --- /dev/null +++ b/ee/katc/table_test.go @@ -0,0 +1,332 @@ +package katc + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/golang/snappy" + "github.com/google/uuid" + "github.com/kolide/launcher/pkg/log/multislogger" + "github.com/osquery/osquery-go/plugin/table" + "github.com/stretchr/testify/require" + + _ "modernc.org/sqlite" +) + +func Test_generate_SqliteBackedIndexedDB(t *testing.T) { + t.Parallel() + + // This test validates generation of table results. It uses a sqlite-backed + // IndexedDB as a source, which means it also exercises functionality from + // sqlite.go, snappy.go, and deserialize_firefox.go. + + // First, set up the data we expect to retrieve. + expectedColumn := "uuid" + u, err := uuid.NewRandom() + require.NoError(t, err, "generating test UUID") + expectedColumnValue := u.String() + + // Serialize the row data, reversing the deserialization operation in + // deserialize_firefox.go. + serializedUuid := []byte(expectedColumnValue) + serializedObj := append([]byte{ + // Header + 0x00, 0x00, 0x00, 0x00, // header tag data -- discarded + 0x00, 0x00, 0xf1, 0xff, // LE `tagHeader` + // Begin object + 0x00, 0x00, 0x00, 0x00, // object tag data -- discarded + 0x08, 0x00, 0xff, 0xff, // LE `tagObject` + // Begin UUID key + 0x04, 0x00, 0x00, 0x80, // LE data about upcoming string: length 4 (remaining bytes), is ASCII + 0x04, 0x00, 0xff, 0xff, // LE `tagString` + 0x75, 0x75, 0x69, 0x64, // "uuid" + 0x00, 0x00, 0x00, 0x00, // padding to get to 8-byte word boundary + // End UUID key + // Begin UUID value + 0x24, 0x00, 0x00, 0x80, // LE data about upcoming string: length 36 (remaining bytes), is ASCII + 0x04, 0x00, 0xff, 0xff, // LE `tagString` + }, + serializedUuid..., + ) + serializedObj = append(serializedObj, + 0x00, 0x00, 0x00, 0x00, // padding to get to 8-byte word boundary for UUID string + // End UUID value + 0x00, 0x00, 0x00, 0x00, // tag data -- discarded + 0x13, 0x00, 0xff, 0xff, // LE `tagEndOfKeys` 0xffff0013 + ) + + // Now compress the serialized row data, reversing the decompression operation + // in snappy.go + compressedObj := snappy.Encode(nil, serializedObj) + + // Now, create a sqlite database to store this data in. + databaseDir := t.TempDir() + sourceFilepath := filepath.Join(databaseDir, "test.sqlite") + f, err := os.Create(sourceFilepath) + require.NoError(t, err, "creating source db") + require.NoError(t, f.Close(), "closing source db file") + conn, err := sql.Open("sqlite", sourceFilepath) + require.NoError(t, err) + _, err = conn.Exec(`CREATE TABLE object_data(data TEXT NOT NULL PRIMARY KEY) WITHOUT ROWID;`) + require.NoError(t, err, "creating test table") + + // Insert compressed object into the database + _, err = conn.Exec("INSERT INTO object_data (data) VALUES (?);", compressedObj) + require.NoError(t, err, "inserting into sqlite database") + require.NoError(t, conn.Close(), "closing sqlite database") + + // At long last, our source is adequately configured. + // Move on to constructing our KATC table. + cfg := katcTableConfig{ + SourceType: katcSourceType{ + name: sqliteSourceType, + dataFunc: sqliteData, + }, + Platform: runtime.GOOS, + Columns: []string{expectedColumn}, + Source: filepath.Join(databaseDir, "%.sqlite"), // All sqlite files in the test directory + Query: "SELECT data FROM object_data;", + RowTransformSteps: []rowTransformStep{ + { + name: snappyDecodeTransformStep, + transformFunc: snappyDecode, + }, + { + name: deserializeFirefoxTransformStep, + transformFunc: deserializeFirefox, + }, + }, + } + testTable, _ := newKatcTable("test_katc_table", cfg, multislogger.NewNopLogger()) + + // Make a query context restricting the source to our exact source sqlite database + queryContext := table.QueryContext{ + Constraints: map[string]table.ConstraintList{ + sourceColumnName: { + Constraints: []table.Constraint{ + { + Operator: table.OperatorEquals, + Expression: sourceFilepath, + }, + }, + }, + }, + } + + // At long last: run a query + results, err := testTable.generate(context.TODO(), queryContext) + require.NoError(t, err) + + // Validate results + require.Equal(t, 1, len(results), "exactly one row expected") + require.Contains(t, results[0], sourceColumnName, "missing source column") + require.Equal(t, sourceFilepath, results[0][sourceColumnName]) + require.Contains(t, results[0], expectedColumn, "expected column missing") + require.Equal(t, expectedColumnValue, results[0][expectedColumn], "data mismatch") +} + +func Test_checkSourcePathConstraints(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + testCaseName string + source string + constraints table.ConstraintList + valid bool + errorExpected bool + }{ + { + testCaseName: "equals", + source: filepath.Join("some", "path", "to", "a", "source"), + constraints: table.ConstraintList{ + Constraints: []table.Constraint{ + { + Operator: table.OperatorEquals, + Expression: filepath.Join("some", "path", "to", "a", "source"), + }, + }, + }, + valid: true, + errorExpected: false, + }, + { + testCaseName: "not equals", + source: filepath.Join("some", "path", "to", "a", "source"), + constraints: table.ConstraintList{ + Constraints: []table.Constraint{ + { + Operator: table.OperatorEquals, + Expression: filepath.Join("a", "path", "to", "a", "different", "source"), + }, + }, + }, + valid: false, + errorExpected: false, + }, + { + testCaseName: "LIKE with % wildcard", + source: filepath.Join("a", "path", "to", "db.sqlite"), + constraints: table.ConstraintList{ + Constraints: []table.Constraint{ + { + Operator: table.OperatorLike, + Expression: filepath.Join("a", "path", "to", "%.sqlite"), + }, + }, + }, + valid: true, + errorExpected: false, + }, + { + testCaseName: "LIKE with underscore wildcard", + source: filepath.Join("a", "path", "to", "db.sqlite"), + constraints: table.ConstraintList{ + Constraints: []table.Constraint{ + { + Operator: table.OperatorLike, + Expression: filepath.Join("_", "path", "to", "db.sqlite"), + }, + }, + }, + valid: true, + errorExpected: false, + }, + { + testCaseName: "LIKE is case-insensitive", + source: filepath.Join("a", "path", "to", "db.sqlite"), + constraints: table.ConstraintList{ + Constraints: []table.Constraint{ + { + Operator: table.OperatorLike, + Expression: filepath.Join("A", "PATH", "TO", "DB.%"), + }, + }, + }, + valid: true, + }, + { + testCaseName: "GLOB with * wildcard", + source: filepath.Join("another", "path", "to", "a", "source"), + constraints: table.ConstraintList{ + Constraints: []table.Constraint{ + { + Operator: table.OperatorGlob, + Expression: filepath.Join("another", "*", "to", "a", "source"), + }, + }, + }, + valid: true, + errorExpected: false, + }, + { + testCaseName: "GLOB with ? wildcard", + source: filepath.Join("another", "path", "to", "a", "source"), + constraints: table.ConstraintList{ + Constraints: []table.Constraint{ + { + Operator: table.OperatorGlob, + Expression: filepath.Join("another", "path", "to", "?", "source"), + }, + }, + }, + valid: true, + errorExpected: false, + }, + { + testCaseName: "regexp", + source: filepath.Join("test", "path", "to", "a", "source"), + constraints: table.ConstraintList{ + Constraints: []table.Constraint{ + { + Operator: table.OperatorRegexp, + Expression: `.*source$`, + }, + }, + }, + valid: true, + errorExpected: false, + }, + { + testCaseName: "invalid regexp", + source: filepath.Join("test", "path", "to", "a", "source"), + constraints: table.ConstraintList{ + Constraints: []table.Constraint{ + { + Operator: table.OperatorRegexp, + Expression: `invalid\`, + }, + }, + }, + valid: false, + errorExpected: true, + }, + { + testCaseName: "unsupported", + source: filepath.Join("test", "path", "to", "a", "source", "2"), + constraints: table.ConstraintList{ + Constraints: []table.Constraint{ + { + Operator: table.OperatorUnique, + Expression: filepath.Join("test", "path", "to", "a", "source", "2"), + }, + }, + }, + valid: false, + errorExpected: true, + }, + { + testCaseName: "multiple constraints where one does not match", + source: filepath.Join("test", "path", "to", "a", "source", "3"), + constraints: table.ConstraintList{ + Constraints: []table.Constraint{ + { + Operator: table.OperatorLike, + Expression: filepath.Join("test", "path", "to", "a", "source", "%"), + }, + { + Operator: table.OperatorEquals, + Expression: filepath.Join("some", "path", "to", "a", "source"), + }, + }, + }, + valid: false, + errorExpected: false, + }, + { + testCaseName: "multiple constraints where all match", + source: filepath.Join("test", "path", "to", "a", "source", "3"), + constraints: table.ConstraintList{ + Constraints: []table.Constraint{ + { + Operator: table.OperatorLike, + Expression: filepath.Join("test", "path", "to", "a", "source", "%"), + }, + { + Operator: table.OperatorEquals, + Expression: filepath.Join("test", "path", "to", "a", "source", "3"), + }, + }, + }, + valid: true, + errorExpected: false, + }, + } { + tt := tt + t.Run(tt.testCaseName, func(t *testing.T) { + t.Parallel() + + valid, err := checkSourceConstraints(tt.source, &tt.constraints) + if tt.errorExpected { + require.Error(t, err, "expected error on checking constraints") + } else { + require.NoError(t, err, "expected no error on checking constraints") + } + + require.Equal(t, tt.valid, valid, "incorrect result checking constraints") + }) + } +} diff --git a/pkg/osquery/table/table.go b/pkg/osquery/table/table.go index 39f3edf6b..e5e2d0517 100644 --- a/pkg/osquery/table/table.go +++ b/pkg/osquery/table/table.go @@ -10,6 +10,7 @@ import ( "github.com/kolide/launcher/ee/agent/startupsettings" "github.com/kolide/launcher/ee/agent/types" "github.com/kolide/launcher/ee/allowedcmd" + "github.com/kolide/launcher/ee/katc" "github.com/kolide/launcher/ee/tables/cryptoinfotable" "github.com/kolide/launcher/ee/tables/dataflattentable" "github.com/kolide/launcher/ee/tables/desktopprocs" @@ -98,29 +99,22 @@ func kolideCustomAtcTables(k types.Knapsack, slogger *slog.Logger) []osquery.Osq } } - // In the future, we would construct the plugins from the configuration here. - // For now, we just log. - slogger.Log(context.TODO(), slog.LevelDebug, - "retrieved Kolide ATC config", - "config", config, - ) - - return nil + return katc.ConstructKATCTables(config, slogger) } func katcFromDb(k types.Knapsack) (map[string]string, error) { if k == nil || k.KatcConfigStore() == nil { return nil, errors.New("stores in knapsack not available") } - loggableConfig := make(map[string]string) + katcCfg := make(map[string]string) if err := k.KatcConfigStore().ForEach(func(k []byte, v []byte) error { - loggableConfig[string(k)] = string(v) + katcCfg[string(k)] = string(v) return nil }); err != nil { return nil, fmt.Errorf("retrieving contents of Kolide ATC config store: %w", err) } - return loggableConfig, nil + return katcCfg, nil } func katcFromStartupSettings(k types.Knapsack) (map[string]string, error) { @@ -135,10 +129,10 @@ func katcFromStartupSettings(k types.Knapsack) (map[string]string, error) { return nil, fmt.Errorf("error getting katc_config from startup settings: %w", err) } - var loggableConfig map[string]string - if err := json.Unmarshal([]byte(katcConfig), &loggableConfig); err != nil { + var katcCfg map[string]string + if err := json.Unmarshal([]byte(katcConfig), &katcCfg); err != nil { return nil, fmt.Errorf("unmarshalling katc_config: %w", err) } - return loggableConfig, nil + return katcCfg, nil }