From a965887fc3a47fc4ba79fb97d1456e926131b180 Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Wed, 3 Jul 2024 12:18:40 -0400 Subject: [PATCH 1/2] Add KATC support for .indexeddb.leveldb files --- ee/indexeddb/values.go | 4 +- ee/indexeddb/values_test.go | 2 +- ee/katc/config.go | 15 ++++++- ee/katc/config_test.go | 14 ++++++ ee/katc/indexeddb_leveldb.go | 71 +++++++++++++++++++++++++++++++ ee/katc/indexeddb_leveldb_test.go | 62 +++++++++++++++++++++++++++ 6 files changed, 163 insertions(+), 5 deletions(-) create mode 100644 ee/katc/indexeddb_leveldb.go create mode 100644 ee/katc/indexeddb_leveldb_test.go diff --git a/ee/indexeddb/values.go b/ee/indexeddb/values.go index d398b5395..a0445678b 100644 --- a/ee/indexeddb/values.go +++ b/ee/indexeddb/values.go @@ -41,9 +41,9 @@ const ( tokenNull byte = 0x30 ) -// deserializeChrome deserializes a JS object that has been stored by Chrome +// DeserializeChrome deserializes a JS object that has been stored by Chrome // in IndexedDB LevelDB-backed databases. -func deserializeChrome(_ context.Context, _ *slog.Logger, row map[string][]byte) (map[string][]byte, error) { +func DeserializeChrome(_ context.Context, _ *slog.Logger, row map[string][]byte) (map[string][]byte, error) { data, ok := row["data"] if !ok { return nil, errors.New("row missing top-level data key") diff --git a/ee/indexeddb/values_test.go b/ee/indexeddb/values_test.go index bedc64b4a..79a4c65d2 100644 --- a/ee/indexeddb/values_test.go +++ b/ee/indexeddb/values_test.go @@ -31,7 +31,7 @@ func Test_deserializeIndexeddbValue(t *testing.T) { 0x01, // properties_written } - obj, err := deserializeChrome(context.TODO(), multislogger.NewNopLogger(), map[string][]byte{"data": testBytes}) + obj, err := DeserializeChrome(context.TODO(), multislogger.NewNopLogger(), map[string][]byte{"data": testBytes}) require.NoError(t, err, "deserializing object") // Confirm we got an id property for the object diff --git a/ee/katc/config.go b/ee/katc/config.go index 18083c84d..62fc3eb58 100644 --- a/ee/katc/config.go +++ b/ee/katc/config.go @@ -7,6 +7,7 @@ import ( "log/slog" "runtime" + "github.com/kolide/launcher/ee/indexeddb" "github.com/osquery/osquery-go" "github.com/osquery/osquery-go/plugin/table" ) @@ -16,7 +17,7 @@ import ( // that performs the query against the source. type katcSourceType struct { name string - dataFunc func(ctx context.Context, slogger *slog.Logger, path string, query string, sourceConstraints *table.ConstraintList) ([]sourceData, error) + dataFunc func(ctx context.Context, slogger *slog.Logger, sourcePattern string, query string, sourceConstraints *table.ConstraintList) ([]sourceData, error) } // sourceData holds the result of calling `katcSourceType.dataFunc`. It maps the @@ -28,7 +29,8 @@ type sourceData struct { } const ( - sqliteSourceType = "sqlite" + sqliteSourceType = "sqlite" + indexeddbLeveldbSourceType = "indexeddb_leveldb" ) func (kst *katcSourceType) UnmarshalJSON(data []byte) error { @@ -43,6 +45,10 @@ func (kst *katcSourceType) UnmarshalJSON(data []byte) error { kst.name = sqliteSourceType kst.dataFunc = sqliteData return nil + case indexeddbLeveldbSourceType: + kst.name = indexeddbLeveldbSourceType + kst.dataFunc = indexeddbLeveldbData + return nil default: return fmt.Errorf("unknown table type %s", s) } @@ -59,6 +65,7 @@ type rowTransformStep struct { const ( snappyDecodeTransformStep = "snappy" deserializeFirefoxTransformStep = "deserialize_firefox" + deserializeChromeTransformStep = "deserialize_chrome" camelToSnakeTransformStep = "camel_to_snake" ) @@ -78,6 +85,10 @@ func (r *rowTransformStep) UnmarshalJSON(data []byte) error { r.name = deserializeFirefoxTransformStep r.transformFunc = deserializeFirefox return nil + case deserializeChromeTransformStep: + r.name = deserializeChromeTransformStep + r.transformFunc = indexeddb.DeserializeChrome + return nil case camelToSnakeTransformStep: r.name = camelToSnakeTransformStep r.transformFunc = camelToSnake diff --git a/ee/katc/config_test.go b/ee/katc/config_test.go index fc29db539..3214412c1 100644 --- a/ee/katc/config_test.go +++ b/ee/katc/config_test.go @@ -32,6 +32,20 @@ func TestConstructKATCTables(t *testing.T) { }, expectedPluginCount: 1, }, + { + testCaseName: "indexeddb_leveldb", + katcConfig: map[string]string{ + "kolide_indexeddb_leveldb_test": fmt.Sprintf(`{ + "source_type": "indexeddb_leveldb", + "platform": "%s", + "columns": ["data"], + "source": "/some/path/to/db.indexeddb.leveldb", + "query": "db.store", + "row_transform_steps": ["deserialize_chrome"] + }`, runtime.GOOS), + }, + expectedPluginCount: 1, + }, { testCaseName: "multiple plugins", katcConfig: map[string]string{ diff --git a/ee/katc/indexeddb_leveldb.go b/ee/katc/indexeddb_leveldb.go new file mode 100644 index 000000000..c1a861328 --- /dev/null +++ b/ee/katc/indexeddb_leveldb.go @@ -0,0 +1,71 @@ +package katc + +import ( + "context" + "fmt" + "log/slog" + "path/filepath" + "strings" + + "github.com/kolide/launcher/ee/indexeddb" + "github.com/osquery/osquery-go/plugin/table" +) + +// indexeddbLeveldbData retrieves data from the LevelDB-backed IndexedDB instances +// found at the filepath in `sourcePattern`. It retrieves all rows from the database +// and object store specified in `query`, which it expects to be in the format +// `.`. +func indexeddbLeveldbData(ctx context.Context, slogger *slog.Logger, sourcePattern string, query string, sourceConstraints *table.ConstraintList) ([]sourceData, error) { + pathPattern := sourcePatternToGlobbablePattern(sourcePattern) + leveldbs, err := filepath.Glob(pathPattern) + if err != nil { + return nil, fmt.Errorf("globbing for leveldb files: %w", err) + } + + // Extract database and table from query + dbName, objectStoreName, err := extractQueryTargets(query) + if err != nil { + return nil, fmt.Errorf("getting db and object store names: %w", err) + } + + // Query databases + results := make([]sourceData, 0) + for _, db := range leveldbs { + // Check to make sure `db` adheres to sourceConstraints + valid, err := checkSourceConstraints(db, sourceConstraints) + if err != nil { + return nil, fmt.Errorf("checking source path constraints: %w", err) + } + if !valid { + continue + } + + rowsFromDb, err := indexeddb.QueryIndexeddbObjectStore(db, dbName, objectStoreName) + if err != nil { + return nil, fmt.Errorf("querying %s: %w", db, err) + } + results = append(results, sourceData{ + path: db, + rows: rowsFromDb, + }) + } + + return results, nil +} + +// extractQueryTargets retrieves the targets of the query (the database name and the object store name) +// from the query. IndexedDB is a NoSQL database, so we expect to retrieve all rows from the given +// object store within the given database name. +func extractQueryTargets(query string) (string, string, error) { + parts := strings.Split(query, ".") + if len(parts) != 2 { + return "", "", fmt.Errorf("unable to extract query targets from query: expected `.`, got `%s`", query) + } + if len(parts[0]) == 0 { + return "", "", fmt.Errorf("missing db name in query `%s`", query) + } + if len(parts[1]) == 0 { + return "", "", fmt.Errorf("missing object store name in query `%s`", query) + } + return parts[0], parts[1], nil +} diff --git a/ee/katc/indexeddb_leveldb_test.go b/ee/katc/indexeddb_leveldb_test.go new file mode 100644 index 000000000..e305ff975 --- /dev/null +++ b/ee/katc/indexeddb_leveldb_test.go @@ -0,0 +1,62 @@ +package katc + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_extractQueryTargets(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + testCaseName string + query string + expectedDbName string + expectedObjectStoreName string + expectErr bool + }{ + { + testCaseName: "correctly formed query", + query: "some_db.some_obj_store", + expectedDbName: "some_db", + expectedObjectStoreName: "some_obj_store", + expectErr: false, + }, + { + testCaseName: "missing db name", + query: ".some_obj_store", + expectErr: true, + }, + { + testCaseName: "missing object store name", + query: "some_db.", + expectErr: true, + }, + { + testCaseName: "query missing separator", + query: "some_db some_obj_store", + expectErr: true, + }, + { + testCaseName: "query has too many components", + query: "some_db.some_obj_store.some_other_component", + expectErr: true, + }, + } { + tt := tt + t.Run(tt.testCaseName, func(t *testing.T) { + t.Parallel() + + dbName, objStoreName, err := extractQueryTargets(tt.query) + + if tt.expectErr { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tt.expectedDbName, dbName) + require.Equal(t, tt.expectedObjectStoreName, objStoreName) + } + }) + } +} From 59af6ef12832ee567290ef08fd730459b2a3dff3 Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Wed, 3 Jul 2024 14:02:18 -0400 Subject: [PATCH 2/2] Fix source pattern matching to ignore underscores --- ee/katc/sqlite.go | 11 ++++------- ee/katc/sqlite_test.go | 9 ++------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/ee/katc/sqlite.go b/ee/katc/sqlite.go index 6ddeea639..f63eb384d 100644 --- a/ee/katc/sqlite.go +++ b/ee/katc/sqlite.go @@ -44,15 +44,12 @@ func sqliteData(ctx context.Context, slogger *slog.Logger, sourcePattern string, return results, nil } -// sourcePatternToGlobbablePattern translates the source pattern, which adheres to LIKE -// sqlite syntax for consistency with other osquery tables, into a pattern that can be +// sourcePatternToGlobbablePattern translates the source pattern, which allows for +// using % wildcards for consistency with other osquery tables, into a pattern that can be // accepted by filepath.Glob. func sourcePatternToGlobbablePattern(sourcePattern string) string { - // % matches zero or more characters in LIKE, corresponds to * in glob syntax - globbablePattern := strings.Replace(sourcePattern, "%", `*`, -1) - // _ matches a single character in LIKE, corresponds to ? in glob syntax - globbablePattern = strings.Replace(globbablePattern, "_", `?`, -1) - return globbablePattern + // % matches zero or more characters, corresponds to * in glob syntax + return strings.Replace(sourcePattern, "%", `*`, -1) } // querySqliteDb queries the database at the given path, returning rows of results diff --git a/ee/katc/sqlite_test.go b/ee/katc/sqlite_test.go index 7f2a8dd0a..25b4f1b98 100644 --- a/ee/katc/sqlite_test.go +++ b/ee/katc/sqlite_test.go @@ -121,15 +121,10 @@ func TestSourcePatternToGlobbablePattern(t *testing.T) { sourcePattern: filepath.Join(rootDir, "path", "to", "%", "directory", "db.sqlite"), expectedPattern: filepath.Join(rootDir, "path", "to", "*", "directory", "db.sqlite"), }, - { - testCaseName: "underscore wildcard", - sourcePattern: filepath.Join(rootDir, "path", "to", "_", "directory", "db.sqlite"), - expectedPattern: filepath.Join(rootDir, "path", "to", "?", "directory", "db.sqlite"), - }, { testCaseName: "multiple wildcards", - sourcePattern: filepath.Join(rootDir, "path", "to", "_", "directory", "%.sqlite"), - expectedPattern: filepath.Join(rootDir, "path", "to", "?", "directory", "*.sqlite"), + sourcePattern: filepath.Join(rootDir, "path", "to", "*", "directory", "%.sqlite"), + expectedPattern: filepath.Join(rootDir, "path", "to", "*", "directory", "*.sqlite"), }, } { tt := tt