From 88097dfe37f1d11efee4b35c908a178a3a70ace2 Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Wed, 3 Jul 2024 14:15:07 -0400 Subject: [PATCH 01/17] Rename source column to path --- ee/katc/sqlite.go | 2 +- ee/katc/table.go | 40 ++++++++++++++++++++-------------------- ee/katc/table_test.go | 34 +++++++++++++++++----------------- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/ee/katc/sqlite.go b/ee/katc/sqlite.go index 6ddeea639..ca9a48776 100644 --- a/ee/katc/sqlite.go +++ b/ee/katc/sqlite.go @@ -23,7 +23,7 @@ func sqliteData(ctx context.Context, slogger *slog.Logger, sourcePattern string, results := make([]sourceData, 0) for _, sqliteDb := range sqliteDbs { // Check to make sure `sqliteDb` adheres to sourceConstraints - valid, err := checkSourceConstraints(sqliteDb, sourceConstraints) + valid, err := checkPathConstraints(sqliteDb, sourceConstraints) if err != nil { return nil, fmt.Errorf("checking source path constraints: %w", err) } diff --git a/ee/katc/table.go b/ee/katc/table.go index d3aa31d0f..e45afc2f3 100644 --- a/ee/katc/table.go +++ b/ee/katc/table.go @@ -10,7 +10,7 @@ import ( "github.com/osquery/osquery-go/plugin/table" ) -const sourceColumnName = "source" +const pathColumnName = "path" // katcTable is a Kolide ATC table. It queries the source and transforms the response data // per the configuration in its `cfg`. @@ -24,12 +24,12 @@ type katcTable struct { func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) (*katcTable, []table.ColumnDefinition) { columns := []table.ColumnDefinition{ { - Name: sourceColumnName, + Name: pathColumnName, Type: table.ColumnTypeText, }, } columnLookup := map[string]struct{}{ - sourceColumnName: {}, + pathColumnName: {}, } for i := 0; i < len(cfg.Columns); i += 1 { columns = append(columns, table.ColumnDefinition{ @@ -62,9 +62,9 @@ func (k *katcTable) generate(ctx context.Context, queryContext table.QueryContex transformedResults := make([]map[string]string, 0) for _, s := range dataRaw { for _, dataRawRow := range s.rows { - // Make sure source is included in row data + // Make sure source's path is included in row data rowData := map[string]string{ - sourceColumnName: s.path, + pathColumnName: s.path, } // Run any needed transformations on the row data @@ -103,30 +103,30 @@ func (k *katcTable) generate(ctx context.Context, queryContext table.QueryContex return filteredResults, nil } -// getSourceConstraint retrieves any constraints against the `source` column +// getSourceConstraint retrieves any constraints against the `path` column func getSourceConstraint(queryContext table.QueryContext) *table.ConstraintList { - sourceConstraint, sourceConstraintExists := queryContext.Constraints[sourceColumnName] + sourceConstraint, sourceConstraintExists := queryContext.Constraints[pathColumnName] if sourceConstraintExists { return &sourceConstraint } return nil } -// checkSourceConstraints validates whether a given `source` matches the given constraints. -func checkSourceConstraints(source string, sourceConstraints *table.ConstraintList) (bool, error) { - if sourceConstraints == nil { +// checkPathConstraints validates whether a given `path` matches the given constraints. +func checkPathConstraints(path string, pathConstraints *table.ConstraintList) (bool, error) { + if pathConstraints == nil { return true, nil } - for _, sourceConstraint := range sourceConstraints.Constraints { - switch sourceConstraint.Operator { + for _, pathConstraint := range pathConstraints.Constraints { + switch pathConstraint.Operator { case table.OperatorEquals: - if source != sourceConstraint.Expression { + if path != pathConstraint.Expression { return false, nil } case table.OperatorLike: // Transform the expression into a regex to test if we have a match. - likeRegexpStr := regexp.QuoteMeta(sourceConstraint.Expression) + likeRegexpStr := regexp.QuoteMeta(pathConstraint.Expression) // % matches zero or more characters likeRegexpStr = strings.Replace(likeRegexpStr, "%", `.*`, -1) // _ matches a single character @@ -137,13 +137,13 @@ func checkSourceConstraints(source string, sourceConstraints *table.ConstraintLi if err != nil { return false, fmt.Errorf("invalid LIKE statement: %w", err) } - if !r.MatchString(source) { + if !r.MatchString(path) { return false, nil } case table.OperatorGlob: // Transform the expression into a regex to test if we have a match. // Unlike LIKE, GLOB is case-sensitive. - globRegexpStr := regexp.QuoteMeta(sourceConstraint.Expression) + globRegexpStr := regexp.QuoteMeta(pathConstraint.Expression) // * matches zero or more characters globRegexpStr = strings.Replace(globRegexpStr, `\*`, `.*`, -1) // ? matches a single character @@ -152,19 +152,19 @@ func checkSourceConstraints(source string, sourceConstraints *table.ConstraintLi if err != nil { return false, fmt.Errorf("invalid GLOB statement: %w", err) } - if !r.MatchString(source) { + if !r.MatchString(path) { return false, nil } case table.OperatorRegexp: - r, err := regexp.Compile(sourceConstraint.Expression) + r, err := regexp.Compile(pathConstraint.Expression) if err != nil { return false, fmt.Errorf("invalid regex: %w", err) } - if !r.MatchString(source) { + if !r.MatchString(path) { return false, nil } default: - return false, fmt.Errorf("operator %v not valid source constraint", sourceConstraint.Operator) + return false, fmt.Errorf("operator %v not valid path constraint", pathConstraint.Operator) } } diff --git a/ee/katc/table_test.go b/ee/katc/table_test.go index 2c3465d09..6ed2b5eff 100644 --- a/ee/katc/table_test.go +++ b/ee/katc/table_test.go @@ -106,7 +106,7 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) { // Make a query context restricting the source to our exact source sqlite database queryContext := table.QueryContext{ Constraints: map[string]table.ConstraintList{ - sourceColumnName: { + pathColumnName: { Constraints: []table.Constraint{ { Operator: table.OperatorEquals, @@ -123,8 +123,8 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) { // Validate results require.Equal(t, 1, len(results), "exactly one row expected") - require.Contains(t, results[0], sourceColumnName, "missing source column") - require.Equal(t, sourceFilepath, results[0][sourceColumnName]) + require.Contains(t, results[0], pathColumnName, "missing source column") + require.Equal(t, sourceFilepath, results[0][pathColumnName]) require.Contains(t, results[0], expectedColumn, "expected column missing") require.Equal(t, expectedColumnValue, results[0][expectedColumn], "data mismatch") } @@ -134,14 +134,14 @@ func Test_checkSourcePathConstraints(t *testing.T) { for _, tt := range []struct { testCaseName string - source string + path string constraints table.ConstraintList valid bool errorExpected bool }{ { testCaseName: "equals", - source: filepath.Join("some", "path", "to", "a", "source"), + path: filepath.Join("some", "path", "to", "a", "source"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -155,7 +155,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "not equals", - source: filepath.Join("some", "path", "to", "a", "source"), + path: filepath.Join("some", "path", "to", "a", "source"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -169,7 +169,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "LIKE with % wildcard", - source: filepath.Join("a", "path", "to", "db.sqlite"), + path: filepath.Join("a", "path", "to", "db.sqlite"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -183,7 +183,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "LIKE with underscore wildcard", - source: filepath.Join("a", "path", "to", "db.sqlite"), + path: filepath.Join("a", "path", "to", "db.sqlite"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -197,7 +197,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "LIKE is case-insensitive", - source: filepath.Join("a", "path", "to", "db.sqlite"), + path: filepath.Join("a", "path", "to", "db.sqlite"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -210,7 +210,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "GLOB with * wildcard", - source: filepath.Join("another", "path", "to", "a", "source"), + path: filepath.Join("another", "path", "to", "a", "source"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -224,7 +224,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "GLOB with ? wildcard", - source: filepath.Join("another", "path", "to", "a", "source"), + path: filepath.Join("another", "path", "to", "a", "source"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -238,7 +238,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "regexp", - source: filepath.Join("test", "path", "to", "a", "source"), + path: filepath.Join("test", "path", "to", "a", "source"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -252,7 +252,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "invalid regexp", - source: filepath.Join("test", "path", "to", "a", "source"), + path: filepath.Join("test", "path", "to", "a", "source"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -266,7 +266,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "unsupported", - source: filepath.Join("test", "path", "to", "a", "source", "2"), + path: filepath.Join("test", "path", "to", "a", "source", "2"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -280,7 +280,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "multiple constraints where one does not match", - source: filepath.Join("test", "path", "to", "a", "source", "3"), + path: filepath.Join("test", "path", "to", "a", "source", "3"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -298,7 +298,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { }, { testCaseName: "multiple constraints where all match", - source: filepath.Join("test", "path", "to", "a", "source", "3"), + path: filepath.Join("test", "path", "to", "a", "source", "3"), constraints: table.ConstraintList{ Constraints: []table.Constraint{ { @@ -319,7 +319,7 @@ func Test_checkSourcePathConstraints(t *testing.T) { t.Run(tt.testCaseName, func(t *testing.T) { t.Parallel() - valid, err := checkSourceConstraints(tt.source, &tt.constraints) + valid, err := checkPathConstraints(tt.path, &tt.constraints) if tt.errorExpected { require.Error(t, err, "expected error on checking constraints") } else { From 93aa764fb4a10e531bd57089cb36997f836096b9 Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Wed, 3 Jul 2024 14:19:43 -0400 Subject: [PATCH 02/17] Rename Source to SourcePaths and support multiple paths --- ee/katc/config.go | 4 ++-- ee/katc/config_test.go | 10 +++++----- ee/katc/sqlite.go | 44 ++++++++++++++++++++++-------------------- ee/katc/sqlite_test.go | 4 ++-- ee/katc/table.go | 4 ++-- ee/katc/table_test.go | 8 ++++---- 6 files changed, 38 insertions(+), 36 deletions(-) diff --git a/ee/katc/config.go b/ee/katc/config.go index 18083c84d..5dadae0cd 100644 --- a/ee/katc/config.go +++ b/ee/katc/config.go @@ -16,7 +16,7 @@ import ( // that performs the query against the source. type katcSourceType struct { name string - dataFunc func(ctx context.Context, slogger *slog.Logger, path string, query string, sourceConstraints *table.ConstraintList) ([]sourceData, error) + dataFunc func(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, sourceConstraints *table.ConstraintList) ([]sourceData, error) } // sourceData holds the result of calling `katcSourceType.dataFunc`. It maps the @@ -91,7 +91,7 @@ func (r *rowTransformStep) UnmarshalJSON(data []byte) error { // sends down these configurations. type katcTableConfig struct { SourceType katcSourceType `json:"source_type"` - Source string `json:"source"` // Describes how to connect to source (e.g. path to db) -- % and _ wildcards supported + SourcePaths []string `json:"source_paths"` // Describes how to connect to source (e.g. path to db) -- % and _ wildcards supported Platform string `json:"platform"` Columns []string `json:"columns"` Query string `json:"query"` // Query to run against `path` diff --git a/ee/katc/config_test.go b/ee/katc/config_test.go index fc29db539..7412ce438 100644 --- a/ee/katc/config_test.go +++ b/ee/katc/config_test.go @@ -25,7 +25,7 @@ func TestConstructKATCTables(t *testing.T) { "source_type": "sqlite", "platform": "%s", "columns": ["data"], - "source": "/some/path/to/db.sqlite", + "source_paths": ["/some/path/to/db.sqlite"], "query": "SELECT data FROM object_data JOIN object_store ON (object_data.object_store_id = object_store.id) WHERE object_store.name=\"testtable\";", "row_transform_steps": ["snappy"] }`, runtime.GOOS), @@ -39,7 +39,7 @@ func TestConstructKATCTables(t *testing.T) { "source_type": "sqlite", "platform": "%s", "columns": ["data"], - "source": "/some/path/to/db.sqlite", + "source_paths": ["/some/path/to/db.sqlite"], "query": "SELECT data FROM object_data;", "row_transform_steps": ["snappy"] }`, runtime.GOOS), @@ -47,7 +47,7 @@ func TestConstructKATCTables(t *testing.T) { "source_type": "sqlite", "platform": "%s", "columns": ["col1", "col2"], - "source": "/some/path/to/a/different/db.sqlite", + "source_paths": ["/some/path/to/a/different/db.sqlite"], "query": "SELECT col1, col2 FROM some_table;", "row_transform_steps": ["camel_to_snake"] }`, runtime.GOOS), @@ -68,7 +68,7 @@ func TestConstructKATCTables(t *testing.T) { "source_type": "unknown_source", "platform": "%s", "columns": ["data"], - "source": "/some/path/to/db.sqlite", + "source_paths": []"/some/path/to/db.sqlite"], "query": "SELECT data FROM object_data;" }`, runtime.GOOS), }, @@ -81,7 +81,7 @@ func TestConstructKATCTables(t *testing.T) { "source_type": "sqlite", "platform": "%s", "columns": ["data"], - "source": "/some/path/to/db.sqlite", + "source_paths": []"/some/path/to/db.sqlite"], "query": "SELECT data FROM object_data;", "row_transform_steps": ["unknown_step"] }`, runtime.GOOS), diff --git a/ee/katc/sqlite.go b/ee/katc/sqlite.go index ca9a48776..5be750477 100644 --- a/ee/katc/sqlite.go +++ b/ee/katc/sqlite.go @@ -13,32 +13,34 @@ import ( ) // sqliteData is the dataFunc for sqlite KATC tables -func sqliteData(ctx context.Context, slogger *slog.Logger, sourcePattern string, query string, sourceConstraints *table.ConstraintList) ([]sourceData, error) { - pathPattern := sourcePatternToGlobbablePattern(sourcePattern) - sqliteDbs, err := filepath.Glob(pathPattern) - if err != nil { - return nil, fmt.Errorf("globbing for files with pattern %s: %w", pathPattern, err) - } - +func sqliteData(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, sourceConstraints *table.ConstraintList) ([]sourceData, error) { results := make([]sourceData, 0) - for _, sqliteDb := range sqliteDbs { - // Check to make sure `sqliteDb` adheres to sourceConstraints - valid, err := checkPathConstraints(sqliteDb, sourceConstraints) + for _, sourcePath := range sourcePaths { + pathPattern := sourcePatternToGlobbablePattern(sourcePath) + sqliteDbs, err := filepath.Glob(pathPattern) if err != nil { - return nil, fmt.Errorf("checking source path constraints: %w", err) - } - if !valid { - continue + return nil, fmt.Errorf("globbing for files with pattern %s: %w", pathPattern, err) } - rowsFromDb, err := querySqliteDb(ctx, slogger, sqliteDb, query) - if err != nil { - return nil, fmt.Errorf("querying %s: %w", sqliteDb, err) + for _, sqliteDb := range sqliteDbs { + // Check to make sure `sqliteDb` adheres to sourceConstraints + valid, err := checkPathConstraints(sqliteDb, sourceConstraints) + if err != nil { + return nil, fmt.Errorf("checking source path constraints: %w", err) + } + if !valid { + continue + } + + rowsFromDb, err := querySqliteDb(ctx, slogger, sqliteDb, query) + if err != nil { + return nil, fmt.Errorf("querying %s: %w", sqliteDb, err) + } + results = append(results, sourceData{ + path: sqliteDb, + rows: rowsFromDb, + }) } - results = append(results, sourceData{ - path: sqliteDb, - rows: rowsFromDb, - }) } return results, nil diff --git a/ee/katc/sqlite_test.go b/ee/katc/sqlite_test.go index 7f2a8dd0a..d2ffb3ef5 100644 --- a/ee/katc/sqlite_test.go +++ b/ee/katc/sqlite_test.go @@ -59,7 +59,7 @@ func Test_sqliteData(t *testing.T) { } // Query data - results, err := sqliteData(context.TODO(), multislogger.NewNopLogger(), filepath.Join(sqliteDir, "*.sqlite"), "SELECT uuid, value FROM test_data;", nil) + results, err := sqliteData(context.TODO(), multislogger.NewNopLogger(), []string{filepath.Join(sqliteDir, "*.sqlite")}, "SELECT uuid, value FROM test_data;", nil) require.NoError(t, err) // Confirm we have the correct number of `sourceData` returned (one per db) @@ -89,7 +89,7 @@ func Test_sqliteData_noSourcesFound(t *testing.T) { t.Parallel() tmpDir := t.TempDir() - results, err := sqliteData(context.TODO(), multislogger.NewNopLogger(), filepath.Join(tmpDir, "db.sqlite"), "SELECT * FROM data;", nil) + results, err := sqliteData(context.TODO(), multislogger.NewNopLogger(), []string{filepath.Join(tmpDir, "db.sqlite")}, "SELECT * FROM data;", nil) require.NoError(t, err) require.Equal(t, 0, len(results)) } diff --git a/ee/katc/table.go b/ee/katc/table.go index e45afc2f3..d23058507 100644 --- a/ee/katc/table.go +++ b/ee/katc/table.go @@ -45,7 +45,7 @@ func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) ( slogger: slogger.With( "table_name", tableName, "table_type", cfg.SourceType, - "table_source", cfg.Source, + "table_source_paths", cfg.SourcePaths, ), }, columns } @@ -53,7 +53,7 @@ func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) ( // generate handles queries against a KATC table. func (k *katcTable) generate(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { // Fetch data from our table source - dataRaw, err := k.cfg.SourceType.dataFunc(ctx, k.slogger, k.cfg.Source, k.cfg.Query, getSourceConstraint(queryContext)) + dataRaw, err := k.cfg.SourceType.dataFunc(ctx, k.slogger, k.cfg.SourcePaths, k.cfg.Query, getSourceConstraint(queryContext)) if err != nil { return nil, fmt.Errorf("fetching data: %w", err) } diff --git a/ee/katc/table_test.go b/ee/katc/table_test.go index 6ed2b5eff..56d7d2d7b 100644 --- a/ee/katc/table_test.go +++ b/ee/katc/table_test.go @@ -86,10 +86,10 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) { name: sqliteSourceType, dataFunc: sqliteData, }, - Platform: runtime.GOOS, - Columns: []string{expectedColumn}, - Source: filepath.Join(databaseDir, "%.sqlite"), // All sqlite files in the test directory - Query: "SELECT data FROM object_data;", + Platform: runtime.GOOS, + Columns: []string{expectedColumn}, + SourcePaths: []string{filepath.Join(databaseDir, "%.sqlite")}, // All sqlite files in the test directory + Query: "SELECT data FROM object_data;", RowTransformSteps: []rowTransformStep{ { name: snappyDecodeTransformStep, From 77a31a6eaa3d8560c18243e745a43607ec7de858 Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Wed, 3 Jul 2024 14:21:06 -0400 Subject: [PATCH 03/17] Rename query to source_query --- ee/katc/config.go | 2 +- ee/katc/config_test.go | 10 +++++----- ee/katc/table.go | 2 +- ee/katc/table_test.go | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ee/katc/config.go b/ee/katc/config.go index 5dadae0cd..5915acbea 100644 --- a/ee/katc/config.go +++ b/ee/katc/config.go @@ -94,7 +94,7 @@ type katcTableConfig struct { SourcePaths []string `json:"source_paths"` // Describes how to connect to source (e.g. path to db) -- % and _ wildcards supported Platform string `json:"platform"` Columns []string `json:"columns"` - Query string `json:"query"` // Query to run against `path` + SourceQuery string `json:"source_query"` // Query to run against each source path RowTransformSteps []rowTransformStep `json:"row_transform_steps"` } diff --git a/ee/katc/config_test.go b/ee/katc/config_test.go index 7412ce438..c5e571d84 100644 --- a/ee/katc/config_test.go +++ b/ee/katc/config_test.go @@ -26,7 +26,7 @@ func TestConstructKATCTables(t *testing.T) { "platform": "%s", "columns": ["data"], "source_paths": ["/some/path/to/db.sqlite"], - "query": "SELECT data FROM object_data JOIN object_store ON (object_data.object_store_id = object_store.id) WHERE object_store.name=\"testtable\";", + "source_query": "SELECT data FROM object_data JOIN object_store ON (object_data.object_store_id = object_store.id) WHERE object_store.name=\"testtable\";", "row_transform_steps": ["snappy"] }`, runtime.GOOS), }, @@ -40,7 +40,7 @@ func TestConstructKATCTables(t *testing.T) { "platform": "%s", "columns": ["data"], "source_paths": ["/some/path/to/db.sqlite"], - "query": "SELECT data FROM object_data;", + "source_query": "SELECT data FROM object_data;", "row_transform_steps": ["snappy"] }`, runtime.GOOS), "test_2": fmt.Sprintf(`{ @@ -48,7 +48,7 @@ func TestConstructKATCTables(t *testing.T) { "platform": "%s", "columns": ["col1", "col2"], "source_paths": ["/some/path/to/a/different/db.sqlite"], - "query": "SELECT col1, col2 FROM some_table;", + "source_query": "SELECT col1, col2 FROM some_table;", "row_transform_steps": ["camel_to_snake"] }`, runtime.GOOS), }, @@ -69,7 +69,7 @@ func TestConstructKATCTables(t *testing.T) { "platform": "%s", "columns": ["data"], "source_paths": []"/some/path/to/db.sqlite"], - "query": "SELECT data FROM object_data;" + "source_query": "SELECT data FROM object_data;" }`, runtime.GOOS), }, expectedPluginCount: 0, @@ -82,7 +82,7 @@ func TestConstructKATCTables(t *testing.T) { "platform": "%s", "columns": ["data"], "source_paths": []"/some/path/to/db.sqlite"], - "query": "SELECT data FROM object_data;", + "source_query": "SELECT data FROM object_data;", "row_transform_steps": ["unknown_step"] }`, runtime.GOOS), }, diff --git a/ee/katc/table.go b/ee/katc/table.go index d23058507..5370e089d 100644 --- a/ee/katc/table.go +++ b/ee/katc/table.go @@ -53,7 +53,7 @@ func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) ( // generate handles queries against a KATC table. func (k *katcTable) generate(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { // Fetch data from our table source - dataRaw, err := k.cfg.SourceType.dataFunc(ctx, k.slogger, k.cfg.SourcePaths, k.cfg.Query, getSourceConstraint(queryContext)) + dataRaw, err := k.cfg.SourceType.dataFunc(ctx, k.slogger, k.cfg.SourcePaths, k.cfg.SourceQuery, getSourceConstraint(queryContext)) if err != nil { return nil, fmt.Errorf("fetching data: %w", err) } diff --git a/ee/katc/table_test.go b/ee/katc/table_test.go index 56d7d2d7b..6261f5feb 100644 --- a/ee/katc/table_test.go +++ b/ee/katc/table_test.go @@ -89,7 +89,7 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) { Platform: runtime.GOOS, Columns: []string{expectedColumn}, SourcePaths: []string{filepath.Join(databaseDir, "%.sqlite")}, // All sqlite files in the test directory - Query: "SELECT data FROM object_data;", + SourceQuery: "SELECT data FROM object_data;", RowTransformSteps: []rowTransformStep{ { name: snappyDecodeTransformStep, From b3f251c4ba1e399bc4c7dc79408784fba1b7afd4 Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Wed, 3 Jul 2024 14:21:31 -0400 Subject: [PATCH 04/17] Fix typos in test --- ee/katc/config_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ee/katc/config_test.go b/ee/katc/config_test.go index c5e571d84..e95fa9e12 100644 --- a/ee/katc/config_test.go +++ b/ee/katc/config_test.go @@ -68,7 +68,7 @@ func TestConstructKATCTables(t *testing.T) { "source_type": "unknown_source", "platform": "%s", "columns": ["data"], - "source_paths": []"/some/path/to/db.sqlite"], + "source_paths": ["/some/path/to/db.sqlite"], "source_query": "SELECT data FROM object_data;" }`, runtime.GOOS), }, @@ -81,7 +81,7 @@ func TestConstructKATCTables(t *testing.T) { "source_type": "sqlite", "platform": "%s", "columns": ["data"], - "source_paths": []"/some/path/to/db.sqlite"], + "source_paths": ["/some/path/to/db.sqlite"], "source_query": "SELECT data FROM object_data;", "row_transform_steps": ["unknown_step"] }`, runtime.GOOS), From 5fa2de95997a5726758edfd6c975674ee87f25c7 Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Wed, 3 Jul 2024 14:22:44 -0400 Subject: [PATCH 05/17] Rename Platform to Filter --- ee/katc/config.go | 5 +++-- ee/katc/config_test.go | 10 +++++----- ee/katc/table_test.go | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ee/katc/config.go b/ee/katc/config.go index 5915acbea..9effc54e8 100644 --- a/ee/katc/config.go +++ b/ee/katc/config.go @@ -92,7 +92,7 @@ func (r *rowTransformStep) UnmarshalJSON(data []byte) error { type katcTableConfig struct { SourceType katcSourceType `json:"source_type"` SourcePaths []string `json:"source_paths"` // Describes how to connect to source (e.g. path to db) -- % and _ wildcards supported - Platform string `json:"platform"` + Filter string `json:"filter"` Columns []string `json:"columns"` SourceQuery string `json:"source_query"` // Query to run against each source path RowTransformSteps []rowTransformStep `json:"row_transform_steps"` @@ -113,7 +113,8 @@ func ConstructKATCTables(config map[string]string, slogger *slog.Logger) []osque continue } - if cfg.Platform != runtime.GOOS { + // For now, the filter is simply the OS + if cfg.Filter != runtime.GOOS { continue } diff --git a/ee/katc/config_test.go b/ee/katc/config_test.go index e95fa9e12..3838a5566 100644 --- a/ee/katc/config_test.go +++ b/ee/katc/config_test.go @@ -23,7 +23,7 @@ func TestConstructKATCTables(t *testing.T) { katcConfig: map[string]string{ "kolide_snappy_sqlite_test": fmt.Sprintf(`{ "source_type": "sqlite", - "platform": "%s", + "filter": "%s", "columns": ["data"], "source_paths": ["/some/path/to/db.sqlite"], "source_query": "SELECT data FROM object_data JOIN object_store ON (object_data.object_store_id = object_store.id) WHERE object_store.name=\"testtable\";", @@ -37,7 +37,7 @@ func TestConstructKATCTables(t *testing.T) { katcConfig: map[string]string{ "test_1": fmt.Sprintf(`{ "source_type": "sqlite", - "platform": "%s", + "filter": "%s", "columns": ["data"], "source_paths": ["/some/path/to/db.sqlite"], "source_query": "SELECT data FROM object_data;", @@ -45,7 +45,7 @@ func TestConstructKATCTables(t *testing.T) { }`, runtime.GOOS), "test_2": fmt.Sprintf(`{ "source_type": "sqlite", - "platform": "%s", + "filter": "%s", "columns": ["col1", "col2"], "source_paths": ["/some/path/to/a/different/db.sqlite"], "source_query": "SELECT col1, col2 FROM some_table;", @@ -66,7 +66,7 @@ func TestConstructKATCTables(t *testing.T) { katcConfig: map[string]string{ "kolide_snappy_test": fmt.Sprintf(`{ "source_type": "unknown_source", - "platform": "%s", + "filter": "%s", "columns": ["data"], "source_paths": ["/some/path/to/db.sqlite"], "source_query": "SELECT data FROM object_data;" @@ -79,7 +79,7 @@ func TestConstructKATCTables(t *testing.T) { katcConfig: map[string]string{ "kolide_snappy_test": fmt.Sprintf(`{ "source_type": "sqlite", - "platform": "%s", + "filter": "%s", "columns": ["data"], "source_paths": ["/some/path/to/db.sqlite"], "source_query": "SELECT data FROM object_data;", diff --git a/ee/katc/table_test.go b/ee/katc/table_test.go index 6261f5feb..fbef23f39 100644 --- a/ee/katc/table_test.go +++ b/ee/katc/table_test.go @@ -86,7 +86,7 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) { name: sqliteSourceType, dataFunc: sqliteData, }, - Platform: runtime.GOOS, + Filter: runtime.GOOS, Columns: []string{expectedColumn}, SourcePaths: []string{filepath.Join(databaseDir, "%.sqlite")}, // All sqlite files in the test directory SourceQuery: "SELECT data FROM object_data;", From 03777f0d486bce7b4ac5dfe196dbb3c5cd0a4675 Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Wed, 3 Jul 2024 14:44:23 -0400 Subject: [PATCH 06/17] Move name inside config and make config a list of tables instead of a map --- ee/katc/config.go | 31 +++++++++-- ee/katc/config_test.go | 124 ++++++++++++++++++++++++++++------------- ee/katc/table.go | 4 +- ee/katc/table_test.go | 3 +- 4 files changed, 115 insertions(+), 47 deletions(-) diff --git a/ee/katc/config.go b/ee/katc/config.go index 9effc54e8..f9cd3e9d2 100644 --- a/ee/katc/config.go +++ b/ee/katc/config.go @@ -90,6 +90,7 @@ func (r *rowTransformStep) UnmarshalJSON(data []byte) error { // katcTableConfig is the configuration for a specific KATC table. The control server // sends down these configurations. type katcTableConfig struct { + Name string `json:"name"` SourceType katcSourceType `json:"source_type"` SourcePaths []string `json:"source_paths"` // Describes how to connect to source (e.g. path to db) -- % and _ wildcards supported Filter string `json:"filter"` @@ -102,12 +103,32 @@ type katcTableConfig struct { // and returns the constructed tables. func ConstructKATCTables(config map[string]string, slogger *slog.Logger) []osquery.OsqueryPlugin { plugins := make([]osquery.OsqueryPlugin, 0) - for tableName, tableConfigStr := range config { + + tableConfigs, tableConfigsExist := config["tables"] + if !tableConfigsExist { + slogger.Log(context.TODO(), slog.LevelWarn, + "missing top-level tables key in KATC config, cannot construct tables", + ) + + return plugins + } + + // We want to unmarshal each table config separately, so that we don't fail to configure all tables + // if only some are malformed. + var rawTableConfigs []json.RawMessage + if err := json.Unmarshal([]byte(tableConfigs), &rawTableConfigs); err != nil { + slogger.Log(context.TODO(), slog.LevelWarn, + "could not unmarshal tables in KATC config", + "err", err, + ) + return plugins + } + + for _, rawTableConfig := range rawTableConfigs { var cfg katcTableConfig - if err := json.Unmarshal([]byte(tableConfigStr), &cfg); err != nil { + if err := json.Unmarshal(rawTableConfig, &cfg); err != nil { slogger.Log(context.TODO(), slog.LevelWarn, "unable to unmarshal config for Kolide ATC table, skipping", - "table_name", tableName, "err", err, ) continue @@ -118,8 +139,8 @@ func ConstructKATCTables(config map[string]string, slogger *slog.Logger) []osque continue } - t, columns := newKatcTable(tableName, cfg, slogger) - plugins = append(plugins, table.NewPlugin(tableName, columns, t.generate)) + t, columns := newKatcTable(cfg, slogger) + plugins = append(plugins, table.NewPlugin(cfg.Name, columns, t.generate)) } return plugins diff --git a/ee/katc/config_test.go b/ee/katc/config_test.go index 3838a5566..471c4e8a2 100644 --- a/ee/katc/config_test.go +++ b/ee/katc/config_test.go @@ -21,39 +21,72 @@ func TestConstructKATCTables(t *testing.T) { { testCaseName: "snappy_sqlite", katcConfig: map[string]string{ - "kolide_snappy_sqlite_test": fmt.Sprintf(`{ - "source_type": "sqlite", - "filter": "%s", - "columns": ["data"], - "source_paths": ["/some/path/to/db.sqlite"], - "source_query": "SELECT data FROM object_data JOIN object_store ON (object_data.object_store_id = object_store.id) WHERE object_store.name=\"testtable\";", - "row_transform_steps": ["snappy"] - }`, runtime.GOOS), + "tables": fmt.Sprintf(`[ + { + "name": "kolide_snappy_sqlite_test", + "source_type": "sqlite", + "filter": "%s", + "columns": ["data"], + "source_paths": ["/some/path/to/db.sqlite"], + "source_query": "SELECT data FROM object_data JOIN object_store ON (object_data.object_store_id = object_store.id) WHERE object_store.name=\"testtable\";", + "row_transform_steps": ["snappy"] + } + ]`, runtime.GOOS), }, expectedPluginCount: 1, }, { testCaseName: "multiple plugins", katcConfig: map[string]string{ - "test_1": fmt.Sprintf(`{ - "source_type": "sqlite", - "filter": "%s", - "columns": ["data"], - "source_paths": ["/some/path/to/db.sqlite"], - "source_query": "SELECT data FROM object_data;", - "row_transform_steps": ["snappy"] - }`, runtime.GOOS), - "test_2": fmt.Sprintf(`{ - "source_type": "sqlite", - "filter": "%s", - "columns": ["col1", "col2"], - "source_paths": ["/some/path/to/a/different/db.sqlite"], - "source_query": "SELECT col1, col2 FROM some_table;", - "row_transform_steps": ["camel_to_snake"] - }`, runtime.GOOS), + "tables": fmt.Sprintf(`[ + { + "name": "test_1", + "source_type": "sqlite", + "filter": "%s", + "columns": ["data"], + "source_paths": ["/some/path/to/db.sqlite"], + "source_query": "SELECT data FROM object_data;", + "row_transform_steps": ["snappy"] + }, + { + "name": "test_2", + "source_type": "sqlite", + "filter": "%s", + "columns": ["col1", "col2"], + "source_paths": ["/some/path/to/a/different/db.sqlite"], + "source_query": "SELECT col1, col2 FROM some_table;", + "row_transform_steps": ["camel_to_snake"] + } + ]`, runtime.GOOS, runtime.GOOS), }, expectedPluginCount: 2, }, + { + testCaseName: "skips invalid tables and returns valid tables", + katcConfig: map[string]string{ + "tables": fmt.Sprintf(`[ + { + "name": "not_a_valid_table", + "source_type": "not a real type", + "filter": "%s", + "columns": ["col1", "col2"], + "source_paths": ["/some/path/to/a/different/db.sqlite"], + "source_query": "SELECT col1, col2 FROM some_table;", + "row_transform_steps": ["not a real row transform step"] + }, + { + "name": "valid_table", + "source_type": "sqlite", + "filter": "%s", + "columns": ["data"], + "source_paths": ["/some/path/to/db.sqlite"], + "source_query": "SELECT data FROM object_data;", + "row_transform_steps": ["snappy"] + } + ]`, runtime.GOOS, runtime.GOOS), + }, + expectedPluginCount: 1, + }, { testCaseName: "malformed config", katcConfig: map[string]string{ @@ -61,30 +94,43 @@ func TestConstructKATCTables(t *testing.T) { }, expectedPluginCount: 0, }, + { + testCaseName: "malformed table", + katcConfig: map[string]string{ + "tables": "this is not a config", + }, + expectedPluginCount: 0, + }, { testCaseName: "invalid table source", katcConfig: map[string]string{ - "kolide_snappy_test": fmt.Sprintf(`{ - "source_type": "unknown_source", - "filter": "%s", - "columns": ["data"], - "source_paths": ["/some/path/to/db.sqlite"], - "source_query": "SELECT data FROM object_data;" - }`, runtime.GOOS), + "tables": fmt.Sprintf(`[ + { + "name": "kolide_snappy_test", + "source_type": "unknown_source", + "filter": "%s", + "columns": ["data"], + "source_paths": ["/some/path/to/db.sqlite"], + "source_query": "SELECT data FROM object_data;" + } + ]`, runtime.GOOS), }, expectedPluginCount: 0, }, { testCaseName: "invalid data processing step type", katcConfig: map[string]string{ - "kolide_snappy_test": fmt.Sprintf(`{ - "source_type": "sqlite", - "filter": "%s", - "columns": ["data"], - "source_paths": ["/some/path/to/db.sqlite"], - "source_query": "SELECT data FROM object_data;", - "row_transform_steps": ["unknown_step"] - }`, runtime.GOOS), + "tables": fmt.Sprintf(`[ + { + "name": "kolide_snappy_test", + "source_type": "sqlite", + "filter": "%s", + "columns": ["data"], + "source_paths": ["/some/path/to/db.sqlite"], + "source_query": "SELECT data FROM object_data;", + "row_transform_steps": ["unknown_step"] + } + ]`, runtime.GOOS), }, expectedPluginCount: 0, }, diff --git a/ee/katc/table.go b/ee/katc/table.go index 5370e089d..cc541994f 100644 --- a/ee/katc/table.go +++ b/ee/katc/table.go @@ -21,7 +21,7 @@ type katcTable struct { } // newKatcTable returns a new table with the given `cfg`, as well as the osquery columns for that table. -func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) (*katcTable, []table.ColumnDefinition) { +func newKatcTable(cfg katcTableConfig, slogger *slog.Logger) (*katcTable, []table.ColumnDefinition) { columns := []table.ColumnDefinition{ { Name: pathColumnName, @@ -43,7 +43,7 @@ func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) ( cfg: cfg, columnLookup: columnLookup, slogger: slogger.With( - "table_name", tableName, + "table_name", cfg.Name, "table_type", cfg.SourceType, "table_source_paths", cfg.SourcePaths, ), diff --git a/ee/katc/table_test.go b/ee/katc/table_test.go index fbef23f39..75d7daeb1 100644 --- a/ee/katc/table_test.go +++ b/ee/katc/table_test.go @@ -82,6 +82,7 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) { // At long last, our source is adequately configured. // Move on to constructing our KATC table. cfg := katcTableConfig{ + Name: "test_katc_table", SourceType: katcSourceType{ name: sqliteSourceType, dataFunc: sqliteData, @@ -101,7 +102,7 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) { }, }, } - testTable, _ := newKatcTable("test_katc_table", cfg, multislogger.NewNopLogger()) + testTable, _ := newKatcTable(cfg, multislogger.NewNopLogger()) // Make a query context restricting the source to our exact source sqlite database queryContext := table.QueryContext{ From 2367637e7cd477fc7d03f80a4459de57866a9764 Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Wed, 3 Jul 2024 15:12:08 -0400 Subject: [PATCH 07/17] Rename sourceConstraints to pathConstraints in function signature --- ee/katc/config.go | 2 +- ee/katc/indexeddb_leveldb.go | 6 +++--- ee/katc/sqlite.go | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ee/katc/config.go b/ee/katc/config.go index 567c6d8ab..f6e0c6926 100644 --- a/ee/katc/config.go +++ b/ee/katc/config.go @@ -17,7 +17,7 @@ import ( // that performs the query against the source. type katcSourceType struct { name string - dataFunc func(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, sourceConstraints *table.ConstraintList) ([]sourceData, error) + dataFunc func(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, pathConstraints *table.ConstraintList) ([]sourceData, error) } // sourceData holds the result of calling `katcSourceType.dataFunc`. It maps the diff --git a/ee/katc/indexeddb_leveldb.go b/ee/katc/indexeddb_leveldb.go index 5325c32af..ed5c7cd71 100644 --- a/ee/katc/indexeddb_leveldb.go +++ b/ee/katc/indexeddb_leveldb.go @@ -15,7 +15,7 @@ import ( // found at the filepath in `sourcePattern`. It retrieves all rows from the database // and object store specified in `query`, which it expects to be in the format // `.`. -func indexeddbLeveldbData(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, sourceConstraints *table.ConstraintList) ([]sourceData, error) { +func indexeddbLeveldbData(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, pathConstraints *table.ConstraintList) ([]sourceData, error) { results := make([]sourceData, 0) for _, sourcePath := range sourcePaths { pathPattern := sourcePatternToGlobbablePattern(sourcePath) @@ -32,8 +32,8 @@ func indexeddbLeveldbData(ctx context.Context, slogger *slog.Logger, sourcePaths // Query databases for _, db := range leveldbs { - // Check to make sure `db` adheres to sourceConstraints - valid, err := checkPathConstraints(db, sourceConstraints) + // Check to make sure `db` adheres to pathConstraints + valid, err := checkPathConstraints(db, pathConstraints) if err != nil { return nil, fmt.Errorf("checking source path constraints: %w", err) } diff --git a/ee/katc/sqlite.go b/ee/katc/sqlite.go index 5bc2f684e..ead532521 100644 --- a/ee/katc/sqlite.go +++ b/ee/katc/sqlite.go @@ -13,7 +13,7 @@ import ( ) // sqliteData is the dataFunc for sqlite KATC tables -func sqliteData(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, sourceConstraints *table.ConstraintList) ([]sourceData, error) { +func sqliteData(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, pathConstraints *table.ConstraintList) ([]sourceData, error) { results := make([]sourceData, 0) for _, sourcePath := range sourcePaths { pathPattern := sourcePatternToGlobbablePattern(sourcePath) @@ -23,8 +23,8 @@ func sqliteData(ctx context.Context, slogger *slog.Logger, sourcePaths []string, } for _, sqliteDb := range sqliteDbs { - // Check to make sure `sqliteDb` adheres to sourceConstraints - valid, err := checkPathConstraints(sqliteDb, sourceConstraints) + // Check to make sure `sqliteDb` adheres to pathConstraints + valid, err := checkPathConstraints(sqliteDb, pathConstraints) if err != nil { return nil, fmt.Errorf("checking source path constraints: %w", err) } From 479571cc02fa9c48da7673df6a5d2e6024bbd24c Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Mon, 8 Jul 2024 09:09:15 -0400 Subject: [PATCH 08/17] Add comment --- ee/katc/config.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ee/katc/config.go b/ee/katc/config.go index f6e0c6926..10a44a11e 100644 --- a/ee/katc/config.go +++ b/ee/katc/config.go @@ -16,7 +16,10 @@ import ( // identifier parsed from the JSON KATC config, and the `dataFunc` is the function // that performs the query against the source. type katcSourceType struct { - name string + name string + // `pathConstraints` comes from the live query or scheduled query, and constrains `sourcePaths` to a particular value. + // For example, `sourcePaths` may allow for querying files belonging to any user ("/Users/%/path/to/file"), and + // `pathConstraints` may narrow the query to a path for a particular user ("/Users/example/path/to/file"). dataFunc func(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, pathConstraints *table.ConstraintList) ([]sourceData, error) } From e2c93da94074e2a8b51a0fb69d5eed8b7815f99c Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Mon, 8 Jul 2024 10:42:56 -0400 Subject: [PATCH 09/17] Implement overlays --- ee/katc/config.go | 66 +++++++---------- ee/katc/config_test.go | 161 ++++++++++++++++++++--------------------- ee/katc/table.go | 62 +++++++++++++--- ee/katc/table_test.go | 14 +++- 4 files changed, 163 insertions(+), 140 deletions(-) diff --git a/ee/katc/config.go b/ee/katc/config.go index 10a44a11e..cdffa110e 100644 --- a/ee/katc/config.go +++ b/ee/katc/config.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "log/slog" - "runtime" "github.com/kolide/launcher/ee/indexeddb" "github.com/osquery/osquery-go" @@ -101,60 +100,45 @@ func (r *rowTransformStep) UnmarshalJSON(data []byte) error { } } -// katcTableConfig is the configuration for a specific KATC table. The control server -// sends down these configurations. -type katcTableConfig struct { - Name string `json:"name"` - SourceType katcSourceType `json:"source_type"` - SourcePaths []string `json:"source_paths"` // Describes how to connect to source (e.g. path to db) -- % and _ wildcards supported - Filter string `json:"filter"` - Columns []string `json:"columns"` - SourceQuery string `json:"source_query"` // Query to run against each source path - RowTransformSteps []rowTransformStep `json:"row_transform_steps"` -} +type ( + // katcTableConfig is the configuration for a specific KATC table. The control server + // sends down these configurations. + katcTableConfig struct { + Columns []string `json:"columns"` + SourceType katcSourceType `json:"source_type"` + SourcePaths []string `json:"source_paths"` // Describes how to connect to source (e.g. path to db) -- % and _ wildcards supported + SourceQuery string `json:"source_query"` // Query to run against each source path + RowTransformSteps []rowTransformStep `json:"row_transform_steps"` + Overlays []katcTableConfigOverlay `json:"overlays"` + } + + katcTableConfigOverlay struct { + Filters map[string]string `json:"filters"` // determines if this overlay is applicable to this launcher installation + SourceType *katcSourceType `json:"source_type,omitempty"` + SourcePaths *[]string `json:"source_paths,omitempty"` // Describes how to connect to source (e.g. path to db) -- % and _ wildcards supported + SourceQuery *string `json:"source_query,omitempty"` // Query to run against each source path + RowTransformSteps *[]rowTransformStep `json:"row_transform_steps,omitempty"` + } +) // ConstructKATCTables takes stored configuration of KATC tables, parses the configuration, // and returns the constructed tables. func ConstructKATCTables(config map[string]string, slogger *slog.Logger) []osquery.OsqueryPlugin { plugins := make([]osquery.OsqueryPlugin, 0) - tableConfigs, tableConfigsExist := config["tables"] - if !tableConfigsExist { - slogger.Log(context.TODO(), slog.LevelWarn, - "missing top-level tables key in KATC config, cannot construct tables", - ) - - return plugins - } - - // We want to unmarshal each table config separately, so that we don't fail to configure all tables - // if only some are malformed. - var rawTableConfigs []json.RawMessage - if err := json.Unmarshal([]byte(tableConfigs), &rawTableConfigs); err != nil { - slogger.Log(context.TODO(), slog.LevelWarn, - "could not unmarshal tables in KATC config", - "err", err, - ) - return plugins - } - - for _, rawTableConfig := range rawTableConfigs { + for tableName, rawTableConfig := range config { var cfg katcTableConfig - if err := json.Unmarshal(rawTableConfig, &cfg); err != nil { + if err := json.Unmarshal([]byte(rawTableConfig), &cfg); err != nil { slogger.Log(context.TODO(), slog.LevelWarn, "unable to unmarshal config for KATC table, skipping", + "table_name", tableName, "err", err, ) continue } - // For now, the filter is simply the OS - if cfg.Filter != runtime.GOOS { - continue - } - - t, columns := newKatcTable(cfg, slogger) - plugins = append(plugins, table.NewPlugin(cfg.Name, columns, t.generate)) + t, columns := newKatcTable(tableName, cfg, slogger) + plugins = append(plugins, table.NewPlugin(tableName, columns, t.generate)) } return plugins diff --git a/ee/katc/config_test.go b/ee/katc/config_test.go index 7abb222de..fed0272e1 100644 --- a/ee/katc/config_test.go +++ b/ee/katc/config_test.go @@ -21,86 +21,93 @@ func TestConstructKATCTables(t *testing.T) { { testCaseName: "snappy_sqlite", katcConfig: map[string]string{ - "tables": fmt.Sprintf(`[ - { - "name": "kolide_snappy_sqlite_test", - "source_type": "sqlite", - "filter": "%s", - "columns": ["data"], - "source_paths": ["/some/path/to/db.sqlite"], - "source_query": "SELECT data FROM object_data JOIN object_store ON (object_data.object_store_id = object_store.id) WHERE object_store.name=\"testtable\";", - "row_transform_steps": ["snappy"] - } - ]`, runtime.GOOS), + "kolide_snappy_sqlite_test": `{ + "source_type": "sqlite", + "columns": ["data"], + "source_paths": ["/some/path/to/db.sqlite"], + "source_query": "SELECT data FROM object_data JOIN object_store ON (object_data.object_store_id = object_store.id) WHERE object_store.name=\"testtable\";", + "row_transform_steps": ["snappy"], + "overlays": [] + }`, }, expectedPluginCount: 1, }, { testCaseName: "indexeddb_leveldb", katcConfig: map[string]string{ - "tables": fmt.Sprintf(`[ - { - "name": "kolide_indexeddb_leveldb_test", - "source_type": "indexeddb_leveldb", - "filter": "%s", - "columns": ["data"], - "source_paths": ["/some/path/to/db.indexeddb.leveldb"], - "source_query": "db.store", - "row_transform_steps": ["deserialize_chrome"] - } - ]`, runtime.GOOS), + "kolide_indexeddb_leveldb_test": `{ + "source_type": "indexeddb_leveldb", + "columns": ["data"], + "source_paths": ["/some/path/to/db.indexeddb.leveldb"], + "source_query": "db.store", + "row_transform_steps": ["deserialize_chrome"], + "overlays": [] + }`, + }, + expectedPluginCount: 1, + }, + { + testCaseName: "overlay", + katcConfig: map[string]string{ + "kolide_overlay_test": fmt.Sprintf(`{ + "source_type": "indexeddb_leveldb", + "columns": ["data"], + "source_paths": ["/some/path/to/db.indexeddb.leveldb"], + "source_query": "db.store", + "row_transform_steps": ["deserialize_chrome"], + "overlays": [ + { + "filters": { + "goos": "%s" + }, + "source_paths": ["/some/different/path/to/db.indexeddb.leveldb"] + } + ] + }`, runtime.GOOS), }, expectedPluginCount: 1, }, { testCaseName: "multiple plugins", katcConfig: map[string]string{ - "tables": fmt.Sprintf(`[ - { - "name": "test_1", - "source_type": "sqlite", - "filter": "%s", - "columns": ["data"], - "source_paths": ["/some/path/to/db.sqlite"], - "source_query": "SELECT data FROM object_data;", - "row_transform_steps": ["snappy"] - }, - { - "name": "test_2", - "source_type": "sqlite", - "filter": "%s", - "columns": ["col1", "col2"], - "source_paths": ["/some/path/to/a/different/db.sqlite"], - "source_query": "SELECT col1, col2 FROM some_table;", - "row_transform_steps": ["camel_to_snake"] - } - ]`, runtime.GOOS, runtime.GOOS), + "test_1": `{ + "source_type": "sqlite", + "columns": ["data"], + "source_paths": ["/some/path/to/db.sqlite"], + "source_query": "SELECT data FROM object_data;", + "row_transform_steps": ["snappy"], + "overlays": [] + }`, + "test_2": `{ + "source_type": "sqlite", + "columns": ["col1", "col2"], + "source_paths": ["/some/path/to/a/different/db.sqlite"], + "source_query": "SELECT col1, col2 FROM some_table;", + "row_transform_steps": ["camel_to_snake"], + "overlays": [] + }`, }, expectedPluginCount: 2, }, { testCaseName: "skips invalid tables and returns valid tables", katcConfig: map[string]string{ - "tables": fmt.Sprintf(`[ - { - "name": "not_a_valid_table", - "source_type": "not a real type", - "filter": "%s", + "not_a_valid_table": `{ + "source_type": "not a real type", "columns": ["col1", "col2"], "source_paths": ["/some/path/to/a/different/db.sqlite"], "source_query": "SELECT col1, col2 FROM some_table;", - "row_transform_steps": ["not a real row transform step"] - }, - { - "name": "valid_table", - "source_type": "sqlite", - "filter": "%s", + "row_transform_steps": ["not a real row transform step"], + "overlays": [] + }`, + "valid_table": `{ + "source_type": "sqlite", "columns": ["data"], "source_paths": ["/some/path/to/db.sqlite"], "source_query": "SELECT data FROM object_data;", - "row_transform_steps": ["snappy"] - } - ]`, runtime.GOOS, runtime.GOOS), + "row_transform_steps": ["snappy"], + "overlays": [] + }`, }, expectedPluginCount: 1, }, @@ -111,43 +118,29 @@ func TestConstructKATCTables(t *testing.T) { }, expectedPluginCount: 0, }, - { - testCaseName: "malformed table", - katcConfig: map[string]string{ - "tables": "this is not a config", - }, - expectedPluginCount: 0, - }, { testCaseName: "invalid table source", katcConfig: map[string]string{ - "tables": fmt.Sprintf(`[ - { - "name": "kolide_snappy_test", - "source_type": "unknown_source", - "filter": "%s", - "columns": ["data"], - "source_paths": ["/some/path/to/db.sqlite"], - "source_query": "SELECT data FROM object_data;" - } - ]`, runtime.GOOS), + "kolide_snappy_test": `{ + "source_type": "unknown_source", + "columns": ["data"], + "source_paths": ["/some/path/to/db.sqlite"], + "source_query": "SELECT data FROM object_data;", + "overlays": [] + }`, }, expectedPluginCount: 0, }, { testCaseName: "invalid data processing step type", katcConfig: map[string]string{ - "tables": fmt.Sprintf(`[ - { - "name": "kolide_snappy_test", - "source_type": "sqlite", - "filter": "%s", - "columns": ["data"], - "source_paths": ["/some/path/to/db.sqlite"], - "source_query": "SELECT data FROM object_data;", - "row_transform_steps": ["unknown_step"] - } - ]`, runtime.GOOS), + "kolide_snappy_test": `{ + "source_type": "sqlite", + "columns": ["data"], + "source_paths": ["/some/path/to/db.sqlite"], + "source_query": "SELECT data FROM object_data;", + "row_transform_steps": ["unknown_step"] + }`, }, expectedPluginCount: 0, }, diff --git a/ee/katc/table.go b/ee/katc/table.go index cc541994f..8192cc7f3 100644 --- a/ee/katc/table.go +++ b/ee/katc/table.go @@ -5,6 +5,7 @@ import ( "fmt" "log/slog" "regexp" + "runtime" "strings" "github.com/osquery/osquery-go/plugin/table" @@ -15,13 +16,16 @@ const pathColumnName = "path" // katcTable is a Kolide ATC table. It queries the source and transforms the response data // per the configuration in its `cfg`. type katcTable struct { - cfg katcTableConfig - columnLookup map[string]struct{} - slogger *slog.Logger + sourceType katcSourceType + sourcePaths []string + sourceQuery string + rowTransformSteps []rowTransformStep + columnLookup map[string]struct{} + slogger *slog.Logger } // newKatcTable returns a new table with the given `cfg`, as well as the osquery columns for that table. -func newKatcTable(cfg katcTableConfig, slogger *slog.Logger) (*katcTable, []table.ColumnDefinition) { +func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) (*katcTable, []table.ColumnDefinition) { columns := []table.ColumnDefinition{ { Name: pathColumnName, @@ -39,21 +43,57 @@ func newKatcTable(cfg katcTableConfig, slogger *slog.Logger) (*katcTable, []tabl columnLookup[cfg.Columns[i]] = struct{}{} } - return &katcTable{ - cfg: cfg, - columnLookup: columnLookup, + k := katcTable{ + sourceType: cfg.SourceType, + sourcePaths: cfg.SourcePaths, + sourceQuery: cfg.SourceQuery, + rowTransformSteps: cfg.RowTransformSteps, + columnLookup: columnLookup, slogger: slogger.With( - "table_name", cfg.Name, + "table_name", tableName, "table_type", cfg.SourceType, "table_source_paths", cfg.SourcePaths, ), - }, columns + } + + // Check overlays to see if any of the filters apply to us; + // use the overlay definition if so. + for _, overlay := range cfg.Overlays { + if !filtersMatch(overlay.Filters) { + continue + } + + if overlay.SourceType != nil { + k.sourceType = *overlay.SourceType + } + if overlay.SourcePaths != nil { + k.sourcePaths = *overlay.SourcePaths + } + if overlay.SourceQuery != nil { + k.sourceQuery = *overlay.SourceQuery + } + if overlay.RowTransformSteps != nil { + k.rowTransformSteps = *overlay.RowTransformSteps + } + + break + } + + return &k, columns +} + +func filtersMatch(filters map[string]string) bool { + // Currently, the only filter we expect is for os. + if goos, goosFound := filters["goos"]; goosFound { + return goos == runtime.GOOS + } + return false } // generate handles queries against a KATC table. func (k *katcTable) generate(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { // Fetch data from our table source - dataRaw, err := k.cfg.SourceType.dataFunc(ctx, k.slogger, k.cfg.SourcePaths, k.cfg.SourceQuery, getSourceConstraint(queryContext)) + dataRaw, err := k.sourceType.dataFunc(ctx, k.slogger, k.sourcePaths, k.sourceQuery, getSourceConstraint(queryContext)) if err != nil { return nil, fmt.Errorf("fetching data: %w", err) } @@ -68,7 +108,7 @@ func (k *katcTable) generate(ctx context.Context, queryContext table.QueryContex } // Run any needed transformations on the row data - for _, step := range k.cfg.RowTransformSteps { + for _, step := range k.rowTransformSteps { dataRawRow, err = step.transformFunc(ctx, k.slogger, dataRawRow) if err != nil { return nil, fmt.Errorf("running transform func %s: %w", step.name, err) diff --git a/ee/katc/table_test.go b/ee/katc/table_test.go index 75d7daeb1..ad3d369a8 100644 --- a/ee/katc/table_test.go +++ b/ee/katc/table_test.go @@ -82,14 +82,12 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) { // At long last, our source is adequately configured. // Move on to constructing our KATC table. cfg := katcTableConfig{ - Name: "test_katc_table", SourceType: katcSourceType{ name: sqliteSourceType, dataFunc: sqliteData, }, - Filter: runtime.GOOS, Columns: []string{expectedColumn}, - SourcePaths: []string{filepath.Join(databaseDir, "%.sqlite")}, // All sqlite files in the test directory + SourcePaths: []string{filepath.Join("some", "incorrect", "path")}, SourceQuery: "SELECT data FROM object_data;", RowTransformSteps: []rowTransformStep{ { @@ -101,8 +99,16 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) { transformFunc: deserializeFirefox, }, }, + Overlays: []katcTableConfigOverlay{ + { + Filters: map[string]string{ + "goos": runtime.GOOS, + }, + SourcePaths: &[]string{filepath.Join(databaseDir, "%.sqlite")}, // All sqlite files in the test directory + }, + }, } - testTable, _ := newKatcTable(cfg, multislogger.NewNopLogger()) + testTable, _ := newKatcTable("test_katc_table", cfg, multislogger.NewNopLogger()) // Make a query context restricting the source to our exact source sqlite database queryContext := table.QueryContext{ From b698ac74b39fcd7ae787f7551750e20b0f392c98 Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Mon, 8 Jul 2024 10:46:35 -0400 Subject: [PATCH 10/17] Cleanup --- ee/katc/config.go | 4 ++-- ee/katc/table.go | 15 +++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ee/katc/config.go b/ee/katc/config.go index cdffa110e..1d8feca32 100644 --- a/ee/katc/config.go +++ b/ee/katc/config.go @@ -126,9 +126,9 @@ type ( func ConstructKATCTables(config map[string]string, slogger *slog.Logger) []osquery.OsqueryPlugin { plugins := make([]osquery.OsqueryPlugin, 0) - for tableName, rawTableConfig := range config { + for tableName, tableConfigStr := range config { var cfg katcTableConfig - if err := json.Unmarshal([]byte(rawTableConfig), &cfg); err != nil { + if err := json.Unmarshal([]byte(tableConfigStr), &cfg); err != nil { slogger.Log(context.TODO(), slog.LevelWarn, "unable to unmarshal config for KATC table, skipping", "table_name", tableName, diff --git a/ee/katc/table.go b/ee/katc/table.go index 8192cc7f3..2b07d03e5 100644 --- a/ee/katc/table.go +++ b/ee/katc/table.go @@ -49,11 +49,7 @@ func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) ( sourceQuery: cfg.SourceQuery, rowTransformSteps: cfg.RowTransformSteps, columnLookup: columnLookup, - slogger: slogger.With( - "table_name", tableName, - "table_type", cfg.SourceType, - "table_source_paths", cfg.SourcePaths, - ), + slogger: slogger, } // Check overlays to see if any of the filters apply to us; @@ -79,11 +75,18 @@ func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) ( break } + // Add extra fields to slogger + k.slogger = slogger.With( + "table_name", tableName, + "table_type", cfg.SourceType, + "table_source_paths", cfg.SourcePaths, + ) + return &k, columns } func filtersMatch(filters map[string]string) bool { - // Currently, the only filter we expect is for os. + // Currently, the only filter we expect is for OS. if goos, goosFound := filters["goos"]; goosFound { return goos == runtime.GOOS } From 2efd17ebe2b265426bbc14707e6360a35afc5b32 Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Thu, 11 Jul 2024 17:08:51 -0400 Subject: [PATCH 11/17] dataFunc should take queryContext instead of constraint list for greater flexibility and clarity --- ee/katc/config.go | 6 ++---- ee/katc/indexeddb_leveldb.go | 9 ++++++--- ee/katc/sqlite.go | 9 ++++++--- ee/katc/sqlite_test.go | 5 +++-- ee/katc/table.go | 11 +++++------ 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/ee/katc/config.go b/ee/katc/config.go index 1d8feca32..3c59f4b38 100644 --- a/ee/katc/config.go +++ b/ee/katc/config.go @@ -16,10 +16,8 @@ import ( // that performs the query against the source. type katcSourceType struct { name string - // `pathConstraints` comes from the live query or scheduled query, and constrains `sourcePaths` to a particular value. - // For example, `sourcePaths` may allow for querying files belonging to any user ("/Users/%/path/to/file"), and - // `pathConstraints` may narrow the query to a path for a particular user ("/Users/example/path/to/file"). - dataFunc func(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, pathConstraints *table.ConstraintList) ([]sourceData, error) + // queryContext contains the constraints from the WHERE clause of the query against the KATC table. + dataFunc func(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, queryContext table.QueryContext) ([]sourceData, error) } // sourceData holds the result of calling `katcSourceType.dataFunc`. It maps the diff --git a/ee/katc/indexeddb_leveldb.go b/ee/katc/indexeddb_leveldb.go index ed5c7cd71..c04b7d902 100644 --- a/ee/katc/indexeddb_leveldb.go +++ b/ee/katc/indexeddb_leveldb.go @@ -15,7 +15,10 @@ import ( // found at the filepath in `sourcePattern`. It retrieves all rows from the database // and object store specified in `query`, which it expects to be in the format // `.`. -func indexeddbLeveldbData(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, pathConstraints *table.ConstraintList) ([]sourceData, error) { +func indexeddbLeveldbData(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, queryContext table.QueryContext) ([]sourceData, error) { + // Pull out path constraints from the query against the KATC table, to avoid querying more leveldb files than we need to. + pathConstraintsFromQuery := getPathConstraint(queryContext) + results := make([]sourceData, 0) for _, sourcePath := range sourcePaths { pathPattern := sourcePatternToGlobbablePattern(sourcePath) @@ -32,8 +35,8 @@ func indexeddbLeveldbData(ctx context.Context, slogger *slog.Logger, sourcePaths // Query databases for _, db := range leveldbs { - // Check to make sure `db` adheres to pathConstraints - valid, err := checkPathConstraints(db, pathConstraints) + // Check to make sure `db` adheres to pathConstraintsFromQuery + valid, err := checkPathConstraints(db, pathConstraintsFromQuery) if err != nil { return nil, fmt.Errorf("checking source path constraints: %w", err) } diff --git a/ee/katc/sqlite.go b/ee/katc/sqlite.go index ead532521..a2afdf17d 100644 --- a/ee/katc/sqlite.go +++ b/ee/katc/sqlite.go @@ -13,7 +13,10 @@ import ( ) // sqliteData is the dataFunc for sqlite KATC tables -func sqliteData(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, pathConstraints *table.ConstraintList) ([]sourceData, error) { +func sqliteData(ctx context.Context, slogger *slog.Logger, sourcePaths []string, query string, queryContext table.QueryContext) ([]sourceData, error) { + // Pull out path constraints from the query against the KATC table, to avoid querying more sqlite dbs than we need to. + pathConstraintsFromQuery := getPathConstraint(queryContext) + results := make([]sourceData, 0) for _, sourcePath := range sourcePaths { pathPattern := sourcePatternToGlobbablePattern(sourcePath) @@ -23,8 +26,8 @@ func sqliteData(ctx context.Context, slogger *slog.Logger, sourcePaths []string, } for _, sqliteDb := range sqliteDbs { - // Check to make sure `sqliteDb` adheres to pathConstraints - valid, err := checkPathConstraints(sqliteDb, pathConstraints) + // Check to make sure `sqliteDb` adheres to pathConstraintsFromQuery + valid, err := checkPathConstraints(sqliteDb, pathConstraintsFromQuery) if err != nil { return nil, fmt.Errorf("checking source path constraints: %w", err) } diff --git a/ee/katc/sqlite_test.go b/ee/katc/sqlite_test.go index 7cdcdde73..c1510510e 100644 --- a/ee/katc/sqlite_test.go +++ b/ee/katc/sqlite_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/kolide/launcher/pkg/log/multislogger" + "github.com/osquery/osquery-go/plugin/table" "github.com/stretchr/testify/require" ) @@ -59,7 +60,7 @@ func Test_sqliteData(t *testing.T) { } // Query data - results, err := sqliteData(context.TODO(), multislogger.NewNopLogger(), []string{filepath.Join(sqliteDir, "*.sqlite")}, "SELECT uuid, value FROM test_data;", nil) + results, err := sqliteData(context.TODO(), multislogger.NewNopLogger(), []string{filepath.Join(sqliteDir, "*.sqlite")}, "SELECT uuid, value FROM test_data;", table.QueryContext{}) require.NoError(t, err) // Confirm we have the correct number of `sourceData` returned (one per db) @@ -89,7 +90,7 @@ func Test_sqliteData_noSourcesFound(t *testing.T) { t.Parallel() tmpDir := t.TempDir() - results, err := sqliteData(context.TODO(), multislogger.NewNopLogger(), []string{filepath.Join(tmpDir, "db.sqlite")}, "SELECT * FROM data;", nil) + results, err := sqliteData(context.TODO(), multislogger.NewNopLogger(), []string{filepath.Join(tmpDir, "db.sqlite")}, "SELECT * FROM data;", table.QueryContext{}) require.NoError(t, err) require.Equal(t, 0, len(results)) } diff --git a/ee/katc/table.go b/ee/katc/table.go index 2b07d03e5..b30ff49c1 100644 --- a/ee/katc/table.go +++ b/ee/katc/table.go @@ -96,7 +96,7 @@ func filtersMatch(filters map[string]string) bool { // generate handles queries against a KATC table. func (k *katcTable) generate(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { // Fetch data from our table source - dataRaw, err := k.sourceType.dataFunc(ctx, k.slogger, k.sourcePaths, k.sourceQuery, getSourceConstraint(queryContext)) + dataRaw, err := k.sourceType.dataFunc(ctx, k.slogger, k.sourcePaths, k.sourceQuery, queryContext) if err != nil { return nil, fmt.Errorf("fetching data: %w", err) } @@ -146,11 +146,10 @@ func (k *katcTable) generate(ctx context.Context, queryContext table.QueryContex return filteredResults, nil } -// getSourceConstraint retrieves any constraints against the `path` column -func getSourceConstraint(queryContext table.QueryContext) *table.ConstraintList { - sourceConstraint, sourceConstraintExists := queryContext.Constraints[pathColumnName] - if sourceConstraintExists { - return &sourceConstraint +// getPathConstraint retrieves any constraints against the `path` column +func getPathConstraint(queryContext table.QueryContext) *table.ConstraintList { + if pathConstraint, pathConstraintExists := queryContext.Constraints[pathColumnName]; pathConstraintExists { + return &pathConstraint } return nil } From 50b478eb5814e99cc1ad65c906509034cfa68d54 Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Fri, 12 Jul 2024 16:19:16 -0400 Subject: [PATCH 12/17] Standardize log messages a little --- pkg/osquery/runtime/runner.go | 4 ++-- pkg/osquery/table/table.go | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/osquery/runtime/runner.go b/pkg/osquery/runtime/runner.go index d96790bf8..a1282c807 100644 --- a/pkg/osquery/runtime/runner.go +++ b/pkg/osquery/runtime/runner.go @@ -187,12 +187,12 @@ func (r *Runner) FlagsChanged(flagKeys ...keys.FlagKey) { // the katc_config subsystem. func (r *Runner) Ping() { r.slogger.Log(context.TODO(), slog.LevelDebug, - "Kolide ATC configuration changed, restarting instance to apply", + "KATC configuration changed, restarting instance to apply", ) if err := r.Restart(); err != nil { r.slogger.Log(context.TODO(), slog.LevelError, - "could not restart osquery instance after Kolide ATC configuration changed", + "could not restart osquery instance after KATC configuration changed", "err", err, ) } diff --git a/pkg/osquery/table/table.go b/pkg/osquery/table/table.go index e5e2d0517..7a72d21ea 100644 --- a/pkg/osquery/table/table.go +++ b/pkg/osquery/table/table.go @@ -77,22 +77,21 @@ func PlatformTables(k types.Knapsack, slogger *slog.Logger, currentOsquerydBinar return tables } -// kolideCustomAtcTables retrieves Kolide ATC config from the appropriate data store(s). -// For now, it just logs the configuration. In the future, it will handle indexeddb tables -// and others. +// kolideCustomAtcTables retrieves Kolide ATC config from the appropriate data store(s), +// then constructs the tables. func kolideCustomAtcTables(k types.Knapsack, slogger *slog.Logger) []osquery.OsqueryPlugin { // Fetch tables from KVStore or from startup settings config, err := katcFromDb(k) if err != nil { slogger.Log(context.TODO(), slog.LevelDebug, - "could not retrieve Kolide ATC config from store, may not have access -- falling back to startup settings", + "could not retrieve KATC config from store, may not have access -- falling back to startup settings", "err", err, ) config, err = katcFromStartupSettings(k) if err != nil { slogger.Log(context.TODO(), slog.LevelWarn, - "could not retrieve Kolide ATC config from startup settings", + "could not retrieve KATC config from startup settings", "err", err, ) return nil From 8894bc2f0d1cb2ce1edeb3132cd96320299016ef Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Fri, 12 Jul 2024 16:19:41 -0400 Subject: [PATCH 13/17] Add config consumer that can handle non-string values --- cmd/launcher/launcher.go | 2 +- .../keyvalueconsumer/config_consumer.go | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 ee/control/consumers/keyvalueconsumer/config_consumer.go diff --git a/cmd/launcher/launcher.go b/cmd/launcher/launcher.go index 9087eb92c..fa487039c 100644 --- a/cmd/launcher/launcher.go +++ b/cmd/launcher/launcher.go @@ -403,7 +403,7 @@ func runLauncher(ctx context.Context, cancel func(), multiSlogger, systemMultiSl // agentFlagConsumer handles agent flags pushed from the control server controlService.RegisterConsumer(agentFlagsSubsystemName, keyvalueconsumer.New(flagController)) // katcConfigConsumer handles updates to Kolide's custom ATC tables - controlService.RegisterConsumer(katcSubsystemName, keyvalueconsumer.New(k.KatcConfigStore())) + controlService.RegisterConsumer(katcSubsystemName, keyvalueconsumer.NewConfigConsumer(k.KatcConfigStore())) controlService.RegisterSubscriber(katcSubsystemName, osqueryRunner) controlService.RegisterSubscriber(katcSubsystemName, startupSettingsWriter) diff --git a/ee/control/consumers/keyvalueconsumer/config_consumer.go b/ee/control/consumers/keyvalueconsumer/config_consumer.go new file mode 100644 index 000000000..21733ad7f --- /dev/null +++ b/ee/control/consumers/keyvalueconsumer/config_consumer.go @@ -0,0 +1,47 @@ +package keyvalueconsumer + +import ( + "encoding/json" + "errors" + "fmt" + "io" + + "github.com/kolide/launcher/ee/agent/types" +) + +type ConfigConsumer struct { + updater types.Updater +} + +func NewConfigConsumer(updater types.Updater) *ConfigConsumer { + c := &ConfigConsumer{ + updater: updater, + } + + return c +} + +func (c *ConfigConsumer) Update(data io.Reader) error { + if c == nil { + return errors.New("key value consumer is nil") + } + + var kvPairs map[string]any + if err := json.NewDecoder(data).Decode(&kvPairs); err != nil { + return fmt.Errorf("failed to decode key-value json: %w", err) + } + + kvStringPairs := make(map[string]string) + for k, v := range kvPairs { + b, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("unable to marshal value for `%s`: %w", k, err) + } + kvStringPairs[k] = string(b) + } + + // Turn the map into a slice of key, value, ... and send it to the thing storing this data + _, err := c.updater.Update(kvStringPairs) + + return err +} From 4b9b56b958e13f7d76569cc1b93297d797deeb12 Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Mon, 15 Jul 2024 11:23:42 -0400 Subject: [PATCH 14/17] Update comments for clarity --- ee/katc/indexeddb_leveldb.go | 4 +++- ee/katc/sqlite.go | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ee/katc/indexeddb_leveldb.go b/ee/katc/indexeddb_leveldb.go index c04b7d902..9c9722b42 100644 --- a/ee/katc/indexeddb_leveldb.go +++ b/ee/katc/indexeddb_leveldb.go @@ -35,7 +35,9 @@ func indexeddbLeveldbData(ctx context.Context, slogger *slog.Logger, sourcePaths // Query databases for _, db := range leveldbs { - // Check to make sure `db` adheres to pathConstraintsFromQuery + // Check to make sure `db` adheres to pathConstraintsFromQuery. This is an + // optimization to avoid work -- no point in querying a leveldb file that a filter + // will exclude anyway. valid, err := checkPathConstraints(db, pathConstraintsFromQuery) if err != nil { return nil, fmt.Errorf("checking source path constraints: %w", err) diff --git a/ee/katc/sqlite.go b/ee/katc/sqlite.go index a2afdf17d..c1c14c32e 100644 --- a/ee/katc/sqlite.go +++ b/ee/katc/sqlite.go @@ -26,7 +26,8 @@ func sqliteData(ctx context.Context, slogger *slog.Logger, sourcePaths []string, } for _, sqliteDb := range sqliteDbs { - // Check to make sure `sqliteDb` adheres to pathConstraintsFromQuery + // Check to make sure `db` adheres to pathConstraintsFromQuery. This is an + // optimization to avoid work, if sqlite filtering is going to exclude it. valid, err := checkPathConstraints(sqliteDb, pathConstraintsFromQuery) if err != nil { return nil, fmt.Errorf("checking source path constraints: %w", err) From 428a872d82b30d655c659ad8eb1801158f8d0fa9 Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Mon, 15 Jul 2024 11:24:05 -0400 Subject: [PATCH 15/17] Remove source paths from logs to avoid overly verbose logs --- ee/katc/table.go | 1 - 1 file changed, 1 deletion(-) diff --git a/ee/katc/table.go b/ee/katc/table.go index b30ff49c1..defa2e8c7 100644 --- a/ee/katc/table.go +++ b/ee/katc/table.go @@ -79,7 +79,6 @@ func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) ( k.slogger = slogger.With( "table_name", tableName, "table_type", cfg.SourceType, - "table_source_paths", cfg.SourcePaths, ) return &k, columns From 8c469279e4ca8dddb092801c5f8955d8fb961ec1 Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Mon, 15 Jul 2024 11:24:59 -0400 Subject: [PATCH 16/17] Fix comments --- ee/katc/indexeddb_leveldb.go | 3 +-- ee/katc/sqlite.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ee/katc/indexeddb_leveldb.go b/ee/katc/indexeddb_leveldb.go index 9c9722b42..41ab7b92b 100644 --- a/ee/katc/indexeddb_leveldb.go +++ b/ee/katc/indexeddb_leveldb.go @@ -36,8 +36,7 @@ func indexeddbLeveldbData(ctx context.Context, slogger *slog.Logger, sourcePaths // Query databases for _, db := range leveldbs { // Check to make sure `db` adheres to pathConstraintsFromQuery. This is an - // optimization to avoid work -- no point in querying a leveldb file that a filter - // will exclude anyway. + // optimization to avoid work, if osquery sqlite filtering is going to exclude it. valid, err := checkPathConstraints(db, pathConstraintsFromQuery) if err != nil { return nil, fmt.Errorf("checking source path constraints: %w", err) diff --git a/ee/katc/sqlite.go b/ee/katc/sqlite.go index c1c14c32e..fe8b3407c 100644 --- a/ee/katc/sqlite.go +++ b/ee/katc/sqlite.go @@ -27,7 +27,7 @@ func sqliteData(ctx context.Context, slogger *slog.Logger, sourcePaths []string, for _, sqliteDb := range sqliteDbs { // Check to make sure `db` adheres to pathConstraintsFromQuery. This is an - // optimization to avoid work, if sqlite filtering is going to exclude it. + // optimization to avoid work, if osquery sqlite filtering is going to exclude it. valid, err := checkPathConstraints(sqliteDb, pathConstraintsFromQuery) if err != nil { return nil, fmt.Errorf("checking source path constraints: %w", err) From 606ade670f5c6509258ad2d0cd280a6e116b4f4c Mon Sep 17 00:00:00 2001 From: RebeccaMahany Date: Mon, 15 Jul 2024 11:37:29 -0400 Subject: [PATCH 17/17] Combine table definition structs --- ee/katc/config.go | 15 ++++++++------- ee/katc/table.go | 26 ++++++++++++++++++++------ ee/katc/table_test.go | 35 ++++++++++++++++++++--------------- 3 files changed, 48 insertions(+), 28 deletions(-) diff --git a/ee/katc/config.go b/ee/katc/config.go index 3c59f4b38..fa45770d4 100644 --- a/ee/katc/config.go +++ b/ee/katc/config.go @@ -102,16 +102,17 @@ type ( // katcTableConfig is the configuration for a specific KATC table. The control server // sends down these configurations. katcTableConfig struct { - Columns []string `json:"columns"` - SourceType katcSourceType `json:"source_type"` - SourcePaths []string `json:"source_paths"` // Describes how to connect to source (e.g. path to db) -- % and _ wildcards supported - SourceQuery string `json:"source_query"` // Query to run against each source path - RowTransformSteps []rowTransformStep `json:"row_transform_steps"` - Overlays []katcTableConfigOverlay `json:"overlays"` + Columns []string `json:"columns"` + katcTableDefinition + Overlays []katcTableConfigOverlay `json:"overlays"` } katcTableConfigOverlay struct { - Filters map[string]string `json:"filters"` // determines if this overlay is applicable to this launcher installation + Filters map[string]string `json:"filters"` // determines if this overlay is applicable to this launcher installation + katcTableDefinition + } + + katcTableDefinition struct { SourceType *katcSourceType `json:"source_type,omitempty"` SourcePaths *[]string `json:"source_paths,omitempty"` // Describes how to connect to source (e.g. path to db) -- % and _ wildcards supported SourceQuery *string `json:"source_query,omitempty"` // Query to run against each source path diff --git a/ee/katc/table.go b/ee/katc/table.go index defa2e8c7..089fddb9b 100644 --- a/ee/katc/table.go +++ b/ee/katc/table.go @@ -2,6 +2,7 @@ package katc import ( "context" + "errors" "fmt" "log/slog" "regexp" @@ -44,12 +45,21 @@ func newKatcTable(tableName string, cfg katcTableConfig, slogger *slog.Logger) ( } k := katcTable{ - sourceType: cfg.SourceType, - sourcePaths: cfg.SourcePaths, - sourceQuery: cfg.SourceQuery, - rowTransformSteps: cfg.RowTransformSteps, - columnLookup: columnLookup, - slogger: slogger, + columnLookup: columnLookup, + slogger: slogger, + } + + if cfg.SourceType != nil { + k.sourceType = *cfg.SourceType + } + if cfg.SourcePaths != nil { + k.sourcePaths = *cfg.SourcePaths + } + if cfg.SourceQuery != nil { + k.sourceQuery = *cfg.SourceQuery + } + if cfg.RowTransformSteps != nil { + k.rowTransformSteps = *cfg.RowTransformSteps } // Check overlays to see if any of the filters apply to us; @@ -94,6 +104,10 @@ func filtersMatch(filters map[string]string) bool { // generate handles queries against a KATC table. func (k *katcTable) generate(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { + if k.sourceType.dataFunc == nil { + return nil, errors.New("table source type not set") + } + // Fetch data from our table source dataRaw, err := k.sourceType.dataFunc(ctx, k.slogger, k.sourcePaths, k.sourceQuery, queryContext) if err != nil { diff --git a/ee/katc/table_test.go b/ee/katc/table_test.go index ad3d369a8..8cba6ce69 100644 --- a/ee/katc/table_test.go +++ b/ee/katc/table_test.go @@ -81,22 +81,25 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) { // At long last, our source is adequately configured. // Move on to constructing our KATC table. + sourceQuery := "SELECT data FROM object_data;" cfg := katcTableConfig{ - SourceType: katcSourceType{ - name: sqliteSourceType, - dataFunc: sqliteData, - }, - Columns: []string{expectedColumn}, - SourcePaths: []string{filepath.Join("some", "incorrect", "path")}, - SourceQuery: "SELECT data FROM object_data;", - RowTransformSteps: []rowTransformStep{ - { - name: snappyDecodeTransformStep, - transformFunc: snappyDecode, + Columns: []string{expectedColumn}, + katcTableDefinition: katcTableDefinition{ + SourceType: &katcSourceType{ + name: sqliteSourceType, + dataFunc: sqliteData, }, - { - name: deserializeFirefoxTransformStep, - transformFunc: deserializeFirefox, + SourcePaths: &[]string{filepath.Join("some", "incorrect", "path")}, + SourceQuery: &sourceQuery, + RowTransformSteps: &[]rowTransformStep{ + { + name: snappyDecodeTransformStep, + transformFunc: snappyDecode, + }, + { + name: deserializeFirefoxTransformStep, + transformFunc: deserializeFirefox, + }, }, }, Overlays: []katcTableConfigOverlay{ @@ -104,7 +107,9 @@ func Test_generate_SqliteBackedIndexedDB(t *testing.T) { Filters: map[string]string{ "goos": runtime.GOOS, }, - SourcePaths: &[]string{filepath.Join(databaseDir, "%.sqlite")}, // All sqlite files in the test directory + katcTableDefinition: katcTableDefinition{ + SourcePaths: &[]string{filepath.Join(databaseDir, "%.sqlite")}, // All sqlite files in the test directory + }, }, }, }