diff --git a/ee/katc/deserialize_firefox_test.go b/ee/katc/deserialize_firefox_test.go new file mode 100644 index 000000000..50b5d2799 --- /dev/null +++ b/ee/katc/deserialize_firefox_test.go @@ -0,0 +1,233 @@ +package katc + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "io" + "testing" + + "github.com/google/uuid" + "github.com/kolide/launcher/pkg/log/multislogger" + "github.com/stretchr/testify/require" +) + +func Test_deserializeFirefox(t *testing.T) { + t.Parallel() + + // Build expected object + u, err := uuid.NewRandom() + require.NoError(t, err, "generating test UUID") + idValue := u.String() + arrWithNestedObj := []string{"{\"id\":\"3\"}"} + nestedArrBytes, err := json.Marshal(arrWithNestedObj) + require.NoError(t, err) + expectedObj := map[string][]byte{ + "id": []byte(idValue), // will exercise deserializeString + "version": []byte("1"), // will exercise int deserialization + "option": nil, // will exercise null/undefined deserialization + "types": nestedArrBytes, // will exercise deserializeArray, deserializeNestedObject + } + + // Build a serialized object to deserialize + serializedObj := []byte{ + // Header + 0x00, 0x00, 0x00, 0x00, // header tag data -- discarded + 0x00, 0x00, 0xf1, 0xff, // LE `tagHeader` + // Begin object + 0x00, 0x00, 0x00, 0x00, // object tag data -- discarded + 0x08, 0x00, 0xff, 0xff, // LE `tagObject` + // Begin `id` key + 0x02, 0x00, 0x00, 0x80, // LE data about upcoming string: length 2 (remaining bytes), is ASCII + 0x04, 0x00, 0xff, 0xff, // LE `tagString` + 0x69, 0x64, // "id" + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // padding to get to 8-byte word boundary + // End `id` key + // Begin `id` value + 0x24, 0x00, 0x00, 0x80, // LE data about upcoming string: length 36 (remaining bytes), is ASCII + 0x04, 0x00, 0xff, 0xff, // LE `tagString` + } + // Append `id` + serializedObj = append(serializedObj, []byte(idValue)...) + // Append `id` padding, add `version` + serializedObj = append(serializedObj, + 0x00, 0x00, 0x00, 0x00, // padding to get to 8-byte word boundary for `id` string + // End `id` value + // Begin `version` key + 0x07, 0x00, 0x00, 0x80, // LE data about upcoming string: length 7 (remaining bytes), is ASCII + 0x04, 0x00, 0xff, 0xff, // LE `tagString` + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, // "version" + 0x00, // padding to get to 8-byte word boundary + // End `version` key + // Begin `version` value + 0x01, 0x00, 0x00, 0x00, // Value `1` + 0x03, 0x00, 0xff, 0xff, // LE `tagInt32` + // End `version` value + // Begin `option` key + 0x06, 0x00, 0x00, 0x80, // LE data about upcoming string: length 6 (remaining bytes), is ASCII + 0x04, 0x00, 0xff, 0xff, // LE `tagString` + 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, // "option" + 0x00, 0x00, // padding to get to 8-byte word boundary + // End `option` key + // Begin `option` value + 0x00, 0x00, 0x00, 0x00, // Unused data, discarded + 0x00, 0x00, 0xff, 0xff, // LE `tagNull` + // End `option` value + // Begin `types` key + 0x05, 0x00, 0x00, 0x80, // LE data about upcoming string: length 5 (remaining bytes), is ASCII + 0x04, 0x00, 0xff, 0xff, // LE `tagString` + 0x74, 0x79, 0x70, 0x65, 0x73, // "types" + 0x00, 0x00, 0x00, // padding to get to 8-byte word boundary + // End `types` key + // Begin `types` value + 0x01, 0x00, 0x00, 0x00, // Array length (1) + 0x07, 0x00, 0xff, 0xff, // LE `tagArrayObject` + // Begin first array item + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // An extra pair that gets discarded, I don't know why + 0x00, 0x00, 0x00, 0x00, // Tag data, discarded + 0x08, 0x00, 0xff, 0xff, // LE `tagObjectObject` + // Begin nested object + // Begin `id` key + 0x02, 0x00, 0x00, 0x80, // LE data about upcoming string: length 2 (remaining bytes), is ASCII + 0x04, 0x00, 0xff, 0xff, // LE `tagString` + 0x69, 0x64, // "id" + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // padding to get to 8-byte word boundary + // End `id` key + // Begin `id` value + 0x03, 0x00, 0x00, 0x00, // Value `3` + 0x03, 0x00, 0xff, 0xff, // LE `tagInt32` + // End `id` value + // Object footer + 0x00, 0x00, 0x00, 0x00, // tag data -- discarded + 0x13, 0x00, 0xff, 0xff, // LE `tagEndOfKeys` 0xffff0013 + // End nested object + // End first array item + // End `types` value + // Object footer + 0x00, 0x00, 0x00, 0x00, // tag data -- discarded + 0x13, 0x00, 0xff, 0xff, // LE `tagEndOfKeys` + ) + + results, err := deserializeFirefox(context.TODO(), multislogger.NewNopLogger(), map[string][]byte{ + "data": serializedObj, + }) + require.NoError(t, err, "expected to be able to deserialize object") + + require.Equal(t, expectedObj, results) +} + +func Test_deserializeFirefox_missingTopLevelDataKey(t *testing.T) { + t.Parallel() + + _, err := deserializeFirefox(context.TODO(), multislogger.NewNopLogger(), map[string][]byte{ + "not_a_data_key": nil, + }) + require.Error(t, err, "expect deserializeFirefox requires top-level data key") +} + +func Test_deserializeFirefox_malformedData(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + testCaseName string + data []byte + }{ + { + testCaseName: "missing header", + data: []byte{ + 0x00, 0x00, 0x00, 0x00, // header tag data -- discarded + 0x00, 0x00, 0xff, 0xff, // LE `tagNull` (`tagHeader` expected instead) + }, + }, + { + testCaseName: "missing top-level object", + data: []byte{ + // Header + 0x00, 0x00, 0x00, 0x00, // header tag data -- discarded + 0x00, 0x00, 0xf1, 0xff, // LE `tagHeader` + // End header + 0x00, 0x00, 0x00, 0x00, // data about tag, not used + 0x04, 0x00, 0xff, 0xff, // LE `tagString` (`tagObject` expected instead) + }, + }, + } { + tt := tt + t.Run(tt.testCaseName, func(t *testing.T) { + t.Parallel() + + _, err := deserializeFirefox(context.TODO(), multislogger.NewNopLogger(), map[string][]byte{ + "data": tt.data, + }) + require.Error(t, err, "expect deserializeFirefox rejects malformed data") + }) + } +} + +// Test_deserializeString tests that deserializeString can handle both ASCII and UTF-16 strings +func Test_deserializeString(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + testCaseName string + expected []byte + stringData []byte + stringBytes []byte + }{ + { + testCaseName: "ascii", + expected: []byte("createdAt"), + stringData: []byte{ + 0x09, 0x00, 0x00, 0x80, // LE data about upcoming string: length 9 (remaining bytes), is ASCII (true) + }, + stringBytes: []byte{ + 0x63, // c + 0x72, // r + 0x65, // e + 0x61, // a + 0x74, // t + 0x65, // e + 0x64, // d + 0x41, // A + 0x74, // t + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // padding to get to 8-byte word boundary + }, + }, + { + testCaseName: "utf-16", + expected: []byte("🏆"), + stringData: []byte{ + 0x02, 0x00, 0x00, 0x00, // LE data about upcoming string: length 2 (remaining bytes), is ASCII (false) + }, + stringBytes: []byte{ + 0x3c, 0xd8, 0xc6, 0xdf, // emoji: UTF-16 LE + 0x00, 0x00, 0x00, 0x00, // padding to get to 8-byte word boundary + }, + }, + } { + tt := tt + t.Run(tt.testCaseName, func(t *testing.T) { + t.Parallel() + + stringDataInt := binary.LittleEndian.Uint32(tt.stringData) + stringReader := bytes.NewReader(tt.stringBytes) + + resultBytes, err := deserializeString(stringDataInt, stringReader) + require.NoError(t, err) + + require.Equal(t, tt.expected, resultBytes) + + // Confirm we read all the padding in as well + _, err = stringReader.ReadByte() + require.Error(t, err) + require.ErrorIs(t, err, io.EOF) + }) + } +} + +func Test_bitMask(t *testing.T) { + t.Parallel() + + var expected uint32 = 0b01111111111111111111111111111111 + require.Equal(t, expected, bitMask(31)) +} diff --git a/ee/katc/snappy_test.go b/ee/katc/snappy_test.go new file mode 100644 index 000000000..d428795c8 --- /dev/null +++ b/ee/katc/snappy_test.go @@ -0,0 +1,30 @@ +package katc + +import ( + "context" + "testing" + + "github.com/golang/snappy" + "github.com/kolide/launcher/pkg/log/multislogger" + "github.com/stretchr/testify/require" +) + +func Test_snappyDecode(t *testing.T) { + t.Parallel() + + expectedRow := map[string][]byte{ + "some_key_a": []byte("some_value_a"), + "some_key_b": []byte("some_value_b"), + } + + encodedRow := map[string][]byte{ + "some_key_a": snappy.Encode(nil, expectedRow["some_key_a"]), + "some_key_b": snappy.Encode(nil, expectedRow["some_key_b"]), + } + + results, err := snappyDecode(context.TODO(), multislogger.NewNopLogger(), encodedRow) + require.NoError(t, err) + + // Validate that the keys are unchanged, and that the data was correctly decoded + require.Equal(t, expectedRow, results) +} diff --git a/ee/katc/sqlite_test.go b/ee/katc/sqlite_test.go new file mode 100644 index 000000000..7f2a8dd0a --- /dev/null +++ b/ee/katc/sqlite_test.go @@ -0,0 +1,150 @@ +package katc + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "testing" + + "github.com/kolide/launcher/pkg/log/multislogger" + "github.com/stretchr/testify/require" +) + +func Test_sqliteData(t *testing.T) { + t.Parallel() + + // Set up two sqlite databases with data in them + sqliteDir := t.TempDir() + dbFilepaths := []string{ + filepath.Join(sqliteDir, "a.sqlite"), + filepath.Join(sqliteDir, "b.sqlite"), + } + uuids := []string{ + "28f3ebd7-0945-4c54-96af-413a0a0d2dd0", + "134def6f-b7e2-4ee1-9461-46e54f627835", + } + values := []string{ + "value one", + "value two", + } + for i, p := range dbFilepaths { + // Create file + f, err := os.Create(p) + require.NoError(t, err) + require.NoError(t, f.Close()) + + // Open connection + conn, err := sql.Open("sqlite", p) + require.NoError(t, err) + + // Create table + _, err = conn.Exec(` + CREATE TABLE IF NOT EXISTS test_data ( + uuid TEXT NOT NULL PRIMARY KEY, + value TEXT, + ignored_column TEXT + ) WITHOUT ROWID; + `) + require.NoError(t, err) + + // Add data to table + _, err = conn.Exec(` + INSERT INTO test_data (uuid, value, ignored_column) + VALUES (?, ?, "ignored value"); + `, uuids[i], values[i]) + require.NoError(t, err) + + conn.Close() + } + + // Query data + results, err := sqliteData(context.TODO(), multislogger.NewNopLogger(), filepath.Join(sqliteDir, "*.sqlite"), "SELECT uuid, value FROM test_data;", nil) + require.NoError(t, err) + + // Confirm we have the correct number of `sourceData` returned (one per db) + require.Equal(t, 2, len(results)) + + // Validate data in each result + for i, sourceResp := range results { + // We don't really care about the ordering of results here, but this ensures we can + // confirm that the data is associated with the correct source + require.Equal(t, dbFilepaths[i], sourceResp.path) + + // Only one row per source + require.Equal(t, 1, len(sourceResp.rows)) + + // Validate keys and their values + require.Contains(t, sourceResp.rows[0], "uuid") + require.Equal(t, uuids[i], string(sourceResp.rows[0]["uuid"])) + require.Contains(t, sourceResp.rows[0], "value") + require.Equal(t, values[i], string(sourceResp.rows[0]["value"])) + + // Confirm we didn't pull the other column + require.NotContains(t, sourceResp.rows[0], "ignored_column") + } +} + +func Test_sqliteData_noSourcesFound(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + results, err := sqliteData(context.TODO(), multislogger.NewNopLogger(), filepath.Join(tmpDir, "db.sqlite"), "SELECT * FROM data;", nil) + require.NoError(t, err) + require.Equal(t, 0, len(results)) +} + +func TestSourcePatternToGlobbablePattern(t *testing.T) { + t.Parallel() + + // Make actual test directories + files so that we can run filepath.Glob + rootDir := t.TempDir() + testDir := filepath.Join(rootDir, "path", "to", "a", "directory") + require.NoError(t, os.MkdirAll(testDir, 0755)) + dbFile := filepath.Join(testDir, "db.sqlite") + f, err := os.Create(dbFile) + require.NoError(t, err) + require.NoError(t, f.Close()) + + for _, tt := range []struct { + testCaseName string + sourcePattern string + expectedPattern string + }{ + { + testCaseName: "no wildcards", + sourcePattern: filepath.Join(rootDir, "path", "to", "a", "directory", "db.sqlite"), + expectedPattern: filepath.Join(rootDir, "path", "to", "a", "directory", "db.sqlite"), + }, + { + testCaseName: "% wildcard", + sourcePattern: filepath.Join(rootDir, "path", "to", "%", "directory", "db.sqlite"), + expectedPattern: filepath.Join(rootDir, "path", "to", "*", "directory", "db.sqlite"), + }, + { + testCaseName: "underscore wildcard", + sourcePattern: filepath.Join(rootDir, "path", "to", "_", "directory", "db.sqlite"), + expectedPattern: filepath.Join(rootDir, "path", "to", "?", "directory", "db.sqlite"), + }, + { + testCaseName: "multiple wildcards", + sourcePattern: filepath.Join(rootDir, "path", "to", "_", "directory", "%.sqlite"), + expectedPattern: filepath.Join(rootDir, "path", "to", "?", "directory", "*.sqlite"), + }, + } { + tt := tt + t.Run(tt.testCaseName, func(t *testing.T) { + t.Parallel() + + // Confirm pattern is as expected + pattern := sourcePatternToGlobbablePattern(tt.sourcePattern) + require.Equal(t, tt.expectedPattern, pattern) + + // Confirm pattern is globbable + matches, err := filepath.Glob(pattern) + require.NoError(t, err) + require.Equal(t, 1, len(matches)) + require.Equal(t, dbFile, matches[0]) + }) + } +}