diff --git a/backend/pkg/database/database.go b/backend/pkg/database/database.go index 2e8c51dbb..da506243d 100644 --- a/backend/pkg/database/database.go +++ b/backend/pkg/database/database.go @@ -38,6 +38,7 @@ func RegisterDrivers() { DBDrivers = map[string]DBDriver{} } DBDrivers[types.DBDriverTypeLocal] = gorm.NewDatabase + DBDrivers[types.DBDriverTypePostgres] = gorm.NewDatabase }) } diff --git a/backend/pkg/database/gorm/database.go b/backend/pkg/database/gorm/database.go index 897109579..f9a37d484 100644 --- a/backend/pkg/database/gorm/database.go +++ b/backend/pkg/database/gorm/database.go @@ -20,10 +20,12 @@ import ( "time" uuid "github.com/satori/go.uuid" + "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" + "github.com/openclarity/vmclarity/backend/pkg/database/odatasql/jsonsql" "github.com/openclarity/vmclarity/backend/pkg/database/types" ) @@ -83,27 +85,27 @@ func initDataBase(config types.DBConfig) (*gorm.DB, error) { // First for all objects index the ID field this speeds up anywhere // we're getting a single object out of the DB, including in PATCH/PUT // etc. - idb := db.Exec("CREATE INDEX IF NOT EXISTS targets_id_idx ON targets(Data -> 'id')") + idb := db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS targets_id_idx ON targets((%s))", SQLVariant.JSONExtract("Data", "$.id"))) if idb.Error != nil { return nil, fmt.Errorf("failed to create index targets_id_idx: %w", idb.Error) } - idb = db.Exec("CREATE INDEX IF NOT EXISTS scan_results_id_idx ON scan_results(Data -> 'id')") + idb = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS scan_results_id_idx ON scan_results((%s))", SQLVariant.JSONExtract("Data", "$.id"))) if idb.Error != nil { return nil, fmt.Errorf("failed to create index scan_results_id_idx: %w", idb.Error) } - idb = db.Exec("CREATE INDEX IF NOT EXISTS scan_configs_id_idx ON scan_configs(Data -> 'id')") + idb = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS scan_configs_id_idx ON scan_configs((%s))", SQLVariant.JSONExtract("Data", "$.id"))) if idb.Error != nil { return nil, fmt.Errorf("failed to create index scan_configs_id_idx: %w", idb.Error) } - idb = db.Exec("CREATE INDEX IF NOT EXISTS scans_id_idx ON scans(Data -> 'id')") + idb = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS scans_id_idx ON scans((%s))", SQLVariant.JSONExtract("Data", "$.id"))) if idb.Error != nil { return nil, fmt.Errorf("failed to create index scans_id_idx: %w", idb.Error) } - idb = db.Exec("CREATE INDEX IF NOT EXISTS findings_id_idx ON findings(Data -> 'id')") + idb = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS findings_id_idx ON findings((%s))", SQLVariant.JSONExtract("Data", "$.id"))) if idb.Error != nil { return nil, fmt.Errorf("failed to create index findings_id_idx: %w", idb.Error) } @@ -111,7 +113,7 @@ func initDataBase(config types.DBConfig) (*gorm.DB, error) { // For processing scan results to findings we need to find all the scan // results by general status and findingsProcessed, so add an index for // that. - idb = db.Exec("CREATE INDEX IF NOT EXISTS scan_results_findings_processed_idx ON scan_results(Data -> 'findingsProcessed', Data -> 'status.general.state')") + idb = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS scan_results_findings_processed_idx ON scan_results((%s), (%s))", SQLVariant.JSONExtract("Data", "$.findingsProcessed"), SQLVariant.JSONExtract("Data", "$.status.general.state"))) if idb.Error != nil { return nil, fmt.Errorf("failed to create index scan_results_findings_processed_idx: %w", idb.Error) } @@ -119,7 +121,7 @@ func initDataBase(config types.DBConfig) (*gorm.DB, error) { // The UI needs to find all the findings for a specific finding type // and the scan result processor needs to filter that list by a // specific asset. So add a combined index for those cases. - idb = db.Exec("CREATE INDEX IF NOT EXISTS findings_by_type_and_asset_idx ON findings(Data -> 'findingInfo.objectType', Data -> 'asset.id')") + idb = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS findings_by_type_and_asset_idx ON findings((%s), (%s))", SQLVariant.JSONExtract("Data", "$.findingInfo.objectType"), SQLVariant.JSONExtract("Data", "$.asset.id"))) if idb.Error != nil { return nil, fmt.Errorf("failed to create index findings_by_type_and_asset_idx: %w", idb.Error) } @@ -133,7 +135,11 @@ func initDataBase(config types.DBConfig) (*gorm.DB, error) { func initDB(config types.DBConfig, dbDriver string, dbLogger logger.Interface) (*gorm.DB, error) { switch dbDriver { case types.DBDriverTypeLocal: + SQLVariant = jsonsql.SQLite return initSqlite(config, dbLogger) + case types.DBDriverTypePostgres: + SQLVariant = jsonsql.Postgres + return initPostgres(config, dbLogger) default: return nil, fmt.Errorf("driver type %s is not supported by GORM driver", dbDriver) } @@ -148,3 +154,17 @@ func initSqlite(config types.DBConfig, dbLogger logger.Interface) (*gorm.DB, err } return db, nil } + +func initPostgres(config types.DBConfig, dbLogger logger.Interface) (*gorm.DB, error) { + dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%s sslmode=disable TimeZone=UTC", + config.DBHost, config.DBUser, config.DBPassword, config.DBName, config.DBPort) + + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{ + Logger: dbLogger, + }) + if err != nil { + return nil, fmt.Errorf("failed to open %s db: %v", config.DBName, err) + } + + return db, nil +} diff --git a/backend/pkg/database/gorm/odata.go b/backend/pkg/database/gorm/odata.go index 41558b23e..a26994fba 100644 --- a/backend/pkg/database/gorm/odata.go +++ b/backend/pkg/database/gorm/odata.go @@ -23,9 +23,12 @@ import ( "gorm.io/gorm" "github.com/openclarity/vmclarity/backend/pkg/database/odatasql" + "github.com/openclarity/vmclarity/backend/pkg/database/odatasql/jsonsql" "github.com/openclarity/vmclarity/runtime_scan/pkg/utils" ) +var SQLVariant jsonsql.Variant + type ODataObject struct { ID uint `gorm:"primarykey"` Data datatypes.JSON @@ -827,7 +830,7 @@ func ODataQuery(db *gorm.DB, schema string, filterString, selectString, expandSt // Build the raw SQL query using the odatasql library, this will also // parse and validate the ODATA query params. - query, err := odatasql.BuildSQLQuery(schemaMetas, schema, filterString, selectString, expandString, orderby, top, skip) + query, err := odatasql.BuildSQLQuery(SQLVariant, schemaMetas, schema, filterString, selectString, expandString, orderby, top, skip) if err != nil { return fmt.Errorf("failed to build query for DB: %w", err) } @@ -849,7 +852,7 @@ func ODataQuery(db *gorm.DB, schema string, filterString, selectString, expandSt } func ODataCount(db *gorm.DB, schema string, filterString *string) (int, error) { - query, err := odatasql.BuildCountQuery(schemaMetas, schema, filterString) + query, err := odatasql.BuildCountQuery(SQLVariant, schemaMetas, schema, filterString) if err != nil { return 0, fmt.Errorf("failed to build query to count objects: %w", err) } diff --git a/backend/pkg/database/odatasql/jsonsql/postgres.go b/backend/pkg/database/odatasql/jsonsql/postgres.go new file mode 100644 index 000000000..dbd7535ad --- /dev/null +++ b/backend/pkg/database/odatasql/jsonsql/postgres.go @@ -0,0 +1,67 @@ +// Copyright © 2023 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jsonsql + +import ( + "fmt" + "strings" +) + +type postgres struct{} + +var Postgres Variant = postgres{} + +func (postgres) JSONObject(parts []string) string { + return fmt.Sprintf("JSONB_BUILD_OBJECT(%s)", strings.Join(parts, ", ")) +} + +func (postgres) JSONArrayAggregate(value string) string { + return fmt.Sprintf("JSONB_AGG(%s)", value) +} + +func (postgres) CastToDateTime(strTime string) string { + return fmt.Sprintf("(%s)::timestamptz", strTime) +} + +func (postgres) JSONEach(source string) string { + return fmt.Sprintf("JSONB_ARRAY_ELEMENTS((%s))", source) +} + +func (postgres) JSONArray(items []string) string { + return fmt.Sprintf("JSONB_BUILD_ARRAY(%s)", strings.Join(items, ", ")) +} + +func convertJSONPathToPostgresPath(jsonPath string) string { + parts := strings.Split(jsonPath, ".") + newParts := []string{} + for _, part := range parts { + if part == "$" { + continue + } + newParts = append(newParts, part) + } + return fmt.Sprintf("{%s}", strings.Join(newParts, ",")) +} + +func (postgres) JSONExtract(source, path string) string { + path = convertJSONPathToPostgresPath(path) + return fmt.Sprintf("%s#>'%s'", source, path) +} + +func (postgres) JSONExtractText(source, path string) string { + path = convertJSONPathToPostgresPath(path) + return fmt.Sprintf("%s#>>'%s'", source, path) +} diff --git a/backend/pkg/database/odatasql/jsonsql/sqlite.go b/backend/pkg/database/odatasql/jsonsql/sqlite.go new file mode 100644 index 000000000..254852502 --- /dev/null +++ b/backend/pkg/database/odatasql/jsonsql/sqlite.go @@ -0,0 +1,53 @@ +// Copyright © 2023 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jsonsql + +import ( + "fmt" + "strings" +) + +type sqlite struct{} + +var SQLite Variant = sqlite{} + +func (sqlite) JSONObject(parts []string) string { + return fmt.Sprintf("JSON_OBJECT(%s)", strings.Join(parts, ", ")) +} + +func (sqlite) JSONArrayAggregate(value string) string { + return fmt.Sprintf("JSON_GROUP_ARRAY(%s)", value) +} + +func (sqlite) CastToDateTime(strTime string) string { + return fmt.Sprintf("datetime(%s)", strTime) +} + +func (sqlite) JSONEach(source string) string { + return fmt.Sprintf("JSON_EACH(%s)", source) +} + +func (sqlite) JSONArray(items []string) string { + return fmt.Sprintf("JSON_ARRAY(%s)", strings.Join(items, ", ")) +} + +func (sqlite) JSONExtract(source, path string) string { + return fmt.Sprintf("%s->'%s'", source, path) +} + +func (sqlite) JSONExtractText(source, path string) string { + return fmt.Sprintf("%s->>'%s'", source, path) +} diff --git a/backend/pkg/database/odatasql/jsonsql/variant.go b/backend/pkg/database/odatasql/jsonsql/variant.go new file mode 100644 index 000000000..e9688f2a7 --- /dev/null +++ b/backend/pkg/database/odatasql/jsonsql/variant.go @@ -0,0 +1,26 @@ +// Copyright © 2023 Cisco Systems, Inc. and its affiliates. +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jsonsql + +type Variant interface { + JSONObject(parts []string) string + JSONArrayAggregate(string) string + CastToDateTime(string) string + JSONEach(source string) string + JSONArray(items []string) string + JSONExtract(source string, path string) string + JSONExtractText(source string, path string) string +} diff --git a/backend/pkg/database/odatasql/query.go b/backend/pkg/database/odatasql/query.go index efd2b3ddc..ef3ab03f1 100644 --- a/backend/pkg/database/odatasql/query.go +++ b/backend/pkg/database/odatasql/query.go @@ -23,12 +23,16 @@ import ( "github.com/CiscoM31/godata" log "github.com/sirupsen/logrus" + + "github.com/openclarity/vmclarity/backend/pkg/database/odatasql/jsonsql" ) +type jsonExtractFunctionType func(string, string) string + var fixSelectToken sync.Once // nolint:cyclop -func BuildCountQuery(schemaMetas map[string]SchemaMeta, schema string, filterString *string) (string, error) { +func BuildCountQuery(sqlVariant jsonsql.Variant, schemaMetas map[string]SchemaMeta, schema string, filterString *string) (string, error) { table := schemaMetas[schema].Table if table == "" { return "", fmt.Errorf("trying to query complex type schema %s with no source table", schema) @@ -48,7 +52,7 @@ func BuildCountQuery(schemaMetas map[string]SchemaMeta, schema string, filterStr } // Build the WHERE conditions based on the $filter tree - conditions, err := buildWhereFromFilter(schemaMetas, rootObject, schema, fmt.Sprintf("%s.Data", table), filterQuery.Tree) + conditions, err := buildWhereFromFilter(sqlVariant, schemaMetas, rootObject, schema, fmt.Sprintf("%s.Data", table), filterQuery.Tree) if err != nil { return "", fmt.Errorf("failed to build DB query from $filter: %w", err) } @@ -60,7 +64,7 @@ func BuildCountQuery(schemaMetas map[string]SchemaMeta, schema string, filterStr } // nolint:cyclop,gocognit -func BuildSQLQuery(schemaMetas map[string]SchemaMeta, schema string, filterString, selectString, expandString, orderbyString *string, top, skip *int) (string, error) { +func BuildSQLQuery(sqlVariant jsonsql.Variant, schemaMetas map[string]SchemaMeta, schema string, filterString, selectString, expandString, orderbyString *string, top, skip *int) (string, error) { // Fix GlobalExpandTokenizer so that it allows for `-` characters in the Literal tokens fixSelectToken.Do(func() { godata.GlobalExpandTokenizer.Add("^[a-zA-Z0-9_\\'\\.:\\$ \\*-]+", godata.ExpandTokenLiteral) @@ -85,7 +89,7 @@ func BuildSQLQuery(schemaMetas map[string]SchemaMeta, schema string, filterStrin } // Build the WHERE conditions based on the $filter tree - conditions, err := buildWhereFromFilter(schemaMetas, rootObject, schema, fmt.Sprintf("%s.Data", table), filterQuery.Tree) + conditions, err := buildWhereFromFilter(sqlVariant, schemaMetas, rootObject, schema, fmt.Sprintf("%s.Data", table), filterQuery.Tree) if err != nil { return "", fmt.Errorf("failed to build DB query from $filter: %w", err) } @@ -100,7 +104,7 @@ func BuildSQLQuery(schemaMetas map[string]SchemaMeta, schema string, filterStrin return "", fmt.Errorf("failed to parse $orderby: %w", err) } - conditions, err := buildOrderByFromOdata(schemaMetas, rootObject, schema, fmt.Sprintf("%s.Data", table), orderbyQuery.OrderByItems) + conditions, err := buildOrderByFromOdata(sqlVariant, schemaMetas, rootObject, schema, fmt.Sprintf("%s.Data", table), orderbyQuery.OrderByItems) if err != nil { return "", fmt.Errorf("failed to build DB query from $orderby: %w", err) } @@ -108,7 +112,7 @@ func BuildSQLQuery(schemaMetas map[string]SchemaMeta, schema string, filterStrin orderby = fmt.Sprintf("ORDER BY %s", conditions) } - selectFields, err := buildSelectFieldsFromSelectAndExpand(schemaMetas, rootObject, schema, fmt.Sprintf("%s.Data", table), selectString, expandString) + selectFields, err := buildSelectFieldsFromSelectAndExpand(sqlVariant, schemaMetas, rootObject, schema, fmt.Sprintf("%s.Data", table), selectString, expandString) if err != nil { return "", fmt.Errorf("failed to construct fields to select: %w", err) } @@ -130,7 +134,7 @@ func BuildSQLQuery(schemaMetas map[string]SchemaMeta, schema string, filterStrin return fmt.Sprintf("SELECT ID, %s AS Data FROM %s %s %s %s", selectFields, table, where, orderby, limitStm), nil } -func buildSelectFieldsFromSelectAndExpand(schemaMetas map[string]SchemaMeta, rootObject FieldMeta, identifier string, source string, selectString, expandString *string) (string, error) { +func buildSelectFieldsFromSelectAndExpand(sqlVariant jsonsql.Variant, schemaMetas map[string]SchemaMeta, rootObject FieldMeta, identifier string, source string, selectString, expandString *string) (string, error) { var selectQuery *godata.GoDataSelectQuery if selectString != nil && *selectString != "" { // NOTE(sambetts): @@ -163,29 +167,29 @@ func buildSelectFieldsFromSelectAndExpand(schemaMetas map[string]SchemaMeta, roo return "", fmt.Errorf("failed to parse select and expand: %w", err) } - return buildSelectFields(schemaMetas, rootObject, identifier, source, "$", selectTree), nil + return buildSelectFields(sqlVariant, schemaMetas, rootObject, identifier, source, "$", selectTree), nil } -func buildSelectFields(schemaMetas map[string]SchemaMeta, field FieldMeta, identifier, source, path string, st *selectNode) string { +func buildSelectFields(sqlVariant jsonsql.Variant, schemaMetas map[string]SchemaMeta, field FieldMeta, identifier, source, path string, st *selectNode) string { switch field.FieldType { case PrimitiveFieldType: // If root of source (path is just $) is primitive just return the source if path == "$" { return source } - return fmt.Sprintf("%s -> '%s'", source, path) + return sqlVariant.JSONExtract(source, path) case CollectionFieldType: if field.CollectionItemMeta.FieldType == RelationshipFieldType { // This is an optimisation to allow us to do a single // aggregate query to the foreign table instead of a // sub query per item in the collection. - return buildSelectFieldsForRelationshipCollectionFieldType(schemaMetas, field, identifier, source, path, st) + return buildSelectFieldsForRelationshipCollectionFieldType(sqlVariant, schemaMetas, field, identifier, source, path, st) } - return buildSelectFieldsForCollectionFieldType(schemaMetas, field, identifier, source, path, st) + return buildSelectFieldsForCollectionFieldType(sqlVariant, schemaMetas, field, identifier, source, path, st) case ComplexFieldType: - return buildSelectFieldsForComplexFieldType(schemaMetas, field, identifier, source, path, st) + return buildSelectFieldsForComplexFieldType(sqlVariant, schemaMetas, field, identifier, source, path, st) case RelationshipFieldType: - return buildSelectFieldsForRelationshipFieldType(schemaMetas, field, identifier, source, path, st) + return buildSelectFieldsForRelationshipFieldType(sqlVariant, schemaMetas, field, identifier, source, path, st) default: log.Errorf("Unsupported field type %v", field.FieldType) // TODO(sambetts) Return an error here @@ -193,19 +197,26 @@ func buildSelectFields(schemaMetas map[string]SchemaMeta, field FieldMeta, ident } } -func buildSelectFieldsForRelationshipCollectionFieldType(schemaMetas map[string]SchemaMeta, field FieldMeta, identifier, source, path string, st *selectNode) string { +func buildSelectFieldsForRelationshipCollectionFieldType(sqlVariant jsonsql.Variant, schemaMetas map[string]SchemaMeta, field FieldMeta, identifier, source, path string, st *selectNode) string { if st == nil || !st.expand { - return fmt.Sprintf("%s -> '%s'", source, path) + return sqlVariant.JSONExtract(source, path) } schemaName := field.CollectionItemMeta.RelationshipSchema schema := schemaMetas[schemaName] newSource := fmt.Sprintf("%s.Data", schema.Table) - where := fmt.Sprintf("WHERE %s -> '$.%s' = %s.value -> '$.%s'", newSource, field.CollectionItemMeta.RelationshipProperty, identifier, field.CollectionItemMeta.RelationshipProperty) + where := fmt.Sprintf( + "WHERE %s = %s", + sqlVariant.JSONExtract(newSource, fmt.Sprintf("$.%s", field.CollectionItemMeta.RelationshipProperty)), + sqlVariant.JSONExtract( + fmt.Sprintf("%s.value", identifier), + fmt.Sprintf("$.%s", field.CollectionItemMeta.RelationshipProperty), + ), + ) if st != nil { if st.filter != nil { - conditions, _ := buildWhereFromFilter(schemaMetas, field, newSource, newSource, st.filter.Tree) + conditions, _ := buildWhereFromFilter(sqlVariant, schemaMetas, field, newSource, newSource, st.filter.Tree) where = fmt.Sprintf("%s and %s", where, conditions) } } @@ -225,18 +236,18 @@ func buildSelectFieldsForRelationshipCollectionFieldType(schemaMetas map[string] } sel := st.children[key] - extract := buildSelectFields(schemaMetas, fm, fmt.Sprintf("%s%s", identifier, key), newSource, fmt.Sprintf("$.%s", key), sel) + extract := buildSelectFields(sqlVariant, schemaMetas, fm, fmt.Sprintf("%s%s", identifier, key), newSource, fmt.Sprintf("$.%s", key), sel) part := fmt.Sprintf("'%s', %s", key, extract) parts = append(parts, part) } - subQuery := fmt.Sprintf("JSON_OBJECT(%s)", strings.Join(parts, ",")) + subQuery := sqlVariant.JSONObject(parts) - return fmt.Sprintf("(SELECT JSON_GROUP_ARRAY(%s) FROM %s,JSON_EACH(%s, '%s') AS %s %s)", subQuery, schema.Table, source, path, identifier, where) + return fmt.Sprintf("(SELECT %s FROM %s,%s AS %s %s)", sqlVariant.JSONArrayAggregate(subQuery), schema.Table, sqlVariant.JSONEach(sqlVariant.JSONExtract(source, path)), identifier, where) } -func buildSelectFieldsForRelationshipFieldType(schemaMetas map[string]SchemaMeta, field FieldMeta, identifier, source, path string, st *selectNode) string { +func buildSelectFieldsForRelationshipFieldType(sqlVariant jsonsql.Variant, schemaMetas map[string]SchemaMeta, field FieldMeta, identifier, source, path string, st *selectNode) string { if st == nil || !st.expand { - return fmt.Sprintf("%s -> '%s'", source, path) + return sqlVariant.JSONExtract(source, path) } schemaName := field.RelationshipSchema @@ -257,13 +268,16 @@ func buildSelectFieldsForRelationshipFieldType(schemaMetas map[string]SchemaMeta } sel := st.children[key] - extract := buildSelectFields(schemaMetas, fm, fmt.Sprintf("%s%s", identifier, key), newsource, fmt.Sprintf("$.%s", key), sel) + extract := buildSelectFields(sqlVariant, schemaMetas, fm, fmt.Sprintf("%s%s", identifier, key), newsource, fmt.Sprintf("$.%s", key), sel) part := fmt.Sprintf("'%s', %s", key, extract) parts = append(parts, part) } - object := fmt.Sprintf("JSON_OBJECT(%s)", strings.Join(parts, ",")) + object := sqlVariant.JSONObject(parts) - return fmt.Sprintf("(SELECT %s FROM %s WHERE %s -> '$.%s' == %s -> '%s.%s')", object, schema.Table, newsource, field.RelationshipProperty, source, path, field.RelationshipProperty) + return fmt.Sprintf("(SELECT %s FROM %s WHERE %s = %s)", object, schema.Table, + sqlVariant.JSONExtract(newsource, fmt.Sprintf("$.%s", field.RelationshipProperty)), + sqlVariant.JSONExtract(source, fmt.Sprintf("%s.%s", path, field.RelationshipProperty)), + ) } func getDiscriminatorValue(schemaName string, field FieldMeta) string { @@ -274,13 +288,13 @@ func getDiscriminatorValue(schemaName string, field FieldMeta) string { } // nolint:cyclop -func buildSelectFieldsForComplexFieldType(schemaMetas map[string]SchemaMeta, field FieldMeta, identifier, source, path string, st *selectNode) string { +func buildSelectFieldsForComplexFieldType(sqlVariant jsonsql.Variant, schemaMetas map[string]SchemaMeta, field FieldMeta, identifier, source, path string, st *selectNode) string { // If there are no children in the select tree for this complex // type, shortcircuit and just return the data from the DB raw, // as there is no need to build the complex query, and it'll // ensure that null values are handled correctly. if st == nil || len(st.children) == 0 { - return fmt.Sprintf("%s -> '%s'", source, path) + return sqlVariant.JSONExtract(source, path) } objects := []string{} @@ -313,11 +327,11 @@ func buildSelectFieldsForComplexFieldType(schemaMetas map[string]SchemaMeta, fie sel = st.children[key] } - extract := buildSelectFields(schemaMetas, fm, fmt.Sprintf("%s%s", identifier, key), source, fmt.Sprintf("%s.%s", path, key), sel) + extract := buildSelectFields(sqlVariant, schemaMetas, fm, fmt.Sprintf("%s%s", identifier, key), source, fmt.Sprintf("%s.%s", path, key), sel) part := fmt.Sprintf("'%s', %s", key, extract) parts = append(parts, part) } - objects = append(objects, fmt.Sprintf("JSON_OBJECT(%s)", strings.Join(parts, ","))) + objects = append(objects, sqlVariant.JSONObject(parts)) } if len(objects) == 1 { @@ -332,12 +346,14 @@ func buildSelectFieldsForComplexFieldType(schemaMetas map[string]SchemaMeta, fie // } return fmt.Sprintf( - "(SELECT %s.value FROM JSON_EACH(JSON_ARRAY(%s)) AS %s WHERE %s.value -> '$.%s' = %s -> '%s.%s')", - identifier, strings.Join(objects, ","), identifier, - identifier, field.DiscriminatorProperty, source, path, field.DiscriminatorProperty) + "(SELECT %s.value FROM %s AS %s WHERE %s = %s)", + identifier, sqlVariant.JSONEach(sqlVariant.JSONArray(objects)), identifier, + sqlVariant.JSONExtract(fmt.Sprintf("%s.value", identifier), fmt.Sprintf("$.%s", field.DiscriminatorProperty)), + sqlVariant.JSONExtract(source, fmt.Sprintf("%s.%s", path, field.DiscriminatorProperty)), + ) } -func buildSelectFieldsForCollectionFieldType(schemaMetas map[string]SchemaMeta, field FieldMeta, identifier, source, path string, st *selectNode) string { +func buildSelectFieldsForCollectionFieldType(sqlVariant jsonsql.Variant, schemaMetas map[string]SchemaMeta, field FieldMeta, identifier, source, path string, st *selectNode) string { newIdentifier := fmt.Sprintf("%sOptions", identifier) newSource := fmt.Sprintf("%s.value", newIdentifier) @@ -346,12 +362,12 @@ func buildSelectFieldsForCollectionFieldType(schemaMetas map[string]SchemaMeta, var newSelectNode *selectNode if st != nil { if st.filter != nil { - conditions, _ := buildWhereFromFilter(schemaMetas, *field.CollectionItemMeta, fmt.Sprintf("%sFilter", identifier), newSource, st.filter.Tree) + conditions, _ := buildWhereFromFilter(sqlVariant, schemaMetas, *field.CollectionItemMeta, fmt.Sprintf("%sFilter", identifier), newSource, st.filter.Tree) where = fmt.Sprintf("WHERE %s", conditions) } if st.orderby != nil { - conditions, err := buildOrderByFromOdata(schemaMetas, *field.CollectionItemMeta, fmt.Sprintf("%sFilter", identifier), newSource, st.orderby.OrderByItems) + conditions, err := buildOrderByFromOdata(sqlVariant, schemaMetas, *field.CollectionItemMeta, fmt.Sprintf("%sFilter", identifier), newSource, st.orderby.OrderByItems) // TODO(sambetts) Add error handling to buildSelectFields if err != nil { log.Errorf("Failed to build DB query from $orderby: %v", err) @@ -369,19 +385,19 @@ func buildSelectFieldsForCollectionFieldType(schemaMetas map[string]SchemaMeta, newSelectNode.orderby = nil } - subQuery := buildSelectFields(schemaMetas, *field.CollectionItemMeta, fmt.Sprintf("%sOptions", newIdentifier), newSource, "$", newSelectNode) + subQuery := buildSelectFields(sqlVariant, schemaMetas, *field.CollectionItemMeta, fmt.Sprintf("%sOptions", newIdentifier), newSource, "$", newSelectNode) // This query will produce an exploded list of items (one row per item) from the collection, selected, filtered and ordered - listQuery := fmt.Sprintf("SELECT %s AS value FROM JSON_EACH(%s, '%s') AS %s %s %s", subQuery, source, path, newIdentifier, where, orderby) + listQuery := fmt.Sprintf("SELECT %s AS value FROM %s AS %s %s %s", subQuery, sqlVariant.JSONEach(sqlVariant.JSONExtract(source, path)), newIdentifier, where, orderby) // Now aggregate all the rows back into a JSON array aggregateValue := fmt.Sprintf("%s.value", identifier) if field.CollectionItemMeta.FieldType != PrimitiveFieldType { // For non-primitives use -> '$' to convert the value back to a // json object in the aggregate. - aggregateValue = fmt.Sprintf("%s -> '$'", aggregateValue) + aggregateValue = sqlVariant.JSONExtract(aggregateValue, "$") } - return fmt.Sprintf("(SELECT JSON_GROUP_ARRAY(%s) FROM (%s) AS %s)", aggregateValue, listQuery, identifier) + return fmt.Sprintf("(SELECT %s FROM (%s) AS %s)", sqlVariant.JSONArrayAggregate(aggregateValue), listQuery, identifier) } var sqlOperators = map[string]string{ @@ -481,7 +497,7 @@ func expandItemsToReachPath(schemaMetas map[string]SchemaMeta, field FieldMeta, // TODO: create a unit test // nolint:cyclop -func buildWhereFromFilter(schemaMetas map[string]SchemaMeta, field FieldMeta, identifier string, source string, node *godata.ParseNode) (string, error) { +func buildWhereFromFilter(sqlVariant jsonsql.Variant, schemaMetas map[string]SchemaMeta, field FieldMeta, identifier string, source string, node *godata.ParseNode) (string, error) { operator := node.Token.Value var query string @@ -494,7 +510,7 @@ func buildWhereFromFilter(schemaMetas map[string]SchemaMeta, field FieldMeta, id return "", fmt.Errorf("unable to covert oData path to json path: %w", err) } - fieldSource, err := sourceFromQueryPath(schemaMetas, field, identifier, source, queryPath) + fieldSource, err := sourceFromQueryPath(sqlVariant, schemaMetas, field, identifier, source, queryPath) if err != nil { return "", fmt.Errorf("unable to build source for filter %w", err) } @@ -502,7 +518,7 @@ func buildWhereFromFilter(schemaMetas map[string]SchemaMeta, field FieldMeta, id queryPath = fmt.Sprintf("$.%s", queryPath) rhs := node.Children[1] - extractFunction := "->" + var extractFunction jsonExtractFunctionType = sqlVariant.JSONExtract sqlOperator := sqlOperators[operator] var value string switch rhs.Token.Type { // TODO: implement all the relevant cases as ExpressionTokenDate and ExpressionTokenDateTime @@ -512,7 +528,7 @@ func buildWhereFromFilter(schemaMetas map[string]SchemaMeta, field FieldMeta, id value = singleQuote(rhs.Token.Value) case godata.ExpressionTokenInteger, godata.ExpressionTokenFloat: value = rhs.Token.Value - extractFunction = "->>" + extractFunction = sqlVariant.JSONExtractText case godata.ExpressionTokenNull: value = "NULL" if operator == "eq" { @@ -524,29 +540,31 @@ func buildWhereFromFilter(schemaMetas map[string]SchemaMeta, field FieldMeta, id } case godata.ExpressionTokenDateTime: value = singleQuote(rhs.Token.Value) - extractFunction = "->>" - return fmt.Sprintf("datetime(%s %s '%s') %s datetime(%s)", source, extractFunction, queryPath, sqlOperator, value), nil + extractFunction = sqlVariant.JSONExtractText + originalTime := sqlVariant.CastToDateTime(extractFunction(source, queryPath)) + timeToCompare := sqlVariant.CastToDateTime(value) + return fmt.Sprintf("%s %s %s", originalTime, sqlOperator, timeToCompare), nil default: return "", fmt.Errorf("unsupported token type %s", node.Children[1].Token.Type) } - query = fmt.Sprintf("%s %s '%s' %s %s", fieldSource, extractFunction, queryPath, sqlOperator, value) + query = fmt.Sprintf("%s %s %s", extractFunction(fieldSource, queryPath), sqlOperator, value) case "and": - left, err := buildWhereFromFilter(schemaMetas, field, identifier, source, node.Children[0]) + left, err := buildWhereFromFilter(sqlVariant, schemaMetas, field, identifier, source, node.Children[0]) if err != nil { return query, err } - right, err := buildWhereFromFilter(schemaMetas, field, identifier, source, node.Children[1]) + right, err := buildWhereFromFilter(sqlVariant, schemaMetas, field, identifier, source, node.Children[1]) if err != nil { return query, err } query = fmt.Sprintf("(%s AND %s)", left, right) case "or": - left, err := buildWhereFromFilter(schemaMetas, field, identifier, source, node.Children[0]) + left, err := buildWhereFromFilter(sqlVariant, schemaMetas, field, identifier, source, node.Children[0]) if err != nil { return query, err } - right, err := buildWhereFromFilter(schemaMetas, field, identifier, source, node.Children[1]) + right, err := buildWhereFromFilter(sqlVariant, schemaMetas, field, identifier, source, node.Children[1]) if err != nil { return query, err } @@ -559,7 +577,7 @@ func buildWhereFromFilter(schemaMetas map[string]SchemaMeta, field FieldMeta, id return "", fmt.Errorf("unable to covert oData path to json path: %w", err) } - fieldSource, err := sourceFromQueryPath(schemaMetas, field, identifier, source, queryPath) + fieldSource, err := sourceFromQueryPath(sqlVariant, schemaMetas, field, identifier, source, queryPath) if err != nil { return "", fmt.Errorf("unable to build source for filter %w", err) } @@ -573,13 +591,17 @@ func buildWhereFromFilter(schemaMetas map[string]SchemaMeta, field FieldMeta, id default: return query, fmt.Errorf("unsupported token type") } - query = fmt.Sprintf("%s ->> '$.%s' LIKE '%s'", fieldSource, queryPath, value) + query = fmt.Sprintf( + "%s LIKE '%s'", + sqlVariant.JSONExtractText(fieldSource, fmt.Sprintf("$.%s", queryPath)), + value, + ) } return query, nil } -func sourceFromQueryPath(schemaMetas map[string]SchemaMeta, field FieldMeta, identifier string, source string, queryPath string) (string, error) { +func sourceFromQueryPath(sqlVariant jsonsql.Variant, schemaMetas map[string]SchemaMeta, field FieldMeta, identifier string, source string, queryPath string) (string, error) { // ODATA path that would be present if we were to $select the // field being filtered selectPath := strings.ReplaceAll(queryPath, ".", "/") @@ -594,7 +616,7 @@ func sourceFromQueryPath(schemaMetas map[string]SchemaMeta, field FieldMeta, ide fieldSource := source if expandItems != "" { var err error - fieldSource, err = buildSelectFieldsFromSelectAndExpand(schemaMetas, field, identifier, source, &selectPath, &expandItems) + fieldSource, err = buildSelectFieldsFromSelectAndExpand(sqlVariant, schemaMetas, field, identifier, source, &selectPath, &expandItems) if err != nil { return "", fmt.Errorf("unable to build source %w", err) } @@ -602,7 +624,7 @@ func sourceFromQueryPath(schemaMetas map[string]SchemaMeta, field FieldMeta, ide return fieldSource, nil } -func buildOrderByFromOdata(schemaMetas map[string]SchemaMeta, field FieldMeta, identifier string, source string, orderbyItems []*godata.OrderByItem) (string, error) { +func buildOrderByFromOdata(sqlVariant jsonsql.Variant, schemaMetas map[string]SchemaMeta, field FieldMeta, identifier string, source string, orderbyItems []*godata.OrderByItem) (string, error) { conditions := []string{} for _, item := range orderbyItems { @@ -611,12 +633,16 @@ func buildOrderByFromOdata(schemaMetas map[string]SchemaMeta, field FieldMeta, i return "", fmt.Errorf("failed to convert odata path to json path: %w", err) } - fieldSource, err := sourceFromQueryPath(schemaMetas, field, identifier, source, queryPath) + fieldSource, err := sourceFromQueryPath(sqlVariant, schemaMetas, field, identifier, source, queryPath) if err != nil { return "", fmt.Errorf("unable to build source for filter %w", err) } - conditions = append(conditions, fmt.Sprintf("%s ->> '$.%s' %s", fieldSource, queryPath, strings.ToUpper(item.Order))) + conditions = append(conditions, fmt.Sprintf( + "%s %s", + sqlVariant.JSONExtractText(fieldSource, fmt.Sprintf("$.%s", queryPath)), + strings.ToUpper(item.Order)), + ) } return strings.Join(conditions, ", "), nil diff --git a/backend/pkg/database/odatasql/query_test.go b/backend/pkg/database/odatasql/query_test.go index dc89b3fcc..07370a89f 100644 --- a/backend/pkg/database/odatasql/query_test.go +++ b/backend/pkg/database/odatasql/query_test.go @@ -30,6 +30,8 @@ import ( "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" + + "github.com/openclarity/vmclarity/backend/pkg/database/odatasql/jsonsql" ) type SubOption struct { @@ -1121,7 +1123,7 @@ func TestBuildSQLQuery(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - query, err := BuildSQLQuery(carSchemaMetas, "Car", tt.args.filterString, tt.args.selectString, tt.args.expandString, tt.args.orderbyString, tt.args.top, tt.args.skip) + query, err := BuildSQLQuery(jsonsql.SQLite, carSchemaMetas, "Car", tt.args.filterString, tt.args.selectString, tt.args.expandString, tt.args.orderbyString, tt.args.top, tt.args.skip) if (err != nil) != tt.wantErr { t.Errorf("BuildSQLQuery() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/backend/pkg/database/types/database.go b/backend/pkg/database/types/database.go index 8007f2741..8f902c3e7 100644 --- a/backend/pkg/database/types/database.go +++ b/backend/pkg/database/types/database.go @@ -22,7 +22,8 @@ import ( ) const ( - DBDriverTypeLocal = "LOCAL" + DBDriverTypeLocal = "LOCAL" + DBDriverTypePostgres = "POSTGRES" ) var ErrNotFound = errors.New("not found") diff --git a/go.mod b/go.mod index 6e38cb538..7553f9c38 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb gopkg.in/yaml.v3 v3.0.1 gorm.io/datatypes v1.2.0 + gorm.io/driver/postgres v1.4.4 gorm.io/driver/sqlite v1.3.6 gorm.io/gorm v1.23.10 gotest.tools/v3 v3.4.0 @@ -223,6 +224,14 @@ require ( github.com/in-toto/in-toto-golang v0.8.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/invopop/yaml v0.1.0 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.13.0 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.1 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgtype v1.12.0 // indirect + github.com/jackc/pgx/v4 v4.17.2 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jdkato/prose v1.1.0 // indirect github.com/jinzhu/copier v0.3.5 // indirect @@ -443,6 +452,11 @@ replace oras.land/oras-go => oras.land/oras-go v1.1.1 // We'll need to keep adding versions until anchore fixes it. exclude ( gorm.io/driver/mysql v1.5.0 + gorm.io/driver/postgres v1.4.5 + gorm.io/driver/postgres v1.4.6 + gorm.io/driver/postgres v1.4.7 + gorm.io/driver/postgres v1.4.8 + gorm.io/driver/postgres v1.5.0 gorm.io/driver/sqlite v1.4.0 gorm.io/driver/sqlite v1.4.1 gorm.io/driver/sqlite v1.4.2 diff --git a/go.sum b/go.sum index 730a56edc..5fefecf5a 100644 --- a/go.sum +++ b/go.sum @@ -549,6 +549,8 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= @@ -669,6 +671,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -855,6 +858,7 @@ github.com/go-gorp/gorp/v3 v3.0.2/go.mod h1:BJ3q1ejpV8cVALtcXvXaXyTOlMmJhWDxTmnc github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= @@ -965,6 +969,7 @@ github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godror/godror v0.24.2/go.mod h1:wZv/9vPiUib6tkoDl+AZ/QLf5YZgMravZ7jxH2eQWAE= +github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.3.1+incompatible h1:0/KbAdpx3UXAx1kEOWHJeOkpbgRFGHVgv+CFIY7dBJI= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= @@ -1239,9 +1244,54 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc= github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= +github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= +github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.13.0 h1:3L1XMNV2Zvca/8BYhzcRFS70Lr0WlDg16Di6SFGAbys= +github.com/jackc/pgconn v1.13.0/go.mod h1:AnowpAqO4CMIIJNZl2VJp+KrkAZciAkhEl0W0JIobpI= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.1 h1:nwj7qwf0S+Q7ISFfBndqeLwSwxs+4DPsbRFjECT1Y4Y= +github.com/jackc/pgproto3/v2 v2.3.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgx/v5 v5.3.1 h1:Fcr8QJ1ZeLi5zsPZqQeUZhNhxfkkKBOgJuYkJHoBOtU= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= +github.com/jackc/pgtype v1.12.0 h1:Dlq8Qvcch7kiehm8wPGIW0W3KsCCHJnRacKW0UM8n5w= +github.com/jackc/pgtype v1.12.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= +github.com/jackc/pgx/v4 v4.17.2 h1:0Ut0rpeKwvIVbMQ1KbMBU4h6wxehBI535LK6Flheh8E= +github.com/jackc/pgx/v4 v4.17.2/go.mod h1:lcxIZN44yMIrWI78a5CpucdD14hX0SBDbNRvjDBItsw= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jdkato/prose v1.1.0 h1:LpvmDGwbKGTgdCH3a8VJL56sr7p/wOFPw/R4lM4PfFg= @@ -1333,6 +1383,7 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -1352,8 +1403,11 @@ github.com/liamg/jfather v0.0.7 h1:Xf78zS263yfT+xr2VSo6+kyAy4ROlCacRqJG7s5jt4k= github.com/liamg/jfather v0.0.7/go.mod h1:xXBGiBoiZ6tmHhfy5Jzw8sugzajwYdi6VosIpB3/cPM= github.com/liamg/memoryfs v1.4.3 h1:+ChjcuPRYpjJSulD13PXDNR3JeJ5HUYKjLHyWVK0bqU= github.com/liamg/memoryfs v1.4.3/go.mod h1:z7mfqXFQS8eSeBBsFjYLlxYRMRyiPktytvYCYTb3BSk= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= @@ -1399,6 +1453,7 @@ github.com/masahiro331/go-xfs-filesystem v0.0.0-20221225060805-c02764233454/go.m github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -1408,6 +1463,8 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= @@ -1700,7 +1757,10 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/rubenv/sql-migrate v1.2.0 h1:fOXMPLMd41sK7Tg75SXDec15k3zg5WNV6SjuDRiNfcU= @@ -1740,6 +1800,7 @@ github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= github.com/shogo82148/go-shuffle v0.0.0-20170808115208-59829097ff3b h1:VI1u+o2KZPZ5AhuPpXY0JBdpQPnkTx6Dd5XJhK/9MYE= github.com/shogo82148/go-shuffle v0.0.0-20170808115208-59829097ff3b/go.mod h1:2htx6lmL0NGLHlO8ZCf+lQBGBHIbEujyywxJArf+2Yc= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -1950,6 +2011,7 @@ github.com/zclconf/go-cty v1.10.0 h1:mp9ZXQeIcN8kAwuqorjH+Q+njbJKjLrvB2yIh4q7U+0 github.com/zclconf/go-cty v1.10.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zclconf/go-cty-yaml v1.0.2 h1:dNyg4QLTrv2IfJpm7Wtxi55ed5gLGOlPrZ6kMd51hY0= github.com/zclconf/go-cty-yaml v1.0.2/go.mod h1:IP3Ylp0wQpYm50IHK8OZWKMu6sPJIUgKa8XhiVHura0= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -1990,16 +2052,23 @@ go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee33 go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= @@ -2011,6 +2080,7 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -2025,9 +2095,11 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -2243,6 +2315,7 @@ golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2410,6 +2483,7 @@ golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -2420,9 +2494,12 @@ golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -2432,6 +2509,7 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -2473,6 +2551,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2742,6 +2822,7 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= @@ -2780,12 +2861,14 @@ gorm.io/datatypes v1.2.0 h1:5YT+eokWdIxhJgWHdrb2zYUimyk0+TaFth+7a0ybzco= gorm.io/datatypes v1.2.0/go.mod h1:o1dh0ZvjIjhH/bngTpypG6lVRJ5chTBxE09FH/71k04= gorm.io/driver/mysql v1.4.7 h1:rY46lkCspzGHn7+IYsNpSfEv9tA+SU4SkkB+GFX125Y= gorm.io/driver/mysql v1.4.7/go.mod h1:SxzItlnT1cb6e1e4ZRpgJN2VYtcqJgqnHxWr4wsP8oc= -gorm.io/driver/postgres v1.5.0 h1:u2FXTy14l45qc3UeCJ7QaAXZmZfDDv0YrthvmRq1l0U= +gorm.io/driver/postgres v1.4.4 h1:zt1fxJ+C+ajparn0SteEnkoPg0BQ6wOWXEQ99bteAmw= +gorm.io/driver/postgres v1.4.4/go.mod h1:whNfh5WhhHs96honoLjBAMwJGYEuA3m1hvgUbNXhPCw= gorm.io/driver/sqlite v1.3.6 h1:Fi8xNYCUplOqWiPa3/GuCeowRNBRGTf62DEmhMDHeQQ= gorm.io/driver/sqlite v1.3.6/go.mod h1:Sg1/pvnKtbQ7jLXxfZa+jSHvoX8hoZA8cn4xllOMTgE= gorm.io/driver/sqlserver v1.4.1 h1:t4r4r6Jam5E6ejqP7N82qAJIJAht27EGT41HyPfXRw0= gorm.io/gorm v1.23.4/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.23.5/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= +gorm.io/gorm v1.23.7/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.23.10 h1:4Ne9ZbzID9GUxRkllxN4WjJKpsHx8YbKvekVdgyWh24= gorm.io/gorm v1.23.10/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= diff --git a/installation/aws/VmClarity.cfn b/installation/aws/VmClarity.cfn index 6710dbadb..89a5505e7 100644 --- a/installation/aws/VmClarity.cfn +++ b/installation/aws/VmClarity.cfn @@ -152,12 +152,78 @@ Resources: apt: docker.io: [] files: - "/etc/vmclarity/render_config.sh": - content: | - #!/bin/bash - set -euo pipefail - local_ip_address="$(curl http://169.254.169.254/latest/meta-data/local-ipv4)" - sed -i "s/__BACKEND_REST_HOST__/${local_ip_address}/" /etc/vmclarity/config.env + "/etc/vmclarity/deploy.sh": + content: + Fn::Sub: | + #!/bin/bash + set -euo pipefail + + # Create the docker network for the VMClarity services if it + # doesn't exist. + if docker network ls | grep vmclarity; then + echo "network already exists" + else + docker network create vmclarity + fi + + # Reload the systemd daemon to ensure that all the VMClarity + # units have been detected. + systemctl daemon-reload + + # Enable and start/restart exploit-db-server + systemctl enable exploit-db-server.service + systemctl restart exploit-db-server.service + + # Enable and start/restart trivy server + systemctl enable trivy_server.service + systemctl restart trivy_server.service + + # Enable and start/restart grype_server + systemctl enable grype_server.service + systemctl restart grype_server.service + + # Enable and start/restart freshclam mirror + systemctl enable vmclarity_freshclam_mirror.service + systemctl restart vmclarity_freshclam_mirror.service + + if [ "${DatabaseToUse}" == "Postgresql" ]; then + # Enable and start/restart postgres + systemctl enable postgres.service + systemctl restart postgres.service + + # Configure the VMClarity backend to use the local postgres + # service + echo "DATABASE_DRIVER=POSTGRES" >> /etc/vmclarity/config.env + echo "DB_NAME=vmclarity" >> /etc/vmclarity/config.env + echo "DB_USER=vmclarity" >> /etc/vmclarity/config.env + echo "DB_PASS=${PostgresDBPassword}" >> /etc/vmclarity/config.env + echo "DB_HOST=postgres.service" >> /etc/vmclarity/config.env + echo "DB_PORT_NUMBER=5432" >> /etc/vmclarity/config.env + elif [ "${DatabaseToUse}" == "External Postgresql" ]; then + # Configure the VMClarity backend to use the postgres + # database configured by the user. + echo "DATABASE_DRIVER=POSTGRES" >> /etc/vmclarity/config.env + echo "DB_NAME=${ExternalDBName}" >> /etc/vmclarity/config.env + echo "DB_USER=${ExternalDBUsername}" >> /etc/vmclarity/config.env + echo "DB_PASS=${ExternalDBPassword}" >> /etc/vmclarity/config.env + echo "DB_HOST=${ExternalDBHost}" >> /etc/vmclarity/config.env + echo "DB_PORT_NUMBER=${ExternalDBPort}" >> /etc/vmclarity/config.env + elif [ "${DatabaseToUse}" == "SQLite" ]; then + # Configure the VMClarity backend to use the SQLite DB + # driver and configure the storage location so that it + # persists. + echo "DATABASE_DRIVER=LOCAL" >> /etc/vmclarity/config.env + echo "LOCAL_DB_PATH=/data/vmclarity.db" >> /etc/vmclarity/config.env + fi + + # Replace anywhere in the config.env __BACKEND_REST_HOST__ + # with the local ipv4 IP address of the VMClarity server. + local_ip_address="$(curl http://169.254.169.254/latest/meta-data/local-ipv4)" + sed -i "s/__BACKEND_REST_HOST__/${!local_ip_address}/" /etc/vmclarity/config.env + + # Enable and start/restart VMClarity backend + systemctl enable vmclarity.service + systemctl restart vmclarity.service mode: "000744" "/etc/vmclarity/config.env": content: @@ -170,8 +236,6 @@ Resources: AWS_INSTANCE_TYPE=${ScannerInstanceType} SCANNER_KEY_PAIR_NAME=${KeyName} AWS_JOB_IMAGE_ID=${JobImageID} - DATABASE_DRIVER=LOCAL - LOCAL_DB_PATH=/data/vmclarity.db BACKEND_REST_HOST=__BACKEND_REST_HOST__ BACKEND_REST_PORT=8888 SCANNER_CONTAINER_IMAGE=${ScannerContainerImage} @@ -213,6 +277,7 @@ Resources: ExecStartPre=/usr/bin/docker pull ${BACKEND_CONTAINER_IMAGE} ExecStart=/usr/bin/docker run \ --rm --name %n \ + --network vmclarity \ -p 0.0.0.0:8888:8888/tcp \ -v /opt/vmclarity:/data \ --env-file /etc/vmclarity/config.env \ @@ -241,6 +306,7 @@ Resources: ExecStartPre=/usr/bin/docker pull ${ExploitDBServerContainerImage} ExecStart=/usr/bin/docker run \ --rm --name %n \ + --network vmclarity \ -p 0.0.0.0:1326:1326/tcp \ -v /opt/exploits:/vuls \ ${ExploitDBServerContainerImage} @@ -272,6 +338,7 @@ Resources: ExecStartPre=/usr/bin/docker pull ${TrivyServerContainerImage} ExecStart=/usr/bin/docker run \ --rm --name %n \ + --network vmclarity \ -p 0.0.0.0:9992:9992/tcp \ -v /opt/trivy-server:/home/scanner/.cache \ --env-file /etc/trivy-server/config.env \ @@ -304,6 +371,7 @@ Resources: ExecStartPre=/usr/bin/docker pull ${GrypeServerContainerImage} ExecStart=/usr/bin/docker run \ --rm --name %n \ + --network vmclarity \ -p 0.0.0.0:9991:9991/tcp \ -v /opt/grype-server:/opt/grype-server \ --env-file /etc/grype-server/config.env \ @@ -330,6 +398,7 @@ Resources: ExecStartPre=/usr/bin/docker pull ${FreshclamMirrorContainerImage} ExecStart=/usr/bin/docker run \ --rm --name %n \ + --network vmclarity \ -p 0.0.0.0:1000:80/tcp \ ${FreshclamMirrorContainerImage} @@ -337,36 +406,37 @@ Resources: WantedBy=multi-user.target - FreshclamMirrorContainerImage: !If [FreshclamMirrorContainerImageOverridden, !Ref FreshclamMirrorContainerImageOverride, "ghcr.io/openclarity/freshclam-mirror:v0.1.0"] mode: "000644" - commands: - 01subsitute_rest_address: - command: /etc/vmclarity/render_config.sh - 02reload_systemctl: - command: systemctl daemon-reload - - 03enable_exploit_db_server: - command: systemctl enable exploit-db-server.service - 04start_restart_exploit_db_server: - command: systemctl restart exploit-db-server.service - - 05enable_trivy_server: - command: systemctl enable trivy_server.service - 06start_restart_trivy_server: - command: systemctl restart trivy_server.service - - 07enable_grype_server: - command: systemctl enable grype_server.service - 08start_restart_grype_server: - command: systemctl restart grype_server.service + "/lib/systemd/system/postgres.service": + content: + Fn::Sub: + - | + [Unit] + Description=Postgresql Database Server + After=docker.service + Requires=docker.service - 09enable_vmclarity_freshclam_mirror: - command: systemctl enable vmclarity_freshclam_mirror.service - 10start_restart_vmclarity_freshclam_mirror: - command: systemctl restart vmclarity_freshclam_mirror.service + [Service] + TimeoutStartSec=0 + Restart=always + ExecStartPre=-/usr/bin/docker stop %n + ExecStartPre=-/usr/bin/docker rm %n + ExecStartPre=/usr/bin/docker pull ${PostgresqlContainerImage} + ExecStart=/usr/bin/docker run \ + --rm --name %n \ + --network vmclarity \ + -e POSTGRESQL_USERNAME=vmclarity \ + -e POSTGRESQL_PASSWORD=${PostgresDBPassword} \ + -e POSTGRESQL_DATABASE=vmclarity \ + -p 127.0.0.1:5432:5432/tcp \ + ${PostgresqlContainerImage} - 11enable_vmclarity: - command: systemctl enable vmclarity.service - 12start_restart_vmclarity: - command: systemctl restart vmclarity.service + [Install] + WantedBy=multi-user.target + - PostgresqlContainerImage: !If [PostgresqlContainerImageOverridden, !Ref PostgresqlContainerImageOverride, "bitnami/postgresql:12.14.0-debian-11-r28"] + mode: "000644" + commands: + 01deploy_vmclarity: + command: /etc/vmclarity/deploy.sh DependsOn: - VmClarityServerPublicRoute @@ -799,6 +869,12 @@ Parameters: "ghcr.io/openclarity/exploit-db-server:v0.1.2" will be used if not overridden. Type: String Default: '' + PostgresqlContainerImageOverride: + Description: > + Name of the container image used for the postgres database server. + "bitnami/postgresql:12.14.0-debian-11-r28" will be used if not overridden. + Type: String + Default: '' AssetScanDeletePolicy: Description: When VMClarity should delete resources after a completed asset scan. Type: String @@ -807,6 +883,38 @@ Parameters: - Always - OnSuccess - Never + DatabaseToUse: + Description: Database VMClarity should use to store its objects. + Type: String + Default: Postgresql + AllowedValues: + - Postgresql + - External Postgresql + - SQLite + PostgresDBPassword: + Description: > + Password to configure Postgresql with on first install. + Required if Postgres is selected as the Database To Use. + Do not change this on stack update. + Type: String + Default: password123 + ExternalDBHost: + Description: Hostname or IP address of the External DB to connect to. Required if an external database type is selected as the Database To Use. + Type: String + ExternalDBPort: + Description: Network Port of the External DB to connect to. Required if an external database type is selected as the Database To Use. + Type: Number + MinValue: 0 + Default: 0 + ExternalDBName: + Description: Name of the Database to use on the External DB. Required if an external database type is selected as the Database To Use. + Type: String + ExternalDBUsername: + Description: Username to use to connect to the External DB. Required if an external database type is selected as the Database To Use. + Type: String + ExternalDBPassword: + Description: Password to use to connect to the External DB. Required if an external database type is selected as the Database To Use. + Type: String Metadata: AWS::CloudFormation::Interface: ParameterGroups: @@ -820,6 +928,16 @@ Metadata: default: Network Configuration Parameters: - SSHLocation + - Label: + default: Database Configuration + Parameters: + - DatabaseToUse + - PostgresDBPassword + - ExternalDBHost + - ExternalDBPort + - ExternalDBName + - ExternalDBUsername + - ExternalDBPassword - Label: default: Advanced Configuration Parameters: @@ -828,6 +946,7 @@ Metadata: - TrivyServerContainerImageOverride - GrypeServerContainerImageOverride - ExploitDBServerContainerImageOverride + - PostgresqlContainerImageOverride - FreshclamMirrorContainerImageOverride - AssetScanDeletePolicy ParameterLabels: @@ -847,8 +966,24 @@ Metadata: default: Exploit DB Server Container Image Override FreshclamMirrorContainerImageOverride: default: freshclam-mirror Container Image Override + PostgresqlContainerImageOverride: + default: Postgresql Container Image Override AssetScanDeletePolicy: default: Asset Scan Delete Policy + DatabaseToUse: + default: Database To Use + PostgresDBPassword: + default: Postgresql DB Password + ExternalDBHost: + default: External DB Hostname or IP + ExternalDBPort: + default: External DB Port + ExternalDBName: + default: External DB Database Name + ExternalDBUsername: + default: External DB Username + ExternalDBPassword: + default: External DB Password Mappings: # For every type we want AWS hardware virtualisation on amd64 (HVM64) AWSInstanceType2Arch: @@ -921,6 +1056,10 @@ Conditions: - !Equals - !Ref ExploitDBServerContainerImageOverride - '' + PostgresqlContainerImageOverridden: !Not + - !Equals + - !Ref PostgresqlContainerImageOverride + - '' Outputs: URL: Value: !Sub "${VmClarityServer.PublicIp}"