Skip to content

Commit

Permalink
Add support for postgres database
Browse files Browse the repository at this point in the history
This commit add better support to odatasql for SQL variants, as well as
updating the gorm database driver to connect to postgres when configured
to do so.

The cloudformation installer has been updated with a new database
configuration section which allows the user to choose the DB driver
(postgres or sqlite), and also configure options to connect to an
external database.
  • Loading branch information
Tehsmash committed May 25, 2023
1 parent 3e45aa3 commit 1a6c3ed
Show file tree
Hide file tree
Showing 12 changed files with 542 additions and 107 deletions.
1 change: 1 addition & 0 deletions backend/pkg/database/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ func RegisterDrivers() {
DBDrivers = map[string]DBDriver{}
}
DBDrivers[types.DBDriverTypeLocal] = gorm.NewDatabase
DBDrivers[types.DBDriverTypePostgres] = gorm.NewDatabase
})
}

Expand Down
34 changes: 27 additions & 7 deletions backend/pkg/database/gorm/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,12 @@ import (
"time"

uuid "github.com/satori/go.uuid"
"gorm.io/driver/postgres"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"

"github.com/openclarity/vmclarity/backend/pkg/database/odatasql/jsonsql"
"github.com/openclarity/vmclarity/backend/pkg/database/types"
)

Expand Down Expand Up @@ -83,43 +85,43 @@ func initDataBase(config types.DBConfig) (*gorm.DB, error) {
// First for all objects index the ID field this speeds up anywhere
// we're getting a single object out of the DB, including in PATCH/PUT
// etc.
idb := db.Exec("CREATE INDEX IF NOT EXISTS targets_id_idx ON targets(Data -> 'id')")
idb := db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS targets_id_idx ON targets((%s))", SQLVariant.JSONExtract("Data", "$.id")))
if idb.Error != nil {
return nil, fmt.Errorf("failed to create index targets_id_idx: %w", idb.Error)
}

idb = db.Exec("CREATE INDEX IF NOT EXISTS scan_results_id_idx ON scan_results(Data -> 'id')")
idb = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS scan_results_id_idx ON scan_results((%s))", SQLVariant.JSONExtract("Data", "$.id")))
if idb.Error != nil {
return nil, fmt.Errorf("failed to create index scan_results_id_idx: %w", idb.Error)
}

idb = db.Exec("CREATE INDEX IF NOT EXISTS scan_configs_id_idx ON scan_configs(Data -> 'id')")
idb = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS scan_configs_id_idx ON scan_configs((%s))", SQLVariant.JSONExtract("Data", "$.id")))
if idb.Error != nil {
return nil, fmt.Errorf("failed to create index scan_configs_id_idx: %w", idb.Error)
}

idb = db.Exec("CREATE INDEX IF NOT EXISTS scans_id_idx ON scans(Data -> 'id')")
idb = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS scans_id_idx ON scans((%s))", SQLVariant.JSONExtract("Data", "$.id")))
if idb.Error != nil {
return nil, fmt.Errorf("failed to create index scans_id_idx: %w", idb.Error)
}

idb = db.Exec("CREATE INDEX IF NOT EXISTS findings_id_idx ON findings(Data -> 'id')")
idb = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS findings_id_idx ON findings((%s))", SQLVariant.JSONExtract("Data", "$.id")))
if idb.Error != nil {
return nil, fmt.Errorf("failed to create index findings_id_idx: %w", idb.Error)
}

// For processing scan results to findings we need to find all the scan
// results by general status and findingsProcessed, so add an index for
// that.
idb = db.Exec("CREATE INDEX IF NOT EXISTS scan_results_findings_processed_idx ON scan_results(Data -> 'findingsProcessed', Data -> 'status.general.state')")
idb = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS scan_results_findings_processed_idx ON scan_results((%s), (%s))", SQLVariant.JSONExtract("Data", "$.findingsProcessed"), SQLVariant.JSONExtract("Data", "$.status.general.state")))
if idb.Error != nil {
return nil, fmt.Errorf("failed to create index scan_results_findings_processed_idx: %w", idb.Error)
}

// The UI needs to find all the findings for a specific finding type
// and the scan result processor needs to filter that list by a
// specific asset. So add a combined index for those cases.
idb = db.Exec("CREATE INDEX IF NOT EXISTS findings_by_type_and_asset_idx ON findings(Data -> 'findingInfo.objectType', Data -> 'asset.id')")
idb = db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS findings_by_type_and_asset_idx ON findings((%s), (%s))", SQLVariant.JSONExtract("Data", "$.findingInfo.objectType"), SQLVariant.JSONExtract("Data", "$.asset.id")))
if idb.Error != nil {
return nil, fmt.Errorf("failed to create index findings_by_type_and_asset_idx: %w", idb.Error)
}
Expand All @@ -133,7 +135,11 @@ func initDataBase(config types.DBConfig) (*gorm.DB, error) {
func initDB(config types.DBConfig, dbDriver string, dbLogger logger.Interface) (*gorm.DB, error) {
switch dbDriver {
case types.DBDriverTypeLocal:
SQLVariant = jsonsql.SQLite
return initSqlite(config, dbLogger)
case types.DBDriverTypePostgres:
SQLVariant = jsonsql.Postgres
return initPostgres(config, dbLogger)
default:
return nil, fmt.Errorf("driver type %s is not supported by GORM driver", dbDriver)
}
Expand All @@ -148,3 +154,17 @@ func initSqlite(config types.DBConfig, dbLogger logger.Interface) (*gorm.DB, err
}
return db, nil
}

func initPostgres(config types.DBConfig, dbLogger logger.Interface) (*gorm.DB, error) {
dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%s sslmode=disable TimeZone=UTC",
config.DBHost, config.DBUser, config.DBPassword, config.DBName, config.DBPort)

db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{
Logger: dbLogger,
})
if err != nil {
return nil, fmt.Errorf("failed to open %s db: %v", config.DBName, err)
}

return db, nil
}
7 changes: 5 additions & 2 deletions backend/pkg/database/gorm/odata.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,12 @@ import (
"gorm.io/gorm"

"github.com/openclarity/vmclarity/backend/pkg/database/odatasql"
"github.com/openclarity/vmclarity/backend/pkg/database/odatasql/jsonsql"
"github.com/openclarity/vmclarity/runtime_scan/pkg/utils"
)

var SQLVariant jsonsql.Variant

type ODataObject struct {
ID uint `gorm:"primarykey"`
Data datatypes.JSON
Expand Down Expand Up @@ -827,7 +830,7 @@ func ODataQuery(db *gorm.DB, schema string, filterString, selectString, expandSt

// Build the raw SQL query using the odatasql library, this will also
// parse and validate the ODATA query params.
query, err := odatasql.BuildSQLQuery(schemaMetas, schema, filterString, selectString, expandString, orderby, top, skip)
query, err := odatasql.BuildSQLQuery(SQLVariant, schemaMetas, schema, filterString, selectString, expandString, orderby, top, skip)
if err != nil {
return fmt.Errorf("failed to build query for DB: %w", err)
}
Expand All @@ -849,7 +852,7 @@ func ODataQuery(db *gorm.DB, schema string, filterString, selectString, expandSt
}

func ODataCount(db *gorm.DB, schema string, filterString *string) (int, error) {
query, err := odatasql.BuildCountQuery(schemaMetas, schema, filterString)
query, err := odatasql.BuildCountQuery(SQLVariant, schemaMetas, schema, filterString)
if err != nil {
return 0, fmt.Errorf("failed to build query to count objects: %w", err)
}
Expand Down
67 changes: 67 additions & 0 deletions backend/pkg/database/odatasql/jsonsql/postgres.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// Copyright © 2023 Cisco Systems, Inc. and its affiliates.
// All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package jsonsql

import (
"fmt"
"strings"
)

type postgres struct{}

var Postgres Variant = postgres{}

func (postgres) JSONObject(parts []string) string {
return fmt.Sprintf("JSONB_BUILD_OBJECT(%s)", strings.Join(parts, ", "))
}

func (postgres) JSONArrayAggregate(value string) string {
return fmt.Sprintf("JSONB_AGG(%s)", value)
}

func (postgres) CastToDateTime(strTime string) string {
return fmt.Sprintf("(%s)::timestamptz", strTime)
}

func (postgres) JSONEach(source string) string {
return fmt.Sprintf("JSONB_ARRAY_ELEMENTS((%s))", source)
}

func (postgres) JSONArray(items []string) string {
return fmt.Sprintf("JSONB_BUILD_ARRAY(%s)", strings.Join(items, ", "))
}

func convertJSONPathToPostgresPath(jsonPath string) string {
parts := strings.Split(jsonPath, ".")
newParts := []string{}
for _, part := range parts {
if part == "$" {
continue
}
newParts = append(newParts, part)
}
return fmt.Sprintf("{%s}", strings.Join(newParts, ","))
}

func (postgres) JSONExtract(source, path string) string {
path = convertJSONPathToPostgresPath(path)
return fmt.Sprintf("%s#>'%s'", source, path)
}

func (postgres) JSONExtractText(source, path string) string {
path = convertJSONPathToPostgresPath(path)
return fmt.Sprintf("%s#>>'%s'", source, path)
}
53 changes: 53 additions & 0 deletions backend/pkg/database/odatasql/jsonsql/sqlite.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Copyright © 2023 Cisco Systems, Inc. and its affiliates.
// All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package jsonsql

import (
"fmt"
"strings"
)

type sqlite struct{}

var SQLite Variant = sqlite{}

func (sqlite) JSONObject(parts []string) string {
return fmt.Sprintf("JSON_OBJECT(%s)", strings.Join(parts, ", "))
}

func (sqlite) JSONArrayAggregate(value string) string {
return fmt.Sprintf("JSON_GROUP_ARRAY(%s)", value)
}

func (sqlite) CastToDateTime(strTime string) string {
return fmt.Sprintf("datetime(%s)", strTime)
}

func (sqlite) JSONEach(source string) string {
return fmt.Sprintf("JSON_EACH(%s)", source)
}

func (sqlite) JSONArray(items []string) string {
return fmt.Sprintf("JSON_ARRAY(%s)", strings.Join(items, ", "))
}

func (sqlite) JSONExtract(source, path string) string {
return fmt.Sprintf("%s->'%s'", source, path)
}

func (sqlite) JSONExtractText(source, path string) string {
return fmt.Sprintf("%s->>'%s'", source, path)
}
26 changes: 26 additions & 0 deletions backend/pkg/database/odatasql/jsonsql/variant.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// Copyright © 2023 Cisco Systems, Inc. and its affiliates.
// All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package jsonsql

type Variant interface {
JSONObject(parts []string) string
JSONArrayAggregate(string) string
CastToDateTime(string) string
JSONEach(source string) string
JSONArray(items []string) string
JSONExtract(source string, path string) string
JSONExtractText(source string, path string) string
}
Loading

0 comments on commit 1a6c3ed

Please sign in to comment.