Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove sqlite support #3084

Merged
merged 4 commits into from
Sep 6, 2022
Merged

Remove sqlite support #3084

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9c47026
maintain: remove db file option
dnephin Aug 31, 2022
c88c78b
maintain: move skipping of tests into the test helper
dnephin Aug 31, 2022
29d1dd0
maintain: remove dbFile command line flag
dnephin Aug 31, 2022
5570805
maintain: remove last of sqlite driver
dnephin Sep 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ require (
gopkg.in/square/go-jose.v2 v2.6.0
gopkg.in/yaml.v2 v2.4.0
gorm.io/driver/postgres v1.3.7
gorm.io/driver/sqlite v1.3.6
gorm.io/gorm v1.23.5
k8s.io/api v0.25.0
k8s.io/apimachinery v0.25.0
Expand Down Expand Up @@ -141,7 +140,6 @@ require (
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/mattn/go-sqlite3 v1.14.12 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
Expand Down
4 changes: 0 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -530,8 +530,6 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ1Z0=
github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
Expand Down Expand Up @@ -1236,8 +1234,6 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/postgres v1.3.7 h1:FKF6sIMDHDEvvMF/XJvbnCl0nu6KSKUaPXevJ4r+VYQ=
gorm.io/driver/postgres v1.3.7/go.mod h1:f02ympjIcgtHEGFMZvdgTxODZ9snAHDb4hXfigBVuNI=
gorm.io/driver/sqlite v1.3.6 h1:Fi8xNYCUplOqWiPa3/GuCeowRNBRGTf62DEmhMDHeQQ=
gorm.io/driver/sqlite v1.3.6/go.mod h1:Sg1/pvnKtbQ7jLXxfZa+jSHvoX8hoZA8cn4xllOMTgE=
gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo=
gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
Expand Down
5 changes: 0 additions & 5 deletions internal/access/access_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,6 @@ import (
func setupDB(t *testing.T) *data.DB {
t.Helper()
driver := database.PostgresDriver(t, "_access")
if driver == nil {
lite, err := data.NewSQLiteDriver("file::memory:")
assert.NilError(t, err)
driver = &database.Driver{Dialector: lite}
}

patch.ModelsSymmetricKey(t)
db, err := data.NewDB(driver.Dialector, nil)
Expand Down
5 changes: 2 additions & 3 deletions internal/cmd/login_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,9 +220,8 @@ func setupServerOptions(t *testing.T, opts *server.Options) {

// TODO: why do tests fail when the same schemaSuffix is used?
suffix := "_cmd_" + t.Name()
if pgDriver := database.PostgresDriver(t, suffix); pgDriver != nil {
opts.DBConnectionString = pgDriver.DSN
}
pgDriver := database.PostgresDriver(t, suffix)
opts.DBConnectionString = pgDriver.DSN
}

func TestLoginCmd_TLSVerify(t *testing.T) {
Expand Down
9 changes: 0 additions & 9 deletions internal/cmd/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,6 @@ func newServerCmd() *cobra.Command {

options.TLSCache = tlsCache

dbFile, err := canonicalPath(options.DBFile)
if err != nil {
return err
}

options.DBFile = dbFile

dbEncryptionKey, err := canonicalPath(options.DBEncryptionKey)
if err != nil {
return err
Expand All @@ -70,7 +63,6 @@ func newServerCmd() *cobra.Command {

cmd.Flags().StringVarP(&configFilename, "config-file", "f", "", "Server configuration file")
cmd.Flags().String("tls-cache", "", "Directory to cache TLS certificates")
cmd.Flags().String("db-file", "", "Path to SQLite 3 database")
cmd.Flags().String("db-name", "", "Database name")
cmd.Flags().String("db-host", "", "Database host")
cmd.Flags().Int("db-port", 0, "Database port")
Expand All @@ -93,7 +85,6 @@ func defaultServerOptions(infraDir string) server.Options {
return server.Options{
Version: 0.2, // update this as the config version changes
TLSCache: filepath.Join(infraDir, "cache"),
DBFile: filepath.Join(infraDir, "sqlite3.db"),
DBEncryptionKey: filepath.Join(infraDir, "sqlite3.db.key"),
DBEncryptionKeyProvider: "native",
EnableTelemetry: true,
Expand Down
7 changes: 3 additions & 4 deletions internal/cmd/server_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (

"github.com/infrahq/infra/internal/cmd/types"
"github.com/infrahq/infra/internal/server"
"github.com/infrahq/infra/internal/testing/database"
)

func TestServerCmd_LoadOptions(t *testing.T) {
Expand Down Expand Up @@ -126,7 +127,6 @@ enableSignup: false # default is true
sessionDuration: 3m
sessionExtensionDeadline: 1m

dbFile: /db/file
dbEncryptionKey: /this-is-the-path
dbEncryptionKeyProvider: the-provider
dbHost: the-host
Expand Down Expand Up @@ -199,7 +199,6 @@ users:

DBEncryptionKey: "/this-is-the-path",
DBEncryptionKeyProvider: "the-provider",
DBFile: "/db/file",
DBHost: "the-host",
DBPort: 5432,
DBParameters: "sslmode=require",
Expand Down Expand Up @@ -285,7 +284,6 @@ users:
setup: func(t *testing.T, cmd *cobra.Command) {
cmd.SetArgs([]string{
"--db-name", "database-name",
"--db-file", "/home/user/database-filename",
"--db-port", "12345",
"--db-host", "thehostname",
"--enable-telemetry=false",
Expand All @@ -297,7 +295,6 @@ users:
expected: func(t *testing.T) server.Options {
expected := defaultServerOptions(filepath.Join(dir, ".infra"))
expected.DBName = "database-name"
expected.DBFile = "/home/user/database-filename"
expected.DBHost = "thehostname"
expected.DBPort = 12345
expected.EnableTelemetry = false
Expand All @@ -317,9 +314,11 @@ users:
}

func TestServerCmd_WithSecretsConfig(t *testing.T) {
pgDriver := database.PostgresDriver(t, "cmd_server")
patchRunServer(t, noServerRun)

content := `
dbConnectionString: ` + pgDriver.DSN + `
addr:
http: "127.0.0.1:0"
https: "127.0.0.1:0"
Expand Down
5 changes: 0 additions & 5 deletions internal/server/authn/authn_method_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,6 @@ import (
func setupDB(t *testing.T) *data.DB {
t.Helper()
driver := database.PostgresDriver(t, "_authn")
if driver == nil {
lite, err := data.NewSQLiteDriver("file::memory:")
assert.NilError(t, err)
driver = &database.Driver{Dialector: lite}
}

patch.ModelsSymmetricKey(t)
db, err := data.NewDB(driver.Dialector, nil)
Expand Down
22 changes: 0 additions & 22 deletions internal/server/data/data.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,13 @@ import (
"database/sql/driver"
"errors"
"fmt"
"net/url"
"os"
"path"
"reflect"
"strings"
"time"
"unicode"

"github.com/jackc/pgconn"
"github.com/jackc/pgerrcode"
"gorm.io/driver/sqlite"
"gorm.io/gorm"

"github.com/infrahq/infra/internal"
Expand Down Expand Up @@ -235,24 +231,6 @@ func initialize(db *DB) error {
return nil
}

func NewSQLiteDriver(connection string) (gorm.Dialector, error) {
if !strings.HasPrefix(connection, "file::memory") {
if err := os.MkdirAll(path.Dir(connection), os.ModePerm); err != nil {
return nil, err
}
}
uri, err := url.Parse(connection)
if err != nil {
return nil, err
}
query := uri.Query()
query.Add("_journal_mode", "WAL")
uri.RawQuery = query.Encode()
connection = uri.String()

return sqlite.Open(connection), nil
}

func getDefaultSortFromType(t interface{}) string {
ty := reflect.TypeOf(t).Elem()
if _, ok := ty.FieldByName("Name"); ok {
Expand Down
23 changes: 3 additions & 20 deletions internal/server/data/data_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package data

import (
"context"
"os"
"testing"

"github.com/rs/zerolog"
Expand All @@ -29,29 +28,13 @@ func setupDB(t *testing.T, driver gorm.Dialector) *DB {
return db
}

var isEnvironmentCI = os.Getenv("CI") != ""

// postgresDriver requires postgres to be available in a CI environment, and
// marks the test as skipped when not in CI environment.
func postgresDriver(t *testing.T) gorm.Dialector {
driver := database.PostgresDriver(t, "")
switch {
case driver == nil && isEnvironmentCI:
t.Fatal("CI must test all drivers, set POSTGRESQL_CONNECTION")
case driver == nil:
t.Skip("Set POSTGRESQL_CONNECTION to test against postgresql")
}
return driver.Dialector
}

// runDBTests against all supported databases. Defaults to only sqlite locally,
// and all supported DBs in CI.
// runDBTests against all supported databases.
// Set POSTGRESQL_CONNECTION to a postgresql connection string to run tests
// against postgresql.
func runDBTests(t *testing.T, run func(t *testing.T, db *DB)) {
t.Run("postgres", func(t *testing.T) {
pgsql := postgresDriver(t)
db := setupDB(t, pgsql)
pgsql := database.PostgresDriver(t, "")
db := setupDB(t, pgsql.Dialector)
run(t, db)
db.Rollback()
})
Expand Down
7 changes: 5 additions & 2 deletions internal/server/data/migrations_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/infrahq/infra/internal/server/data/migrator"
"github.com/infrahq/infra/internal/server/data/schema"
"github.com/infrahq/infra/internal/server/models"
"github.com/infrahq/infra/internal/testing/database"
"github.com/infrahq/infra/internal/testing/patch"
"github.com/infrahq/infra/uid"
)
Expand Down Expand Up @@ -575,7 +576,7 @@ DELETE FROM settings WHERE id=24567;
var initialSchema string
runStep(t, "initial schema", func(t *testing.T) {
patch.ModelsSymmetricKey(t)
rawDB, err := newRawDB(postgresDriver(t))
rawDB, err := newRawDB(database.PostgresDriver(t, "").Dialector)
assert.NilError(t, err)

db := &DB{DB: rawDB}
Expand All @@ -589,7 +590,7 @@ DELETE FROM settings WHERE id=24567;
assert.NilError(t, err)
})

db, err := newRawDB(postgresDriver(t))
db, err := newRawDB(database.PostgresDriver(t, "").Dialector)
assert.NilError(t, err)
for i, tc := range testCases {
runStep(t, tc.label.Name, func(t *testing.T) {
Expand Down Expand Up @@ -646,6 +647,8 @@ type testCaseLabel struct {
Line string
}

var isEnvironmentCI = os.Getenv("CI") != ""
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this variable live closer to there it is used? I dont see it in migrations_test.go

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's used 6 lines below here. The diff looks weird because I removed it from a different place in the data package.


func dumpSchema(t *testing.T, conn string) string {
t.Helper()
if _, err := exec.LookPath("pg_dump"); err != nil {
Expand Down
20 changes: 6 additions & 14 deletions internal/server/data/migrator/migrator_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,15 @@ package migrator
import (
"database/sql"
"database/sql/driver"
"os"
"path/filepath"
"testing"

"gorm.io/driver/postgres"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gotest.tools/v3/assert"

"github.com/infrahq/infra/internal/testing/database"
)

type database struct {
type dbDriver struct {
dialect string
driver gorm.Dialector
}
Expand Down Expand Up @@ -361,16 +359,10 @@ func migrationCount(t *testing.T, db DB) (count int64) {
}

func runDBTests(t *testing.T, fn func(t *testing.T, db DB)) {
dir := t.TempDir()

databases := []database{
{dialect: "sqlite3", driver: sqlite.Open("file:" + filepath.Join(dir, "sqlite3.db"))},
}
databases := []dbDriver{}

if pg := os.Getenv("POSTGRESQL_CONNECTION"); pg != "" {
databases = append(databases, database{
dialect: "postgres", driver: postgres.Open(pg),
})
if pg := database.PostgresDriver(t, "_migrator"); pg != nil {
databases = append(databases, dbDriver{dialect: "postgres", driver: pg.Dialector})
}

for _, database := range databases {
Expand Down
4 changes: 3 additions & 1 deletion internal/server/data/sqlfunc_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"testing"

"gotest.tools/v3/assert"

"github.com/infrahq/infra/internal/testing/database"
)

func TestSQLUidStrToIntRoundTrip(t *testing.T) {
Expand All @@ -13,7 +15,7 @@ func TestSQLUidStrToIntRoundTrip(t *testing.T) {
intval int64
err string
}
db := setupDB(t, postgresDriver(t))
db := setupDB(t, database.PostgresDriver(t, "").Dialector)

run := func(t *testing.T, tc testCase) {
var i int64
Expand Down
5 changes: 0 additions & 5 deletions internal/server/middleware_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,6 @@ import (
func setupDB(t *testing.T) *data.DB {
t.Helper()
driver := database.PostgresDriver(t, "_server")
if driver == nil {
lite, err := data.NewSQLiteDriver("file::memory:")
assert.NilError(t, err)
driver = &database.Driver{Dialector: lite}
}

tpatch.ModelsSymmetricKey(t)
db, err := data.NewDB(driver.Dialector, nil)
Expand Down
9 changes: 4 additions & 5 deletions internal/server/models/encryption_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"github.com/infrahq/infra/internal/server/data"
"github.com/infrahq/infra/internal/server/models"
"github.com/infrahq/infra/internal/testing/database"
"github.com/infrahq/infra/internal/testing/patch"
"github.com/infrahq/infra/uid"
)
Expand All @@ -20,18 +21,16 @@ type StructForTesting struct {
func (s StructForTesting) Schema() string {
return `
CREATE TABLE struct_for_testings (
id integer PRIMARY KEY,
id bigint PRIMARY KEY,
a_secret text
);`
}

func TestEncryptedAtRest(t *testing.T) {
patch.ModelsSymmetricKey(t)

driver, err := data.NewSQLiteDriver("file::memory:")
assert.NilError(t, err)

db, err := data.NewDB(driver, nil)
pg := database.PostgresDriver(t, "_models")
db, err := data.NewDB(pg.Dialector, nil)
assert.NilError(t, err)

_, err = db.Exec(StructForTesting{}.Schema())
Expand Down
Loading