feat: migration from mongo to postgres data storage

.golangci.yml

@@ -99,6 +99,11 @@ issues:
         - goerr113
         - varnamelen
 
+    - path: internal/app/storage/migrations/
+      linters:
+        - gochecknoinits
+        - varnamelen
+
     - linters:
       - nolintlint
       text: "should be written without leading space"

cmd/migrate.go

@@ -0,0 +1,67 @@
+package cmd
+
+import (
+	"fmt"
+	"log"
+
+	// allow blank import to initiate migrations.
+	_ "github.com/formancehq/payments/internal/app/storage/migrations"
+	_ "github.com/lib/pq"
+
+	"github.com/pressly/goose/v3"
+	"github.com/spf13/cobra"
+)
+
+func newMigrate() *cobra.Command {
+	return &cobra.Command{
+		Use:   "migrate",
+		Short: "Run migrations",
+		RunE:  runMigrate,
+	}
+}
+
+// Usage: `go run cmd/main.go migrate --postgres-uri {uri} {command}`
+/*
+Commands:
+    up                   Migrate the DB to the most recent version available
+    up-by-one            Migrate the DB up by 1
+    up-to VERSION        Migrate the DB to a specific VERSION
+    down                 Roll back the version by 1
+    down-to VERSION      Roll back to a specific VERSION
+    redo                 Re-run the latest migration
+    reset                Roll back all migrations
+    status               Dump the migration status for the current DB
+    version              Print the current version of the database
+    create NAME [sql|go] Creates new migration file with the current timestamp
+    fix                  Apply sequential ordering to migrations
+*/
+
+func runMigrate(cmd *cobra.Command, args []string) error {
+	postgresURI := cmd.Flag(postgresURIFlag).Value.String()
+	if postgresURI == "" {
+		return fmt.Errorf("postgres uri is not set")
+	}
+
+	database, err := goose.OpenDBWithDriver("postgres", postgresURI)
+	if err != nil {
+		return fmt.Errorf("failed to open database: %w", err)
+	}
+
+	defer func() {
+		if err = database.Close(); err != nil {
+			log.Fatalf("failed to close DB: %v\n", err)
+		}
+	}()
+
+	if len(args) == 0 {
+		return fmt.Errorf("missing migration direction")
+	}
+
+	command := args[0]
+
+	if err = goose.Run(command, database, ".", args[1:]...); err != nil {
+		log.Printf("migrate %v: %v", command, err)
+	}
+
+	return nil
+}

cmd/root.go

@@ -35,12 +35,18 @@ func rootCommand() *cobra.Command {
 	root.AddCommand(version)
 
 	server := newServer()
-	root.AddCommand(server)
+	root.AddCommand(newServer())
+
+	migrate := newMigrate()
+	root.AddCommand(migrate)
 
 	root.PersistentFlags().Bool(debugFlag, false, "Debug mode")
+
+	migrate.Flags().String(postgresURIFlag, "postgres://localhost/payments", "PostgreSQL DB address")
+
 	server.Flags().BoolP("toggle", "t", false, "Help message for toggle")
-	server.Flags().String(mongodbURIFlag, "mongodb://localhost:27017", "MongoDB address")
-	server.Flags().String(mongodbDatabaseFlag, "payments", "MongoDB database name")
+	server.Flags().String(postgresURIFlag, "postgres://localhost/payments", "PostgreSQL DB address")
+	server.Flags().String(postgresDatabaseName, "payments", "PostgreSQL database name")
 	server.Flags().String(envFlag, "local", "Environment")
 	server.Flags().Bool(publisherKafkaEnabledFlag, false, "Publish write events to kafka")
 	server.Flags().StringSlice(publisherKafkaBrokerFlag, []string{}, "Kafka address is kafka enabled")

cmd/server.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/bombsimon/logrusr/v3"
 	"github.com/formancehq/payments/internal/app/api"
-	"github.com/formancehq/payments/internal/app/database"
+	"github.com/formancehq/payments/internal/app/storage"
 	"github.com/pkg/errors"
 	"go.opentelemetry.io/otel"
 
@@ -27,34 +27,27 @@ import (
 
 //nolint:gosec // false positive
 const (
-	mongodbURIFlag                       = "mongodb-uri"
-	mongodbDatabaseFlag                  = "mongodb-database"
-	otelTracesFlag                       = "otel-traces"
-	otelTracesExporterFlag               = "otel-traces-exporter"
-	otelTracesExporterJaegerEndpointFlag = "otel-traces-exporter-jaeger-endpoint"
-	otelTracesExporterJaegerUserFlag     = "otel-traces-exporter-jaeger-user"
-	otelTracesExporterJaegerPasswordFlag = "otel-traces-exporter-jaeger-password"
-	otelTracesExporterOTLPModeFlag       = "otel-traces-exporter-otlp-mode"
-	otelTracesExporterOTLPEndpointFlag   = "otel-traces-exporter-otlp-endpoint"
-	otelTracesExporterOTLPInsecureFlag   = "otel-traces-exporter-otlp-insecure"
-	envFlag                              = "env"
-	publisherKafkaEnabledFlag            = "publisher-kafka-enabled"
-	publisherKafkaBrokerFlag             = "publisher-kafka-broker"
-	publisherKafkaSASLEnabled            = "publisher-kafka-sasl-enabled"
-	publisherKafkaSASLUsername           = "publisher-kafka-sasl-username"
-	publisherKafkaSASLPassword           = "publisher-kafka-sasl-password"
-	publisherKafkaSASLMechanism          = "publisher-kafka-sasl-mechanism"
-	publisherKafkaSASLScramSHASize       = "publisher-kafka-sasl-scram-sha-size"
-	publisherKafkaTLSEnabled             = "publisher-kafka-tls-enabled"
-	publisherTopicMappingFlag            = "publisher-topic-mapping"
-	publisherHTTPEnabledFlag             = "publisher-http-enabled"
-	authBasicEnabledFlag                 = "auth-basic-enabled"
-	authBasicCredentialsFlag             = "auth-basic-credentials"
-	authBearerEnabledFlag                = "auth-bearer-enabled"
-	authBearerIntrospectURLFlag          = "auth-bearer-introspect-url"
-	authBearerAudienceFlag               = "auth-bearer-audience"
-	authBearerAudiencesWildcardFlag      = "auth-bearer-audiences-wildcard"
-	authBearerUseScopesFlag              = "auth-bearer-use-scopes"
+	postgresURIFlag                 = "postgres-uri"
+	postgresDatabaseName            = "postgres-database-name"
+	otelTracesFlag                  = "otel-traces"
+	envFlag                         = "env"
+	publisherKafkaEnabledFlag       = "publisher-kafka-enabled"
+	publisherKafkaBrokerFlag        = "publisher-kafka-broker"
+	publisherKafkaSASLEnabled       = "publisher-kafka-sasl-enabled"
+	publisherKafkaSASLUsername      = "publisher-kafka-sasl-username"
+	publisherKafkaSASLPassword      = "publisher-kafka-sasl-password"
+	publisherKafkaSASLMechanism     = "publisher-kafka-sasl-mechanism"
+	publisherKafkaSASLScramSHASize  = "publisher-kafka-sasl-scram-sha-size"
+	publisherKafkaTLSEnabled        = "publisher-kafka-tls-enabled"
+	publisherTopicMappingFlag       = "publisher-topic-mapping"
+	publisherHTTPEnabledFlag        = "publisher-http-enabled"
+	authBasicEnabledFlag            = "auth-basic-enabled"
+	authBasicCredentialsFlag        = "auth-basic-credentials"
+	authBearerEnabledFlag           = "auth-bearer-enabled"
+	authBearerIntrospectURLFlag     = "auth-bearer-introspect-url"
+	authBearerAudienceFlag          = "auth-bearer-audience"
+	authBearerAudiencesWildcardFlag = "auth-bearer-audiences-wildcard"
+	authBearerUseScopesFlag         = "auth-bearer-use-scopes"
 
 	serviceName = "Payments"
 )
@@ -156,17 +149,17 @@ func setLogger() {
 }
 
 func prepareDatabaseOptions() (fx.Option, error) {
-	mongodbURI := viper.GetString(mongodbURIFlag)
-	if mongodbURI == "" {
-		return nil, errors.New("missing mongodb uri")
+	postgresURI := viper.GetString(postgresURIFlag)
+	if postgresURI == "" {
+		return nil, errors.New("missing postgres uri")
 	}
 
-	mongodbDatabase := viper.GetString(mongodbDatabaseFlag)
-	if mongodbDatabase == "" {
-		return nil, errors.New("missing mongodb database name")
+	postgresDBName := viper.GetString(postgresDatabaseName)
+	if postgresDBName == "" {
+		return nil, errors.New("missing postgres DB name")
 	}
 
-	return database.MongoModule(mongodbURI, mongodbDatabase), nil
+	return storage.Module(postgresURI, postgresDBName), nil
 }
 
 func topicsMapping() map[string]string {

go.mod

@@ -13,43 +13,40 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/iancoleman/strcase v0.2.0
-	github.com/ory/dockertest v3.3.5+incompatible
+	github.com/jackc/pgx/v5 v5.2.0
+	github.com/lib/pq v1.10.6
 	github.com/pborman/uuid v1.2.1
 	github.com/pkg/errors v0.9.1
+	github.com/pressly/goose/v3 v3.7.0
 	github.com/rs/cors v1.8.2
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/afero v1.9.2
 	github.com/spf13/cobra v1.5.0
 	github.com/spf13/viper v1.12.0
 	github.com/stretchr/testify v1.8.1
 	github.com/stripe/stripe-go/v72 v72.122.0
+	github.com/uptrace/bun v1.1.9
+	github.com/uptrace/bun/dialect/pgdialect v1.1.9
+	github.com/uptrace/bun/extra/bunotel v1.1.9
 	github.com/uptrace/opentelemetry-go-extra/otellogrus v0.1.15
 	github.com/xdg-go/scram v1.1.1
 	go.mongodb.org/mongo-driver v1.10.3
 	go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.34.0
-	go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver/mongo/otelmongo v0.36.4
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.4
 	go.opentelemetry.io/otel v1.11.1
 	go.opentelemetry.io/otel/trace v1.11.1
 	go.uber.org/dig v1.15.0
 	go.uber.org/fx v1.18.1
-	golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde
+	golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7
 )
 
 require (
-	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
-	github.com/Microsoft/go-winio v0.5.1 // indirect
-	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
 	github.com/ThreeDotsLabs/watermill-http v1.1.4 // indirect
 	github.com/ThreeDotsLabs/watermill-kafka/v2 v2.2.2 // indirect
 	github.com/ajg/form v1.5.1 // indirect
-	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cenkalti/backoff/v4 v4.2.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/containerd/continuity v0.2.2 // indirect
 	github.com/dgraph-io/ristretto v0.1.0 // indirect
-	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/eapache/go-resiliency v1.3.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
@@ -65,28 +62,26 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.14.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/gofork v1.7.6 // indirect
 	github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/klauspost/compress v1.15.9 // indirect
-	github.com/lib/pq v1.10.5 // indirect
 	github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/montanaflynn/stats v0.6.6 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.2 // indirect
-	github.com/opencontainers/runc v1.1.0 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.3 // indirect
 	github.com/pierrec/lz4/v4 v4.1.15 // indirect
@@ -96,7 +91,11 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.4.1 // indirect
+	github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
+	github.com/uptrace/opentelemetry-go-extra/otelsql v0.1.17 // indirect
 	github.com/uptrace/opentelemetry-go-extra/otelutil v0.1.15 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/stringprep v1.0.3 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
@@ -114,7 +113,7 @@ require (
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.23.0 // indirect
-	golang.org/x/crypto v0.0.0-20220824171710-5757bc0c5503 // indirect
+	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
 	golang.org/x/net v0.2.0 // indirect
 	golang.org/x/sys v0.2.0 // indirect
 	golang.org/x/text v0.4.0 // indirect
@@ -124,5 +123,4 @@ require (
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	gotest.tools v2.2.0+incompatible // indirect
 )