Merge branch 'main' into fulghum/binlog-dev-2

go/cmd/dolt/commands/engine/sqlengine.go

@@ -44,6 +44,7 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/mysql_file_handler"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/statsnoms"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/statspro"
+	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/writer"
 	"github.com/dolthub/dolt/go/libraries/utils/config"
 	"github.com/dolthub/dolt/go/store/types"
 )
@@ -397,7 +398,7 @@ func sqlContextFactory() contextFactory {
 // doltSessionFactory returns a sessionFactory that creates a new DoltSession
 func doltSessionFactory(pro *dsqle.DoltDatabaseProvider, statsPro sql.StatsProvider, config config.ReadWriteConfig, bc *branch_control.Controller, autocommit bool) sessionFactory {
 	return func(mysqlSess *sql.BaseSession, provider sql.DatabaseProvider) (*dsess.DoltSession, error) {
-		doltSession, err := dsess.NewDoltSession(mysqlSess, pro, config, bc, statsPro)
+		doltSession, err := dsess.NewDoltSession(mysqlSess, pro, config, bc, statsPro, writer.NewWriteSession)
 		if err != nil {
 			return nil, err
 		}

go/cmd/dolt/commands/filter-branch.go

@@ -34,6 +34,7 @@ import (
 	"github.com/dolthub/dolt/go/libraries/doltcore/rebase"
 	dsqle "github.com/dolthub/dolt/go/libraries/doltcore/sqle"
 	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
+	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/writer"
 	"github.com/dolthub/dolt/go/libraries/doltcore/table/editor"
 	"github.com/dolthub/dolt/go/libraries/utils/argparser"
 	"github.com/dolthub/dolt/go/store/hash"
@@ -292,7 +293,7 @@ func rebaseSqlEngine(ctx context.Context, dEnv *env.DoltEnv, cm *doltdb.Commit)
 		return nil, nil, err
 	}
 
-	sess := dsess.DefaultSession(pro)
+	sess := dsess.DefaultSession(pro, writer.NewWriteSession)
 
 	sqlCtx := sql.NewContext(ctx, sql.WithSession(sess))
 	err = sqlCtx.SetSessionVariable(sqlCtx, sql.AutoCommitSessionVar, false)

go/cmd/dolt/commands/remote.go

@@ -16,11 +16,17 @@ package commands
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
+	"fmt"
 	"strings"
 
+	"github.com/dolthub/go-mysql-server/sql"
+	"github.com/dolthub/go-mysql-server/sql/types"
+	"github.com/gocraft/dbr/v2"
+	"github.com/gocraft/dbr/v2/dialect"
+
 	"github.com/dolthub/dolt/go/cmd/dolt/cli"
+	"github.com/dolthub/dolt/go/cmd/dolt/commands/engine"
 	"github.com/dolthub/dolt/go/cmd/dolt/errhand"
 	eventsapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi/v1alpha1"
 	"github.com/dolthub/dolt/go/libraries/doltcore/dbfactory"
@@ -109,80 +115,92 @@ func (cmd RemoteCmd) Exec(ctx context.Context, commandStr string, args []string,
 	help, usage := cli.HelpAndUsagePrinters(cli.CommandDocsForCommandString(commandStr, remoteDocs, ap))
 	apr := cli.ParseArgsOrDie(ap, args, help)
 
-	var verr errhand.VerboseError
+	queryist, sqlCtx, closeFunc, err := cliCtx.QueryEngine(ctx)
+	if err != nil {
+		return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
+	}
+	if closeFunc != nil {
+		defer closeFunc()
+	}
 
+	var verr errhand.VerboseError
 	switch {
 	case apr.NArg() == 0:
-		verr = printRemotes(dEnv, apr)
+		verr = printRemotes(sqlCtx, queryist, apr)
 	case apr.Arg(0) == addRemoteId:
-		verr = addRemote(dEnv, apr)
-	case apr.Arg(0) == removeRemoteId:
-		verr = removeRemote(ctx, dEnv, apr)
-	case apr.Arg(0) == removeRemoteShortId:
-		verr = removeRemote(ctx, dEnv, apr)
+		verr = addRemote(sqlCtx, queryist, dEnv, apr)
+	case apr.Arg(0) == removeRemoteId, apr.Arg(0) == removeRemoteShortId:
+		verr = removeRemote(sqlCtx, queryist, apr)
 	default:
 		verr = errhand.BuildDError("").SetPrintUsage().Build()
 	}
 
 	return HandleVErrAndExitCode(verr, usage)
 }
 
-func removeRemote(ctx context.Context, dEnv *env.DoltEnv, apr *argparser.ArgParseResults) errhand.VerboseError {
+func removeRemote(sqlCtx *sql.Context, qureyist cli.Queryist, apr *argparser.ArgParseResults) errhand.VerboseError {
 	if apr.NArg() != 2 {
 		return errhand.BuildDError("").SetPrintUsage().Build()
 	}
+	toRemove := strings.TrimSpace(apr.Arg(1))
 
-	old := strings.TrimSpace(apr.Arg(1))
-	err := dEnv.RemoveRemote(ctx, old)
-
-	switch err {
-	case nil:
-		return nil
-	case env.ErrFailedToWriteRepoState:
-		return errhand.BuildDError("error: failed to save change to repo state").AddCause(err).Build()
-	case env.ErrFailedToDeleteRemote:
-		return errhand.BuildDError("error: failed to delete remote tracking ref").AddCause(err).Build()
-	case env.ErrFailedToReadFromDb:
-		return errhand.BuildDError("error: failed to read from db").AddCause(err).Build()
-	case env.ErrRemoteNotFound:
-		return errhand.BuildDError("error: unknown remote: '%s' ", old).Build()
-	default:
-		return errhand.BuildDError("error: unknown error").AddCause(err).Build()
+	err := callSQLRemoteRemove(sqlCtx, qureyist, toRemove)
+	if err != nil {
+		return errhand.BuildDError("error: Unable to remove remote.").AddCause(err).Build()
 	}
+	return nil
 }
 
-func addRemote(dEnv *env.DoltEnv, apr *argparser.ArgParseResults) errhand.VerboseError {
+func addRemote(sqlCtx *sql.Context, queryist cli.Queryist, dEnv *env.DoltEnv, apr *argparser.ArgParseResults) errhand.VerboseError {
 	if apr.NArg() != 3 {
 		return errhand.BuildDError("").SetPrintUsage().Build()
 	}
 
 	remoteName := strings.TrimSpace(apr.Arg(1))
 
 	remoteUrl := apr.Arg(2)
+
 	scheme, absRemoteUrl, err := env.GetAbsRemoteUrl(dEnv.FS, dEnv.Config, remoteUrl)
 	if err != nil {
 		return errhand.BuildDError("error: '%s' is not valid.", remoteUrl).AddCause(err).Build()
 	}
-
 	params, verr := parseRemoteArgs(apr, scheme, absRemoteUrl)
 	if verr != nil {
 		return verr
 	}
 
-	r := env.NewRemote(remoteName, remoteUrl, params)
-	err = dEnv.AddRemote(r)
+	if len(params) == 0 {
+		err := callSQLRemoteAdd(sqlCtx, queryist, remoteName, remoteUrl)
+		if err != nil {
+			return errhand.BuildDError("error: Unable to add remote.").AddCause(err).Build()
+		}
+	} else {
+		// We only support adding remotes with parameters in the local configuration
+		if _, ok := queryist.(*engine.SqlEngine); !ok {
+			return errhand.BuildDError("error: remote add failed. sql-server running while attempting to use advanced remote parameters. Stop server and re-run").Build()
+		}
+		return addRemoteLocaly(remoteName, absRemoteUrl, params, dEnv)
+	}
+	return nil
+}
+
+// addRemoteLocal adds a remote to the local configuration, which should only be used in the event that there
+// are AWS/GCP/OSS parameters. These are not supported in the SQL interface
+func addRemoteLocaly(remoteName, remoteUrl string, params map[string]string, dEnv *env.DoltEnv) errhand.VerboseError {
+	rmot := env.NewRemote(remoteName, remoteUrl, params)
+	err := dEnv.AddRemote(rmot)
 
 	switch err {
 	case nil:
 		return nil
 	case env.ErrRemoteAlreadyExists:
-		return errhand.BuildDError("error: a remote named '%s' already exists.", r.Name).AddDetails("remove it before running this command again").Build()
+		return errhand.BuildDError("error: a remote named '%s' already exists.", rmot.Name).AddDetails("remove it before running this command again").Build()
 	case env.ErrRemoteNotFound:
-		return errhand.BuildDError("error: unknown remote: '%s' ", r.Name).Build()
+		return errhand.BuildDError("error: unknown remote: '%s' ", rmot.Name).Build()
 	case env.ErrInvalidRemoteURL:
-		return errhand.BuildDError("error: '%s' is not valid.", r.Url).AddCause(err).Build()
+		return errhand.BuildDError("error: '%s' is not valid.", rmot.Url).AddCause(err).Build()
 	case env.ErrInvalidRemoteName:
-		return errhand.BuildDError("error: invalid remote name: " + r.Name).Build()
+		return errhand.BuildDError("error: invalid remote name: " + rmot.Name).Build()
 	default:
 		return errhand.BuildDError("error: Unable to save changes.").AddCause(err).Build()
 	}
@@ -207,21 +225,98 @@ func parseRemoteArgs(apr *argparser.ArgParseResults, scheme, remoteUrl string) (
 	return params, nil
 }
 
-func printRemotes(dEnv *env.DoltEnv, apr *argparser.ArgParseResults) errhand.VerboseError {
-	remotes, err := dEnv.GetRemotes()
+// callSQLRemoteAdd calls the SQL function `call `dolt_remote('add', remoteName, remoteUrl)`
+func callSQLRemoteAdd(sqlCtx *sql.Context, queryist cli.Queryist, remoteName, remoteUrl string) error {
+	qry, err := dbr.InterpolateForDialect("call dolt_remote('add', ?, ?)", []interface{}{remoteName, remoteUrl}, dialect.MySQL)
+	if err != nil {
+		return err
+	}
+
+	_, err = GetRowsForSql(queryist, sqlCtx, qry)
+	return err
+}
 
+// callSQLRemoteRemove calls the SQL function `call `dolt_remote('remove', remoteName)`
+func callSQLRemoteRemove(sqlCtxe *sql.Context, queryist cli.Queryist, remoteName string) error {
+	qry, err := dbr.InterpolateForDialect("call dolt_remote('remove', ?)", []interface{}{remoteName}, dialect.MySQL)
+	if err != nil {
+		return err
+	}
+
+	_, err = GetRowsForSql(queryist, sqlCtxe, qry)
+	return err
+}
+
+type remote struct {
+	Name   string
+	Url    string
+	Params string
+}
+
+func getRemotesSQL(sqlCtx *sql.Context, queryist cli.Queryist) ([]remote, error) {
+	qry := "select name,url,params from dolt_remotes"
+	rows, err := GetRowsForSql(queryist, sqlCtx, qry)
+	if err != nil {
+		return nil, err
+	}
+
+	remotes := make([]remote, 0, len(rows))
+	for _, r := range rows {
+		name, ok := r[0].(string)
+		if !ok {
+			return nil, fmt.Errorf("invalid remote name")
+		}
+
+		url, ok := r[1].(string)
+		if !ok {
+			return nil, fmt.Errorf("invalid remote url")
+		}
+
+		params, err := getJsonAsString(sqlCtx, r[2])
+		if err != nil {
+			return nil, fmt.Errorf("invalid params")
+		}
+
+		remotes = append(remotes, remote{
+			Name:   name,
+			Url:    url,
+			Params: params,
+		})
+	}
+
+	return remotes, nil
+}
+
+// getJsonAsString returns a string representation of the given interface{}, which can either be a string or a JSONDocument.
+// If it is a string, it gets returned as is. If it is a JSONDocument, it gets converted to a string.
+// SQLEngine and the remote connection behave a little differently here, which is the reason for needing this.
+func getJsonAsString(sqlCtx *sql.Context, params interface{}) (string, error) {
+	switch p := params.(type) {
+	case string:
+		return p, nil
+	case sql.JSONWrapper:
+		json, err := types.StringifyJSON(p)
+		if err != nil {
+			return "", err
+		}
+		if json == "{}" {
+			return "", nil
+		}
+		return json, nil
+	default:
+		return "", fmt.Errorf("unexpected interface{} type: %T", p)
+	}
+}
+
+func printRemotes(sqlCtx *sql.Context, queryist cli.Queryist, apr *argparser.ArgParseResults) errhand.VerboseError {
+	remotes, err := getRemotesSQL(sqlCtx, queryist)
 	if err != nil {
 		return errhand.BuildDError("Unable to get remotes from the local directory").AddCause(err).Build()
 	}
 
-	for _, r := range remotes.Snapshot() {
+	for _, r := range remotes {
 		if apr.Contains(cli.VerboseFlag) {
-			paramStr := make([]byte, 0)
-			if len(r.Params) > 0 {
-				paramStr, _ = json.Marshal(r.Params)
-			}
-
-			cli.Printf("%s %s %s\n", r.Name, r.Url, paramStr)
+			cli.Printf("%s %s %s\n", r.Name, r.Url, r.Params)
 		} else {
 			cli.Println(r.Name)
 		}

go/cmd/dolt/dolt.go

@@ -130,7 +130,6 @@ var commandsWithoutCliCtx = []cli.Command{
 	admin.Commands,
 	sqlserver.SqlServerCmd{VersionStr: doltversion.Version},
 	commands.CloneCmd{},
-	commands.RemoteCmd{},
 	commands.BackupCmd{},
 	commands.LoginCmd{},
 	credcmds.Commands,

go/cmd/dolt/doltversion/version.go

@@ -16,5 +16,5 @@
 package doltversion
 
 const (
-	Version = "1.38.0"
+	Version = "1.38.2"
 )

go/go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/dolthub/fslock v0.0.3
 	github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488
 	github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81
-	github.com/dolthub/vitess v0.0.0-20240515231645-966f8cb81cfe
+	github.com/dolthub/vitess v0.0.0-20240521104846-e778edc6deb3
 	github.com/dustin/go-humanize v1.0.1
 	github.com/fatih/color v1.13.0
 	github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
@@ -57,7 +57,7 @@ require (
 	github.com/cespare/xxhash v1.1.0
 	github.com/creasty/defaults v1.6.0
 	github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
-	github.com/dolthub/go-mysql-server v0.18.2-0.20240516060849-27785083f114
+	github.com/dolthub/go-mysql-server v0.18.2-0.20240521114548-05792f2565eb
 	github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63
 	github.com/dolthub/swiss v0.1.0
 	github.com/goccy/go-json v0.10.2
@@ -168,4 +168,4 @@ require (
 
 replace github.com/dolthub/dolt/go/gen/proto/dolt/services/eventsapi => ./gen/proto/dolt/services/eventsapi
 
-go 1.22
+go 1.22.2

go/go.sum

@@ -183,8 +183,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
 github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
 github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e h1:kPsT4a47cw1+y/N5SSCkma7FhAPw7KeGmD6c9PBZW9Y=
 github.com/dolthub/go-icu-regex v0.0.0-20230524105445-af7e7991c97e/go.mod h1:KPUcpx070QOfJK1gNe0zx4pA5sicIK1GMikIGLKC168=
-github.com/dolthub/go-mysql-server v0.18.2-0.20240516060849-27785083f114 h1:2BFdyCFqZ8x2iw/dvFBAqhu+iB0GuFTO+7wNAZKS7hc=
-github.com/dolthub/go-mysql-server v0.18.2-0.20240516060849-27785083f114/go.mod h1:zj+j1Gt3iiUWDkZ5qpV3C0up2YS+3gmr1+IGsinM+XI=
+github.com/dolthub/go-mysql-server v0.18.2-0.20240521114548-05792f2565eb h1:Hw0R/XRKHbfWooNYfXSMSVHmEPdZZX2akZ+wRpJTGAU=
+github.com/dolthub/go-mysql-server v0.18.2-0.20240521114548-05792f2565eb/go.mod h1:6oKOQFcUXF2aKjsBIq4/5mqCzNZkSTGCIcf6o96fIdY=
 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI=
 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q=
 github.com/dolthub/ishell v0.0.0-20221214210346-d7db0b066488 h1:0HHu0GWJH0N6a6keStrHhUAK5/o9LVfkh44pvsV4514=
@@ -197,8 +197,8 @@ github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81 h1:7/v8q9X
 github.com/dolthub/sqllogictest/go v0.0.0-20201107003712-816f3ae12d81/go.mod h1:siLfyv2c92W1eN/R4QqG/+RjjX5W2+gCTRjZxBjI3TY=
 github.com/dolthub/swiss v0.1.0 h1:EaGQct3AqeP/MjASHLiH6i4TAmgbG/c4rA6a1bzCOPc=
 github.com/dolthub/swiss v0.1.0/go.mod h1:BeucyB08Vb1G9tumVN3Vp/pyY4AMUnr9p7Rz7wJ7kAQ=
-github.com/dolthub/vitess v0.0.0-20240515231645-966f8cb81cfe h1:40//2vXX3xDDud5bgrw6pcNHSZBexqvdlHvIoSA2dyA=
-github.com/dolthub/vitess v0.0.0-20240515231645-966f8cb81cfe/go.mod h1:uBvlRluuL+SbEWTCZ68o0xvsdYZER3CEG/35INdzfJM=
+github.com/dolthub/vitess v0.0.0-20240521104846-e778edc6deb3 h1:2Q6j9yI1XjDH5UOY0EAhaXoGIdN11jItAzpRGLl6O8o=
+github.com/dolthub/vitess v0.0.0-20240521104846-e778edc6deb3/go.mod h1:uBvlRluuL+SbEWTCZ68o0xvsdYZER3CEG/35INdzfJM=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=