From ba07f4c3a9206dae4fbd7125f645cd5ea483642f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20Olav=20L=C3=B8ite?= Date: Fri, 25 Mar 2022 15:44:04 +0100 Subject: [PATCH 1/3] feat: add option use execParams instead of prepare Adds an option to skip the DescribeStatement step when the statement cache has been disabled. This reduces the number of round trips to the server by one. --- .gitignore | 1 + conn.go | 88 ++++++++++++++++++++++++++++++++++++++++++++++------ conn_test.go | 11 ++++++- 3 files changed, 90 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 39175a965..7500e87e8 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ _testmain.go *.exe .envrc +.idea diff --git a/conn.go b/conn.go index 4bc5db6bb..f5753809f 100644 --- a/conn.go +++ b/conn.go @@ -37,6 +37,12 @@ type ConnConfig struct { // QueryExOptions.SimpleProtocol. PreferSimpleProtocol bool + // PreferExecParams disables the describe statement round trip when there is no statement cache in use. Instead, pgx + // will execute the statement directly in extended mode by sending the messages Parse-Bind-Execute directly. This + // setting only has effect if statement caching has been disabled, either by setting BuildStatementCache to nil or + // by setting 'statement_cache_capacity=0' in the connection string. + PreferExecParams bool + createdByParseConfig bool // Used to enforce created by ParseConfig rule. } @@ -127,6 +133,9 @@ func ConnectConfig(ctx context.Context, connConfig *ConnConfig) (*Conn, error) { // // prefer_simple_protocol // Possible values: "true" and "false". Use the simple protocol instead of extended protocol. Default: false +// +// prefer_exec_params +// Possible values: "true" and "false". Skip DescribeStatement when statement cache is disabled. Default: true func ParseConfig(connString string) (*ConnConfig, error) { config, err := pgconn.ParseConfig(connString) if err != nil { @@ -173,12 +182,23 @@ func ParseConfig(connString string) (*ConnConfig, error) { } } + preferExecParams := true + if s, ok := config.RuntimeParams["prefer_exec_params"]; ok { + delete(config.RuntimeParams, "prefer_exec_params") + if b, err := strconv.ParseBool(s); err == nil { + preferExecParams = b + } else { + return nil, fmt.Errorf("invalid prefer_exec_params: %v", err) + } + } + connConfig := &ConnConfig{ Config: *config, createdByParseConfig: true, LogLevel: LogLevelInfo, BuildStatementCache: buildStatementCache, PreferSimpleProtocol: preferSimpleProtocol, + PreferExecParams: preferExecParams, connString: connString, } @@ -440,11 +460,23 @@ optionLoop: return c.execPrepared(ctx, sd, arguments) } - sd, err := c.Prepare(ctx, "", sql) - if err != nil { - return nil, err + return c.execUnprepared(ctx, sql, arguments) + + //sd, err := c.Prepare(ctx, "", sql) + //if err != nil { + // return nil, err + //} + //return c.execPrepared(ctx, sd, arguments) +} + +func (c *Conn) paramOIDsFromArguments(arguments []interface{}) []uint32 { + paramOIDs := make([]uint32, len(arguments)) + for i, arg := range arguments { + if dt, ok := c.ConnInfo().DataTypeForValue(arg); ok { + paramOIDs[i] = dt.OID + } } - return c.execPrepared(ctx, sd, arguments) + return paramOIDs } func (c *Conn) execSimpleProtocol(ctx context.Context, sql string, arguments []interface{}) (commandTag pgconn.CommandTag, err error) { @@ -489,6 +521,21 @@ func (c *Conn) execParamsAndPreparedPrefix(sd *pgconn.StatementDescription, argu return nil } +func (c *Conn) execUnprepared(ctx context.Context, sql string, arguments []interface{}) (pgconn.CommandTag, error) { + sd := &pgconn.StatementDescription{ + SQL: sql, + ParamOIDs: c.paramOIDsFromArguments(arguments), + } + err := c.execParamsAndPreparedPrefix(sd, arguments) + if err != nil { + return nil, err + } + + result := c.pgConn.ExecParams(ctx, sd.SQL, c.eqb.paramValues, sd.ParamOIDs, c.eqb.paramFormats, c.eqb.resultFormats).Read() + c.eqb.Reset() // Allow c.eqb internal memory to be GC'ed as soon as possible. + return result.CommandTag, result.Err +} + func (c *Conn) execParams(ctx context.Context, sd *pgconn.StatementDescription, arguments []interface{}) (pgconn.CommandTag, error) { err := c.execParamsAndPreparedPrefix(sd, arguments) if err != nil { @@ -601,11 +648,16 @@ optionLoop: return rows, rows.err } } else { - sd, err = c.pgConn.Prepare(ctx, "", sql, nil) - if err != nil { - rows.fatal(err) - return rows, rows.err + sd = &pgconn.StatementDescription{ + SQL: sql, + ParamOIDs: c.paramOIDsFromArguments(args), } + + //sd, err = c.pgConn.Prepare(ctx, "", sql, nil) + //if err != nil { + // rows.fatal(err) + // return rows, rows.err + //} } } if len(sd.ParamOIDs) != len(args) { @@ -644,7 +696,7 @@ optionLoop: resultFormats = c.eqb.resultFormats } - if c.stmtcache != nil && c.stmtcache.Mode() == stmtcache.ModeDescribe { + if c.stmtcache == nil || (c.stmtcache != nil && c.stmtcache.Mode() == stmtcache.ModeDescribe) { rows.resultReader = c.pgConn.ExecParams(ctx, sql, c.eqb.paramValues, sd.ParamOIDs, c.eqb.paramFormats, resultFormats) } else { rows.resultReader = c.pgConn.ExecPrepared(ctx, sd.Name, c.eqb.paramValues, c.eqb.paramFormats, resultFormats) @@ -655,6 +707,24 @@ optionLoop: return rows, rows.err } +func (c *Conn) queryUnprepared(ctx context.Context, rows *connRows, sql string, args ...interface{}) (Rows, error) { + sd := &pgconn.StatementDescription{ + SQL: sql, + ParamOIDs: c.paramOIDsFromArguments(args), + } + err := c.execParamsAndPreparedPrefix(sd, args) + if err != nil { + return nil, err + } + + rows.sql = sql + rows.resultReader = c.pgConn.ExecParams(ctx, sql, c.eqb.paramValues, sd.ParamOIDs, c.eqb.paramFormats, c.eqb.resultFormats) + + c.eqb.Reset() // Allow c.eqb internal memory to be GC'ed as soon as possible. + + return rows, rows.err +} + // QueryRow is a convenience wrapper over Query. Any error that occurs while // querying is deferred until calling Scan on the returned Row. That Row will // error with ErrNoRows if no rows are returned. diff --git a/conn_test.go b/conn_test.go index beddcdcd5..74377f8f0 100644 --- a/conn_test.go +++ b/conn_test.go @@ -328,10 +328,17 @@ func TestExecStatementCacheModes(t *testing.T) { tests := []struct { name string buildStatementCache pgx.BuildStatementCacheFunc + preferExecParams bool }{ { - name: "disabled", + name: "disabled - execPrepared", buildStatementCache: nil, + preferExecParams: false, + }, + { + name: "disabled - execParams", + buildStatementCache: nil, + preferExecParams: true, }, { name: "prepare", @@ -344,12 +351,14 @@ func TestExecStatementCacheModes(t *testing.T) { buildStatementCache: func(conn *pgconn.PgConn) stmtcache.Cache { return stmtcache.New(conn, stmtcache.ModeDescribe, 32) }, + preferExecParams: false, }, } for _, tt := range tests { func() { config.BuildStatementCache = tt.buildStatementCache + config.PreferExecParams = tt.preferExecParams conn := mustConnect(t, config) defer closeConn(t, conn) From 5f849bfd8c5929e33330c0175bfa2945441397f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20Olav=20L=C3=B8ite?= Date: Fri, 25 Mar 2022 15:52:39 +0100 Subject: [PATCH 2/3] chore: cleanup --- conn.go | 65 +++++++++++++++++---------------------------------------- 1 file changed, 19 insertions(+), 46 deletions(-) diff --git a/conn.go b/conn.go index f5753809f..a31bce0a2 100644 --- a/conn.go +++ b/conn.go @@ -460,13 +460,19 @@ optionLoop: return c.execPrepared(ctx, sd, arguments) } - return c.execUnprepared(ctx, sql, arguments) + if c.config.PreferExecParams { + sd := &pgconn.StatementDescription{ + SQL: sql, + ParamOIDs: c.paramOIDsFromArguments(arguments), + } + return c.execParams(ctx, sd, arguments) + } - //sd, err := c.Prepare(ctx, "", sql) - //if err != nil { - // return nil, err - //} - //return c.execPrepared(ctx, sd, arguments) + sd, err := c.Prepare(ctx, "", sql) + if err != nil { + return nil, err + } + return c.execPrepared(ctx, sd, arguments) } func (c *Conn) paramOIDsFromArguments(arguments []interface{}) []uint32 { @@ -521,21 +527,6 @@ func (c *Conn) execParamsAndPreparedPrefix(sd *pgconn.StatementDescription, argu return nil } -func (c *Conn) execUnprepared(ctx context.Context, sql string, arguments []interface{}) (pgconn.CommandTag, error) { - sd := &pgconn.StatementDescription{ - SQL: sql, - ParamOIDs: c.paramOIDsFromArguments(arguments), - } - err := c.execParamsAndPreparedPrefix(sd, arguments) - if err != nil { - return nil, err - } - - result := c.pgConn.ExecParams(ctx, sd.SQL, c.eqb.paramValues, sd.ParamOIDs, c.eqb.paramFormats, c.eqb.resultFormats).Read() - c.eqb.Reset() // Allow c.eqb internal memory to be GC'ed as soon as possible. - return result.CommandTag, result.Err -} - func (c *Conn) execParams(ctx context.Context, sd *pgconn.StatementDescription, arguments []interface{}) (pgconn.CommandTag, error) { err := c.execParamsAndPreparedPrefix(sd, arguments) if err != nil { @@ -647,17 +638,17 @@ optionLoop: rows.fatal(err) return rows, rows.err } - } else { + } else if c.config.PreferExecParams { sd = &pgconn.StatementDescription{ SQL: sql, ParamOIDs: c.paramOIDsFromArguments(args), } - - //sd, err = c.pgConn.Prepare(ctx, "", sql, nil) - //if err != nil { - // rows.fatal(err) - // return rows, rows.err - //} + } else { + sd, err = c.pgConn.Prepare(ctx, "", sql, nil) + if err != nil { + rows.fatal(err) + return rows, rows.err + } } } if len(sd.ParamOIDs) != len(args) { @@ -707,24 +698,6 @@ optionLoop: return rows, rows.err } -func (c *Conn) queryUnprepared(ctx context.Context, rows *connRows, sql string, args ...interface{}) (Rows, error) { - sd := &pgconn.StatementDescription{ - SQL: sql, - ParamOIDs: c.paramOIDsFromArguments(args), - } - err := c.execParamsAndPreparedPrefix(sd, args) - if err != nil { - return nil, err - } - - rows.sql = sql - rows.resultReader = c.pgConn.ExecParams(ctx, sql, c.eqb.paramValues, sd.ParamOIDs, c.eqb.paramFormats, c.eqb.resultFormats) - - c.eqb.Reset() // Allow c.eqb internal memory to be GC'ed as soon as possible. - - return rows, rows.err -} - // QueryRow is a convenience wrapper over Query. Any error that occurs while // querying is deferred until calling Scan on the returned Row. That Row will // error with ErrNoRows if no rows are returned. From 9938d035dbcab7a1066b72829d72ea7b7aef613c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20Olav=20L=C3=B8ite?= Date: Fri, 25 Mar 2022 15:57:57 +0100 Subject: [PATCH 3/3] fix: use config for queries --- conn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conn.go b/conn.go index a31bce0a2..5f365771b 100644 --- a/conn.go +++ b/conn.go @@ -687,7 +687,7 @@ optionLoop: resultFormats = c.eqb.resultFormats } - if c.stmtcache == nil || (c.stmtcache != nil && c.stmtcache.Mode() == stmtcache.ModeDescribe) { + if (c.stmtcache == nil && c.config.PreferExecParams) || (c.stmtcache != nil && c.stmtcache.Mode() == stmtcache.ModeDescribe) { rows.resultReader = c.pgConn.ExecParams(ctx, sql, c.eqb.paramValues, sd.ParamOIDs, c.eqb.paramFormats, resultFormats) } else { rows.resultReader = c.pgConn.ExecPrepared(ctx, sd.Name, c.eqb.paramValues, c.eqb.paramFormats, resultFormats)