feat(analyzer): Analyze queries using a running PostgreSQL database (#…

…2805) * feat(analyzer): Analyze queries using a running PostgreSQL database 94 of the open issues on sqlc are related to the analyzer. There are many cases where the current analyzer produces false positives or false negatives. sqlx and some other projects have proven that it's possible to extract query metadata from a running database. This approach is a bit different, in that the database analysis is layered on top of the existing query analyzer. We use the new analysis to provide better type information and support a wider set of cases when the existing analyzer fails. * test(analyzer): Update endtoend tests for new analyzer Add a `contexts` key to exec.json to opt certain tests into or out of database-backed analysis. Fix many incorrect test cases that didn't run against an actual database.
sqlc-dev · Oct 12, 2023 · bb84596 · bb84596
1 parent 4b7fddd
commit bb84596
Show file tree

Hide file tree

Showing 478 changed files with 4,422 additions and 996 deletions.
diff --git a/examples/authors/sqlc.yaml b/examples/authors/sqlc.yaml
@@ -7,6 +7,8 @@ sql:
   engine: postgresql
   database:
     managed: true
+  analyzer:
+    database: false
   rules:
   - sqlc/db-prepare
   - postgresql-query-too-costly

diff --git a/examples/booktest/sqlc.json b/examples/booktest/sqlc.json
@@ -13,6 +13,9 @@
       "database": {
         "managed": true
       },
+      "analyzer": {
+        "database": false
+      },
       "rules": [
         "sqlc/db-prepare"
       ]

diff --git a/examples/jets/sqlc.json b/examples/jets/sqlc.json
@@ -13,6 +13,9 @@
       "database": {
         "managed": true
       },
+      "analyzer": {
+        "database": false
+      },
       "rules": [
         "sqlc/db-prepare"
       ]

diff --git a/examples/ondeck/sqlc.json b/examples/ondeck/sqlc.json
@@ -13,6 +13,9 @@
       "database": {
         "managed": true
       },
+      "analyzer": {
+        "database": false
+      },
       "rules": [
         "sqlc/db-prepare"
       ],
@@ -21,7 +24,7 @@
       "emit_interface": true
     },
     {
-      "path": "mysql",
+    "path": "mysql",
       "name": "ondeck",
       "schema": "mysql/schema",
       "queries": "mysql/query",

diff --git a/go.mod b/go.mod
@@ -29,6 +29,7 @@ require (
 
 require (
 	github.com/benbjohnson/clock v1.3.5 // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect

diff --git a/go.sum b/go.sum
@@ -100,7 +100,10 @@ github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSlj
 github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0=
 github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
+github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=

diff --git a/internal/analyzer/analyzer.go b/internal/analyzer/analyzer.go
@@ -0,0 +1,46 @@
+package analyzer
+
+import (
+	"context"
+
+	"github.com/sqlc-dev/sqlc/internal/sql/ast"
+	"github.com/sqlc-dev/sqlc/internal/sql/named"
+)
+
+type Column struct {
+	Name         string
+	OriginalName string
+	DataType     string
+	NotNull      bool
+	Unsigned     bool
+	IsArray      bool
+	ArrayDims    int
+	Comment      string
+	Length       *int
+	IsNamedParam bool
+	IsFuncCall   bool
+
+	// XXX: Figure out what PostgreSQL calls `foo.id`
+	Scope      string
+	Table      *ast.TableName
+	TableAlias string
+	Type       *ast.TypeName
+	EmbedTable *ast.TableName
+
+	IsSqlcSlice bool // is this sqlc.slice()
+}
+
+type Parameter struct {
+	Number int
+	Column *Column
+}
+
+type Analysis struct {
+	Columns []Column
+	Params  []Parameter
+}
+
+type Analyzer interface {
+	Analyze(context.Context, ast.Node, string, []string, *named.ParamSet) (*Analysis, error)
+	Close(context.Context) error
+}
diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go
@@ -197,7 +197,10 @@ var genCmd = &cobra.Command{
 		defer trace.StartRegion(cmd.Context(), "generate").End()
 		stderr := cmd.ErrOrStderr()
 		dir, name := getConfigPath(stderr, cmd.Flag("file"))
-		output, err := Generate(cmd.Context(), ParseEnv(cmd), dir, name, stderr)
+		output, err := Generate(cmd.Context(), dir, name, &Options{
+			Env:    ParseEnv(cmd),
+			Stderr: stderr,
+		})
 		if err != nil {
 			os.Exit(1)
 		}
@@ -219,7 +222,11 @@ var uploadCmd = &cobra.Command{
 	RunE: func(cmd *cobra.Command, args []string) error {
 		stderr := cmd.ErrOrStderr()
 		dir, name := getConfigPath(stderr, cmd.Flag("file"))
-		if err := createPkg(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
+		opts := &Options{
+			Env:    ParseEnv(cmd),
+			Stderr: stderr,
+		}
+		if err := createPkg(cmd.Context(), dir, name, opts); err != nil {
 			fmt.Fprintf(stderr, "error uploading: %s\n", err)
 			os.Exit(1)
 		}
@@ -234,7 +241,11 @@ var checkCmd = &cobra.Command{
 		defer trace.StartRegion(cmd.Context(), "compile").End()
 		stderr := cmd.ErrOrStderr()
 		dir, name := getConfigPath(stderr, cmd.Flag("file"))
-		if _, err := Generate(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
+		_, err := Generate(cmd.Context(), dir, name, &Options{
+			Env:    ParseEnv(cmd),
+			Stderr: stderr,
+		})
+		if err != nil {
 			os.Exit(1)
 		}
 		return nil
@@ -277,7 +288,11 @@ var diffCmd = &cobra.Command{
 		defer trace.StartRegion(cmd.Context(), "diff").End()
 		stderr := cmd.ErrOrStderr()
 		dir, name := getConfigPath(stderr, cmd.Flag("file"))
-		if err := Diff(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
+		opts := &Options{
+			Env:    ParseEnv(cmd),
+			Stderr: stderr,
+		}
+		if err := Diff(cmd.Context(), dir, name, opts); err != nil {
 			os.Exit(1)
 		}
 		return nil

diff --git a/internal/cmd/diff.go b/internal/cmd/diff.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io"
 	"os"
 	"runtime/trace"
 	"sort"
@@ -13,8 +12,9 @@ import (
 	"github.com/cubicdaiya/gonp"
 )
 
-func Diff(ctx context.Context, e Env, dir, name string, stderr io.Writer) error {
-	output, err := Generate(ctx, e, dir, name, stderr)
+func Diff(ctx context.Context, dir, name string, opts *Options) error {
+	stderr := opts.Stderr
+	output, err := Generate(ctx, dir, name, opts)
 	if err != nil {
 		return err
 	}

diff --git a/internal/cmd/generate.go b/internal/cmd/generate.go
@@ -139,8 +139,11 @@ func readConfig(stderr io.Writer, dir, filename string) (string, *config.Config,
 	return configPath, &conf, nil
 }
 
-func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer) (map[string]string, error) {
-	configPath, conf, err := readConfig(stderr, dir, filename)
+func Generate(ctx context.Context, dir, filename string, o *Options) (map[string]string, error) {
+	e := o.Env
+	stderr := o.Stderr
+
+	configPath, conf, err := o.ReadConfig(dir, filename)
 	if err != nil {
 		return nil, err
 	}
@@ -343,7 +346,12 @@ func remoteGenerate(ctx context.Context, configPath string, conf *config.Config,
 
 func parse(ctx context.Context, name, dir string, sql config.SQL, combo config.CombinedSettings, parserOpts opts.Parser, stderr io.Writer) (*compiler.Result, bool) {
 	defer trace.StartRegion(ctx, "parse").End()
-	c := compiler.NewCompiler(sql, combo)
+	c, err := compiler.NewCompiler(sql, combo)
+	defer c.Close(ctx)
+	if err != nil {
+		fmt.Fprintf(stderr, "error creating compiler: %s\n", err)
+		return nil, true
+	}
 	if err := c.ParseCatalog(sql.Schema); err != nil {
 		fmt.Fprintf(stderr, "# package %s\n", name)
 		if parserErr, ok := err.(*multierr.Error); ok {

diff --git a/internal/cmd/options.go b/internal/cmd/options.go
@@ -0,0 +1,24 @@
+package cmd
+
+import (
+	"io"
+
+	"github.com/sqlc-dev/sqlc/internal/config"
+)
+
+type Options struct {
+	Env          Env
+	Stderr       io.Writer
+	MutateConfig func(*config.Config)
+}
+
+func (o *Options) ReadConfig(dir, filename string) (string, *config.Config, error) {
+	path, conf, err := readConfig(o.Stderr, dir, filename)
+	if err != nil {
+		return path, conf, err
+	}
+	if o.MutateConfig != nil {
+		o.MutateConfig(conf)
+	}
+	return path, conf, nil
+}
diff --git a/internal/cmd/package.go b/internal/cmd/package.go
@@ -2,13 +2,14 @@ package cmd
 
 import (
 	"context"
-	"io"
 	"os"
 
 	"github.com/sqlc-dev/sqlc/internal/bundler"
 )
 
-func createPkg(ctx context.Context, e Env, dir, filename string, stderr io.Writer) error {
+func createPkg(ctx context.Context, dir, filename string, opts *Options) error {
+	e := opts.Env
+	stderr := opts.Stderr
 	configPath, conf, err := readConfig(stderr, dir, filename)
 	if err != nil {
 		return err
@@ -17,7 +18,7 @@ func createPkg(ctx context.Context, e Env, dir, filename string, stderr io.Write
 	if err := up.Validate(); err != nil {
 		return err
 	}
-	output, err := Generate(ctx, e, dir, filename, stderr)
+	output, err := Generate(ctx, dir, filename, opts)
 	if err != nil {
 		os.Exit(1)
 	}

diff --git a/internal/cmd/vet.go b/internal/cmd/vet.go
@@ -47,8 +47,12 @@ func NewCmdVet() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			defer trace.StartRegion(cmd.Context(), "vet").End()
 			stderr := cmd.ErrOrStderr()
+			opts := &Options{
+				Env:    ParseEnv(cmd),
+				Stderr: stderr,
+			}
 			dir, name := getConfigPath(stderr, cmd.Flag("file"))
-			if err := Vet(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
+			if err := Vet(cmd.Context(), dir, name, opts); err != nil {
 				if !errors.Is(err, ErrFailedChecks) {
 					fmt.Fprintf(stderr, "%s\n", err)
 				}
@@ -59,7 +63,9 @@ func NewCmdVet() *cobra.Command {
 	}
 }
 
-func Vet(ctx context.Context, e Env, dir, filename string, stderr io.Writer) error {
+func Vet(ctx context.Context, dir, filename string, opts *Options) error {
+	e := opts.Env
+	stderr := opts.Stderr
 	configPath, conf, err := readConfig(stderr, dir, filename)
 	if err != nil {
 		return err

diff --git a/internal/codegen/golang/result.go b/internal/codegen/golang/result.go
@@ -249,9 +249,7 @@ func buildQueries(req *plugin.CodeGenRequest, options *opts, structs []Struct) (
 		if len(query.Columns) == 1 && query.Columns[0].EmbedTable == nil {
 			c := query.Columns[0]
 			name := columnName(c, 0)
-			if c.IsFuncCall {
-				name = strings.Replace(name, "$", "_", -1)
-			}
+			name = strings.Replace(name, "$", "_", -1)
 			gq.Ret = QueryValue{
 				Name:      name,
 				DBName:    name,