Skip to content

Commit

Permalink
ast.Errors
Browse files Browse the repository at this point in the history
This commit introduces ast.Errors- an error type that should be used
whenever a generated error refers to a place in the query source. This
commit switches the parser package to use ast.Error but a follow up
commit will use ast.Errors for the semantic package as well.
  • Loading branch information
mattnibs committed May 13, 2024
1 parent 183e432 commit 261d933
Show file tree
Hide file tree
Showing 22 changed files with 373 additions and 224 deletions.
28 changes: 18 additions & 10 deletions api/client/connection.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,16 @@ import (
"github.com/brimdata/zed"
"github.com/brimdata/zed/api"
"github.com/brimdata/zed/api/client/auth0"
"github.com/brimdata/zed/compiler/parser"
"github.com/brimdata/zed/compiler/ast"
"github.com/brimdata/zed/lake"
"github.com/brimdata/zed/lake/branches"
"github.com/brimdata/zed/lakeparse"
"github.com/brimdata/zed/pkg/unpack"
"github.com/brimdata/zed/runtime/exec"
"github.com/brimdata/zed/zio/zngio"
"github.com/brimdata/zed/zson"
"github.com/segmentio/ksuid"
"go.uber.org/multierr"
)

const (
Expand Down Expand Up @@ -292,27 +294,33 @@ func (c *Connection) Revert(ctx context.Context, poolID ksuid.KSUID, branchName
return commit, err
}

var queryErrUnpacker = unpack.New(ast.Error{})

// Query assembles a query from src and filenames and runs it.
//
// As for Connection.Do, if the returned error is nil, the user is expected to
// call Response.Body.Close.
func (c *Connection) Query(ctx context.Context, head *lakeparse.Commitish, src string, filenames ...string) (*Response, error) {
src, srcInfo, err := parser.ConcatSource(filenames, src)
if err != nil {
return nil, err
}
func (c *Connection) Query(ctx context.Context, head *lakeparse.Commitish, src string) (*Response, error) {
body := api.QueryRequest{Query: src}
if head != nil {
body.Head = *head
}
req := c.NewRequest(ctx, http.MethodPost, "/query?ctrl=T", body)
res, err := c.Do(req)
var ae *api.Error
if errors.As(err, &ae) {
if m, ok := ae.Info.(map[string]interface{}); ok {
if offset, ok := m["parse_error_offset"].(float64); ok {
return res, parser.NewError(src, srcInfo, int(offset))
if errors.As(err, &ae) && ae.Info != nil {
if list, ok := ae.Info.([]any); ok {
var errs error
for _, l := range list {
var lerr *ast.Error
if queryErrUnpacker.UnmarshalObject(l, &lerr) != nil {
// If an error is encountered here just return the parent
// error since this is more interesting.
return nil, err
}
errs = multierr.Append(errs, lerr)
}
return nil, errs
}
}
return res, err
Expand Down
69 changes: 69 additions & 0 deletions cli/clierrors/format.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package clierrors

import (
"errors"
"fmt"
"strings"

"github.com/brimdata/zed/compiler/ast"
"github.com/brimdata/zed/compiler/parser"
"go.uber.org/multierr"
)

func Format(set *parser.SourceSet, err error) error {
if err == nil {
return err
}
var errs []error
for _, err := range multierr.Errors(err) {
if asterr, ok := err.(*ast.Error); ok {
err = formatASTError(set, asterr)
}
errs = append(errs, err)
}
return errors.Join(errs...)
}

func formatASTError(set *parser.SourceSet, err *ast.Error) error {
src := set.SourceOf(err.Pos)
start := src.Position(err.Pos)
end := src.Position(err.End)
var b strings.Builder
b.WriteString(err.Error())
b.WriteString(" (")
if src.Filename != "" {
fmt.Fprintf(&b, "%s: ", src.Filename)
}
fmt.Fprintf(&b, "line %d, ", start.Line)
fmt.Fprintf(&b, "column %d):\n", start.Column)
line := src.LineOfPos(set.Contents, err.Pos)
b.WriteString(line + "\n")
if end.IsValid() {
formatSpanError(&b, line, start, end)
} else {
formatPointError(&b, start)
}
return errors.New(b.String())
}

func formatSpanError(b *strings.Builder, line string, start, end parser.Position) {
col := start.Column - 1
b.WriteString(strings.Repeat(" ", col))
n := len(line) - col
if start.Line == end.Line {
n = end.Column - col
}
b.WriteString(strings.Repeat("~", n))
}

func formatPointError(b *strings.Builder, start parser.Position) {
col := start.Column - 1
for k := 0; k < col; k++ {
if k >= col-4 && k != col-1 {
b.WriteByte('=')
} else {
b.WriteByte(' ')
}
}
b.WriteString("^ ===")
}
53 changes: 34 additions & 19 deletions cli/queryflags/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,15 @@ import (
"strings"

"github.com/brimdata/zed/cli"
"github.com/brimdata/zed/compiler"
"github.com/brimdata/zed/cli/clierrors"
"github.com/brimdata/zed/compiler/ast"
"github.com/brimdata/zed/compiler/data"
"github.com/brimdata/zed/compiler/parser"
"github.com/brimdata/zed/compiler/semantic"
"github.com/brimdata/zed/pkg/storage"
"github.com/brimdata/zed/zbuf"
"github.com/brimdata/zed/zson"
"go.uber.org/multierr"
)

type Flags struct {
Expand All @@ -32,7 +33,7 @@ func (f *Flags) SetFlags(fs *flag.FlagSet) {
fs.Var(&f.Includes, "I", "source file containing Zed query text (may be used multiple times)")
}

func (f *Flags) ParseSourcesAndInputs(paths []string) ([]string, ast.Seq, bool, error) {
func (f *Flags) ParseSourcesAndInputs(paths []string) ([]string, ast.Seq, *parser.SourceSet, bool, error) {
var src string
if len(paths) != 0 && !cli.FileExists(paths[0]) && !isURLWithKnownScheme(paths[0], "http", "https", "s3") {
src = paths[0]
Expand All @@ -41,25 +42,34 @@ func (f *Flags) ParseSourcesAndInputs(paths []string) ([]string, ast.Seq, bool,
// Consider a lone argument to be a query if it compiles
// and appears to start with a from or yield operator.
// Otherwise, consider it a file.
query, err := compiler.Parse(src, f.Includes...)
query, set, err := parse(src, f.Includes...)
if err == nil {
if s, err := semantic.Analyze(context.Background(), query, data.NewSource(storage.NewLocalEngine(), nil), nil); err == nil {
if semantic.HasSource(s) {
return nil, query, false, nil
return nil, query, set, false, nil
}
if semantic.StartsWithYield(s) {
return nil, query, true, nil
return nil, query, set, true, nil
}
}
}
return nil, nil, false, singleArgError(src, err)
return nil, nil, nil, false, singleArgError(src, set, err)
}
}
query, err := compiler.Parse(src, f.Includes...)
query, set, err := parse(src, f.Includes...)
if err != nil {
return nil, nil, false, err
return nil, nil, set, false, clierrors.Format(set, err)
}
return paths, query, false, nil
return paths, query, set, false, nil
}

func parse(src string, includes ...string) (ast.Seq, *parser.SourceSet, error) {
set, err := parser.ConcatSource(includes, src)
if err != nil {
return nil, nil, err
}
seq, err := parser.ParseZed(string(set.Contents))
return seq, set, err
}

func isURLWithKnownScheme(path string, schemes ...string) bool {
Expand All @@ -80,25 +90,30 @@ func (f *Flags) PrintStats(stats zbuf.Progress) {
}
}

func singleArgError(src string, err error) error {
func singleArgError(src string, set *parser.SourceSet, err error) error {
var b strings.Builder
b.WriteString("could not invoke zq with a single argument because:")
if len(src) > 20 {
src = src[:20] + "..."
}
fmt.Fprintf(&b, "\n - a file could not be found with the name %q", src)
var perr *parser.Error
if errors.As(err, &perr) {
b.WriteString("\n - the argument could not be compiled as a valid Zed query due to parse error (")
if perr.LineNum > 0 {
fmt.Fprintf(&b, "line %d, ", perr.LineNum)
}
fmt.Fprintf(&b, "column %d):", perr.Column)
for _, l := range strings.Split(perr.ParseErrorContext(), "\n") {
fmt.Fprintf(&b, "\n %s", l)
if hasASTErrors(err) {
b.WriteString("\n - the argument could not be compiled as a valid Zed query:")
for _, line := range strings.Split(clierrors.Format(set, err).Error(), "\n") {
fmt.Fprintf(&b, "\n %s", line)
}
} else {
b.WriteString("\n - the argument did not parse as a valid Zed query")
}
return errors.New(b.String())
}

func hasASTErrors(errs error) bool {
for _, err := range multierr.Errors(errs) {
var aerr *ast.Error
if errors.As(err, &aerr) {
return true
}
}
return false
}
5 changes: 3 additions & 2 deletions cli/zq/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/brimdata/zed"
"github.com/brimdata/zed/cli"
"github.com/brimdata/zed/cli/clierrors"
"github.com/brimdata/zed/cli/inputflags"
"github.com/brimdata/zed/cli/outputflags"
"github.com/brimdata/zed/cli/queryflags"
Expand Down Expand Up @@ -129,7 +130,7 @@ func (c *Command) Run(args []string) error {
// Prevent ParseSourcesAndInputs from treating args[0] as a path.
args = append(args, "-")
}
paths, flowgraph, null, err := c.queryFlags.ParseSourcesAndInputs(args)
paths, flowgraph, set, null, err := c.queryFlags.ParseSourcesAndInputs(args)
if err != nil {
return fmt.Errorf("zq: %w", err)
}
Expand All @@ -156,7 +157,7 @@ func (c *Command) Run(args []string) error {
comp := compiler.NewFileSystemCompiler(local)
query, err := runtime.CompileQuery(ctx, zctx, comp, flowgraph, readers)
if err != nil {
return err
return clierrors.Format(set, err)
}
defer query.Pull(true)
err = zbuf.CopyPuller(writer, query)
Expand Down
49 changes: 23 additions & 26 deletions cmd/zed/dev/compile/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ import (
"errors"
"flag"
"fmt"
"os"
"strings"

"github.com/brimdata/zed/cli/clierrors"
"github.com/brimdata/zed/cmd/zed/dev"
"github.com/brimdata/zed/cmd/zed/root"
"github.com/brimdata/zed/compiler"
Expand Down Expand Up @@ -112,25 +112,18 @@ func (c *Command) Run(args []string) error {
c.pigeon = true
}
}
var src string
if len(c.includes) > 0 {
for _, path := range c.includes {
b, err := os.ReadFile(path)
if err != nil {
return err
}
src += "\n" + string(b)
}
set, err := parser.ConcatSource(c.includes, strings.Join(args, " "))
if err != nil {
return err
}
src += strings.Join(args, " ")
var lk *lake.Root
if c.semantic || c.optimize || c.parallel != 0 {
lakeAPI, err := c.LakeFlags.Open(ctx)
if err == nil {
lk = lakeAPI.Root()
}
}
return c.parse(src, lk)
return c.parse(set, lk)
}

func (c *Command) header(msg string) {
Expand All @@ -142,33 +135,33 @@ func (c *Command) header(msg string) {
}
}

func (c *Command) parse(z string, lk *lake.Root) error {
func (c *Command) parse(set *parser.SourceSet, lk *lake.Root) error {
if c.pigeon {
s, err := parsePigeon(z)
s, err := parsePigeon(set)
if err != nil {
return err
}
c.header("pigeon")
fmt.Println(s)
}
if c.proc {
seq, err := compiler.Parse(z)
seq, err := compiler.Parse(string(set.Contents))
if err != nil {
return err
}
c.header("proc")
c.writeAST(seq)
}
if c.semantic {
runtime, err := c.compile(z, lk)
runtime, err := c.compile(set, lk)
if err != nil {
return err
}
c.header("semantic")
c.writeDAG(runtime.Entry())
}
if c.optimize {
runtime, err := c.compile(z, lk)
runtime, err := c.compile(set, lk)
if err != nil {
return err
}
Expand All @@ -179,7 +172,7 @@ func (c *Command) parse(z string, lk *lake.Root) error {
c.writeDAG(runtime.Entry())
}
if c.parallel > 0 {
runtime, err := c.compile(z, lk)
runtime, err := c.compile(set, lk)
if err != nil {
return err
}
Expand Down Expand Up @@ -213,12 +206,16 @@ func (c *Command) writeDAG(seq dag.Seq) {
}
}

func (c *Command) compile(z string, lk *lake.Root) (*compiler.Job, error) {
p, err := compiler.Parse(z)
func (c *Command) compile(set *parser.SourceSet, lk *lake.Root) (*compiler.Job, error) {
p, err := compiler.Parse(string(set.Contents))
if err != nil {
return nil, err
return nil, clierrors.Format(set, err)
}
return compiler.NewJob(runtime.DefaultContext(), p, data.NewSource(nil, lk), nil)
job, err := compiler.NewJob(runtime.DefaultContext(), p, data.NewSource(nil, lk), nil)
if err != nil {
return nil, clierrors.Format(set, err)
}
return job, nil
}

func normalize(b []byte) (string, error) {
Expand Down Expand Up @@ -253,14 +250,14 @@ func dagFmt(seq dag.Seq, canon bool) (string, error) {
return normalize(dagJSON)
}

func parsePigeon(z string) (string, error) {
ast, err := parser.Parse("", []byte(z))
func parsePigeon(set *parser.SourceSet) (string, error) {
ast, err := parser.ParseZed(string(set.Contents))
if err != nil {
return "", err
return "", clierrors.Format(set, err)
}
goPEGJSON, err := json.Marshal(ast)
if err != nil {
return "", errors.New("go peg parser returned bad value for: " + z)
return "", errors.New("go peg parser returned bad value for: " + string(set.Contents))
}
return normalize(goPEGJSON)
}
Loading

0 comments on commit 261d933

Please sign in to comment.