improve UX for metadata queries (#4377)

This commit improves the UX for metadata queries by using the
metadata type name to distinguish between commit-level queries
and pool-level queries rather than requiring the presence of
an @Branch commit tag for commit-level queries.  This way you
can query the objects on the main branch of a pool without
specifying @main.  We also fixed a bug in the optimization logic
where the layout of metadata queries was presumed to be the
same layout of the pool.  The fix is to presume the layout
of metadata queries is unknown.

compiler/ast/dag/operator.go

@@ -200,6 +200,25 @@ type (
 	}
 )
 
+var LakeMetas = map[string]struct{}{
+	"branches":    {},
+	"index_rules": {},
+	"pools":       {},
+}
+
+var PoolMetas = map[string]struct{}{
+	"branches": {},
+}
+
+var CommitMetas = map[string]struct{}{
+	"indexes":    {},
+	"log":        {},
+	"objects":    {},
+	"partitions": {},
+	"rawlog":     {},
+	"vectors":    {},
+}
+
 type Source interface {
 	Source()
 }

compiler/optimizer/optimizer.go

@@ -173,8 +173,10 @@ func (o *Optimizer) getLayout(s dag.Source, parent order.Layout) (order.Layout,
 		return s.Layout, nil
 	case *dag.HTTP:
 		return s.Layout, nil
-	case *dag.Pool, *dag.LakeMeta, *dag.PoolMeta, *dag.CommitMeta:
+	case *dag.Pool:
 		return o.source.Layout(o.ctx, s), nil
+	case *dag.LakeMeta, *dag.PoolMeta, *dag.CommitMeta:
+		return order.Nil, nil
 	case *dag.Pass:
 		return parent, nil
 	case *kernel.Reader:

compiler/semantic/op.go

@@ -165,9 +165,13 @@ func semPoolWithName(ctx context.Context, scope *Scope, p *ast.Pool, poolName st
 		commit = head.Branch
 	}
 	if poolName == "" {
-		if p.Spec.Meta == "" {
+		meta := p.Spec.Meta
+		if meta == "" {
 			return nil, errors.New("pool name missing")
 		}
+		if _, ok := dag.LakeMetas[meta]; !ok {
+			return nil, fmt.Errorf("unknown lake metadata type %q in from operator", meta)
+		}
 		return &dag.LakeMeta{
 			Kind: "LakeMeta",
 			Meta: p.Spec.Meta,
@@ -176,9 +180,7 @@ func semPoolWithName(ctx context.Context, scope *Scope, p *ast.Pool, poolName st
 	// If a name appears as an 0x bytes ksuid, convert it to the
 	// ksuid string form since the backend doesn't parse the 0x format.
 	poolID, err := lakeparse.ParseID(poolName)
-	if err == nil {
-		poolName = poolID.String()
-	} else {
+	if err != nil {
 		poolID, err = ds.PoolID(ctx, poolName)
 		if err != nil {
 			return nil, err
@@ -203,20 +205,29 @@ func semPoolWithName(ctx context.Context, scope *Scope, p *ast.Pool, poolName st
 			}
 		}
 	}
-	if p.Spec.Meta != "" {
-		if commit != "" {
+	if meta := p.Spec.Meta; meta != "" {
+		if _, ok := dag.CommitMetas[meta]; ok {
+			if commitID == ksuid.Nil {
+				commitID, err = ds.CommitObject(ctx, poolID, "main")
+				if err != nil {
+					return nil, err
+				}
+			}
 			return &dag.CommitMeta{
 				Kind:   "CommitMeta",
-				Meta:   p.Spec.Meta,
+				Meta:   meta,
 				Pool:   poolID,
 				Commit: commitID,
 			}, nil
 		}
-		return &dag.PoolMeta{
-			Kind: "PoolMeta",
-			Meta: p.Spec.Meta,
-			ID:   poolID,
-		}, nil
+		if _, ok := dag.PoolMetas[meta]; ok {
+			return &dag.PoolMeta{
+				Kind: "PoolMeta",
+				Meta: meta,
+				ID:   poolID,
+			}, nil
+		}
+		return nil, fmt.Errorf("unknown metadata type %q in from operator", meta)
 	}
 	if commitID == ksuid.Nil {
 		// This trick here allows us to default to the main branch when

lake/ztests/meta.yaml

@@ -8,7 +8,7 @@ script: |
   zed query -Z 'from :pools | drop id | sort name | drop ts'
   echo ===
   zed query -Z 'from poolA@main:objects | cut nameof(this),meta'
-  zed query -Z 'from poolA@main:log | cut nameof(this) | drop ts'
+  zed query -Z 'from poolA:log | cut nameof(this) | drop ts'
   echo ===
   zed index create -q Rule field a
   zed query -Z 'from :index_rules | nameof:=nameof(this) | drop ts,id'