CLI w/ embedded prefix: disambiguating

* add `dop`, `oltp`  structures; refactor
* compensate for s3 delete deficiency
* consolidate all errors in cli/err.go
* prefetch to warn "does not exist"; auto-confirm
* simplify and refactor
* part two, prev. commit: 1bf78c8

Signed-off-by: Alex Aizman <alex.aizman@gmail.com>

ais/proxy.go

@@ -2238,8 +2238,9 @@ func (p *proxy) listBuckets(w http.ResponseWriter, r *http.Request, qbck *cmn.Qu
 	hdr := w.Header()
 	hdr.Set(cos.HdrContentType, res.header.Get(cos.HdrContentType))
 	hdr.Set(cos.HdrContentLength, strconv.Itoa(len(res.bytes)))
-	_, err = w.Write(res.bytes)
-	debug.AssertNoErr(err)
+	if _, err := w.Write(res.bytes); err != nil {
+		nlog.Warningln(err) // (broken pipe; benign)
+	}
 }
 
 func (p *proxy) redirectURL(r *http.Request, si *meta.Snode, ts time.Time, netIntra string, netPubs ...string) (redirect string) {

ais/test/scripts/remais-prefetch-latest.sh

@@ -123,17 +123,19 @@ cnt3=$(ais show performance counters --regex $cold_counter -H | awk '{sum+=$2;}E
 echo "9. out-of-band DELETE"
 AIS_ENDPOINT=$rendpoint ais object rm "$rbucket/lorem-duis" 1>/dev/null || exit $?
 
-echo "10. prefetch without '--latest': expecting no changes"
-ais prefetch "$bucket/lorem-duis" --wait
+## '--yes' to auto-confirm non-existence
+echo "10. prefetch without '--latesti --yes': expecting no changes"
+ais prefetch "$bucket/lorem-duis" --wait --yes
 checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
 [[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
 
 echo "11. remember 'remote-deleted' counter, enable version synchronization on the bucket"
 cnt4=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
 ais bucket props set $bucket versioning.synchronize=true
 
-echo "12. run 'prefetch --latest' one last time, and make sure the object \"disappears\""
-ais prefetch "$bucket/lorem-duis" --latest --wait 2>/dev/null
+## '--yes' ditto
+echo "12. run 'prefetch --latest --yes' one last time, and make sure the object \"disappears\""
+ais prefetch "$bucket/lorem-duis" --latest --wait --yes 2>/dev/null
 [[ $? == 0 ]] || { echo "FAIL: expecting 'prefetch --wait' to return Ok, got $?"; exit 1; }
 
 echo "13. 'remote-deleted' counter must increment"

ais/test/scripts/s3-prefetch-latest-prefix.sh

@@ -105,17 +105,19 @@ cnt3=$(ais show performance counters --regex ${cold_counter} -H | awk '{sum+=$2;
 echo "9. out-of-band DELETE"
 s3cmd del "$bucket/lorem-duis" $host 1>/dev/null || exit $?
 
-echo "10. prefetch without '--latest': expecting no changes"
-ais prefetch "$bucket/lorem-duis" --wait
+## '--yes' to auto-confirm non-existence
+echo "10. prefetch without '--latest --yes': expecting no changes"
+ais prefetch "$bucket/lorem-duis" --wait --yes
 checksum=$(ais ls "$bucket/lorem-duis" --cached -H -props checksum | awk '{print $2}')
 [[ "$checksum" == "$sum2"  ]] || { echo "FAIL: $checksum != $sum2"; exit 1; }
 
 echo "11. remember 'remote-deleted' counter _and_ enable version synchronization"
 cnt4=$(ais show performance counters --regex REMOTE-DEL -H | awk '{sum+=$2;}END{print sum;}')
 ais bucket props set $bucket versioning.synchronize=true
 
-echo "12. run 'prefetch --latest' one last time, and make sure the object \"disappears\""
-ais prefetch "$bucket/lorem-duis" --latest --wait 2>/dev/null
+## '--yes' ditto
+echo "12. run 'prefetch --latest --yes' one last time, and make sure the object \"disappears\""
+ais prefetch "$bucket/lorem-duis" --latest --wait --yes 2>/dev/null
 [[ $? == 0 ]] || { echo "FAIL: expecting 'prefetch --wait' to return Ok, got $?"; exit 1; }
 
 echo "13. 'remote-deleted' counter must increment (because 'versioning.synchronize=true')"

cmd/cli/cli/bucket_hdlr.go

@@ -130,7 +130,7 @@ var (
 			waitJobXactFinishedFlag,
 			latestVerFlag,
 			syncFlag,
-			noRecursFlag, // (embedded prefix parseObjListTemplate dop)
+			noRecursFlag, // (embedded prefix dopOLTP)
 			nonverboseFlag,
 		},
 		commandRename: {
@@ -143,7 +143,7 @@ var (
 			keepMDFlag,
 			verbObjPrefixFlag, // to disambiguate bucket/prefix vs bucket/objName
 			dryRunFlag,
-			noRecursFlag, // (embedded prefix parseObjListTemplate dop)
+			noRecursFlag, // (embedded prefix dopOLTP)
 			verboseFlag,  // not yet used
 			nonverboseFlag,
 		),

cmd/cli/cli/err.go

@@ -426,3 +426,49 @@ func isStartingUp(err error) bool {
 	}
 	return false
 }
+
+//
+// misplaced or mistyped flag(s)
+//
+
+func errArgIsFlag(c *cli.Context, arg string) (err error) {
+	if len(arg) > 1 && arg[0] == '-' {
+		err = incorrectUsageMsg(c, "missing command line argument (hint: flag '%s' misplaced?)", arg)
+	}
+	return err
+}
+
+func errTailArgsContainFlag(tail []string) error {
+	for _, arg := range tail {
+		if len(arg) > 1 && arg[0] == '-' {
+			return fmt.Errorf("unrecognized or misplaced option %q", arg)
+		}
+	}
+	return nil
+}
+
+//
+// range read
+//
+
+func errRangeReadArch(what string) error {
+	return fmt.Errorf("cannot range-read (%s, %s) archived content (%s) - "+NIY, qflprn(lengthFlag), qflprn(offsetFlag), what)
+}
+
+//
+// parse uri
+//
+
+func errBucketNameInvalid(c *cli.Context, arg string, err error) error {
+	if errV := errArgIsFlag(c, arg); errV != nil {
+		return errV
+	}
+	if strings.Contains(err.Error(), cos.OnlyPlus) && strings.Contains(err.Error(), "bucket name") {
+		if strings.Contains(arg, ":/") && !strings.Contains(arg, apc.BckProviderSeparator) {
+			a := strings.Replace(arg, ":/", apc.BckProviderSeparator, 1)
+			return fmt.Errorf("bucket name in %q is invalid: (did you mean %q?)", arg, a)
+		}
+		return fmt.Errorf("bucket name in %q is invalid: "+cos.OnlyPlus, arg)
+	}
+	return nil
+}

cmd/cli/cli/get.go

@@ -730,8 +730,3 @@ func (ex *extractor) _write(filename string, size int64, wfh *os.File, reader io
 func discardOutput(outf string) bool {
 	return outf == "/dev/null" || outf == "dev/null" || outf == "dev/nil"
 }
-
-// rr
-func errRangeReadArch(what string) error {
-	return fmt.Errorf("cannot range-read (%s, %s) archived content (%s) - "+NIY, qflprn(lengthFlag), qflprn(offsetFlag), what)
-}

cmd/cli/cli/job_hdlr.go

@@ -101,8 +101,9 @@ var (
 			dryRunFlag,
 			verbObjPrefixFlag,
 			latestVerFlag,
-			noRecursFlag, // (embedded prefix parseObjListTemplate dop)
+			noRecursFlag, // (embedded prefix dopOLTP)
 			blobThresholdFlag,
+			yesFlag,
 			numListRangeWorkersFlag,
 		),
 		cmdBlobDownload: {

cmd/cli/cli/multiobj.go

@@ -173,22 +173,22 @@ func _evictOne(c *cli.Context, shift int) error {
 	if _, err := headBucket(bck, false /* don't add */); err != nil {
 		return err
 	}
-	objName, listObjs, tmplObjs, err := parseObjListTemplate(c, bck, objNameOrTmpl)
+	oltp, err := dopOLTP(c, bck, objNameOrTmpl)
 	if err != nil {
 		return err
 	}
 
 	switch {
-	case listObjs != "" || tmplObjs != "": // 1. multi-obj
-		lrCtx := &lrCtx{listObjs, tmplObjs, bck}
+	case oltp.list != "" || oltp.tmpl != "": // 1. multi-obj
+		lrCtx := &lrCtx{oltp.list, oltp.tmpl, bck}
 		return lrCtx.do(c)
-	case objName == "": // 2. entire bucket
+	case oltp.objName == "": // 2. entire bucket
 		return evictBucket(c, bck)
 	default: // 3. one(?) obj to evict
-		err := api.EvictObject(apiBP, bck, objName)
+		err := api.EvictObject(apiBP, bck, oltp.objName)
 		if err == nil {
 			if !flagIsSet(c, nonverboseFlag) {
-				fmt.Fprintf(c.App.Writer, "evicted %q from %s\n", objName, bck.Cname(""))
+				fmt.Fprintf(c.App.Writer, "evicted %q from %s\n", oltp.objName, bck.Cname(""))
 			}
 			return nil
 		}
@@ -201,7 +201,7 @@ func _evictOne(c *cli.Context, shift int) error {
 		if !argIsFlag(c, 1) {
 			suffix = " (hint: missing double or single quotes?)"
 		}
-		return &errDoesNotExist{name: bck.Cname(objName), suffix: suffix}
+		return &errDoesNotExist{name: bck.Cname(oltp.objName), suffix: suffix}
 	}
 }
 
@@ -230,18 +230,19 @@ func _rmOne(c *cli.Context, shift int) error {
 	if _, err := headBucket(bck, false /* don't add */); err != nil {
 		return err
 	}
-	// NOTE: passing empty bck _not_ to interpret embedded objName as prefix
-	// TODO: instead of HEAD(obj) do list-objects(prefix=objNameOrTmpl)  - here and everywhere
-	objName, listObjs, tmplObjs, err := parseObjListTemplate(c, bck, objNameOrTmpl)
+	// [NOTE]
+	// - passing empty bck _not_ to interpret embedded objName as prefix
+	// - instead of HEAD(obj) do list-objects(prefix=objNameOrTmpl)  - here and everywhere
+	oltp, err := dopOLTP(c, bck, objNameOrTmpl)
 	if err != nil {
 		return err
 	}
 
 	switch {
-	case listObjs != "" || tmplObjs != "": // 1. multi-obj
-		lrCtx := &lrCtx{listObjs, tmplObjs, bck}
+	case oltp.list != "" || oltp.tmpl != "": // 1. multi-obj
+		lrCtx := &lrCtx{oltp.list, oltp.tmpl, bck}
 		return lrCtx.do(c)
-	case objName == "": // 2. all objects
+	case oltp.objName == "": // 2. all objects
 		if flagIsSet(c, rmrfFlag) {
 			if !flagIsSet(c, yesFlag) {
 				warn := fmt.Sprintf("will remove all objects from %s. The operation cannot be undone!", bck)
@@ -251,13 +252,19 @@ func _rmOne(c *cli.Context, shift int) error {
 			}
 			return rmRfAllObjects(c, bck)
 		}
-		return incorrectUsageMsg(c, "use one of: (%s or %s or %s) to indicate _which_ objects to remove",
+		return incorrectUsageMsg(c, "to select objects to be removed use one of: (%s or %s or %s)",
 			qflprn(listFlag), qflprn(templateFlag), qflprn(rmrfFlag))
 	default: // 3. one obj
-		err := api.DeleteObject(apiBP, bck, objName)
+		err := api.DeleteObject(apiBP, bck, oltp.objName)
+		if err == nil && oltp.notFound {
+			// [NOTE]
+			// - certain backends return OK when specified object does not exist (see aws.go)
+			// - compensate here
+			return cos.NewErrNotFound(nil, bck.Cname(oltp.objName))
+		}
 		if err == nil {
 			if !flagIsSet(c, nonverboseFlag) {
-				fmt.Fprintf(c.App.Writer, "deleted %q from %s\n", objName, bck.Cname(""))
+				fmt.Fprintf(c.App.Writer, "deleted %q from %s\n", oltp.objName, bck.Cname(""))
 			}
 			return nil
 		}
@@ -270,7 +277,7 @@ func _rmOne(c *cli.Context, shift int) error {
 		if c.NArg() > 1 {
 			suffix = " (hint: missing double or single quotes?)"
 		}
-		return &errDoesNotExist{name: bck.Cname(objName), suffix: suffix}
+		return &errDoesNotExist{name: bck.Cname(oltp.objName), suffix: suffix}
 	}
 }
 
@@ -303,15 +310,25 @@ func _prefetchOne(c *cli.Context, shift int) error {
 	if !bck.IsRemote() {
 		return fmt.Errorf("expecting remote bucket (have %s)", bck.Cname(""))
 	}
-	objName, listObjs, tmplObjs, err := parseObjListTemplate(c, bck, objNameOrTmpl)
+	oltp, err := dopOLTP(c, bck, objNameOrTmpl)
 	if err != nil {
 		return err
 	}
+	if oltp.notFound { // (true only when list-objects says so)
+		err := cos.NewErrNotFound(nil, "\""+uri+"\"")
+		if !flagIsSet(c, yesFlag) {
+			if ok := confirm(c, err.Error()+" - proceed anyway?"); !ok {
+				return err
+			}
+		} else {
+			actionWarn(c, err.Error()+" - proceeding anyway")
+		}
+	}
 
-	if listObjs == "" && tmplObjs == "" {
-		listObjs = objName
+	if oltp.list == "" && oltp.tmpl == "" {
+		oltp.list = oltp.objName // ("prefetch" is not one of those primitive verbs)
 	}
-	lrCtx := &lrCtx{listObjs, tmplObjs, bck}
+	lrCtx := &lrCtx{oltp.list, oltp.tmpl, bck}
 	return lrCtx.do(c)
 }
 

cmd/cli/cli/object_hdlr.go

@@ -81,7 +81,7 @@ var (
 			rmrfFlag,
 			verboseFlag, // rm -rf
 			nonverboseFlag,
-			noRecursFlag, // (embedded prefix parseObjListTemplate dop)
+			noRecursFlag, // (embedded prefix dopOLTP dop)
 			yesFlag,
 		),
 		commandRename: {},