Add CLI tools for debugging and analyzing caching issues

cli/src/alluxio.org/cli/cmd/fs/checkcaching.go

@@ -0,0 +1,65 @@
+/*
+ * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+ * (the "License"). You may not use this work except in compliance with the License, which is
+ * available at www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied, as more fully set forth in the License.
+ *
+ * See the NOTICE file distributed with this work for information regarding copyright ownership.
+ */
+
+package fs
+
+import (
+    "alluxio.org/cli/env"
+    "github.com/spf13/cobra"
+    "strconv"
+)
+
+func CheckCaching(className string) env.Command {
+    return &CheckCachingCommand{
+        BaseJavaCommand: &env.BaseJavaCommand{
+            CommandName:   "checkCaching",
+            JavaClassName: className,
+            Parameters:    []string{"checkCaching"},
+        },
+    }
+}
+
+type CheckCachingCommand struct {
+    *env.BaseJavaCommand
+
+    sample int
+    limit  int
+}
+
+func (c *CheckCachingCommand) Base() *env.BaseJavaCommand {
+    return c.BaseJavaCommand
+}
+
+func (c *CheckCachingCommand) ToCommand() *cobra.Command {
+    cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
+        Use:   "checkCaching [path]",
+        Short: "Checks if files under a path have been cached in alluxio.",
+        Args:  cobra.ExactArgs(1),
+        RunE: func(cmd *cobra.Command, args []string) error {
+            return c.Run(args)
+        },
+    })
+    cmd.Flags().IntVar(&c.sample, "sample", 0, "Sample ratio, 10 means sample 1 in every 10 files.")
+    cmd.Flags().IntVar(&c.limit, "limit", 0, "limit, default 1000")
+    return cmd
+}
+
+func (c *CheckCachingCommand) Run(args []string) error {
+    javaArgs := []string{"checkCaching"}
+    if c.sample != 0 {
+        javaArgs = append(javaArgs, "--sample", strconv.Itoa(c.sample))
+    }
+    if c.limit != 0 {
+        javaArgs = append(javaArgs, "--limit", strconv.Itoa(c.limit))
+    }
+    javaArgs = append(javaArgs, args...)
+    return c.Base().Run(args)
+}

cli/src/alluxio.org/cli/cmd/fs/consistanthash.go

@@ -0,0 +1,69 @@
+/*
+ * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+ * (the "License"). You may not use this work except in compliance with the License, which is
+ * available at www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied, as more fully set forth in the License.
+ *
+ * See the NOTICE file distributed with this work for information regarding copyright ownership.
+ */
+
+package fs
+
+import (
+    "alluxio.org/cli/env"
+    "github.com/spf13/cobra"
+)
+
+func ConsistentHash(className string) env.Command {
+    return &ConsistentHashCommand{
+        BaseJavaCommand: &env.BaseJavaCommand{
+            CommandName:   "consistentHash",
+            JavaClassName: className,
+            Parameters:    []string{"consistentHash"},
+        },
+    }
+}
+
+type ConsistentHashCommand struct {
+    *env.BaseJavaCommand
+
+    createCheckFile   bool
+    compareCheckFiles bool
+    cleanCheckData    bool
+}
+
+func (c *ConsistentHashCommand) Base() *env.BaseJavaCommand {
+    return c.BaseJavaCommand
+}
+
+func (c *ConsistentHashCommand) ToCommand() *cobra.Command {
+    cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
+        Use:   "consistentHash [--createCheckFile]|[--compareCheckFiles <1stCheckFilePath> <2ndCheckFilePath>]|[--cleanCheckData] ",
+        Short: "This command is for checking whether the consistent hash ring is changed or not. ",
+        RunE: func(cmd *cobra.Command, args []string) error {
+            return c.Run(args)
+        },
+    })
+    cmd.Flags().BoolVarP(&c.createCheckFile, "createCheckFile", "c", false, "Generate check file.")
+    cmd.Flags().BoolVarP(&c.compareCheckFiles, "compareCheckFiles", "C", false, "Compare check files to see if the hash ring has changed and if data lost.")
+    cmd.Flags().BoolVarP(&c.cleanCheckData, "cleanCheckData", "d", false, "Clean all check data.")
+    return cmd
+}
+
+func (c *ConsistentHashCommand) Run(args []string) error {
+    javaArgs := []string{}
+    if c.createCheckFile {
+        javaArgs = append(javaArgs, "--createCheckFile")
+    }
+    if c.compareCheckFiles {
+        javaArgs = append(javaArgs, "--compareCheckFiles")
+    }
+    if c.cleanCheckData {
+        javaArgs = append(javaArgs, "--cleanCheckData")
+    }
+
+    javaArgs = append(javaArgs, args...)
+    return c.Base().Run(javaArgs)
+}

cli/src/alluxio.org/cli/cmd/fs/fs.go

@@ -12,37 +12,40 @@
 package fs
 
 import (
-	"alluxio.org/cli/cmd/names"
-	"alluxio.org/cli/env"
+    "alluxio.org/cli/cmd/names"
+    "alluxio.org/cli/env"
 )
 
 var Service = &env.Service{
-	Name:        "fs",
-	Description: "Operations to interface with the Alluxio filesystem",
-	Commands:    Cmds(names.FileSystemShellJavaClass),
+    Name:        "fs",
+    Description: "Operations to interface with the Alluxio filesystem",
+    Commands:    Cmds(names.FileSystemShellJavaClass),
 }
 
 func Cmds(className string) []env.Command {
-	var ret []env.Command
-	for _, c := range []func(string) env.Command{
-		Cat,
-		Checksum,
-		Chgrp,
-		Chmod,
-		Chown,
-		Cp,
-		Head,
-		Ls,
-		Mkdir,
-		Mv,
-		Rm,
-		Stat,
-		Tail,
-		Test,
-		Touch,
-	} {
-		ret = append(ret, c(className))
-	}
+    var ret []env.Command
+    for _, c := range []func(string) env.Command{
+        Cat,
+        Checksum,
+        Chgrp,
+        Chmod,
+        Chown,
+        Cp,
+        Head,
+        Ls,
+        Mkdir,
+        Mv,
+        Rm,
+        Stat,
+        Tail,
+        Test,
+        Touch,
+        Location,
+        CheckCaching,
+        ConsistentHash,
+    } {
+        ret = append(ret, c(className))
+    }
 
-	return ret
+    return ret
 }

cli/src/alluxio.org/cli/cmd/fs/location.go

@@ -0,0 +1,52 @@
+/*
+ * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+ * (the "License"). You may not use this work except in compliance with the License, which is
+ * available at www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied, as more fully set forth in the License.
+ *
+ * See the NOTICE file distributed with this work for information regarding copyright ownership.
+ */
+
+package fs
+
+import (
+    "github.com/spf13/cobra"
+
+    "alluxio.org/cli/env"
+)
+
+func Location(className string) env.Command {
+    return &LocationCommand{
+        BaseJavaCommand: &env.BaseJavaCommand{
+            CommandName:   "location",
+            JavaClassName: className,
+            Parameters:    []string{"location"},
+        },
+    }
+}
+
+type LocationCommand struct {
+    *env.BaseJavaCommand
+}
+
+func (c *LocationCommand) Base() *env.BaseJavaCommand {
+    return c.BaseJavaCommand
+}
+
+func (c *LocationCommand) ToCommand() *cobra.Command {
+    cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{
+        Use:   "location [path]",
+        Short: "Displays the list of hosts storing the specified file.",
+        Args:  cobra.ExactArgs(1),
+        RunE: func(cmd *cobra.Command, args []string) error {
+            return c.Run(args)
+        },
+    })
+    return cmd
+}
+
+func (c *LocationCommand) Run(args []string) error {
+    return c.Base().Run(args)
+}

dora/core/client/fs/src/main/java/alluxio/client/file/DelegatingFileSystem.java

@@ -13,6 +13,7 @@
 
 import alluxio.AlluxioURI;
 import alluxio.PositionReader;
+import alluxio.client.file.ufs.UfsBaseFileSystem;
 import alluxio.conf.AlluxioConfiguration;
 import alluxio.exception.AlluxioException;
 import alluxio.exception.DirectoryNotEmptyException;
@@ -50,6 +51,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Consumer;
+import javax.annotation.Nullable;
 
 /**
  * A wrapper of a FileSystem instance.
@@ -260,6 +262,18 @@ public void close() throws IOException {
     mDelegatedFileSystem.close();
   }
 
+  @Nullable
+  @Override
+  public DoraCacheFileSystem getDoraCacheFileSystem() {
+    return mDelegatedFileSystem.getDoraCacheFileSystem();
+  }
+
+  @Nullable
+  @Override
+  public UfsBaseFileSystem getUfsBaseFileSystem() {
+    return mDelegatedFileSystem.getUfsBaseFileSystem();
+  }
+
   /**
    * @return the underlying fileSystem
    */