Add Dir::GlobOptions to control Dir.glob's behavior (crystal-lang…

…#13550)

spec/std/dir_spec.cr

@@ -25,6 +25,20 @@ private def unset_tempdir(&)
   {% end %}
 end
 
+{% if flag?(:win32) %}
+  private def make_hidden(path)
+    wstr = Crystal::System.to_wstr(path)
+    attributes = LibC.GetFileAttributesW(wstr)
+    LibC.SetFileAttributesW(wstr, attributes | LibC::FILE_ATTRIBUTE_HIDDEN)
+  end
+
+  private def make_system(path)
+    wstr = Crystal::System.to_wstr(path)
+    attributes = LibC.GetFileAttributesW(wstr)
+    LibC.SetFileAttributesW(wstr, attributes | LibC::FILE_ATTRIBUTE_SYSTEM)
+  end
+{% end %}
+
 private def it_raises_on_null_byte(operation, &block)
   it "errors on #{operation}" do
     expect_raises(ArgumentError, "String contains null byte") do
@@ -438,26 +452,86 @@ describe "Dir" do
       ].sort
     end
 
-    context "match_hidden: true" do
-      it "matches hidden files" do
+    context "match: :dot_files / match_hidden" do
+      it "matches dot files" do
+        Dir.glob("#{datapath}/dir/dots/**/*", match: :dot_files).sort.should eq [
+          datapath("dir", "dots", ".dot.hidden"),
+          datapath("dir", "dots", ".hidden"),
+          datapath("dir", "dots", ".hidden", "f1.txt"),
+        ].sort
         Dir.glob("#{datapath}/dir/dots/**/*", match_hidden: true).sort.should eq [
           datapath("dir", "dots", ".dot.hidden"),
           datapath("dir", "dots", ".hidden"),
           datapath("dir", "dots", ".hidden", "f1.txt"),
         ].sort
       end
-    end
 
-    context "match_hidden: false" do
       it "ignores hidden files" do
-        Dir.glob("#{datapath}/dir/dots/*", match_hidden: false).size.should eq 0
+        Dir.glob("#{datapath}/dir/dots/*", match: :none).should be_empty
+        Dir.glob("#{datapath}/dir/dots/*", match_hidden: false).should be_empty
       end
 
       it "ignores hidden files recursively" do
-        Dir.glob("#{datapath}/dir/dots/**/*", match_hidden: false).size.should eq 0
+        Dir.glob("#{datapath}/dir/dots/**/*", match: :none).should be_empty
+        Dir.glob("#{datapath}/dir/dots/**/*", match_hidden: false).should be_empty
       end
     end
 
+    {% if flag?(:win32) %}
+      it "respects `NativeHidden` and `OSHidden`" do
+        with_tempfile("glob-system-hidden") do |path|
+          FileUtils.mkdir_p(path)
+
+          visible_txt = File.join(path, "visible.txt")
+          hidden_txt = File.join(path, "hidden.txt")
+          system_txt = File.join(path, "system.txt")
+          system_hidden_txt = File.join(path, "system_hidden.txt")
+
+          File.write(visible_txt, "")
+          File.write(hidden_txt, "")
+          File.write(system_txt, "")
+          File.write(system_hidden_txt, "")
+          make_hidden(hidden_txt)
+          make_hidden(system_hidden_txt)
+          make_system(system_txt)
+          make_system(system_hidden_txt)
+
+          visible_dir = File.join(path, "visible_dir")
+          hidden_dir = File.join(path, "hidden_dir")
+          system_dir = File.join(path, "system_dir")
+          system_hidden_dir = File.join(path, "system_hidden_dir")
+
+          Dir.mkdir(visible_dir)
+          Dir.mkdir(hidden_dir)
+          Dir.mkdir(system_dir)
+          Dir.mkdir(system_hidden_dir)
+          make_hidden(hidden_dir)
+          make_hidden(system_hidden_dir)
+          make_system(system_dir)
+          make_system(system_hidden_dir)
+
+          inside_visible = File.join(visible_dir, "inside.txt")
+          inside_hidden = File.join(hidden_dir, "inside.txt")
+          inside_system = File.join(system_dir, "inside.txt")
+          inside_system_hidden = File.join(system_hidden_dir, "inside.txt")
+
+          File.write(inside_visible, "")
+          File.write(inside_hidden, "")
+          File.write(inside_system, "")
+          File.write(inside_system_hidden, "")
+
+          expected = [visible_txt, visible_dir, inside_visible, system_txt, system_dir, inside_system].sort!
+          expected_hidden = (expected + [hidden_txt, hidden_dir, inside_hidden]).sort!
+          expected_system_hidden = (expected_hidden + [system_hidden_txt, system_hidden_dir, inside_system_hidden]).sort!
+
+          Dir.glob("#{path}/**/*", match: :none).sort.should eq(expected)
+          Dir.glob("#{path}/**/*", match: :native_hidden).sort.should eq(expected_hidden)
+          Dir.glob("#{path}/**/*", match: :os_hidden).sort.should eq(expected)
+          Dir.glob("#{path}/**/*", match: File::MatchOptions[NativeHidden, OSHidden]).sort.should eq(expected_system_hidden)
+        end
+      end
+    {% end %}
+
     context "with path" do
       expected = [
         datapath("dir", "f1.txt"),

src/crystal/system/dir.cr

@@ -4,17 +4,19 @@ module Crystal::System::Dir
   #
   # Information about a directory entry.
   #
-  # In particular we only care about the name and whether its
-  # a directory or not to improve the performance of Dir.glob
-  # by avoid having to call File.info on every directory entry.
+  # In particular we only care about the name, whether it's a directory, and
+  # whether any hidden file attributes are set to improve the performance of
+  # `Dir.glob` by not having to call `File.info` on every directory entry.
   # If dir is nil, the type is unknown.
   # In the future we might change Dir's API to expose these entries
   # with more info but right now it's not necessary.
   struct Entry
-    getter name
-    getter? dir
+    getter name : String
+    getter? dir : Bool?
+    getter? native_hidden : Bool
+    getter? os_hidden : Bool
 
-    def initialize(@name : String, @dir : Bool?)
+    def initialize(@name, @dir, @native_hidden, @os_hidden = false)
     end
   end
 

src/crystal/system/unix/dir.cr

@@ -19,7 +19,11 @@ module Crystal::System::Dir
             when LibC::DT_UNKNOWN, LibC::DT_LNK then nil
             else                                     false
             end
-      Entry.new(name, dir)
+
+      # TODO: support `st_flags & UF_HIDDEN` on BSD-like systems: https://man.freebsd.org/cgi/man.cgi?query=stat&sektion=2
+      # TODO: support hidden file attributes on macOS / HFS+: https://stackoverflow.com/a/15236292
+      # (are these the same?)
+      Entry.new(name, dir, false)
     elsif Errno.value != Errno::NONE
       raise ::File::Error.from_errno("Error reading directory entries", file: path)
     else

src/crystal/system/wasi/dir.cr

@@ -56,7 +56,7 @@ module Crystal::System::Dir
              else                                 false
              end
 
-    Entry.new(name, is_dir)
+    Entry.new(name, is_dir, false)
   end
 
   def self.rewind(dir) : Nil

src/crystal/system/win32/dir.cr

@@ -55,9 +55,11 @@ module Crystal::System::Dir
   def self.data_to_entry(data)
     name = String.from_utf16(data.cFileName.to_unsafe)[0]
     unless data.dwFileAttributes.bits_set?(LibC::FILE_ATTRIBUTE_REPARSE_POINT) && data.dwReserved0 == LibC::IO_REPARSE_TAG_SYMLINK
-      dir = (data.dwFileAttributes & LibC::FILE_ATTRIBUTE_DIRECTORY) != 0
+      dir = data.dwFileAttributes.bits_set?(LibC::FILE_ATTRIBUTE_DIRECTORY)
     end
-    Entry.new(name, dir)
+    native_hidden = data.dwFileAttributes.bits_set?(LibC::FILE_ATTRIBUTE_HIDDEN)
+    os_hidden = native_hidden && data.dwFileAttributes.bits_set?(LibC::FILE_ATTRIBUTE_SYSTEM)
+    Entry.new(name, dir, native_hidden, os_hidden)
   end
 
   def self.rewind(dir : DirHandle) : Nil

src/dir/glob.cr

@@ -4,30 +4,76 @@ class Dir
   # The pattern syntax is similar to shell filename globbing, see `File.match?` for details.
   #
   # NOTE: Path separator in patterns needs to be always `/`. The returned file names use system-specific path separators.
-  def self.[](*patterns : Path | String, match_hidden = false, follow_symlinks = false) : Array(String)
-    glob(patterns, match_hidden: match_hidden, follow_symlinks: follow_symlinks)
+  def self.[](*patterns : Path | String, match : File::MatchOptions = File::MatchOptions.glob_default, follow_symlinks : Bool = false) : Array(String)
+    glob(patterns, match: match, follow_symlinks: follow_symlinks)
   end
 
   # :ditto:
-  def self.[](patterns : Enumerable, match_hidden = false, follow_symlinks = false) : Array(String)
-    glob(patterns, match_hidden: match_hidden, follow_symlinks: follow_symlinks)
+  def self.[](patterns : Enumerable, match : File::MatchOptions = File::MatchOptions.glob_default, follow_symlinks : Bool = false) : Array(String)
+    glob(patterns, match: match, follow_symlinks: follow_symlinks)
+  end
+
+  # :ditto:
+  #
+  # For compatibility, a falsey *match_hidden* argument is equivalent to passing
+  # `match: File::MatchOptions.glob_default`, and a truthy *match_hidden* is
+  # equivalent to
+  # `match: File::MatchOptions.glob_default | File::MatchOptions::DotFiles`.
+  @[Deprecated("Use the overload with a `match` parameter instead")]
+  def self.[](*patterns : Path | String, match_hidden, follow_symlinks = false) : Array(String)
+    glob(patterns, match: match_hidden_to_options(match_hidden), follow_symlinks: follow_symlinks)
+  end
+
+  # :ditto:
+  #
+  # For compatibility, a falsey *match_hidden* argument is equivalent to passing
+  # `match: File::MatchOptions.glob_default`, and a truthy *match_hidden* is
+  # equivalent to
+  # `match: File::MatchOptions.glob_default | File::MatchOptions::DotFiles`.
+  @[Deprecated("Use the overload with a `match` parameter instead")]
+  def self.[](patterns : Enumerable, match_hidden, follow_symlinks = false) : Array(String)
+    glob(patterns, match: match_hidden_to_options(match_hidden), follow_symlinks: follow_symlinks)
   end
 
   # Returns an array of all files that match against any of *patterns*.
   #
   # The pattern syntax is similar to shell filename globbing, see `File.match?` for details.
   #
-  # If *match_hidden* is `true` the pattern will match hidden files and folders.
-  #
   # NOTE: Path separator in patterns needs to be always `/`. The returned file names use system-specific path separators.
-  def self.glob(*patterns : Path | String, match_hidden = false, follow_symlinks = false) : Array(String)
-    glob(patterns, match_hidden: match_hidden, follow_symlinks: follow_symlinks)
+  def self.glob(*patterns : Path | String, match : File::MatchOptions = File::MatchOptions.glob_default, follow_symlinks : Bool = false) : Array(String)
+    glob(patterns, match: match, follow_symlinks: follow_symlinks)
   end
 
   # :ditto:
-  def self.glob(patterns : Enumerable, match_hidden = false, follow_symlinks = false) : Array(String)
+  def self.glob(patterns : Enumerable, match : File::MatchOptions = File::MatchOptions.glob_default, follow_symlinks : Bool = false) : Array(String)
     paths = [] of String
-    glob(patterns, match_hidden: match_hidden, follow_symlinks: follow_symlinks) do |path|
+    glob(patterns, match: match, follow_symlinks: follow_symlinks) do |path|
+      paths << path
+    end
+    paths
+  end
+
+  # :ditto:
+  #
+  # For compatibility, a falsey *match_hidden* argument is equivalent to passing
+  # `match: File::MatchOptions.glob_default`, and a truthy *match_hidden* is
+  # equivalent to
+  # `match: File::MatchOptions.glob_default | File::MatchOptions::DotFiles`.
+  @[Deprecated("Use the overload with a `match` parameter instead")]
+  def self.glob(*patterns : Path | String, match_hidden, follow_symlinks = false) : Array(String)
+    glob(patterns, match: match_hidden_to_options(match_hidden), follow_symlinks: follow_symlinks)
+  end
+
+  # :ditto:
+  #
+  # For compatibility, a falsey *match_hidden* argument is equivalent to passing
+  # `match: File::MatchOptions.glob_default`, and a truthy *match_hidden* is
+  # equivalent to
+  # `match: File::MatchOptions.glob_default | File::MatchOptions::DotFiles`.
+  @[Deprecated("Use the overload with a `match` parameter instead")]
+  def self.glob(patterns : Enumerable, match_hidden, follow_symlinks = false) : Array(String)
+    paths = [] of String
+    glob(patterns, match: match_hidden_to_options(match_hidden), follow_symlinks: follow_symlinks) do |path|
       paths << path
     end
     paths
@@ -37,22 +83,52 @@ class Dir
   #
   # The pattern syntax is similar to shell filename globbing, see `File.match?` for details.
   #
-  # If *match_hidden* is `true` the pattern will match hidden files and folders.
-  #
   # NOTE: Path separator in patterns needs to be always `/`. The returned file names use system-specific path separators.
-  def self.glob(*patterns : Path | String, match_hidden = false, follow_symlinks = false, &block : String -> _)
-    glob(patterns, match_hidden: match_hidden, follow_symlinks: follow_symlinks) do |path|
+  def self.glob(*patterns : Path | String, match : File::MatchOptions = File::MatchOptions.glob_default, follow_symlinks : Bool = false, &block : String -> _)
+    glob(patterns, match: match, follow_symlinks: follow_symlinks) do |path|
+      yield path
+    end
+  end
+
+  # :ditto:
+  def self.glob(patterns : Enumerable, match : File::MatchOptions = File::MatchOptions.glob_default, follow_symlinks : Bool = false, &block : String -> _)
+    Globber.glob(patterns, match: match, follow_symlinks: follow_symlinks) do |path|
       yield path
     end
   end
 
   # :ditto:
-  def self.glob(patterns : Enumerable, match_hidden = false, follow_symlinks = false, &block : String -> _)
-    Globber.glob(patterns, match_hidden: match_hidden, follow_symlinks: follow_symlinks) do |path|
+  #
+  # For compatibility, a falsey *match_hidden* argument is equivalent to passing
+  # `match: File::MatchOptions.glob_default`, and a truthy *match_hidden* is
+  # equivalent to
+  # `match: File::MatchOptions.glob_default | File::MatchOptions::DotFiles`.
+  @[Deprecated("Use the overload with a `match` parameter instead")]
+  def self.glob(*patterns : Path | String, match_hidden, follow_symlinks = false, &block : String -> _)
+    glob(patterns, match: match_hidden_to_options(match_hidden), follow_symlinks: follow_symlinks) do |path|
       yield path
     end
   end
 
+  # :ditto:
+  #
+  # For compatibility, a falsey *match_hidden* argument is equivalent to passing
+  # `match: File::MatchOptions.glob_default`, and a truthy *match_hidden* is
+  # equivalent to
+  # `match: File::MatchOptions.glob_default | File::MatchOptions::DotFiles`.
+  @[Deprecated("Use the overload with a `match` parameter instead")]
+  def self.glob(patterns : Enumerable, match_hidden, follow_symlinks = false, &block : String -> _)
+    Globber.glob(patterns, match: match_hidden_to_options(match_hidden), follow_symlinks: follow_symlinks) do |path|
+      yield path
+    end
+  end
+
+  private def self.match_hidden_to_options(match_hidden)
+    options = File::MatchOptions.glob_default
+    options |= File::MatchOptions::DotFiles if match_hidden
+    options
+  end
+
   # :nodoc:
   module Globber
     record DirectoriesOnly
@@ -72,7 +148,7 @@ class Dir
     end
     alias PatternType = DirectoriesOnly | ConstantEntry | EntryMatch | RecursiveDirectories | ConstantDirectory | RootDirectory | DirectoryMatch
 
-    def self.glob(patterns : Enumerable, *, match_hidden, follow_symlinks, &block : String -> _)
+    def self.glob(patterns : Enumerable, *, match, follow_symlinks, &block : String -> _)
       patterns.each do |pattern|
         if pattern.is_a?(Path)
           pattern = pattern.to_posix.to_s
@@ -81,11 +157,11 @@ class Dir
 
         sequences.each do |sequence|
           if sequence.count(&.is_a?(RecursiveDirectories)) > 1
-            run_tracking(sequence, match_hidden: match_hidden, follow_symlinks: follow_symlinks) do |match|
+            run_tracking(sequence, match: match, follow_symlinks: follow_symlinks) do |match|
               yield match
             end
           else
-            run(sequence, match_hidden: match_hidden, follow_symlinks: follow_symlinks) do |match|
+            run(sequence, match: match, follow_symlinks: follow_symlinks) do |match|
               yield match
             end
           end
@@ -153,17 +229,17 @@ class Dir
       true
     end
 
-    private def self.run_tracking(sequence, match_hidden, follow_symlinks, &block : String -> _)
+    private def self.run_tracking(sequence, match, follow_symlinks, &block : String -> _)
       result_tracker = Set(String).new
 
-      run(sequence, match_hidden, follow_symlinks) do |result|
+      run(sequence, match, follow_symlinks) do |result|
         if result_tracker.add?(result)
           yield result
         end
       end
     end
 
-    private def self.run(sequence, match_hidden, follow_symlinks, &block : String -> _)
+    private def self.run(sequence, match, follow_symlinks, &block : String -> _)
       return if sequence.empty?
 
       path_stack = [] of Tuple(Int32, String?, Crystal::System::Dir::Entry?)
@@ -195,7 +271,7 @@ class Dir
         in EntryMatch
           next if sequence[pos + 1]?.is_a?(RecursiveDirectories)
           each_child(path) do |entry|
-            next if !match_hidden && entry.name.starts_with?('.')
+            next unless matches_file?(entry, match)
             yield join(path, entry.name) if cmd.matches?(entry.name)
           end
         in DirectoryMatch
@@ -255,7 +331,7 @@ class Dir
 
             if entry = read_entry(dir)
               next if entry.name.in?(".", "..")
-              next if !match_hidden && entry.name.starts_with?('.')
+              next unless matches_file?(entry, match)
 
               if dir_path.bytesize == 0
                 fullpath = entry.name
@@ -340,5 +416,12 @@ class Dir
       # call File.info? which is really expensive.
       Crystal::System::Dir.next_entry(dir.@dir, dir.path)
     end
+
+    private def self.matches_file?(entry, match)
+      return false if entry.name.starts_with?('.') && !match.dot_files?
+      return false if entry.native_hidden? && !match.native_hidden?
+      return false if entry.os_hidden? && !match.os_hidden?
+      true
+    end
   end
 end