diff --git a/spec/support/split_specs.rb b/spec/support/split_specs.rb
index 18b9557ff..d554bd3d8 100644
--- a/spec/support/split_specs.rb
+++ b/spec/support/split_specs.rb
@@ -14,10 +14,16 @@
 raise 'expected "groupN"' unless num =~ /group(\d+)/
 num = $1.to_i - 1
 
-list = Dir['spec/**/*_spec.rb']
-slice_size = (list.size / groups.to_f).ceil
+raise "expected 1-#{groups}" unless (0...groups.to_i).include?(num)
+
+list = Dir['spec/**/*_spec.rb', 'test/**/*_test.rb']
+
+rand = Random.new(42)
+sorted_list = list.sort_by { |item| rand.rand }
+
+slice_size = (sorted_list.size / groups.to_f).ceil
 groups = []
-list.each_slice(slice_size).each do |group|
+sorted_list.each_slice(slice_size).each do |group|
   groups << group
 end
 
diff --git a/test/ruby/natalie_test.rb b/test/ruby/natalie_test.rb
index 23e05aa17..f1bf50157 100644
--- a/test/ruby/natalie_test.rb
+++ b/test/ruby/natalie_test.rb
@@ -10,7 +10,16 @@
   parallelize_me!
 
   Dir.chdir File.expand_path('../..', __dir__)
-  Dir['test/natalie/**/*_test.rb'].each do |path|
+  files = Dir['test/natalie/**/*_test.rb']
+
+  if !(glob = ENV['GLOB']).to_s.empty?
+    glob_files = files & Dir[*glob.split(',')]
+    puts "Matched #{glob_files.size} out of #{files.size} total test files:"
+    puts glob_files.to_a
+    files = glob_files
+  end
+
+  files.each do |path|
     code = File.read(path, encoding: 'utf-8')
     describe path do
       if code =~ /# skip-ruby/
diff --git a/test/ruby/ruby_specs_test.rb b/test/ruby/ruby_specs_test.rb
index d4172de46..6b66e4fe1 100644
--- a/test/ruby/ruby_specs_test.rb
+++ b/test/ruby/ruby_specs_test.rb
@@ -17,20 +17,23 @@ def spec_timeout(path)
   end
 
   Dir.chdir File.expand_path('../..', __dir__)
-  glob = if ENV['DEBUG_COMPILER']
-           # I use this when I'm working on the compiler,
-           # as it catches 99% of bugs and finishes a lot quicker.
-           Dir['spec/language/*_spec.rb']
-         elsif !(glob = ENV['GLOB']).to_s.empty?
-           # GLOB="spec/core/io/*_spec.rb,spec/core/thread/*_spec.rb" rake test
-           Dir[*glob.split(',')].tap do |files|
-             puts 'Matched files:'
-             puts files.to_a
-           end
-         else
-           Dir['spec/**/*_spec.rb']
-         end
-  glob.each do |path|
+  files = if ENV['DEBUG_COMPILER']
+            # I use this when I'm working on the compiler,
+            # as it catches 99% of bugs and finishes a lot quicker.
+            Dir['spec/language/*_spec.rb']
+          else
+            Dir['spec/**/*_spec.rb']
+          end
+
+  if !(glob = ENV['GLOB']).to_s.empty?
+    # GLOB="spec/core/io/*_spec.rb,spec/core/thread/*_spec.rb" rake test
+    glob_files = files & Dir[*glob.split(',')]
+    puts "Matched #{glob_files.size} out of #{files.size} total spec files:"
+    puts glob_files.to_a
+    files = glob_files
+  end
+
+  files.each do |path|
     describe path do
       it 'passes all specs' do
         out_nat = Timeout.timeout(spec_timeout(path), nil, "execution expired running: #{path}") do