diff --git a/.gitignore b/.gitignore
index 6a513a061947e..70bbd4ecd1e92 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,4 @@
 .DS_Store
 bin/configlet
 bin/configlet.exe
+tmp/
diff --git a/exercises/.keep b/exercises/.keep
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/exercises/complex-numbers/HINTS.md b/exercises/complex-numbers/.meta/hints.md
similarity index 100%
rename from exercises/complex-numbers/HINTS.md
rename to exercises/complex-numbers/.meta/hints.md
diff --git a/exercises/custom-set/HINTS.md b/exercises/custom-set/.meta/hints.md
similarity index 100%
rename from exercises/custom-set/HINTS.md
rename to exercises/custom-set/.meta/hints.md
diff --git a/exercises/robot-name/HINTS.md b/exercises/robot-name/.meta/hints.md
similarity index 100%
rename from exercises/robot-name/HINTS.md
rename to exercises/robot-name/.meta/hints.md
diff --git a/exercises/rotational-cipher/HINTS.md b/exercises/rotational-cipher/.meta/hints.md
similarity index 100%
rename from exercises/rotational-cipher/HINTS.md
rename to exercises/rotational-cipher/.meta/hints.md
diff --git a/runtests.jl b/runtests.jl
index 4aad3283f10da..ff116cca69b9e 100644
--- a/runtests.jl
+++ b/runtests.jl
@@ -13,32 +13,31 @@ macro test_broken(ex)
     @test eval(current_module(), ex)
 end
 
-for (root, dirs, files) in walkdir("exercises")
-    for exercise in dirs
-        # Allow only testing specified execises
-        if !isempty(ARGS) && !(exercise in ARGS)
-            continue
-        end
+for exercise in readdir("exercises")
+    # Allow only testing specified execises
+    if !isempty(ARGS) && !(exercise in ARGS)
+        continue
+    end
+
+    exercise_path = joinpath("exercises", exercise)
+    isdir(exercise_path) || continue
+
+    # Create temporary directory
+    temp_path = mktempdir(".")
+
+    # Copy tests & example to the temporary directory
+    cp(joinpath(exercise_path, "example.jl"), joinpath(temp_path, "$exercise.jl"))
+    cp(joinpath(exercise_path, "runtests.jl"), joinpath(temp_path, "runtests.jl"))
 
-        exercise_path = joinpath("exercises", exercise)
-
-        # Create temporary directory
-        temp_path = mktempdir(root)
-
-        # Copy tests & example to the temporary directory
-        cp(joinpath(exercise_path, "example.jl"), joinpath(temp_path, "$exercise.jl"))
-        cp(joinpath(exercise_path, "runtests.jl"), joinpath(temp_path, "runtests.jl"))
-
-        try
-            # Run the tests
-            @testset "$exercise example" begin
-                # Run the tests within an anonymous module to prevent definitions from
-                # one exercise leaking into another.
-                eval(Module(), :(include(joinpath($temp_path, "runtests.jl"))))
-            end
-        finally
-            # Delete the temporary directory
-            rm(temp_path, recursive=true)
+    try
+        # Run the tests
+        @testset "$exercise example" begin
+            # Run the tests within an anonymous module to prevent definitions from
+            # one exercise leaking into another.
+            eval(Module(), :(include(joinpath($temp_path, "runtests.jl"))))
         end
+    finally
+        # Delete the temporary directory
+        rm(temp_path, recursive=true)
     end
 end