diff --git a/pkg/ruler/base/mapper.go b/pkg/ruler/base/mapper.go
index 06ce873c47f2..9da3e43aebad 100644
--- a/pkg/ruler/base/mapper.go
+++ b/pkg/ruler/base/mapper.go
@@ -3,6 +3,7 @@ package base
 import (
 	"crypto/md5"
 	"net/url"
+	"os"
 	"path/filepath"
 	"sort"
 
@@ -60,6 +61,11 @@ func (m *mapper) users() ([]string, error) {
 	var result []string
 
 	dirs, err := afero.ReadDir(m.FS, m.Path)
+	if os.IsNotExist(err) {
+		// The directory may have not been created yet. With regards to this function
+		// it's like the ruler has no tenants and it shouldn't be considered an error.
+		return nil, nil
+	}
 	for _, u := range dirs {
 		if u.IsDir() {
 			result = append(result, u.Name())
diff --git a/pkg/ruler/base/mapper_test.go b/pkg/ruler/base/mapper_test.go
index 35586641763b..a5519e1448c0 100644
--- a/pkg/ruler/base/mapper_test.go
+++ b/pkg/ruler/base/mapper_test.go
@@ -429,3 +429,15 @@ func TestYamlFormatting(t *testing.T) {
 
 	require.Equal(t, expected, string(data))
 }
+
+func Test_mapper_CleanupShouldNotFailIfPathDoesNotExist(t *testing.T) {
+	m := &mapper{
+		Path:   "/path-does-not-exist",
+		FS:     afero.NewMemMapFs(),
+		logger: log.NewNopLogger(),
+	}
+
+	actual, err := m.users()
+	require.NoError(t, err)
+	require.Empty(t, actual)
+}