diff --git a/pkg/v1/remote/descriptor.go b/pkg/v1/remote/descriptor.go
index 527a742a4..7db91a743 100644
--- a/pkg/v1/remote/descriptor.go
+++ b/pkg/v1/remote/descriptor.go
@@ -330,13 +330,14 @@ func (f *fetcher) fetchReferrers(ctx context.Context, filter map[string]string,
 	// The registry doesn't support the Referrers API endpoint, so we'll use the fallback tag scheme.
 	b, _, err := f.fetchManifest(ctx, fallbackTag(d), []types.MediaType{types.OCIImageIndex})
 	if err != nil {
+		var terr *transport.Error
+		if ok := errors.As(err, &terr); ok && terr.StatusCode == http.StatusNotFound {
+			// Not found just means there are no attachments yet. Start with an empty manifest.
+			return &v1.IndexManifest{MediaType: types.OCIImageIndex}, nil
+		}
+
 		return nil, err
 	}
-	var terr *transport.Error
-	if ok := errors.As(err, &terr); ok && terr.StatusCode == http.StatusNotFound {
-		// Not found just means there are no attachments yet. Start with an empty manifest.
-		return &v1.IndexManifest{MediaType: types.OCIImageIndex}, nil
-	}
 
 	var im v1.IndexManifest
 	if err := json.Unmarshal(b, &im); err != nil {
diff --git a/pkg/v1/remote/referrers_test.go b/pkg/v1/remote/referrers_test.go
index 91f9edcc6..75521df2f 100644
--- a/pkg/v1/remote/referrers_test.go
+++ b/pkg/v1/remote/referrers_test.go
@@ -94,6 +94,16 @@ func TestReferrers(t *testing.T) {
 		rootDesc := descriptor(rootImg)
 		t.Logf("root image is %s", rootDesc.Digest)
 
+		// Before pushing referrers, try to get the referrers of the root image.
+		rootRefDigest := rootRef.Context().Digest(rootDesc.Digest.String())
+		index, err := remote.Referrers(rootRefDigest)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if numManifests := len(index.Manifests); numManifests != 0 {
+			t.Fatalf("expected index to contain 0 manifests, but had %d", numManifests)
+		}
+
 		// Push an image that refers to the root image as its subject.
 		leafRef, err := name.ParseReference(fmt.Sprintf("%s/repo:leaf", u.Host))
 		if err != nil {
@@ -112,8 +122,7 @@ func TestReferrers(t *testing.T) {
 		t.Logf("leaf image is %s", leafDesc.Digest)
 
 		// Get the referrers of the root image, by digest.
-		rootRefDigest := rootRef.Context().Digest(rootDesc.Digest.String())
-		index, err := remote.Referrers(rootRefDigest)
+		index, err = remote.Referrers(rootRefDigest)
 		if err != nil {
 			t.Fatal(err)
 		}