diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index bdb4fa6556e..f19fb20bf1c 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -69,7 +69,7 @@ int isValidPath(char * path)
 SV * queryReferences(char * path)
     PPCODE:
         try {
-            for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->referencesPossiblyToSelf())
+            for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references)
                 XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0)));
         } catch (Error & e) {
             croak("%s", e.what());
@@ -110,7 +110,7 @@ SV * queryPathInfo(char * path, int base32)
             mXPUSHi(info->registrationTime);
             mXPUSHi(info->narSize);
             AV * refs = newAV();
-            for (auto & i : info->referencesPossiblyToSelf())
+            for (auto & i : info->references)
                 av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0));
             XPUSHs(sv_2mortal(newRV((SV *) refs)));
             AV * sigs = newAV();
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index ae573cf4d75..3b32625b1f7 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1544,8 +1544,7 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V
     StorePathSet refs;
     if (state.store->isInStore(path)) {
         try {
-            // FIXME: Are self references becoming non-self references OK?
-            refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->referencesPossiblyToSelf();
+            refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references;
         } catch (Error &) { // FIXME: should be InvalidPathError
         }
         // Re-scan references to filter down to just the ones that actually occur in the file.
@@ -1980,7 +1979,7 @@ static void addPath(
             try {
                 auto [storePath, subPath] = state.store->toStorePath(path);
                 // FIXME: we should scanForReferences on the path before adding it
-                refs = state.store->queryPathInfo(storePath)->referencesPossiblyToSelf();
+                refs = state.store->queryPathInfo(storePath)->references;
                 path = state.store->toRealPath(storePath) + subPath;
             } catch (Error &) { // FIXME: should be InvalidPathError
             }
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 087b376551a..ac41add2c73 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -180,11 +180,11 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
         duration);
 
     /* Verify that all references are valid. This may do some .narinfo
-       reads, but typically they'll already be cached. Note that
-       self-references are always valid. */
-    for (auto & ref : info.references.others)
+       reads, but typically they'll already be cached. */
+    for (auto & ref : info.references)
         try {
-            queryPathInfo(ref);
+            if (ref != info.path)
+                queryPathInfo(ref);
         } catch (InvalidPath &) {
             throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid",
                 printStorePath(info.path), printStorePath(ref));
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index d96858fc076..bb4f929898e 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -2523,7 +2523,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
                 auto narHashAndSize = hashPath(htSHA256, actualPath);
                 ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
                 newInfo0.narSize = narHashAndSize.second;
-                newInfo0.references = rewriteRefs();
+                auto refs = rewriteRefs();
+                newInfo0.references = std::move(refs.others);
+                if (refs.self)
+                    newInfo0.references.insert(newInfo0.path);
                 return newInfo0;
             },
 
@@ -2774,12 +2777,12 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
                 auto i = outputsByPath.find(worker.store.printStorePath(path));
                 if (i != outputsByPath.end()) {
                     closureSize += i->second.narSize;
-                    for (auto & ref : i->second.referencesPossiblyToSelf())
+                    for (auto & ref : i->second.references)
                         pathsLeft.push(ref);
                 } else {
                     auto info = worker.store.queryPathInfo(path);
                     closureSize += info->narSize;
-                    for (auto & ref : info->referencesPossiblyToSelf())
+                    for (auto & ref : info->references)
                         pathsLeft.push(ref);
                 }
             }
@@ -2819,7 +2822,7 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
 
                 auto used = recursive
                     ? getClosure(info.path).first
-                    : info.referencesPossiblyToSelf();
+                    : info.references;
 
                 if (recursive && checks.ignoreSelfRefs)
                     used.erase(info.path);
diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
index 36b0ea7a70e..994cb4ac2f6 100644
--- a/src/libstore/build/substitution-goal.cc
+++ b/src/libstore/build/substitution-goal.cc
@@ -165,8 +165,9 @@ void PathSubstitutionGoal::tryNext()
 
     /* To maintain the closure invariant, we first have to realise the
        paths referenced by this one. */
-    for (auto & i : info->references.others)
-        addWaitee(worker.makePathSubstitutionGoal(i));
+    for (auto & i : info->references)
+        if (i != storePath) /* ignore self-references */
+            addWaitee(worker.makePathSubstitutionGoal(i));
 
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
         referencesValid();
@@ -187,8 +188,9 @@ void PathSubstitutionGoal::referencesValid()
         return;
     }
 
-    for (auto & i : info->references.others)
-        assert(worker.store.isValidPath(i));
+    for (auto & i : info->references)
+        if (i != storePath) /* ignore self-references */
+            assert(worker.store.isValidPath(i));
 
     state = &PathSubstitutionGoal::tryToRun;
     worker.wakeUp(shared_from_this());
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index 3b8a773b74c..a98e34cb8a5 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -151,6 +151,16 @@ Hash getContentAddressHash(const ContentAddress & ca)
     }, ca);
 }
 
+bool StoreReferences::empty() const
+{
+    return !self && others.empty();
+}
+
+size_t StoreReferences::size() const
+{
+    return (self ? 1 : 0) + others.size();
+}
+
 ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) {
     return std::visit(overloaded {
         [&](const TextHash & h) -> ContentAddressWithReferences {
diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh
index f8a4d53701d..4a50bbee0f3 100644
--- a/src/libstore/content-address.hh
+++ b/src/libstore/content-address.hh
@@ -4,7 +4,6 @@
 #include "hash.hh"
 #include "path.hh"
 #include "comparator.hh"
-#include "reference-set.hh"
 
 namespace nix {
 
@@ -95,7 +94,15 @@ Hash getContentAddressHash(const ContentAddress & ca);
  * References set
  */
 
-typedef References<StorePath> StoreReferences;
+struct StoreReferences {
+    StorePathSet others;
+    bool self = false;
+
+    bool empty() const;
+    size_t size() const;
+
+    GENERATE_CMP(StoreReferences, me->self, me->others);
+};
 
 /*
  * Full content address
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 605f871fc20..12596ba4983 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -336,7 +336,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         logger->startWork();
         StorePathSet paths;
         if (op == wopQueryReferences)
-            for (auto & i : store->queryPathInfo(path)->referencesPossiblyToSelf())
+            for (auto & i : store->queryPathInfo(path)->references)
                 paths.insert(i);
         else if (op == wopQueryReferrers)
             store->queryReferrers(path, paths);
@@ -758,7 +758,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         else {
             to << 1
                << (i->second.deriver ? store->printStorePath(*i->second.deriver) : "");
-            worker_proto::write(*store, to, i->second.references.possiblyToSelf(path));
+            worker_proto::write(*store, to, i->second.references);
             to << i->second.downloadSize
                << i->second.narSize;
         }
@@ -781,7 +781,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         for (auto & i : infos) {
             to << store->printStorePath(i.first)
                << (i.second.deriver ? store->printStorePath(*i.second.deriver) : "");
-            worker_proto::write(*store, to, i.second.references.possiblyToSelf(i.first));
+            worker_proto::write(*store, to, i.second.references);
             to << i.second.downloadSize << i.second.narSize;
         }
         break;
@@ -863,7 +863,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         ValidPathInfo info { path, narHash };
         if (deriver != "")
             info.deriver = store->parseStorePath(deriver);
-        info.setReferencesPossiblyToSelf(worker_proto::read(*store, from, Phantom<StorePathSet> {}));
+        info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {});
         from >> info.registrationTime >> info.narSize >> info.ultimate;
         info.sigs = readStrings<StringSet>(from);
         info.ca = parseContentAddressOpt(readString(from));
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 4adf5157338..9875da90906 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
     teeSink
         << exportMagic
         << printStorePath(path);
-    worker_proto::write(*this, teeSink, info->referencesPossiblyToSelf());
+    worker_proto::write(*this, teeSink, info->references);
     teeSink
         << (info->deriver ? printStorePath(*info->deriver) : "")
         << 0;
@@ -80,7 +80,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
         ValidPathInfo info { path, narHash };
         if (deriver != "")
             info.deriver = parseStorePath(deriver);
-        info.setReferencesPossiblyToSelf(std::move(references));
+        info.references = references;
         info.narSize = saved.s.size();
 
         // Ignore optional legacy signature.
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 6a694f03427..2c9dd26808a 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -137,7 +137,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info->deriver = parseStorePath(deriver);
-            info->setReferencesPossiblyToSelf(worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}));
+            info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
             readLongLong(conn->from); // download size
             info->narSize = readLongLong(conn->from);
 
@@ -171,7 +171,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
                 << printStorePath(info.path)
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << info.narHash.to_string(Base16, false);
-            worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf());
+            worker_proto::write(*this, conn->to, info.references);
             conn->to
                 << info.registrationTime
                 << info.narSize
@@ -200,7 +200,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             conn->to
                 << exportMagic
                 << printStorePath(info.path);
-            worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf());
+            worker_proto::write(*this, conn->to, info.references);
             conn->to
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index b32953f3ff9..2d03d2d8b6a 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -938,8 +938,7 @@ std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & s
     auto useQueryReferences(state.stmts->QueryReferences.use()(info->id));
 
     while (useQueryReferences.next())
-        info->insertReferencePossiblyToSelf(
-            parseStorePath(useQueryReferences.getStr(0)));
+        info->references.insert(parseStorePath(useQueryReferences.getStr(0)));
 
     return info;
 }
@@ -1206,7 +1205,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
 
         for (auto & [_, i] : infos) {
             auto referrer = queryValidPathId(*state, i.path);
-            for (auto & j : i.referencesPossiblyToSelf())
+            for (auto & j : i.references)
                 state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
         }
 
@@ -1227,7 +1226,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
         topoSort(paths,
             {[&](const StorePath & path) {
                 auto i = infos.find(path);
-                return i == infos.end() ? StorePathSet() : i->second.references.others;
+                return i == infos.end() ? StorePathSet() : i->second.references;
             }},
             {[&](const StorePath & path, const StorePath & parent) {
                 return BuildError(
@@ -1525,8 +1524,7 @@ StorePath LocalStore::addTextToStore(
 
             ValidPathInfo info { dstPath, narHash };
             info.narSize = sink.s.size();
-            // No self reference allowed with text-hashing
-            info.references.others = references;
+            info.references = references;
             info.ca = TextHash { .hash = hash };
             registerValidPath(info);
         }
diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc
index 5d7945eb1c7..09f615439cb 100644
--- a/src/libstore/make-content-addressed.cc
+++ b/src/libstore/make-content-addressed.cc
@@ -28,8 +28,9 @@ std::map<StorePath, StorePath> makeContentAddressed(
         StringMap rewrites;
 
         StoreReferences refs;
-        refs.self = oldInfo->references.self;
-        for (auto & ref : oldInfo->references.others) {
+        for (auto & ref : oldInfo->references) {
+            if (ref == path)
+                refs.self = true;
             auto i = remappings.find(ref);
             auto replacement = i != remappings.end() ? i->second : ref;
             // FIXME: warn about unremapped paths?
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 70e97569a9a..da96dcebcd7 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -40,8 +40,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
                         std::future<ref<const ValidPathInfo>> & fut) {
             StorePathSet res;
             auto info = fut.get();
-            for (auto & ref : info->references.others)
-                res.insert(ref);
+            for (auto & ref : info->references)
+                if (ref != path)
+                    res.insert(ref);
 
             if (includeOutputs && path.isDerivation())
                 for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
@@ -223,7 +224,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
                 state->narSize += info->second.narSize;
             }
 
-            for (auto & ref : info->second.references.others)
+            for (auto & ref : info->second.references)
                 pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref }));
           },
         }, req.raw());
@@ -241,7 +242,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
     return topoSort(paths,
         {[&](const StorePath & path) {
             try {
-                return queryPathInfo(path)->references.others;
+                return queryPathInfo(path)->references;
             } catch (InvalidPath &) {
                 return StorePathSet();
             }
@@ -297,7 +298,7 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
 
     auto info = store.queryPathInfo(outputPath);
 
-    return drvOutputReferences(Realisation::closure(store, inputRealisations), info->referencesPossiblyToSelf());
+    return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
 }
 
 OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 8d4a21daf81..3e0689534b6 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -248,7 +248,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache
             narInfo->fileSize = queryNAR.getInt(5);
             narInfo->narSize = queryNAR.getInt(7);
             for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
-                narInfo->insertReferencePossiblyToSelf(StorePath(r));
+                narInfo->references.insert(StorePath(r));
             if (!queryNAR.isNull(9))
                 narInfo->deriver = StorePath(queryNAR.getStr(9));
             for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index f54e8f1fc3f..071d8355e5c 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -63,7 +63,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
             auto refs = tokenizeString<Strings>(value, " ");
             if (!references.empty()) throw corrupt();
             for (auto & r : refs)
-                insertReferencePossiblyToSelf(StorePath(r));
+                references.insert(StorePath(r));
         }
         else if (name == "Deriver") {
             if (value != "unknown-deriver")
diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc
index 2972c0bbeef..93f91e702fd 100644
--- a/src/libstore/path-info.cc
+++ b/src/libstore/path-info.cc
@@ -12,7 +12,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
         "1;" + store.printStorePath(path) + ";"
         + narHash.to_string(Base32, true) + ";"
         + std::to_string(narSize) + ";"
-        + concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf()));
+        + concatStringsSep(",", store.printStorePathSet(references));
 }
 
 
@@ -30,16 +30,25 @@ std::optional<StorePathDescriptor> ValidPathInfo::fullStorePathDescriptorOpt() c
         .name = std::string { path.name() },
         .info = std::visit(overloaded {
             [&](const TextHash & th) -> ContentAddressWithReferences {
-                assert(!references.self);
+                assert(references.count(path) == 0);
                 return TextInfo {
                     th,
-                    .references = references.others,
+                    .references = references,
                 };
             },
             [&](const FixedOutputHash & foh) -> ContentAddressWithReferences {
+                auto refs = references;
+                bool hasSelfReference = false;
+                if (refs.count(path)) {
+                    hasSelfReference = true;
+                    refs.erase(path);
+                }
                 return FixedOutputInfo {
                     foh,
-                    .references = references,
+                    .references = {
+                        .others = std::move(refs),
+                        .self = hasSelfReference,
+                    },
                 };
             },
         }, *ca),
@@ -85,7 +94,7 @@ bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publi
 Strings ValidPathInfo::shortRefs() const
 {
     Strings refs;
-    for (auto & r : referencesPossiblyToSelf())
+    for (auto & r : references)
         refs.push_back(std::string(r.to_string()));
     return refs;
 }
@@ -100,36 +109,19 @@ ValidPathInfo::ValidPathInfo(
 {
     std::visit(overloaded {
         [this](TextInfo && ti) {
-            this->references = {
-                .others = std::move(ti.references),
-                .self = false,
-            };
+            this->references = std::move(ti.references);
             this->ca = std::move((TextHash &&) ti);
         },
         [this](FixedOutputInfo && foi) {
-            this->references = std::move(foi.references);
+            this->references = std::move(foi.references.others);
+            if (foi.references.self)
+                this->references.insert(path);
             this->ca = std::move((FixedOutputHash &&) foi);
         },
     }, std::move(info.info));
 }
 
 
-StorePathSet ValidPathInfo::referencesPossiblyToSelf() const
-{
-    return references.possiblyToSelf(path);
-}
-
-void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref)
-{
-    return references.insertPossiblyToSelf(path, std::move(ref));
-}
-
-void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs)
-{
-    return references.setPossiblyToSelf(path, std::move(refs));
-}
-
-
 ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format)
 {
     return read(source, store, format, store.parseStorePath(readString(source)));
@@ -141,7 +133,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned
     auto narHash = Hash::parseAny(readString(source), htSHA256);
     ValidPathInfo info(path, narHash);
     if (deriver != "") info.deriver = store.parseStorePath(deriver);
-    info.setReferencesPossiblyToSelf(worker_proto::read(store, source, Phantom<StorePathSet> {}));
+    info.references = worker_proto::read(store, source, Phantom<StorePathSet> {});
     source >> info.registrationTime >> info.narSize;
     if (format >= 16) {
         source >> info.ultimate;
@@ -162,7 +154,7 @@ void ValidPathInfo::write(
         sink << store.printStorePath(path);
     sink << (deriver ? store.printStorePath(*deriver) : "")
          << narHash.to_string(Base16, false);
-    worker_proto::write(store, sink, referencesPossiblyToSelf());
+    worker_proto::write(store, sink, references);
     sink << registrationTime << narSize;
     if (format >= 16) {
         sink << ultimate
diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh
index 9254835b77d..476df79c2af 100644
--- a/src/libstore/path-info.hh
+++ b/src/libstore/path-info.hh
@@ -17,19 +17,20 @@ class Store;
 struct SubstitutablePathInfo
 {
     std::optional<StorePath> deriver;
-    StoreReferences references;
+    StorePathSet references;
     uint64_t downloadSize; /* 0 = unknown or inapplicable */
     uint64_t narSize; /* 0 = unknown */
 };
 
 typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
 
+
 struct ValidPathInfo
 {
     StorePath path;
     std::optional<StorePath> deriver;
     Hash narHash;
-    StoreReferences references;
+    StorePathSet references;
     time_t registrationTime = 0;
     uint64_t narSize = 0; // 0 = unknown
     uint64_t id; // internal use only
@@ -81,12 +82,6 @@ struct ValidPathInfo
     /* Return true iff the path is verifiably content-addressed. */
     bool isContentAddressed(const Store & store) const;
 
-    /* Functions to view references + hasSelfReference as one set, mainly for
-       compatibility's sake. */
-    StorePathSet referencesPossiblyToSelf() const;
-    void insertReferencePossiblyToSelf(StorePath && ref);
-    void setReferencesPossiblyToSelf(StorePathSet && refs);
-
     static const size_t maxSigs = std::numeric_limits<size_t>::max();
 
     /* Return the number of signatures on this .narinfo that were
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 8ea126c65a5..ff57a77ca28 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -402,7 +402,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references.setPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}));
+            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
             infos.insert_or_assign(i.first, std::move(info));
@@ -421,12 +421,11 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S
         conn.processStderr();
         size_t count = readNum<size_t>(conn->from);
         for (size_t n = 0; n < count; n++) {
-            auto path = parseStorePath(readString(conn->from));
-            SubstitutablePathInfo & info { infos[path] };
+            SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]);
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info.deriver = parseStorePath(deriver);
-            info.references.setPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}));
+            info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
             info.downloadSize = readLongLong(conn->from);
             info.narSize = readLongLong(conn->from);
         }
@@ -634,7 +633,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
             sink
                 << exportMagic
                 << printStorePath(info.path);
-            worker_proto::write(*this, sink, info.referencesPossiblyToSelf());
+            worker_proto::write(*this, sink, info.references);
             sink
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0 // == no legacy signature
@@ -645,7 +644,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
         conn.processStderr(0, source2.get());
 
         auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
-        assert(importedPaths.empty() == 0); // doesn't include possible self reference
+        assert(importedPaths.size() <= 1);
     }
 
     else {
@@ -653,7 +652,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
                  << printStorePath(info.path)
                  << (info.deriver ? printStorePath(*info.deriver) : "")
                  << info.narHash.to_string(Base16, false);
-        worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf());
+        worker_proto::write(*this, conn->to, info.references);
         conn->to << info.registrationTime << info.narSize
                  << info.ultimate << info.sigs << renderContentAddress(info.ca)
                  << repair << !checkSigs;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 9446ad132f5..c39e50d1451 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -200,7 +200,10 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons
 {
     assert(info.hash.type == htSHA256);
     return makeStorePath(
-        makeType(*this, "text", StoreReferences { info.references }),
+        makeType(*this, "text", StoreReferences {
+            .others = info.references,
+            .self = false,
+        }),
         info.hash,
         name);
 }
@@ -310,7 +313,7 @@ void Store::addMultipleToStore(
             bytesExpected += info.narSize;
             act.setExpected(actCopyPath, bytesExpected);
 
-            return info.references.others;
+            return info.references;
         },
 
         [&](const StorePath & path) {
@@ -815,7 +818,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths,
 
         s += (format("%1%\n") % info->references.size()).str();
 
-        for (auto & j : info->referencesPossiblyToSelf())
+        for (auto & j : info->references)
             s += printStorePath(j) + "\n";
     }
 
@@ -877,7 +880,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths,
 
             {
                 auto& jsonRefs = (jsonPath["references"] = json::array());
-                for (auto & ref : info->referencesPossiblyToSelf())
+                for (auto & ref : info->references)
                     jsonRefs.emplace_back(printStorePath(ref));
             }
 
@@ -1205,7 +1208,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
     if (!n) throw Error("number expected");
     while ((*n)--) {
         getline(str, s);
-        info.insertReferencePossiblyToSelf(store.parseStorePath(s));
+        info.references.insert(store.parseStorePath(s));
     }
     if (!str || str.eof()) throw Error("missing input");
     return std::optional<ValidPathInfo>(std::move(info));
@@ -1228,6 +1231,7 @@ std::string showPaths(const PathSet & paths)
     return concatStringsSep(", ", quoteStrings(paths));
 }
 
+
 Derivation Store::derivationFromPath(const StorePath & drvPath)
 {
     ensurePath(drvPath);
diff --git a/src/libutil/reference-set.hh b/src/libutil/reference-set.hh
deleted file mode 100644
index ac4a9994e5a..00000000000
--- a/src/libutil/reference-set.hh
+++ /dev/null
@@ -1,68 +0,0 @@
-#pragma once
-
-#include "comparator.hh"
-
-#include <set>
-
-namespace nix {
-
-template<typename Ref>
-struct References
-{
-    std::set<Ref> others;
-    bool self = false;
-
-    bool empty() const;
-    size_t size() const;
-
-    /* Functions to view references + self as one set, mainly for
-       compatibility's sake. */
-    std::set<Ref> possiblyToSelf(const Ref & self) const;
-    void insertPossiblyToSelf(const Ref & self, Ref && ref);
-    void setPossiblyToSelf(const Ref & self, std::set<Ref> && refs);
-
-    GENERATE_CMP(References<Ref>, me->others, me->self);
-};
-
-template<typename Ref>
-bool References<Ref>::empty() const
-{
-    return !self && others.empty();
-}
-
-template<typename Ref>
-size_t References<Ref>::size() const
-{
-    return (self ? 1 : 0) + others.size();
-}
-
-template<typename Ref>
-std::set<Ref> References<Ref>::possiblyToSelf(const Ref & selfRef) const
-{
-    std::set<Ref> refs { others };
-    if (self)
-        refs.insert(selfRef);
-    return refs;
-}
-
-template<typename Ref>
-void References<Ref>::insertPossiblyToSelf(const Ref & selfRef, Ref && ref)
-{
-    if (ref == selfRef)
-        self = true;
-    else
-        others.insert(std::move(ref));
-}
-
-template<typename Ref>
-void References<Ref>::setPossiblyToSelf(const Ref & selfRef, std::set<Ref> && refs)
-{
-    if (refs.count(selfRef)) {
-        self = true;
-        refs.erase(selfRef);
-    }
-
-    others = refs;
-}
-
-}
diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc
index 36d774dca76..577cadceb31 100644
--- a/src/nix-store/dotgraph.cc
+++ b/src/nix-store/dotgraph.cc
@@ -56,7 +56,7 @@ void printDotGraph(ref<Store> store, StorePathSet && roots)
 
         cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000");
 
-        for (auto & p : store->queryPathInfo(path)->referencesPossiblyToSelf()) {
+        for (auto & p : store->queryPathInfo(path)->references) {
             if (p != path) {
                 workList.insert(p);
                 cout << makeEdge(std::string(p.to_string()), std::string(path.to_string()));
diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc
index d2eebca7aff..425d61e53ed 100644
--- a/src/nix-store/graphml.cc
+++ b/src/nix-store/graphml.cc
@@ -71,7 +71,7 @@ void printGraphML(ref<Store> store, StorePathSet && roots)
         auto info = store->queryPathInfo(path);
         cout << makeNode(*info);
 
-        for (auto & p : info->referencesPossiblyToSelf()) {
+        for (auto & p : info->references) {
             if (p != path) {
                 workList.insert(p);
                 cout << makeEdge(path.to_string(), p.to_string());
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 5cb5aa53afa..5b261ecc6d3 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -263,7 +263,7 @@ static void printTree(const StorePath & path,
        closure(B).  That is, if derivation A is an (possibly indirect)
        input of B, then A is printed first.  This has the effect of
        flattening the tree, preventing deeply nested structures.  */
-    auto sorted = store->topoSortPaths(info->referencesPossiblyToSelf());
+    auto sorted = store->topoSortPaths(info->references);
     reverse(sorted.begin(), sorted.end());
 
     for (const auto &[n, i] : enumerate(sorted)) {
@@ -344,7 +344,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
                 for (auto & j : ps) {
                     if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs);
                     else if (query == qReferences) {
-                        for (auto & p : store->queryPathInfo(j)->referencesPossiblyToSelf())
+                        for (auto & p : store->queryPathInfo(j)->references)
                             paths.insert(p);
                     }
                     else if (query == qReferrers) {
@@ -867,7 +867,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                         auto info = store->queryPathInfo(i);
                         out << store->printStorePath(info->path)
                             << (info->deriver ? store->printStorePath(*info->deriver) : "");
-                        worker_proto::write(*store, out, info->referencesPossiblyToSelf());
+                        worker_proto::write(*store, out, info->references);
                         // !!! Maybe we want compression?
                         out << info->narSize // downloadSize
                             << info->narSize;
@@ -964,7 +964,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 };
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
-                info.setReferencesPossiblyToSelf(worker_proto::read(*store, in, Phantom<StorePathSet> {}));
+                info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {});
                 in >> info.registrationTime >> info.narSize >> info.ultimate;
                 info.sigs = readStrings<StringSet>(in);
                 info.ca = parseContentAddressOpt(readString(in));
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 33cd1360033..76125e5e43d 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -136,7 +136,7 @@ struct CmdWhyDepends : SourceExprCommand
         for (auto & path : closure)
             graph.emplace(path, Node {
                 .path = path,
-                .refs = store->queryPathInfo(path)->references.others,
+                .refs = store->queryPathInfo(path)->references,
                 .dist = path == dependencyPath ? 0 : inf
             });