Make fetch* source derivation names (optionally) more descriptive and homogenize them across fetchers

lib/default.nix

@@ -125,7 +125,7 @@ let
     inherit (self.sources) cleanSourceFilter
       cleanSource sourceByRegex sourceFilesBySuffices
       commitIdFromGitRepo cleanSourceWith pathHasContext
-      canCleanSource pathIsGitRepo;
+      canCleanSource pathIsGitRepo repoRevToName;
     inherit (self.modules) evalModules setDefaultModuleLocation
       unifyModuleSyntax applyModuleArgsIfFunction mergeModules
       mergeModules' mergeOptionDecls mergeDefinitions

lib/sources.nix

@@ -254,6 +254,62 @@ let
       outPath = builtins.path { inherit filter name; path = origSrc; };
     };
 
+  # urlToName : (URL | Path | String) -> String
+  #
+  # Transform a URL (or path, or string) into a clean package name.
+  urlToName = url:
+    let
+      inherit (builtins) stringLength;
+      base = baseNameOf (lib.removeSuffix "/" (lib.last (lib.splitString ":" (toString url))));
+      # chop away one git or archive-related extension
+      removeExt = name: let
+          matchExt = builtins.match "(.*)\\.(git|tar|zip|gz|tgz|bz|tbz|bz2|tbz2|lzma|txz|xz|zstd)" name;
+        in if matchExt != null then lib.head matchExt else name;
+      # apply function f to string x while the result shrinks
+      shrink = f: x: let v = f x; in if stringLength v < stringLength x then shrink f v else x;
+    in shrink removeExt base;
+
+  # shortRev : (String | Integer) -> String
+  #
+  # Given a package revision (like "refs/tags/v12.0"), produce a short revision ("12.0").
+  shortRev = rev:
+    let
+      baseRev = baseNameOf (toString rev);
+      matchHash = builtins.match "[a-f0-9]+" baseRev;
+      matchVer = builtins.match "([A-Za-z]+[-_. ]?)*(v)?([0-9.]+.*)" baseRev;
+    in
+      if matchHash != null then builtins.substring 0 7 baseRev
+      else if matchVer != null then lib.last matchVer
+      else baseRev;
+
+  # repoRevToNameFull : (URL | Path | String) -> (String | Integer | null) -> (String | null) -> String
+  #
+  # See `repoRevToName` below.
+  repoRevToNameFull = repo_: rev_: suffix_:
+    let
+      repo = urlToName repo_;
+      rev = if rev_ != null then "-${shortRev rev_}" else "";
+      suffix = if suffix_ != null then "-${suffix_}" else "";
+    in "${repo}${rev}${suffix}-source";
+
+  # repoRevToName : (Bool | String) -> (URL | Path | String) -> (String | Integer | null) -> String -> String
+  #
+  # Produce derivation.name attribute for a given repository URL/path/name and (optionally) its revision/version tag.
+  #
+  # This is used by fetch(zip|git|FromGitHub|hg|svn|etc) to generate discoverable
+  # /nix/store paths.
+  #
+  # This uses a different implementation depending on the `pretty` argument:
+  #  false  -> name everything as "source"
+  #  true   -> name everything as "${repo}-${rev}-source"
+  #  "full" -> name everything as "${repo}-${rev}-${fetcher}-source"
+  repoRevToName = pretty:
+    # match on `pretty` first to minimize the thunk
+    if pretty == false then (repo: rev: suffix: "source")
+    else if pretty == true then (repo: rev: suffix: repoRevToNameFull repo rev null)
+    else if pretty == "full" then repoRevToNameFull
+    else throw "invalid value of `pretty'";
+
 in {
 
   pathType = lib.warnIf (lib.isInOldestRelease 2305)
@@ -278,6 +334,10 @@ in {
     pathHasContext
     canCleanSource
 
+    urlToName
+    shortRev
+    repoRevToName
+
     sourceByRegex
     sourceFilesBySuffices
 

pkgs/build-support/fetchbitbucket/default.nix

@@ -1,7 +1,8 @@
-{ fetchzip, lib }:
+{ lib, repoRevToNameMaybe, fetchzip }:
 
 lib.makeOverridable (
-{ owner, repo, rev, name ? "source"
+{ owner, repo, rev
+, name ? repoRevToNameMaybe repo rev "bitbucket"
 , ... # For hash agility
 }@args: fetchzip ({
   inherit name;

pkgs/build-support/fetchgit/default.nix

@@ -1,22 +1,13 @@
-{lib, stdenvNoCC, git, git-lfs, cacert}: let
-  urlToName = url: rev: let
-    inherit (lib) removeSuffix splitString last;
-    base = last (splitString ":" (baseNameOf (removeSuffix "/" url)));
+{lib, repoRevToNameMaybe, stdenvNoCC, git, git-lfs, cacert}:
 
-    matched = builtins.match "(.*)\\.git" base;
-
-    short = builtins.substring 0 7 rev;
-
-    appendShort = lib.optionalString ((builtins.match "[a-f0-9]*" rev) != null) "-${short}";
-  in "${if matched == null then base else builtins.head matched}${appendShort}";
-in
 lib.makeOverridable (
-{ url, rev ? "HEAD", sha256 ? "", hash ? "", leaveDotGit ? deepClone
+{ url, rev ? "HEAD"
+, name ? repoRevToNameMaybe url rev "git"
+, sha256 ? "", hash ? "", leaveDotGit ? deepClone
 , fetchSubmodules ? true, deepClone ? false
 , branchName ? null
 , sparseCheckout ? []
 , nonConeMode ? false
-, name ? urlToName url rev
 , # Shell code executed after the file has been fetched
   # successfully. This can do things like check or transform the file.
   postFetch ? ""

pkgs/build-support/fetchgithub/default.nix

@@ -1,7 +1,8 @@
-{ lib, fetchgit, fetchzip }:
+{ lib, repoRevToNameMaybe, fetchgit, fetchzip }:
 
 lib.makeOverridable (
-{ owner, repo, rev, name ? "source"
+{ owner, repo, rev
+, name ? repoRevToNameMaybe repo rev "github"
 , fetchSubmodules ? false, leaveDotGit ? null
 , deepClone ? false, private ? false, forceFetchGit ? false
 , sparseCheckout ? []

pkgs/build-support/fetchgitiles/default.nix

@@ -1,7 +1,10 @@
-{ fetchzip, lib }:
+{ fetchzip, repoRevToNameMaybe, lib }:
 
 lib.makeOverridable (
-{ url, rev, name ? "source", ... } @ args:
+{ url, rev
+, name ? repoRevToNameMaybe url rev "gitiles"
+, ...
+} @ args:
 
 fetchzip ({
   inherit name;

pkgs/build-support/fetchgitlab/default.nix

@@ -1,8 +1,9 @@
-{ lib, fetchgit, fetchzip }:
+{ lib, repoRevToNameMaybe, fetchgit, fetchzip }:
 
 lib.makeOverridable (
 # gitlab example
-{ owner, repo, rev, protocol ? "https", domain ? "gitlab.com", name ? "source", group ? null
+{ owner, repo, rev, protocol ? "https", domain ? "gitlab.com", group ? null
+, name ? repoRevToNameMaybe repo rev "gitlab"
 , fetchSubmodules ? false, leaveDotGit ? false
 , deepClone ? false, forceFetchGit ? false
 , sparseCheckout ? []

pkgs/build-support/fetchhg/default.nix

@@ -1,18 +1,19 @@
-{ lib, stdenvNoCC, mercurial }:
-{ name ? null
-, url
+{ lib, repoRevToNameMaybe, stdenvNoCC, mercurial }:
+{ url
 , rev ? null
+, name ? repoRevToNameMaybe url rev "hg"
 , sha256 ? null
 , hash ? null
 , fetchSubrepos ? false
-, preferLocalBuild ? true }:
+, preferLocalBuild ? true
+}:
 
 if hash != null && sha256 != null then
   throw "Only one of sha256 or hash can be set"
 else
 # TODO: statically check if mercurial as the https support if the url starts woth https.
 stdenvNoCC.mkDerivation {
-  name = "hg-archive" + (lib.optionalString (name != null) "-${name}");
+  inherit name;
   builder = ./builder.sh;
   nativeBuildInputs = [mercurial];
 

pkgs/build-support/fetchrepoorcz/default.nix

@@ -1,7 +1,8 @@
-{ fetchzip }:
+{ lib, repoRevToNameMaybe, fetchzip }:
 
 # gitweb example, snapshot support is optional in gitweb
-{ repo, rev, name ? "source"
+{ repo, rev
+, name ? repoRevToNameMaybe repo rev "repoorcz"
 , ... # For hash agility
 }@args: fetchzip ({
   inherit name;

pkgs/build-support/fetchsavannah/default.nix

@@ -1,8 +1,9 @@
-{ fetchzip, lib }:
+{ lib, repoRevToNameMaybe, fetchzip }:
 
 lib.makeOverridable (
 # cgit example, snapshot support is optional in cgit
-{ repo, rev, name ? "source"
+{ repo, rev
+, name ? repoRevToNameMaybe repo rev "savannah"
 , ... # For hash agility
 }@args: fetchzip ({
   inherit name;

pkgs/build-support/fetchsourcehut/default.nix

@@ -1,4 +1,4 @@
-{ fetchgit, fetchhg, fetchzip, lib }:
+{ lib, repoRevToNameMaybe, fetchgit, fetchhg, fetchzip }:
 
 let
   inherit (lib)
@@ -13,7 +13,7 @@ makeOverridable (
 , repo, rev
 , domain ? "sr.ht"
 , vc ? "git"
-, name ? "source"
+, name ? repoRevToNameMaybe repo rev "sourcehut"
 , fetchSubmodules ? false
 , ... # For hash agility
 } @ args:

pkgs/build-support/fetchsvn/default.nix

@@ -1,42 +1,42 @@
-{ lib, stdenvNoCC, buildPackages
+{ lib, config, stdenvNoCC, buildPackages
 , subversion, glibcLocales, sshSupport ? true, openssh ? null
 }:
 
-{ url, rev ? "HEAD", sha256 ? "", hash ? ""
-, ignoreExternals ? false, ignoreKeywords ? false, name ? null
+let
+  repoToName = url: rev:
+    let
+      inherit (lib) removeSuffix splitString reverseList head last elemAt;
+      base = removeSuffix "/" (last (splitString ":" url));
+      path = reverseList (splitString "/" base);
+      repoName =
+        # ../repo/trunk -> repo
+        if head path == "trunk" then elemAt path 1
+        # ../repo/branches/branch -> repo-branch
+        else if elemAt path 1 == "branches" then "${elemAt path 2}-${head path}"
+        # ../repo/tags/tag -> repo-tag
+        else if elemAt path 1 == "tags" then "${elemAt path 2}-${head path}"
+        # ../repo (no trunk) -> repo
+        else head path;
+      pretty = config.nameSourcesPrettily;
+      suffix = lib.optionalString (pretty == "full") "-svn";
+    in if pretty == false then "source"
+       else "${repoName}-r${toString rev}${suffix}-source";
+in
+
+{ url, rev ? "HEAD"
+, name ? repoToName url rev
+, sha256 ? "", hash ? ""
+, ignoreExternals ? false, ignoreKeywords ? false
 , preferLocalBuild ? true
 }:
 
 assert sshSupport -> openssh != null;
 
-let
-  repoName = with lib;
-    let
-      fst = head;
-      snd = l: head (tail l);
-      trd = l: head (tail (tail l));
-      path_ =
-        (p: if head p == "" then tail p else p) # ~ drop final slash if any
-        (reverseList (splitString "/" url));
-      path = [ (removeSuffix "/" (head path_)) ] ++ (tail path_);
-    in
-      # ../repo/trunk -> repo
-      if fst path == "trunk" then snd path
-      # ../repo/branches/branch -> repo-branch
-      else if snd path == "branches" then "${trd path}-${fst path}"
-      # ../repo/tags/tag -> repo-tag
-      else if snd path == "tags" then     "${trd path}-${fst path}"
-      # ../repo (no trunk) -> repo
-      else fst path;
-
-  name_ = if name == null then "${repoName}-r${toString rev}" else name;
-in
-
 if hash != "" && sha256 != "" then
   throw "Only one of sha256 or hash can be set"
 else
 stdenvNoCC.mkDerivation {
-  name = name_;
+  inherit name;
   builder = ./builder.sh;
   nativeBuildInputs = [ subversion glibcLocales ]
     ++ lib.optional sshSupport openssh;

pkgs/build-support/fetchzip/default.nix

@@ -5,11 +5,14 @@
 # (e.g. due to minor changes in the compression algorithm, or changes
 # in timestamps).
 
-{ lib, fetchurl, withUnzip ? true, unzip, glibcLocalesUtf8 }:
+{ lib, repoRevToNameMaybe, fetchurl
+, withUnzip ? true, unzip
+, glibcLocalesUtf8
+}:
 
-{ name ? "source"
-, url ? ""
+{ url ? ""
 , urls ? []
+, name ? repoRevToNameMaybe (if url != "" then url else builtins.head urls) null "unpacked"
 , nativeBuildInputs ? []
 , postFetch ? ""
 , extraPostFetch ? ""
@@ -22,7 +25,8 @@
 , extension ? null
 
 # the rest are given to fetchurl as is
-, ... } @ args:
+, ...
+} @ args:
 
 assert (extraPostFetch != "") -> lib.warn "use 'postFetch' instead of 'extraPostFetch' with 'fetchzip' and 'fetchFromGitHub' or 'fetchFromGitLab'." true;
 

pkgs/top-level/all-packages.nix

@@ -1090,6 +1090,9 @@ with pkgs;
 
   broadlink-cli = callPackage ../tools/misc/broadlink-cli { };
 
+  # this is used by most `fetch*` functions
+  repoRevToNameMaybe = lib.repoRevToName config.nameSourcesPrettily;
+
   fetchpatch = callPackage ../build-support/fetchpatch {
     # 0.3.4 would change hashes: https://github.com/NixOS/nixpkgs/issues/25154
     patchutils = buildPackages.patchutils_0_3_3;

pkgs/top-level/config.nix

@@ -50,6 +50,39 @@ let
       default = false;
     };
 
+    nameSourcesPrettily =  mkOption {
+      type = types.uniq (types.either types.bool (types.enum ["full"]));
+      default = false;
+      description = lib.mdDoc ''
+        This controls the default derivation name attribute generated by the
+        `fetch*` (like `fetchzip`, `fetchFromGitHub`, etc) functions.
+
+        Possible values and the resulting `.name`:
+
+        - `false` -> `"source"`
+        - `true` -> `"''${repo}-''${rev}-source"`
+        - `"full"` -> `"''${repo}-''${rev}-''${fetcherName}-source"`
+
+        The default `false` is the best choice for minimal rebuilds, it will
+        ignore any non-hash changes (like branches being renamed, source URLs
+        changing, etc).
+
+        Setting this to `true` greatly helps with discoverability of sources
+        in `/nix/store` at the cost of a single mass-rebuild for all `src`
+        derivations (but not their referrers, as all `src`s are fixed-output)
+        and occasional rebuild when a source changes some of its non-hash
+        attributes.
+
+        Setting this to `"full"` is similar to setting it to `true`, but the
+        use of `fetcherName` in the derivation name will force a rebuild when
+        `src` switches between `fetch*` functions, thus forcing `nix` to check
+        new derivation's `outputHash`, which is useful for debugging.
+
+        Also, `"full"` is useful for easy collection and tracking of
+        statistics of where the packages you use are hosted.
+      '';
+    };
+
     doCheckByDefault = mkMassRebuild {
       feature = "run `checkPhase` by default";
     };